@toolkit-cli/toolkode 1.3.8 → 1.3.9

package/src/storage/json-migration.ts

@@ -1,425 +0,0 @@
-import { Database } from "bun:sqlite"
-import { drizzle } from "drizzle-orm/bun-sqlite"
-import { Global } from "../global"
-import { Log } from "../util/log"
-import { ProjectTable } from "../project/project.sql"
-import { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
-import { SessionShareTable } from "../share/share.sql"
-import path from "path"
-import { existsSync } from "fs"
-import { Filesystem } from "../util/filesystem"
-import { Glob } from "../util/glob"
-
-export namespace JsonMigration {
-  const log = Log.create({ service: "json-migration" })
-
-  export type Progress = {
-    current: number
-    total: number
-    label: string
-  }
-
-  type Options = {
-    progress?: (event: Progress) => void
-  }
-
-  export async function run(sqlite: Database, options?: Options) {
-    const storageDir = path.join(Global.Path.data, "storage")
-
-    if (!existsSync(storageDir)) {
-      log.info("storage directory does not exist, skipping migration")
-      return {
-        projects: 0,
-        sessions: 0,
-        messages: 0,
-        parts: 0,
-        todos: 0,
-        permissions: 0,
-        shares: 0,
-        errors: [] as string[],
-      }
-    }
-
-    log.info("starting json to sqlite migration", { storageDir })
-    const start = performance.now()
-
-    const db = drizzle({ client: sqlite })
-
-    // Optimize SQLite for bulk inserts
-    sqlite.exec("PRAGMA journal_mode = WAL")
-    sqlite.exec("PRAGMA synchronous = OFF")
-    sqlite.exec("PRAGMA cache_size = 10000")
-    sqlite.exec("PRAGMA temp_store = MEMORY")
-    const stats = {
-      projects: 0,
-      sessions: 0,
-      messages: 0,
-      parts: 0,
-      todos: 0,
-      permissions: 0,
-      shares: 0,
-      errors: [] as string[],
-    }
-    const orphans = {
-      sessions: 0,
-      todos: 0,
-      permissions: 0,
-      shares: 0,
-    }
-    const errs = stats.errors
-
-    const batchSize = 1000
-    const now = Date.now()
-
-    async function list(pattern: string) {
-      return Glob.scan(pattern, { cwd: storageDir, absolute: true })
-    }
-
-    async function read(files: string[], start: number, end: number) {
-      const count = end - start
-      const tasks = new Array(count)
-      for (let i = 0; i < count; i++) {
-        tasks[i] = Filesystem.readJson(files[start + i])
-      }
-      const results = await Promise.allSettled(tasks)
-      const items = new Array(count)
-      for (let i = 0; i < results.length; i++) {
-        const result = results[i]
-        if (result.status === "fulfilled") {
-          items[i] = result.value
-          continue
-        }
-        errs.push(`failed to read ${files[start + i]}: ${result.reason}`)
-      }
-      return items
-    }
-
-    function insert(values: any[], table: any, label: string) {
-      if (values.length === 0) return 0
-      try {
-        db.insert(table).values(values).onConflictDoNothing().run()
-        return values.length
-      } catch (e) {
-        errs.push(`failed to migrate ${label} batch: ${e}`)
-        return 0
-      }
-    }
-
-    // Pre-scan all files upfront to avoid repeated glob operations
-    log.info("scanning files...")
-    const [projectFiles, sessionFiles, messageFiles, partFiles, todoFiles, permFiles, shareFiles] = await Promise.all([
-      list("project/*.json"),
-      list("session/*/*.json"),
-      list("message/*/*.json"),
-      list("part/*/*.json"),
-      list("todo/*.json"),
-      list("permission/*.json"),
-      list("session_share/*.json"),
-    ])
-
-    log.info("file scan complete", {
-      projects: projectFiles.length,
-      sessions: sessionFiles.length,
-      messages: messageFiles.length,
-      parts: partFiles.length,
-      todos: todoFiles.length,
-      permissions: permFiles.length,
-      shares: shareFiles.length,
-    })
-
-    const total = Math.max(
-      1,
-      projectFiles.length +
-        sessionFiles.length +
-        messageFiles.length +
-        partFiles.length +
-        todoFiles.length +
-        permFiles.length +
-        shareFiles.length,
-    )
-    const progress = options?.progress
-    let current = 0
-    const step = (label: string, count: number) => {
-      current = Math.min(total, current + count)
-      progress?.({ current, total, label })
-    }
-
-    progress?.({ current, total, label: "starting" })
-
-    sqlite.exec("BEGIN TRANSACTION")
-
-    // Migrate projects first (no FK deps)
-    // Derive all IDs from file paths, not JSON content
-    const projectIds = new Set<string>()
-    const projectValues = [] as any[]
-    for (let i = 0; i < projectFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, projectFiles.length)
-      const batch = await read(projectFiles, i, end)
-      projectValues.length = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const id = path.basename(projectFiles[i + j], ".json")
-        projectIds.add(id)
-        projectValues.push({
-          id,
-          worktree: data.worktree ?? "/",
-          vcs: data.vcs,
-          name: data.name ?? undefined,
-          icon_url: data.icon?.url,
-          icon_color: data.icon?.color,
-          time_created: data.time?.created ?? now,
-          time_updated: data.time?.updated ?? now,
-          time_initialized: data.time?.initialized,
-          sandboxes: data.sandboxes ?? [],
-          commands: data.commands,
-        })
-      }
-      stats.projects += insert(projectValues, ProjectTable, "project")
-      step("projects", end - i)
-    }
-    log.info("migrated projects", { count: stats.projects, duration: Math.round(performance.now() - start) })
-
-    // Migrate sessions (depends on projects)
-    // Derive all IDs from directory/file paths, not JSON content, since earlier
-    // migrations may have moved sessions to new directories without updating the JSON
-    const sessionProjects = sessionFiles.map((file) => path.basename(path.dirname(file)))
-    const sessionIds = new Set<string>()
-    const sessionValues = [] as any[]
-    for (let i = 0; i < sessionFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, sessionFiles.length)
-      const batch = await read(sessionFiles, i, end)
-      sessionValues.length = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const id = path.basename(sessionFiles[i + j], ".json")
-        const projectID = sessionProjects[i + j]
-        if (!projectIds.has(projectID)) {
-          orphans.sessions++
-          continue
-        }
-        sessionIds.add(id)
-        sessionValues.push({
-          id,
-          project_id: projectID,
-          parent_id: data.parentID ?? null,
-          slug: data.slug ?? "",
-          directory: data.directory ?? "",
-          title: data.title ?? "",
-          version: data.version ?? "",
-          share_url: data.share?.url ?? null,
-          summary_additions: data.summary?.additions ?? null,
-          summary_deletions: data.summary?.deletions ?? null,
-          summary_files: data.summary?.files ?? null,
-          summary_diffs: data.summary?.diffs ?? null,
-          revert: data.revert ?? null,
-          permission: data.permission ?? null,
-          time_created: data.time?.created ?? now,
-          time_updated: data.time?.updated ?? now,
-          time_compacting: data.time?.compacting ?? null,
-          time_archived: data.time?.archived ?? null,
-        })
-      }
-      stats.sessions += insert(sessionValues, SessionTable, "session")
-      step("sessions", end - i)
-    }
-    log.info("migrated sessions", { count: stats.sessions })
-    if (orphans.sessions > 0) {
-      log.warn("skipped orphaned sessions", { count: orphans.sessions })
-    }
-
-    // Migrate messages using pre-scanned file map
-    const allMessageFiles = [] as string[]
-    const allMessageSessions = [] as string[]
-    const messageSessions = new Map<string, string>()
-    for (const file of messageFiles) {
-      const sessionID = path.basename(path.dirname(file))
-      if (!sessionIds.has(sessionID)) continue
-      allMessageFiles.push(file)
-      allMessageSessions.push(sessionID)
-    }
-
-    for (let i = 0; i < allMessageFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, allMessageFiles.length)
-      const batch = await read(allMessageFiles, i, end)
-      const values = new Array(batch.length)
-      let count = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const file = allMessageFiles[i + j]
-        const id = path.basename(file, ".json")
-        const sessionID = allMessageSessions[i + j]
-        messageSessions.set(id, sessionID)
-        const rest = data
-        delete rest.id
-        delete rest.sessionID
-        values[count++] = {
-          id,
-          session_id: sessionID,
-          time_created: data.time?.created ?? now,
-          time_updated: data.time?.updated ?? now,
-          data: rest,
-        }
-      }
-      values.length = count
-      stats.messages += insert(values, MessageTable, "message")
-      step("messages", end - i)
-    }
-    log.info("migrated messages", { count: stats.messages })
-
-    // Migrate parts using pre-scanned file map
-    for (let i = 0; i < partFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, partFiles.length)
-      const batch = await read(partFiles, i, end)
-      const values = new Array(batch.length)
-      let count = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const file = partFiles[i + j]
-        const id = path.basename(file, ".json")
-        const messageID = path.basename(path.dirname(file))
-        const sessionID = messageSessions.get(messageID)
-        if (!sessionID) {
-          errs.push(`part missing message session: ${file}`)
-          continue
-        }
-        if (!sessionIds.has(sessionID)) continue
-        const rest = data
-        delete rest.id
-        delete rest.messageID
-        delete rest.sessionID
-        values[count++] = {
-          id,
-          message_id: messageID,
-          session_id: sessionID,
-          time_created: data.time?.created ?? now,
-          time_updated: data.time?.updated ?? now,
-          data: rest,
-        }
-      }
-      values.length = count
-      stats.parts += insert(values, PartTable, "part")
-      step("parts", end - i)
-    }
-    log.info("migrated parts", { count: stats.parts })
-
-    // Migrate todos
-    const todoSessions = todoFiles.map((file) => path.basename(file, ".json"))
-    for (let i = 0; i < todoFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, todoFiles.length)
-      const batch = await read(todoFiles, i, end)
-      const values = [] as any[]
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const sessionID = todoSessions[i + j]
-        if (!sessionIds.has(sessionID)) {
-          orphans.todos++
-          continue
-        }
-        if (!Array.isArray(data)) {
-          errs.push(`todo not an array: ${todoFiles[i + j]}`)
-          continue
-        }
-        for (let position = 0; position < data.length; position++) {
-          const todo = data[position]
-          if (!todo?.content || !todo?.status || !todo?.priority) continue
-          values.push({
-            session_id: sessionID,
-            content: todo.content,
-            status: todo.status,
-            priority: todo.priority,
-            position,
-            time_created: now,
-            time_updated: now,
-          })
-        }
-      }
-      stats.todos += insert(values, TodoTable, "todo")
-      step("todos", end - i)
-    }
-    log.info("migrated todos", { count: stats.todos })
-    if (orphans.todos > 0) {
-      log.warn("skipped orphaned todos", { count: orphans.todos })
-    }
-
-    // Migrate permissions
-    const permProjects = permFiles.map((file) => path.basename(file, ".json"))
-    const permValues = [] as any[]
-    for (let i = 0; i < permFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, permFiles.length)
-      const batch = await read(permFiles, i, end)
-      permValues.length = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const projectID = permProjects[i + j]
-        if (!projectIds.has(projectID)) {
-          orphans.permissions++
-          continue
-        }
-        permValues.push({ project_id: projectID, data })
-      }
-      stats.permissions += insert(permValues, PermissionTable, "permission")
-      step("permissions", end - i)
-    }
-    log.info("migrated permissions", { count: stats.permissions })
-    if (orphans.permissions > 0) {
-      log.warn("skipped orphaned permissions", { count: orphans.permissions })
-    }
-
-    // Migrate session shares
-    const shareSessions = shareFiles.map((file) => path.basename(file, ".json"))
-    const shareValues = [] as any[]
-    for (let i = 0; i < shareFiles.length; i += batchSize) {
-      const end = Math.min(i + batchSize, shareFiles.length)
-      const batch = await read(shareFiles, i, end)
-      shareValues.length = 0
-      for (let j = 0; j < batch.length; j++) {
-        const data = batch[j]
-        if (!data) continue
-        const sessionID = shareSessions[i + j]
-        if (!sessionIds.has(sessionID)) {
-          orphans.shares++
-          continue
-        }
-        if (!data?.id || !data?.secret || !data?.url) {
-          errs.push(`session_share missing id/secret/url: ${shareFiles[i + j]}`)
-          continue
-        }
-        shareValues.push({ session_id: sessionID, id: data.id, secret: data.secret, url: data.url })
-      }
-      stats.shares += insert(shareValues, SessionShareTable, "session_share")
-      step("shares", end - i)
-    }
-    log.info("migrated session shares", { count: stats.shares })
-    if (orphans.shares > 0) {
-      log.warn("skipped orphaned session shares", { count: orphans.shares })
-    }
-
-    sqlite.exec("COMMIT")
-
-    log.info("json migration complete", {
-      projects: stats.projects,
-      sessions: stats.sessions,
-      messages: stats.messages,
-      parts: stats.parts,
-      todos: stats.todos,
-      permissions: stats.permissions,
-      shares: stats.shares,
-      errorCount: stats.errors.length,
-      duration: Math.round(performance.now() - start),
-    })
-
-    if (stats.errors.length > 0) {
-      log.warn("migration errors", { errors: stats.errors.slice(0, 20) })
-    }
-
-    progress?.({ current: total, total, label: "complete" })
-
-    return stats
-  }
-}

package/src/storage/schema.sql.ts

@@ -1,10 +0,0 @@
-import { integer } from "drizzle-orm/sqlite-core"
-
-export const Timestamps = {
-  time_created: integer()
-    .notNull()
-    .$default(() => Date.now()),
-  time_updated: integer()
-    .notNull()
-    .$onUpdate(() => Date.now()),
-}

package/src/storage/schema.ts

@@ -1,5 +0,0 @@
-export { AccountTable, AccountStateTable, ControlAccountTable } from "../account/account.sql"
-export { ProjectTable } from "../project/project.sql"
-export { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
-export { SessionShareTable } from "../share/share.sql"
-export { WorkspaceTable } from "../control-plane/workspace.sql"

package/src/storage/storage.ts

@@ -1,217 +0,0 @@
-import { Log } from "../util/log"
-import path from "path"
-import fs from "fs/promises"
-import { Global } from "../global"
-import { Filesystem } from "../util/filesystem"
-import { lazy } from "../util/lazy"
-import { Lock } from "../util/lock"
-import { NamedError } from "@toolkit-cli/util/error"
-import z from "zod"
-import { Glob } from "../util/glob"
-import { git } from "@/util/git"
-
-export namespace Storage {
-  const log = Log.create({ service: "storage" })
-
-  type Migration = (dir: string) => Promise<void>
-
-  export const NotFoundError = NamedError.create(
-    "NotFoundError",
-    z.object({
-      message: z.string(),
-    }),
-  )
-
-  const MIGRATIONS: Migration[] = [
-    async (dir) => {
-      const project = path.resolve(dir, "../project")
-      if (!(await Filesystem.isDir(project))) return
-      const projectDirs = await Glob.scan("*", {
-        cwd: project,
-        include: "all",
-      })
-      for (const projectDir of projectDirs) {
-        const fullPath = path.join(project, projectDir)
-        if (!(await Filesystem.isDir(fullPath))) continue
-        log.info(`migrating project ${projectDir}`)
-        let projectID = projectDir
-        const fullProjectDir = path.join(project, projectDir)
-        let worktree = "/"
-
-        if (projectID !== "global") {
-          for (const msgFile of await Glob.scan("storage/session/message/*/*.json", {
-            cwd: path.join(project, projectDir),
-            absolute: true,
-          })) {
-            const json = await Filesystem.readJson<any>(msgFile)
-            worktree = json.path?.root
-            if (worktree) break
-          }
-          if (!worktree) continue
-          if (!(await Filesystem.isDir(worktree))) continue
-          const result = await git(["rev-list", "--max-parents=0", "--all"], {
-            cwd: worktree,
-          })
-          const [id] = result
-            .text()
-            .split("\n")
-            .filter(Boolean)
-            .map((x) => x.trim())
-            .toSorted()
-          if (!id) continue
-          projectID = id
-
-          await Filesystem.writeJson(path.join(dir, "project", projectID + ".json"), {
-            id,
-            vcs: "git",
-            worktree,
-            time: {
-              created: Date.now(),
-              initialized: Date.now(),
-            },
-          })
-
-          log.info(`migrating sessions for project ${projectID}`)
-          for (const sessionFile of await Glob.scan("storage/session/info/*.json", {
-            cwd: fullProjectDir,
-            absolute: true,
-          })) {
-            const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
-            log.info("copying", {
-              sessionFile,
-              dest,
-            })
-            const session = await Filesystem.readJson<any>(sessionFile)
-            await Filesystem.writeJson(dest, session)
-            log.info(`migrating messages for session ${session.id}`)
-            for (const msgFile of await Glob.scan(`storage/session/message/${session.id}/*.json`, {
-              cwd: fullProjectDir,
-              absolute: true,
-            })) {
-              const dest = path.join(dir, "message", session.id, path.basename(msgFile))
-              log.info("copying", {
-                msgFile,
-                dest,
-              })
-              const message = await Filesystem.readJson<any>(msgFile)
-              await Filesystem.writeJson(dest, message)
-
-              log.info(`migrating parts for message ${message.id}`)
-              for (const partFile of await Glob.scan(`storage/session/part/${session.id}/${message.id}/*.json`, {
-                cwd: fullProjectDir,
-                absolute: true,
-              })) {
-                const dest = path.join(dir, "part", message.id, path.basename(partFile))
-                const part = await Filesystem.readJson(partFile)
-                log.info("copying", {
-                  partFile,
-                  dest,
-                })
-                await Filesystem.writeJson(dest, part)
-              }
-            }
-          }
-        }
-      }
-    },
-    async (dir) => {
-      for (const item of await Glob.scan("session/*/*.json", {
-        cwd: dir,
-        absolute: true,
-      })) {
-        const session = await Filesystem.readJson<any>(item)
-        if (!session.projectID) continue
-        if (!session.summary?.diffs) continue
-        const { diffs } = session.summary
-        await Filesystem.write(path.join(dir, "session_diff", session.id + ".json"), JSON.stringify(diffs))
-        await Filesystem.writeJson(path.join(dir, "session", session.projectID, session.id + ".json"), {
-          ...session,
-          summary: {
-            additions: diffs.reduce((sum: any, x: any) => sum + x.additions, 0),
-            deletions: diffs.reduce((sum: any, x: any) => sum + x.deletions, 0),
-          },
-        })
-      }
-    },
-  ]
-
-  const state = lazy(async () => {
-    const dir = path.join(Global.Path.data, "storage")
-    const migration = await Filesystem.readJson<string>(path.join(dir, "migration"))
-      .then((x) => parseInt(x))
-      .catch(() => 0)
-    for (let index = migration; index < MIGRATIONS.length; index++) {
-      log.info("running migration", { index })
-      const migration = MIGRATIONS[index]
-      await migration(dir).catch(() => log.error("failed to run migration", { index }))
-      await Filesystem.write(path.join(dir, "migration"), (index + 1).toString())
-    }
-    return {
-      dir,
-    }
-  })
-
-  export async function remove(key: string[]) {
-    const dir = await state().then((x) => x.dir)
-    const target = path.join(dir, ...key) + ".json"
-    return withErrorHandling(async () => {
-      await fs.unlink(target).catch(() => {})
-    })
-  }
-
-  export async function read<T>(key: string[]) {
-    const dir = await state().then((x) => x.dir)
-    const target = path.join(dir, ...key) + ".json"
-    return withErrorHandling(async () => {
-      using _ = await Lock.read(target)
-      const result = await Filesystem.readJson<T>(target)
-      return result as T
-    })
-  }
-
-  export async function update<T>(key: string[], fn: (draft: T) => void) {
-    const dir = await state().then((x) => x.dir)
-    const target = path.join(dir, ...key) + ".json"
-    return withErrorHandling(async () => {
-      using _ = await Lock.write(target)
-      const content = await Filesystem.readJson<T>(target)
-      fn(content as T)
-      await Filesystem.writeJson(target, content)
-      return content
-    })
-  }
-
-  export async function write<T>(key: string[], content: T) {
-    const dir = await state().then((x) => x.dir)
-    const target = path.join(dir, ...key) + ".json"
-    return withErrorHandling(async () => {
-      using _ = await Lock.write(target)
-      await Filesystem.writeJson(target, content)
-    })
-  }
-
-  async function withErrorHandling<T>(body: () => Promise<T>) {
-    return body().catch((e) => {
-      if (!(e instanceof Error)) throw e
-      const errnoException = e as NodeJS.ErrnoException
-      if (errnoException.code === "ENOENT") {
-        throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` })
-      }
-      throw e
-    })
-  }
-
-  export async function list(prefix: string[]) {
-    const dir = await state().then((x) => x.dir)
-    try {
-      const result = await Glob.scan("**/*", {
-        cwd: path.join(dir, ...prefix),
-        include: "file",
-      }).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]))
-      result.sort()
-      return result
-    } catch {
-      return []
-    }
-  }
-}