@toolkit-cli/toolkode 1.3.7

package/src/storage/json-migration.ts

@@ -0,0 +1,425 @@
+import { Database } from "bun:sqlite"
+import { drizzle } from "drizzle-orm/bun-sqlite"
+import { Global } from "../global"
+import { Log } from "../util/log"
+import { ProjectTable } from "../project/project.sql"
+import { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
+import { SessionShareTable } from "../share/share.sql"
+import path from "path"
+import { existsSync } from "fs"
+import { Filesystem } from "../util/filesystem"
+import { Glob } from "../util/glob"
+
+export namespace JsonMigration {
+  const log = Log.create({ service: "json-migration" })
+
+  export type Progress = {
+    current: number
+    total: number
+    label: string
+  }
+
+  type Options = {
+    progress?: (event: Progress) => void
+  }
+
+  export async function run(sqlite: Database, options?: Options) {
+    const storageDir = path.join(Global.Path.data, "storage")
+
+    if (!existsSync(storageDir)) {
+      log.info("storage directory does not exist, skipping migration")
+      return {
+        projects: 0,
+        sessions: 0,
+        messages: 0,
+        parts: 0,
+        todos: 0,
+        permissions: 0,
+        shares: 0,
+        errors: [] as string[],
+      }
+    }
+
+    log.info("starting json to sqlite migration", { storageDir })
+    const start = performance.now()
+
+    const db = drizzle({ client: sqlite })
+
+    // Optimize SQLite for bulk inserts
+    sqlite.exec("PRAGMA journal_mode = WAL")
+    sqlite.exec("PRAGMA synchronous = OFF")
+    sqlite.exec("PRAGMA cache_size = 10000")
+    sqlite.exec("PRAGMA temp_store = MEMORY")
+    const stats = {
+      projects: 0,
+      sessions: 0,
+      messages: 0,
+      parts: 0,
+      todos: 0,
+      permissions: 0,
+      shares: 0,
+      errors: [] as string[],
+    }
+    const orphans = {
+      sessions: 0,
+      todos: 0,
+      permissions: 0,
+      shares: 0,
+    }
+    const errs = stats.errors
+
+    const batchSize = 1000
+    const now = Date.now()
+
+    async function list(pattern: string) {
+      return Glob.scan(pattern, { cwd: storageDir, absolute: true })
+    }
+
+    async function read(files: string[], start: number, end: number) {
+      const count = end - start
+      const tasks = new Array(count)
+      for (let i = 0; i < count; i++) {
+        tasks[i] = Filesystem.readJson(files[start + i])
+      }
+      const results = await Promise.allSettled(tasks)
+      const items = new Array(count)
+      for (let i = 0; i < results.length; i++) {
+        const result = results[i]
+        if (result.status === "fulfilled") {
+          items[i] = result.value
+          continue
+        }
+        errs.push(`failed to read ${files[start + i]}: ${result.reason}`)
+      }
+      return items
+    }
+
+    function insert(values: any[], table: any, label: string) {
+      if (values.length === 0) return 0
+      try {
+        db.insert(table).values(values).onConflictDoNothing().run()
+        return values.length
+      } catch (e) {
+        errs.push(`failed to migrate ${label} batch: ${e}`)
+        return 0
+      }
+    }
+
+    // Pre-scan all files upfront to avoid repeated glob operations
+    log.info("scanning files...")
+    const [projectFiles, sessionFiles, messageFiles, partFiles, todoFiles, permFiles, shareFiles] = await Promise.all([
+      list("project/*.json"),
+      list("session/*/*.json"),
+      list("message/*/*.json"),
+      list("part/*/*.json"),
+      list("todo/*.json"),
+      list("permission/*.json"),
+      list("session_share/*.json"),
+    ])
+
+    log.info("file scan complete", {
+      projects: projectFiles.length,
+      sessions: sessionFiles.length,
+      messages: messageFiles.length,
+      parts: partFiles.length,
+      todos: todoFiles.length,
+      permissions: permFiles.length,
+      shares: shareFiles.length,
+    })
+
+    const total = Math.max(
+      1,
+      projectFiles.length +
+        sessionFiles.length +
+        messageFiles.length +
+        partFiles.length +
+        todoFiles.length +
+        permFiles.length +
+        shareFiles.length,
+    )
+    const progress = options?.progress
+    let current = 0
+    const step = (label: string, count: number) => {
+      current = Math.min(total, current + count)
+      progress?.({ current, total, label })
+    }
+
+    progress?.({ current, total, label: "starting" })
+
+    sqlite.exec("BEGIN TRANSACTION")
+
+    // Migrate projects first (no FK deps)
+    // Derive all IDs from file paths, not JSON content
+    const projectIds = new Set<string>()
+    const projectValues = [] as any[]
+    for (let i = 0; i < projectFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, projectFiles.length)
+      const batch = await read(projectFiles, i, end)
+      projectValues.length = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const id = path.basename(projectFiles[i + j], ".json")
+        projectIds.add(id)
+        projectValues.push({
+          id,
+          worktree: data.worktree ?? "/",
+          vcs: data.vcs,
+          name: data.name ?? undefined,
+          icon_url: data.icon?.url,
+          icon_color: data.icon?.color,
+          time_created: data.time?.created ?? now,
+          time_updated: data.time?.updated ?? now,
+          time_initialized: data.time?.initialized,
+          sandboxes: data.sandboxes ?? [],
+          commands: data.commands,
+        })
+      }
+      stats.projects += insert(projectValues, ProjectTable, "project")
+      step("projects", end - i)
+    }
+    log.info("migrated projects", { count: stats.projects, duration: Math.round(performance.now() - start) })
+
+    // Migrate sessions (depends on projects)
+    // Derive all IDs from directory/file paths, not JSON content, since earlier
+    // migrations may have moved sessions to new directories without updating the JSON
+    const sessionProjects = sessionFiles.map((file) => path.basename(path.dirname(file)))
+    const sessionIds = new Set<string>()
+    const sessionValues = [] as any[]
+    for (let i = 0; i < sessionFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, sessionFiles.length)
+      const batch = await read(sessionFiles, i, end)
+      sessionValues.length = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const id = path.basename(sessionFiles[i + j], ".json")
+        const projectID = sessionProjects[i + j]
+        if (!projectIds.has(projectID)) {
+          orphans.sessions++
+          continue
+        }
+        sessionIds.add(id)
+        sessionValues.push({
+          id,
+          project_id: projectID,
+          parent_id: data.parentID ?? null,
+          slug: data.slug ?? "",
+          directory: data.directory ?? "",
+          title: data.title ?? "",
+          version: data.version ?? "",
+          share_url: data.share?.url ?? null,
+          summary_additions: data.summary?.additions ?? null,
+          summary_deletions: data.summary?.deletions ?? null,
+          summary_files: data.summary?.files ?? null,
+          summary_diffs: data.summary?.diffs ?? null,
+          revert: data.revert ?? null,
+          permission: data.permission ?? null,
+          time_created: data.time?.created ?? now,
+          time_updated: data.time?.updated ?? now,
+          time_compacting: data.time?.compacting ?? null,
+          time_archived: data.time?.archived ?? null,
+        })
+      }
+      stats.sessions += insert(sessionValues, SessionTable, "session")
+      step("sessions", end - i)
+    }
+    log.info("migrated sessions", { count: stats.sessions })
+    if (orphans.sessions > 0) {
+      log.warn("skipped orphaned sessions", { count: orphans.sessions })
+    }
+
+    // Migrate messages using pre-scanned file map
+    const allMessageFiles = [] as string[]
+    const allMessageSessions = [] as string[]
+    const messageSessions = new Map<string, string>()
+    for (const file of messageFiles) {
+      const sessionID = path.basename(path.dirname(file))
+      if (!sessionIds.has(sessionID)) continue
+      allMessageFiles.push(file)
+      allMessageSessions.push(sessionID)
+    }
+
+    for (let i = 0; i < allMessageFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, allMessageFiles.length)
+      const batch = await read(allMessageFiles, i, end)
+      const values = new Array(batch.length)
+      let count = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const file = allMessageFiles[i + j]
+        const id = path.basename(file, ".json")
+        const sessionID = allMessageSessions[i + j]
+        messageSessions.set(id, sessionID)
+        const rest = data
+        delete rest.id
+        delete rest.sessionID
+        values[count++] = {
+          id,
+          session_id: sessionID,
+          time_created: data.time?.created ?? now,
+          time_updated: data.time?.updated ?? now,
+          data: rest,
+        }
+      }
+      values.length = count
+      stats.messages += insert(values, MessageTable, "message")
+      step("messages", end - i)
+    }
+    log.info("migrated messages", { count: stats.messages })
+
+    // Migrate parts using pre-scanned file map
+    for (let i = 0; i < partFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, partFiles.length)
+      const batch = await read(partFiles, i, end)
+      const values = new Array(batch.length)
+      let count = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const file = partFiles[i + j]
+        const id = path.basename(file, ".json")
+        const messageID = path.basename(path.dirname(file))
+        const sessionID = messageSessions.get(messageID)
+        if (!sessionID) {
+          errs.push(`part missing message session: ${file}`)
+          continue
+        }
+        if (!sessionIds.has(sessionID)) continue
+        const rest = data
+        delete rest.id
+        delete rest.messageID
+        delete rest.sessionID
+        values[count++] = {
+          id,
+          message_id: messageID,
+          session_id: sessionID,
+          time_created: data.time?.created ?? now,
+          time_updated: data.time?.updated ?? now,
+          data: rest,
+        }
+      }
+      values.length = count
+      stats.parts += insert(values, PartTable, "part")
+      step("parts", end - i)
+    }
+    log.info("migrated parts", { count: stats.parts })
+
+    // Migrate todos
+    const todoSessions = todoFiles.map((file) => path.basename(file, ".json"))
+    for (let i = 0; i < todoFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, todoFiles.length)
+      const batch = await read(todoFiles, i, end)
+      const values = [] as any[]
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const sessionID = todoSessions[i + j]
+        if (!sessionIds.has(sessionID)) {
+          orphans.todos++
+          continue
+        }
+        if (!Array.isArray(data)) {
+          errs.push(`todo not an array: ${todoFiles[i + j]}`)
+          continue
+        }
+        for (let position = 0; position < data.length; position++) {
+          const todo = data[position]
+          if (!todo?.content || !todo?.status || !todo?.priority) continue
+          values.push({
+            session_id: sessionID,
+            content: todo.content,
+            status: todo.status,
+            priority: todo.priority,
+            position,
+            time_created: now,
+            time_updated: now,
+          })
+        }
+      }
+      stats.todos += insert(values, TodoTable, "todo")
+      step("todos", end - i)
+    }
+    log.info("migrated todos", { count: stats.todos })
+    if (orphans.todos > 0) {
+      log.warn("skipped orphaned todos", { count: orphans.todos })
+    }
+
+    // Migrate permissions
+    const permProjects = permFiles.map((file) => path.basename(file, ".json"))
+    const permValues = [] as any[]
+    for (let i = 0; i < permFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, permFiles.length)
+      const batch = await read(permFiles, i, end)
+      permValues.length = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const projectID = permProjects[i + j]
+        if (!projectIds.has(projectID)) {
+          orphans.permissions++
+          continue
+        }
+        permValues.push({ project_id: projectID, data })
+      }
+      stats.permissions += insert(permValues, PermissionTable, "permission")
+      step("permissions", end - i)
+    }
+    log.info("migrated permissions", { count: stats.permissions })
+    if (orphans.permissions > 0) {
+      log.warn("skipped orphaned permissions", { count: orphans.permissions })
+    }
+
+    // Migrate session shares
+    const shareSessions = shareFiles.map((file) => path.basename(file, ".json"))
+    const shareValues = [] as any[]
+    for (let i = 0; i < shareFiles.length; i += batchSize) {
+      const end = Math.min(i + batchSize, shareFiles.length)
+      const batch = await read(shareFiles, i, end)
+      shareValues.length = 0
+      for (let j = 0; j < batch.length; j++) {
+        const data = batch[j]
+        if (!data) continue
+        const sessionID = shareSessions[i + j]
+        if (!sessionIds.has(sessionID)) {
+          orphans.shares++
+          continue
+        }
+        if (!data?.id || !data?.secret || !data?.url) {
+          errs.push(`session_share missing id/secret/url: ${shareFiles[i + j]}`)
+          continue
+        }
+        shareValues.push({ session_id: sessionID, id: data.id, secret: data.secret, url: data.url })
+      }
+      stats.shares += insert(shareValues, SessionShareTable, "session_share")
+      step("shares", end - i)
+    }
+    log.info("migrated session shares", { count: stats.shares })
+    if (orphans.shares > 0) {
+      log.warn("skipped orphaned session shares", { count: orphans.shares })
+    }
+
+    sqlite.exec("COMMIT")
+
+    log.info("json migration complete", {
+      projects: stats.projects,
+      sessions: stats.sessions,
+      messages: stats.messages,
+      parts: stats.parts,
+      todos: stats.todos,
+      permissions: stats.permissions,
+      shares: stats.shares,
+      errorCount: stats.errors.length,
+      duration: Math.round(performance.now() - start),
+    })
+
+    if (stats.errors.length > 0) {
+      log.warn("migration errors", { errors: stats.errors.slice(0, 20) })
+    }
+
+    progress?.({ current: total, total, label: "complete" })
+
+    return stats
+  }
+}

package/src/storage/schema.sql.ts

@@ -0,0 +1,10 @@
+import { integer } from "drizzle-orm/sqlite-core"
+
+export const Timestamps = {
+  time_created: integer()
+    .notNull()
+    .$default(() => Date.now()),
+  time_updated: integer()
+    .notNull()
+    .$onUpdate(() => Date.now()),
+}

package/src/storage/schema.ts

@@ -0,0 +1,5 @@
+export { AccountTable, AccountStateTable, ControlAccountTable } from "../account/account.sql"
+export { ProjectTable } from "../project/project.sql"
+export { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
+export { SessionShareTable } from "../share/share.sql"
+export { WorkspaceTable } from "../control-plane/workspace.sql"

package/src/storage/storage.ts

@@ -0,0 +1,217 @@
+import { Log } from "../util/log"
+import path from "path"
+import fs from "fs/promises"
+import { Global } from "../global"
+import { Filesystem } from "../util/filesystem"
+import { lazy } from "../util/lazy"
+import { Lock } from "../util/lock"
+import { NamedError } from "@toolkit-cli/util/error"
+import z from "zod"
+import { Glob } from "../util/glob"
+import { git } from "@/util/git"
+
+export namespace Storage {
+  const log = Log.create({ service: "storage" })
+
+  type Migration = (dir: string) => Promise<void>
+
+  export const NotFoundError = NamedError.create(
+    "NotFoundError",
+    z.object({
+      message: z.string(),
+    }),
+  )
+
+  const MIGRATIONS: Migration[] = [
+    async (dir) => {
+      const project = path.resolve(dir, "../project")
+      if (!(await Filesystem.isDir(project))) return
+      const projectDirs = await Glob.scan("*", {
+        cwd: project,
+        include: "all",
+      })
+      for (const projectDir of projectDirs) {
+        const fullPath = path.join(project, projectDir)
+        if (!(await Filesystem.isDir(fullPath))) continue
+        log.info(`migrating project ${projectDir}`)
+        let projectID = projectDir
+        const fullProjectDir = path.join(project, projectDir)
+        let worktree = "/"
+
+        if (projectID !== "global") {
+          for (const msgFile of await Glob.scan("storage/session/message/*/*.json", {
+            cwd: path.join(project, projectDir),
+            absolute: true,
+          })) {
+            const json = await Filesystem.readJson<any>(msgFile)
+            worktree = json.path?.root
+            if (worktree) break
+          }
+          if (!worktree) continue
+          if (!(await Filesystem.isDir(worktree))) continue
+          const result = await git(["rev-list", "--max-parents=0", "--all"], {
+            cwd: worktree,
+          })
+          const [id] = result
+            .text()
+            .split("\n")
+            .filter(Boolean)
+            .map((x) => x.trim())
+            .toSorted()
+          if (!id) continue
+          projectID = id
+
+          await Filesystem.writeJson(path.join(dir, "project", projectID + ".json"), {
+            id,
+            vcs: "git",
+            worktree,
+            time: {
+              created: Date.now(),
+              initialized: Date.now(),
+            },
+          })
+
+          log.info(`migrating sessions for project ${projectID}`)
+          for (const sessionFile of await Glob.scan("storage/session/info/*.json", {
+            cwd: fullProjectDir,
+            absolute: true,
+          })) {
+            const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
+            log.info("copying", {
+              sessionFile,
+              dest,
+            })
+            const session = await Filesystem.readJson<any>(sessionFile)
+            await Filesystem.writeJson(dest, session)
+            log.info(`migrating messages for session ${session.id}`)
+            for (const msgFile of await Glob.scan(`storage/session/message/${session.id}/*.json`, {
+              cwd: fullProjectDir,
+              absolute: true,
+            })) {
+              const dest = path.join(dir, "message", session.id, path.basename(msgFile))
+              log.info("copying", {
+                msgFile,
+                dest,
+              })
+              const message = await Filesystem.readJson<any>(msgFile)
+              await Filesystem.writeJson(dest, message)
+
+              log.info(`migrating parts for message ${message.id}`)
+              for (const partFile of await Glob.scan(`storage/session/part/${session.id}/${message.id}/*.json`, {
+                cwd: fullProjectDir,
+                absolute: true,
+              })) {
+                const dest = path.join(dir, "part", message.id, path.basename(partFile))
+                const part = await Filesystem.readJson(partFile)
+                log.info("copying", {
+                  partFile,
+                  dest,
+                })
+                await Filesystem.writeJson(dest, part)
+              }
+            }
+          }
+        }
+      }
+    },
+    async (dir) => {
+      for (const item of await Glob.scan("session/*/*.json", {
+        cwd: dir,
+        absolute: true,
+      })) {
+        const session = await Filesystem.readJson<any>(item)
+        if (!session.projectID) continue
+        if (!session.summary?.diffs) continue
+        const { diffs } = session.summary
+        await Filesystem.write(path.join(dir, "session_diff", session.id + ".json"), JSON.stringify(diffs))
+        await Filesystem.writeJson(path.join(dir, "session", session.projectID, session.id + ".json"), {
+          ...session,
+          summary: {
+            additions: diffs.reduce((sum: any, x: any) => sum + x.additions, 0),
+            deletions: diffs.reduce((sum: any, x: any) => sum + x.deletions, 0),
+          },
+        })
+      }
+    },
+  ]
+
+  const state = lazy(async () => {
+    const dir = path.join(Global.Path.data, "storage")
+    const migration = await Filesystem.readJson<string>(path.join(dir, "migration"))
+      .then((x) => parseInt(x))
+      .catch(() => 0)
+    for (let index = migration; index < MIGRATIONS.length; index++) {
+      log.info("running migration", { index })
+      const migration = MIGRATIONS[index]
+      await migration(dir).catch(() => log.error("failed to run migration", { index }))
+      await Filesystem.write(path.join(dir, "migration"), (index + 1).toString())
+    }
+    return {
+      dir,
+    }
+  })
+
+  export async function remove(key: string[]) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      await fs.unlink(target).catch(() => {})
+    })
+  }
+
+  export async function read<T>(key: string[]) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.read(target)
+      const result = await Filesystem.readJson<T>(target)
+      return result as T
+    })
+  }
+
+  export async function update<T>(key: string[], fn: (draft: T) => void) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      const content = await Filesystem.readJson<T>(target)
+      fn(content as T)
+      await Filesystem.writeJson(target, content)
+      return content
+    })
+  }
+
+  export async function write<T>(key: string[], content: T) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      await Filesystem.writeJson(target, content)
+    })
+  }
+
+  async function withErrorHandling<T>(body: () => Promise<T>) {
+    return body().catch((e) => {
+      if (!(e instanceof Error)) throw e
+      const errnoException = e as NodeJS.ErrnoException
+      if (errnoException.code === "ENOENT") {
+        throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` })
+      }
+      throw e
+    })
+  }
+
+  export async function list(prefix: string[]) {
+    const dir = await state().then((x) => x.dir)
+    try {
+      const result = await Glob.scan("**/*", {
+        cwd: path.join(dir, ...prefix),
+        include: "file",
+      }).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]))
+      result.sort()
+      return result
+    } catch {
+      return []
+    }
+  }
+}