@toolkit-cli/toolkode 1.3.7

package/src/snapshot/index.ts

@@ -0,0 +1,489 @@
+import { NodeFileSystem, NodePath } from "@effect/platform-node"
+import { Cause, Duration, Effect, Layer, Schedule, Semaphore, ServiceMap, Stream } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"
+import path from "path"
+import z from "zod"
+import * as CrossSpawnSpawner from "@/effect/cross-spawn-spawner"
+import { InstanceState } from "@/effect/instance-state"
+import { makeRuntime } from "@/effect/run-service"
+import { AppFileSystem } from "@/filesystem"
+import { Hash } from "@/util/hash"
+import { Config } from "../config/config"
+import { Global } from "../global"
+import { Log } from "../util/log"
+
+export namespace Snapshot {
+  export const Patch = z.object({
+    hash: z.string(),
+    files: z.string().array(),
+  })
+  export type Patch = z.infer<typeof Patch>
+
+  export const FileDiff = z
+    .object({
+      file: z.string(),
+      before: z.string(),
+      after: z.string(),
+      additions: z.number(),
+      deletions: z.number(),
+      status: z.enum(["added", "deleted", "modified"]).optional(),
+    })
+    .meta({
+      ref: "FileDiff",
+    })
+  export type FileDiff = z.infer<typeof FileDiff>
+
+  const log = Log.create({ service: "snapshot" })
+  const prune = "7.days"
+  const limit = 2 * 1024 * 1024
+  const core = ["-c", "core.longpaths=true", "-c", "core.symlinks=true"]
+  const cfg = ["-c", "core.autocrlf=false", ...core]
+  const quote = [...cfg, "-c", "core.quotepath=false"]
+  interface GitResult {
+    readonly code: ChildProcessSpawner.ExitCode
+    readonly text: string
+    readonly stderr: string
+  }
+
+  type State = Omit<Interface, "init">
+
+  export interface Interface {
+    readonly init: () => Effect.Effect<void>
+    readonly cleanup: () => Effect.Effect<void>
+    readonly track: () => Effect.Effect<string | undefined>
+    readonly patch: (hash: string) => Effect.Effect<Snapshot.Patch>
+    readonly restore: (snapshot: string) => Effect.Effect<void>
+    readonly revert: (patches: Snapshot.Patch[]) => Effect.Effect<void>
+    readonly diff: (hash: string) => Effect.Effect<string>
+    readonly diffFull: (from: string, to: string) => Effect.Effect<Snapshot.FileDiff[]>
+  }
+
+  export class Service extends ServiceMap.Service<Service, Interface>()("@opencode/Snapshot") {}
+
+  export const layer: Layer.Layer<
+    Service,
+    never,
+    AppFileSystem.Service | ChildProcessSpawner.ChildProcessSpawner | Config.Service
+  > = Layer.effect(
+    Service,
+    Effect.gen(function* () {
+      const fs = yield* AppFileSystem.Service
+      const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
+      const config = yield* Config.Service
+      const locks = new Map<string, Semaphore.Semaphore>()
+
+      const lock = (key: string) => {
+        const hit = locks.get(key)
+        if (hit) return hit
+
+        const next = Semaphore.makeUnsafe(1)
+        locks.set(key, next)
+        return next
+      }
+
+      const state = yield* InstanceState.make<State>(
+        Effect.fn("Snapshot.state")(function* (ctx) {
+          const state = {
+            directory: ctx.directory,
+            worktree: ctx.worktree,
+            gitdir: path.join(Global.Path.data, "snapshot", ctx.project.id, Hash.fast(ctx.worktree)),
+            vcs: ctx.project.vcs,
+          }
+
+          const args = (cmd: string[]) => ["--git-dir", state.gitdir, "--work-tree", state.worktree, ...cmd]
+
+          const git = Effect.fnUntraced(
+            function* (cmd: string[], opts?: { cwd?: string; env?: Record<string, string> }) {
+              const proc = ChildProcess.make("git", cmd, {
+                cwd: opts?.cwd,
+                env: opts?.env,
+                extendEnv: true,
+              })
+              const handle = yield* spawner.spawn(proc)
+              const [text, stderr] = yield* Effect.all(
+                [Stream.mkString(Stream.decodeText(handle.stdout)), Stream.mkString(Stream.decodeText(handle.stderr))],
+                { concurrency: 2 },
+              )
+              const code = yield* handle.exitCode
+              return { code, text, stderr } satisfies GitResult
+            },
+            Effect.scoped,
+            Effect.catch((err) =>
+              Effect.succeed({
+                code: ChildProcessSpawner.ExitCode(1),
+                text: "",
+                stderr: String(err),
+              }),
+            ),
+          )
+
+          const exists = (file: string) => fs.exists(file).pipe(Effect.orDie)
+          const read = (file: string) => fs.readFileString(file).pipe(Effect.catch(() => Effect.succeed("")))
+          const remove = (file: string) => fs.remove(file).pipe(Effect.catch(() => Effect.void))
+          const locked = <A, E, R>(fx: Effect.Effect<A, E, R>) => lock(state.gitdir).withPermits(1)(fx)
+
+          const enabled = Effect.fnUntraced(function* () {
+            if (state.vcs !== "git") return false
+            return (yield* config.get()).snapshot !== false
+          })
+
+          const excludes = Effect.fnUntraced(function* () {
+            const result = yield* git(["rev-parse", "--path-format=absolute", "--git-path", "info/exclude"], {
+              cwd: state.worktree,
+            })
+            const file = result.text.trim()
+            if (!file) return
+            if (!(yield* exists(file))) return
+            return file
+          })
+
+          const sync = Effect.fnUntraced(function* (list: string[] = []) {
+            const file = yield* excludes()
+            const target = path.join(state.gitdir, "info", "exclude")
+            const text = [
+              file ? (yield* read(file)).trimEnd() : "",
+              ...list.map((item) => `/${item.replaceAll("\\", "/")}`),
+            ]
+              .filter(Boolean)
+              .join("\n")
+            yield* fs.ensureDir(path.join(state.gitdir, "info")).pipe(Effect.orDie)
+            yield* fs.writeFileString(target, text ? `${text}\n` : "").pipe(Effect.orDie)
+          })
+
+          const add = Effect.fnUntraced(function* () {
+            yield* sync()
+            const [diff, other] = yield* Effect.all(
+              [
+                git([...quote, ...args(["diff-files", "--name-only", "-z", "--", "."])], {
+                  cwd: state.directory,
+                }),
+                git([...quote, ...args(["ls-files", "--others", "--exclude-standard", "-z", "--", "."])], {
+                  cwd: state.directory,
+                }),
+              ],
+              { concurrency: 2 },
+            )
+            if (diff.code !== 0 || other.code !== 0) {
+              log.warn("failed to list snapshot files", {
+                diffCode: diff.code,
+                diffStderr: diff.stderr,
+                otherCode: other.code,
+                otherStderr: other.stderr,
+              })
+              return
+            }
+
+            const tracked = diff.text.split("\0").filter(Boolean)
+            const all = Array.from(new Set([...tracked, ...other.text.split("\0").filter(Boolean)]))
+            if (!all.length) return
+
+            const large = (yield* Effect.all(
+              all.map((item) =>
+                fs
+                  .stat(path.join(state.directory, item))
+                  .pipe(Effect.catch(() => Effect.void))
+                  .pipe(
+                    Effect.map((stat) => {
+                      if (!stat || stat.type !== "File") return
+                      const size = typeof stat.size === "bigint" ? Number(stat.size) : stat.size
+                      return size > limit ? item : undefined
+                    }),
+                  ),
+              ),
+              { concurrency: 8 },
+            )).filter((item): item is string => Boolean(item))
+            yield* sync(large)
+            const result = yield* git([...cfg, ...args(["add", "--sparse", "."])], { cwd: state.directory })
+            if (result.code !== 0) {
+              log.warn("failed to add snapshot files", {
+                exitCode: result.code,
+                stderr: result.stderr,
+              })
+            }
+          })
+
+          const cleanup = Effect.fnUntraced(function* () {
+            return yield* locked(
+              Effect.gen(function* () {
+                if (!(yield* enabled())) return
+                if (!(yield* exists(state.gitdir))) return
+                const result = yield* git(args(["gc", `--prune=${prune}`]), { cwd: state.directory })
+                if (result.code !== 0) {
+                  log.warn("cleanup failed", {
+                    exitCode: result.code,
+                    stderr: result.stderr,
+                  })
+                  return
+                }
+                log.info("cleanup", { prune })
+              }),
+            )
+          })
+
+          const track = Effect.fnUntraced(function* () {
+            return yield* locked(
+              Effect.gen(function* () {
+                if (!(yield* enabled())) return
+                const existed = yield* exists(state.gitdir)
+                yield* fs.ensureDir(state.gitdir).pipe(Effect.orDie)
+                if (!existed) {
+                  yield* git(["init"], {
+                    env: { GIT_DIR: state.gitdir, GIT_WORK_TREE: state.worktree },
+                  })
+                  yield* git(["--git-dir", state.gitdir, "config", "core.autocrlf", "false"])
+                  yield* git(["--git-dir", state.gitdir, "config", "core.longpaths", "true"])
+                  yield* git(["--git-dir", state.gitdir, "config", "core.symlinks", "true"])
+                  yield* git(["--git-dir", state.gitdir, "config", "core.fsmonitor", "false"])
+                  log.info("initialized")
+                }
+                yield* add()
+                const result = yield* git(args(["write-tree"]), { cwd: state.directory })
+                const hash = result.text.trim()
+                log.info("tracking", { hash, cwd: state.directory, git: state.gitdir })
+                return hash
+              }),
+            )
+          })
+
+          const patch = Effect.fnUntraced(function* (hash: string) {
+            return yield* locked(
+              Effect.gen(function* () {
+                yield* add()
+                const result = yield* git(
+                  [...quote, ...args(["diff", "--cached", "--no-ext-diff", "--name-only", hash, "--", "."])],
+                  {
+                    cwd: state.directory,
+                  },
+                )
+                if (result.code !== 0) {
+                  log.warn("failed to get diff", { hash, exitCode: result.code })
+                  return { hash, files: [] }
+                }
+                return {
+                  hash,
+                  files: result.text
+                    .trim()
+                    .split("\n")
+                    .map((x) => x.trim())
+                    .filter(Boolean)
+                    .map((x) => path.join(state.worktree, x).replaceAll("\\", "/")),
+                }
+              }),
+            )
+          })
+
+          const restore = Effect.fnUntraced(function* (snapshot: string) {
+            return yield* locked(
+              Effect.gen(function* () {
+                log.info("restore", { commit: snapshot })
+                const result = yield* git([...core, ...args(["read-tree", snapshot])], { cwd: state.worktree })
+                if (result.code === 0) {
+                  const checkout = yield* git([...core, ...args(["checkout-index", "-a", "-f"])], {
+                    cwd: state.worktree,
+                  })
+                  if (checkout.code === 0) return
+                  log.error("failed to restore snapshot", {
+                    snapshot,
+                    exitCode: checkout.code,
+                    stderr: checkout.stderr,
+                  })
+                  return
+                }
+                log.error("failed to restore snapshot", {
+                  snapshot,
+                  exitCode: result.code,
+                  stderr: result.stderr,
+                })
+              }),
+            )
+          })
+
+          const revert = Effect.fnUntraced(function* (patches: Snapshot.Patch[]) {
+            return yield* locked(
+              Effect.gen(function* () {
+                const seen = new Set<string>()
+                for (const item of patches) {
+                  for (const file of item.files) {
+                    if (seen.has(file)) continue
+                    seen.add(file)
+                    log.info("reverting", { file, hash: item.hash })
+                    const result = yield* git([...core, ...args(["checkout", item.hash, "--", file])], {
+                      cwd: state.worktree,
+                    })
+                    if (result.code !== 0) {
+                      const rel = path.relative(state.worktree, file)
+                      const tree = yield* git([...core, ...args(["ls-tree", item.hash, "--", rel])], {
+                        cwd: state.worktree,
+                      })
+                      if (tree.code === 0 && tree.text.trim()) {
+                        log.info("file existed in snapshot but checkout failed, keeping", { file })
+                      } else {
+                        log.info("file did not exist in snapshot, deleting", { file })
+                        yield* remove(file)
+                      }
+                    }
+                  }
+                }
+              }),
+            )
+          })
+
+          const diff = Effect.fnUntraced(function* (hash: string) {
+            return yield* locked(
+              Effect.gen(function* () {
+                yield* add()
+                const result = yield* git([...quote, ...args(["diff", "--cached", "--no-ext-diff", hash, "--", "."])], {
+                  cwd: state.worktree,
+                })
+                if (result.code !== 0) {
+                  log.warn("failed to get diff", {
+                    hash,
+                    exitCode: result.code,
+                    stderr: result.stderr,
+                  })
+                  return ""
+                }
+                return result.text.trim()
+              }),
+            )
+          })
+
+          const diffFull = Effect.fnUntraced(function* (from: string, to: string) {
+            return yield* locked(
+              Effect.gen(function* () {
+                const result: Snapshot.FileDiff[] = []
+                const status = new Map<string, "added" | "deleted" | "modified">()
+
+                const statuses = yield* git(
+                  [...quote, ...args(["diff", "--no-ext-diff", "--name-status", "--no-renames", from, to, "--", "."])],
+                  { cwd: state.directory },
+                )
+
+                for (const line of statuses.text.trim().split("\n")) {
+                  if (!line) continue
+                  const [code, file] = line.split("\t")
+                  if (!code || !file) continue
+                  status.set(file, code.startsWith("A") ? "added" : code.startsWith("D") ? "deleted" : "modified")
+                }
+
+                const numstat = yield* git(
+                  [...quote, ...args(["diff", "--no-ext-diff", "--no-renames", "--numstat", from, to, "--", "."])],
+                  {
+                    cwd: state.directory,
+                  },
+                )
+
+                for (const line of numstat.text.trim().split("\n")) {
+                  if (!line) continue
+                  const [adds, dels, file] = line.split("\t")
+                  if (!file) continue
+                  const binary = adds === "-" && dels === "-"
+                  const [before, after] = binary
+                    ? ["", ""]
+                    : yield* Effect.all(
+                        [
+                          git([...cfg, ...args(["show", `${from}:${file}`])]).pipe(Effect.map((item) => item.text)),
+                          git([...cfg, ...args(["show", `${to}:${file}`])]).pipe(Effect.map((item) => item.text)),
+                        ],
+                        { concurrency: 2 },
+                      )
+                  const additions = binary ? 0 : parseInt(adds)
+                  const deletions = binary ? 0 : parseInt(dels)
+                  result.push({
+                    file,
+                    before,
+                    after,
+                    additions: Number.isFinite(additions) ? additions : 0,
+                    deletions: Number.isFinite(deletions) ? deletions : 0,
+                    status: status.get(file) ?? "modified",
+                  })
+                }
+
+                return result
+              }),
+            )
+          })
+
+          yield* cleanup().pipe(
+            Effect.catchCause((cause) => {
+              log.error("cleanup loop failed", { cause: Cause.pretty(cause) })
+              return Effect.void
+            }),
+            Effect.repeat(Schedule.spaced(Duration.hours(1))),
+            Effect.delay(Duration.minutes(1)),
+            Effect.forkScoped,
+          )
+
+          return { cleanup, track, patch, restore, revert, diff, diffFull }
+        }),
+      )
+
+      return Service.of({
+        init: Effect.fn("Snapshot.init")(function* () {
+          yield* InstanceState.get(state)
+        }),
+        cleanup: Effect.fn("Snapshot.cleanup")(function* () {
+          return yield* InstanceState.useEffect(state, (s) => s.cleanup())
+        }),
+        track: Effect.fn("Snapshot.track")(function* () {
+          return yield* InstanceState.useEffect(state, (s) => s.track())
+        }),
+        patch: Effect.fn("Snapshot.patch")(function* (hash: string) {
+          return yield* InstanceState.useEffect(state, (s) => s.patch(hash))
+        }),
+        restore: Effect.fn("Snapshot.restore")(function* (snapshot: string) {
+          return yield* InstanceState.useEffect(state, (s) => s.restore(snapshot))
+        }),
+        revert: Effect.fn("Snapshot.revert")(function* (patches: Snapshot.Patch[]) {
+          return yield* InstanceState.useEffect(state, (s) => s.revert(patches))
+        }),
+        diff: Effect.fn("Snapshot.diff")(function* (hash: string) {
+          return yield* InstanceState.useEffect(state, (s) => s.diff(hash))
+        }),
+        diffFull: Effect.fn("Snapshot.diffFull")(function* (from: string, to: string) {
+          return yield* InstanceState.useEffect(state, (s) => s.diffFull(from, to))
+        }),
+      })
+    }),
+  )
+
+  export const defaultLayer = layer.pipe(
+    Layer.provide(CrossSpawnSpawner.defaultLayer),
+    Layer.provide(AppFileSystem.defaultLayer),
+    Layer.provide(Config.defaultLayer),
+  )
+
+  const { runPromise } = makeRuntime(Service, defaultLayer)
+
+  export async function init() {
+    return runPromise((svc) => svc.init())
+  }
+
+  export async function cleanup() {
+    return runPromise((svc) => svc.cleanup())
+  }
+
+  export async function track() {
+    return runPromise((svc) => svc.track())
+  }
+
+  export async function patch(hash: string) {
+    return runPromise((svc) => svc.patch(hash))
+  }
+
+  export async function restore(snapshot: string) {
+    return runPromise((svc) => svc.restore(snapshot))
+  }
+
+  export async function revert(patches: Patch[]) {
+    return runPromise((svc) => svc.revert(patches))
+  }
+
+  export async function diff(hash: string) {
+    return runPromise((svc) => svc.diff(hash))
+  }
+
+  export async function diffFull(from: string, to: string) {
+    return runPromise((svc) => svc.diffFull(from, to))
+  }
+}

package/src/sql.d.ts

@@ -0,0 +1,4 @@
+declare module "*.sql" {
+  const content: string
+  export default content
+}

package/src/storage/db.bun.ts

@@ -0,0 +1,8 @@
+import { Database } from "bun:sqlite"
+import { drizzle } from "drizzle-orm/bun-sqlite"
+
+export function init(path: string) {
+  const sqlite = new Database(path, { create: true })
+  const db = drizzle({ client: sqlite })
+  return db
+}

package/src/storage/db.node.ts

@@ -0,0 +1,8 @@
+import { DatabaseSync } from "node:sqlite"
+import { drizzle } from "drizzle-orm/node-sqlite"
+
+export function init(path: string) {
+  const sqlite = new DatabaseSync(path)
+  const db = drizzle({ client: sqlite })
+  return db
+}

package/src/storage/db.ts

@@ -0,0 +1,177 @@
+import { type SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
+import { migrate } from "drizzle-orm/bun-sqlite/migrator"
+import { type SQLiteTransaction } from "drizzle-orm/sqlite-core"
+export * from "drizzle-orm"
+import { Context } from "../util/context"
+import { lazy } from "../util/lazy"
+import { Global } from "../global"
+import { Log } from "../util/log"
+import { NamedError } from "@toolkit-cli/util/error"
+import z from "zod"
+import path from "path"
+import { readFileSync, readdirSync, existsSync } from "fs"
+import { Installation } from "../installation"
+import { Flag } from "../flag/flag"
+import { iife } from "@/util/iife"
+import { init } from "#db"
+
+declare const OPENCODE_MIGRATIONS: { sql: string; timestamp: number; name: string }[] | undefined
+
+export const NotFoundError = NamedError.create(
+  "NotFoundError",
+  z.object({
+    message: z.string(),
+  }),
+)
+
+const log = Log.create({ service: "db" })
+
+export namespace Database {
+  export function getChannelPath() {
+    const channel = Installation.CHANNEL
+    if (["latest", "beta"].includes(channel) || Flag.OPENCODE_DISABLE_CHANNEL_DB)
+      return path.join(Global.Path.data, "toolkode.db")
+    const safe = channel.replace(/[^a-zA-Z0-9._-]/g, "-")
+    return path.join(Global.Path.data, `toolkode-${safe}.db`)
+  }
+
+  export const Path = iife(() => {
+    if (Flag.OPENCODE_DB) {
+      if (Flag.OPENCODE_DB === ":memory:" || path.isAbsolute(Flag.OPENCODE_DB)) return Flag.OPENCODE_DB
+      return path.join(Global.Path.data, Flag.OPENCODE_DB)
+    }
+    return getChannelPath()
+  })
+
+  export type Transaction = SQLiteTransaction<"sync", void>
+
+  type Client = SQLiteBunDatabase
+
+  type Journal = { sql: string; timestamp: number; name: string }[]
+
+  function time(tag: string) {
+    const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/.exec(tag)
+    if (!match) return 0
+    return Date.UTC(
+      Number(match[1]),
+      Number(match[2]) - 1,
+      Number(match[3]),
+      Number(match[4]),
+      Number(match[5]),
+      Number(match[6]),
+    )
+  }
+
+  function migrations(dir: string): Journal {
+    const dirs = readdirSync(dir, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+
+    const sql = dirs
+      .map((name) => {
+        const file = path.join(dir, name, "migration.sql")
+        if (!existsSync(file)) return
+        return {
+          sql: readFileSync(file, "utf-8"),
+          timestamp: time(name),
+          name,
+        }
+      })
+      .filter(Boolean) as Journal
+
+    return sql.sort((a, b) => a.timestamp - b.timestamp)
+  }
+
+  export const Client = lazy(() => {
+    log.info("opening database", { path: Path })
+
+    const db = init(Path)
+
+    db.run("PRAGMA journal_mode = WAL")
+    db.run("PRAGMA synchronous = NORMAL")
+    db.run("PRAGMA busy_timeout = 5000")
+    db.run("PRAGMA cache_size = -64000")
+    db.run("PRAGMA foreign_keys = ON")
+    db.run("PRAGMA wal_checkpoint(PASSIVE)")
+
+    // Apply schema migrations
+    const entries =
+      typeof OPENCODE_MIGRATIONS !== "undefined"
+        ? OPENCODE_MIGRATIONS
+        : migrations(path.join(import.meta.dirname, "../../migration"))
+    if (entries.length > 0) {
+      log.info("applying migrations", {
+        count: entries.length,
+        mode: typeof OPENCODE_MIGRATIONS !== "undefined" ? "bundled" : "dev",
+      })
+      if (Flag.OPENCODE_SKIP_MIGRATIONS) {
+        for (const item of entries) {
+          item.sql = "select 1;"
+        }
+      }
+      migrate(db, entries)
+    }
+
+    return db
+  })
+
+  export function close() {
+    Client().$client.close()
+    Client.reset()
+  }
+
+  export type TxOrDb = Transaction | Client
+
+  const ctx = Context.create<{
+    tx: TxOrDb
+    effects: (() => void | Promise<void>)[]
+  }>("database")
+
+  export function use<T>(callback: (trx: TxOrDb) => T): T {
+    try {
+      return callback(ctx.use().tx)
+    } catch (err) {
+      if (err instanceof Context.NotFound) {
+        const effects: (() => void | Promise<void>)[] = []
+        const result = ctx.provide({ effects, tx: Client() }, () => callback(Client()))
+        for (const effect of effects) effect()
+        return result
+      }
+      throw err
+    }
+  }
+
+  export function effect(fn: () => any | Promise<any>) {
+    try {
+      ctx.use().effects.push(fn)
+    } catch {
+      fn()
+    }
+  }
+
+  type NotPromise<T> = T extends Promise<any> ? never : T
+
+  export function transaction<T>(
+    callback: (tx: TxOrDb) => NotPromise<T>,
+    options?: {
+      behavior?: "deferred" | "immediate" | "exclusive"
+    },
+  ): NotPromise<T> {
+    try {
+      return callback(ctx.use().tx)
+    } catch (err) {
+      if (err instanceof Context.NotFound) {
+        const effects: (() => void | Promise<void>)[] = []
+        const result = Client().transaction(
+          (tx: TxOrDb) => {
+            return ctx.provide({ tx, effects }, () => callback(tx))
+          },
+          { behavior: options?.behavior },
+        )
+        for (const effect of effects) effect()
+        return result as NotPromise<T>
+      }
+      throw err
+    }
+  }
+}