easc-cli 1.1.28

package/src/snapshot/index.ts

@@ -0,0 +1,236 @@
+import { $ } from "bun"
+import path from "path"
+import fs from "fs/promises"
+import { Log } from "../util/log"
+import { Global } from "../global"
+import z from "zod"
+import { Config } from "../config/config"
+import { Instance } from "../project/instance"
+import { Scheduler } from "../scheduler"
+
+export namespace Snapshot {
+  const log = Log.create({ service: "snapshot" })
+  const hour = 60 * 60 * 1000
+  const prune = "7.days"
+
+  export function init() {
+    Scheduler.register({
+      id: "snapshot.cleanup",
+      interval: hour,
+      run: cleanup,
+      scope: "instance",
+    })
+  }
+
+  export async function cleanup() {
+    if (Instance.project.vcs !== "git") return
+    const cfg = await Config.get()
+    if (cfg.snapshot === false) return
+    const git = gitdir()
+    const exists = await fs
+      .stat(git)
+      .then(() => true)
+      .catch(() => false)
+    if (!exists) return
+    const result = await $`git --git-dir ${git} --work-tree ${Instance.worktree} gc --prune=${prune}`
+      .quiet()
+      .cwd(Instance.directory)
+      .nothrow()
+    if (result.exitCode !== 0) {
+      log.warn("cleanup failed", {
+        exitCode: result.exitCode,
+        stderr: result.stderr.toString(),
+        stdout: result.stdout.toString(),
+      })
+      return
+    }
+    log.info("cleanup", { prune })
+  }
+
+  export async function track() {
+    if (Instance.project.vcs !== "git") return
+    const cfg = await Config.get()
+    if (cfg.snapshot === false) return
+    const git = gitdir()
+    if (await fs.mkdir(git, { recursive: true })) {
+      await $`git init`
+        .env({
+          ...process.env,
+          GIT_DIR: git,
+          GIT_WORK_TREE: Instance.worktree,
+        })
+        .quiet()
+        .nothrow()
+      // Configure git to not convert line endings on Windows
+      await $`git --git-dir ${git} config core.autocrlf false`.quiet().nothrow()
+      log.info("initialized")
+    }
+    await $`git --git-dir ${git} --work-tree ${Instance.worktree} add .`.quiet().cwd(Instance.directory).nothrow()
+    const hash = await $`git --git-dir ${git} --work-tree ${Instance.worktree} write-tree`
+      .quiet()
+      .cwd(Instance.directory)
+      .nothrow()
+      .text()
+    log.info("tracking", { hash, cwd: Instance.directory, git })
+    return hash.trim()
+  }
+
+  export const Patch = z.object({
+    hash: z.string(),
+    files: z.string().array(),
+  })
+  export type Patch = z.infer<typeof Patch>
+
+  export async function patch(hash: string): Promise<Patch> {
+    const git = gitdir()
+    await $`git --git-dir ${git} --work-tree ${Instance.worktree} add .`.quiet().cwd(Instance.directory).nothrow()
+    const result =
+      await $`git -c core.autocrlf=false --git-dir ${git} --work-tree ${Instance.worktree} diff --no-ext-diff --name-only ${hash} -- .`
+        .quiet()
+        .cwd(Instance.directory)
+        .nothrow()
+
+    // If git diff fails, return empty patch
+    if (result.exitCode !== 0) {
+      log.warn("failed to get diff", { hash, exitCode: result.exitCode })
+      return { hash, files: [] }
+    }
+
+    const files = result.text()
+    return {
+      hash,
+      files: files
+        .trim()
+        .split("\n")
+        .map((x) => x.trim())
+        .filter(Boolean)
+        .map((x) => path.join(Instance.worktree, x)),
+    }
+  }
+
+  export async function restore(snapshot: string) {
+    log.info("restore", { commit: snapshot })
+    const git = gitdir()
+    const result =
+      await $`git --git-dir ${git} --work-tree ${Instance.worktree} read-tree ${snapshot} && git --git-dir ${git} --work-tree ${Instance.worktree} checkout-index -a -f`
+        .quiet()
+        .cwd(Instance.worktree)
+        .nothrow()
+
+    if (result.exitCode !== 0) {
+      log.error("failed to restore snapshot", {
+        snapshot,
+        exitCode: result.exitCode,
+        stderr: result.stderr.toString(),
+        stdout: result.stdout.toString(),
+      })
+    }
+  }
+
+  export async function revert(patches: Patch[]) {
+    const files = new Set<string>()
+    const git = gitdir()
+    for (const item of patches) {
+      for (const file of item.files) {
+        if (files.has(file)) continue
+        log.info("reverting", { file, hash: item.hash })
+        const result = await $`git --git-dir ${git} --work-tree ${Instance.worktree} checkout ${item.hash} -- ${file}`
+          .quiet()
+          .cwd(Instance.worktree)
+          .nothrow()
+        if (result.exitCode !== 0) {
+          const relativePath = path.relative(Instance.worktree, file)
+          const checkTree =
+            await $`git --git-dir ${git} --work-tree ${Instance.worktree} ls-tree ${item.hash} -- ${relativePath}`
+              .quiet()
+              .cwd(Instance.worktree)
+              .nothrow()
+          if (checkTree.exitCode === 0 && checkTree.text().trim()) {
+            log.info("file existed in snapshot but checkout failed, keeping", {
+              file,
+            })
+          } else {
+            log.info("file did not exist in snapshot, deleting", { file })
+            await fs.unlink(file).catch(() => {})
+          }
+        }
+        files.add(file)
+      }
+    }
+  }
+
+  export async function diff(hash: string) {
+    const git = gitdir()
+    await $`git --git-dir ${git} --work-tree ${Instance.worktree} add .`.quiet().cwd(Instance.directory).nothrow()
+    const result =
+      await $`git -c core.autocrlf=false --git-dir ${git} --work-tree ${Instance.worktree} diff --no-ext-diff ${hash} -- .`
+        .quiet()
+        .cwd(Instance.worktree)
+        .nothrow()
+
+    if (result.exitCode !== 0) {
+      log.warn("failed to get diff", {
+        hash,
+        exitCode: result.exitCode,
+        stderr: result.stderr.toString(),
+        stdout: result.stdout.toString(),
+      })
+      return ""
+    }
+
+    return result.text().trim()
+  }
+
+  export const FileDiff = z
+    .object({
+      file: z.string(),
+      before: z.string(),
+      after: z.string(),
+      additions: z.number(),
+      deletions: z.number(),
+    })
+    .meta({
+      ref: "FileDiff",
+    })
+  export type FileDiff = z.infer<typeof FileDiff>
+  export async function diffFull(from: string, to: string): Promise<FileDiff[]> {
+    const git = gitdir()
+    const result: FileDiff[] = []
+    for await (const line of $`git -c core.autocrlf=false --git-dir ${git} --work-tree ${Instance.worktree} diff --no-ext-diff --no-renames --numstat ${from} ${to} -- .`
+      .quiet()
+      .cwd(Instance.directory)
+      .nothrow()
+      .lines()) {
+      if (!line) continue
+      const [additions, deletions, file] = line.split("\t")
+      const isBinaryFile = additions === "-" && deletions === "-"
+      const before = isBinaryFile
+        ? ""
+        : await $`git -c core.autocrlf=false --git-dir ${git} --work-tree ${Instance.worktree} show ${from}:${file}`
+            .quiet()
+            .nothrow()
+            .text()
+      const after = isBinaryFile
+        ? ""
+        : await $`git -c core.autocrlf=false --git-dir ${git} --work-tree ${Instance.worktree} show ${to}:${file}`
+            .quiet()
+            .nothrow()
+            .text()
+      const added = isBinaryFile ? 0 : parseInt(additions)
+      const deleted = isBinaryFile ? 0 : parseInt(deletions)
+      result.push({
+        file,
+        before,
+        after,
+        additions: Number.isFinite(added) ? added : 0,
+        deletions: Number.isFinite(deleted) ? deleted : 0,
+      })
+    }
+    return result
+  }
+
+  function gitdir() {
+    const project = Instance.project
+    return path.join(Global.Path.data, "snapshot", project.id)
+  }
+}

package/src/storage/storage.ts

@@ -0,0 +1,227 @@
+import { Log } from "../util/log"
+import path from "path"
+import fs from "fs/promises"
+import { Global } from "../global"
+import { Filesystem } from "../util/filesystem"
+import { lazy } from "../util/lazy"
+import { Lock } from "../util/lock"
+import { $ } from "bun"
+import { NamedError } from "@eliseart.ai/util/error"
+import z from "zod"
+
+export namespace Storage {
+  const log = Log.create({ service: "storage" })
+
+  type Migration = (dir: string) => Promise<void>
+
+  export const NotFoundError = NamedError.create(
+    "NotFoundError",
+    z.object({
+      message: z.string(),
+    }),
+  )
+
+  const MIGRATIONS: Migration[] = [
+    async (dir) => {
+      const project = path.resolve(dir, "../project")
+      if (!(await Filesystem.isDir(project))) return
+      for await (const projectDir of new Bun.Glob("*").scan({
+        cwd: project,
+        onlyFiles: false,
+      })) {
+        log.info(`migrating project ${projectDir}`)
+        let projectID = projectDir
+        const fullProjectDir = path.join(project, projectDir)
+        let worktree = "/"
+
+        if (projectID !== "global") {
+          for await (const msgFile of new Bun.Glob("storage/session/message/*/*.json").scan({
+            cwd: path.join(project, projectDir),
+            absolute: true,
+          })) {
+            const json = await Bun.file(msgFile).json()
+            worktree = json.path?.root
+            if (worktree) break
+          }
+          if (!worktree) continue
+          if (!(await Filesystem.isDir(worktree))) continue
+          const [id] = await $`git rev-list --max-parents=0 --all`
+            .quiet()
+            .nothrow()
+            .cwd(worktree)
+            .text()
+            .then((x) =>
+              x
+                .split("\n")
+                .filter(Boolean)
+                .map((x) => x.trim())
+                .toSorted(),
+            )
+          if (!id) continue
+          projectID = id
+
+          await Bun.write(
+            path.join(dir, "project", projectID + ".json"),
+            JSON.stringify({
+              id,
+              vcs: "git",
+              worktree,
+              time: {
+                created: Date.now(),
+                initialized: Date.now(),
+              },
+            }),
+          )
+
+          log.info(`migrating sessions for project ${projectID}`)
+          for await (const sessionFile of new Bun.Glob("storage/session/info/*.json").scan({
+            cwd: fullProjectDir,
+            absolute: true,
+          })) {
+            const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
+            log.info("copying", {
+              sessionFile,
+              dest,
+            })
+            const session = await Bun.file(sessionFile).json()
+            await Bun.write(dest, JSON.stringify(session))
+            log.info(`migrating messages for session ${session.id}`)
+            for await (const msgFile of new Bun.Glob(`storage/session/message/${session.id}/*.json`).scan({
+              cwd: fullProjectDir,
+              absolute: true,
+            })) {
+              const dest = path.join(dir, "message", session.id, path.basename(msgFile))
+              log.info("copying", {
+                msgFile,
+                dest,
+              })
+              const message = await Bun.file(msgFile).json()
+              await Bun.write(dest, JSON.stringify(message))
+
+              log.info(`migrating parts for message ${message.id}`)
+              for await (const partFile of new Bun.Glob(`storage/session/part/${session.id}/${message.id}/*.json`).scan(
+                {
+                  cwd: fullProjectDir,
+                  absolute: true,
+                },
+              )) {
+                const dest = path.join(dir, "part", message.id, path.basename(partFile))
+                const part = await Bun.file(partFile).json()
+                log.info("copying", {
+                  partFile,
+                  dest,
+                })
+                await Bun.write(dest, JSON.stringify(part))
+              }
+            }
+          }
+        }
+      }
+    },
+    async (dir) => {
+      for await (const item of new Bun.Glob("session/*/*.json").scan({
+        cwd: dir,
+        absolute: true,
+      })) {
+        const session = await Bun.file(item).json()
+        if (!session.projectID) continue
+        if (!session.summary?.diffs) continue
+        const { diffs } = session.summary
+        await Bun.file(path.join(dir, "session_diff", session.id + ".json")).write(JSON.stringify(diffs))
+        await Bun.file(path.join(dir, "session", session.projectID, session.id + ".json")).write(
+          JSON.stringify({
+            ...session,
+            summary: {
+              additions: diffs.reduce((sum: any, x: any) => sum + x.additions, 0),
+              deletions: diffs.reduce((sum: any, x: any) => sum + x.deletions, 0),
+            },
+          }),
+        )
+      }
+    },
+  ]
+
+  const state = lazy(async () => {
+    const dir = path.join(Global.Path.data, "storage")
+    const migration = await Bun.file(path.join(dir, "migration"))
+      .json()
+      .then((x) => parseInt(x))
+      .catch(() => 0)
+    for (let index = migration; index < MIGRATIONS.length; index++) {
+      log.info("running migration", { index })
+      const migration = MIGRATIONS[index]
+      await migration(dir).catch(() => log.error("failed to run migration", { index }))
+      await Bun.write(path.join(dir, "migration"), (index + 1).toString())
+    }
+    return {
+      dir,
+    }
+  })
+
+  export async function remove(key: string[]) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      await fs.unlink(target).catch(() => {})
+    })
+  }
+
+  export async function read<T>(key: string[]) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.read(target)
+      const result = await Bun.file(target).json()
+      return result as T
+    })
+  }
+
+  export async function update<T>(key: string[], fn: (draft: T) => void) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      const content = await Bun.file(target).json()
+      fn(content)
+      await Bun.write(target, JSON.stringify(content, null, 2))
+      return content as T
+    })
+  }
+
+  export async function write<T>(key: string[], content: T) {
+    const dir = await state().then((x) => x.dir)
+    const target = path.join(dir, ...key) + ".json"
+    return withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      await Bun.write(target, JSON.stringify(content, null, 2))
+    })
+  }
+
+  async function withErrorHandling<T>(body: () => Promise<T>) {
+    return body().catch((e) => {
+      if (!(e instanceof Error)) throw e
+      const errnoException = e as NodeJS.ErrnoException
+      if (errnoException.code === "ENOENT") {
+        throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` })
+      }
+      throw e
+    })
+  }
+
+  const glob = new Bun.Glob("**/*")
+  export async function list(prefix: string[]) {
+    const dir = await state().then((x) => x.dir)
+    try {
+      const result = await Array.fromAsync(
+        glob.scan({
+          cwd: path.join(dir, ...prefix),
+          onlyFiles: true,
+        }),
+      ).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]))
+      result.sort()
+      return result
+    } catch {
+      return []
+    }
+  }
+}