@dench.com/cli 0.4.3 → 0.4.5

package/home-sync-lib.ts

@@ -0,0 +1,185 @@
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+export const DEFAULT_LIVE_HOME = "/tmp/.dench-home";
+export const DEFAULT_SNAPSHOT_DIR = "/workspace/.dench/home/snapshots";
+export const DEFAULT_DEBOUNCE_MS = 2_000;
+export const DEFAULT_KEEP_SNAPSHOTS = 8;
+export const DEFAULT_STATUS_PATH = "/tmp/dench-home-sync.status.json";
+
+export type HomeSyncArgs = {
+  liveHome: string;
+  snapshotDir: string;
+  debounceMs: number;
+  keepSnapshots: number;
+  statusPath: string;
+  once: boolean;
+};
+
+export function parseHomeSyncArgs(argv: string[]): HomeSyncArgs {
+  const out: Partial<HomeSyncArgs> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i];
+    const value = argv[i + 1];
+    switch (flag) {
+      case "--live-home":
+        out.liveHome = value;
+        i++;
+        break;
+      case "--snapshot-dir":
+        out.snapshotDir = value;
+        i++;
+        break;
+      case "--debounce-ms":
+        out.debounceMs = Number.parseInt(value, 10);
+        i++;
+        break;
+      case "--keep-snapshots":
+        out.keepSnapshots = Number.parseInt(value, 10);
+        i++;
+        break;
+      case "--status-path":
+        out.statusPath = value;
+        i++;
+        break;
+      case "--once":
+        out.once = true;
+        break;
+      default:
+        // Ignore unknown flags so newer bootstraps can run on older CLIs.
+        break;
+    }
+  }
+
+  return {
+    liveHome: out.liveHome ?? DEFAULT_LIVE_HOME,
+    snapshotDir:
+      out.snapshotDir ??
+      process.env.DENCH_HOME_SNAPSHOT_DIR?.trim() ??
+      DEFAULT_SNAPSHOT_DIR,
+    debounceMs: Number.isFinite(out.debounceMs)
+      ? Math.max(250, out.debounceMs ?? DEFAULT_DEBOUNCE_MS)
+      : DEFAULT_DEBOUNCE_MS,
+    keepSnapshots: Number.isFinite(out.keepSnapshots)
+      ? Math.max(1, out.keepSnapshots ?? DEFAULT_KEEP_SNAPSHOTS)
+      : DEFAULT_KEEP_SNAPSHOTS,
+    statusPath: out.statusPath ?? DEFAULT_STATUS_PATH,
+    once: out.once ?? false,
+  };
+}
+
+const EXCLUDED_HOME_PATTERNS = [
+  ".cache",
+  ".cache/**",
+  ".npm/_cacache",
+  ".npm/_cacache/**",
+  ".npm/_logs",
+  ".npm/_logs/**",
+  ".bun/install/cache",
+  ".bun/install/cache/**",
+  ".cargo/registry/cache",
+  ".cargo/registry/cache/**",
+  ".cargo/registry/index",
+  ".cargo/registry/index/**",
+  ".cargo/git/checkouts",
+  ".cargo/git/checkouts/**",
+  ".cargo/git/db",
+  ".cargo/git/db/**",
+  "go/pkg/mod/cache",
+  "go/pkg/mod/cache/**",
+  ".config/google-chrome",
+  ".config/google-chrome/**",
+  ".config/chromium",
+  ".config/chromium/**",
+];
+
+export function homeSnapshotExcludes(): string[] {
+  return [...EXCLUDED_HOME_PATTERNS];
+}
+
+export function shouldIgnoreHomePath(path: string, liveHome: string): boolean {
+  const rel = relative(liveHome, path).replaceAll("\\", "/");
+  if (!rel || rel === ".") return false;
+  if (rel.startsWith("..")) return true;
+  return EXCLUDED_HOME_PATTERNS.some((pattern) => {
+    if (pattern.endsWith("/**")) {
+      const prefix = pattern.slice(0, -3);
+      return rel === prefix || rel.startsWith(`${prefix}/`);
+    }
+    return rel === pattern;
+  });
+}
+
+export function buildTarCreateArgs(args: {
+  liveHome: string;
+  archivePath: string;
+}): string[] {
+  return [
+    "--create",
+    "--file",
+    args.archivePath,
+    "--directory",
+    args.liveHome,
+    ...EXCLUDED_HOME_PATTERNS.map((pattern) => `--exclude=${pattern}`),
+    ".",
+  ];
+}
+
+export function buildSnapshotName(now: Date, pid = process.pid): string {
+  const stamp = now
+    .toISOString()
+    .replaceAll("-", "")
+    .replaceAll(":", "")
+    .replace(".", "");
+  return `home-${stamp}-${pid}.tar`;
+}
+
+export async function ensureHomeSyncDirs(args: {
+  liveHome: string;
+  snapshotDir: string;
+}): Promise<void> {
+  await mkdir(args.liveHome, { recursive: true });
+  await mkdir(args.snapshotDir, { recursive: true });
+}
+
+export async function listHomeSnapshots(
+  snapshotDir: string,
+): Promise<string[]> {
+  let entries: string[];
+  try {
+    entries = await readdir(snapshotDir);
+  } catch {
+    return [];
+  }
+
+  const snapshots = await Promise.all(
+    entries
+      .filter((entry) => /^home-\d{8}T\d{9}Z-\d+\.tar$/.test(entry))
+      .map(async (entry) => {
+        const path = join(snapshotDir, entry);
+        try {
+          const info = await stat(path);
+          return { path, mtimeMs: info.mtimeMs };
+        } catch {
+          return null;
+        }
+      }),
+  );
+
+  return snapshots
+    .filter((snapshot): snapshot is { path: string; mtimeMs: number } =>
+      Boolean(snapshot),
+    )
+    .sort((a, b) => b.mtimeMs - a.mtimeMs)
+    .map((snapshot) => snapshot.path);
+}
+
+export async function pruneHomeSnapshots(args: {
+  snapshotDir: string;
+  keepSnapshots: number;
+}): Promise<string[]> {
+  const snapshots = await listHomeSnapshots(args.snapshotDir);
+  const stale = snapshots.slice(args.keepSnapshots);
+  await Promise.all(stale.map((path) => rm(path, { force: true })));
+  return stale;
+}

package/home-sync.ts

@@ -0,0 +1,243 @@
+#!/usr/bin/env bun
+/**
+ * dench home sync
+ *
+ * Persists the sandbox's live home overlay. The live tree lives on
+ * /tmp (real overlayfs) so tool installers can use rename(2), symlink(2),
+ * chmod, sqlite, keyrings, npm globals, etc. This daemon snapshots that
+ * tree into tar archives on /workspace, which avoids Mountpoint-for-S3's
+ * symlink/rename limitations while keeping future sandboxes restorable.
+ */
+import { spawn } from "node:child_process";
+import { createReadStream, createWriteStream } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pipeline } from "node:stream/promises";
+import {
+  buildSnapshotName,
+  buildTarCreateArgs,
+  ensureHomeSyncDirs,
+  type HomeSyncArgs,
+  parseHomeSyncArgs,
+  pruneHomeSnapshots,
+  shouldIgnoreHomePath,
+} from "./home-sync-lib";
+
+type HomeSyncStatus = {
+  pid: number;
+  liveHome: string;
+  snapshotDir: string;
+  lastSnapshotAt: string | null;
+  lastSnapshotPath: string | null;
+  lastError: string | null;
+  pending: boolean;
+  inFlight: boolean;
+};
+
+const EXIT_CONFIG_ERROR = 78;
+
+function runCommand(command: string, args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      stdout += String(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += String(chunk);
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(
+        new Error(
+          `${command} exited ${code ?? "unknown"}: ${stderr || stdout}`.trim(),
+        ),
+      );
+    });
+  });
+}
+
+async function copyBytesOnly(
+  source: string,
+  destination: string,
+): Promise<void> {
+  await pipeline(
+    createReadStream(source),
+    createWriteStream(destination, { flags: "wx" }),
+  );
+}
+
+async function writeStatus(
+  path: string,
+  status: HomeSyncStatus,
+): Promise<void> {
+  await writeFile(path, `${JSON.stringify(status, null, 2)}\n`);
+}
+
+class HomeSyncDaemon {
+  private pendingTimer: NodeJS.Timeout | null = null;
+  private pending = false;
+  private inFlight = false;
+  private status: HomeSyncStatus;
+
+  constructor(private readonly args: HomeSyncArgs) {
+    this.status = {
+      pid: process.pid,
+      liveHome: args.liveHome,
+      snapshotDir: args.snapshotDir,
+      lastSnapshotAt: null,
+      lastSnapshotPath: null,
+      lastError: null,
+      pending: false,
+      inFlight: false,
+    };
+  }
+
+  async start(): Promise<void> {
+    await ensureHomeSyncDirs(this.args);
+    await writeStatus(this.args.statusPath, this.status);
+
+    if (this.args.once) {
+      await this.snapshotNow("once");
+      return;
+    }
+
+    let chokidar: typeof import("chokidar");
+    try {
+      chokidar = (await import("chokidar")) as never;
+    } catch (error) {
+      console.error(
+        `[dench-home-sync] chokidar not available; refusing to start (${
+          error instanceof Error ? error.message : String(error)
+        })`,
+      );
+      process.exit(EXIT_CONFIG_ERROR);
+    }
+
+    const watcher = chokidar.watch(this.args.liveHome, {
+      ignoreInitial: true,
+      persistent: true,
+      usePolling: false,
+      awaitWriteFinish: { stabilityThreshold: 250, pollInterval: 50 },
+      ignored: (path: string) => shouldIgnoreHomePath(path, this.args.liveHome),
+    });
+
+    const schedule = (event: string, path: string) => {
+      if (shouldIgnoreHomePath(path, this.args.liveHome)) return;
+      this.scheduleSnapshot(`${event}:${path}`);
+    };
+
+    watcher.on("add", (path: string) => schedule("add", path));
+    watcher.on("change", (path: string) => schedule("change", path));
+    watcher.on("unlink", (path: string) => schedule("unlink", path));
+    watcher.on("addDir", (path: string) => schedule("addDir", path));
+    watcher.on("unlinkDir", (path: string) => schedule("unlinkDir", path));
+    watcher.on("error", (error) => {
+      console.error(`[dench-home-sync] watcher error: ${String(error)}`);
+      this.status.lastError = String(error);
+      void writeStatus(this.args.statusPath, this.status);
+    });
+
+    process.on("SIGTERM", () => {
+      void this.shutdown(0);
+    });
+    process.on("SIGINT", () => {
+      void this.shutdown(0);
+    });
+
+    console.log(
+      `[dench-home-sync] watching ${this.args.liveHome} -> ${this.args.snapshotDir}`,
+    );
+  }
+
+  private scheduleSnapshot(reason: string): void {
+    this.pending = true;
+    this.status.pending = true;
+    void writeStatus(this.args.statusPath, this.status);
+
+    if (this.pendingTimer) clearTimeout(this.pendingTimer);
+    this.pendingTimer = setTimeout(() => {
+      this.pendingTimer = null;
+      void this.snapshotNow(reason);
+    }, this.args.debounceMs);
+  }
+
+  private async snapshotNow(reason: string): Promise<void> {
+    if (this.inFlight) {
+      this.scheduleSnapshot(`queued:${reason}`);
+      return;
+    }
+
+    this.pending = false;
+    this.inFlight = true;
+    this.status.pending = false;
+    this.status.inFlight = true;
+    await writeStatus(this.args.statusPath, this.status);
+
+    const tempDir = await mkdtemp(join(tmpdir(), "dench-home-sync-"));
+    const archivePath = join(tempDir, "home.tar");
+    const snapshotName = buildSnapshotName(new Date());
+    const durablePath = join(this.args.snapshotDir, snapshotName);
+
+    try {
+      await runCommand(
+        "tar",
+        buildTarCreateArgs({ liveHome: this.args.liveHome, archivePath }),
+      );
+      await copyBytesOnly(archivePath, durablePath);
+      await pruneHomeSnapshots({
+        snapshotDir: this.args.snapshotDir,
+        keepSnapshots: this.args.keepSnapshots,
+      });
+      this.status.lastSnapshotAt = new Date().toISOString();
+      this.status.lastSnapshotPath = durablePath;
+      this.status.lastError = null;
+      console.log(
+        `[dench-home-sync] snapshotted home (${reason}) -> ${durablePath}`,
+      );
+    } catch (error) {
+      this.status.lastError =
+        error instanceof Error ? error.message : String(error);
+      console.error(
+        `[dench-home-sync] snapshot failed: ${this.status.lastError}`,
+      );
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+      this.inFlight = false;
+      this.status.inFlight = false;
+      this.status.pending = this.pending;
+      await writeStatus(this.args.statusPath, this.status);
+    }
+  }
+
+  private async shutdown(exitCode: number): Promise<void> {
+    if (this.pendingTimer) {
+      clearTimeout(this.pendingTimer);
+      this.pendingTimer = null;
+    }
+    if (this.pending || this.inFlight) {
+      await this.snapshotNow("shutdown");
+    }
+    process.exit(exitCode);
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseHomeSyncArgs(process.argv.slice(2));
+  await new HomeSyncDaemon(args).start();
+}
+
+main().catch((error) => {
+  console.error(
+    `[dench-home-sync] fatal: ${
+      error instanceof Error ? (error.stack ?? error.message) : String(error)
+    }`,
+  );
+  process.exit(1);
+});