@dench.com/cli 0.4.2 → 0.4.4

package/fs-daemon-binary.ts

@@ -0,0 +1,64 @@
+/**
+ * Binary-content detection for `dench-fs-daemon`.
+ *
+ * Background: the daemon's `commitFileUpload` historically inlined every
+ * file ≤ `MAX_INLINE_TEXT_BYTES` (100 KB) into `fileContents.text` via
+ * `TextDecoder("utf-8", { fatal: false }).decode(bytes)`. The
+ * `fatal: false` mode silently substitutes U+FFFD (replacement char)
+ * for any invalid byte sequence — perfect for ad-hoc text rendering,
+ * disastrous when a small PNG / PDF / .zip lands in the cache:
+ *   • The storage blob is fine (bytes preserved verbatim in `_storage`).
+ *   • The inline text cache is corrupt (replacement chars everywhere).
+ *   • The viewer that pulls from `fileContents.text` for fast paths
+ *     renders garbled mojibake until something forces a re-write.
+ *
+ * The agent's `commitConvexFileBytes` in chat-turn.ts has had a sibling
+ * `detectBinaryBuffer` for a while — this file ports it verbatim so
+ * the daemon can route binary blobs to the no-inline path. Keeping it
+ * in its own module (no chokidar / Convex deps) means the unit tests
+ * can pin behavior without booting the daemon.
+ */
+
+const BINARY_PROBE_BYTES = 4096;
+
+/**
+ * Sniff a buffer for binary content. PNG / PDF / Office / .zip all
+ * contain NUL bytes within the first KB; valid UTF-8 text never does.
+ * We deliberately limit the scan to the first 4 KiB so the check stays
+ * cheap on multi-MB blobs, and treat invalid UTF-8 in that prefix as a
+ * binary signal too — silently round-tripping garbage Unicode through
+ * the inline-text cache is what corrupts the rendered text for
+ * downstream viewers.
+ *
+ * Returns `true` when the buffer looks binary (skip inline text cache),
+ * `false` when it looks like clean UTF-8 text (safe to inline).
+ */
+export function detectBinaryBuffer(buffer: Uint8Array): boolean {
+  const slice =
+    buffer.byteLength > BINARY_PROBE_BYTES
+      ? buffer.subarray(0, BINARY_PROBE_BYTES)
+      : buffer;
+  for (const byte of slice) {
+    if (byte === 0) return true;
+  }
+  try {
+    new TextDecoder("utf-8", { fatal: true }).decode(slice);
+    return false;
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * Convenience: decode bytes to a UTF-8 string ONLY when the buffer
+ * looks like clean text. Returns `undefined` for binary content so
+ * callers can pass it straight into `commitFileUpload({ text })` —
+ * Convex schema marks `text` optional and skips the inline cache when
+ * it's missing.
+ */
+export function decodeInlineTextOrUndefined(
+  buffer: Uint8Array,
+): string | undefined {
+  if (detectBinaryBuffer(buffer)) return undefined;
+  return new TextDecoder("utf-8", { fatal: false }).decode(buffer);
+}

package/fs-daemon-flush.ts

@@ -0,0 +1,140 @@
+/**
+ * Pure polling loop for `dench-fs-daemon flush`.
+ *
+ * Lives in its own module so the unit tests can pull `runFlushPolling`
+ * without importing `cli/fs-daemon.ts`'s top-level `main()` (which
+ * would try to start the actual daemon and crash the test runner).
+ *
+ * The flush command's whole job is to make `execute_bash` writes
+ * deterministically visible in Convex BEFORE chat-turn returns the
+ * tool result to the model. The polling loop is responsible for:
+ *
+ *   1. Bouncing immediately when no daemon is registered or the
+ *      registered pid is dead. Otherwise chat-turn would hang for
+ *      timeoutMs every turn on a host where the daemon never came up.
+ *   2. Sending exactly one SIGUSR1 (the daemon's runDaemon registers a
+ *      handler that triggers an immediate reconcile tick).
+ *   3. Treating the flush as complete only when BOTH conditions hold:
+ *      a) `lastReconcile.finishedAt > t0` (proves the signal-driven
+ *         tick actually ran and not just an old scheduled tick).
+ *      b) `pendingDrain.{pendingPaths, inFlightUploads}` are both 0
+ *         (proves every per-path debounce timer fired AND the upload
+ *         it triggered drained — without (b) we'd race a pending
+ *         300 ms debounce or an in-flight Convex Storage POST).
+ *   4. Surfacing a structured timeout result with the last-seen drain
+ *      counts so the caller (chat-turn) can decide whether to flag
+ *      drift to the user.
+ */
+
+/**
+ * Snapshot of how much work the daemon still has in flight. The
+ * `flush` subcommand polls this via the status file and only declares
+ * success when both numbers are zero AND a fresh reconcile has
+ * completed since the flush request.
+ */
+export type DrainStatus = {
+  /** Paths with a debounce timer set but the upload hasn't started. */
+  pendingPaths: number;
+  /** Upload calls currently mid-flight (between fn() start and finally). */
+  inFlightUploads: number;
+};
+
+/**
+ * Subset of the daemon status file the flush poller needs. The full
+ * shape lives in `cli/fs-daemon.ts`; we accept any superset here so
+ * upstream changes to the status file don't ripple into this module.
+ */
+export type FlushPollingStatus = {
+  pid: number;
+  lastReconcile: { finishedAt: number } | null;
+  pendingDrain: DrainStatus;
+};
+
+export type FlushOutcome =
+  | { ok: true; reason: "flushed"; pid: number; durationMs: number }
+  | { ok: false; reason: "no_daemon"; pid: number | null; durationMs: number }
+  | {
+      ok: false;
+      reason: "signal_failed";
+      pid: number;
+      durationMs: number;
+      error: string;
+    }
+  | {
+      ok: false;
+      reason: "timeout";
+      pid: number;
+      durationMs: number;
+      drain: DrainStatus;
+    };
+
+export const FLUSH_POLL_INTERVAL_MS = 50;
+export const FLUSH_DEFAULT_TIMEOUT_MS = 5_000;
+
+export type FlushPollingArgs = {
+  timeoutMs: number;
+  readStatus: () => Promise<FlushPollingStatus | null>;
+  isPidAlive: (pid: number) => boolean;
+  signal: (pid: number) => void;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  pollIntervalMs?: number;
+};
+
+export async function runFlushPolling(
+  args: FlushPollingArgs,
+): Promise<FlushOutcome> {
+  const now = args.now ?? (() => Date.now());
+  const sleep =
+    args.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
+  const pollMs = args.pollIntervalMs ?? FLUSH_POLL_INTERVAL_MS;
+  const t0 = now();
+
+  const initial = await args.readStatus();
+  if (!initial) {
+    return {
+      ok: false,
+      reason: "no_daemon",
+      pid: null,
+      durationMs: now() - t0,
+    };
+  }
+  const pid = initial.pid;
+  if (!args.isPidAlive(pid)) {
+    return { ok: false, reason: "no_daemon", pid, durationMs: now() - t0 };
+  }
+  try {
+    args.signal(pid);
+  } catch (error) {
+    return {
+      ok: false,
+      reason: "signal_failed",
+      pid,
+      durationMs: now() - t0,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+
+  const deadline = t0 + args.timeoutMs;
+  let lastDrain: DrainStatus = initial.pendingDrain;
+  while (now() < deadline) {
+    await sleep(pollMs);
+    const current = await args.readStatus();
+    if (!current) continue;
+    lastDrain = current.pendingDrain;
+    const finishedAfterT0 = (current.lastReconcile?.finishedAt ?? 0) > t0;
+    const drained =
+      current.pendingDrain.pendingPaths === 0 &&
+      current.pendingDrain.inFlightUploads === 0;
+    if (finishedAfterT0 && drained) {
+      return { ok: true, reason: "flushed", pid, durationMs: now() - t0 };
+    }
+  }
+  return {
+    ok: false,
+    reason: "timeout",
+    pid,
+    durationMs: now() - t0,
+    drain: lastDrain,
+  };
+}

package/fs-daemon.ts

@@ -108,6 +108,7 @@ function shouldIgnore(relPath: string): boolean {
   const parts = relPath.split("/").filter(Boolean);
   const blockedDirs = new Set([
     ".chrome-profile",
+    ".dench",
     ".git",
     ".next",
     ".vercel",

package/home-sync

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+const entrypoint = new URL("./home-sync.ts", import.meta.url).href;
+
+if (process.versions.bun) {
+  await import(entrypoint);
+} else {
+  const { tsImport } = await import("tsx/esm/api");
+  await tsImport(entrypoint, import.meta.url);
+}

package/home-sync-lib.ts

@@ -0,0 +1,185 @@
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+export const DEFAULT_LIVE_HOME = "/tmp/.dench-home";
+export const DEFAULT_SNAPSHOT_DIR = "/workspace/.dench/home/snapshots";
+export const DEFAULT_DEBOUNCE_MS = 2_000;
+export const DEFAULT_KEEP_SNAPSHOTS = 8;
+export const DEFAULT_STATUS_PATH = "/tmp/dench-home-sync.status.json";
+
+export type HomeSyncArgs = {
+  liveHome: string;
+  snapshotDir: string;
+  debounceMs: number;
+  keepSnapshots: number;
+  statusPath: string;
+  once: boolean;
+};
+
+export function parseHomeSyncArgs(argv: string[]): HomeSyncArgs {
+  const out: Partial<HomeSyncArgs> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i];
+    const value = argv[i + 1];
+    switch (flag) {
+      case "--live-home":
+        out.liveHome = value;
+        i++;
+        break;
+      case "--snapshot-dir":
+        out.snapshotDir = value;
+        i++;
+        break;
+      case "--debounce-ms":
+        out.debounceMs = Number.parseInt(value, 10);
+        i++;
+        break;
+      case "--keep-snapshots":
+        out.keepSnapshots = Number.parseInt(value, 10);
+        i++;
+        break;
+      case "--status-path":
+        out.statusPath = value;
+        i++;
+        break;
+      case "--once":
+        out.once = true;
+        break;
+      default:
+        // Ignore unknown flags so newer bootstraps can run on older CLIs.
+        break;
+    }
+  }
+
+  return {
+    liveHome: out.liveHome ?? DEFAULT_LIVE_HOME,
+    snapshotDir:
+      out.snapshotDir ??
+      process.env.DENCH_HOME_SNAPSHOT_DIR?.trim() ??
+      DEFAULT_SNAPSHOT_DIR,
+    debounceMs: Number.isFinite(out.debounceMs)
+      ? Math.max(250, out.debounceMs ?? DEFAULT_DEBOUNCE_MS)
+      : DEFAULT_DEBOUNCE_MS,
+    keepSnapshots: Number.isFinite(out.keepSnapshots)
+      ? Math.max(1, out.keepSnapshots ?? DEFAULT_KEEP_SNAPSHOTS)
+      : DEFAULT_KEEP_SNAPSHOTS,
+    statusPath: out.statusPath ?? DEFAULT_STATUS_PATH,
+    once: out.once ?? false,
+  };
+}
+
+const EXCLUDED_HOME_PATTERNS = [
+  ".cache",
+  ".cache/**",
+  ".npm/_cacache",
+  ".npm/_cacache/**",
+  ".npm/_logs",
+  ".npm/_logs/**",
+  ".bun/install/cache",
+  ".bun/install/cache/**",
+  ".cargo/registry/cache",
+  ".cargo/registry/cache/**",
+  ".cargo/registry/index",
+  ".cargo/registry/index/**",
+  ".cargo/git/checkouts",
+  ".cargo/git/checkouts/**",
+  ".cargo/git/db",
+  ".cargo/git/db/**",
+  "go/pkg/mod/cache",
+  "go/pkg/mod/cache/**",
+  ".config/google-chrome",
+  ".config/google-chrome/**",
+  ".config/chromium",
+  ".config/chromium/**",
+];
+
+export function homeSnapshotExcludes(): string[] {
+  return [...EXCLUDED_HOME_PATTERNS];
+}
+
+export function shouldIgnoreHomePath(path: string, liveHome: string): boolean {
+  const rel = relative(liveHome, path).replaceAll("\\", "/");
+  if (!rel || rel === ".") return false;
+  if (rel.startsWith("..")) return true;
+  return EXCLUDED_HOME_PATTERNS.some((pattern) => {
+    if (pattern.endsWith("/**")) {
+      const prefix = pattern.slice(0, -3);
+      return rel === prefix || rel.startsWith(`${prefix}/`);
+    }
+    return rel === pattern;
+  });
+}
+
+export function buildTarCreateArgs(args: {
+  liveHome: string;
+  archivePath: string;
+}): string[] {
+  return [
+    "--create",
+    "--file",
+    args.archivePath,
+    "--directory",
+    args.liveHome,
+    ...EXCLUDED_HOME_PATTERNS.map((pattern) => `--exclude=${pattern}`),
+    ".",
+  ];
+}
+
+export function buildSnapshotName(now: Date, pid = process.pid): string {
+  const stamp = now
+    .toISOString()
+    .replaceAll("-", "")
+    .replaceAll(":", "")
+    .replace(".", "");
+  return `home-${stamp}-${pid}.tar`;
+}
+
+export async function ensureHomeSyncDirs(args: {
+  liveHome: string;
+  snapshotDir: string;
+}): Promise<void> {
+  await mkdir(args.liveHome, { recursive: true });
+  await mkdir(args.snapshotDir, { recursive: true });
+}
+
+export async function listHomeSnapshots(
+  snapshotDir: string,
+): Promise<string[]> {
+  let entries: string[];
+  try {
+    entries = await readdir(snapshotDir);
+  } catch {
+    return [];
+  }
+
+  const snapshots = await Promise.all(
+    entries
+      .filter((entry) => /^home-\d{8}T\d{9}Z-\d+\.tar$/.test(entry))
+      .map(async (entry) => {
+        const path = join(snapshotDir, entry);
+        try {
+          const info = await stat(path);
+          return { path, mtimeMs: info.mtimeMs };
+        } catch {
+          return null;
+        }
+      }),
+  );
+
+  return snapshots
+    .filter((snapshot): snapshot is { path: string; mtimeMs: number } =>
+      Boolean(snapshot),
+    )
+    .sort((a, b) => b.mtimeMs - a.mtimeMs)
+    .map((snapshot) => snapshot.path);
+}
+
+export async function pruneHomeSnapshots(args: {
+  snapshotDir: string;
+  keepSnapshots: number;
+}): Promise<string[]> {
+  const snapshots = await listHomeSnapshots(args.snapshotDir);
+  const stale = snapshots.slice(args.keepSnapshots);
+  await Promise.all(stale.map((path) => rm(path, { force: true })));
+  return stale;
+}

package/home-sync.ts

@@ -0,0 +1,231 @@
+#!/usr/bin/env bun
+/**
+ * dench home sync
+ *
+ * Persists the sandbox's live home overlay. The live tree lives on
+ * /tmp (real overlayfs) so tool installers can use rename(2), symlink(2),
+ * chmod, sqlite, keyrings, npm globals, etc. This daemon snapshots that
+ * tree into tar archives on /workspace, which avoids Mountpoint-for-S3's
+ * symlink/rename limitations while keeping future sandboxes restorable.
+ */
+import { spawn } from "node:child_process";
+import { cp, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  buildSnapshotName,
+  buildTarCreateArgs,
+  ensureHomeSyncDirs,
+  type HomeSyncArgs,
+  parseHomeSyncArgs,
+  pruneHomeSnapshots,
+  shouldIgnoreHomePath,
+} from "./home-sync-lib";
+
+type HomeSyncStatus = {
+  pid: number;
+  liveHome: string;
+  snapshotDir: string;
+  lastSnapshotAt: string | null;
+  lastSnapshotPath: string | null;
+  lastError: string | null;
+  pending: boolean;
+  inFlight: boolean;
+};
+
+const EXIT_CONFIG_ERROR = 78;
+
+function runCommand(command: string, args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      stdout += String(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += String(chunk);
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(
+        new Error(
+          `${command} exited ${code ?? "unknown"}: ${stderr || stdout}`.trim(),
+        ),
+      );
+    });
+  });
+}
+
+async function writeStatus(
+  path: string,
+  status: HomeSyncStatus,
+): Promise<void> {
+  await writeFile(path, `${JSON.stringify(status, null, 2)}\n`);
+}
+
+class HomeSyncDaemon {
+  private pendingTimer: NodeJS.Timeout | null = null;
+  private pending = false;
+  private inFlight = false;
+  private status: HomeSyncStatus;
+
+  constructor(private readonly args: HomeSyncArgs) {
+    this.status = {
+      pid: process.pid,
+      liveHome: args.liveHome,
+      snapshotDir: args.snapshotDir,
+      lastSnapshotAt: null,
+      lastSnapshotPath: null,
+      lastError: null,
+      pending: false,
+      inFlight: false,
+    };
+  }
+
+  async start(): Promise<void> {
+    await ensureHomeSyncDirs(this.args);
+    await writeStatus(this.args.statusPath, this.status);
+
+    if (this.args.once) {
+      await this.snapshotNow("once");
+      return;
+    }
+
+    let chokidar: typeof import("chokidar");
+    try {
+      chokidar = (await import("chokidar")) as never;
+    } catch (error) {
+      console.error(
+        `[dench-home-sync] chokidar not available; refusing to start (${
+          error instanceof Error ? error.message : String(error)
+        })`,
+      );
+      process.exit(EXIT_CONFIG_ERROR);
+    }
+
+    const watcher = chokidar.watch(this.args.liveHome, {
+      ignoreInitial: true,
+      persistent: true,
+      usePolling: false,
+      awaitWriteFinish: { stabilityThreshold: 250, pollInterval: 50 },
+      ignored: (path: string) => shouldIgnoreHomePath(path, this.args.liveHome),
+    });
+
+    const schedule = (event: string, path: string) => {
+      if (shouldIgnoreHomePath(path, this.args.liveHome)) return;
+      this.scheduleSnapshot(`${event}:${path}`);
+    };
+
+    watcher.on("add", (path: string) => schedule("add", path));
+    watcher.on("change", (path: string) => schedule("change", path));
+    watcher.on("unlink", (path: string) => schedule("unlink", path));
+    watcher.on("addDir", (path: string) => schedule("addDir", path));
+    watcher.on("unlinkDir", (path: string) => schedule("unlinkDir", path));
+    watcher.on("error", (error) => {
+      console.error(`[dench-home-sync] watcher error: ${String(error)}`);
+      this.status.lastError = String(error);
+      void writeStatus(this.args.statusPath, this.status);
+    });
+
+    process.on("SIGTERM", () => {
+      void this.shutdown(0);
+    });
+    process.on("SIGINT", () => {
+      void this.shutdown(0);
+    });
+
+    console.log(
+      `[dench-home-sync] watching ${this.args.liveHome} -> ${this.args.snapshotDir}`,
+    );
+  }
+
+  private scheduleSnapshot(reason: string): void {
+    this.pending = true;
+    this.status.pending = true;
+    void writeStatus(this.args.statusPath, this.status);
+
+    if (this.pendingTimer) clearTimeout(this.pendingTimer);
+    this.pendingTimer = setTimeout(() => {
+      this.pendingTimer = null;
+      void this.snapshotNow(reason);
+    }, this.args.debounceMs);
+  }
+
+  private async snapshotNow(reason: string): Promise<void> {
+    if (this.inFlight) {
+      this.scheduleSnapshot(`queued:${reason}`);
+      return;
+    }
+
+    this.pending = false;
+    this.inFlight = true;
+    this.status.pending = false;
+    this.status.inFlight = true;
+    await writeStatus(this.args.statusPath, this.status);
+
+    const tempDir = await mkdtemp(join(tmpdir(), "dench-home-sync-"));
+    const archivePath = join(tempDir, "home.tar");
+    const snapshotName = buildSnapshotName(new Date());
+    const durablePath = join(this.args.snapshotDir, snapshotName);
+
+    try {
+      await runCommand(
+        "tar",
+        buildTarCreateArgs({ liveHome: this.args.liveHome, archivePath }),
+      );
+      await cp(archivePath, durablePath, { force: false });
+      await pruneHomeSnapshots({
+        snapshotDir: this.args.snapshotDir,
+        keepSnapshots: this.args.keepSnapshots,
+      });
+      this.status.lastSnapshotAt = new Date().toISOString();
+      this.status.lastSnapshotPath = durablePath;
+      this.status.lastError = null;
+      console.log(
+        `[dench-home-sync] snapshotted home (${reason}) -> ${durablePath}`,
+      );
+    } catch (error) {
+      this.status.lastError =
+        error instanceof Error ? error.message : String(error);
+      console.error(
+        `[dench-home-sync] snapshot failed: ${this.status.lastError}`,
+      );
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+      this.inFlight = false;
+      this.status.inFlight = false;
+      this.status.pending = this.pending;
+      await writeStatus(this.args.statusPath, this.status);
+    }
+  }
+
+  private async shutdown(exitCode: number): Promise<void> {
+    if (this.pendingTimer) {
+      clearTimeout(this.pendingTimer);
+      this.pendingTimer = null;
+    }
+    if (this.pending || this.inFlight) {
+      await this.snapshotNow("shutdown");
+    }
+    process.exit(exitCode);
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseHomeSyncArgs(process.argv.slice(2));
+  await new HomeSyncDaemon(args).start();
+}
+
+main().catch((error) => {
+  console.error(
+    `[dench-home-sync] fatal: ${
+      error instanceof Error ? (error.stack ?? error.message) : String(error)
+    }`,
+  );
+  process.exit(1);
+});