@openparachute/vault 0.7.4 → 0.7.5-rc.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/vault",
-  "version": "0.7.4",
+  "version": "0.7.5-rc.5",
   "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
   "module": "src/cli.ts",
   "type": "module",

package/src/cli.ts

@@ -175,6 +175,21 @@ import {
   type CliBuildResult,
 } from "./transcription/build.ts";
 import { downloadTo } from "./transcription/download.ts";
+import {
+  describeWhisperPlan,
+  planWhisperInstall,
+} from "./transcription/install-whisper-cpp.ts";
+import {
+  ensureBinaries,
+  ensureModel,
+  verifyTranscription,
+} from "./transcription/install-whisper-exec.ts";
+import {
+  binaryNameFor,
+  candidateBinDirs,
+  resolveCliBinary,
+  resolveFfmpeg,
+} from "./transcription/resolve-binary.ts";
 import { selectDefaultProvider, NOMINAL_SLACK_GB, type TierPlan } from "./transcription/tiers.ts";
 import {
   PYTHON_PROVIDERS,
@@ -3605,6 +3620,93 @@ function printTranscriptionPlan(plan: InstallPlan): void {
   console.log(`        ~${m.approxSizeMb}MB download, ~${m.approxRuntimeGb}GB peak RAM while transcribing`);
 }
 
+
+/**
+ * `parachute-vault transcription install` — the whisper.cpp path.
+ *
+ * Plan → binaries → model → VERIFY → activate. The verify step is the
+ * load-bearing one: the provider this replaces was activated by an install
+ * that never checked whether what it configured could run, which is how
+ * `TRANSCRIPTION_PROVIDER` came to point at a CLI that has never existed.
+ * `TRANSCRIPTION_PROVIDER` flips only after a real CLI transcribes a real
+ * file, so an install that reports success is one that works.
+ */
+async function runWhisperCppInstall(opts: {
+  modelId?: string;
+  dryRun: boolean;
+  yes: boolean;
+}): Promise<void> {
+  const plan = planWhisperInstall(opts.modelId, {
+    resolveExisting: (engine) => resolveCliBinary(engine),
+  });
+  for (const line of describeWhisperPlan(plan)) console.log(line);
+
+  if (!plan.supported) {
+    console.error("\nCan't install automatically on this host — see above.");
+    process.exit(1);
+  }
+  if (opts.dryRun) {
+    console.log("\n(dry run — nothing downloaded or changed)");
+    return;
+  }
+  if (!opts.yes) {
+    // Bun's global `confirm()` reads stdin. In a non-TTY — which is exactly
+    // how the unified setup script and any CI invocation call this — it would
+    // block forever, so require an explicit --yes there instead of hanging.
+    if (!process.stdin.isTTY) {
+      console.error(
+        "\nNot a terminal — re-run with --yes to install without confirmation.",
+      );
+      process.exit(1);
+    }
+    if (!confirm("\nProceed?")) {
+      console.log("Cancelled.");
+      return;
+    }
+  }
+
+  const log = (l: string) => console.log(`  ${l}`);
+
+  const bin = await ensureBinaries(plan, { log });
+  console.log(`${bin.ok ? "✓" : "✗"} ${bin.message}`);
+  if (!bin.ok) process.exit(1);
+
+  const model = await ensureModel(plan, { log });
+  console.log(`${model.ok ? "✓" : "✗"} ${model.message}`);
+  if (!model.ok) process.exit(1);
+
+  // Re-resolve AFTER install — a brew install just changed what's on disk.
+  const binPath = resolveCliBinary(plan.model.engine);
+  if (!binPath) {
+    console.error(
+      `✗ ${binaryNameFor(plan.model.engine)} still isn't resolvable after install. ` +
+        `Searched: ${candidateBinDirs().slice(0, 5).join(", ")}`,
+    );
+    process.exit(1);
+  }
+
+  const verified = await verifyTranscription(plan, binPath, { log });
+  console.log(`${verified.ok ? "✓" : "✗"} ${verified.message}`);
+  if (!verified.ok) {
+    console.error("\nNot activating — an install that can't transcribe isn't installed.");
+    process.exit(1);
+  }
+
+  setEnvVar("TRANSCRIPTION_PROVIDER", "whisper-cpp");
+  setEnvVar("TRANSCRIPTION_MODEL", plan.model.id);
+  console.log(`\n✓ Activated: whisper-cpp with ${plan.model.label}.`);
+
+  if (!resolveFfmpeg()) {
+    console.log(
+      "\n! ffmpeg isn't installed. Audio has to be transcoded to 16 kHz mono WAV before\n" +
+        "  transcription, so voice memos will fail until you install it:\n" +
+        "    macOS:  brew install ffmpeg\n" +
+        "    Debian: sudo apt install ffmpeg",
+    );
+  }
+  console.log("\nRestart the vault to apply (`parachute restart vault`).");
+}
+
 async function cmdTranscriptionInstall(args: string[]) {
   const dryRun = args.includes("--dry-run") || args.includes("--plan");
   const force = args.includes("--force");
@@ -3612,6 +3714,16 @@ async function cmdTranscriptionInstall(args: string[]) {
   const overrideModel = takeArgValue(args, "--model").value;
   const providerArg = takeArgValue(args, "--provider").value;
 
+  // whisper-cpp is the local path now, and the DEFAULT when no --provider is
+  // given. The legacy tier table below still serves an explicit
+  // `--provider transcribe-cpp|parakeet-mlx|onnx-asr`, but nothing routes
+  // there by default any more: transcribe-cpp's CLI has never shipped, and
+  // the Python providers need a venv plus a multi-GB model.
+  if (!providerArg || providerArg === "whisper-cpp") {
+    await runWhisperCppInstall({ modelId: overrideModel, dryRun, yes });
+    return;
+  }
+
   if (providerArg === "scribe-http") {
     // Just flip config back to the remote provider — no download.
     if (dryRun) {
@@ -3628,7 +3740,7 @@ async function cmdTranscriptionInstall(args: string[]) {
   if (providerArg) {
     if (!["transcribe-cpp", "parakeet-mlx", "onnx-asr"].includes(providerArg)) {
       console.error(
-        `Unknown --provider "${providerArg}". Valid: transcribe-cpp, parakeet-mlx, onnx-asr, scribe-http.`,
+        `Unknown --provider "${providerArg}". Valid: whisper-cpp (default), transcribe-cpp, parakeet-mlx, onnx-asr, scribe-http.`,
       );
       process.exit(1);
     }

package/src/mirror-import-jobs.ts

@@ -0,0 +1,197 @@
+/**
+ * In-process job registry for git imports (vault#640).
+ *
+ * **Why this exists.** `POST /vault/<name>/.parachute/mirror/import` used to
+ * run the whole clone-and-import inside the request and return the result.
+ * That shape has a hard ceiling that has nothing to do with how long the work
+ * legitimately takes:
+ *
+ *   - hub proxies the vault behind `Bun.serve({ idleTimeout: 255 })`, so any
+ *     import past ~4 minutes dies in the proxy no matter what vault does;
+ *   - browsers and intermediate proxies impose their own limits;
+ *   - a dropped connection lost the import's outcome entirely, even when the
+ *     import itself had succeeded.
+ *
+ * So the old handler papered over it with a 60s clone timeout and a docstring
+ * promising "if/when bigger vaults arrive we promote to async polling." They
+ * arrived. This is that promotion.
+ *
+ * **Shape.** POST starts a job and returns `202 { job_id }` immediately. The
+ * work continues on the server independent of the request that started it.
+ * `GET .../mirror/import/<job_id>` returns the current record; the SPA polls
+ * it for stage + progress and renders the terminal state.
+ *
+ * **Deliberately in-memory.** Jobs do not survive a vault restart, and that's
+ * the honest design rather than a shortcut: the work itself can't survive a
+ * restart either (the clone lives in a tempdir, the importer holds an open
+ * store handle), so a persisted "running" row would only ever resurrect as a
+ * lie. A restart mid-import surfaces as `job_not_found`, which the SPA reads
+ * as "the server restarted — check your vault and retry." Whatever the
+ * importer had already committed is committed; merge mode makes a retry safe.
+ *
+ * **Concurrency.** One running import per vault, enforced here so the 409
+ * lands before a tempdir is created. `cloneAndImport` keeps its own inFlight
+ * guard as the inner belt for direct (CLI / test) callers.
+ */
+
+import type {
+  ImportProgress,
+  ImportResult,
+  ImportStage,
+} from "./mirror-import.ts";
+
+/** How long a finished job stays queryable before GC. */
+export const FINISHED_JOB_TTL_MS = 60 * 60_000;
+
+/** Terminal + non-terminal job states. */
+export type ImportJobStatus = "running" | "succeeded" | "failed";
+
+/**
+ * Error detail for a failed job. Mirrors the `error_type` vocabulary the
+ * synchronous handler used to return, so the SPA's error branching is
+ * unchanged — it just reads them off the job record instead of the POST
+ * response.
+ */
+export interface ImportJobError {
+  error_type:
+    | "git_not_installed"
+    | "concurrent_import"
+    | "not_a_vault_export"
+    | "clone_failed"
+    | "internal";
+  message: string;
+}
+
+/** The record the status endpoint serves. */
+export interface ImportJob {
+  job_id: string;
+  vault_name: string;
+  status: ImportJobStatus;
+  stage: ImportStage;
+  /** Latest progress line for the current stage, when there is one. */
+  detail?: string;
+  started_at: string;
+  updated_at: string;
+  finished_at?: string;
+  /** Present iff `status === "succeeded"`. */
+  result?: ImportResult;
+  /** Present iff `status === "failed"`. */
+  error?: ImportJobError;
+}
+
+/** Thrown by `startImportJob` when this vault already has one running. */
+export class ImportJobConflictError extends Error {
+  constructor(public readonly vaultName: string) {
+    super(
+      `An import is already running for vault "${vaultName}". Wait for it to finish, or reload to watch its progress.`,
+    );
+    this.name = "ImportJobConflictError";
+  }
+}
+
+/** job_id → record. */
+const jobs = new Map<string, ImportJob>();
+/** vault name → job_id of the RUNNING job, if any. */
+const running = new Map<string, string>();
+
+/**
+ * Drop finished jobs past their TTL. Called opportunistically on every
+ * create/read rather than on a timer — a timer would keep the event loop
+ * alive and there is no correctness need for prompt collection.
+ */
+function gc(now: number): void {
+  for (const [id, job] of jobs) {
+    if (job.status === "running") continue;
+    const finished = job.finished_at ? Date.parse(job.finished_at) : 0;
+    if (Number.isFinite(finished) && now - finished > FINISHED_JOB_TTL_MS) {
+      jobs.delete(id);
+    }
+  }
+}
+
+/** The running job for a vault, or undefined. */
+export function getRunningImportJob(vaultName: string): ImportJob | undefined {
+  const id = running.get(vaultName);
+  return id ? jobs.get(id) : undefined;
+}
+
+/**
+ * Look up a job by id, scoped to a vault. The vault scope matters: the route
+ * is authorized against `vault:<name>:admin`, so an admin of vault A must not
+ * be able to read vault B's import record by guessing an id.
+ */
+export function getImportJob(
+  vaultName: string,
+  jobId: string,
+): ImportJob | undefined {
+  gc(Date.now());
+  const job = jobs.get(jobId);
+  if (!job || job.vault_name !== vaultName) return undefined;
+  return job;
+}
+
+/**
+ * Register a job and start `work` on it. Returns the initial record
+ * synchronously so the route can 202 immediately; `work` runs detached.
+ *
+ * `work` receives a progress sink to call as it moves through stages. Its
+ * resolved value becomes `result`; a throw becomes `error` via `classify`.
+ */
+export function startImportJob(
+  vaultName: string,
+  work: (onProgress: (update: ImportProgress) => void) => Promise<ImportResult>,
+  classify: (err: unknown) => ImportJobError,
+): ImportJob {
+  const now = Date.now();
+  gc(now);
+  if (running.has(vaultName)) {
+    throw new ImportJobConflictError(vaultName);
+  }
+
+  const nowIso = new Date(now).toISOString();
+  const job: ImportJob = {
+    job_id: crypto.randomUUID(),
+    vault_name: vaultName,
+    status: "running",
+    stage: "cloning",
+    started_at: nowIso,
+    updated_at: nowIso,
+  };
+  jobs.set(job.job_id, job);
+  running.set(vaultName, job.job_id);
+
+  const onProgress = (update: ImportProgress) => {
+    // A tick that arrives after the job finished (a late stderr line racing
+    // the exit) must not resurrect a terminal record.
+    if (job.status !== "running") return;
+    job.stage = update.stage;
+    if (update.detail !== undefined) job.detail = update.detail;
+    else delete job.detail;
+    job.updated_at = new Date().toISOString();
+  };
+
+  const finish = (patch: Partial<ImportJob>) => {
+    Object.assign(job, patch);
+    job.finished_at = new Date().toISOString();
+    job.updated_at = job.finished_at;
+    delete job.detail;
+    running.delete(vaultName);
+  };
+
+  // Detached on purpose — the response has already been sent by the time this
+  // settles. Every rejection path is funnelled through `classify` so an
+  // unexpected throw becomes a readable job error rather than an unhandled
+  // rejection that leaves the job "running" forever.
+  void work(onProgress).then(
+    (result) => finish({ status: "succeeded", result }),
+    (err) => finish({ status: "failed", error: classify(err) }),
+  );
+
+  return job;
+}
+
+/** Test seam: forget every job. */
+export function _resetImportJobsForTest(): void {
+  jobs.clear();
+  running.clear();
+}

package/src/mirror-import.test.ts

@@ -24,6 +24,7 @@ import { SqliteStore } from "../core/src/store.ts";
 import { exportVaultToDir } from "../core/src/portable-md.ts";
 import {
   CloneFailedError,
+  DEFAULT_CLONE_STALL_TIMEOUT_MS,
   ImportConflictError,
   NotAVaultExportError,
   _isImportInFlight,
@@ -437,7 +438,7 @@ describe("cloneAndImport — failures", () => {
     }
   });
 
-  test("clone timeout → CloneFailedError mentioning timeout", async () => {
+  test("absolute clone timeout → CloneFailedError naming the limit", async () => {
     await expect(
       cloneAndImport({
         vaultName: "default",
@@ -449,7 +450,106 @@ describe("cloneAndImport — failures", () => {
         spawn: spawnCloneTimeout,
         cloneTimeoutMs: 100,
       }),
-    ).rejects.toThrow(/timed out/);
+    ).rejects.toThrow(/exceeded its .*limit/);
+  });
+
+  // vault#640 — a stalled clone and a slow one are different failures and get
+  // different copy. The old code could only say "timed out after 60s", which
+  // was actively misleading on a big vault: nothing was wrong except that the
+  // vault was large.
+  test("stalled clone → CloneFailedError says STALLED, not 'too slow'", async () => {
+    const spawnStalled: GitSpawn = async () => ({
+      exitCode: 143,
+      stderr: "",
+      timedOut: true,
+      stalled: true,
+    });
+    await expect(
+      cloneAndImport({
+        vaultName: "default",
+        remoteUrl: "https://github.com/owner/repo.git",
+        auth: { kind: "none" },
+        mode: "merge",
+        store,
+        assetsDir,
+        spawn: spawnStalled,
+        cloneStallTimeoutMs: 600_000,
+      }),
+    ).rejects.toThrow(/stalled — no progress for 10 minutes/);
+  });
+
+  // The regression that made this whole change necessary: a clone that takes
+  // longer than a minute must NOT be killed. Previously `cloneTimeoutMs`
+  // defaulted to 60s, so any vault big enough to matter failed here.
+  test("no absolute timeout by default — a long clone is allowed to finish", async () => {
+    const localFixture = await buildExportFixture();
+    let observedTimeout: number | undefined;
+    let observedStall: number | undefined;
+    const spawnSlowButFine: GitSpawn = async (argv, options) => {
+      observedTimeout = options.timeoutMs;
+      observedStall = options.stallTimeoutMs;
+      const dest = argv[argv.length - 1]!;
+      cpSync(localFixture, dest, { recursive: true });
+      return { exitCode: 0, stderr: "", timedOut: false };
+    };
+    const result = await cloneAndImport({
+      vaultName: "default",
+      remoteUrl: "https://github.com/owner/repo.git",
+      auth: { kind: "none" },
+      mode: "merge",
+      store,
+      assetsDir,
+      spawn: spawnSlowButFine,
+    });
+    expect(result.notes_imported).toBeGreaterThan(0);
+    // 0 == disabled. The stall bound is what guards a wedged clone.
+    expect(observedTimeout).toBe(0);
+    expect(observedStall).toBe(DEFAULT_CLONE_STALL_TIMEOUT_MS);
+  });
+
+  test("clone runs with --progress so the stall timer has something to watch", async () => {
+    const localFixture = await buildExportFixture();
+    let argvSeen: string[] = [];
+    const spawnCapture: GitSpawn = async (argv) => {
+      argvSeen = argv;
+      const dest = argv[argv.length - 1]!;
+      cpSync(localFixture, dest, { recursive: true });
+      return { exitCode: 0, stderr: "", timedOut: false };
+    };
+    await cloneAndImport({
+      vaultName: "default",
+      remoteUrl: "https://github.com/owner/repo.git",
+      auth: { kind: "none" },
+      mode: "merge",
+      store,
+      assetsDir,
+      spawn: spawnCapture,
+    });
+    expect(argvSeen).toContain("--progress");
+  });
+
+  test("onProgress reports stage transitions and clone detail", async () => {
+    const localFixture = await buildExportFixture();
+    const seen: string[] = [];
+    const spawnWithProgress: GitSpawn = async (argv, options) => {
+      options.onProgress?.("Receiving objects:  47% (470/1000)");
+      const dest = argv[argv.length - 1]!;
+      cpSync(localFixture, dest, { recursive: true });
+      return { exitCode: 0, stderr: "", timedOut: false };
+    };
+    await cloneAndImport({
+      vaultName: "default",
+      remoteUrl: "https://github.com/owner/repo.git",
+      auth: { kind: "none" },
+      mode: "merge",
+      store,
+      assetsDir,
+      spawn: spawnWithProgress,
+      onProgress: (u) => seen.push(`${u.stage}:${u.detail ?? ""}`),
+    });
+    expect(seen).toContain("cloning:");
+    expect(seen).toContain("cloning:Receiving objects:  47% (470/1000)");
+    expect(seen).toContain("importing:");
   });
 
   test("clone target lacks .parachute/vault.yaml → NotAVaultExportError", async () => {