@openparachute/vault 0.6.5-rc.2 → 0.6.5-rc.9

package/core/src/sql-in.ts

@@ -0,0 +1,76 @@
+/**
+ * SQL `IN`-list helpers sized for the smallest backing SQLite we run on.
+ *
+ * ## The DO 100-bound-parameter cap
+ *
+ * Cloudflare Durable Object SQLite (the cloud vault's backing store) rejects
+ * any statement carrying **more than 100 bound parameters** with
+ * `too many SQL variables`. Standard SQLite — including bun:sqlite (self-host)
+ * — tolerates 999 on old builds and 32766 on new ones, which is why an
+ * `IN (?, ?, …)` list sized for the 999 floor shipped fine on self-host and
+ * only 500'd on cloud vaults: a >100-note export page passed >100 ids in one
+ * statement and DO SQLite rejected it. See vault export-500 diagnosis.
+ *
+ * Two DO-safe shapes, depending on whether the id-list is the *only* thing
+ * in the statement:
+ *
+ * 1. **Standalone id-list** (its own `SELECT … WHERE id IN (…)`, no other
+ *    params, no pagination): chunk at {@link IN_PARAM_CHUNK} and union the
+ *    results in JS. Use {@link chunkForInClause}.
+ *
+ * 2. **Embedded id-set** (an `IN` filter inside a larger statement that also
+ *    carries `ORDER BY` / `LIMIT` / `OFFSET` / other params — chunking would
+ *    break the shared pagination window): bind the whole set as a SINGLE JSON
+ *    array param via {@link IN_VIA_JSON_EACH} + {@link jsonEachParam}. One
+ *    param regardless of set size, so it can never trip the cap. Requires the
+ *    JSON1 extension, which both bun:sqlite and DO SQLite ship (the vault
+ *    already relies on `json_extract` on DO for metadata queries).
+ *
+ * ## Future refinement — `maxBoundParams`
+ *
+ * The clean long-term shape is for the Store / DB shim to advertise a
+ * `maxBoundParams` (DO = 100, bun = 999) and chunk to
+ * `min(IN_PARAM_CHUNK, maxBoundParams - reservedParams)`, so self-host keeps
+ * large-batch efficiency instead of paying the DO tax. Until then a global
+ * 90 is SAFE on bun — it just issues more cheap, fully-indexed IN queries —
+ * and is the conservative floor everywhere.
+ */
+
+/**
+ * Chunk size for standalone `IN (?, ?, …)` id-lists. 90 leaves headroom
+ * under the DO 100-param cap for statements whose only bound params are the
+ * IN-list (the extra 10 covers the rare embedded extra param). Never raise
+ * this above 100 without teaching every embedded callsite the DO cap — see
+ * the module header.
+ */
+export const IN_PARAM_CHUNK = 90;
+
+/**
+ * Split `items` into consecutive chunks of at most `size` (default
+ * {@link IN_PARAM_CHUNK}) for feeding to a chunked `IN (?, …)` query. Empty
+ * input yields no chunks. This is the single place that decides IN-list chunk
+ * boundaries, so a regression that raises the size is caught by one test.
+ */
+export function chunkForInClause<T>(items: readonly T[], size = IN_PARAM_CHUNK): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    chunks.push(items.slice(i, i + size));
+  }
+  return chunks;
+}
+
+/**
+ * SQL fragment for a single-param `IN` filter over an id-set embedded in a
+ * larger statement: `WHERE col IN ${IN_VIA_JSON_EACH}`. Bind the matching
+ * `?` with {@link jsonEachParam}. json_each expands the bound JSON array into
+ * a one-column table, so the whole set costs exactly ONE bound parameter and
+ * can never exceed the DO cap. Callers must guarantee the set is non-empty
+ * (`json_each('[]')` yields no rows — correct for `IN`, but an empty set is
+ * usually short-circuited to a `0 = 1` no-match upstream to skip the query).
+ */
+export const IN_VIA_JSON_EACH = "(SELECT value FROM json_each(?))";
+
+/** Serialize an id-set for the single {@link IN_VIA_JSON_EACH} bound param. */
+export function jsonEachParam(values: readonly (string | number)[]): string {
+  return JSON.stringify(values);
+}

package/core/src/transcription/provider.ts

@@ -0,0 +1,141 @@
+/**
+ * Transcription provider seam (scribe-fold Phase 1).
+ *
+ * A `TranscriptionProvider` is the single, runtime-agnostic contract for
+ * turning an audio blob into text. It exists so vault's transcription worker
+ * doesn't hard-code "POST to scribe" — the worker resolves a provider and
+ * calls `transcribe()`, and the provider owns HOW the bytes become text.
+ *
+ * ## Why this lives in core (not src)
+ *
+ * The interface must be importable from BOTH runtimes without dragging in
+ * Bun- or Node-only APIs:
+ *
+ *   - the Bun vault (`src/transcription/providers/scribe-http.ts` — the
+ *     whisper-compatible remote provider that reproduces today's scribe call);
+ *   - the cloud vault (a future Cloudflare Workers-AI provider implemented
+ *     against THIS interface for the metered voice build).
+ *
+ * So it is pure types + one dependency-free error class — no `fs`, no
+ * `FormData` serialization, no `fetch`. `audio` arrives as a `Uint8Array`
+ * (both runtimes have it); a provider decides how to ship it. Core has no
+ * package `exports` map, so consumers import the src subpath directly
+ * (`../core/src/transcription/provider.ts`), the same convention as
+ * `core/src/seed-packs.ts`.
+ *
+ * ## The fold (context)
+ *
+ * Ratified 2026-07-03 (Aaron): scribe folds into vault as a built-in
+ * transcription feature. Phase 1 (this seam) is behavior-preserving — the
+ * default `scribe-http` provider does EXACTLY what today's `callScribe` did,
+ * so existing scribe installs keep working unchanged. Phase 2 moves local-ASR
+ * backends behind the same interface. See
+ * `design/2026-07-03-transcription-provider-seam.md`.
+ */
+
+/**
+ * A single context entry attached alongside the audio (person/project notes,
+ * etc.). Structurally identical to `src/context.ts`'s `ContextEntry` — the
+ * same shape scribe already consumes over the `context` multipart part — so a
+ * `ContextPayload` produced there is assignable here without conversion. The
+ * interface stays runtime-agnostic: a provider decides whether/how to
+ * serialize it.
+ */
+export interface TranscriptionContextEntry {
+  /** Note path basename, or note id if no path. */
+  name: string;
+  /** Whitelisted metadata fields carried through from the vault query. */
+  [key: string]: unknown;
+}
+
+/** The `{ entries: [...] }` context blob a provider may forward with the audio. */
+export interface TranscriptionContext {
+  entries: TranscriptionContextEntry[];
+}
+
+/** The audio + descriptors handed to a provider for one transcription. */
+export interface TranscribeInput {
+  /** Raw audio bytes. Both runtimes expose `Uint8Array`; a `Buffer` is one. */
+  audio: Uint8Array;
+  /** Original filename (used for the multipart `file` part / provider hints). */
+  filename: string;
+  /** MIME type of the audio (e.g. `audio/webm`). */
+  mimeType: string;
+  /**
+   * Optional vault-provided context (person/project notes) to forward with the
+   * audio so the provider/model can use it. `null`/`undefined` means none.
+   */
+  context?: TranscriptionContext | null;
+}
+
+/** What a provider returns on a successful transcription. */
+export interface TranscribeResult {
+  /** The transcript text. */
+  text: string;
+  /**
+   * Duration of the transcribed AUDIO in seconds, when the provider reports it.
+   * This is a metered/plan concern (the cloud voice build will decrement a
+   * minutes balance by it); self-host providers that don't measure it leave it
+   * unset. NOT the wall-clock request time — the worker measures that itself.
+   */
+  audioSeconds?: number;
+}
+
+/** Provider readiness probe result. */
+export interface ProviderAvailability {
+  /** True when the provider can service `transcribe()` calls right now. */
+  ok: boolean;
+  /** Human-readable cause when `ok` is false (e.g. "no scribe URL configured"). */
+  reason?: string;
+}
+
+/**
+ * The transcription contract. Implementations own the audio→text step; the
+ * worker owns queueing, backoff/retry, transcript-note materialization, and
+ * retention — none of which belong here.
+ */
+export interface TranscriptionProvider {
+  /** Stable provider identifier, surfaced on the capability flag (e.g. "scribe-http"). */
+  readonly name: string;
+  /** Turn audio into text (or throw — see `TranscriptionError` for the retry contract). */
+  transcribe(input: TranscribeInput): Promise<TranscribeResult>;
+  /**
+   * Cheap, side-effect-free readiness check. MUST NOT perform the actual
+   * transcription or a network round-trip unless that is genuinely cheap —
+   * it gates the capability flag on the vault landing, called on config reads.
+   */
+  available(): Promise<ProviderAvailability>;
+}
+
+/**
+ * Structured, provider-agnostic transcription failure.
+ *
+ * The worker's terminal-vs-retry decision keys off `retriable`: a
+ * non-retriable error (a 4xx the operator must fix — no provider configured,
+ * bad auth) fails immediately with no backoff; a retriable one (5xx, transient
+ * upstream) is retried with exponential backoff. `code` is a stable machine
+ * string (e.g. `missing_provider`) surfaced on the transcript note's
+ * frontmatter and the attachment's `transcribe_error_code` so callers branch
+ * on it instead of regex-matching message text.
+ *
+ * Providers throw this for structured failures; a plain `Error` (network drop,
+ * timeout, malformed response) is treated as retriable by the worker.
+ */
+export class TranscriptionError extends Error {
+  /** Stable machine code (e.g. "missing_provider"), when the provider supplied one. */
+  readonly code?: string;
+  /** Upstream HTTP status, when the failure came from an HTTP provider. */
+  readonly httpStatus?: number;
+  /** Whether retrying the same audio could plausibly succeed. */
+  readonly retriable: boolean;
+  constructor(
+    message: string,
+    opts: { code?: string; httpStatus?: number; retriable: boolean },
+  ) {
+    super(message);
+    this.name = "TranscriptionError";
+    this.code = opts.code;
+    this.httpStatus = opts.httpStatus;
+    this.retriable = opts.retriable;
+  }
+}

package/core/src/vault-projection.ts

@@ -33,7 +33,7 @@ import {
 import { DEFAULT_TAG_NAME } from "./tag-hierarchy.ts";
 import * as noteOps from "./notes.ts";
 import type { VaultStats } from "./types.ts";
-import { GETTING_STARTED_PATH } from "./onboarding.ts";
+import { GETTING_STARTED_PATH } from "./seed-packs.ts";
 
 /**
  * Does a note live at the seeded onboarding path? Used to gate the "Start here"

package/core/src/wikilinks.ts

@@ -1,5 +1,6 @@
 import { Database } from "bun:sqlite";
 import * as linkOps from "./links.js";
+import { chunkForInClause } from "./sql-in.js";
 
 // ---------------------------------------------------------------------------
 // Parser — extract [[wikilinks]] from markdown content
@@ -272,10 +273,15 @@ export function listUnresolvedWikilinks(db: Database, limit = 50): { unresolved:
   if (rows.length === 0) return { unresolved: [], count: total };
 
   const sourceIds = [...new Set(rows.map((r) => r.source_id))];
-  const placeholders = sourceIds.map(() => "?").join(", ");
-  const pathRows = db.prepare(
-    `SELECT id, path FROM notes WHERE id IN (${placeholders})`,
-  ).all(...sourceIds) as { id: string; path: string | null }[];
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts) — with a large
+  // `limit` this hydrates >100 source ids in one IN-list otherwise.
+  const pathRows: { id: string; path: string | null }[] = [];
+  for (const chunk of chunkForInClause(sourceIds)) {
+    const placeholders = chunk.map(() => "?").join(", ");
+    pathRows.push(...db.prepare(
+      `SELECT id, path FROM notes WHERE id IN (${placeholders})`,
+    ).all(...chunk) as { id: string; path: string | null }[]);
+  }
   const pathMap = new Map(pathRows.map((r) => [r.id, r.path]));
 
   const unresolved: UnresolvedWikilink[] = rows.map((r) => ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/vault",
-  "version": "0.6.5-rc.2",
+  "version": "0.6.5-rc.9",
   "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
   "module": "src/cli.ts",
   "type": "module",

package/src/add-pack.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Integration tests for `parachute-vault add-pack <pack> [--vault <name>]`.
+ *
+ * Spawns the CLI in a temp `PARACHUTE_HOME` (same harness shape as
+ * vault-create.test.ts). Covers: listing on no-arg / unknown pack, applying
+ * `surface-starter` onto a default-seeded vault, idempotent re-runs, the
+ * missing-vault error, and `--vault` targeting.
+ *
+ * No in-test `Bun.serve` is involved, so `Bun.spawnSync` is fine here (see
+ * CLAUDE.md "Subprocess tests + Bun.serve").
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { resolve } from "path";
+import { mkdtempSync, rmSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+
+const CLI = resolve(import.meta.dir, "cli.ts");
+
+function runCli(
+  args: string[],
+  env: Record<string, string>,
+): { exitCode: number; stdout: string; stderr: string } {
+  // Hermetic: don't inherit the dev/CI box's PARACHUTE_HUB_ORIGIN (see
+  // vault-create.test.ts for the rationale).
+  const baseEnv: Record<string, string | undefined> = { ...process.env };
+  delete baseEnv.PARACHUTE_HUB_ORIGIN;
+  const proc = Bun.spawnSync({
+    cmd: ["bun", CLI, ...args],
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...baseEnv, ...env },
+  });
+  return {
+    exitCode: proc.exitCode ?? -1,
+    stdout: new TextDecoder().decode(proc.stdout),
+    stderr: new TextDecoder().decode(proc.stderr),
+  };
+}
+
+/** Read all note paths from a vault's SQLite DB inside the temp home. */
+function readNotePaths(home: string, name: string): (string | null)[] {
+  const db = new Database(join(home, "vault", "data", name, "vault.db"), {
+    readonly: true,
+  });
+  try {
+    return (db.query("SELECT path FROM notes").all() as { path: string | null }[]).map(
+      (r) => r.path,
+    );
+  } finally {
+    db.close();
+  }
+}
+
+let home: string;
+
+beforeEach(() => {
+  home = mkdtempSync(join(tmpdir(), "add-pack-test-"));
+});
+
+afterEach(() => {
+  rmSync(home, { recursive: true, force: true });
+});
+
+describe("add-pack — listing + errors", () => {
+  test("no arguments → usage + the three packs, exit 1", () => {
+    const { exitCode, stderr } = runCli(["add-pack"], { PARACHUTE_HOME: home });
+    expect(exitCode).toBe(1);
+    expect(stderr).toContain("Usage: parachute-vault add-pack");
+    expect(stderr).toContain("Available packs:");
+    expect(stderr).toContain("welcome");
+    expect(stderr).toContain("getting-started");
+    expect(stderr).toContain("surface-starter");
+  });
+
+  test("unknown pack → error + listing, exit 1", () => {
+    const { exitCode, stderr } = runCli(["add-pack", "does-not-exist"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(1);
+    expect(stderr).toContain('Unknown pack: "does-not-exist"');
+    expect(stderr).toContain("Available packs:");
+  });
+
+  test("missing vault → actionable error, exit 1", () => {
+    const { exitCode, stderr } = runCli(
+      ["add-pack", "surface-starter", "--vault", "ghost"],
+      { PARACHUTE_HOME: home },
+    );
+    expect(exitCode).toBe(1);
+    expect(stderr).toContain('Vault "ghost" not found');
+    expect(stderr).toContain("parachute-vault create ghost");
+  });
+});
+
+describe("add-pack surface-starter", () => {
+  test("applies onto a default-seeded vault and reports what was added", () => {
+    expect(runCli(["create", "packed", "--json"], { PARACHUTE_HOME: home }).exitCode).toBe(0);
+    // Default seed does NOT include Surface Starter.
+    expect(readNotePaths(home, "packed")).not.toContain("Surface Starter");
+
+    const { exitCode, stdout } = runCli(
+      ["add-pack", "surface-starter", "--vault", "packed"],
+      { PARACHUTE_HOME: home },
+    );
+    expect(exitCode).toBe(0);
+    expect(stdout).toContain('Pack "surface-starter" applied to vault "packed"');
+    expect(stdout).toContain("+ note  Surface Starter");
+    expect(readNotePaths(home, "packed")).toContain("Surface Starter");
+  });
+
+  test("idempotent: a re-run skips (exit 0) and doesn't duplicate", () => {
+    runCli(["create", "packed", "--json"], { PARACHUTE_HOME: home });
+    runCli(["add-pack", "surface-starter", "--vault", "packed"], { PARACHUTE_HOME: home });
+
+    const { exitCode, stdout } = runCli(
+      ["add-pack", "surface-starter", "--vault", "packed"],
+      { PARACHUTE_HOME: home },
+    );
+    expect(exitCode).toBe(0);
+    expect(stdout).toContain("= note  Surface Starter (already exists — left untouched)");
+    expect(stdout).not.toContain("+ note");
+
+    const paths = readNotePaths(home, "packed");
+    expect(paths.filter((p) => p === "Surface Starter")).toHaveLength(1);
+  });
+
+  test("re-applying a default pack is a clean no-op-style run (welcome on a fresh vault)", () => {
+    runCli(["create", "packed", "--json"], { PARACHUTE_HOME: home });
+    const { exitCode, stdout } = runCli(
+      ["add-pack", "welcome", "--vault", "packed"],
+      { PARACHUTE_HOME: home },
+    );
+    expect(exitCode).toBe(0);
+    // Notes were already default-seeded → all skipped; tags re-upsert.
+    expect(stdout).not.toContain("+ note");
+    expect(stdout).toContain("= note  Welcome to your vault 🪂 (already exists — left untouched)");
+    expect(stdout).toContain("~ tag   capture (upserted)");
+  });
+});