@openparachute/vault 0.6.4 → 0.6.5-rc.10

package/core/src/__fixtures__/golden-vault.ts

@@ -0,0 +1,125 @@
+/**
+ * Shared deterministic vault builder + export-tree serializer for the
+ * portable-md "old bytes == new bytes" golden fixture (Phase-1 streaming
+ * export refactor).
+ *
+ * Both the one-shot capture script (scripts/capture-portable-golden.ts,
+ * run against the PRE-refactor code) and the regression test
+ * (portable-md-golden.test.ts, run against the POST-refactor code) build
+ * the SAME vault here and serialize the export tree the SAME way — so a
+ * byte diff between the committed golden JSON and a fresh export is a real
+ * drift, not a fixture-vs-test mismatch.
+ *
+ * Determinism requirements met here:
+ *   - explicit note ids + pinned created_at/updated_at (no wall-clock)
+ *   - export called with `exportedAt` + `caseSensitiveOverride: true`
+ *     (so the bytes don't depend on the runner's filesystem)
+ *   - content that exercises every format field: pathed / unpathed /
+ *     empty-content notes, a non-md (csv) note that forces a notes-meta
+ *     sidecar, tags with schema + relationships, typed links, multi-line
+ *     + control-character metadata, and created_at != updated_at.
+ */
+
+import { readdirSync, readFileSync, statSync } from "fs";
+import { join } from "path";
+import type { Store } from "../types.js";
+
+/** Fixed export timestamp — pins vault.yaml's `exported_at`. */
+export const GOLDEN_EXPORTED_AT = "2026-07-02T00:00:00.000Z";
+
+/**
+ * Populate `store` with a deterministic, format-exhaustive vault. Callers
+ * export it with `{ exportedAt: GOLDEN_EXPORTED_AT, caseSensitiveOverride: true }`.
+ */
+export async function buildGoldenVault(store: Store): Promise<void> {
+  // Tag schema + opaque relationship vocabulary (vault#428 shape).
+  await store.upsertTagSchema("project", {
+    description: "A long-running effort",
+    fields: { status: { type: "string", enum: ["active", "done"], indexed: true } },
+  });
+  await store.upsertTagRecord("project", {
+    relationships: {
+      "works-on": { from: "person", to: "project" },
+      "based-at": { from: "project", to: "place", note: "freeform" },
+    },
+  });
+
+  const t0 = "2026-01-01T00:00:00.000Z";
+  const t1 = "2026-01-01T00:01:00.000Z";
+
+  await store.createNote("alpha body", {
+    id: "01HX001",
+    path: "Inbox/alpha",
+    tags: ["project", "z-other"],
+    metadata: {
+      priority: "high",
+      notes: "line1\nline2\nline3",
+      // control character exercises the \xNN escape path (vault#317 F1)
+      ctrl: "a\tb",
+      status: "active",
+    },
+    created_at: t0,
+  });
+  await store.createNote("beta body", {
+    id: "01HX002",
+    path: "Inbox/beta",
+    tags: ["project"],
+    metadata: { status: "done" },
+    created_at: t0,
+  });
+  await store.createNote("unpathed jot", { id: "01HX003", created_at: t0 });
+  // Empty-content skeleton note (vault#323).
+  await store.createNote("", {
+    id: "01HX004",
+    path: "Inbox/skeleton",
+    tags: ["project"],
+    created_at: t0,
+  });
+  // Non-md note → forces a .parachute/notes-meta/<id>.yaml sidecar.
+  await store.createNote("col1,col2\n1,2\n", {
+    id: "01HX005",
+    path: "Data/table",
+    extension: "csv",
+    tags: ["z-other"],
+    metadata: { rows: 2 },
+    created_at: t0,
+  });
+
+  await store.createLink("01HX001", "01HX002", "derived-from", { source: "git://example" });
+
+  // Pin every timestamp so the export bytes are wall-clock-independent.
+  // n1 gets a divergent updated_at to exercise restoreNoteTimestamps.
+  await store.restoreNoteTimestamps("01HX001", t0, t1);
+  for (const id of ["01HX002", "01HX003", "01HX004", "01HX005"]) {
+    await store.restoreNoteTimestamps(id, t0, t0);
+  }
+}
+
+/**
+ * Recursively serialize an export directory into a sorted map of
+ * `<relative-posix-path>` → file content. Directories are recorded as
+ * entries with a trailing `/` and an empty value so an intentionally
+ * empty dir (e.g. `.parachute/schemas/` on a schema-less vault) still
+ * shows up in the byte comparison.
+ */
+export function serializeExportTree(dir: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  const walk = (abs: string, rel: string): void => {
+    const entries = readdirSync(abs).sort();
+    if (entries.length === 0 && rel !== "") {
+      out[rel + "/"] = "";
+      return;
+    }
+    for (const entry of entries) {
+      const childAbs = join(abs, entry);
+      const childRel = rel === "" ? entry : `${rel}/${entry}`;
+      if (statSync(childAbs).isDirectory()) {
+        walk(childAbs, childRel);
+      } else {
+        out[childRel] = readFileSync(childAbs, "utf-8");
+      }
+    }
+  };
+  walk(dir, "");
+  return out;
+}

package/core/src/__fixtures__/portable-export-golden.json

@@ -0,0 +1,10 @@
+{
+  ".parachute/notes-meta/01HX005.yaml": "id: 01HX005\npath: Data/table\nextension: csv\ntags:\n  - z-other\nmetadata:\n  rows: 2\ncreated_at: 2026-01-01T00:00:00.000Z\nupdated_at: 2026-01-01T00:00:00.000Z\n",
+  ".parachute/schemas/project.yaml": "description: A long-running effort\nfields:\n  status:\n    enum:\n      - active\n      - done\n    indexed: true\n    type: string\nname: project\nrelationships:\n  based-at:\n    from: project\n    note: freeform\n    to: place\n  works-on:\n    from: person\n    to: project\n",
+  ".parachute/vault.yaml": "description: portable-md byte-stability fixture\nexport_format_version: 1\nexported_at: 2026-07-02T00:00:00.000Z\nname: golden\n",
+  "Data/table.csv": "col1,col2\n1,2\n",
+  "Inbox/alpha.md": "---\nid: 01HX001\npath: Inbox/alpha\ntags:\n  - project\n  - z-other\nmetadata:\n  ctrl: \"a\\tb\"\n  notes: \"line1\\nline2\\nline3\"\n  priority: high\n  status: active\nlinks:\n  - metadata:\n      source: git://example\n    relationship: derived-from\n    target: 01HX002\ncreated_at: 2026-01-01T00:00:00.000Z\nupdated_at: 2026-01-01T00:01:00.000Z\n---\nalpha body\n",
+  "Inbox/beta.md": "---\nid: 01HX002\npath: Inbox/beta\ntags:\n  - project\nmetadata:\n  status: done\ncreated_at: 2026-01-01T00:00:00.000Z\nupdated_at: 2026-01-01T00:00:00.000Z\n---\nbeta body\n",
+  "Inbox/skeleton.md": "---\nid: 01HX004\npath: Inbox/skeleton\ntags:\n  - project\ncreated_at: 2026-01-01T00:00:00.000Z\nupdated_at: 2026-01-01T00:00:00.000Z\n---\n",
+  "_unpathed/01HX003.md": "---\nid: 01HX003\ncreated_at: 2026-01-01T00:00:00.000Z\nupdated_at: 2026-01-01T00:00:00.000Z\n---\nunpathed jot\n"
+}

package/core/src/do-param-cap.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Behavioral proof that the id-list query paths stay under the Cloudflare
+ * Durable Object SQLite 100-bound-param cap over a real >100-note vault.
+ *
+ * bun:sqlite tolerates 999+ bound params, so it CANNOT reproduce the DO
+ * `too many SQL variables` 500 directly — the ultimate DO-constraint proof is
+ * the cloud staging export re-verify (orchestrator's post-merge step). What we
+ * CAN prove locally, and what actually catches the regression that let the bug
+ * ship, is the *shape* of the SQL each path issues: no single prepared
+ * statement binds more than the chunk size (chunked lists) or a handful of
+ * params (json_each id-sets). A prepare-spy records the max bind count per
+ * statement; if a future edit raises the chunk size back over the cap, the
+ * placeholder-IN assertion here fails.
+ *
+ * Root cause recap: DO SQLite caps bound params at 100/statement. The export
+ * walk pages 500 notes at a time and hydrated each page's rows + tags with an
+ * `id IN (?×page)` list sized for standard SQLite's 999 floor → >100 params →
+ * DO 500. Fix: chunk id-lists at 90 (sql-in.ts) and bind embedded id-sets
+ * (e.g. `near` neighborhoods) as one json_each param.
+ */
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { existsSync, mkdtempSync, rmSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+
+import { SqliteStore } from "./store.js";
+import { exportVaultToDir } from "./portable-md.js";
+import { IN_PARAM_CHUNK } from "./sql-in.js";
+
+/** Zero-padded so lexicographic id order == numeric order. */
+const pad = (i: number): string => String(i).padStart(5, "0");
+
+/** Wrap `db.prepare` to record the max bind-arg count seen per SQL string.
+ *  Install AFTER seeding so bulk-insert params don't pollute the sample. */
+function installBindSpy(db: Database) {
+  const perSql = new Map<string, number>();
+  const origPrepare = db.prepare.bind(db);
+  (db as unknown as { prepare: (sql: string) => unknown }).prepare = (sql: string) => {
+    const stmt = origPrepare(sql) as Record<string, unknown>;
+    const record = (n: number) => perSql.set(sql, Math.max(perSql.get(sql) ?? 0, n));
+    for (const m of ["all", "get", "run", "values", "iterate"] as const) {
+      const orig = stmt[m];
+      if (typeof orig === "function") {
+        stmt[m] = (...args: unknown[]) => {
+          record(args.length);
+          return (orig as (...a: unknown[]) => unknown).apply(stmt, args);
+        };
+      }
+    }
+    return stmt;
+  };
+  return {
+    restore() {
+      (db as unknown as { prepare: unknown }).prepare = origPrepare;
+    },
+    /** Max binds over statements carrying a literal `IN (?, …)` placeholder
+     *  list — the chunked kind that must never exceed the chunk size. */
+    maxPlaceholderInBinds(): number {
+      let max = 0;
+      for (const [sql, n] of perSql) if (/\bIN \(\?/.test(sql)) max = Math.max(max, n);
+      return max;
+    },
+    /** Max binds over ALL statements — must stay under the DO cap. */
+    maxAnyBinds(): number {
+      let max = 0;
+      for (const n of perSql.values()) max = Math.max(max, n);
+      return max;
+    },
+    usedJsonEach(): boolean {
+      for (const sql of perSql.keys()) if (/json_each\(/.test(sql)) return true;
+      return false;
+    },
+  };
+}
+
+const DO_PARAM_CAP = 100;
+
+describe("id-list query paths stay under the DO 100-bound-param cap", () => {
+  let store: SqliteStore;
+  let db: Database;
+
+  beforeEach(async () => {
+    db = new Database(":memory:");
+    store = new SqliteStore(db);
+    // 250 notes > the DO cap and > 2× the chunk size, so chunking spans
+    // multiple chunks (250 = 90 + 90 + 70).
+    const inputs = [];
+    for (let i = 0; i < 250; i++) {
+      inputs.push({ content: `note ${i}`, id: `n${pad(i)}`, path: `notes/n${pad(i)}` });
+    }
+    await store.createNotes(inputs);
+  });
+
+  it("getNotes chunks a >100-id fetch (correct rows, no statement over the cap)", async () => {
+    const ids = Array.from({ length: 250 }, (_, i) => `n${pad(i)}`);
+    const spy = installBindSpy(db);
+    try {
+      const notes = await store.getNotes(ids);
+      // Correctness: every requested note returned, ordered by created_at then id.
+      expect(notes.length).toBe(250);
+      expect(notes.map((n) => n.id)).toEqual(ids);
+    } finally {
+      spy.restore();
+    }
+    // Regression guard: no IN-list statement binds more than the chunk size...
+    expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+    // ...and nothing at all trips the DO cap.
+    expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+  });
+
+  it("queryNotes over a >100-id set (the `near` path) binds the set as one json_each param", async () => {
+    const ids = Array.from({ length: 150 }, (_, i) => `n${pad(i)}`);
+    const spy = installBindSpy(db);
+    let notes;
+    try {
+      notes = await store.queryNotes({ ids, limit: 1000, sort: "asc" });
+    } finally {
+      spy.restore();
+    }
+    // Correctness: all 150 in-set notes returned.
+    expect(notes.length).toBe(150);
+    // The id-set filter rode a single json_each param, not 150 placeholders...
+    expect(spy.usedJsonEach()).toBe(true);
+    // ...so no statement — including the phase-1 id select carrying the set —
+    // exceeds the DO cap, and the phase-2 chunked hydration stays under it.
+    expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+    expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+  });
+
+  it("empty id-set still short-circuits (no json_each, no rows)", async () => {
+    const notes = await store.queryNotes({ ids: [], limit: 10 });
+    expect(notes.length).toBe(0);
+  });
+
+  describe("export a >100-note vault", () => {
+    let outDir: string;
+    beforeEach(() => { outDir = mkdtempSync(join(tmpdir(), "do-cap-export-")); });
+    afterEach(() => { try { rmSync(outDir, { recursive: true, force: true }); } catch {} });
+
+    it("exports all 250 notes with no statement over the cap (the shipped bug)", async () => {
+      const spy = installBindSpy(db);
+      let stats;
+      try {
+        stats = await exportVaultToDir(store, { outDir, exportedAt: "2026-07-03T00:00:00.000Z" });
+      } finally {
+        spy.restore();
+      }
+      // Correctness: every note on disk.
+      expect(stats.notes).toBe(250);
+      expect(existsSync(join(outDir, "notes/n00000.md"))).toBe(true);
+      expect(existsSync(join(outDir, "notes/n00090.md"))).toBe(true); // first chunk boundary
+      expect(existsSync(join(outDir, "notes/n00249.md"))).toBe(true); // tail
+      // The bug: the export page's `id IN (?×page)` hydration exceeded 100 on
+      // DO. Now every IN-list statement stays under the chunk size...
+      expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+      // ...and no statement trips the DO cap.
+      expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+    });
+  });
+});

package/core/src/links.ts

@@ -1,6 +1,7 @@
 import { Database } from "bun:sqlite";
 import type { Link, NoteSummary, HydratedLink } from "./types.js";
 import { getNoteTagsForNotes } from "./notes.js";
+import { chunkForInClause } from "./sql-in.js";
 
 export function createLink(
   db: Database,
@@ -103,16 +104,13 @@ function parseMetadata(raw: string | null): Record<string, unknown> | undefined
   try { return JSON.parse(raw); } catch { return undefined; }
 }
 
-/** IN-list chunk size — matches getLinkCounts' conservative bound-variable floor. */
-const IN_CHUNK = 900;
-
 function getNoteSummaries(db: Database, noteIds: string[]): Map<string, NoteSummary> {
   const map = new Map<string, NoteSummary>();
   if (noteIds.length === 0) return map;
   const ids = [...new Set(noteIds)];
   const rows: SummaryRow[] = [];
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts).
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     rows.push(...db.prepare(
       `SELECT id, path, metadata, created_at, updated_at FROM notes WHERE id IN (${placeholders})`,
@@ -193,8 +191,8 @@ export function getLinksHydratedForNotes(
   // (source, target, relationship) primary key so a link whose endpoints
   // are both on the page is fetched once.
   const rowsByKey = new Map<string, LinkRow>();
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts).
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     for (const column of ["source_id", "target_id"] as const) {
       const rows = db.prepare(
@@ -265,12 +263,9 @@ export function getLinkCounts(
   const wantOutbound = direction === "outbound" || direction === "both";
   const wantInbound = direction === "inbound" || direction === "both";
 
-  // SQLite's default SQLITE_MAX_VARIABLE_NUMBER is 999 on older builds and
-  // 32766 on newer ones; chunk well under the conservative floor so the
-  // IN-list never trips the bind limit on a large page.
-  const CHUNK = 900;
-  for (let i = 0; i < ids.length; i += CHUNK) {
-    const chunk = ids.slice(i, i + CHUNK);
+  // Chunk under the Cloudflare Durable Object SQLite 100-bound-param cap
+  // (bun:sqlite tolerates 999+, DO SQLite rejects >100). See sql-in.ts.
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
 
     if (wantOutbound) {