@openparachute/vault 0.6.5-rc.2 → 0.6.5

package/.parachute/module.json

@@ -12,6 +12,7 @@
   "focus": "core",
   "adminCapabilities": ["config", "credentials"],
   "startCmd": ["parachute-vault", "serve"],
+  "websocket": true,
   "scopes": {
     "defines": ["vault:read", "vault:write", "vault:admin"]
   },

package/core/src/core.test.ts

@@ -4046,7 +4046,7 @@ describe("query-notes link expansion", async () => {
 });
 
 // ---------------------------------------------------------------------------
-// Tag hierarchy via tags.parent_names (post-v14, patterns/tag-data-model.md)
+// Tag hierarchy via tags.parent_names (post-v14, docs/contracts/tag-data-model.md)
 // ---------------------------------------------------------------------------
 
 describe("tag hierarchy (tags.parent_names)", async () => {
@@ -5107,7 +5107,7 @@ describe("schema inheritance via parent_names (vault#270)", async () => {
   });
 });
 
-describe("expandTagsWithDescendants (tag-scoped tokens — patterns/tag-scoped-tokens.md)", async () => {
+describe("expandTagsWithDescendants (tag-scoped tokens — docs/contracts/tag-scoped-tokens.md)", async () => {
   it("returns the union of root + every descendant per tags.parent_names", async () => {
     await store.upsertTagRecord("health/food", { parent_names: ["health"] });
     await store.upsertTagRecord("health/food/breakfast", { parent_names: ["health/food"] });
@@ -5143,10 +5143,10 @@ describe("expandTagsWithDescendants (tag-scoped tokens — patterns/tag-scoped-t
 });
 
 // ---------------------------------------------------------------------------
-// Tag record API — patterns/tag-data-model.md
+// Tag record API — docs/contracts/tag-data-model.md
 // ---------------------------------------------------------------------------
 
-describe("tag record API (patterns/tag-data-model.md)", async () => {
+describe("tag record API (docs/contracts/tag-data-model.md)", async () => {
   it("upsertTagRecord persists description + fields + relationships + parent_names", async () => {
     await store.upsertTagRecord("project", {
       description: "long-running deliverable",
@@ -5430,7 +5430,7 @@ describe("tag record API (patterns/tag-data-model.md)", async () => {
 });
 
 // ---------------------------------------------------------------------------
-// Schema migration v13 → v14 — patterns/tag-data-model.md
+// Schema migration v13 → v14 — docs/contracts/tag-data-model.md
 // ---------------------------------------------------------------------------
 
 describe("schema migration v13 → v14", async () => {
@@ -5809,7 +5809,7 @@ describe("schema migration v15 → v16", async () => {
 });
 
 // ---------------------------------------------------------------------------
-// Tag-scope auth post-v14 — patterns/tag-scoped-tokens.md
+// Tag-scope auth post-v14 — docs/contracts/tag-scoped-tokens.md
 // ---------------------------------------------------------------------------
 
 describe("tag-scope auth (post-v14 hierarchy)", async () => {
@@ -5824,7 +5824,7 @@ describe("tag-scope auth (post-v14 hierarchy)", async () => {
   });
 
   it("orphan sub-tag fallback: token for `health` still sees `#health/food` even with no declared hierarchy", async () => {
-    // Per patterns/tag-scoped-tokens.md §Storage details, the auth check
+    // Per docs/contracts/tag-scoped-tokens.md §Storage details, the auth check
     // also splits on '/' and matches the root verbatim against the raw
     // allowlist. This survives the v14 source-of-truth swap because the
     // fallback lives in src/tag-scope.ts, not in the resolver.

package/core/src/do-param-cap.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Behavioral proof that the id-list query paths stay under the Cloudflare
+ * Durable Object SQLite 100-bound-param cap over a real >100-note vault.
+ *
+ * bun:sqlite tolerates 999+ bound params, so it CANNOT reproduce the DO
+ * `too many SQL variables` 500 directly — the ultimate DO-constraint proof is
+ * the cloud staging export re-verify (orchestrator's post-merge step). What we
+ * CAN prove locally, and what actually catches the regression that let the bug
+ * ship, is the *shape* of the SQL each path issues: no single prepared
+ * statement binds more than the chunk size (chunked lists) or a handful of
+ * params (json_each id-sets). A prepare-spy records the max bind count per
+ * statement; if a future edit raises the chunk size back over the cap, the
+ * placeholder-IN assertion here fails.
+ *
+ * Root cause recap: DO SQLite caps bound params at 100/statement. The export
+ * walk pages 500 notes at a time and hydrated each page's rows + tags with an
+ * `id IN (?×page)` list sized for standard SQLite's 999 floor → >100 params →
+ * DO 500. Fix: chunk id-lists at 90 (sql-in.ts) and bind embedded id-sets
+ * (e.g. `near` neighborhoods) as one json_each param.
+ */
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { existsSync, mkdtempSync, rmSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+
+import { SqliteStore } from "./store.js";
+import { exportVaultToDir } from "./portable-md.js";
+import { IN_PARAM_CHUNK } from "./sql-in.js";
+
+/** Zero-padded so lexicographic id order == numeric order. */
+const pad = (i: number): string => String(i).padStart(5, "0");
+
+/** Wrap `db.prepare` to record the max bind-arg count seen per SQL string.
+ *  Install AFTER seeding so bulk-insert params don't pollute the sample. */
+function installBindSpy(db: Database) {
+  const perSql = new Map<string, number>();
+  const origPrepare = db.prepare.bind(db);
+  (db as unknown as { prepare: (sql: string) => unknown }).prepare = (sql: string) => {
+    const stmt = origPrepare(sql) as Record<string, unknown>;
+    const record = (n: number) => perSql.set(sql, Math.max(perSql.get(sql) ?? 0, n));
+    for (const m of ["all", "get", "run", "values", "iterate"] as const) {
+      const orig = stmt[m];
+      if (typeof orig === "function") {
+        stmt[m] = (...args: unknown[]) => {
+          record(args.length);
+          return (orig as (...a: unknown[]) => unknown).apply(stmt, args);
+        };
+      }
+    }
+    return stmt;
+  };
+  return {
+    restore() {
+      (db as unknown as { prepare: unknown }).prepare = origPrepare;
+    },
+    /** Max binds over statements carrying a literal `IN (?, …)` placeholder
+     *  list — the chunked kind that must never exceed the chunk size. */
+    maxPlaceholderInBinds(): number {
+      let max = 0;
+      for (const [sql, n] of perSql) if (/\bIN \(\?/.test(sql)) max = Math.max(max, n);
+      return max;
+    },
+    /** Max binds over ALL statements — must stay under the DO cap. */
+    maxAnyBinds(): number {
+      let max = 0;
+      for (const n of perSql.values()) max = Math.max(max, n);
+      return max;
+    },
+    usedJsonEach(): boolean {
+      for (const sql of perSql.keys()) if (/json_each\(/.test(sql)) return true;
+      return false;
+    },
+  };
+}
+
+const DO_PARAM_CAP = 100;
+
+describe("id-list query paths stay under the DO 100-bound-param cap", () => {
+  let store: SqliteStore;
+  let db: Database;
+
+  beforeEach(async () => {
+    db = new Database(":memory:");
+    store = new SqliteStore(db);
+    // 250 notes > the DO cap and > 2× the chunk size, so chunking spans
+    // multiple chunks (250 = 90 + 90 + 70).
+    const inputs = [];
+    for (let i = 0; i < 250; i++) {
+      inputs.push({ content: `note ${i}`, id: `n${pad(i)}`, path: `notes/n${pad(i)}` });
+    }
+    await store.createNotes(inputs);
+  });
+
+  it("getNotes chunks a >100-id fetch (correct rows, no statement over the cap)", async () => {
+    const ids = Array.from({ length: 250 }, (_, i) => `n${pad(i)}`);
+    const spy = installBindSpy(db);
+    try {
+      const notes = await store.getNotes(ids);
+      // Correctness: every requested note returned, ordered by created_at then id.
+      expect(notes.length).toBe(250);
+      expect(notes.map((n) => n.id)).toEqual(ids);
+    } finally {
+      spy.restore();
+    }
+    // Regression guard: no IN-list statement binds more than the chunk size...
+    expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+    // ...and nothing at all trips the DO cap.
+    expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+  });
+
+  it("queryNotes over a >100-id set (the `near` path) binds the set as one json_each param", async () => {
+    const ids = Array.from({ length: 150 }, (_, i) => `n${pad(i)}`);
+    const spy = installBindSpy(db);
+    let notes;
+    try {
+      notes = await store.queryNotes({ ids, limit: 1000, sort: "asc" });
+    } finally {
+      spy.restore();
+    }
+    // Correctness: all 150 in-set notes returned.
+    expect(notes.length).toBe(150);
+    // The id-set filter rode a single json_each param, not 150 placeholders...
+    expect(spy.usedJsonEach()).toBe(true);
+    // ...so no statement — including the phase-1 id select carrying the set —
+    // exceeds the DO cap, and the phase-2 chunked hydration stays under it.
+    expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+    expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+  });
+
+  it("empty id-set still short-circuits (no json_each, no rows)", async () => {
+    const notes = await store.queryNotes({ ids: [], limit: 10 });
+    expect(notes.length).toBe(0);
+  });
+
+  describe("export a >100-note vault", () => {
+    let outDir: string;
+    beforeEach(() => { outDir = mkdtempSync(join(tmpdir(), "do-cap-export-")); });
+    afterEach(() => { try { rmSync(outDir, { recursive: true, force: true }); } catch {} });
+
+    it("exports all 250 notes with no statement over the cap (the shipped bug)", async () => {
+      const spy = installBindSpy(db);
+      let stats;
+      try {
+        stats = await exportVaultToDir(store, { outDir, exportedAt: "2026-07-03T00:00:00.000Z" });
+      } finally {
+        spy.restore();
+      }
+      // Correctness: every note on disk.
+      expect(stats.notes).toBe(250);
+      expect(existsSync(join(outDir, "notes/n00000.md"))).toBe(true);
+      expect(existsSync(join(outDir, "notes/n00090.md"))).toBe(true); // first chunk boundary
+      expect(existsSync(join(outDir, "notes/n00249.md"))).toBe(true); // tail
+      // The bug: the export page's `id IN (?×page)` hydration exceeded 100 on
+      // DO. Now every IN-list statement stays under the chunk size...
+      expect(spy.maxPlaceholderInBinds()).toBeLessThanOrEqual(IN_PARAM_CHUNK);
+      // ...and no statement trips the DO cap.
+      expect(spy.maxAnyBinds()).toBeLessThanOrEqual(DO_PARAM_CAP);
+    });
+  });
+});

package/core/src/links.ts

@@ -1,6 +1,7 @@
 import { Database } from "bun:sqlite";
 import type { Link, NoteSummary, HydratedLink } from "./types.js";
 import { getNoteTagsForNotes } from "./notes.js";
+import { chunkForInClause } from "./sql-in.js";
 
 export function createLink(
   db: Database,
@@ -103,16 +104,13 @@ function parseMetadata(raw: string | null): Record<string, unknown> | undefined
   try { return JSON.parse(raw); } catch { return undefined; }
 }
 
-/** IN-list chunk size — matches getLinkCounts' conservative bound-variable floor. */
-const IN_CHUNK = 900;
-
 function getNoteSummaries(db: Database, noteIds: string[]): Map<string, NoteSummary> {
   const map = new Map<string, NoteSummary>();
   if (noteIds.length === 0) return map;
   const ids = [...new Set(noteIds)];
   const rows: SummaryRow[] = [];
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts).
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     rows.push(...db.prepare(
       `SELECT id, path, metadata, created_at, updated_at FROM notes WHERE id IN (${placeholders})`,
@@ -193,8 +191,8 @@ export function getLinksHydratedForNotes(
   // (source, target, relationship) primary key so a link whose endpoints
   // are both on the page is fetched once.
   const rowsByKey = new Map<string, LinkRow>();
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts).
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     for (const column of ["source_id", "target_id"] as const) {
       const rows = db.prepare(
@@ -265,12 +263,9 @@ export function getLinkCounts(
   const wantOutbound = direction === "outbound" || direction === "both";
   const wantInbound = direction === "inbound" || direction === "both";
 
-  // SQLite's default SQLITE_MAX_VARIABLE_NUMBER is 999 on older builds and
-  // 32766 on newer ones; chunk well under the conservative floor so the
-  // IN-list never trips the bind limit on a large page.
-  const CHUNK = 900;
-  for (let i = 0; i < ids.length; i += CHUNK) {
-    const chunk = ids.slice(i, i + CHUNK);
+  // Chunk under the Cloudflare Durable Object SQLite 100-bound-param cap
+  // (bun:sqlite tolerates 999+, DO SQLite rejects >100). See sql-in.ts.
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
 
     if (wantOutbound) {

package/core/src/mcp.ts

@@ -1408,7 +1408,7 @@ Write-attribution (vault#298): every result carries \`createdBy\`/\`createdVia\`
     {
       name: "update-tag",
       requiredVerb: "write",
-      description: "Create or update a tag's identity row: description, indexed-field schemas, relationship-vocabulary map, and hierarchy parents. If the tag doesn't exist, it's created. Fields are merged (new keys added, existing keys replaced); relationships and parent_names are replaced wholesale when provided. Pass null for fields/relationships/parent_names to clear that column. See parachute-patterns/patterns/tag-data-model.md.",
+      description: "Create or update a tag's identity row: description, indexed-field schemas, relationship-vocabulary map, and hierarchy parents. If the tag doesn't exist, it's created. Fields are merged (new keys added, existing keys replaced); relationships and parent_names are replaced wholesale when provided. Pass null for fields/relationships/parent_names to clear that column. See parachute-vault/docs/contracts/tag-data-model.md.",
       inputSchema: {
         type: "object",
         properties: {

package/core/src/notes.ts

@@ -21,6 +21,7 @@ import {
 } from "./cursor.js";
 import { getIndexedField, releaseField } from "./indexed-fields.js";
 import { stripTagHash } from "./tag-hierarchy.js";
+import { chunkForInClause, IN_VIA_JSON_EACH, jsonEachParam } from "./sql-in.js";
 
 let idCounter = 0;
 
@@ -178,10 +179,26 @@ export function getNoteByPath(db: Database, path: string, extension?: string): N
 
 export function getNotes(db: Database, ids: string[]): Note[] {
   if (ids.length === 0) return [];
-  const placeholders = ids.map(() => "?").join(", ");
-  const rows = db.prepare(
-    `SELECT * FROM notes WHERE id IN (${placeholders}) ORDER BY created_at`,
-  ).all(...ids) as NoteRow[];
+  // Dedupe before chunking: a duplicate id straddling two chunk boundaries
+  // would otherwise be fetched (and returned) twice, unlike the old single
+  // IN-list which returned each matching row once regardless of duplicate
+  // params. Matches getNoteTagsForNotes / getNoteSummaries.
+  const uniqueIds = [...new Set(ids)];
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts). Each chunk is its
+  // own IN-list query; results are merged and re-sorted by created_at (with id
+  // as a deterministic tiebreak) to preserve the single-statement ORDER BY.
+  const rows: NoteRow[] = [];
+  for (const chunk of chunkForInClause(uniqueIds)) {
+    const placeholders = chunk.map(() => "?").join(", ");
+    rows.push(...db.prepare(
+      `SELECT * FROM notes WHERE id IN (${placeholders})`,
+    ).all(...chunk) as NoteRow[]);
+  }
+  rows.sort((a, b) =>
+    a.created_at < b.created_at ? -1
+    : a.created_at > b.created_at ? 1
+    : a.id < b.id ? -1 : a.id > b.id ? 1 : 0,
+  );
   return notesWithTags(db, rows);
 }
 
@@ -756,9 +773,13 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
       // with an always-false condition; building `IN ()` would be a SQL error.
       conditions.push("0 = 1");
     } else {
-      const placeholders = opts.ids.map(() => "?").join(", ");
-      conditions.push(`n.id IN (${placeholders})`);
-      params.push(...opts.ids);
+      // Bind the id-set as ONE json_each param, not one `?` per id: this IN is
+      // embedded in a paginated statement (shared ORDER BY / LIMIT / OFFSET),
+      // so it can't be chunked, and `near` neighborhoods routinely exceed the
+      // DO 100-bound-param cap. json_each keeps the whole set at a single
+      // bound param. See sql-in.ts.
+      conditions.push(`n.id IN ${IN_VIA_JSON_EACH}`);
+      params.push(jsonEachParam(opts.ids));
     }
   }
 
@@ -1044,10 +1065,6 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
   return fetchNotesByIdsOrdered(db, idRows.map((r) => r.id));
 }
 
-/** Chunk size for IN-list queries — comfortably under SQLite's conservative
- *  999 bound-variable floor (older builds), matching getLinkCounts. */
-const IN_CHUNK = 900;
-
 /**
  * Fetch full note rows for `ids`, preserving the input order, with tags
  * hydrated via ONE batched query per chunk (not one per note). Ids not
@@ -1056,8 +1073,9 @@ const IN_CHUNK = 900;
 function fetchNotesByIdsOrdered(db: Database, ids: string[]): Note[] {
   if (ids.length === 0) return [];
   const rowsById = new Map<string, NoteRow>();
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts) — a single export
+  // page (EXPORT_BATCH_SIZE=500) or a large query result exceeds it otherwise.
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     const rows = db.prepare(
       `SELECT * FROM notes WHERE id IN (${placeholders})`,
@@ -1084,8 +1102,8 @@ export function getNoteTagsForNotes(db: Database, noteIds: string[]): Map<string
   if (noteIds.length === 0) return map;
   const ids = [...new Set(noteIds)];
   for (const id of ids) map.set(id, []);
-  for (let i = 0; i < ids.length; i += IN_CHUNK) {
-    const chunk = ids.slice(i, i + IN_CHUNK);
+  // Chunk under the DO 100-bound-param cap (see sql-in.ts).
+  for (const chunk of chunkForInClause(ids)) {
     const placeholders = chunk.map(() => "?").join(", ");
     const rows = db.prepare(
       `SELECT note_id, tag_name FROM note_tags WHERE note_id IN (${placeholders}) ORDER BY tag_name`,