@openparachute/vault 0.6.0 → 0.6.1

package/core/src/query-perf-routing.test.ts

@@ -0,0 +1,208 @@
+/**
+ * Regression pins for the 2026-06-10 query-perf work:
+ *
+ *  1. Plain `{field: value}` metadata equality on an INDEXED field now routes
+ *     through the generated column as an index prefilter, keeping the
+ *     original json_extract clause as a residual predicate. Results must be
+ *     IDENTICAL to the JSON-scan path — property-tested here by giving every
+ *     note twin fields (`*_idx` indexed, `*_raw` not) carrying the SAME
+ *     value, then asserting every probe returns the same ids through both.
+ *
+ *  2. Tag membership is a semijoin (no `JOIN note_tags` + `DISTINCT n.*`),
+ *     so a note matched by SEVERAL tags in one query must still appear once.
+ *
+ *  3. `getLinksHydratedForNotes` (the batched include_links path) must agree
+ *     with per-note `getLinksHydrated`, including self-loops and links whose
+ *     endpoints are both on the page.
+ */
+import { describe, it, expect, beforeEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { SqliteStore } from "./store.js";
+import { queryNotes } from "./notes.js";
+import { getLinksHydrated, getLinksHydratedForNotes } from "./links.js";
+
+let db: Database;
+let store: SqliteStore;
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  store = new SqliteStore(db);
+});
+
+describe("plain metadata equality: indexed routing is result-identical to the JSON scan", () => {
+  /**
+   * Edge values exercised through BOTH paths. Some of these never match
+   * (today's documented scan behavior — e.g. a JS number binds as its JSON
+   * text and SQLite won't equate INTEGER 5 with TEXT '5'); the pin is that
+   * indexed routing reproduces the scan's verdict EXACTLY, match or not.
+   */
+  const STORED_VALUES: unknown[] = [
+    "pending",
+    "",
+    "5",            // numeric-looking string
+    5,              // JSON number — affinity edge case vs "5"
+    0,
+    -1.5,
+    true,
+    false,
+    null,
+    "naïve-Ünïcode ✨",
+    { nested: { deep: 1 } },
+    ["a", "b"],
+    undefined,      // field absent from metadata
+  ];
+
+  const PROBES: unknown[] = [
+    "pending",
+    "",
+    "5",
+    5,
+    0,
+    -1.5,
+    true,
+    false,
+    null,
+    "naïve-Ünïcode ✨",
+    "missing-everywhere",
+  ];
+
+  async function seedTwinFieldFixture(sqliteType: "string" | "integer") {
+    // `status_idx` is indexed (string) via a tag schema; `status_raw` is not.
+    await store.upsertTagRecord("thing", {
+      fields: { status_idx: { type: sqliteType, indexed: true } },
+    });
+    for (let i = 0; i < STORED_VALUES.length; i++) {
+      const v = STORED_VALUES[i];
+      const metadata: Record<string, unknown> = { seq: i };
+      if (v !== undefined) {
+        metadata.status_idx = v;
+        metadata.status_raw = v;
+      }
+      await store.createNote(`note ${i}`, { tags: ["thing"], metadata });
+    }
+  }
+
+  for (const sqliteType of ["string", "integer"] as const) {
+    it(`every probe matches the same notes through the indexed and scan paths (${sqliteType} column)`, async () => {
+      await seedTwinFieldFixture(sqliteType);
+      for (const probe of PROBES) {
+        const viaIndexed = queryNotes(db, { metadata: { status_idx: probe } }).map((n) => n.id);
+        const viaScan = queryNotes(db, { metadata: { status_raw: probe } }).map((n) => n.id);
+        expect(viaIndexed).toEqual(viaScan);
+      }
+    });
+  }
+
+  it("string probe on the indexed field actually matches (sanity: not vacuous)", async () => {
+    await seedTwinFieldFixture("string");
+    const hits = queryNotes(db, { metadata: { status_idx: "pending" } });
+    expect(hits.length).toBe(1);
+    expect(hits[0]!.metadata?.seq).toBe(0);
+  });
+
+  it("indexed plain equality agrees with the operator form for string values", async () => {
+    await seedTwinFieldFixture("string");
+    for (const probe of ["pending", "", "missing-everywhere", "naïve-Ünïcode ✨"]) {
+      const plain = queryNotes(db, { metadata: { status_idx: probe } }).map((n) => n.id);
+      const operator = queryNotes(db, { metadata: { status_idx: { eq: probe } } }).map((n) => n.id);
+      expect(plain).toEqual(operator);
+    }
+  });
+
+  it("plain equality on a non-indexed field still works (no FIELD_NOT_INDEXED error)", async () => {
+    await store.createNote("a", { metadata: { color: "blue" } });
+    await store.createNote("b", { metadata: { color: "red" } });
+    const hits = queryNotes(db, { metadata: { color: "blue" } });
+    expect(hits.length).toBe(1);
+  });
+});
+
+describe("tag semijoin: no duplicate rows without DISTINCT", () => {
+  it("a note carrying several matching tags appears once under tag_match=any", async () => {
+    await store.upsertTagRecord("capture/voice", { parent_names: ["capture"] });
+    await store.upsertTagRecord("capture/text", { parent_names: ["capture"] });
+    // Tagged with the parent AND two children — the old JOIN produced 3 rows.
+    const note = await store.createNote("multi", {
+      tags: ["capture", "capture/voice", "capture/text"],
+    });
+    await store.createNote("single", { tags: ["capture/voice"] });
+
+    const viaExpansion = await store.queryNotes({ tags: ["capture"] });
+    expect(viaExpansion.map((n) => n.id).filter((id) => id === note.id).length).toBe(1);
+    expect(viaExpansion.length).toBe(2);
+
+    const viaAny = await store.queryNotes({
+      tags: ["capture", "capture/voice", "capture/text"],
+      tagMatch: "any",
+    });
+    expect(viaAny.map((n) => n.id).filter((id) => id === note.id).length).toBe(1);
+  });
+
+  it("tag_match=all still requires every input tag", async () => {
+    const both = await store.createNote("both", { tags: ["a", "b"] });
+    await store.createNote("only-a", { tags: ["a"] });
+    const hits = await store.queryNotes({ tags: ["a", "b"], tagMatch: "all" });
+    expect(hits.map((n) => n.id)).toEqual([both.id]);
+  });
+});
+
+describe("getLinksHydratedForNotes agrees with per-note getLinksHydrated", () => {
+  it("matches per-note hydration across self-loops, shared links, and isolated notes", async () => {
+    const a = await store.createNote("a", { tags: ["x"] });
+    const b = await store.createNote("b", { tags: ["y"], metadata: { k: 1 } });
+    const c = await store.createNote("c", { path: "Notes/c" });
+    const lonely = await store.createNote("lonely");
+
+    await store.createLink(a.id, b.id, "mentions");      // both endpoints on page
+    await store.createLink(b.id, a.id, "replies");       // reverse direction
+    await store.createLink(a.id, a.id, "self");          // self-loop
+    await store.createLink(c.id, a.id, "references");    // inbound from page note
+    await store.createLink(b.id, c.id, "cites", { via: "test" });
+
+    const pageIds = [a.id, b.id, c.id, lonely.id];
+    const batch = getLinksHydratedForNotes(db, pageIds);
+
+    for (const id of pageIds) {
+      const single = getLinksHydrated(db, id);
+      const batched = batch.get(id)!;
+      const key = (l: { sourceId: string; targetId: string; relationship: string }) =>
+        `${l.sourceId}→${l.targetId}:${l.relationship}`;
+      // Same link set per note…
+      expect(batched.map(key).sort()).toEqual(single.map(key).sort());
+      // …ordered newest-first like the single-note SQL.
+      const stamps = batched.map((l) => l.createdAt);
+      expect([...stamps].sort().reverse()).toEqual(stamps);
+      // …with identical hydrated summaries (path, tags, metadata).
+      const summaries = (ls: typeof single) =>
+        new Map(ls.map((l) => [key(l), JSON.stringify([l.sourceNote, l.targetNote])]));
+      expect(summaries(batched)).toEqual(summaries(single));
+    }
+
+    // The self-loop appears once in a's list (not duplicated by the
+    // source/target double-walk), matching the single-note query.
+    const selfLoops = batch.get(a.id)!.filter((l) => l.sourceId === a.id && l.targetId === a.id);
+    expect(selfLoops.length).toBe(1);
+
+    expect(batch.get(lonely.id)).toEqual([]);
+  });
+});
+
+describe("v22 keyset index", () => {
+  it("exists after initSchema and covers (updated_at, id)", () => {
+    const row = db
+      .prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_notes_updated'")
+      .get();
+    expect(row).toBeTruthy();
+    const plan = db
+      .prepare(
+        `EXPLAIN QUERY PLAN
+         SELECT n.id FROM notes n
+         WHERE (n.updated_at > ? OR (n.updated_at = ? AND n.id > ?))
+         ORDER BY n.updated_at ASC, n.id ASC LIMIT 10`,
+      )
+      .all("2025-01-01", "2025-01-01", "x") as { detail: string }[];
+    const details = plan.map((r) => r.detail).join(" | ");
+    expect(details).toContain("idx_notes_updated");
+    expect(details).not.toContain("TEMP B-TREE");
+  });
+});

package/core/src/schema.ts

@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
 import { normalizePath } from "./paths.js";
 import { rebuildIndexes } from "./indexed-fields.js";
 
-export const SCHEMA_VERSION = 21;
+export const SCHEMA_VERSION = 22;
 
 export const SCHEMA_SQL = `
 -- Notes: the universal record.
@@ -474,6 +474,11 @@ export function initSchema(db: Database): void {
   // above, so this is a defensive confirmation hook for upgrading vaults.
   migrateToV21(db);
 
+  // Migrate v21 → v22: composite index notes(updated_at, id) backing cursor
+  // keyset pagination and date_filter on updated_at — both were full table
+  // scans. See the 2026-06-10 query-perf measurements.
+  migrateToV22(db);
+
   // Rebuild any generated columns + indexes declared in indexed_fields.
   // No-op for a fresh vault; idempotent on existing vaults.
   rebuildIndexes(db);
@@ -1121,6 +1126,30 @@ function migrateToV21(db: Database): void {
   `);
 }
 
+/**
+ * Migrate v21 → v22: composite B-tree on notes(updated_at, id).
+ *
+ * Two query shapes ride it (2026-06-10 perf measurements — both were full
+ * table scans + temp-B-tree sorts before):
+ *
+ *   1. Cursor keyset pagination (vault#313): the predicate
+ *      `updated_at > ? OR (updated_at = ? AND id > ?)` with
+ *      `ORDER BY updated_at ASC, id ASC` matches the composite key exactly,
+ *      so a "since last checked" poll seeks straight to the watermark and
+ *      streams in order — no scan, no sort.
+ *   2. `date_filter: { field: "updated_at" }` range queries (incremental
+ *      rebuild flows, vault#285 1.5).
+ *
+ * Lives here (not SCHEMA_SQL) following the idx_tokens_vault_name precedent:
+ * migrations run after every column-shape change, so the index statement
+ * never races an older table shape. Fresh vaults reach this through the
+ * same initSchema path — CREATE INDEX IF NOT EXISTS is idempotent.
+ */
+function migrateToV22(db: Database): void {
+  if (!hasTable(db, "notes")) return;
+  db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated ON notes(updated_at, id)");
+}
+
 function hasTable(db: Database, name: string): boolean {
   const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
   return !!row;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/vault",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
   "module": "src/cli.ts",
   "type": "module",

package/src/content-range-routes.test.ts

@@ -0,0 +1,178 @@
+/**
+ * REST face of content range / pagination (bounded reads for large notes).
+ *
+ * Exercises all four GET shapes that accept `content_offset` /
+ * `content_length`:
+ *   - GET /notes?id=…           (single, folded into the collection route)
+ *   - GET /notes/:idOrPath      (single point read)
+ *   - GET /notes?…              (structured list)
+ *   - GET /notes?search=…       (full-text list)
+ *
+ * Slice mechanics (codepoint boundaries, reassembly invariant) are pinned
+ * in core/src/content-range.test.ts; this suite covers the HTTP wiring:
+ * param parsing, 400s, per-shape application, and the no-params
+ * regression. Fully sandboxed — in-memory SQLite, no daemon.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { BunStore } from "./vault-store.ts";
+import { handleNotes } from "./routes.ts";
+
+let db: Database;
+let store: BunStore;
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  store = new BunStore(db);
+});
+
+afterEach(() => {
+  db.close();
+});
+
+const BASE = "http://localhost/api";
+
+function get(path: string): Promise<Response> {
+  return handleNotes(new Request(`${BASE}/notes${path}`, { method: "GET" }), store, pathSub(path));
+}
+
+/** Extract the handleNotes subpath ("/<id>" for point reads, "" otherwise). */
+function pathSub(path: string): string {
+  const m = path.match(/^\/([^?/]+)/);
+  return m ? `/${m[1]}` : "";
+}
+
+describe("REST content range — single note", () => {
+  it("GET /notes?id=… honors content_length and adds the range fields", async () => {
+    const note = await store.createNote("0123456789");
+    const res = await get(`?id=${note.id}&content_length=4`);
+    expect(res.status).toBe(200);
+    const body: any = await res.json();
+    expect(body.content).toBe("0123");
+    expect(body.content_offset).toBe(0);
+    expect(body.content_total_length).toBe(10);
+    expect(body.content_next_offset).toBe(4);
+  });
+
+  it("GET /notes/:id honors content_offset (tail read to the end)", async () => {
+    const note = await store.createNote("hello world", { path: "tail-note" });
+    const res = await get(`/${note.id}?content_offset=6`);
+    expect(res.status).toBe(200);
+    const body: any = await res.json();
+    expect(body.content).toBe("world");
+    expect(body.content_total_length).toBe(11);
+    expect(body.content_next_offset).toBeNull();
+  });
+
+  it("paged loop over REST reassembles multi-byte content byte-identically", async () => {
+    const content = "ab\u{1F600}cd 你好 ".repeat(20).trim();
+    const note = await store.createNote(content);
+    let offset = 0;
+    let assembled = "";
+    for (;;) {
+      const res = await get(`/${note.id}?content_offset=${offset}&content_length=16`);
+      expect(res.status).toBe(200);
+      const body: any = await res.json();
+      expect(Buffer.byteLength(body.content, "utf8")).toBeLessThanOrEqual(16);
+      assembled += body.content;
+      if (body.content_next_offset === null) break;
+      offset = body.content_next_offset;
+    }
+    expect(assembled).toBe(content);
+  });
+
+  it("offset past end → 200 with empty slice and null next_offset", async () => {
+    const note = await store.createNote("abc");
+    const res = await get(`?id=${note.id}&content_offset=500`);
+    expect(res.status).toBe(200);
+    const body: any = await res.json();
+    expect(body.content).toBe("");
+    expect(body.content_next_offset).toBeNull();
+    expect(body.content_total_length).toBe(3);
+  });
+
+  it("include_content=false + range params → 400 INVALID_QUERY", async () => {
+    const note = await store.createNote("abc");
+    const res = await get(`?id=${note.id}&include_content=false&content_length=8`);
+    expect(res.status).toBe(400);
+    const body: any = await res.json();
+    expect(body.code).toBe("INVALID_QUERY");
+    expect(body.error).toContain("include_content");
+  });
+
+  it("zero / sub-minimum / malformed budgets → 400 INVALID_QUERY", async () => {
+    const note = await store.createNote("abc");
+    for (const qs of ["content_length=0", "content_length=2", "content_length=abc", "content_offset=-1"]) {
+      const res = await get(`?id=${note.id}&${qs}`);
+      expect(res.status).toBe(400);
+      const body: any = await res.json();
+      expect(body.code).toBe("INVALID_QUERY");
+    }
+  });
+
+  it("no range params → response shape unchanged (regression)", async () => {
+    const note = await store.createNote("plain body", { path: "plain" });
+    const res = await get(`/${note.id}`);
+    expect(res.status).toBe(200);
+    const body: any = await res.json();
+    expect(body.content).toBe("plain body");
+    expect("content_total_length" in body).toBe(false);
+    expect("content_next_offset" in body).toBe(false);
+    expect("content_offset" in body).toBe(false);
+  });
+});
+
+describe("REST content range — list shapes", () => {
+  it("structured list with include_content=true applies the window per note", async () => {
+    await store.createNote("first body first body", { tags: ["paged"] });
+    await store.createNote("second body second body", { tags: ["paged"] });
+    const res = await get(`?tag=paged&include_content=true&content_length=6`);
+    expect(res.status).toBe(200);
+    const body: any[] = await res.json();
+    expect(body.length).toBe(2);
+    for (const n of body) {
+      expect(Buffer.byteLength(n.content, "utf8")).toBeLessThanOrEqual(6);
+      expect(typeof n.content_total_length).toBe("number");
+      expect(n.content_next_offset).toBe(6);
+    }
+  });
+
+  it("structured list on the lean default + range params → 400", async () => {
+    await store.createNote("body", { tags: ["paged"] });
+    const res = await get(`?tag=paged&content_length=8`);
+    expect(res.status).toBe(400);
+    const body: any = await res.json();
+    expect(body.code).toBe("INVALID_QUERY");
+    expect(body.error).toContain("include_content");
+  });
+
+  it("search list with include_content=true applies the window per note", async () => {
+    await store.createNote("the quick brown fox jumps over the lazy dog");
+    const res = await get(`?search=fox&include_content=true&content_length=9`);
+    expect(res.status).toBe(200);
+    const body: any[] = await res.json();
+    expect(body.length).toBe(1);
+    expect(Buffer.byteLength(body[0].content, "utf8")).toBeLessThanOrEqual(9);
+    expect(body[0].content_total_length).toBe(
+      Buffer.byteLength("the quick brown fox jumps over the lazy dog", "utf8"),
+    );
+  });
+
+  it("search list on the lean default + range params → 400", async () => {
+    await store.createNote("the quick brown fox");
+    const res = await get(`?search=fox&content_length=8`);
+    expect(res.status).toBe(400);
+    const body: any = await res.json();
+    expect(body.code).toBe("INVALID_QUERY");
+  });
+
+  it("list without range params → no range fields injected (regression)", async () => {
+    await store.createNote("body here", { tags: ["paged"] });
+    const res = await get(`?tag=paged&include_content=true`);
+    expect(res.status).toBe(200);
+    const body: any[] = await res.json();
+    expect("content_total_length" in body[0]).toBe(false);
+    expect("content_next_offset" in body[0]).toBe(false);
+  });
+});