@openparachute/vault 0.7.2 → 0.7.3-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +5 -3
  2. package/core/src/attachment/policy.test.ts +66 -0
  3. package/core/src/attachment/policy.ts +131 -0
  4. package/core/src/attachment/tickets.test.ts +45 -0
  5. package/core/src/attachment/tickets.ts +117 -0
  6. package/core/src/attachment-tickets-tool.test.ts +286 -0
  7. package/core/src/conformance.ts +2 -1
  8. package/core/src/contract-typed-index.test.ts +4 -3
  9. package/core/src/core.test.ts +607 -11
  10. package/core/src/display-title.test.ts +190 -0
  11. package/core/src/embedding/chunker.test.ts +97 -0
  12. package/core/src/embedding/chunker.ts +180 -0
  13. package/core/src/embedding/provider.ts +108 -0
  14. package/core/src/embedding/staleness.test.ts +87 -0
  15. package/core/src/embedding/staleness.ts +83 -0
  16. package/core/src/embedding/vector-codec.test.ts +99 -0
  17. package/core/src/embedding/vector-codec.ts +60 -0
  18. package/core/src/embedding/vectors.test.ts +163 -0
  19. package/core/src/embedding/vectors.ts +135 -0
  20. package/core/src/expand.ts +11 -3
  21. package/core/src/indexed-fields.test.ts +9 -3
  22. package/core/src/indexed-fields.ts +9 -1
  23. package/core/src/lede.test.ts +96 -0
  24. package/core/src/mcp-semantic-search.test.ts +160 -0
  25. package/core/src/mcp.ts +405 -10
  26. package/core/src/notes.semantic-search.test.ts +131 -0
  27. package/core/src/notes.ts +319 -7
  28. package/core/src/query-warnings.ts +40 -0
  29. package/core/src/schema-defaults.ts +85 -1
  30. package/core/src/schema-v27-note-vectors.test.ts +178 -0
  31. package/core/src/schema.ts +81 -1
  32. package/core/src/search-fts-v25.test.ts +9 -1
  33. package/core/src/search-query.test.ts +42 -0
  34. package/core/src/search-query.ts +27 -0
  35. package/core/src/search-title-boost.test.ts +125 -0
  36. package/core/src/seed-packs.test.ts +117 -1
  37. package/core/src/seed-packs.ts +217 -1
  38. package/core/src/store.semantic-search.test.ts +236 -0
  39. package/core/src/store.ts +162 -5
  40. package/core/src/tag-schemas.ts +27 -12
  41. package/core/src/types.ts +63 -1
  42. package/core/src/vault-projection.ts +48 -0
  43. package/package.json +6 -1
  44. package/src/add-pack.test.ts +32 -0
  45. package/src/attachment-tickets.test.ts +350 -0
  46. package/src/attachment-tickets.ts +264 -0
  47. package/src/auth.ts +8 -2
  48. package/src/cli.ts +5 -1
  49. package/src/contract-errors.test.ts +2 -1
  50. package/src/contract-honest-queries.test.ts +88 -0
  51. package/src/contract-search.test.ts +41 -0
  52. package/src/embedding/capability.test.ts +33 -0
  53. package/src/embedding/capability.ts +34 -0
  54. package/src/embedding/external-api.test.ts +154 -0
  55. package/src/embedding/external-api.ts +144 -0
  56. package/src/embedding/onnx-transformers.test.ts +113 -0
  57. package/src/embedding/onnx-transformers.ts +141 -0
  58. package/src/embedding/select.test.ts +99 -0
  59. package/src/embedding/select.ts +92 -0
  60. package/src/embedding-worker.test.ts +300 -0
  61. package/src/embedding-worker.ts +226 -0
  62. package/src/mcp-http.ts +13 -1
  63. package/src/mcp-tools.ts +49 -14
  64. package/src/onboarding-seed.test.ts +64 -0
  65. package/src/routes.ts +173 -92
  66. package/src/routing.ts +17 -0
  67. package/src/scribe-discovery.test.ts +76 -1
  68. package/src/scribe-discovery.ts +28 -9
  69. package/src/semantic-search-routes.test.ts +161 -0
  70. package/src/server.ts +21 -2
  71. package/src/vault-embeddings-capability.test.ts +82 -0
  72. package/src/vault-store-embedding-wiring.test.ts +69 -0
  73. package/src/vault-store.ts +53 -2
  74. package/src/vault.test.ts +62 -13
@@ -41,6 +41,7 @@
41
41
  */
42
42
 
43
43
  import { Database } from "bun:sqlite";
44
+ import { timestampToMs } from "./cursor.js";
44
45
 
45
46
  // ---------------------------------------------------------------------------
46
47
  // Types
@@ -57,7 +58,7 @@ export interface SchemaField {
57
58
  * `type_mismatch` warning that previously fired on every integer-shaped
58
59
  * field because the validator had no `"integer"` case. See vault#310.
59
60
  */
60
- type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
61
+ type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference" | "date";
61
62
  enum?: string[];
62
63
  description?: string;
63
64
  /**
@@ -320,6 +321,82 @@ export function resolveNoteSchemas(
320
321
  return { effectiveTags, mergedFields, conflicts };
321
322
  }
322
323
 
324
+ /**
325
+ * Normalize `date`-typed field VALUES in `metadata` to canonical UTC ISO
326
+ * form (`Z`-suffixed) BEFORE persisting (vault#date-field-type — mixed-
327
+ * offset corruption, caught in review).
328
+ *
329
+ * The bug: every consumer of an indexed `date` field — `buildOperatorClause`
330
+ * (query-operators.ts), `date_filter` and `order_by` (notes.ts) — compares
331
+ * the generated `meta_<field>` column as raw TEXT. ISO-8601 strings sort
332
+ * correctly under a TEXT compare ONLY when every value shares the same
333
+ * offset representation. `timestampToMs` (cursor.ts) — the validator both
334
+ * `defaultMatchesType` and `valueMatchesType` reuse — correctly ACCEPTS an
335
+ * explicit `±HH:MM` offset (validation was never the gap), but a value like
336
+ * `"2026-07-16T10:00:00+02:00"` (= `08:00Z`) persisted VERBATIM sorts AFTER
337
+ * `"2026-07-16T09:00:00Z"` (= `09:00Z`) under a raw string compare, even
338
+ * though the actual instant is earlier — a mixed-offset vault silently gets
339
+ * wrong range-query/order_by/date_filter results. This is the exact bug
340
+ * class `updated_at` got a dedicated ms-mirror column for (vault#585/#586,
341
+ * see `notes.ts`'s `dateFilter` block) that never extended to user-declared
342
+ * `date` fields.
343
+ *
344
+ * The fix is normalize-on-write, not reject-on-write (matching the
345
+ * "paths are normalized on write" precedent — instant preserved,
346
+ * representation canonicalized) — rejecting offsets would defeat the
347
+ * type's own motivation (calendar integrations and other emitters commonly
348
+ * produce offset timestamps, not always `Z`). Only a FULL timestamp (has a
349
+ * time component) is rewritten, to `new Date(ms).toISOString()` — always
350
+ * UTC, millisecond precision, `Z`-suffixed. A bare `YYYY-MM-DD` value is
351
+ * left untouched: it has no offset to normalize, is already canonical, and
352
+ * prefix-sorts correctly against full timestamps sharing the same calendar
353
+ * day. A value that fails `timestampToMs` is left untouched too — that's a
354
+ * `type_mismatch` for `valueMatchesType` to catch, not this function's job.
355
+ *
356
+ * COPY-ON-WRITE, never mutates the input `metadata` object (vault#date-
357
+ * field-type review round 2) — `core/` is a published library; a direct
358
+ * embedder holding a reference to the object THEY passed in must never see
359
+ * it change out from under them. Returns the SAME reference when nothing
360
+ * needs rewriting (the common case — no unnecessary allocation on the
361
+ * no-op path) or `metadata` is undefined; returns a freshly shallow-copied
362
+ * object, with only the rewritten field(s) replaced, the first time a
363
+ * rewrite is actually needed. Every call site must use the RETURNED value
364
+ * (not assume its input was mutated) — see `store.createNote`/`updateNote`/
365
+ * `createNotes`/`createNoteRaw`, all of which reassign their local
366
+ * `opts`/`updates`/`input` binding from the return rather than relying on a
367
+ * side effect.
368
+ *
369
+ * Called from `store.createNote`/`updateNote`/`createNotes`/`createNoteRaw`
370
+ * — the lowest chokepoint every write path (MCP, REST, import) funnels
371
+ * through — BEFORE the row is persisted, so the value that's validated by
372
+ * `validateNote`/`valueMatchesType` upstream (offset-tolerant, so
373
+ * normalization can't newly fail a write) and the value that's written +
374
+ * echoed back on the response are the SAME normalized string.
375
+ */
376
+ export function normalizeDateFields(
377
+ resolved: ResolvedSchemas,
378
+ note: { tags?: string[]; metadata?: Record<string, unknown> },
379
+ ): Record<string, unknown> | undefined {
380
+ const metadata = note.metadata;
381
+ if (!metadata) return metadata;
382
+ const { mergedFields } = resolveNoteSchemas(resolved, note);
383
+ let copy: Record<string, unknown> | undefined;
384
+ for (const [fieldName, { spec }] of mergedFields) {
385
+ if (spec.type !== "date") continue;
386
+ const value = metadata[fieldName];
387
+ if (typeof value !== "string") continue;
388
+ // Bare `YYYY-MM-DD` (no time component) has no offset to normalize.
389
+ if (/^\d{4}-\d{2}-\d{2}$/.test(value)) continue;
390
+ const ms = timestampToMs(value);
391
+ if (ms === null) continue; // unparseable — valueMatchesType's job, not ours
392
+ const canonical = new Date(ms).toISOString();
393
+ if (canonical === value) continue;
394
+ if (!copy) copy = { ...metadata }; // lazy: only copy once a rewrite is due
395
+ copy[fieldName] = canonical;
396
+ }
397
+ return copy ?? metadata;
398
+ }
399
+
323
400
  function fieldSpecsEqual(a: SchemaField, b: SchemaField): boolean {
324
401
  if (a.type !== b.type) return false;
325
402
  if (!stringArraysEqual(a.enum, b.enum)) return false;
@@ -390,6 +467,13 @@ function valueMatchesType(value: unknown, type: SchemaField["type"], cardinality
390
467
  return Array.isArray(value) && value.every((item) => typeof item === "string");
391
468
  }
392
469
  return typeof value === "string";
470
+ // `date` validates like `string` — an ISO-8601 date (`YYYY-MM-DD`) or
471
+ // full RFC3339 timestamp. Reuses `cursor.ts`'s `timestampToMs` (the SAME
472
+ // UTC-correct parser `date_filter`'s `updated_at` bound uses), not a
473
+ // second date parser — see tag-schemas.ts's `VALID_FIELD_TYPES` doc
474
+ // comment.
475
+ case "date":
476
+ return typeof value === "string" && timestampToMs(value) !== null;
393
477
  }
394
478
  }
395
479
 
@@ -0,0 +1,178 @@
1
+ /**
2
+ * Migration v26 → v27: the `note_vectors` table (semantic search MVP —
3
+ * `SEMANTIC-MVP-PLAN.md`). Exercised against a REALISTIC pre-v27 fixture
4
+ * (a hand-built v26-shape DB, not a fresh vault — per the real-install-
5
+ * fixture discipline: fresh-DB-only testing misses the "existing vault
6
+ * upgrading" path, which is the one that matters in production).
7
+ */
8
+ import { describe, it, expect } from "bun:test";
9
+ import { Database } from "bun:sqlite";
10
+ import { initSchema, SCHEMA_VERSION } from "./schema.js";
11
+ import { encodeVector, normalize } from "./embedding/vector-codec.js";
12
+
13
+ /** A v26-shape vault: everything through vault#586, no `note_vectors` at all. */
14
+ function buildPreV27Vault(): Database {
15
+ const db = new Database(":memory:");
16
+ db.exec(`
17
+ CREATE TABLE notes (
18
+ id TEXT PRIMARY KEY,
19
+ content TEXT DEFAULT '',
20
+ path TEXT,
21
+ metadata TEXT DEFAULT '{}',
22
+ created_at TEXT NOT NULL,
23
+ updated_at TEXT,
24
+ extension TEXT NOT NULL DEFAULT 'md',
25
+ created_by TEXT,
26
+ created_via TEXT,
27
+ last_updated_by TEXT,
28
+ last_updated_via TEXT,
29
+ updated_at_ms INTEGER
30
+ );
31
+ CREATE TABLE tags (
32
+ name TEXT PRIMARY KEY,
33
+ description TEXT,
34
+ fields TEXT,
35
+ relationships TEXT,
36
+ parent_names TEXT,
37
+ created_at TEXT,
38
+ updated_at TEXT
39
+ );
40
+ CREATE TABLE note_tags (
41
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
42
+ tag_name TEXT NOT NULL REFERENCES tags(name),
43
+ PRIMARY KEY (note_id, tag_name)
44
+ );
45
+ CREATE TABLE attachments (
46
+ id TEXT PRIMARY KEY,
47
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
48
+ path TEXT NOT NULL,
49
+ mime_type TEXT NOT NULL,
50
+ metadata TEXT DEFAULT '{}',
51
+ created_at TEXT NOT NULL
52
+ );
53
+ CREATE TABLE links (
54
+ source_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
55
+ target_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
56
+ relationship TEXT NOT NULL,
57
+ metadata TEXT DEFAULT '{}',
58
+ created_at TEXT NOT NULL,
59
+ UNIQUE(source_id, target_id, relationship)
60
+ );
61
+ CREATE INDEX idx_notes_updated_ms ON notes(updated_at_ms, id);
62
+ -- A REAL v26 vault already went through migrateToV25's notes_fts
63
+ -- rebuild+repopulate long ago — this fixture must include that
64
+ -- already-populated state too, or a post-migration DELETE against
65
+ -- notes hits the notes_fts_delete trigger with an external-content FTS5
66
+ -- index that was never populated for these rows, which SQLite reports
67
+ -- as disk-image corruption (a fixture-fidelity issue, not a real one).
68
+ CREATE VIRTUAL TABLE notes_fts USING fts5(
69
+ path, content, content='notes', content_rowid='rowid', tokenize='porter unicode61'
70
+ );
71
+ CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
72
+ INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
73
+ END;
74
+ CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
75
+ INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
76
+ END;
77
+ CREATE TRIGGER notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
78
+ INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
79
+ INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
80
+ END;
81
+ CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
82
+ INSERT INTO schema_version (version, applied_at) VALUES (26, '2026-07-01T00:00:00.000Z');
83
+ `);
84
+ const insert = db.prepare(
85
+ "INSERT INTO notes (id, content, created_at, updated_at, updated_at_ms) VALUES (?, ?, ?, ?, ?)",
86
+ );
87
+ insert.run("note-1", "some real morning-pages content", "2026-06-01T00:00:00.000Z", "2026-06-01T00:00:00.000Z", Date.UTC(2026, 5, 1));
88
+ insert.run("note-2", "another pre-existing note", "2026-06-02T00:00:00.000Z", "2026-06-02T00:00:00.000Z", Date.UTC(2026, 5, 2));
89
+ return db;
90
+ }
91
+
92
+ function hasTable(db: Database, name: string): boolean {
93
+ return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
94
+ }
95
+
96
+ function hasIndex(db: Database, name: string): boolean {
97
+ return !!db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name=?").get(name);
98
+ }
99
+
100
+ describe("migrateToV27 — creates note_vectors on a real pre-v27 fixture", () => {
101
+ it("has no note_vectors table before migration; has one (+ its index) after", () => {
102
+ const db = buildPreV27Vault();
103
+ expect(hasTable(db, "note_vectors")).toBe(false);
104
+ initSchema(db);
105
+ expect(hasTable(db, "note_vectors")).toBe(true);
106
+ expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
107
+ expect(
108
+ (db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v,
109
+ ).toBe(SCHEMA_VERSION);
110
+ });
111
+
112
+ it("does NOT touch existing note rows — no data movement, unlike migrateToV26's backfill", () => {
113
+ const db = buildPreV27Vault();
114
+ const before = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
115
+ initSchema(db);
116
+ const after = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
117
+ expect(after).toEqual(before);
118
+ });
119
+
120
+ it("every pre-existing note starts with ZERO note_vectors rows — the implicit 'needs embedding' backfill signal", () => {
121
+ const db = buildPreV27Vault();
122
+ initSchema(db);
123
+ const count = (
124
+ db.prepare("SELECT COUNT(*) AS n FROM note_vectors").get() as { n: number }
125
+ ).n;
126
+ expect(count).toBe(0);
127
+ });
128
+
129
+ it("is idempotent — a second initSchema neither throws nor duplicates the table/index", () => {
130
+ const db = buildPreV27Vault();
131
+ initSchema(db);
132
+ expect(() => initSchema(db)).not.toThrow();
133
+ expect(hasTable(db, "note_vectors")).toBe(true);
134
+ });
135
+
136
+ it("note_vectors rows cascade-delete when their note is deleted (ON DELETE CASCADE)", () => {
137
+ const db = buildPreV27Vault();
138
+ initSchema(db);
139
+ const vec = encodeVector(normalize(new Float32Array([1, 2, 3, 4])));
140
+ db.prepare(
141
+ `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
142
+ VALUES (?, 0, ?, 4, 'test-model', 'deadbeef', ?)`,
143
+ ).run("note-1", vec, new Date().toISOString());
144
+ expect(
145
+ (db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
146
+ ).toBe(1);
147
+
148
+ db.prepare("DELETE FROM notes WHERE id = ?").run("note-1");
149
+
150
+ expect(
151
+ (db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
152
+ ).toBe(0);
153
+ // note-2 is unaffected.
154
+ expect(hasTable(db, "notes")).toBe(true);
155
+ expect(
156
+ (db.prepare("SELECT COUNT(*) AS n FROM notes WHERE id = ?").get("note-2") as { n: number }).n,
157
+ ).toBe(1);
158
+ });
159
+
160
+ it("PRIMARY KEY (note_id, chunk_ix) rejects a duplicate chunk row (upsert-by-replace is the caller's job)", () => {
161
+ const db = buildPreV27Vault();
162
+ initSchema(db);
163
+ const vec = encodeVector(normalize(new Float32Array([1, 0, 0, 0])));
164
+ const insert = db.prepare(
165
+ `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
166
+ VALUES (?, 0, ?, 4, 'test-model', 'hash-a', ?)`,
167
+ );
168
+ insert.run("note-1", vec, new Date().toISOString());
169
+ expect(() => insert.run("note-1", vec, new Date().toISOString())).toThrow();
170
+ });
171
+
172
+ it("a fresh vault (no prior schema_version row) gets note_vectors directly from SCHEMA_SQL", () => {
173
+ const db = new Database(":memory:");
174
+ initSchema(db);
175
+ expect(hasTable(db, "note_vectors")).toBe(true);
176
+ expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
177
+ });
178
+ });
@@ -5,7 +5,7 @@ import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
5
5
  import { transaction } from "./txn.js";
6
6
  import { timestampToMs } from "./cursor.js";
7
7
 
8
- export const SCHEMA_VERSION = 26;
8
+ export const SCHEMA_VERSION = 27;
9
9
 
10
10
  /**
11
11
  * Deterministic last-resort epoch for a note whose `updated_at` AND
@@ -112,6 +112,35 @@ CREATE TABLE IF NOT EXISTS links (
112
112
  UNIQUE(source_id, target_id, relationship)
113
113
  );
114
114
 
115
+ -- note_vectors (v27, semantic search MVP — EXPERIMENTAL): one row per
116
+ -- embedded CHUNK of a note (see core/src/embedding/chunker.ts). chunk_ix=0
117
+ -- is either the whole note (short notes, the degenerate/v1-equivalent
118
+ -- case) or the first section of a long, multi-section note.
119
+ --
120
+ -- vector is a Float32Array serialized to raw bytes (core/src/embedding/
121
+ -- vector-codec.ts), stored L2-NORMALIZED so ranking is a plain dot
122
+ -- product. model is recorded per-row so a provider/model change is
123
+ -- detectable (see core/src/embedding/staleness.ts) — a chunk_ix's row is
124
+ -- REPLACED wholesale on re-embed, never appended, so a vault only ever
125
+ -- carries vectors from ONE model at a time. content_hash is the freshness
126
+ -- gate: a create/update whose chunk text hashes the same as the stored
127
+ -- row is a no-op (no wasted embed on a metadata-only save).
128
+ -- ON DELETE CASCADE mirrors attachments/links above — a deleted note
129
+ -- drops its vectors for free.
130
+ CREATE TABLE IF NOT EXISTS note_vectors (
131
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
132
+ chunk_ix INTEGER NOT NULL DEFAULT 0,
133
+ vector BLOB NOT NULL,
134
+ dims INTEGER NOT NULL,
135
+ model TEXT NOT NULL,
136
+ content_hash TEXT NOT NULL,
137
+ embedded_at TEXT NOT NULL,
138
+ PRIMARY KEY (note_id, chunk_ix)
139
+ );
140
+ -- Backs the staleness diff (per-note row fetch) and the model-mismatch
141
+ -- sweep (a model <> ? scan) the backfill/drain use to find pending work.
142
+ CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash);
143
+
115
144
  -- tag_schemas (v6) was retired in v14; description + fields lifted onto the
116
145
  -- tags row directly. The CREATE TABLE was removed from SCHEMA_SQL after the
117
146
  -- v14 data migration drops the table; existing v6+ vaults pick up the
@@ -554,6 +583,12 @@ export function initSchema(db: Database): void {
554
583
  // `updated_at` string with a UTC-correct parse. See vault#586.
555
584
  migrateToV26(db);
556
585
 
586
+ // Migrate v26 → v27: create the `note_vectors` table (semantic search
587
+ // MVP, EXPERIMENTAL) — one row per embedded note chunk. No backfill loop:
588
+ // every pre-existing note simply has zero rows, which the staleness gate
589
+ // already reads as "needs embedding." See vault semantic-search MVP plan.
590
+ migrateToV27(db);
591
+
557
592
  // Rebuild any generated columns + indexes declared in indexed_fields.
558
593
  // No-op for a fresh vault; idempotent on existing vaults.
559
594
  rebuildIndexes(db);
@@ -1617,6 +1652,51 @@ function migrateToV26(db: Database): void {
1617
1652
  db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated_ms ON notes(updated_at_ms, id)");
1618
1653
  }
1619
1654
 
1655
+ /**
1656
+ * Migrate v26 → v27: the `note_vectors` table (semantic search MVP,
1657
+ * EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Created by SCHEMA_SQL's
1658
+ * `CREATE TABLE IF NOT EXISTS` above, so on a FRESH vault this is a no-op
1659
+ * confirmation hook (same pattern as `migrateToV7`/`V20`/`V21` — the table
1660
+ * is new, not a column added to existing rows). On an EXISTING vault
1661
+ * upgrading through this version, the `CREATE TABLE IF NOT EXISTS` +
1662
+ * `CREATE INDEX IF NOT EXISTS` calls create it for the first time.
1663
+ *
1664
+ * Deliberately NO backfill loop, and no embedding call of any kind — this
1665
+ * function never resolves an `EmbeddingProvider` or does network/CPU work.
1666
+ * "Backfill" for existing notes is implicit and free: every note that
1667
+ * predates this migration simply has ZERO rows in the (now-created, empty)
1668
+ * `note_vectors` table, which `core/src/embedding/staleness.ts:planStaleness`
1669
+ * already treats as "stale — needs embedding" (no existing row at a given
1670
+ * `chunk_ix` ⇒ stale). The async drain (self-host: the `onNote` hook +
1671
+ * its sweep; cloud: the DO-alarm drain) discovers this pending work by
1672
+ * scanning for notes lacking a fresh vector row — the exact same "queue is
1673
+ * implicit in the diff, not a separate table" property the `updated_at_ms`
1674
+ * self-heal above relies on. So this migration is intentionally ALL
1675
+ * schema, zero data movement — the whole point of "enqueue, never embed
1676
+ * inside the migration."
1677
+ *
1678
+ * Wrapped in a transaction for symmetry with `migrateToV25`/`V26` (a
1679
+ * multi-statement DDL block should commit or roll back as one unit), even
1680
+ * though there's no row data to lose here.
1681
+ */
1682
+ function migrateToV27(db: Database): void {
1683
+ transaction(db, () => {
1684
+ db.exec(`
1685
+ CREATE TABLE IF NOT EXISTS note_vectors (
1686
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
1687
+ chunk_ix INTEGER NOT NULL DEFAULT 0,
1688
+ vector BLOB NOT NULL,
1689
+ dims INTEGER NOT NULL,
1690
+ model TEXT NOT NULL,
1691
+ content_hash TEXT NOT NULL,
1692
+ embedded_at TEXT NOT NULL,
1693
+ PRIMARY KEY (note_id, chunk_ix)
1694
+ )
1695
+ `);
1696
+ db.exec("CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash)");
1697
+ });
1698
+ }
1699
+
1620
1700
  function hasTable(db: Database, name: string): boolean {
1621
1701
  const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
1622
1702
  return !!row;
@@ -132,7 +132,15 @@ describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
132
132
  );
133
133
  insert.run(
134
134
  "legacy-1",
135
- "a dedicated writeup mentioning propolis only once in passing text",
135
+ // Deliberately multi-line, with the "propolis" mention on a LATER
136
+ // line — this fixture isolates the PATH-weighted bm25 axis (below)
137
+ // from the separate content-title-boost axis (title axis, ratified
138
+ // 2026-07-17): that boost derives a note's "title" from its FIRST
139
+ // content line, and a single-line note IS its own title in full, so
140
+ // a one-line body mentioning "propolis" would (correctly, under the
141
+ // newer axis) count as a title match and confound this test's
142
+ // path-only signal. See `search-title-boost.test.ts` for that axis.
143
+ "Notes on foraging patterns\n\na dedicated writeup mentioning propolis only once in passing text",
136
144
  "beekeeping-notes",
137
145
  "2026-01-01T00:00:00.000Z",
138
146
  "2026-01-01T00:00:00.000Z",
@@ -4,6 +4,7 @@ import {
4
4
  escapeFtsToken,
5
5
  buildLiteralSearchQuery,
6
6
  isValidSearchMode,
7
+ extractLiteralBoostTerms,
7
8
  } from "./search-query.js";
8
9
 
9
10
  describe("escapeFtsToken", () => {
@@ -147,3 +148,44 @@ describe("isValidSearchMode / SEARCH_MODES", () => {
147
148
  expect(isValidSearchMode(["literal"])).toBe(false);
148
149
  });
149
150
  });
151
+
152
+ describe("extractLiteralBoostTerms (search title-boost input, title axis)", () => {
153
+ it("lowercases and whitespace-splits", () => {
154
+ expect(extractLiteralBoostTerms("Budget Review")).toEqual(["budget", "review"]);
155
+ });
156
+
157
+ it("collapses whitespace runs", () => {
158
+ expect(extractLiteralBoostTerms("widgets gadgets")).toEqual(["widgets", "gadgets"]);
159
+ });
160
+
161
+ it("trims leading/trailing whitespace", () => {
162
+ expect(extractLiteralBoostTerms(" widgets ")).toEqual(["widgets"]);
163
+ });
164
+
165
+ it("strips literal quote characters a caller typed, unlike buildLiteralSearchQuery", () => {
166
+ // buildLiteralSearchQuery treats a manually-typed `"` as ordinary
167
+ // content (escaped into the FTS phrase); the boost-term extractor
168
+ // strips it instead, since a title-string comparison shouldn't
169
+ // require the title to contain a literal quote mark.
170
+ expect(extractLiteralBoostTerms(`"eleven-day capping delay"`)).toEqual([
171
+ "eleven-day",
172
+ "capping",
173
+ "delay",
174
+ ]);
175
+ });
176
+
177
+ it("preserves punctuation that isn't a quote (hyphen, period, apostrophe)", () => {
178
+ expect(extractLiteralBoostTerms("eleven-day")).toEqual(["eleven-day"]);
179
+ expect(extractLiteralBoostTerms("18.6")).toEqual(["18.6"]);
180
+ expect(extractLiteralBoostTerms("didn't")).toEqual(["didn't"]);
181
+ });
182
+
183
+ it("returns an empty array for blank/whitespace-only input", () => {
184
+ expect(extractLiteralBoostTerms("")).toEqual([]);
185
+ expect(extractLiteralBoostTerms(" ")).toEqual([]);
186
+ });
187
+
188
+ it("sanitizes control characters as separators, matching buildLiteralSearchQuery", () => {
189
+ expect(extractLiteralBoostTerms("hello\x00world")).toEqual(["hello", "world"]);
190
+ });
191
+ });
@@ -153,3 +153,30 @@ export function buildLiteralSearchQuery(raw: string): LiteralSearchQuery {
153
153
  const tokens = cleaned.trim().split(/\s+/).filter(Boolean);
154
154
  return { query: tokens.map(escapeFtsToken).join(" "), isEmpty: false };
155
155
  }
156
+
157
+ /**
158
+ * Extract lowercase whitespace-delimited terms from raw literal-mode search
159
+ * text, for the search title-boost post-rank (title axis, ratified
160
+ * 2026-07-17) — NOT used to build the FTS5 MATCH query itself (that's
161
+ * `buildLiteralSearchQuery`, above). Sanitizes control characters the same
162
+ * way, plus strips any literal `"` a caller typed (kept as ordinary content
163
+ * by `buildLiteralSearchQuery`, but a display-title substring check
164
+ * shouldn't require the title to contain a literal quote mark).
165
+ *
166
+ * This is a heuristic re-rank signal, not a second index — it deliberately
167
+ * does not attempt FTS5-tokenizer parity (stemming, unicode normalization).
168
+ * It only needs to answer "does the note's first line plausibly contain the
169
+ * query," not reproduce FTS5's match semantics exactly. Restricted to
170
+ * literal-mode callers by convention (see `core/src/notes.ts` `searchNotes`)
171
+ * — `search_mode: "advanced"` text carries FTS5 boolean/column/prefix
172
+ * operators (`AND`, `path:`, `foo*`) that would leak into the term list as
173
+ * false "content" if tokenized the same naive way.
174
+ */
175
+ export function extractLiteralBoostTerms(raw: string): string[] {
176
+ const cleaned = raw.replace(CONTROL_CHARS, " ").toLowerCase();
177
+ return cleaned
178
+ .trim()
179
+ .split(/\s+/)
180
+ .map((t) => t.replace(/^"+|"+$/g, ""))
181
+ .filter(Boolean);
182
+ }
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Search title-boost (title axis, ratified 2026-07-17): literal-mode
3
+ * search results whose `displayTitle` (first non-empty content line, NOT
4
+ * `path`) contains every query term are post-ranked ahead of body-only
5
+ * matches. No-migration — an in-memory re-rank of the already-fetched
6
+ * page, not a `notes_fts` schema change (see `applySearchTitleBoost` /
7
+ * `boostTitleMatches` in `notes.ts`).
8
+ *
9
+ * This is a DIFFERENT axis from the existing `path`-weighted bm25 ranking
10
+ * covered in `search-fts-v25.test.ts` — a note's title here is its first
11
+ * CONTENT line, not its `path`.
12
+ */
13
+ import { describe, it, expect, beforeEach } from "bun:test";
14
+ import { Database } from "bun:sqlite";
15
+ import { SqliteStore } from "./store.js";
16
+
17
+ let store: SqliteStore;
18
+ let db: Database;
19
+
20
+ beforeEach(() => {
21
+ db = new Database(":memory:");
22
+ store = new SqliteStore(db);
23
+ });
24
+
25
+ describe("search title-boost", () => {
26
+ it("ranks a first-line (title) match above a body-only match", async () => {
27
+ // Body-only match: "budget" appears only deep in the body, first line
28
+ // is unrelated.
29
+ await store.createNote(
30
+ "Weekly Standup\nlots of unrelated notes here, but somewhere we discuss the budget in passing",
31
+ { path: "standup-notes" },
32
+ );
33
+ // First-line (title) match: "Budget" is the first line itself.
34
+ await store.createNote("Budget Review\nQ3 numbers", { path: "budget-review" });
35
+
36
+ const hits = await store.searchNotes("budget");
37
+ expect(hits.length).toBeGreaterThanOrEqual(2);
38
+ expect(hits[0].path).toBe("budget-review");
39
+ });
40
+
41
+ it("requires ALL query terms to be present in the first line to boost (a partial title match doesn't qualify)", async () => {
42
+ // "budget" is in the title but "review" is only in the body — a
43
+ // PARTIAL title match, which stays in the body-only tier.
44
+ await store.createNote("Budget notes\nsee the quarterly review for details", { path: "partial" });
45
+ // Both "budget" and "review" are in the title — a FULL title match.
46
+ await store.createNote("Budget Review\nQ3 numbers", { path: "full-match" });
47
+
48
+ const hits = await store.searchNotes("budget review");
49
+ expect(hits.map((n) => n.path)).toEqual(["full-match", "partial"]);
50
+ });
51
+
52
+ it("ties (both notes in the same tier) keep the existing relevance order", async () => {
53
+ // Two title-matching notes plus one body-only note. Comparing literal
54
+ // mode (boosted) against advanced mode with the same plain-word query
55
+ // (identical FTS5 syntax, so identical underlying bm25 order, but no
56
+ // boost pass) isolates the boost's effect: within the title-match
57
+ // tier, the boost must preserve whatever relative order the
58
+ // underlying relevance ranking already gave — a stable sort, not a
59
+ // fresh comparator that could reshuffle ties.
60
+ await store.createNote("Project Widget\nfirst, mentions widget again for relevance", { path: "w1" });
61
+ await store.createNote("Project Widget\nsecond", { path: "w2" });
62
+ await store.createNote("Unrelated\nbody-only mention of widget project", { path: "body-only" });
63
+
64
+ const boosted = await store.searchNotes("widget project");
65
+ const rawOrder = await store.searchNotes("widget project", { mode: "advanced" });
66
+
67
+ const titleTierBoosted = boosted.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
68
+ const titleTierRaw = rawOrder.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
69
+ expect(titleTierBoosted).toEqual(titleTierRaw);
70
+ // The title-match tier precedes the body-only note in the boosted result.
71
+ expect(boosted.map((n) => n.path).indexOf("body-only")).toBeGreaterThan(
72
+ Math.max(...["w1", "w2"].map((p) => boosted.map((n) => n.path).indexOf(p))),
73
+ );
74
+ });
75
+
76
+ it("does not disturb ordering when an explicit sort is requested", async () => {
77
+ await store.createNote("Zebra body-only mention of gadget", { path: "z" });
78
+ await store.createNote("Gadget Launch\nfirst line title match", { path: "a" });
79
+
80
+ const asc = await store.searchNotes("gadget", { sort: "asc" });
81
+ // created_at ASC means "z" (created first) comes before "a" — the
82
+ // title-boost must NOT override an explicit sort.
83
+ expect(asc.map((n) => n.path)).toEqual(["z", "a"]);
84
+ });
85
+
86
+ it("does not apply the boost under search_mode: advanced (raw FTS5 syntax, not naively tokenizable)", async () => {
87
+ await store.createNote("Body only mentions gadget in passing text here", { path: "body-only" });
88
+ await store.createNote("Gadget\nfirst line title match", { path: "title-match" });
89
+
90
+ const advanced = await store.searchNotes("gadget", { mode: "advanced" });
91
+ const literal = await store.searchNotes("gadget");
92
+
93
+ // Literal mode boosts the title match to the front.
94
+ expect(literal[0].path).toBe("title-match");
95
+ // Advanced mode is unaffected by the boost — order comes from bm25
96
+ // relevance alone (title-match still likely ranks first via the
97
+ // EXISTING path-weighted bm25, but that's a different mechanism; the
98
+ // test here just confirms advanced mode doesn't throw and returns
99
+ // both notes, not that ordering matches/mismatches literal mode).
100
+ expect(advanced.map((n) => n.path).sort()).toEqual(["body-only", "title-match"]);
101
+ });
102
+
103
+ it("a query with no results is unaffected (empty array in, empty array out)", async () => {
104
+ const hits = await store.searchNotes("nonexistent_term_xyz");
105
+ expect(hits).toEqual([]);
106
+ });
107
+
108
+ it("boosts on the post-frontmatter title line, not the `---` delimiter (shares computeDisplayTitle's frontmatter skip)", async () => {
109
+ // Direct-create misuse path: raw frontmatter-bearing content. Without
110
+ // the frontmatter skip, the derived "title" would be the literal
111
+ // string "---", which never matches any query term, so this note
112
+ // would wrongly land in the body-only tier.
113
+ await store.createNote(
114
+ "---\ntitle: irrelevant\n---\n# Budget Review\nQ3 numbers",
115
+ { path: "frontmatter-led" },
116
+ );
117
+ await store.createNote(
118
+ "Weekly Standup\nsomewhere we discuss the budget in passing",
119
+ { path: "body-only" },
120
+ );
121
+
122
+ const hits = await store.searchNotes("budget review");
123
+ expect(hits[0].path).toBe("frontmatter-led");
124
+ });
125
+ });