@openparachute/vault 0.7.3-rc.1 → 0.7.3-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +5 -3
  2. package/core/src/attachment/policy.test.ts +66 -0
  3. package/core/src/attachment/policy.ts +131 -0
  4. package/core/src/attachment/tickets.test.ts +45 -0
  5. package/core/src/attachment/tickets.ts +117 -0
  6. package/core/src/attachment-tickets-tool.test.ts +286 -0
  7. package/core/src/conformance.ts +2 -1
  8. package/core/src/contract-typed-index.test.ts +4 -3
  9. package/core/src/core.test.ts +521 -4
  10. package/core/src/display-title.test.ts +190 -0
  11. package/core/src/embedding/chunker.test.ts +97 -0
  12. package/core/src/embedding/chunker.ts +180 -0
  13. package/core/src/embedding/provider.ts +108 -0
  14. package/core/src/embedding/staleness.test.ts +87 -0
  15. package/core/src/embedding/staleness.ts +83 -0
  16. package/core/src/embedding/vector-codec.test.ts +99 -0
  17. package/core/src/embedding/vector-codec.ts +60 -0
  18. package/core/src/embedding/vectors.test.ts +163 -0
  19. package/core/src/embedding/vectors.ts +135 -0
  20. package/core/src/expand.ts +11 -3
  21. package/core/src/indexed-fields.test.ts +9 -3
  22. package/core/src/indexed-fields.ts +9 -1
  23. package/core/src/lede.test.ts +96 -0
  24. package/core/src/mcp-semantic-search.test.ts +160 -0
  25. package/core/src/mcp.ts +381 -10
  26. package/core/src/notes.semantic-search.test.ts +131 -0
  27. package/core/src/notes.ts +313 -6
  28. package/core/src/query-warnings.ts +20 -0
  29. package/core/src/schema-defaults.ts +85 -1
  30. package/core/src/schema-v27-note-vectors.test.ts +178 -0
  31. package/core/src/schema.ts +81 -1
  32. package/core/src/search-fts-v25.test.ts +9 -1
  33. package/core/src/search-query.test.ts +42 -0
  34. package/core/src/search-query.ts +27 -0
  35. package/core/src/search-title-boost.test.ts +125 -0
  36. package/core/src/seed-packs.test.ts +117 -1
  37. package/core/src/seed-packs.ts +217 -1
  38. package/core/src/store.semantic-search.test.ts +236 -0
  39. package/core/src/store.ts +162 -5
  40. package/core/src/tag-schemas.ts +27 -12
  41. package/core/src/types.ts +55 -1
  42. package/core/src/vault-projection.ts +48 -0
  43. package/package.json +6 -1
  44. package/src/add-pack.test.ts +32 -0
  45. package/src/attachment-tickets.test.ts +350 -0
  46. package/src/attachment-tickets.ts +264 -0
  47. package/src/cli.ts +5 -1
  48. package/src/contract-errors.test.ts +2 -1
  49. package/src/embedding/capability.test.ts +33 -0
  50. package/src/embedding/capability.ts +34 -0
  51. package/src/embedding/external-api.test.ts +154 -0
  52. package/src/embedding/external-api.ts +144 -0
  53. package/src/embedding/onnx-transformers.test.ts +113 -0
  54. package/src/embedding/onnx-transformers.ts +141 -0
  55. package/src/embedding/select.test.ts +99 -0
  56. package/src/embedding/select.ts +92 -0
  57. package/src/embedding-worker.test.ts +300 -0
  58. package/src/embedding-worker.ts +226 -0
  59. package/src/mcp-http.ts +13 -1
  60. package/src/mcp-tools.ts +49 -14
  61. package/src/onboarding-seed.test.ts +64 -0
  62. package/src/routes.ts +146 -92
  63. package/src/routing.ts +17 -0
  64. package/src/semantic-search-routes.test.ts +161 -0
  65. package/src/server.ts +21 -2
  66. package/src/vault-embeddings-capability.test.ts +82 -0
  67. package/src/vault-store-embedding-wiring.test.ts +69 -0
  68. package/src/vault-store.ts +53 -2
  69. package/src/vault.test.ts +35 -11
@@ -0,0 +1,178 @@
1
+ /**
2
+ * Migration v26 → v27: the `note_vectors` table (semantic search MVP —
3
+ * `SEMANTIC-MVP-PLAN.md`). Exercised against a REALISTIC pre-v27 fixture
4
+ * (a hand-built v26-shape DB, not a fresh vault — per the real-install-
5
+ * fixture discipline: fresh-DB-only testing misses the "existing vault
6
+ * upgrading" path, which is the one that matters in production).
7
+ */
8
+ import { describe, it, expect } from "bun:test";
9
+ import { Database } from "bun:sqlite";
10
+ import { initSchema, SCHEMA_VERSION } from "./schema.js";
11
+ import { encodeVector, normalize } from "./embedding/vector-codec.js";
12
+
13
+ /** A v26-shape vault: everything through vault#586, no `note_vectors` at all. */
14
+ function buildPreV27Vault(): Database {
15
+ const db = new Database(":memory:");
16
+ db.exec(`
17
+ CREATE TABLE notes (
18
+ id TEXT PRIMARY KEY,
19
+ content TEXT DEFAULT '',
20
+ path TEXT,
21
+ metadata TEXT DEFAULT '{}',
22
+ created_at TEXT NOT NULL,
23
+ updated_at TEXT,
24
+ extension TEXT NOT NULL DEFAULT 'md',
25
+ created_by TEXT,
26
+ created_via TEXT,
27
+ last_updated_by TEXT,
28
+ last_updated_via TEXT,
29
+ updated_at_ms INTEGER
30
+ );
31
+ CREATE TABLE tags (
32
+ name TEXT PRIMARY KEY,
33
+ description TEXT,
34
+ fields TEXT,
35
+ relationships TEXT,
36
+ parent_names TEXT,
37
+ created_at TEXT,
38
+ updated_at TEXT
39
+ );
40
+ CREATE TABLE note_tags (
41
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
42
+ tag_name TEXT NOT NULL REFERENCES tags(name),
43
+ PRIMARY KEY (note_id, tag_name)
44
+ );
45
+ CREATE TABLE attachments (
46
+ id TEXT PRIMARY KEY,
47
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
48
+ path TEXT NOT NULL,
49
+ mime_type TEXT NOT NULL,
50
+ metadata TEXT DEFAULT '{}',
51
+ created_at TEXT NOT NULL
52
+ );
53
+ CREATE TABLE links (
54
+ source_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
55
+ target_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
56
+ relationship TEXT NOT NULL,
57
+ metadata TEXT DEFAULT '{}',
58
+ created_at TEXT NOT NULL,
59
+ UNIQUE(source_id, target_id, relationship)
60
+ );
61
+ CREATE INDEX idx_notes_updated_ms ON notes(updated_at_ms, id);
62
+ -- A REAL v26 vault already went through migrateToV25's notes_fts
63
+ -- rebuild+repopulate long ago — this fixture must include that
64
+ -- already-populated state too, or a post-migration DELETE against
65
+ -- notes hits the notes_fts_delete trigger with an external-content FTS5
66
+ -- index that was never populated for these rows, which SQLite reports
67
+ -- as disk-image corruption (a fixture-fidelity issue, not a real one).
68
+ CREATE VIRTUAL TABLE notes_fts USING fts5(
69
+ path, content, content='notes', content_rowid='rowid', tokenize='porter unicode61'
70
+ );
71
+ CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
72
+ INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
73
+ END;
74
+ CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
75
+ INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
76
+ END;
77
+ CREATE TRIGGER notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
78
+ INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
79
+ INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
80
+ END;
81
+ CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
82
+ INSERT INTO schema_version (version, applied_at) VALUES (26, '2026-07-01T00:00:00.000Z');
83
+ `);
84
+ const insert = db.prepare(
85
+ "INSERT INTO notes (id, content, created_at, updated_at, updated_at_ms) VALUES (?, ?, ?, ?, ?)",
86
+ );
87
+ insert.run("note-1", "some real morning-pages content", "2026-06-01T00:00:00.000Z", "2026-06-01T00:00:00.000Z", Date.UTC(2026, 5, 1));
88
+ insert.run("note-2", "another pre-existing note", "2026-06-02T00:00:00.000Z", "2026-06-02T00:00:00.000Z", Date.UTC(2026, 5, 2));
89
+ return db;
90
+ }
91
+
92
+ function hasTable(db: Database, name: string): boolean {
93
+ return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
94
+ }
95
+
96
+ function hasIndex(db: Database, name: string): boolean {
97
+ return !!db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name=?").get(name);
98
+ }
99
+
100
+ describe("migrateToV27 — creates note_vectors on a real pre-v27 fixture", () => {
101
+ it("has no note_vectors table before migration; has one (+ its index) after", () => {
102
+ const db = buildPreV27Vault();
103
+ expect(hasTable(db, "note_vectors")).toBe(false);
104
+ initSchema(db);
105
+ expect(hasTable(db, "note_vectors")).toBe(true);
106
+ expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
107
+ expect(
108
+ (db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v,
109
+ ).toBe(SCHEMA_VERSION);
110
+ });
111
+
112
+ it("does NOT touch existing note rows — no data movement, unlike migrateToV26's backfill", () => {
113
+ const db = buildPreV27Vault();
114
+ const before = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
115
+ initSchema(db);
116
+ const after = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
117
+ expect(after).toEqual(before);
118
+ });
119
+
120
+ it("every pre-existing note starts with ZERO note_vectors rows — the implicit 'needs embedding' backfill signal", () => {
121
+ const db = buildPreV27Vault();
122
+ initSchema(db);
123
+ const count = (
124
+ db.prepare("SELECT COUNT(*) AS n FROM note_vectors").get() as { n: number }
125
+ ).n;
126
+ expect(count).toBe(0);
127
+ });
128
+
129
+ it("is idempotent — a second initSchema neither throws nor duplicates the table/index", () => {
130
+ const db = buildPreV27Vault();
131
+ initSchema(db);
132
+ expect(() => initSchema(db)).not.toThrow();
133
+ expect(hasTable(db, "note_vectors")).toBe(true);
134
+ });
135
+
136
+ it("note_vectors rows cascade-delete when their note is deleted (ON DELETE CASCADE)", () => {
137
+ const db = buildPreV27Vault();
138
+ initSchema(db);
139
+ const vec = encodeVector(normalize(new Float32Array([1, 2, 3, 4])));
140
+ db.prepare(
141
+ `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
142
+ VALUES (?, 0, ?, 4, 'test-model', 'deadbeef', ?)`,
143
+ ).run("note-1", vec, new Date().toISOString());
144
+ expect(
145
+ (db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
146
+ ).toBe(1);
147
+
148
+ db.prepare("DELETE FROM notes WHERE id = ?").run("note-1");
149
+
150
+ expect(
151
+ (db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
152
+ ).toBe(0);
153
+ // note-2 is unaffected.
154
+ expect(hasTable(db, "notes")).toBe(true);
155
+ expect(
156
+ (db.prepare("SELECT COUNT(*) AS n FROM notes WHERE id = ?").get("note-2") as { n: number }).n,
157
+ ).toBe(1);
158
+ });
159
+
160
+ it("PRIMARY KEY (note_id, chunk_ix) rejects a duplicate chunk row (upsert-by-replace is the caller's job)", () => {
161
+ const db = buildPreV27Vault();
162
+ initSchema(db);
163
+ const vec = encodeVector(normalize(new Float32Array([1, 0, 0, 0])));
164
+ const insert = db.prepare(
165
+ `INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
166
+ VALUES (?, 0, ?, 4, 'test-model', 'hash-a', ?)`,
167
+ );
168
+ insert.run("note-1", vec, new Date().toISOString());
169
+ expect(() => insert.run("note-1", vec, new Date().toISOString())).toThrow();
170
+ });
171
+
172
+ it("a fresh vault (no prior schema_version row) gets note_vectors directly from SCHEMA_SQL", () => {
173
+ const db = new Database(":memory:");
174
+ initSchema(db);
175
+ expect(hasTable(db, "note_vectors")).toBe(true);
176
+ expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
177
+ });
178
+ });
@@ -5,7 +5,7 @@ import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
5
5
  import { transaction } from "./txn.js";
6
6
  import { timestampToMs } from "./cursor.js";
7
7
 
8
- export const SCHEMA_VERSION = 26;
8
+ export const SCHEMA_VERSION = 27;
9
9
 
10
10
  /**
11
11
  * Deterministic last-resort epoch for a note whose `updated_at` AND
@@ -112,6 +112,35 @@ CREATE TABLE IF NOT EXISTS links (
112
112
  UNIQUE(source_id, target_id, relationship)
113
113
  );
114
114
 
115
+ -- note_vectors (v27, semantic search MVP — EXPERIMENTAL): one row per
116
+ -- embedded CHUNK of a note (see core/src/embedding/chunker.ts). chunk_ix=0
117
+ -- is either the whole note (short notes, the degenerate/v1-equivalent
118
+ -- case) or the first section of a long, multi-section note.
119
+ --
120
+ -- vector is a Float32Array serialized to raw bytes (core/src/embedding/
121
+ -- vector-codec.ts), stored L2-NORMALIZED so ranking is a plain dot
122
+ -- product. model is recorded per-row so a provider/model change is
123
+ -- detectable (see core/src/embedding/staleness.ts) — a chunk_ix's row is
124
+ -- REPLACED wholesale on re-embed, never appended, so a vault only ever
125
+ -- carries vectors from ONE model at a time. content_hash is the freshness
126
+ -- gate: a create/update whose chunk text hashes the same as the stored
127
+ -- row is a no-op (no wasted embed on a metadata-only save).
128
+ -- ON DELETE CASCADE mirrors attachments/links above — a deleted note
129
+ -- drops its vectors for free.
130
+ CREATE TABLE IF NOT EXISTS note_vectors (
131
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
132
+ chunk_ix INTEGER NOT NULL DEFAULT 0,
133
+ vector BLOB NOT NULL,
134
+ dims INTEGER NOT NULL,
135
+ model TEXT NOT NULL,
136
+ content_hash TEXT NOT NULL,
137
+ embedded_at TEXT NOT NULL,
138
+ PRIMARY KEY (note_id, chunk_ix)
139
+ );
140
+ -- Backs the staleness diff (per-note row fetch) and the model-mismatch
141
+ -- sweep (a model <> ? scan) the backfill/drain use to find pending work.
142
+ CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash);
143
+
115
144
  -- tag_schemas (v6) was retired in v14; description + fields lifted onto the
116
145
  -- tags row directly. The CREATE TABLE was removed from SCHEMA_SQL after the
117
146
  -- v14 data migration drops the table; existing v6+ vaults pick up the
@@ -554,6 +583,12 @@ export function initSchema(db: Database): void {
554
583
  // `updated_at` string with a UTC-correct parse. See vault#586.
555
584
  migrateToV26(db);
556
585
 
586
+ // Migrate v26 → v27: create the `note_vectors` table (semantic search
587
+ // MVP, EXPERIMENTAL) — one row per embedded note chunk. No backfill loop:
588
+ // every pre-existing note simply has zero rows, which the staleness gate
589
+ // already reads as "needs embedding." See vault semantic-search MVP plan.
590
+ migrateToV27(db);
591
+
557
592
  // Rebuild any generated columns + indexes declared in indexed_fields.
558
593
  // No-op for a fresh vault; idempotent on existing vaults.
559
594
  rebuildIndexes(db);
@@ -1617,6 +1652,51 @@ function migrateToV26(db: Database): void {
1617
1652
  db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated_ms ON notes(updated_at_ms, id)");
1618
1653
  }
1619
1654
 
1655
+ /**
1656
+ * Migrate v26 → v27: the `note_vectors` table (semantic search MVP,
1657
+ * EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Created by SCHEMA_SQL's
1658
+ * `CREATE TABLE IF NOT EXISTS` above, so on a FRESH vault this is a no-op
1659
+ * confirmation hook (same pattern as `migrateToV7`/`V20`/`V21` — the table
1660
+ * is new, not a column added to existing rows). On an EXISTING vault
1661
+ * upgrading through this version, the `CREATE TABLE IF NOT EXISTS` +
1662
+ * `CREATE INDEX IF NOT EXISTS` calls create it for the first time.
1663
+ *
1664
+ * Deliberately NO backfill loop, and no embedding call of any kind — this
1665
+ * function never resolves an `EmbeddingProvider` or does network/CPU work.
1666
+ * "Backfill" for existing notes is implicit and free: every note that
1667
+ * predates this migration simply has ZERO rows in the (now-created, empty)
1668
+ * `note_vectors` table, which `core/src/embedding/staleness.ts:planStaleness`
1669
+ * already treats as "stale — needs embedding" (no existing row at a given
1670
+ * `chunk_ix` ⇒ stale). The async drain (self-host: the `onNote` hook +
1671
+ * its sweep; cloud: the DO-alarm drain) discovers this pending work by
1672
+ * scanning for notes lacking a fresh vector row — the exact same "queue is
1673
+ * implicit in the diff, not a separate table" property the `updated_at_ms`
1674
+ * self-heal above relies on. So this migration is intentionally ALL
1675
+ * schema, zero data movement — the whole point of "enqueue, never embed
1676
+ * inside the migration."
1677
+ *
1678
+ * Wrapped in a transaction for symmetry with `migrateToV25`/`V26` (a
1679
+ * multi-statement DDL block should commit or roll back as one unit), even
1680
+ * though there's no row data to lose here.
1681
+ */
1682
+ function migrateToV27(db: Database): void {
1683
+ transaction(db, () => {
1684
+ db.exec(`
1685
+ CREATE TABLE IF NOT EXISTS note_vectors (
1686
+ note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
1687
+ chunk_ix INTEGER NOT NULL DEFAULT 0,
1688
+ vector BLOB NOT NULL,
1689
+ dims INTEGER NOT NULL,
1690
+ model TEXT NOT NULL,
1691
+ content_hash TEXT NOT NULL,
1692
+ embedded_at TEXT NOT NULL,
1693
+ PRIMARY KEY (note_id, chunk_ix)
1694
+ )
1695
+ `);
1696
+ db.exec("CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash)");
1697
+ });
1698
+ }
1699
+
1620
1700
  function hasTable(db: Database, name: string): boolean {
1621
1701
  const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
1622
1702
  return !!row;
@@ -132,7 +132,15 @@ describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
132
132
  );
133
133
  insert.run(
134
134
  "legacy-1",
135
- "a dedicated writeup mentioning propolis only once in passing text",
135
+ // Deliberately multi-line, with the "propolis" mention on a LATER
136
+ // line — this fixture isolates the PATH-weighted bm25 axis (below)
137
+ // from the separate content-title-boost axis (title axis, ratified
138
+ // 2026-07-17): that boost derives a note's "title" from its FIRST
139
+ // content line, and a single-line note IS its own title in full, so
140
+ // a one-line body mentioning "propolis" would (correctly, under the
141
+ // newer axis) count as a title match and confound this test's
142
+ // path-only signal. See `search-title-boost.test.ts` for that axis.
143
+ "Notes on foraging patterns\n\na dedicated writeup mentioning propolis only once in passing text",
136
144
  "beekeeping-notes",
137
145
  "2026-01-01T00:00:00.000Z",
138
146
  "2026-01-01T00:00:00.000Z",
@@ -4,6 +4,7 @@ import {
4
4
  escapeFtsToken,
5
5
  buildLiteralSearchQuery,
6
6
  isValidSearchMode,
7
+ extractLiteralBoostTerms,
7
8
  } from "./search-query.js";
8
9
 
9
10
  describe("escapeFtsToken", () => {
@@ -147,3 +148,44 @@ describe("isValidSearchMode / SEARCH_MODES", () => {
147
148
  expect(isValidSearchMode(["literal"])).toBe(false);
148
149
  });
149
150
  });
151
+
152
+ describe("extractLiteralBoostTerms (search title-boost input, title axis)", () => {
153
+ it("lowercases and whitespace-splits", () => {
154
+ expect(extractLiteralBoostTerms("Budget Review")).toEqual(["budget", "review"]);
155
+ });
156
+
157
+ it("collapses whitespace runs", () => {
158
+ expect(extractLiteralBoostTerms("widgets gadgets")).toEqual(["widgets", "gadgets"]);
159
+ });
160
+
161
+ it("trims leading/trailing whitespace", () => {
162
+ expect(extractLiteralBoostTerms(" widgets ")).toEqual(["widgets"]);
163
+ });
164
+
165
+ it("strips literal quote characters a caller typed, unlike buildLiteralSearchQuery", () => {
166
+ // buildLiteralSearchQuery treats a manually-typed `"` as ordinary
167
+ // content (escaped into the FTS phrase); the boost-term extractor
168
+ // strips it instead, since a title-string comparison shouldn't
169
+ // require the title to contain a literal quote mark.
170
+ expect(extractLiteralBoostTerms(`"eleven-day capping delay"`)).toEqual([
171
+ "eleven-day",
172
+ "capping",
173
+ "delay",
174
+ ]);
175
+ });
176
+
177
+ it("preserves punctuation that isn't a quote (hyphen, period, apostrophe)", () => {
178
+ expect(extractLiteralBoostTerms("eleven-day")).toEqual(["eleven-day"]);
179
+ expect(extractLiteralBoostTerms("18.6")).toEqual(["18.6"]);
180
+ expect(extractLiteralBoostTerms("didn't")).toEqual(["didn't"]);
181
+ });
182
+
183
+ it("returns an empty array for blank/whitespace-only input", () => {
184
+ expect(extractLiteralBoostTerms("")).toEqual([]);
185
+ expect(extractLiteralBoostTerms(" ")).toEqual([]);
186
+ });
187
+
188
+ it("sanitizes control characters as separators, matching buildLiteralSearchQuery", () => {
189
+ expect(extractLiteralBoostTerms("hello\x00world")).toEqual(["hello", "world"]);
190
+ });
191
+ });
@@ -153,3 +153,30 @@ export function buildLiteralSearchQuery(raw: string): LiteralSearchQuery {
153
153
  const tokens = cleaned.trim().split(/\s+/).filter(Boolean);
154
154
  return { query: tokens.map(escapeFtsToken).join(" "), isEmpty: false };
155
155
  }
156
+
157
+ /**
158
+ * Extract lowercase whitespace-delimited terms from raw literal-mode search
159
+ * text, for the search title-boost post-rank (title axis, ratified
160
+ * 2026-07-17) — NOT used to build the FTS5 MATCH query itself (that's
161
+ * `buildLiteralSearchQuery`, above). Sanitizes control characters the same
162
+ * way, plus strips any literal `"` a caller typed (kept as ordinary content
163
+ * by `buildLiteralSearchQuery`, but a display-title substring check
164
+ * shouldn't require the title to contain a literal quote mark).
165
+ *
166
+ * This is a heuristic re-rank signal, not a second index — it deliberately
167
+ * does not attempt FTS5-tokenizer parity (stemming, unicode normalization).
168
+ * It only needs to answer "does the note's first line plausibly contain the
169
+ * query," not reproduce FTS5's match semantics exactly. Restricted to
170
+ * literal-mode callers by convention (see `core/src/notes.ts` `searchNotes`)
171
+ * — `search_mode: "advanced"` text carries FTS5 boolean/column/prefix
172
+ * operators (`AND`, `path:`, `foo*`) that would leak into the term list as
173
+ * false "content" if tokenized the same naive way.
174
+ */
175
+ export function extractLiteralBoostTerms(raw: string): string[] {
176
+ const cleaned = raw.replace(CONTROL_CHARS, " ").toLowerCase();
177
+ return cleaned
178
+ .trim()
179
+ .split(/\s+/)
180
+ .map((t) => t.replace(/^"+|"+$/g, ""))
181
+ .filter(Boolean);
182
+ }
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Search title-boost (title axis, ratified 2026-07-17): literal-mode
3
+ * search results whose `displayTitle` (first non-empty content line, NOT
4
+ * `path`) contains every query term are post-ranked ahead of body-only
5
+ * matches. No-migration — an in-memory re-rank of the already-fetched
6
+ * page, not a `notes_fts` schema change (see `applySearchTitleBoost` /
7
+ * `boostTitleMatches` in `notes.ts`).
8
+ *
9
+ * This is a DIFFERENT axis from the existing `path`-weighted bm25 ranking
10
+ * covered in `search-fts-v25.test.ts` — a note's title here is its first
11
+ * CONTENT line, not its `path`.
12
+ */
13
+ import { describe, it, expect, beforeEach } from "bun:test";
14
+ import { Database } from "bun:sqlite";
15
+ import { SqliteStore } from "./store.js";
16
+
17
+ let store: SqliteStore;
18
+ let db: Database;
19
+
20
+ beforeEach(() => {
21
+ db = new Database(":memory:");
22
+ store = new SqliteStore(db);
23
+ });
24
+
25
+ describe("search title-boost", () => {
26
+ it("ranks a first-line (title) match above a body-only match", async () => {
27
+ // Body-only match: "budget" appears only deep in the body, first line
28
+ // is unrelated.
29
+ await store.createNote(
30
+ "Weekly Standup\nlots of unrelated notes here, but somewhere we discuss the budget in passing",
31
+ { path: "standup-notes" },
32
+ );
33
+ // First-line (title) match: "Budget" is the first line itself.
34
+ await store.createNote("Budget Review\nQ3 numbers", { path: "budget-review" });
35
+
36
+ const hits = await store.searchNotes("budget");
37
+ expect(hits.length).toBeGreaterThanOrEqual(2);
38
+ expect(hits[0].path).toBe("budget-review");
39
+ });
40
+
41
+ it("requires ALL query terms to be present in the first line to boost (a partial title match doesn't qualify)", async () => {
42
+ // "budget" is in the title but "review" is only in the body — a
43
+ // PARTIAL title match, which stays in the body-only tier.
44
+ await store.createNote("Budget notes\nsee the quarterly review for details", { path: "partial" });
45
+ // Both "budget" and "review" are in the title — a FULL title match.
46
+ await store.createNote("Budget Review\nQ3 numbers", { path: "full-match" });
47
+
48
+ const hits = await store.searchNotes("budget review");
49
+ expect(hits.map((n) => n.path)).toEqual(["full-match", "partial"]);
50
+ });
51
+
52
+ it("ties (both notes in the same tier) keep the existing relevance order", async () => {
53
+ // Two title-matching notes plus one body-only note. Comparing literal
54
+ // mode (boosted) against advanced mode with the same plain-word query
55
+ // (identical FTS5 syntax, so identical underlying bm25 order, but no
56
+ // boost pass) isolates the boost's effect: within the title-match
57
+ // tier, the boost must preserve whatever relative order the
58
+ // underlying relevance ranking already gave — a stable sort, not a
59
+ // fresh comparator that could reshuffle ties.
60
+ await store.createNote("Project Widget\nfirst, mentions widget again for relevance", { path: "w1" });
61
+ await store.createNote("Project Widget\nsecond", { path: "w2" });
62
+ await store.createNote("Unrelated\nbody-only mention of widget project", { path: "body-only" });
63
+
64
+ const boosted = await store.searchNotes("widget project");
65
+ const rawOrder = await store.searchNotes("widget project", { mode: "advanced" });
66
+
67
+ const titleTierBoosted = boosted.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
68
+ const titleTierRaw = rawOrder.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
69
+ expect(titleTierBoosted).toEqual(titleTierRaw);
70
+ // The title-match tier precedes the body-only note in the boosted result.
71
+ expect(boosted.map((n) => n.path).indexOf("body-only")).toBeGreaterThan(
72
+ Math.max(...["w1", "w2"].map((p) => boosted.map((n) => n.path).indexOf(p))),
73
+ );
74
+ });
75
+
76
+ it("does not disturb ordering when an explicit sort is requested", async () => {
77
+ await store.createNote("Zebra body-only mention of gadget", { path: "z" });
78
+ await store.createNote("Gadget Launch\nfirst line title match", { path: "a" });
79
+
80
+ const asc = await store.searchNotes("gadget", { sort: "asc" });
81
+ // created_at ASC means "z" (created first) comes before "a" — the
82
+ // title-boost must NOT override an explicit sort.
83
+ expect(asc.map((n) => n.path)).toEqual(["z", "a"]);
84
+ });
85
+
86
+ it("does not apply the boost under search_mode: advanced (raw FTS5 syntax, not naively tokenizable)", async () => {
87
+ await store.createNote("Body only mentions gadget in passing text here", { path: "body-only" });
88
+ await store.createNote("Gadget\nfirst line title match", { path: "title-match" });
89
+
90
+ const advanced = await store.searchNotes("gadget", { mode: "advanced" });
91
+ const literal = await store.searchNotes("gadget");
92
+
93
+ // Literal mode boosts the title match to the front.
94
+ expect(literal[0].path).toBe("title-match");
95
+ // Advanced mode is unaffected by the boost — order comes from bm25
96
+ // relevance alone (title-match still likely ranks first via the
97
+ // EXISTING path-weighted bm25, but that's a different mechanism; the
98
+ // test here just confirms advanced mode doesn't throw and returns
99
+ // both notes, not that ordering matches/mismatches literal mode).
100
+ expect(advanced.map((n) => n.path).sort()).toEqual(["body-only", "title-match"]);
101
+ });
102
+
103
+ it("a query with no results is unaffected (empty array in, empty array out)", async () => {
104
+ const hits = await store.searchNotes("nonexistent_term_xyz");
105
+ expect(hits).toEqual([]);
106
+ });
107
+
108
+ it("boosts on the post-frontmatter title line, not the `---` delimiter (shares computeDisplayTitle's frontmatter skip)", async () => {
109
+ // Direct-create misuse path: raw frontmatter-bearing content. Without
110
+ // the frontmatter skip, the derived "title" would be the literal
111
+ // string "---", which never matches any query term, so this note
112
+ // would wrongly land in the body-only tier.
113
+ await store.createNote(
114
+ "---\ntitle: irrelevant\n---\n# Budget Review\nQ3 numbers",
115
+ { path: "frontmatter-led" },
116
+ );
117
+ await store.createNote(
118
+ "Weekly Standup\nsomewhere we discuss the budget in passing",
119
+ { path: "body-only" },
120
+ );
121
+
122
+ const hits = await store.searchNotes("budget review");
123
+ expect(hits[0].path).toBe("frontmatter-led");
124
+ });
125
+ });
@@ -20,7 +20,10 @@
20
20
 
21
21
  import { describe, test, expect } from "bun:test";
22
22
  import {
23
+ ALL_NOTES_VIEW_PATH,
23
24
  applySeedPack,
25
+ ARCHIVE_VIEW_PATH,
26
+ ARCHIVED_TAG,
24
27
  CAPTURE_ANYTHING_PATH,
25
28
  CONNECT_AI_PATH,
26
29
  DEFAULT_VAULT_DESCRIPTION,
@@ -33,11 +36,17 @@ import {
33
36
  GUIDE_TAG,
34
37
  listSeedPacks,
35
38
  NOTES_REQUIRED_TAGS,
39
+ PINNED_TAG,
40
+ PINNED_VIEW_PATH,
41
+ RECENT_VIEW_PATH,
36
42
  SEED_PACK_NAMES,
43
+ STARTER_ONTOLOGY_PACK,
37
44
  SURFACE_STARTER_CONTENT,
38
45
  SURFACE_STARTER_PACK,
39
46
  SURFACE_STARTER_PATH,
40
47
  TAGS_GRAPH_PATH,
48
+ VIEW_TAG,
49
+ VIEWS_PATH_PREFIX,
41
50
  WELCOME_PATH,
42
51
  welcomePack,
43
52
  YOURS_TO_KEEP_PATH,
@@ -335,12 +344,119 @@ describe("surface-starter pack", () => {
335
344
  });
336
345
  });
337
346
 
347
+ describe("starter-ontology pack", () => {
348
+ test("exactly four tags, in constitution order: view, archived, pinned, capture", () => {
349
+ expect(STARTER_ONTOLOGY_PACK.name).toBe("starter-ontology");
350
+ expect(STARTER_ONTOLOGY_PACK.tags.map((t) => t.name)).toEqual([
351
+ "view",
352
+ "archived",
353
+ "pinned",
354
+ "capture",
355
+ ]);
356
+ });
357
+
358
+ test("opt-in, not seeded by default", () => {
359
+ expect(STARTER_ONTOLOGY_PACK.description).toContain("Opt-in — not seeded by default.");
360
+ });
361
+
362
+ test("view is the one meta tag — carries the kind/query/lane_by/date_field schema", () => {
363
+ expect(VIEW_TAG.name).toBe("view");
364
+ expect(VIEW_TAG.description).toContain("meta tag");
365
+ expect(VIEW_TAG.fields).toBeDefined();
366
+ expect(Object.keys(VIEW_TAG.fields!)).toEqual([
367
+ "kind",
368
+ "query",
369
+ "lane_by",
370
+ "date_field",
371
+ ]);
372
+
373
+ const kind = VIEW_TAG.fields!.kind!;
374
+ expect(kind.type).toBe("string");
375
+ expect(kind.enum).toEqual(["list", "board", "calendar", "gallery"]);
376
+ expect(kind.default).toBe("list");
377
+
378
+ const query = VIEW_TAG.fields!.query!;
379
+ expect(query.type).toBe("string");
380
+
381
+ // lane_by / date_field are optional (no default — absent stays absent).
382
+ expect(VIEW_TAG.fields!.lane_by!.default).toBeUndefined();
383
+ expect(VIEW_TAG.fields!.date_field!.default).toBeUndefined();
384
+ });
385
+
386
+ test("archived + pinned are plain tags — no schema", () => {
387
+ expect(ARCHIVED_TAG.name).toBe("archived");
388
+ expect(ARCHIVED_TAG.fields).toBeUndefined();
389
+ expect(ARCHIVED_TAG.description).toContain("out of the flow of the present");
390
+
391
+ expect(PINNED_TAG.name).toBe("pinned");
392
+ expect(PINNED_TAG.fields).toBeUndefined();
393
+ expect(PINNED_TAG.description).toContain("partition");
394
+ });
395
+
396
+ test("capture is reused verbatim from NOTES_REQUIRED_TAGS — byte-equal, no drift risk", () => {
397
+ const capture = STARTER_ONTOLOGY_PACK.tags.find((t) => t.name === "capture");
398
+ expect(capture).toBe(NOTES_REQUIRED_TAGS[0]); // SAME object reference, not just equal content
399
+ expect(capture!.description).toBe(NOTES_REQUIRED_TAGS[0]!.description);
400
+ });
401
+
402
+ test("four seed #view notes mirroring the app's default pages", () => {
403
+ expect(STARTER_ONTOLOGY_PACK.notes.map((n) => n.path)).toEqual([
404
+ ALL_NOTES_VIEW_PATH,
405
+ RECENT_VIEW_PATH,
406
+ PINNED_VIEW_PATH,
407
+ ARCHIVE_VIEW_PATH,
408
+ ]);
409
+ for (const note of STARTER_ONTOLOGY_PACK.notes) {
410
+ expect(note.path.startsWith(VIEWS_PATH_PREFIX)).toBe(true);
411
+ expect(note.tags).toEqual(["view"]);
412
+ expect(note.metadata?.kind).toBe("list");
413
+ expect(typeof note.metadata?.query).toBe("string");
414
+ // Every seeded query must itself be valid JSON — a downstream reader
415
+ // parses it as query-notes params.
416
+ expect(() => JSON.parse(note.metadata!.query as string)).not.toThrow();
417
+ }
418
+ });
419
+
420
+ test("All notes / Recent exclude archived explicitly (authoring-time default, not magic)", () => {
421
+ for (const path of [ALL_NOTES_VIEW_PATH, RECENT_VIEW_PATH]) {
422
+ const note = noteAt(STARTER_ONTOLOGY_PACK, path);
423
+ const query = JSON.parse(note.metadata!.query as string);
424
+ expect(query.exclude_tags).toEqual(["archived"]);
425
+ }
426
+ });
427
+
428
+ test("Recent orders by updated_at desc", () => {
429
+ const query = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, RECENT_VIEW_PATH).metadata!.query as string);
430
+ expect(query.order_by).toBe("updated_at");
431
+ expect(query.sort).toBe("desc");
432
+ });
433
+
434
+ test("Pinned queries tag:pinned; Archive queries tag:archived (the deliberate exception to exclude_tags)", () => {
435
+ const pinnedQuery = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, PINNED_VIEW_PATH).metadata!.query as string);
436
+ expect(pinnedQuery.tag).toBe("pinned");
437
+
438
+ const archiveQuery = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, ARCHIVE_VIEW_PATH).metadata!.query as string);
439
+ expect(archiveQuery.tag).toBe("archived");
440
+ expect(archiveQuery.exclude_tags).toBeUndefined();
441
+ });
442
+
443
+ test("no dangling wikilinks within the pack", () => {
444
+ const seededPaths = new Set(STARTER_ONTOLOGY_PACK.notes.map((n) => n.path));
445
+ for (const note of STARTER_ONTOLOGY_PACK.notes) {
446
+ for (const target of wikilinkTargets(note.content)) {
447
+ expect(seededPaths.has(target)).toBe(true);
448
+ }
449
+ }
450
+ });
451
+ });
452
+
338
453
  describe("pack registry", () => {
339
- test("lists exactly the three packs, in order", () => {
454
+ test("lists exactly the four packs, in order", () => {
340
455
  expect([...SEED_PACK_NAMES]).toEqual([
341
456
  "welcome",
342
457
  "getting-started",
343
458
  "surface-starter",
459
+ "starter-ontology",
344
460
  ]);
345
461
  expect(listSeedPacks().map((p) => p.name)).toEqual([...SEED_PACK_NAMES]);
346
462
  for (const p of listSeedPacks()) {