@openparachute/vault 0.7.2 → 0.7.3-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/core/src/attachment/policy.test.ts +66 -0
- package/core/src/attachment/policy.ts +131 -0
- package/core/src/attachment/tickets.test.ts +45 -0
- package/core/src/attachment/tickets.ts +117 -0
- package/core/src/attachment-tickets-tool.test.ts +286 -0
- package/core/src/conformance.ts +2 -1
- package/core/src/contract-typed-index.test.ts +4 -3
- package/core/src/core.test.ts +607 -11
- package/core/src/display-title.test.ts +190 -0
- package/core/src/embedding/chunker.test.ts +97 -0
- package/core/src/embedding/chunker.ts +180 -0
- package/core/src/embedding/provider.ts +108 -0
- package/core/src/embedding/staleness.test.ts +87 -0
- package/core/src/embedding/staleness.ts +83 -0
- package/core/src/embedding/vector-codec.test.ts +99 -0
- package/core/src/embedding/vector-codec.ts +60 -0
- package/core/src/embedding/vectors.test.ts +163 -0
- package/core/src/embedding/vectors.ts +135 -0
- package/core/src/expand.ts +11 -3
- package/core/src/indexed-fields.test.ts +9 -3
- package/core/src/indexed-fields.ts +9 -1
- package/core/src/lede.test.ts +96 -0
- package/core/src/mcp-semantic-search.test.ts +160 -0
- package/core/src/mcp.ts +405 -10
- package/core/src/notes.semantic-search.test.ts +131 -0
- package/core/src/notes.ts +319 -7
- package/core/src/query-warnings.ts +40 -0
- package/core/src/schema-defaults.ts +85 -1
- package/core/src/schema-v27-note-vectors.test.ts +178 -0
- package/core/src/schema.ts +81 -1
- package/core/src/search-fts-v25.test.ts +9 -1
- package/core/src/search-query.test.ts +42 -0
- package/core/src/search-query.ts +27 -0
- package/core/src/search-title-boost.test.ts +125 -0
- package/core/src/seed-packs.test.ts +117 -1
- package/core/src/seed-packs.ts +217 -1
- package/core/src/store.semantic-search.test.ts +236 -0
- package/core/src/store.ts +162 -5
- package/core/src/tag-schemas.ts +27 -12
- package/core/src/types.ts +63 -1
- package/core/src/vault-projection.ts +48 -0
- package/package.json +6 -1
- package/src/add-pack.test.ts +32 -0
- package/src/attachment-tickets.test.ts +350 -0
- package/src/attachment-tickets.ts +264 -0
- package/src/auth.ts +8 -2
- package/src/cli.ts +5 -1
- package/src/contract-errors.test.ts +2 -1
- package/src/contract-honest-queries.test.ts +88 -0
- package/src/contract-search.test.ts +41 -0
- package/src/embedding/capability.test.ts +33 -0
- package/src/embedding/capability.ts +34 -0
- package/src/embedding/external-api.test.ts +154 -0
- package/src/embedding/external-api.ts +144 -0
- package/src/embedding/onnx-transformers.test.ts +113 -0
- package/src/embedding/onnx-transformers.ts +141 -0
- package/src/embedding/select.test.ts +99 -0
- package/src/embedding/select.ts +92 -0
- package/src/embedding-worker.test.ts +300 -0
- package/src/embedding-worker.ts +226 -0
- package/src/mcp-http.ts +13 -1
- package/src/mcp-tools.ts +49 -14
- package/src/onboarding-seed.test.ts +64 -0
- package/src/routes.ts +173 -92
- package/src/routing.ts +17 -0
- package/src/scribe-discovery.test.ts +76 -1
- package/src/scribe-discovery.ts +28 -9
- package/src/semantic-search-routes.test.ts +161 -0
- package/src/server.ts +21 -2
- package/src/vault-embeddings-capability.test.ts +82 -0
- package/src/vault-store-embedding-wiring.test.ts +69 -0
- package/src/vault-store.ts +53 -2
- package/src/vault.test.ts +62 -13
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { Database } from "bun:sqlite";
|
|
44
|
+
import { timestampToMs } from "./cursor.js";
|
|
44
45
|
|
|
45
46
|
// ---------------------------------------------------------------------------
|
|
46
47
|
// Types
|
|
@@ -57,7 +58,7 @@ export interface SchemaField {
|
|
|
57
58
|
* `type_mismatch` warning that previously fired on every integer-shaped
|
|
58
59
|
* field because the validator had no `"integer"` case. See vault#310.
|
|
59
60
|
*/
|
|
60
|
-
type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
|
|
61
|
+
type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference" | "date";
|
|
61
62
|
enum?: string[];
|
|
62
63
|
description?: string;
|
|
63
64
|
/**
|
|
@@ -320,6 +321,82 @@ export function resolveNoteSchemas(
|
|
|
320
321
|
return { effectiveTags, mergedFields, conflicts };
|
|
321
322
|
}
|
|
322
323
|
|
|
324
|
+
/**
|
|
325
|
+
* Normalize `date`-typed field VALUES in `metadata` to canonical UTC ISO
|
|
326
|
+
* form (`Z`-suffixed) BEFORE persisting (vault#date-field-type — mixed-
|
|
327
|
+
* offset corruption, caught in review).
|
|
328
|
+
*
|
|
329
|
+
* The bug: every consumer of an indexed `date` field — `buildOperatorClause`
|
|
330
|
+
* (query-operators.ts), `date_filter` and `order_by` (notes.ts) — compares
|
|
331
|
+
* the generated `meta_<field>` column as raw TEXT. ISO-8601 strings sort
|
|
332
|
+
* correctly under a TEXT compare ONLY when every value shares the same
|
|
333
|
+
* offset representation. `timestampToMs` (cursor.ts) — the validator both
|
|
334
|
+
* `defaultMatchesType` and `valueMatchesType` reuse — correctly ACCEPTS an
|
|
335
|
+
* explicit `±HH:MM` offset (validation was never the gap), but a value like
|
|
336
|
+
* `"2026-07-16T10:00:00+02:00"` (= `08:00Z`) persisted VERBATIM sorts AFTER
|
|
337
|
+
* `"2026-07-16T09:00:00Z"` (= `09:00Z`) under a raw string compare, even
|
|
338
|
+
* though the actual instant is earlier — a mixed-offset vault silently gets
|
|
339
|
+
* wrong range-query/order_by/date_filter results. This is the exact bug
|
|
340
|
+
* class `updated_at` got a dedicated ms-mirror column for (vault#585/#586,
|
|
341
|
+
* see `notes.ts`'s `dateFilter` block) that never extended to user-declared
|
|
342
|
+
* `date` fields.
|
|
343
|
+
*
|
|
344
|
+
* The fix is normalize-on-write, not reject-on-write (matching the
|
|
345
|
+
* "paths are normalized on write" precedent — instant preserved,
|
|
346
|
+
* representation canonicalized) — rejecting offsets would defeat the
|
|
347
|
+
* type's own motivation (calendar integrations and other emitters commonly
|
|
348
|
+
* produce offset timestamps, not always `Z`). Only a FULL timestamp (has a
|
|
349
|
+
* time component) is rewritten, to `new Date(ms).toISOString()` — always
|
|
350
|
+
* UTC, millisecond precision, `Z`-suffixed. A bare `YYYY-MM-DD` value is
|
|
351
|
+
* left untouched: it has no offset to normalize, is already canonical, and
|
|
352
|
+
* prefix-sorts correctly against full timestamps sharing the same calendar
|
|
353
|
+
* day. A value that fails `timestampToMs` is left untouched too — that's a
|
|
354
|
+
* `type_mismatch` for `valueMatchesType` to catch, not this function's job.
|
|
355
|
+
*
|
|
356
|
+
* COPY-ON-WRITE, never mutates the input `metadata` object (vault#date-
|
|
357
|
+
* field-type review round 2) — `core/` is a published library; a direct
|
|
358
|
+
* embedder holding a reference to the object THEY passed in must never see
|
|
359
|
+
* it change out from under them. Returns the SAME reference when nothing
|
|
360
|
+
* needs rewriting (the common case — no unnecessary allocation on the
|
|
361
|
+
* no-op path) or `metadata` is undefined; returns a freshly shallow-copied
|
|
362
|
+
* object, with only the rewritten field(s) replaced, the first time a
|
|
363
|
+
* rewrite is actually needed. Every call site must use the RETURNED value
|
|
364
|
+
* (not assume its input was mutated) — see `store.createNote`/`updateNote`/
|
|
365
|
+
* `createNotes`/`createNoteRaw`, all of which reassign their local
|
|
366
|
+
* `opts`/`updates`/`input` binding from the return rather than relying on a
|
|
367
|
+
* side effect.
|
|
368
|
+
*
|
|
369
|
+
* Called from `store.createNote`/`updateNote`/`createNotes`/`createNoteRaw`
|
|
370
|
+
* — the lowest chokepoint every write path (MCP, REST, import) funnels
|
|
371
|
+
* through — BEFORE the row is persisted, so the value that's validated by
|
|
372
|
+
* `validateNote`/`valueMatchesType` upstream (offset-tolerant, so
|
|
373
|
+
* normalization can't newly fail a write) and the value that's written +
|
|
374
|
+
* echoed back on the response are the SAME normalized string.
|
|
375
|
+
*/
|
|
376
|
+
export function normalizeDateFields(
|
|
377
|
+
resolved: ResolvedSchemas,
|
|
378
|
+
note: { tags?: string[]; metadata?: Record<string, unknown> },
|
|
379
|
+
): Record<string, unknown> | undefined {
|
|
380
|
+
const metadata = note.metadata;
|
|
381
|
+
if (!metadata) return metadata;
|
|
382
|
+
const { mergedFields } = resolveNoteSchemas(resolved, note);
|
|
383
|
+
let copy: Record<string, unknown> | undefined;
|
|
384
|
+
for (const [fieldName, { spec }] of mergedFields) {
|
|
385
|
+
if (spec.type !== "date") continue;
|
|
386
|
+
const value = metadata[fieldName];
|
|
387
|
+
if (typeof value !== "string") continue;
|
|
388
|
+
// Bare `YYYY-MM-DD` (no time component) has no offset to normalize.
|
|
389
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(value)) continue;
|
|
390
|
+
const ms = timestampToMs(value);
|
|
391
|
+
if (ms === null) continue; // unparseable — valueMatchesType's job, not ours
|
|
392
|
+
const canonical = new Date(ms).toISOString();
|
|
393
|
+
if (canonical === value) continue;
|
|
394
|
+
if (!copy) copy = { ...metadata }; // lazy: only copy once a rewrite is due
|
|
395
|
+
copy[fieldName] = canonical;
|
|
396
|
+
}
|
|
397
|
+
return copy ?? metadata;
|
|
398
|
+
}
|
|
399
|
+
|
|
323
400
|
function fieldSpecsEqual(a: SchemaField, b: SchemaField): boolean {
|
|
324
401
|
if (a.type !== b.type) return false;
|
|
325
402
|
if (!stringArraysEqual(a.enum, b.enum)) return false;
|
|
@@ -390,6 +467,13 @@ function valueMatchesType(value: unknown, type: SchemaField["type"], cardinality
|
|
|
390
467
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
391
468
|
}
|
|
392
469
|
return typeof value === "string";
|
|
470
|
+
// `date` validates like `string` — an ISO-8601 date (`YYYY-MM-DD`) or
|
|
471
|
+
// full RFC3339 timestamp. Reuses `cursor.ts`'s `timestampToMs` (the SAME
|
|
472
|
+
// UTC-correct parser `date_filter`'s `updated_at` bound uses), not a
|
|
473
|
+
// second date parser — see tag-schemas.ts's `VALID_FIELD_TYPES` doc
|
|
474
|
+
// comment.
|
|
475
|
+
case "date":
|
|
476
|
+
return typeof value === "string" && timestampToMs(value) !== null;
|
|
393
477
|
}
|
|
394
478
|
}
|
|
395
479
|
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration v26 → v27: the `note_vectors` table (semantic search MVP —
|
|
3
|
+
* `SEMANTIC-MVP-PLAN.md`). Exercised against a REALISTIC pre-v27 fixture
|
|
4
|
+
* (a hand-built v26-shape DB, not a fresh vault — per the real-install-
|
|
5
|
+
* fixture discipline: fresh-DB-only testing misses the "existing vault
|
|
6
|
+
* upgrading" path, which is the one that matters in production).
|
|
7
|
+
*/
|
|
8
|
+
import { describe, it, expect } from "bun:test";
|
|
9
|
+
import { Database } from "bun:sqlite";
|
|
10
|
+
import { initSchema, SCHEMA_VERSION } from "./schema.js";
|
|
11
|
+
import { encodeVector, normalize } from "./embedding/vector-codec.js";
|
|
12
|
+
|
|
13
|
+
/** A v26-shape vault: everything through vault#586, no `note_vectors` at all. */
|
|
14
|
+
function buildPreV27Vault(): Database {
|
|
15
|
+
const db = new Database(":memory:");
|
|
16
|
+
db.exec(`
|
|
17
|
+
CREATE TABLE notes (
|
|
18
|
+
id TEXT PRIMARY KEY,
|
|
19
|
+
content TEXT DEFAULT '',
|
|
20
|
+
path TEXT,
|
|
21
|
+
metadata TEXT DEFAULT '{}',
|
|
22
|
+
created_at TEXT NOT NULL,
|
|
23
|
+
updated_at TEXT,
|
|
24
|
+
extension TEXT NOT NULL DEFAULT 'md',
|
|
25
|
+
created_by TEXT,
|
|
26
|
+
created_via TEXT,
|
|
27
|
+
last_updated_by TEXT,
|
|
28
|
+
last_updated_via TEXT,
|
|
29
|
+
updated_at_ms INTEGER
|
|
30
|
+
);
|
|
31
|
+
CREATE TABLE tags (
|
|
32
|
+
name TEXT PRIMARY KEY,
|
|
33
|
+
description TEXT,
|
|
34
|
+
fields TEXT,
|
|
35
|
+
relationships TEXT,
|
|
36
|
+
parent_names TEXT,
|
|
37
|
+
created_at TEXT,
|
|
38
|
+
updated_at TEXT
|
|
39
|
+
);
|
|
40
|
+
CREATE TABLE note_tags (
|
|
41
|
+
note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
42
|
+
tag_name TEXT NOT NULL REFERENCES tags(name),
|
|
43
|
+
PRIMARY KEY (note_id, tag_name)
|
|
44
|
+
);
|
|
45
|
+
CREATE TABLE attachments (
|
|
46
|
+
id TEXT PRIMARY KEY,
|
|
47
|
+
note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
48
|
+
path TEXT NOT NULL,
|
|
49
|
+
mime_type TEXT NOT NULL,
|
|
50
|
+
metadata TEXT DEFAULT '{}',
|
|
51
|
+
created_at TEXT NOT NULL
|
|
52
|
+
);
|
|
53
|
+
CREATE TABLE links (
|
|
54
|
+
source_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
55
|
+
target_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
56
|
+
relationship TEXT NOT NULL,
|
|
57
|
+
metadata TEXT DEFAULT '{}',
|
|
58
|
+
created_at TEXT NOT NULL,
|
|
59
|
+
UNIQUE(source_id, target_id, relationship)
|
|
60
|
+
);
|
|
61
|
+
CREATE INDEX idx_notes_updated_ms ON notes(updated_at_ms, id);
|
|
62
|
+
-- A REAL v26 vault already went through migrateToV25's notes_fts
|
|
63
|
+
-- rebuild+repopulate long ago — this fixture must include that
|
|
64
|
+
-- already-populated state too, or a post-migration DELETE against
|
|
65
|
+
-- notes hits the notes_fts_delete trigger with an external-content FTS5
|
|
66
|
+
-- index that was never populated for these rows, which SQLite reports
|
|
67
|
+
-- as disk-image corruption (a fixture-fidelity issue, not a real one).
|
|
68
|
+
CREATE VIRTUAL TABLE notes_fts USING fts5(
|
|
69
|
+
path, content, content='notes', content_rowid='rowid', tokenize='porter unicode61'
|
|
70
|
+
);
|
|
71
|
+
CREATE TRIGGER notes_fts_insert AFTER INSERT ON notes BEGIN
|
|
72
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
73
|
+
END;
|
|
74
|
+
CREATE TRIGGER notes_fts_delete AFTER DELETE ON notes BEGIN
|
|
75
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
76
|
+
END;
|
|
77
|
+
CREATE TRIGGER notes_fts_update AFTER UPDATE OF content, path ON notes BEGIN
|
|
78
|
+
INSERT INTO notes_fts(notes_fts, rowid, path, content) VALUES('delete', old.rowid, COALESCE(old.path, ''), old.content);
|
|
79
|
+
INSERT INTO notes_fts(rowid, path, content) VALUES (new.rowid, COALESCE(new.path, ''), new.content);
|
|
80
|
+
END;
|
|
81
|
+
CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT);
|
|
82
|
+
INSERT INTO schema_version (version, applied_at) VALUES (26, '2026-07-01T00:00:00.000Z');
|
|
83
|
+
`);
|
|
84
|
+
const insert = db.prepare(
|
|
85
|
+
"INSERT INTO notes (id, content, created_at, updated_at, updated_at_ms) VALUES (?, ?, ?, ?, ?)",
|
|
86
|
+
);
|
|
87
|
+
insert.run("note-1", "some real morning-pages content", "2026-06-01T00:00:00.000Z", "2026-06-01T00:00:00.000Z", Date.UTC(2026, 5, 1));
|
|
88
|
+
insert.run("note-2", "another pre-existing note", "2026-06-02T00:00:00.000Z", "2026-06-02T00:00:00.000Z", Date.UTC(2026, 5, 2));
|
|
89
|
+
return db;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function hasTable(db: Database, name: string): boolean {
|
|
93
|
+
return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function hasIndex(db: Database, name: string): boolean {
|
|
97
|
+
return !!db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name=?").get(name);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
describe("migrateToV27 — creates note_vectors on a real pre-v27 fixture", () => {
|
|
101
|
+
it("has no note_vectors table before migration; has one (+ its index) after", () => {
|
|
102
|
+
const db = buildPreV27Vault();
|
|
103
|
+
expect(hasTable(db, "note_vectors")).toBe(false);
|
|
104
|
+
initSchema(db);
|
|
105
|
+
expect(hasTable(db, "note_vectors")).toBe(true);
|
|
106
|
+
expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
|
|
107
|
+
expect(
|
|
108
|
+
(db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { v: number }).v,
|
|
109
|
+
).toBe(SCHEMA_VERSION);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("does NOT touch existing note rows — no data movement, unlike migrateToV26's backfill", () => {
|
|
113
|
+
const db = buildPreV27Vault();
|
|
114
|
+
const before = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
|
|
115
|
+
initSchema(db);
|
|
116
|
+
const after = db.prepare("SELECT id, content, created_at, updated_at FROM notes ORDER BY id").all();
|
|
117
|
+
expect(after).toEqual(before);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("every pre-existing note starts with ZERO note_vectors rows — the implicit 'needs embedding' backfill signal", () => {
|
|
121
|
+
const db = buildPreV27Vault();
|
|
122
|
+
initSchema(db);
|
|
123
|
+
const count = (
|
|
124
|
+
db.prepare("SELECT COUNT(*) AS n FROM note_vectors").get() as { n: number }
|
|
125
|
+
).n;
|
|
126
|
+
expect(count).toBe(0);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it("is idempotent — a second initSchema neither throws nor duplicates the table/index", () => {
|
|
130
|
+
const db = buildPreV27Vault();
|
|
131
|
+
initSchema(db);
|
|
132
|
+
expect(() => initSchema(db)).not.toThrow();
|
|
133
|
+
expect(hasTable(db, "note_vectors")).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("note_vectors rows cascade-delete when their note is deleted (ON DELETE CASCADE)", () => {
|
|
137
|
+
const db = buildPreV27Vault();
|
|
138
|
+
initSchema(db);
|
|
139
|
+
const vec = encodeVector(normalize(new Float32Array([1, 2, 3, 4])));
|
|
140
|
+
db.prepare(
|
|
141
|
+
`INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
|
|
142
|
+
VALUES (?, 0, ?, 4, 'test-model', 'deadbeef', ?)`,
|
|
143
|
+
).run("note-1", vec, new Date().toISOString());
|
|
144
|
+
expect(
|
|
145
|
+
(db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
|
|
146
|
+
).toBe(1);
|
|
147
|
+
|
|
148
|
+
db.prepare("DELETE FROM notes WHERE id = ?").run("note-1");
|
|
149
|
+
|
|
150
|
+
expect(
|
|
151
|
+
(db.prepare("SELECT COUNT(*) AS n FROM note_vectors WHERE note_id = ?").get("note-1") as { n: number }).n,
|
|
152
|
+
).toBe(0);
|
|
153
|
+
// note-2 is unaffected.
|
|
154
|
+
expect(hasTable(db, "notes")).toBe(true);
|
|
155
|
+
expect(
|
|
156
|
+
(db.prepare("SELECT COUNT(*) AS n FROM notes WHERE id = ?").get("note-2") as { n: number }).n,
|
|
157
|
+
).toBe(1);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("PRIMARY KEY (note_id, chunk_ix) rejects a duplicate chunk row (upsert-by-replace is the caller's job)", () => {
|
|
161
|
+
const db = buildPreV27Vault();
|
|
162
|
+
initSchema(db);
|
|
163
|
+
const vec = encodeVector(normalize(new Float32Array([1, 0, 0, 0])));
|
|
164
|
+
const insert = db.prepare(
|
|
165
|
+
`INSERT INTO note_vectors (note_id, chunk_ix, vector, dims, model, content_hash, embedded_at)
|
|
166
|
+
VALUES (?, 0, ?, 4, 'test-model', 'hash-a', ?)`,
|
|
167
|
+
);
|
|
168
|
+
insert.run("note-1", vec, new Date().toISOString());
|
|
169
|
+
expect(() => insert.run("note-1", vec, new Date().toISOString())).toThrow();
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("a fresh vault (no prior schema_version row) gets note_vectors directly from SCHEMA_SQL", () => {
|
|
173
|
+
const db = new Database(":memory:");
|
|
174
|
+
initSchema(db);
|
|
175
|
+
expect(hasTable(db, "note_vectors")).toBe(true);
|
|
176
|
+
expect(hasIndex(db, "idx_note_vectors_stale")).toBe(true);
|
|
177
|
+
});
|
|
178
|
+
});
|
package/core/src/schema.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
|
|
|
5
5
|
import { transaction } from "./txn.js";
|
|
6
6
|
import { timestampToMs } from "./cursor.js";
|
|
7
7
|
|
|
8
|
-
export const SCHEMA_VERSION =
|
|
8
|
+
export const SCHEMA_VERSION = 27;
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* Deterministic last-resort epoch for a note whose `updated_at` AND
|
|
@@ -112,6 +112,35 @@ CREATE TABLE IF NOT EXISTS links (
|
|
|
112
112
|
UNIQUE(source_id, target_id, relationship)
|
|
113
113
|
);
|
|
114
114
|
|
|
115
|
+
-- note_vectors (v27, semantic search MVP — EXPERIMENTAL): one row per
|
|
116
|
+
-- embedded CHUNK of a note (see core/src/embedding/chunker.ts). chunk_ix=0
|
|
117
|
+
-- is either the whole note (short notes, the degenerate/v1-equivalent
|
|
118
|
+
-- case) or the first section of a long, multi-section note.
|
|
119
|
+
--
|
|
120
|
+
-- vector is a Float32Array serialized to raw bytes (core/src/embedding/
|
|
121
|
+
-- vector-codec.ts), stored L2-NORMALIZED so ranking is a plain dot
|
|
122
|
+
-- product. model is recorded per-row so a provider/model change is
|
|
123
|
+
-- detectable (see core/src/embedding/staleness.ts) — a chunk_ix's row is
|
|
124
|
+
-- REPLACED wholesale on re-embed, never appended, so a vault only ever
|
|
125
|
+
-- carries vectors from ONE model at a time. content_hash is the freshness
|
|
126
|
+
-- gate: a create/update whose chunk text hashes the same as the stored
|
|
127
|
+
-- row is a no-op (no wasted embed on a metadata-only save).
|
|
128
|
+
-- ON DELETE CASCADE mirrors attachments/links above — a deleted note
|
|
129
|
+
-- drops its vectors for free.
|
|
130
|
+
CREATE TABLE IF NOT EXISTS note_vectors (
|
|
131
|
+
note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
132
|
+
chunk_ix INTEGER NOT NULL DEFAULT 0,
|
|
133
|
+
vector BLOB NOT NULL,
|
|
134
|
+
dims INTEGER NOT NULL,
|
|
135
|
+
model TEXT NOT NULL,
|
|
136
|
+
content_hash TEXT NOT NULL,
|
|
137
|
+
embedded_at TEXT NOT NULL,
|
|
138
|
+
PRIMARY KEY (note_id, chunk_ix)
|
|
139
|
+
);
|
|
140
|
+
-- Backs the staleness diff (per-note row fetch) and the model-mismatch
|
|
141
|
+
-- sweep (a model <> ? scan) the backfill/drain use to find pending work.
|
|
142
|
+
CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash);
|
|
143
|
+
|
|
115
144
|
-- tag_schemas (v6) was retired in v14; description + fields lifted onto the
|
|
116
145
|
-- tags row directly. The CREATE TABLE was removed from SCHEMA_SQL after the
|
|
117
146
|
-- v14 data migration drops the table; existing v6+ vaults pick up the
|
|
@@ -554,6 +583,12 @@ export function initSchema(db: Database): void {
|
|
|
554
583
|
// `updated_at` string with a UTC-correct parse. See vault#586.
|
|
555
584
|
migrateToV26(db);
|
|
556
585
|
|
|
586
|
+
// Migrate v26 → v27: create the `note_vectors` table (semantic search
|
|
587
|
+
// MVP, EXPERIMENTAL) — one row per embedded note chunk. No backfill loop:
|
|
588
|
+
// every pre-existing note simply has zero rows, which the staleness gate
|
|
589
|
+
// already reads as "needs embedding." See vault semantic-search MVP plan.
|
|
590
|
+
migrateToV27(db);
|
|
591
|
+
|
|
557
592
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
558
593
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
559
594
|
rebuildIndexes(db);
|
|
@@ -1617,6 +1652,51 @@ function migrateToV26(db: Database): void {
|
|
|
1617
1652
|
db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated_ms ON notes(updated_at_ms, id)");
|
|
1618
1653
|
}
|
|
1619
1654
|
|
|
1655
|
+
/**
|
|
1656
|
+
* Migrate v26 → v27: the `note_vectors` table (semantic search MVP,
|
|
1657
|
+
* EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Created by SCHEMA_SQL's
|
|
1658
|
+
* `CREATE TABLE IF NOT EXISTS` above, so on a FRESH vault this is a no-op
|
|
1659
|
+
* confirmation hook (same pattern as `migrateToV7`/`V20`/`V21` — the table
|
|
1660
|
+
* is new, not a column added to existing rows). On an EXISTING vault
|
|
1661
|
+
* upgrading through this version, the `CREATE TABLE IF NOT EXISTS` +
|
|
1662
|
+
* `CREATE INDEX IF NOT EXISTS` calls create it for the first time.
|
|
1663
|
+
*
|
|
1664
|
+
* Deliberately NO backfill loop, and no embedding call of any kind — this
|
|
1665
|
+
* function never resolves an `EmbeddingProvider` or does network/CPU work.
|
|
1666
|
+
* "Backfill" for existing notes is implicit and free: every note that
|
|
1667
|
+
* predates this migration simply has ZERO rows in the (now-created, empty)
|
|
1668
|
+
* `note_vectors` table, which `core/src/embedding/staleness.ts:planStaleness`
|
|
1669
|
+
* already treats as "stale — needs embedding" (no existing row at a given
|
|
1670
|
+
* `chunk_ix` ⇒ stale). The async drain (self-host: the `onNote` hook +
|
|
1671
|
+
* its sweep; cloud: the DO-alarm drain) discovers this pending work by
|
|
1672
|
+
* scanning for notes lacking a fresh vector row — the exact same "queue is
|
|
1673
|
+
* implicit in the diff, not a separate table" property the `updated_at_ms`
|
|
1674
|
+
* self-heal above relies on. So this migration is intentionally ALL
|
|
1675
|
+
* schema, zero data movement — the whole point of "enqueue, never embed
|
|
1676
|
+
* inside the migration."
|
|
1677
|
+
*
|
|
1678
|
+
* Wrapped in a transaction for symmetry with `migrateToV25`/`V26` (a
|
|
1679
|
+
* multi-statement DDL block should commit or roll back as one unit), even
|
|
1680
|
+
* though there's no row data to lose here.
|
|
1681
|
+
*/
|
|
1682
|
+
function migrateToV27(db: Database): void {
|
|
1683
|
+
transaction(db, () => {
|
|
1684
|
+
db.exec(`
|
|
1685
|
+
CREATE TABLE IF NOT EXISTS note_vectors (
|
|
1686
|
+
note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE,
|
|
1687
|
+
chunk_ix INTEGER NOT NULL DEFAULT 0,
|
|
1688
|
+
vector BLOB NOT NULL,
|
|
1689
|
+
dims INTEGER NOT NULL,
|
|
1690
|
+
model TEXT NOT NULL,
|
|
1691
|
+
content_hash TEXT NOT NULL,
|
|
1692
|
+
embedded_at TEXT NOT NULL,
|
|
1693
|
+
PRIMARY KEY (note_id, chunk_ix)
|
|
1694
|
+
)
|
|
1695
|
+
`);
|
|
1696
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_note_vectors_stale ON note_vectors(model, content_hash)");
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1620
1700
|
function hasTable(db: Database, name: string): boolean {
|
|
1621
1701
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1622
1702
|
return !!row;
|
|
@@ -132,7 +132,15 @@ describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
|
|
|
132
132
|
);
|
|
133
133
|
insert.run(
|
|
134
134
|
"legacy-1",
|
|
135
|
-
|
|
135
|
+
// Deliberately multi-line, with the "propolis" mention on a LATER
|
|
136
|
+
// line — this fixture isolates the PATH-weighted bm25 axis (below)
|
|
137
|
+
// from the separate content-title-boost axis (title axis, ratified
|
|
138
|
+
// 2026-07-17): that boost derives a note's "title" from its FIRST
|
|
139
|
+
// content line, and a single-line note IS its own title in full, so
|
|
140
|
+
// a one-line body mentioning "propolis" would (correctly, under the
|
|
141
|
+
// newer axis) count as a title match and confound this test's
|
|
142
|
+
// path-only signal. See `search-title-boost.test.ts` for that axis.
|
|
143
|
+
"Notes on foraging patterns\n\na dedicated writeup mentioning propolis only once in passing text",
|
|
136
144
|
"beekeeping-notes",
|
|
137
145
|
"2026-01-01T00:00:00.000Z",
|
|
138
146
|
"2026-01-01T00:00:00.000Z",
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
escapeFtsToken,
|
|
5
5
|
buildLiteralSearchQuery,
|
|
6
6
|
isValidSearchMode,
|
|
7
|
+
extractLiteralBoostTerms,
|
|
7
8
|
} from "./search-query.js";
|
|
8
9
|
|
|
9
10
|
describe("escapeFtsToken", () => {
|
|
@@ -147,3 +148,44 @@ describe("isValidSearchMode / SEARCH_MODES", () => {
|
|
|
147
148
|
expect(isValidSearchMode(["literal"])).toBe(false);
|
|
148
149
|
});
|
|
149
150
|
});
|
|
151
|
+
|
|
152
|
+
describe("extractLiteralBoostTerms (search title-boost input, title axis)", () => {
|
|
153
|
+
it("lowercases and whitespace-splits", () => {
|
|
154
|
+
expect(extractLiteralBoostTerms("Budget Review")).toEqual(["budget", "review"]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("collapses whitespace runs", () => {
|
|
158
|
+
expect(extractLiteralBoostTerms("widgets gadgets")).toEqual(["widgets", "gadgets"]);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it("trims leading/trailing whitespace", () => {
|
|
162
|
+
expect(extractLiteralBoostTerms(" widgets ")).toEqual(["widgets"]);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("strips literal quote characters a caller typed, unlike buildLiteralSearchQuery", () => {
|
|
166
|
+
// buildLiteralSearchQuery treats a manually-typed `"` as ordinary
|
|
167
|
+
// content (escaped into the FTS phrase); the boost-term extractor
|
|
168
|
+
// strips it instead, since a title-string comparison shouldn't
|
|
169
|
+
// require the title to contain a literal quote mark.
|
|
170
|
+
expect(extractLiteralBoostTerms(`"eleven-day capping delay"`)).toEqual([
|
|
171
|
+
"eleven-day",
|
|
172
|
+
"capping",
|
|
173
|
+
"delay",
|
|
174
|
+
]);
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it("preserves punctuation that isn't a quote (hyphen, period, apostrophe)", () => {
|
|
178
|
+
expect(extractLiteralBoostTerms("eleven-day")).toEqual(["eleven-day"]);
|
|
179
|
+
expect(extractLiteralBoostTerms("18.6")).toEqual(["18.6"]);
|
|
180
|
+
expect(extractLiteralBoostTerms("didn't")).toEqual(["didn't"]);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("returns an empty array for blank/whitespace-only input", () => {
|
|
184
|
+
expect(extractLiteralBoostTerms("")).toEqual([]);
|
|
185
|
+
expect(extractLiteralBoostTerms(" ")).toEqual([]);
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
it("sanitizes control characters as separators, matching buildLiteralSearchQuery", () => {
|
|
189
|
+
expect(extractLiteralBoostTerms("hello\x00world")).toEqual(["hello", "world"]);
|
|
190
|
+
});
|
|
191
|
+
});
|
package/core/src/search-query.ts
CHANGED
|
@@ -153,3 +153,30 @@ export function buildLiteralSearchQuery(raw: string): LiteralSearchQuery {
|
|
|
153
153
|
const tokens = cleaned.trim().split(/\s+/).filter(Boolean);
|
|
154
154
|
return { query: tokens.map(escapeFtsToken).join(" "), isEmpty: false };
|
|
155
155
|
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Extract lowercase whitespace-delimited terms from raw literal-mode search
|
|
159
|
+
* text, for the search title-boost post-rank (title axis, ratified
|
|
160
|
+
* 2026-07-17) — NOT used to build the FTS5 MATCH query itself (that's
|
|
161
|
+
* `buildLiteralSearchQuery`, above). Sanitizes control characters the same
|
|
162
|
+
* way, plus strips any literal `"` a caller typed (kept as ordinary content
|
|
163
|
+
* by `buildLiteralSearchQuery`, but a display-title substring check
|
|
164
|
+
* shouldn't require the title to contain a literal quote mark).
|
|
165
|
+
*
|
|
166
|
+
* This is a heuristic re-rank signal, not a second index — it deliberately
|
|
167
|
+
* does not attempt FTS5-tokenizer parity (stemming, unicode normalization).
|
|
168
|
+
* It only needs to answer "does the note's first line plausibly contain the
|
|
169
|
+
* query," not reproduce FTS5's match semantics exactly. Restricted to
|
|
170
|
+
* literal-mode callers by convention (see `core/src/notes.ts` `searchNotes`)
|
|
171
|
+
* — `search_mode: "advanced"` text carries FTS5 boolean/column/prefix
|
|
172
|
+
* operators (`AND`, `path:`, `foo*`) that would leak into the term list as
|
|
173
|
+
* false "content" if tokenized the same naive way.
|
|
174
|
+
*/
|
|
175
|
+
export function extractLiteralBoostTerms(raw: string): string[] {
|
|
176
|
+
const cleaned = raw.replace(CONTROL_CHARS, " ").toLowerCase();
|
|
177
|
+
return cleaned
|
|
178
|
+
.trim()
|
|
179
|
+
.split(/\s+/)
|
|
180
|
+
.map((t) => t.replace(/^"+|"+$/g, ""))
|
|
181
|
+
.filter(Boolean);
|
|
182
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search title-boost (title axis, ratified 2026-07-17): literal-mode
|
|
3
|
+
* search results whose `displayTitle` (first non-empty content line, NOT
|
|
4
|
+
* `path`) contains every query term are post-ranked ahead of body-only
|
|
5
|
+
* matches. No-migration — an in-memory re-rank of the already-fetched
|
|
6
|
+
* page, not a `notes_fts` schema change (see `applySearchTitleBoost` /
|
|
7
|
+
* `boostTitleMatches` in `notes.ts`).
|
|
8
|
+
*
|
|
9
|
+
* This is a DIFFERENT axis from the existing `path`-weighted bm25 ranking
|
|
10
|
+
* covered in `search-fts-v25.test.ts` — a note's title here is its first
|
|
11
|
+
* CONTENT line, not its `path`.
|
|
12
|
+
*/
|
|
13
|
+
import { describe, it, expect, beforeEach } from "bun:test";
|
|
14
|
+
import { Database } from "bun:sqlite";
|
|
15
|
+
import { SqliteStore } from "./store.js";
|
|
16
|
+
|
|
17
|
+
let store: SqliteStore;
|
|
18
|
+
let db: Database;
|
|
19
|
+
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
db = new Database(":memory:");
|
|
22
|
+
store = new SqliteStore(db);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
describe("search title-boost", () => {
|
|
26
|
+
it("ranks a first-line (title) match above a body-only match", async () => {
|
|
27
|
+
// Body-only match: "budget" appears only deep in the body, first line
|
|
28
|
+
// is unrelated.
|
|
29
|
+
await store.createNote(
|
|
30
|
+
"Weekly Standup\nlots of unrelated notes here, but somewhere we discuss the budget in passing",
|
|
31
|
+
{ path: "standup-notes" },
|
|
32
|
+
);
|
|
33
|
+
// First-line (title) match: "Budget" is the first line itself.
|
|
34
|
+
await store.createNote("Budget Review\nQ3 numbers", { path: "budget-review" });
|
|
35
|
+
|
|
36
|
+
const hits = await store.searchNotes("budget");
|
|
37
|
+
expect(hits.length).toBeGreaterThanOrEqual(2);
|
|
38
|
+
expect(hits[0].path).toBe("budget-review");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("requires ALL query terms to be present in the first line to boost (a partial title match doesn't qualify)", async () => {
|
|
42
|
+
// "budget" is in the title but "review" is only in the body — a
|
|
43
|
+
// PARTIAL title match, which stays in the body-only tier.
|
|
44
|
+
await store.createNote("Budget notes\nsee the quarterly review for details", { path: "partial" });
|
|
45
|
+
// Both "budget" and "review" are in the title — a FULL title match.
|
|
46
|
+
await store.createNote("Budget Review\nQ3 numbers", { path: "full-match" });
|
|
47
|
+
|
|
48
|
+
const hits = await store.searchNotes("budget review");
|
|
49
|
+
expect(hits.map((n) => n.path)).toEqual(["full-match", "partial"]);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("ties (both notes in the same tier) keep the existing relevance order", async () => {
|
|
53
|
+
// Two title-matching notes plus one body-only note. Comparing literal
|
|
54
|
+
// mode (boosted) against advanced mode with the same plain-word query
|
|
55
|
+
// (identical FTS5 syntax, so identical underlying bm25 order, but no
|
|
56
|
+
// boost pass) isolates the boost's effect: within the title-match
|
|
57
|
+
// tier, the boost must preserve whatever relative order the
|
|
58
|
+
// underlying relevance ranking already gave — a stable sort, not a
|
|
59
|
+
// fresh comparator that could reshuffle ties.
|
|
60
|
+
await store.createNote("Project Widget\nfirst, mentions widget again for relevance", { path: "w1" });
|
|
61
|
+
await store.createNote("Project Widget\nsecond", { path: "w2" });
|
|
62
|
+
await store.createNote("Unrelated\nbody-only mention of widget project", { path: "body-only" });
|
|
63
|
+
|
|
64
|
+
const boosted = await store.searchNotes("widget project");
|
|
65
|
+
const rawOrder = await store.searchNotes("widget project", { mode: "advanced" });
|
|
66
|
+
|
|
67
|
+
const titleTierBoosted = boosted.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
|
|
68
|
+
const titleTierRaw = rawOrder.map((n) => n.path).filter((p) => p === "w1" || p === "w2");
|
|
69
|
+
expect(titleTierBoosted).toEqual(titleTierRaw);
|
|
70
|
+
// The title-match tier precedes the body-only note in the boosted result.
|
|
71
|
+
expect(boosted.map((n) => n.path).indexOf("body-only")).toBeGreaterThan(
|
|
72
|
+
Math.max(...["w1", "w2"].map((p) => boosted.map((n) => n.path).indexOf(p))),
|
|
73
|
+
);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("does not disturb ordering when an explicit sort is requested", async () => {
|
|
77
|
+
await store.createNote("Zebra body-only mention of gadget", { path: "z" });
|
|
78
|
+
await store.createNote("Gadget Launch\nfirst line title match", { path: "a" });
|
|
79
|
+
|
|
80
|
+
const asc = await store.searchNotes("gadget", { sort: "asc" });
|
|
81
|
+
// created_at ASC means "z" (created first) comes before "a" — the
|
|
82
|
+
// title-boost must NOT override an explicit sort.
|
|
83
|
+
expect(asc.map((n) => n.path)).toEqual(["z", "a"]);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("does not apply the boost under search_mode: advanced (raw FTS5 syntax, not naively tokenizable)", async () => {
|
|
87
|
+
await store.createNote("Body only mentions gadget in passing text here", { path: "body-only" });
|
|
88
|
+
await store.createNote("Gadget\nfirst line title match", { path: "title-match" });
|
|
89
|
+
|
|
90
|
+
const advanced = await store.searchNotes("gadget", { mode: "advanced" });
|
|
91
|
+
const literal = await store.searchNotes("gadget");
|
|
92
|
+
|
|
93
|
+
// Literal mode boosts the title match to the front.
|
|
94
|
+
expect(literal[0].path).toBe("title-match");
|
|
95
|
+
// Advanced mode is unaffected by the boost — order comes from bm25
|
|
96
|
+
// relevance alone (title-match still likely ranks first via the
|
|
97
|
+
// EXISTING path-weighted bm25, but that's a different mechanism; the
|
|
98
|
+
// test here just confirms advanced mode doesn't throw and returns
|
|
99
|
+
// both notes, not that ordering matches/mismatches literal mode).
|
|
100
|
+
expect(advanced.map((n) => n.path).sort()).toEqual(["body-only", "title-match"]);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("a query with no results is unaffected (empty array in, empty array out)", async () => {
|
|
104
|
+
const hits = await store.searchNotes("nonexistent_term_xyz");
|
|
105
|
+
expect(hits).toEqual([]);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("boosts on the post-frontmatter title line, not the `---` delimiter (shares computeDisplayTitle's frontmatter skip)", async () => {
|
|
109
|
+
// Direct-create misuse path: raw frontmatter-bearing content. Without
|
|
110
|
+
// the frontmatter skip, the derived "title" would be the literal
|
|
111
|
+
// string "---", which never matches any query term, so this note
|
|
112
|
+
// would wrongly land in the body-only tier.
|
|
113
|
+
await store.createNote(
|
|
114
|
+
"---\ntitle: irrelevant\n---\n# Budget Review\nQ3 numbers",
|
|
115
|
+
{ path: "frontmatter-led" },
|
|
116
|
+
);
|
|
117
|
+
await store.createNote(
|
|
118
|
+
"Weekly Standup\nsomewhere we discuss the budget in passing",
|
|
119
|
+
{ path: "body-only" },
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
const hits = await store.searchNotes("budget review");
|
|
123
|
+
expect(hits[0].path).toBe("frontmatter-led");
|
|
124
|
+
});
|
|
125
|
+
});
|