@retiform/paperless-mcp 2.3.6 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +301 -4
  2. package/build/api/PaperlessAPI.js +1 -6
  3. package/build/search/build-outcome.d.ts +28 -0
  4. package/build/search/build-outcome.js +31 -0
  5. package/build/search/chunk.d.ts +16 -0
  6. package/build/search/chunk.js +64 -0
  7. package/build/search/config.d.ts +120 -0
  8. package/build/search/config.js +264 -0
  9. package/build/search/embedder.d.ts +346 -0
  10. package/build/search/embedder.js +734 -0
  11. package/build/search/rank.d.ts +11 -0
  12. package/build/search/rank.js +87 -0
  13. package/build/search/store.d.ts +406 -0
  14. package/build/search/store.js +847 -0
  15. package/build/search/sync.d.ts +204 -0
  16. package/build/search/sync.js +819 -0
  17. package/build/search/webhook.d.ts +193 -0
  18. package/build/search/webhook.js +445 -0
  19. package/build/server.d.ts +29 -0
  20. package/build/server.js +77 -0
  21. package/build/tools/documents.d.ts +13 -3
  22. package/build/tools/documents.js +23 -6
  23. package/build/tools/search.d.ts +518 -0
  24. package/build/tools/search.js +1199 -0
  25. package/build/tools/utils/middlewares.d.ts +5 -0
  26. package/build/tools/utils/middlewares.js +282 -9
  27. package/package.json +9 -3
  28. package/paperless-mcp.dxt +0 -0
  29. package/build/api/documentEnhancer.test.d.ts +0 -1
  30. package/build/api/documentEnhancer.test.js +0 -80
  31. package/build/manifest.test.d.ts +0 -1
  32. package/build/manifest.test.js +0 -66
  33. package/build/resources/documents.test.d.ts +0 -1
  34. package/build/resources/documents.test.js +0 -124
  35. package/build/server.test.d.ts +0 -1
  36. package/build/server.test.js +0 -54
  37. package/build/tools/boundedContent.test.d.ts +0 -1
  38. package/build/tools/boundedContent.test.js +0 -565
  39. package/build/tools/documents.test.d.ts +0 -1
  40. package/build/tools/documents.test.js +0 -658
  41. package/build/tools/mail.test.d.ts +0 -1
  42. package/build/tools/mail.test.js +0 -212
  43. package/build/tools/notes.test.d.ts +0 -1
  44. package/build/tools/notes.test.js +0 -177
  45. package/build/tools/schemaCompat.test.d.ts +0 -1
  46. package/build/tools/schemaCompat.test.js +0 -59
  47. package/build/tools/utils/monetary.test.d.ts +0 -1
  48. package/build/tools/utils/monetary.test.js +0 -25
  49. package/build/tools/utils/resourceUri.test.d.ts +0 -1
  50. package/build/tools/utils/resourceUri.test.js +0 -50
  51. package/build/tools/utils/selectFields.test.d.ts +0 -1
  52. package/build/tools/utils/selectFields.test.js +0 -164
@@ -0,0 +1,847 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.SearchIndex = exports.DeleteCountError = exports.LexicalUnavailableError = exports.SchemaIncompatibleError = exports.VECTOR_CANDIDATE_CEILING = exports.HAS_FTS5 = exports.CHUNK_ROWID_STRIDE = exports.EMBEDDING_DIMENSIONS = exports.SCHEMA_VERSION = void 0;
7
+ exports.chunkRowid = chunkRowid;
8
+ exports.decodeChunkRowid = decodeChunkRowid;
9
+ exports.toFtsMatch = toFtsMatch;
10
+ const node_fs_1 = require("node:fs");
11
+ const node_path_1 = require("node:path");
12
+ const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
13
+ const sqlite_vec_1 = require("sqlite-vec");
14
+ const config_1 = require("./config");
15
+ Object.defineProperty(exports, "CHUNK_ROWID_STRIDE", { enumerable: true, get: function () { return config_1.CHUNK_ROWID_STRIDE; } });
16
+ const embedder_1 = require("./embedder");
17
+ /**
18
+ * The vector + lexical index for the Paperless corpus, in one SQLite file.
19
+ *
20
+ * This is a REWRITE, not a port. The reference implementation
21
+ * (`jaimemachado/paperless-mcp@50f7bee:src/search/store.ts`) is Postgres with
22
+ * pgvector: `public.vector(1024)`, HNSW over `halfvec`, `to_tsvector('simple',
23
+ * public.unaccent(...))`, GIN on a stored `tsvector`, and
24
+ * `SET LOCAL hnsw.iterative_scan`. None of that survives to SQLite, and none of
25
+ * it should — this store exists so the corpus can be searched without a second
26
+ * web app and without a database server.
27
+ *
28
+ * ── Schema ────────────────────────────────────────────────────────────────
29
+ * documents id, modified, title, chunk_count, truncated, metadata_at
30
+ * chunks doc_id, position, kind, text (PK: doc_id, position)
31
+ * vec_chunks embedding float[768] distance_metric=cosine
32
+ * fts FTS5 over chunks.text, contentless, rowid = chunk rowid
33
+ * index_meta key/value: schema_version, dimensions, model, prefix_mode,
34
+ * plus the sync watermark (watermark_count, watermark_modified,
35
+ * watermark_at) — see Watermark below
36
+ *
37
+ * ── The five measured traps, each handled explicitly ──────────────────────
38
+ * Every one of these was reproduced on this box under the gateway child's
39
+ * runtime (node v24.21.0, better-sqlite3 13.2.0, sqlite-vec 0.1.9) before a
40
+ * line of this file was written. Two of the five turned out differently from
41
+ * the recon notes, and the notes are corrected here rather than in a drawer.
42
+ *
43
+ * 1. ROWID PARAMETERS MUST BE CAST. `insert into v(rowid, embedding) values
44
+ * (?, ?)` fails with `SQLITE_ERROR: Only integers are allows for primary
45
+ * key values on v` while both the literal form and `CAST(? AS INTEGER)`
46
+ * succeed. Verified: `k = ?` does NOT need the cast (it worked at
47
+ * `k = ?` bound to 2), so the cast is applied to rowid contexts only, not
48
+ * sprayed everywhere.
49
+ *
50
+ * 2. `changes` IS RELIABLE — the recon note was WRONG about this. The note
51
+ * recorded `delete from v where rowid = ? [5]` returning `changes: 0` and
52
+ * concluded that a bound-rowid delete silently no-ops. Re-measured five
53
+ * ways: the row simply did not exist. A bound delete against an EXISTING
54
+ * rowid returns `changes: 1` for a number, a float, a string, and a bigint
55
+ * alike, and a range delete returns the true count (3 of 3). So `changes` is
56
+ * a real instrument, and the discipline it buys is the right one anyway —
57
+ * ASSERT ON IT, never on the absence of a thrown error — but the mechanism
58
+ * in the note ("vec0 silently ignores bound rowids") does not exist.
59
+ *
60
+ * 3. NO `INSERT OR REPLACE` ON vec0. Verified: a duplicate rowid throws
61
+ * `UNIQUE constraint failed on v primary key`, and `insert or replace`
62
+ * throws the same. A replace is DELETE-then-INSERT inside one transaction.
63
+ * (The cascade/rollback path was verified: a vec0 insert inside an explicit
64
+ * transaction disappears on ROLLBACK.)
65
+ *
66
+ * 4. `rowid IN (…)` COMPOSES WITH `MATCH`. Verified. This store does not
67
+ * currently push visibility filters into the kNN statement, because the
68
+ * caller-side filter belongs to the tool lap, but the mechanism is
69
+ * available and the note is accurate.
70
+ *
71
+ * 5. DEFAULT METRIC IS L2 — `distance_metric=cosine` is declared explicitly.
72
+ * Verified: cosine gives self 0 and orthogonal 0.50098 (= 1 − cos 90°),
73
+ * while a default table gives self 0 and orthogonal 16 for the same unit
74
+ * vectors. Similarity is `1 − distance`, and `vectorHits` returns the
75
+ * similarity, never the raw distance.
76
+ *
77
+ * ── One further finding, which changed the schema ─────────────────────────
78
+ * A rowid of `doc_id * 1001 + position` — the shape the recon notes proposed —
79
+ * COLLIDES as soon as `MAX_CHUNKS` exceeds 1000. With 4001 rows per document
80
+ * the arithmetic produces 15,000 collisions across just six documents:
81
+ * (doc 1, pos 1001) and (doc 2, pos 0) both map to 2002. Since `MAX_CHUNKS`
82
+ * is being raised to 4000, the stride is 8192 instead: a power of two (so the
83
+ * decode is a mask rather than a division that has to be exactly right), and
84
+ * comfortably above the maximum chunk count, with zero collisions verified at
85
+ * 4001 rows per document.
86
+ */
87
+ exports.SCHEMA_VERSION = 1;
88
+ exports.EMBEDDING_DIMENSIONS = 768;
89
+ /** Whether FTS5 is compiled into the bundled SQLite. Measured, not assumed. */
90
+ exports.HAS_FTS5 = detectFts5();
91
+ /**
92
+ * The hard ceiling on kNN candidates for one query.
93
+ *
94
+ * It is **4096 because sqlite-vec refuses more**, not because that is a good
95
+ * number: `k value in knn query too large, provided 6400 and the limit is 4096`
96
+ * is the measured error, and it fires as a hard SQLITE_ERROR rather than
97
+ * clamping silently. Discovered by writing this widening loop against a ceiling
98
+ * of 20,000 and watching eleven previously-green tests fail -- which is the
99
+ * argument for keeping the number here, derived from the constraint, rather
100
+ * than somewhere in the widening logic.
101
+ *
102
+ * ⚠️ THE CONSEQUENCE, stated because it is a real limit on defect 5's fix: with
103
+ * a 4,000-chunk document occupying the top of the ranking, a query can find at
104
+ * most `4096 - 4000 + 1 = 97` distinct documents no matter how far it widens.
105
+ * That is far more than any caller's `limit` (MAX_LIMIT is 50), so the widening
106
+ * does its job; but it is a ceiling, not an absence of one.
107
+ */
108
+ exports.VECTOR_CANDIDATE_CEILING = 4096;
109
+ function detectFts5() {
110
+ const probe = new better_sqlite3_1.default(":memory:");
111
+ try {
112
+ const options = probe
113
+ .prepare("pragma compile_options")
114
+ .all();
115
+ const compileFlags = options.map((row) => row.compile_options.toUpperCase());
116
+ if (!compileFlags.some((flag) => flag.includes("ENABLE_FTS5")))
117
+ return false;
118
+ probe.exec("create virtual table fts5_probe using fts5(x)");
119
+ probe.exec("drop table fts5_probe");
120
+ return true;
121
+ }
122
+ catch (_a) {
123
+ return false;
124
+ }
125
+ finally {
126
+ probe.close();
127
+ }
128
+ }
129
+ class SchemaIncompatibleError extends Error {
130
+ constructor(message) {
131
+ super(message);
132
+ this.name = "SchemaIncompatibleError";
133
+ }
134
+ }
135
+ exports.SchemaIncompatibleError = SchemaIncompatibleError;
136
+ /**
137
+ * Thrown when the lexical half is asked for on a build without FTS5.
138
+ *
139
+ * Returning an empty array would be indistinguishable from "the query matched
140
+ * nothing", which would let the ranker believe it had searched lexically and
141
+ * found silence. That is the silent degradation the plan forbids, so the absence
142
+ * is a refusal the caller must handle, not a quiet empty list.
143
+ */
144
+ class LexicalUnavailableError extends Error {
145
+ constructor() {
146
+ super("This build of SQLite has no FTS5 module, so lexical search is " +
147
+ "unavailable. Check `HAS_FTS5` or `counts().ftsRows === null` before " +
148
+ "calling textHits if you want to degrade gracefully.");
149
+ this.name = "LexicalUnavailableError";
150
+ }
151
+ }
152
+ exports.LexicalUnavailableError = LexicalUnavailableError;
153
+ class DeleteCountError extends Error {
154
+ constructor(message) {
155
+ super(message);
156
+ this.name = "DeleteCountError";
157
+ }
158
+ }
159
+ exports.DeleteCountError = DeleteCountError;
160
+ /** Encode a chunk's rowid. Verified collision-free for 8191 chunks per doc. */
161
+ function chunkRowid(docId, position) {
162
+ if (!Number.isInteger(docId) || docId < 1) {
163
+ throw new Error(`docId must be a positive integer, not ${docId}.`);
164
+ }
165
+ if (!Number.isInteger(position) || position < 0) {
166
+ throw new Error(`position must be a non-negative integer, not ${position}.`);
167
+ }
168
+ if (position >= config_1.CHUNK_ROWID_STRIDE) {
169
+ throw new Error(`position ${position} does not fit the rowid stride ` +
170
+ `${config_1.CHUNK_ROWID_STRIDE}. Raise CHUNK_ROWID_STRIDE before indexing a ` +
171
+ `document this long.`);
172
+ }
173
+ return docId * config_1.CHUNK_ROWID_STRIDE + position;
174
+ }
175
+ function decodeChunkRowid(rowid) {
176
+ return {
177
+ docId: Math.floor(rowid / config_1.CHUNK_ROWID_STRIDE),
178
+ position: rowid % config_1.CHUNK_ROWID_STRIDE,
179
+ };
180
+ }
181
+ function toBlob(embedding) {
182
+ if (embedding.length !== exports.EMBEDDING_DIMENSIONS) {
183
+ throw new Error(`Embedding has ${embedding.length} dimensions; the index stores ` +
184
+ `${exports.EMBEDDING_DIMENSIONS}.`);
185
+ }
186
+ const values = Float32Array.from(embedding);
187
+ return Buffer.from(values.buffer, values.byteOffset, values.byteLength);
188
+ }
189
+ const SCHEMA_DDL = `
190
+ CREATE TABLE IF NOT EXISTS documents (
191
+ id INTEGER PRIMARY KEY,
192
+ modified TEXT NOT NULL,
193
+ title TEXT NOT NULL,
194
+ chunk_count INTEGER NOT NULL,
195
+ truncated INTEGER NOT NULL DEFAULT 0,
196
+ metadata_at TEXT NOT NULL
197
+ );
198
+
199
+ CREATE TABLE IF NOT EXISTS chunks (
200
+ doc_id INTEGER NOT NULL,
201
+ position INTEGER NOT NULL,
202
+ kind TEXT NOT NULL CHECK (kind IN ('metadata', 'content')),
203
+ text TEXT NOT NULL,
204
+ PRIMARY KEY (doc_id, position)
205
+ );
206
+
207
+ CREATE INDEX IF NOT EXISTS chunks_doc_id ON chunks (doc_id);
208
+
209
+ CREATE TABLE IF NOT EXISTS index_meta (
210
+ key TEXT PRIMARY KEY,
211
+ value TEXT NOT NULL
212
+ );
213
+ `;
214
+ const FTS_DDL = `
215
+ CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
216
+ text,
217
+ content='',
218
+ contentless_delete=1,
219
+ tokenize='unicode61 remove_diacritics 2'
220
+ );
221
+ `;
222
+ const VEC_DDL = `
223
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
224
+ embedding float[${exports.EMBEDDING_DIMENSIONS}] distance_metric=cosine
225
+ );
226
+ `;
227
+ /**
228
+ * A lazily-opened index.
229
+ *
230
+ * Lazy is not tidiness — it is a named risk in the plan. The gateway launches
231
+ * this package's child at startup, and opening a ~1 GB SQLite file (plus
232
+ * loading the vec0 extension, which is a dynamic shared object) at module
233
+ * import time could exceed the gateway's child-startup tolerance. `open()` is
234
+ * therefore explicit, `close()` is explicit, and no code in this module runs
235
+ * on import beyond the FTS5 probe, which is a `:memory:` connection that is
236
+ * closed immediately.
237
+ */
238
+ class SearchIndex {
239
+ constructor(path, options = {}) {
240
+ var _a;
241
+ this.path = path;
242
+ this.options = options;
243
+ this.hasFts5 = (_a = options.fts5) !== null && _a !== void 0 ? _a : exports.HAS_FTS5;
244
+ }
245
+ /**
246
+ * Whether the lexical half can actually run in this build.
247
+ *
248
+ * Exposed because the caller's "is anything searchable at all" question is a
249
+ * question about CAPABILITY, not about whether a list came back empty. Those
250
+ * are different facts and conflating them produced a false statement of
251
+ * mechanism: a legitimate no-match with FTS5 present was reported as
252
+ * "no lexical fallback exists".
253
+ */
254
+ get lexicalAvailable() {
255
+ return this.hasFts5;
256
+ }
257
+ get isOpen() {
258
+ return this.db !== undefined;
259
+ }
260
+ /** Open the database, creating and migrating it when writable. */
261
+ open() {
262
+ if (this.db)
263
+ return this.db;
264
+ if (!this.options.readonly) {
265
+ (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(this.path), { recursive: true });
266
+ }
267
+ // better-sqlite3 validates the PRESENCE of the key, not just its value:
268
+ // passing { readonly: undefined } throws "Expected the readonly option to
269
+ // be a boolean" rather than defaulting. Measured. So the key is omitted
270
+ // entirely unless it is explicitly true.
271
+ const db = this.options.readonly
272
+ ? new better_sqlite3_1.default(this.path, { readonly: true })
273
+ : new better_sqlite3_1.default(this.path);
274
+ try {
275
+ // WAL lets readers proceed while a sync writes. busy_timeout makes the
276
+ // second writer WAIT instead of failing: two writers on one sqlite file
277
+ // is a failure this arc has already hit once, and SQLITE_BUSY presents
278
+ // as a mysterious crash rather than as a lock.
279
+ if (!this.options.readonly) {
280
+ db.pragma("journal_mode = WAL");
281
+ db.pragma("synchronous = NORMAL");
282
+ }
283
+ db.pragma("busy_timeout = 10000");
284
+ db.pragma("foreign_keys = ON");
285
+ // The vec0 virtual-table type lives on the CONNECTION, not in the file,
286
+ // so this is required even for a readonly handle. Measured: without it,
287
+ // the first `prepare` on a vec0 table throws `no such module: vec0`.
288
+ (0, sqlite_vec_1.load)(db);
289
+ }
290
+ catch (err) {
291
+ db.close();
292
+ throw err;
293
+ }
294
+ if (!this.options.readonly)
295
+ this.migrate(db);
296
+ this.db = db;
297
+ if (this.options.expected) {
298
+ try {
299
+ // A FRESH index has no metadata and must stay openable: that is the
300
+ // first build, and refusing it would make `sync_embeddings` unable to
301
+ // create the thing it exists to create. `assertSchemaCompatible` is
302
+ // only meaningful once there is a stored index to disagree with.
303
+ if (this.hasMetaRows(db))
304
+ this.assertSchemaCompatible(this.options.expected);
305
+ }
306
+ catch (err) {
307
+ // Close the handle we just opened before propagating: a mismatch must
308
+ // not leave a live connection to an index this process refuses to use.
309
+ this.db = undefined;
310
+ db.close();
311
+ throw err;
312
+ }
313
+ }
314
+ return db;
315
+ }
316
+ /**
317
+ * Whether the index has ever been BUILT — i.e. carries metadata ROWS.
318
+ *
319
+ * Not "does the table exist": `migrate()` creates the table on every open, so
320
+ * a table check is true on a brand-new index and would make
321
+ * `assertSchemaCompatible` throw "no metadata, build it first" on the very
322
+ * run that exists to build it. The distinction that matters is rows, because
323
+ * rows mean vectors with a stated provenance.
324
+ */
325
+ hasMetaRows(db) {
326
+ const row = db
327
+ .prepare("SELECT count(*) AS c FROM sqlite_master WHERE type = 'table' AND name = 'index_meta'")
328
+ .get();
329
+ if (row.c === 0)
330
+ return false;
331
+ const rows = db.prepare("SELECT count(*) AS c FROM index_meta").get();
332
+ return rows.c > 0;
333
+ }
334
+ close() {
335
+ var _a;
336
+ (_a = this.db) === null || _a === void 0 ? void 0 : _a.close();
337
+ this.db = undefined;
338
+ }
339
+ handle() {
340
+ const db = this.db;
341
+ if (!db) {
342
+ throw new Error("The search index is not open. Call open() before using it.");
343
+ }
344
+ return db;
345
+ }
346
+ migrate(db) {
347
+ db.exec(SCHEMA_DDL);
348
+ if (this.hasFts5)
349
+ db.exec(FTS_DDL);
350
+ db.exec(VEC_DDL);
351
+ }
352
+ /** Read the stored metadata, or undefined on a fresh index. */
353
+ meta() {
354
+ var _a, _b;
355
+ const rows = this.handle()
356
+ .prepare("SELECT key, value FROM index_meta")
357
+ .all();
358
+ if (rows.length === 0)
359
+ return undefined;
360
+ const map = new Map(rows.map((row) => [row.key, row.value]));
361
+ return {
362
+ schemaVersion: Number(map.get("schema_version")),
363
+ dimensions: Number(map.get("dimensions")),
364
+ model: (_a = map.get("model")) !== null && _a !== void 0 ? _a : "",
365
+ prefixMode: ((_b = map.get("prefix_mode")) !== null && _b !== void 0 ? _b : ""),
366
+ };
367
+ }
368
+ /** Write the metadata for a fresh index. Refuses to overwrite silently. */
369
+ writeMeta(meta) {
370
+ const db = this.handle();
371
+ const existing = this.meta();
372
+ if (existing && existing.schemaVersion !== meta.schemaVersion) {
373
+ throw new SchemaIncompatibleError(`The index reports schema_version ${existing.schemaVersion} but this ` +
374
+ `build writes ${meta.schemaVersion}. Refusing to reuse it; delete ` +
375
+ `the index and rebuild.`);
376
+ }
377
+ const upsert = db.prepare("INSERT INTO index_meta (key, value) VALUES (?, ?) " +
378
+ "ON CONFLICT (key) DO UPDATE SET value = excluded.value");
379
+ const write = db.transaction(() => {
380
+ upsert.run("schema_version", String(meta.schemaVersion));
381
+ upsert.run("dimensions", String(meta.dimensions));
382
+ upsert.run("model", meta.model);
383
+ upsert.run("prefix_mode", meta.prefixMode);
384
+ });
385
+ write();
386
+ }
387
+ /**
388
+ * Record what the remote corpus looked like when this sync finished.
389
+ *
390
+ * Written through the same upsert as `writeMeta` and in its own transaction,
391
+ * because the two are separate facts with separate lifetimes: the identity is
392
+ * written BEFORE the work (so the index never holds vectors whose provenance
393
+ * it cannot state) while the watermark can only be written AFTER it. A single
394
+ * transaction spanning both would mean the identity could not be committed
395
+ * until the corpus had been read.
396
+ */
397
+ writeWatermark(watermark) {
398
+ const db = this.handle();
399
+ const upsert = db.prepare("INSERT INTO index_meta (key, value) VALUES (?, ?) " +
400
+ "ON CONFLICT (key) DO UPDATE SET value = excluded.value");
401
+ const write = db.transaction(() => {
402
+ var _a;
403
+ upsert.run("watermark_count", String(watermark.count));
404
+ // An empty corpus has no `max(modified)`. `null` is stored as the literal
405
+ // "null" rather than as an absent row, so "the index was synced against an
406
+ // empty corpus" stays distinguishable from "the index has never been
407
+ // synced" — the two read identically to `watermark()` otherwise, and the
408
+ // first is a real state that a caller may legitimately see.
409
+ upsert.run("watermark_modified", (_a = watermark.modified) !== null && _a !== void 0 ? _a : "null");
410
+ upsert.run("watermark_at", watermark.at);
411
+ });
412
+ write();
413
+ }
414
+ /**
415
+ * Read the stored watermark, or undefined when this index has never recorded
416
+ * one.
417
+ *
418
+ * Undefined is the honest answer for an index built before the watermark
419
+ * existed, and it must STAY honest: the caller runs the full diff in that
420
+ * case. Defaulting to `{count: 0}` would claim the corpus was empty, and the
421
+ * first sync after an upgrade would then skip on nothing having changed —
422
+ * silently, permanently.
423
+ */
424
+ watermark() {
425
+ const rows = this.handle()
426
+ .prepare("SELECT key, value FROM index_meta WHERE key IN " +
427
+ "('watermark_count', 'watermark_modified', 'watermark_at')")
428
+ .all();
429
+ if (rows.length === 0)
430
+ return undefined;
431
+ const map = new Map(rows.map((row) => [row.key, row.value]));
432
+ const count = map.get("watermark_count");
433
+ const modified = map.get("watermark_modified");
434
+ const at = map.get("watermark_at");
435
+ // All three or none. A partial set means the rows were written by different
436
+ // builds, and a half-watermark is not a measurement of anything.
437
+ if (count === undefined || modified === undefined || at === undefined) {
438
+ return undefined;
439
+ }
440
+ return {
441
+ count: Number(count),
442
+ modified: modified === "null" ? null : modified,
443
+ at,
444
+ };
445
+ }
446
+ /**
447
+ * Refuse to operate on an index whose schema or dimension we do not know.
448
+ * An unknown schema version is the one condition that must never degrade.
449
+ */
450
+ assertSchemaCompatible(expected) {
451
+ const meta = this.meta();
452
+ if (!meta) {
453
+ throw new SchemaIncompatibleError("The index has no metadata. Build it before querying it.");
454
+ }
455
+ if (meta.schemaVersion !== exports.SCHEMA_VERSION) {
456
+ throw new SchemaIncompatibleError(`The index reports schema_version ${meta.schemaVersion}, and this ` +
457
+ `build only understands ${exports.SCHEMA_VERSION}. Refusing to read a ` +
458
+ `schema it does not know — a wrong guess here returns plausible ` +
459
+ `results from the wrong vectors.`);
460
+ }
461
+ if (meta.dimensions !== exports.EMBEDDING_DIMENSIONS) {
462
+ throw new SchemaIncompatibleError(`The index stores ${meta.dimensions}-dimensional vectors and this ` +
463
+ `build writes ${exports.EMBEDDING_DIMENSIONS}.`);
464
+ }
465
+ if ((expected === null || expected === void 0 ? void 0 : expected.dimensions) !== undefined) {
466
+ if (expected.dimensions !== meta.dimensions) {
467
+ throw new SchemaIncompatibleError(`The embedder produces ${expected.dimensions} dimensions and the ` +
468
+ `index stores ${meta.dimensions}.`);
469
+ }
470
+ }
471
+ if ((expected === null || expected === void 0 ? void 0 : expected.model) !== undefined && expected.model !== meta.model) {
472
+ throw new SchemaIncompatibleError(`The index was built with model "${meta.model}" and this process is ` +
473
+ `configured for "${expected.model}". Two models put vectors in ` +
474
+ `different spaces; a mixed index is silently meaningless.`);
475
+ }
476
+ if ((expected === null || expected === void 0 ? void 0 : expected.prefixMode) !== undefined) {
477
+ (0, embedder_1.assertPrefixModeMatches)(expected.prefixMode, meta.prefixMode);
478
+ }
479
+ return meta;
480
+ }
481
+ /** `{ docId → modified }` for every indexed document. */
482
+ indexed() {
483
+ const rows = this.handle()
484
+ .prepare("SELECT id, modified FROM documents")
485
+ .all();
486
+ return new Map(rows.map((row) => [row.id, row.modified]));
487
+ }
488
+ /** `{ docId → metadataAt }`, needed by the 24 h metadata refresh. */
489
+ metadataStamps() {
490
+ const rows = this.handle()
491
+ .prepare("SELECT id, metadata_at FROM documents")
492
+ .all();
493
+ return new Map(rows.map((row) => [row.id, row.metadata_at]));
494
+ }
495
+ /**
496
+ * Delete every chunk of a document. Returns the number of vector rows removed.
497
+ *
498
+ * `changes` is asserted rather than ignored. The recon note claimed a bound
499
+ * delete silently returns 0 — re-measurement showed `changes` is accurate,
500
+ * but the discipline is worth keeping regardless: a delete that reports
501
+ * success and removes nothing is the same failure family as a `matches: 0`
502
+ * on a guessed pattern, and the instrument is what proves it landed.
503
+ */
504
+ deleteDocumentChunks(docId) {
505
+ const db = this.handle();
506
+ const removed = db
507
+ .prepare("DELETE FROM vec_chunks WHERE rowid BETWEEN CAST(? AS INTEGER) AND " +
508
+ "CAST(? AS INTEGER)")
509
+ .run(chunkRowid(docId, 0), chunkRowid(docId, config_1.CHUNK_ROWID_STRIDE - 1));
510
+ const expected = db
511
+ .prepare("SELECT count(*) AS c FROM chunks WHERE doc_id = ?")
512
+ .get(docId).c;
513
+ if (removed.changes !== expected) {
514
+ throw new DeleteCountError(`Deleting the vectors for document ${docId} removed ` +
515
+ `${removed.changes} rows but the chunk table holds ${expected}. ` +
516
+ `The vector index is out of step with the chunk table.`);
517
+ }
518
+ db.prepare("DELETE FROM chunks WHERE doc_id = ?").run(docId);
519
+ if (this.hasFts5) {
520
+ db.prepare("DELETE FROM fts WHERE rowid BETWEEN CAST(? AS INTEGER) AND " +
521
+ "CAST(? AS INTEGER)").run(chunkRowid(docId, 0), chunkRowid(docId, config_1.CHUNK_ROWID_STRIDE - 1));
522
+ }
523
+ return removed.changes;
524
+ }
525
+ /**
526
+ * Replace a document and all of its chunks atomically.
527
+ *
528
+ * A vec0 rowid cannot be overwritten — verified, a duplicate throws a UNIQUE
529
+ * constraint and `insert or replace` throws it too — so a replace is always
530
+ * a delete followed by an insert, and both must be in one transaction or a
531
+ * crashed sync leaves an index that is missing a document body while its
532
+ * `documents` row still claims it is complete.
533
+ */
534
+ replaceDocument(doc, chunks) {
535
+ const db = this.handle();
536
+ if (chunks.length !== doc.chunkCount) {
537
+ throw new Error(`Document ${doc.id} declares ${doc.chunkCount} chunks but received ` +
538
+ `${chunks.length}.`);
539
+ }
540
+ const insertDoc = db.prepare("INSERT INTO documents (id, modified, title, chunk_count, truncated, metadata_at) " +
541
+ "VALUES (?, ?, ?, ?, ?, ?) " +
542
+ "ON CONFLICT (id) DO UPDATE SET modified = excluded.modified, " +
543
+ "title = excluded.title, chunk_count = excluded.chunk_count, " +
544
+ "truncated = excluded.truncated, metadata_at = excluded.metadata_at");
545
+ const insertChunk = db.prepare("INSERT INTO chunks (doc_id, position, kind, text) VALUES (?, ?, ?, ?)");
546
+ const insertVec = db.prepare("INSERT INTO vec_chunks (rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
547
+ const insertFts = this.hasFts5
548
+ ? db.prepare("INSERT INTO fts (rowid, text) VALUES (CAST(? AS INTEGER), ?)")
549
+ : undefined;
550
+ const run = db.transaction(() => {
551
+ this.deleteDocumentChunks(doc.id);
552
+ insertDoc.run(doc.id, doc.modified, doc.title, doc.chunkCount, doc.truncated ? 1 : 0, doc.metadataAt);
553
+ chunks.forEach((chunk, position) => {
554
+ if (position >= config_1.CHUNK_ROWID_STRIDE) {
555
+ throw new Error(`Document ${doc.id} has ${chunks.length} chunks, beyond the ` +
556
+ `rowid stride ${config_1.CHUNK_ROWID_STRIDE}.`);
557
+ }
558
+ const rowid = chunkRowid(doc.id, position);
559
+ insertChunk.run(doc.id, position, chunk.kind, chunk.text);
560
+ insertVec.run(rowid, toBlob(chunk.embedding));
561
+ insertFts === null || insertFts === void 0 ? void 0 : insertFts.run(rowid, chunk.text);
562
+ });
563
+ });
564
+ run();
565
+ }
566
+ /**
567
+ * Replace only the metadata chunk (position 0) and stamp `metadata_at`.
568
+ * Used by the 24 h metadata refresh, which exists because tags,
569
+ * correspondents and document types change without touching `modified`.
570
+ */
571
+ replaceMetadata(docId, chunk, metadataAt) {
572
+ const db = this.handle();
573
+ if (chunk.kind !== "metadata") {
574
+ throw new Error("replaceMetadata requires a chunk of kind 'metadata'.");
575
+ }
576
+ const rowid = chunkRowid(docId, 0);
577
+ const update = db.prepare("UPDATE documents SET metadata_at = ? WHERE id = ?");
578
+ const insertChunk = db.prepare("INSERT INTO chunks (doc_id, position, kind, text) VALUES (?, 0, ?, ?) " +
579
+ "ON CONFLICT (doc_id, position) DO UPDATE SET kind = excluded.kind, " +
580
+ "text = excluded.text");
581
+ const insertVec = db.prepare("INSERT INTO vec_chunks (rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
582
+ const insertFts = this.hasFts5
583
+ ? db.prepare("INSERT INTO fts (rowid, text) VALUES (CAST(? AS INTEGER), ?)")
584
+ : undefined;
585
+ const deleteFts = this.hasFts5
586
+ ? db.prepare("DELETE FROM fts WHERE rowid = CAST(? AS INTEGER)")
587
+ : undefined;
588
+ const countVec = db.prepare("SELECT count(*) AS c FROM vec_chunks WHERE rowid = CAST(? AS INTEGER)");
589
+ const run = db.transaction(() => {
590
+ // The DELETE's own row count is NOT asserted: a fresh document may not yet
591
+ // have position 0 in the vector table, and `changes: 0` is legitimate
592
+ // then. What IS asserted is the post-condition -- exactly one vector row
593
+ // for this document's metadata chunk after the delete-then-insert.
594
+ //
595
+ // This assertion is the comment that used to sit here, made real. The
596
+ // comment claimed it and the code said `void del`, which is the
597
+ // comment-vs-code lie class: a reader trusts a guarantee that has no
598
+ // enforcement, and the failure it describes (a duplicated vector row
599
+ // making one document score twice) has no other detector.
600
+ db.prepare("DELETE FROM vec_chunks WHERE rowid = CAST(? AS INTEGER)").run(rowid);
601
+ // The lexical index needs the same delete-then-insert. Without it the OLD
602
+ // metadata text stays in `fts` forever and remains searchable alongside
603
+ // the new text, so one document matches two contradictory strings.
604
+ // Caught by a test, not by review: "old metadata" still hit.
605
+ deleteFts === null || deleteFts === void 0 ? void 0 : deleteFts.run(rowid);
606
+ insertChunk.run(docId, chunk.kind, chunk.text);
607
+ insertVec.run(rowid, toBlob(chunk.embedding));
608
+ insertFts === null || insertFts === void 0 ? void 0 : insertFts.run(rowid, chunk.text);
609
+ update.run(metadataAt, docId);
610
+ const after = countVec.get(rowid).c;
611
+ if (after !== 1) {
612
+ // Thrown inside the transaction, so the whole replace rolls back: a
613
+ // document whose metadata vector is duplicated is better left with its
614
+ // previous metadata than with two contradictory ones.
615
+ throw new Error(`replaceMetadata left ${after} vector rows for document ${docId}; expected exactly 1.`);
616
+ }
617
+ });
618
+ run();
619
+ }
620
+ /** Delete documents and everything belonging to them, atomically. */
621
+ deleteDocuments(ids) {
622
+ if (ids.length === 0)
623
+ return;
624
+ const db = this.handle();
625
+ const run = db.transaction(() => {
626
+ for (const id of ids)
627
+ this.deleteDocumentChunks(id);
628
+ const drop = db.prepare("DELETE FROM documents WHERE id = ?");
629
+ for (const id of ids)
630
+ drop.run(id);
631
+ });
632
+ run();
633
+ }
634
+ /**
635
+ * The stored chunks of one document, in position order.
636
+ *
637
+ * A read accessor, and it exists because `textHits`/`vectorHits` return only
638
+ * the BEST chunk per document -- correct for ranking, useless for asserting
639
+ * what a document actually contains. Without it a test can only observe the
640
+ * metadata chunk by getting lucky with a query.
641
+ */
642
+ chunksFor(docId) {
643
+ return this.handle()
644
+ .prepare("SELECT position, kind, text FROM chunks WHERE doc_id = ? ORDER BY position ASC")
645
+ .all(docId);
646
+ }
647
+ /**
648
+ * How many indexed chunks contain each term.
649
+ *
650
+ * Exists so the passage anchor can pick the MOST DISTINCTIVE query term
651
+ * instead of merely the longest. Measured on the live corpus, length is a bad
652
+ * proxy: for "what did the doctor say about the ankle" it chose `doctor`
653
+ * (1,903 chunks) over `ankle` (808) purely because it is one character
654
+ * longer, and for "the practice of a wrathful deity with a lion head" it
655
+ * chose `practice` (36,459 chunks) over `wrathful` (4,657). Rarity agreed
656
+ * with a human read in 3 of 3 cases; length in 0 of 3.
657
+ *
658
+ * One indexed lookup per term, and `toFtsMatch` already caps a query at 32
659
+ * terms, so this cannot become an unbounded scan. A term absent from the
660
+ * index reports 0 rather than throwing -- "not in the corpus" is a legitimate
661
+ * and very informative answer here.
662
+ */
663
+ termChunkCounts(terms) {
664
+ const out = new Map();
665
+ const db = this.handle();
666
+ if (!this.hasFts5)
667
+ return out;
668
+ const stmt = db.prepare("SELECT count(*) AS c FROM fts WHERE fts MATCH ?");
669
+ for (const term of new Set(terms)) {
670
+ try {
671
+ out.set(term, stmt.get(`"${term.replace(/"/g, '""')}"`).c);
672
+ }
673
+ catch (_a) {
674
+ // A term FTS5 cannot parse is simply unscored; the anchor falls back
675
+ // to length for it rather than failing the whole search.
676
+ }
677
+ }
678
+ return out;
679
+ }
680
+ /**
681
+ * How many indexed documents were cut off at the chunk cap.
682
+ *
683
+ * Exposed because truncation is the one index fact that changes what a caller
684
+ * can conclude from a miss: a truncated document lost its tail, so "no such
685
+ * text" may be a false negative rather than an absence.
686
+ */
687
+ truncatedCount() {
688
+ return this.handle().prepare("SELECT count(*) AS c FROM documents WHERE truncated = 1").get().c;
689
+ }
690
+ counts() {
691
+ const db = this.handle();
692
+ const one = (sql) => db.prepare(sql).get().c;
693
+ return {
694
+ documents: one("SELECT count(*) AS c FROM documents"),
695
+ chunks: one("SELECT count(*) AS c FROM chunks"),
696
+ vectors: one("SELECT count(*) AS c FROM vec_chunks"),
697
+ ftsRows: this.hasFts5 ? one("SELECT count(*) AS c FROM fts") : null,
698
+ };
699
+ }
700
+ /**
701
+ * The best chunk of each document by vector similarity.
702
+ *
703
+ * Per-document dedupe is deliberate: a chunk list lets one long document fill
704
+ * the result set, and a document that matches only by meaning then drops out.
705
+ * The reference implementation does this in Postgres with `DISTINCT ON`; here
706
+ * it is a window function over the kNN result.
707
+ *
708
+ * `score` is `1 − distance` — a similarity. The raw `distance` is never
709
+ * handed to a caller as a score, because on a cosine table 0 means identical
710
+ * and 1 means orthogonal, which reads exactly backwards.
711
+ */
712
+ vectorHits(embedding, limit) {
713
+ const db = this.handle();
714
+ if (limit <= 0)
715
+ return [];
716
+ // A FIXED over-fetch is not enough, and the reason is arithmetic rather
717
+ // than a matter of degree. This store returns one hit per document, so `k`
718
+ // candidates containing D distinct documents can only ever yield D results.
719
+ // With `max(limit*20, 200)` and a 4,000-chunk document, a caller asking for
720
+ // 10 got **1** -- all 200 (or 800) nearest rows belonged to that document,
721
+ // measured against a corpus of 60 documents. The live corpus has 21
722
+ // documents over 2M chars, so this is a real scale risk, not a corner case.
723
+ //
724
+ // So the budget WIDENS until either `limit` distinct documents are in hand
725
+ // or the pool is exhausted. Doubling keeps the number of round trips
726
+ // logarithmic in the worst case, and VECTOR_CANDIDATE_CEILING bounds it so
727
+ // a corpus with one enormous document cannot make a query unbounded.
728
+ // Clamped from the start: `limit * 20` at MAX_LIMIT 50 is 1000, but a
729
+ // caller is not the only input -- `runSemanticSearch` multiplies by 4
730
+ // before it gets here, so an unclamped initial value could already exceed
731
+ // sqlite-vec's k cap before the widening loop ran once.
732
+ let candidates = Math.min(Math.max(limit * 20, 200), exports.VECTOR_CANDIDATE_CEILING);
733
+ let hits = [];
734
+ for (;;) {
735
+ hits = this.vectorHitsAt(embedding, candidates, limit);
736
+ // `hits.length` is a count of DISTINCT documents (per-document dedupe),
737
+ // so this compares the right quantity against the right target.
738
+ if (hits.length >= limit || candidates >= exports.VECTOR_CANDIDATE_CEILING)
739
+ return hits;
740
+ candidates = Math.min(candidates * 2, exports.VECTOR_CANDIDATE_CEILING);
741
+ }
742
+ }
743
+ /** One kNN round trip at a given candidate budget. */
744
+ vectorHitsAt(embedding, candidates, limit) {
745
+ const db = this.handle();
746
+ const sql = `
747
+ WITH nearest AS (
748
+ SELECT rowid, distance
749
+ FROM vec_chunks
750
+ WHERE embedding MATCH CAST(? AS BLOB) AND k = CAST(? AS INTEGER)
751
+ ), ranked AS (
752
+ SELECT rowid, distance,
753
+ (rowid / ${config_1.CHUNK_ROWID_STRIDE}) AS doc_id,
754
+ ROW_NUMBER() OVER (
755
+ PARTITION BY (rowid / ${config_1.CHUNK_ROWID_STRIDE})
756
+ ORDER BY distance ASC, rowid ASC
757
+ ) AS rank_in_doc
758
+ FROM nearest
759
+ )
760
+ SELECT c.doc_id AS documentId, c.position AS position, c.text AS text,
761
+ 1.0 - r.distance AS score
762
+ FROM ranked r
763
+ JOIN chunks c
764
+ ON c.doc_id = r.doc_id
765
+ AND c.position = (r.rowid % ${config_1.CHUNK_ROWID_STRIDE})
766
+ WHERE r.rank_in_doc = 1
767
+ ORDER BY r.distance ASC, c.doc_id ASC
768
+ LIMIT CAST(? AS INTEGER)
769
+ `;
770
+ return db
771
+ .prepare(sql)
772
+ .all(toBlob(embedding), candidates, limit);
773
+ }
774
+ /**
775
+ * The best chunk of each document by BM25.
776
+ *
777
+ * FTS5's `bm25()` returns a NEGATIVE score where more negative is better, so
778
+ * it is negated to a positive "higher is better" value. It is NOT comparable
779
+ * to the vector similarity in magnitude — only in order — which is why the
780
+ * ranker fuses the two lists by rank rather than by score.
781
+ */
782
+ textHits(query, limit) {
783
+ if (!this.hasFts5)
784
+ throw new LexicalUnavailableError();
785
+ const match = toFtsMatch(query);
786
+ if (!match)
787
+ return [];
788
+ const db = this.handle();
789
+ const sql = `
790
+ WITH matched AS (
791
+ SELECT rowid, bm25(fts) AS bm
792
+ FROM fts
793
+ WHERE fts MATCH CAST(? AS TEXT)
794
+ ), ranked AS (
795
+ SELECT rowid, bm,
796
+ (rowid / ${config_1.CHUNK_ROWID_STRIDE}) AS doc_id,
797
+ ROW_NUMBER() OVER (
798
+ PARTITION BY (rowid / ${config_1.CHUNK_ROWID_STRIDE})
799
+ ORDER BY bm ASC, rowid ASC
800
+ ) AS rank_in_doc
801
+ FROM matched
802
+ )
803
+ SELECT c.doc_id AS documentId, c.position AS position, c.text AS text,
804
+ -r.bm AS score
805
+ FROM ranked r
806
+ JOIN chunks c
807
+ ON c.doc_id = r.doc_id
808
+ AND c.position = (r.rowid % ${config_1.CHUNK_ROWID_STRIDE})
809
+ WHERE r.rank_in_doc = 1
810
+ ORDER BY r.bm ASC, c.doc_id ASC
811
+ LIMIT CAST(? AS INTEGER)
812
+ `;
813
+ return db.prepare(sql).all(match, limit);
814
+ }
815
+ }
816
+ exports.SearchIndex = SearchIndex;
817
+ /**
818
+ * Turn a user's words into an FTS5 MATCH expression.
819
+ *
820
+ * Raw user text is NOT a valid MATCH expression — measured: `c++ (test)` throws
821
+ * `fts5: syntax error near "+"`, `a-b` throws `no such column: b`, and an empty
822
+ * string throws. So the query is tokenised into bare terms and every term is
823
+ * quoted, which makes FTS5 treat it as a literal rather than as query syntax.
824
+ * Terms are OR-ed because a user typing three words wants documents matching
825
+ * any of them ranked by BM25, not only those matching all three.
826
+ *
827
+ * `remove_diacritics 2` means `titulo` matches `Título`; verified both
828
+ * directions on this build.
829
+ */
830
+ function toFtsMatch(query) {
831
+ const terms = query
832
+ .toLowerCase()
833
+ // \p{M} is REQUIRED, not decorative. A Tibetan syllable is a base letter
834
+ // plus combining marks -- U+0FB3 and U+0F74 are category Mn, not L -- so
835
+ // splitting on [^\p{L}\p{N}_] tears "རླུང" into "ར" and "ང": two fragments,
836
+ // neither of which is the word. Measured on this build. This corpus is
837
+ // heavily Tibetan, so the bug would have made those terms unfindable
838
+ // lexically while looking like an ordinary no-match.
839
+ .split(/[^\p{L}\p{N}\p{M}_]+/u)
840
+ .filter((term) => term.length > 0);
841
+ if (terms.length === 0)
842
+ return null;
843
+ // Cap the expression: a 500-term OR is pathological and slower than the
844
+ // answer is worth. The first terms are the ones the user typed first.
845
+ const capped = terms.slice(0, 32);
846
+ return capped.map((term) => `"${term.replace(/"/g, '""')}"`).join(" OR ");
847
+ }