@retiform/paperless-mcp 2.3.5 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +10 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,847 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SearchIndex = exports.DeleteCountError = exports.LexicalUnavailableError = exports.SchemaIncompatibleError = exports.VECTOR_CANDIDATE_CEILING = exports.HAS_FTS5 = exports.CHUNK_ROWID_STRIDE = exports.EMBEDDING_DIMENSIONS = exports.SCHEMA_VERSION = void 0;
|
|
7
|
+
exports.chunkRowid = chunkRowid;
|
|
8
|
+
exports.decodeChunkRowid = decodeChunkRowid;
|
|
9
|
+
exports.toFtsMatch = toFtsMatch;
|
|
10
|
+
const node_fs_1 = require("node:fs");
|
|
11
|
+
const node_path_1 = require("node:path");
|
|
12
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
13
|
+
const sqlite_vec_1 = require("sqlite-vec");
|
|
14
|
+
const config_1 = require("./config");
|
|
15
|
+
Object.defineProperty(exports, "CHUNK_ROWID_STRIDE", { enumerable: true, get: function () { return config_1.CHUNK_ROWID_STRIDE; } });
|
|
16
|
+
const embedder_1 = require("./embedder");
|
|
17
|
+
/**
|
|
18
|
+
* The vector + lexical index for the Paperless corpus, in one SQLite file.
|
|
19
|
+
*
|
|
20
|
+
* This is a REWRITE, not a port. The reference implementation
|
|
21
|
+
* (`jaimemachado/paperless-mcp@50f7bee:src/search/store.ts`) is Postgres with
|
|
22
|
+
* pgvector: `public.vector(1024)`, HNSW over `halfvec`, `to_tsvector('simple',
|
|
23
|
+
* public.unaccent(...))`, GIN on a stored `tsvector`, and
|
|
24
|
+
* `SET LOCAL hnsw.iterative_scan`. None of that survives to SQLite, and none of
|
|
25
|
+
* it should — this store exists so the corpus can be searched without a second
|
|
26
|
+
* web app and without a database server.
|
|
27
|
+
*
|
|
28
|
+
* ── Schema ────────────────────────────────────────────────────────────────
|
|
29
|
+
* documents id, modified, title, chunk_count, truncated, metadata_at
|
|
30
|
+
* chunks doc_id, position, kind, text (PK: doc_id, position)
|
|
31
|
+
* vec_chunks embedding float[768] distance_metric=cosine
|
|
32
|
+
* fts FTS5 over chunks.text, contentless, rowid = chunk rowid
|
|
33
|
+
* index_meta key/value: schema_version, dimensions, model, prefix_mode,
|
|
34
|
+
* plus the sync watermark (watermark_count, watermark_modified,
|
|
35
|
+
* watermark_at) — see Watermark below
|
|
36
|
+
*
|
|
37
|
+
* ── The five measured traps, each handled explicitly ──────────────────────
|
|
38
|
+
* Every one of these was reproduced on this box under the gateway child's
|
|
39
|
+
* runtime (node v24.21.0, better-sqlite3 13.2.0, sqlite-vec 0.1.9) before a
|
|
40
|
+
* line of this file was written. Two of the five turned out differently from
|
|
41
|
+
* the recon notes, and the notes are corrected here rather than in a drawer.
|
|
42
|
+
*
|
|
43
|
+
* 1. ROWID PARAMETERS MUST BE CAST. `insert into v(rowid, embedding) values
|
|
44
|
+
* (?, ?)` fails with `SQLITE_ERROR: Only integers are allows for primary
|
|
45
|
+
* key values on v` while both the literal form and `CAST(? AS INTEGER)`
|
|
46
|
+
* succeed. Verified: `k = ?` does NOT need the cast (it worked at
|
|
47
|
+
* `k = ?` bound to 2), so the cast is applied to rowid contexts only, not
|
|
48
|
+
* sprayed everywhere.
|
|
49
|
+
*
|
|
50
|
+
* 2. `changes` IS RELIABLE — the recon note was WRONG about this. The note
|
|
51
|
+
* recorded `delete from v where rowid = ? [5]` returning `changes: 0` and
|
|
52
|
+
* concluded that a bound-rowid delete silently no-ops. Re-measured five
|
|
53
|
+
* ways: the row simply did not exist. A bound delete against an EXISTING
|
|
54
|
+
* rowid returns `changes: 1` for a number, a float, a string, and a bigint
|
|
55
|
+
* alike, and a range delete returns the true count (3 of 3). So `changes` is
|
|
56
|
+
* a real instrument, and the discipline it buys is the right one anyway —
|
|
57
|
+
* ASSERT ON IT, never on the absence of a thrown error — but the mechanism
|
|
58
|
+
* in the note ("vec0 silently ignores bound rowids") does not exist.
|
|
59
|
+
*
|
|
60
|
+
* 3. NO `INSERT OR REPLACE` ON vec0. Verified: a duplicate rowid throws
|
|
61
|
+
* `UNIQUE constraint failed on v primary key`, and `insert or replace`
|
|
62
|
+
* throws the same. A replace is DELETE-then-INSERT inside one transaction.
|
|
63
|
+
* (The cascade/rollback path was verified: a vec0 insert inside an explicit
|
|
64
|
+
* transaction disappears on ROLLBACK.)
|
|
65
|
+
*
|
|
66
|
+
* 4. `rowid IN (…)` COMPOSES WITH `MATCH`. Verified. This store does not
|
|
67
|
+
* currently push visibility filters into the kNN statement, because the
|
|
68
|
+
* caller-side filter belongs to the tool lap, but the mechanism is
|
|
69
|
+
* available and the note is accurate.
|
|
70
|
+
*
|
|
71
|
+
* 5. DEFAULT METRIC IS L2 — `distance_metric=cosine` is declared explicitly.
|
|
72
|
+
* Verified: cosine gives self 0 and orthogonal 0.50098 (= 1 − cos 90°),
|
|
73
|
+
* while a default table gives self 0 and orthogonal 16 for the same unit
|
|
74
|
+
* vectors. Similarity is `1 − distance`, and `vectorHits` returns the
|
|
75
|
+
* similarity, never the raw distance.
|
|
76
|
+
*
|
|
77
|
+
* ── One further finding, which changed the schema ─────────────────────────
|
|
78
|
+
* A rowid of `doc_id * 1001 + position` — the shape the recon notes proposed —
|
|
79
|
+
* COLLIDES as soon as `MAX_CHUNKS` exceeds 1000. With 4001 rows per document
|
|
80
|
+
* the arithmetic produces 15,000 collisions across just six documents:
|
|
81
|
+
* (doc 1, pos 1001) and (doc 2, pos 0) both map to 2002. Since `MAX_CHUNKS`
|
|
82
|
+
* is being raised to 4000, the stride is 8192 instead: a power of two (so the
|
|
83
|
+
* decode is a mask rather than a division that has to be exactly right), and
|
|
84
|
+
* comfortably above the maximum chunk count, with zero collisions verified at
|
|
85
|
+
* 4001 rows per document.
|
|
86
|
+
*/
|
|
87
|
+
exports.SCHEMA_VERSION = 1;
|
|
88
|
+
exports.EMBEDDING_DIMENSIONS = 768;
|
|
89
|
+
/** Whether FTS5 is compiled into the bundled SQLite. Measured, not assumed. */
|
|
90
|
+
exports.HAS_FTS5 = detectFts5();
|
|
91
|
+
/**
|
|
92
|
+
* The hard ceiling on kNN candidates for one query.
|
|
93
|
+
*
|
|
94
|
+
* It is **4096 because sqlite-vec refuses more**, not because that is a good
|
|
95
|
+
* number: `k value in knn query too large, provided 6400 and the limit is 4096`
|
|
96
|
+
* is the measured error, and it fires as a hard SQLITE_ERROR rather than
|
|
97
|
+
* clamping silently. Discovered by writing this widening loop against a ceiling
|
|
98
|
+
* of 20,000 and watching eleven previously-green tests fail -- which is the
|
|
99
|
+
* argument for keeping the number here, derived from the constraint, rather
|
|
100
|
+
* than somewhere in the widening logic.
|
|
101
|
+
*
|
|
102
|
+
* ⚠️ THE CONSEQUENCE, stated because it is a real limit on defect 5's fix: with
|
|
103
|
+
* a 4,000-chunk document occupying the top of the ranking, a query can find at
|
|
104
|
+
* most `4096 - 4000 + 1 = 97` distinct documents no matter how far it widens.
|
|
105
|
+
* That is far more than any caller's `limit` (MAX_LIMIT is 50), so the widening
|
|
106
|
+
* does its job; but it is a ceiling, not an absence of one.
|
|
107
|
+
*/
|
|
108
|
+
exports.VECTOR_CANDIDATE_CEILING = 4096;
|
|
109
|
+
function detectFts5() {
|
|
110
|
+
const probe = new better_sqlite3_1.default(":memory:");
|
|
111
|
+
try {
|
|
112
|
+
const options = probe
|
|
113
|
+
.prepare("pragma compile_options")
|
|
114
|
+
.all();
|
|
115
|
+
const compileFlags = options.map((row) => row.compile_options.toUpperCase());
|
|
116
|
+
if (!compileFlags.some((flag) => flag.includes("ENABLE_FTS5")))
|
|
117
|
+
return false;
|
|
118
|
+
probe.exec("create virtual table fts5_probe using fts5(x)");
|
|
119
|
+
probe.exec("drop table fts5_probe");
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
catch (_a) {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
probe.close();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
class SchemaIncompatibleError extends Error {
|
|
130
|
+
constructor(message) {
|
|
131
|
+
super(message);
|
|
132
|
+
this.name = "SchemaIncompatibleError";
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
exports.SchemaIncompatibleError = SchemaIncompatibleError;
|
|
136
|
+
/**
|
|
137
|
+
* Thrown when the lexical half is asked for on a build without FTS5.
|
|
138
|
+
*
|
|
139
|
+
* Returning an empty array would be indistinguishable from "the query matched
|
|
140
|
+
* nothing", which would let the ranker believe it had searched lexically and
|
|
141
|
+
* found silence. That is the silent degradation the plan forbids, so the absence
|
|
142
|
+
* is a refusal the caller must handle, not a quiet empty list.
|
|
143
|
+
*/
|
|
144
|
+
class LexicalUnavailableError extends Error {
|
|
145
|
+
constructor() {
|
|
146
|
+
super("This build of SQLite has no FTS5 module, so lexical search is " +
|
|
147
|
+
"unavailable. Check `HAS_FTS5` or `counts().ftsRows === null` before " +
|
|
148
|
+
"calling textHits if you want to degrade gracefully.");
|
|
149
|
+
this.name = "LexicalUnavailableError";
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
exports.LexicalUnavailableError = LexicalUnavailableError;
|
|
153
|
+
class DeleteCountError extends Error {
|
|
154
|
+
constructor(message) {
|
|
155
|
+
super(message);
|
|
156
|
+
this.name = "DeleteCountError";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
exports.DeleteCountError = DeleteCountError;
|
|
160
|
+
/** Encode a chunk's rowid. Verified collision-free for 8191 chunks per doc. */
|
|
161
|
+
function chunkRowid(docId, position) {
|
|
162
|
+
if (!Number.isInteger(docId) || docId < 1) {
|
|
163
|
+
throw new Error(`docId must be a positive integer, not ${docId}.`);
|
|
164
|
+
}
|
|
165
|
+
if (!Number.isInteger(position) || position < 0) {
|
|
166
|
+
throw new Error(`position must be a non-negative integer, not ${position}.`);
|
|
167
|
+
}
|
|
168
|
+
if (position >= config_1.CHUNK_ROWID_STRIDE) {
|
|
169
|
+
throw new Error(`position ${position} does not fit the rowid stride ` +
|
|
170
|
+
`${config_1.CHUNK_ROWID_STRIDE}. Raise CHUNK_ROWID_STRIDE before indexing a ` +
|
|
171
|
+
`document this long.`);
|
|
172
|
+
}
|
|
173
|
+
return docId * config_1.CHUNK_ROWID_STRIDE + position;
|
|
174
|
+
}
|
|
175
|
+
function decodeChunkRowid(rowid) {
|
|
176
|
+
return {
|
|
177
|
+
docId: Math.floor(rowid / config_1.CHUNK_ROWID_STRIDE),
|
|
178
|
+
position: rowid % config_1.CHUNK_ROWID_STRIDE,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function toBlob(embedding) {
|
|
182
|
+
if (embedding.length !== exports.EMBEDDING_DIMENSIONS) {
|
|
183
|
+
throw new Error(`Embedding has ${embedding.length} dimensions; the index stores ` +
|
|
184
|
+
`${exports.EMBEDDING_DIMENSIONS}.`);
|
|
185
|
+
}
|
|
186
|
+
const values = Float32Array.from(embedding);
|
|
187
|
+
return Buffer.from(values.buffer, values.byteOffset, values.byteLength);
|
|
188
|
+
}
|
|
189
|
+
const SCHEMA_DDL = `
|
|
190
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
191
|
+
id INTEGER PRIMARY KEY,
|
|
192
|
+
modified TEXT NOT NULL,
|
|
193
|
+
title TEXT NOT NULL,
|
|
194
|
+
chunk_count INTEGER NOT NULL,
|
|
195
|
+
truncated INTEGER NOT NULL DEFAULT 0,
|
|
196
|
+
metadata_at TEXT NOT NULL
|
|
197
|
+
);
|
|
198
|
+
|
|
199
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
200
|
+
doc_id INTEGER NOT NULL,
|
|
201
|
+
position INTEGER NOT NULL,
|
|
202
|
+
kind TEXT NOT NULL CHECK (kind IN ('metadata', 'content')),
|
|
203
|
+
text TEXT NOT NULL,
|
|
204
|
+
PRIMARY KEY (doc_id, position)
|
|
205
|
+
);
|
|
206
|
+
|
|
207
|
+
CREATE INDEX IF NOT EXISTS chunks_doc_id ON chunks (doc_id);
|
|
208
|
+
|
|
209
|
+
CREATE TABLE IF NOT EXISTS index_meta (
|
|
210
|
+
key TEXT PRIMARY KEY,
|
|
211
|
+
value TEXT NOT NULL
|
|
212
|
+
);
|
|
213
|
+
`;
|
|
214
|
+
const FTS_DDL = `
|
|
215
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
|
|
216
|
+
text,
|
|
217
|
+
content='',
|
|
218
|
+
contentless_delete=1,
|
|
219
|
+
tokenize='unicode61 remove_diacritics 2'
|
|
220
|
+
);
|
|
221
|
+
`;
|
|
222
|
+
const VEC_DDL = `
|
|
223
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
|
|
224
|
+
embedding float[${exports.EMBEDDING_DIMENSIONS}] distance_metric=cosine
|
|
225
|
+
);
|
|
226
|
+
`;
|
|
227
|
+
/**
|
|
228
|
+
* A lazily-opened index.
|
|
229
|
+
*
|
|
230
|
+
* Lazy is not tidiness — it is a named risk in the plan. The gateway launches
|
|
231
|
+
* this package's child at startup, and opening a ~1 GB SQLite file (plus
|
|
232
|
+
* loading the vec0 extension, which is a dynamic shared object) at module
|
|
233
|
+
* import time could exceed the gateway's child-startup tolerance. `open()` is
|
|
234
|
+
* therefore explicit, `close()` is explicit, and no code in this module runs
|
|
235
|
+
* on import beyond the FTS5 probe, which is a `:memory:` connection that is
|
|
236
|
+
* closed immediately.
|
|
237
|
+
*/
|
|
238
|
+
class SearchIndex {
|
|
239
|
+
constructor(path, options = {}) {
|
|
240
|
+
var _a;
|
|
241
|
+
this.path = path;
|
|
242
|
+
this.options = options;
|
|
243
|
+
this.hasFts5 = (_a = options.fts5) !== null && _a !== void 0 ? _a : exports.HAS_FTS5;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Whether the lexical half can actually run in this build.
|
|
247
|
+
*
|
|
248
|
+
* Exposed because the caller's "is anything searchable at all" question is a
|
|
249
|
+
* question about CAPABILITY, not about whether a list came back empty. Those
|
|
250
|
+
* are different facts and conflating them produced a false statement of
|
|
251
|
+
* mechanism: a legitimate no-match with FTS5 present was reported as
|
|
252
|
+
* "no lexical fallback exists".
|
|
253
|
+
*/
|
|
254
|
+
get lexicalAvailable() {
|
|
255
|
+
return this.hasFts5;
|
|
256
|
+
}
|
|
257
|
+
get isOpen() {
|
|
258
|
+
return this.db !== undefined;
|
|
259
|
+
}
|
|
260
|
+
/** Open the database, creating and migrating it when writable. */
|
|
261
|
+
open() {
|
|
262
|
+
if (this.db)
|
|
263
|
+
return this.db;
|
|
264
|
+
if (!this.options.readonly) {
|
|
265
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(this.path), { recursive: true });
|
|
266
|
+
}
|
|
267
|
+
// better-sqlite3 validates the PRESENCE of the key, not just its value:
|
|
268
|
+
// passing { readonly: undefined } throws "Expected the readonly option to
|
|
269
|
+
// be a boolean" rather than defaulting. Measured. So the key is omitted
|
|
270
|
+
// entirely unless it is explicitly true.
|
|
271
|
+
const db = this.options.readonly
|
|
272
|
+
? new better_sqlite3_1.default(this.path, { readonly: true })
|
|
273
|
+
: new better_sqlite3_1.default(this.path);
|
|
274
|
+
try {
|
|
275
|
+
// WAL lets readers proceed while a sync writes. busy_timeout makes the
|
|
276
|
+
// second writer WAIT instead of failing: two writers on one sqlite file
|
|
277
|
+
// is a failure this arc has already hit once, and SQLITE_BUSY presents
|
|
278
|
+
// as a mysterious crash rather than as a lock.
|
|
279
|
+
if (!this.options.readonly) {
|
|
280
|
+
db.pragma("journal_mode = WAL");
|
|
281
|
+
db.pragma("synchronous = NORMAL");
|
|
282
|
+
}
|
|
283
|
+
db.pragma("busy_timeout = 10000");
|
|
284
|
+
db.pragma("foreign_keys = ON");
|
|
285
|
+
// The vec0 virtual-table type lives on the CONNECTION, not in the file,
|
|
286
|
+
// so this is required even for a readonly handle. Measured: without it,
|
|
287
|
+
// the first `prepare` on a vec0 table throws `no such module: vec0`.
|
|
288
|
+
(0, sqlite_vec_1.load)(db);
|
|
289
|
+
}
|
|
290
|
+
catch (err) {
|
|
291
|
+
db.close();
|
|
292
|
+
throw err;
|
|
293
|
+
}
|
|
294
|
+
if (!this.options.readonly)
|
|
295
|
+
this.migrate(db);
|
|
296
|
+
this.db = db;
|
|
297
|
+
if (this.options.expected) {
|
|
298
|
+
try {
|
|
299
|
+
// A FRESH index has no metadata and must stay openable: that is the
|
|
300
|
+
// first build, and refusing it would make `sync_embeddings` unable to
|
|
301
|
+
// create the thing it exists to create. `assertSchemaCompatible` is
|
|
302
|
+
// only meaningful once there is a stored index to disagree with.
|
|
303
|
+
if (this.hasMetaRows(db))
|
|
304
|
+
this.assertSchemaCompatible(this.options.expected);
|
|
305
|
+
}
|
|
306
|
+
catch (err) {
|
|
307
|
+
// Close the handle we just opened before propagating: a mismatch must
|
|
308
|
+
// not leave a live connection to an index this process refuses to use.
|
|
309
|
+
this.db = undefined;
|
|
310
|
+
db.close();
|
|
311
|
+
throw err;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return db;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Whether the index has ever been BUILT — i.e. carries metadata ROWS.
|
|
318
|
+
*
|
|
319
|
+
* Not "does the table exist": `migrate()` creates the table on every open, so
|
|
320
|
+
* a table check is true on a brand-new index and would make
|
|
321
|
+
* `assertSchemaCompatible` throw "no metadata, build it first" on the very
|
|
322
|
+
* run that exists to build it. The distinction that matters is rows, because
|
|
323
|
+
* rows mean vectors with a stated provenance.
|
|
324
|
+
*/
|
|
325
|
+
hasMetaRows(db) {
|
|
326
|
+
const row = db
|
|
327
|
+
.prepare("SELECT count(*) AS c FROM sqlite_master WHERE type = 'table' AND name = 'index_meta'")
|
|
328
|
+
.get();
|
|
329
|
+
if (row.c === 0)
|
|
330
|
+
return false;
|
|
331
|
+
const rows = db.prepare("SELECT count(*) AS c FROM index_meta").get();
|
|
332
|
+
return rows.c > 0;
|
|
333
|
+
}
|
|
334
|
+
close() {
|
|
335
|
+
var _a;
|
|
336
|
+
(_a = this.db) === null || _a === void 0 ? void 0 : _a.close();
|
|
337
|
+
this.db = undefined;
|
|
338
|
+
}
|
|
339
|
+
handle() {
|
|
340
|
+
const db = this.db;
|
|
341
|
+
if (!db) {
|
|
342
|
+
throw new Error("The search index is not open. Call open() before using it.");
|
|
343
|
+
}
|
|
344
|
+
return db;
|
|
345
|
+
}
|
|
346
|
+
migrate(db) {
|
|
347
|
+
db.exec(SCHEMA_DDL);
|
|
348
|
+
if (this.hasFts5)
|
|
349
|
+
db.exec(FTS_DDL);
|
|
350
|
+
db.exec(VEC_DDL);
|
|
351
|
+
}
|
|
352
|
+
/** Read the stored metadata, or undefined on a fresh index. */
|
|
353
|
+
meta() {
|
|
354
|
+
var _a, _b;
|
|
355
|
+
const rows = this.handle()
|
|
356
|
+
.prepare("SELECT key, value FROM index_meta")
|
|
357
|
+
.all();
|
|
358
|
+
if (rows.length === 0)
|
|
359
|
+
return undefined;
|
|
360
|
+
const map = new Map(rows.map((row) => [row.key, row.value]));
|
|
361
|
+
return {
|
|
362
|
+
schemaVersion: Number(map.get("schema_version")),
|
|
363
|
+
dimensions: Number(map.get("dimensions")),
|
|
364
|
+
model: (_a = map.get("model")) !== null && _a !== void 0 ? _a : "",
|
|
365
|
+
prefixMode: ((_b = map.get("prefix_mode")) !== null && _b !== void 0 ? _b : ""),
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
/** Write the metadata for a fresh index. Refuses to overwrite silently. */
|
|
369
|
+
writeMeta(meta) {
|
|
370
|
+
const db = this.handle();
|
|
371
|
+
const existing = this.meta();
|
|
372
|
+
if (existing && existing.schemaVersion !== meta.schemaVersion) {
|
|
373
|
+
throw new SchemaIncompatibleError(`The index reports schema_version ${existing.schemaVersion} but this ` +
|
|
374
|
+
`build writes ${meta.schemaVersion}. Refusing to reuse it; delete ` +
|
|
375
|
+
`the index and rebuild.`);
|
|
376
|
+
}
|
|
377
|
+
const upsert = db.prepare("INSERT INTO index_meta (key, value) VALUES (?, ?) " +
|
|
378
|
+
"ON CONFLICT (key) DO UPDATE SET value = excluded.value");
|
|
379
|
+
const write = db.transaction(() => {
|
|
380
|
+
upsert.run("schema_version", String(meta.schemaVersion));
|
|
381
|
+
upsert.run("dimensions", String(meta.dimensions));
|
|
382
|
+
upsert.run("model", meta.model);
|
|
383
|
+
upsert.run("prefix_mode", meta.prefixMode);
|
|
384
|
+
});
|
|
385
|
+
write();
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Record what the remote corpus looked like when this sync finished.
|
|
389
|
+
*
|
|
390
|
+
* Written through the same upsert as `writeMeta` and in its own transaction,
|
|
391
|
+
* because the two are separate facts with separate lifetimes: the identity is
|
|
392
|
+
* written BEFORE the work (so the index never holds vectors whose provenance
|
|
393
|
+
* it cannot state) while the watermark can only be written AFTER it. A single
|
|
394
|
+
* transaction spanning both would mean the identity could not be committed
|
|
395
|
+
* until the corpus had been read.
|
|
396
|
+
*/
|
|
397
|
+
writeWatermark(watermark) {
|
|
398
|
+
const db = this.handle();
|
|
399
|
+
const upsert = db.prepare("INSERT INTO index_meta (key, value) VALUES (?, ?) " +
|
|
400
|
+
"ON CONFLICT (key) DO UPDATE SET value = excluded.value");
|
|
401
|
+
const write = db.transaction(() => {
|
|
402
|
+
var _a;
|
|
403
|
+
upsert.run("watermark_count", String(watermark.count));
|
|
404
|
+
// An empty corpus has no `max(modified)`. `null` is stored as the literal
|
|
405
|
+
// "null" rather than as an absent row, so "the index was synced against an
|
|
406
|
+
// empty corpus" stays distinguishable from "the index has never been
|
|
407
|
+
// synced" — the two read identically to `watermark()` otherwise, and the
|
|
408
|
+
// first is a real state that a caller may legitimately see.
|
|
409
|
+
upsert.run("watermark_modified", (_a = watermark.modified) !== null && _a !== void 0 ? _a : "null");
|
|
410
|
+
upsert.run("watermark_at", watermark.at);
|
|
411
|
+
});
|
|
412
|
+
write();
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Read the stored watermark, or undefined when this index has never recorded
|
|
416
|
+
* one.
|
|
417
|
+
*
|
|
418
|
+
* Undefined is the honest answer for an index built before the watermark
|
|
419
|
+
* existed, and it must STAY honest: the caller runs the full diff in that
|
|
420
|
+
* case. Defaulting to `{count: 0}` would claim the corpus was empty, and the
|
|
421
|
+
* first sync after an upgrade would then skip on nothing having changed —
|
|
422
|
+
* silently, permanently.
|
|
423
|
+
*/
|
|
424
|
+
watermark() {
|
|
425
|
+
const rows = this.handle()
|
|
426
|
+
.prepare("SELECT key, value FROM index_meta WHERE key IN " +
|
|
427
|
+
"('watermark_count', 'watermark_modified', 'watermark_at')")
|
|
428
|
+
.all();
|
|
429
|
+
if (rows.length === 0)
|
|
430
|
+
return undefined;
|
|
431
|
+
const map = new Map(rows.map((row) => [row.key, row.value]));
|
|
432
|
+
const count = map.get("watermark_count");
|
|
433
|
+
const modified = map.get("watermark_modified");
|
|
434
|
+
const at = map.get("watermark_at");
|
|
435
|
+
// All three or none. A partial set means the rows were written by different
|
|
436
|
+
// builds, and a half-watermark is not a measurement of anything.
|
|
437
|
+
if (count === undefined || modified === undefined || at === undefined) {
|
|
438
|
+
return undefined;
|
|
439
|
+
}
|
|
440
|
+
return {
|
|
441
|
+
count: Number(count),
|
|
442
|
+
modified: modified === "null" ? null : modified,
|
|
443
|
+
at,
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Refuse to operate on an index whose schema or dimension we do not know.
|
|
448
|
+
* An unknown schema version is the one condition that must never degrade.
|
|
449
|
+
*/
|
|
450
|
+
assertSchemaCompatible(expected) {
|
|
451
|
+
const meta = this.meta();
|
|
452
|
+
if (!meta) {
|
|
453
|
+
throw new SchemaIncompatibleError("The index has no metadata. Build it before querying it.");
|
|
454
|
+
}
|
|
455
|
+
if (meta.schemaVersion !== exports.SCHEMA_VERSION) {
|
|
456
|
+
throw new SchemaIncompatibleError(`The index reports schema_version ${meta.schemaVersion}, and this ` +
|
|
457
|
+
`build only understands ${exports.SCHEMA_VERSION}. Refusing to read a ` +
|
|
458
|
+
`schema it does not know — a wrong guess here returns plausible ` +
|
|
459
|
+
`results from the wrong vectors.`);
|
|
460
|
+
}
|
|
461
|
+
if (meta.dimensions !== exports.EMBEDDING_DIMENSIONS) {
|
|
462
|
+
throw new SchemaIncompatibleError(`The index stores ${meta.dimensions}-dimensional vectors and this ` +
|
|
463
|
+
`build writes ${exports.EMBEDDING_DIMENSIONS}.`);
|
|
464
|
+
}
|
|
465
|
+
if ((expected === null || expected === void 0 ? void 0 : expected.dimensions) !== undefined) {
|
|
466
|
+
if (expected.dimensions !== meta.dimensions) {
|
|
467
|
+
throw new SchemaIncompatibleError(`The embedder produces ${expected.dimensions} dimensions and the ` +
|
|
468
|
+
`index stores ${meta.dimensions}.`);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
if ((expected === null || expected === void 0 ? void 0 : expected.model) !== undefined && expected.model !== meta.model) {
|
|
472
|
+
throw new SchemaIncompatibleError(`The index was built with model "${meta.model}" and this process is ` +
|
|
473
|
+
`configured for "${expected.model}". Two models put vectors in ` +
|
|
474
|
+
`different spaces; a mixed index is silently meaningless.`);
|
|
475
|
+
}
|
|
476
|
+
if ((expected === null || expected === void 0 ? void 0 : expected.prefixMode) !== undefined) {
|
|
477
|
+
(0, embedder_1.assertPrefixModeMatches)(expected.prefixMode, meta.prefixMode);
|
|
478
|
+
}
|
|
479
|
+
return meta;
|
|
480
|
+
}
|
|
481
|
+
/** `{ docId → modified }` for every indexed document. */
|
|
482
|
+
indexed() {
|
|
483
|
+
const rows = this.handle()
|
|
484
|
+
.prepare("SELECT id, modified FROM documents")
|
|
485
|
+
.all();
|
|
486
|
+
return new Map(rows.map((row) => [row.id, row.modified]));
|
|
487
|
+
}
|
|
488
|
+
/** `{ docId → metadataAt }`, needed by the 24 h metadata refresh. */
|
|
489
|
+
metadataStamps() {
|
|
490
|
+
const rows = this.handle()
|
|
491
|
+
.prepare("SELECT id, metadata_at FROM documents")
|
|
492
|
+
.all();
|
|
493
|
+
return new Map(rows.map((row) => [row.id, row.metadata_at]));
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Delete every chunk of a document. Returns the number of vector rows removed.
|
|
497
|
+
*
|
|
498
|
+
* `changes` is asserted rather than ignored. The recon note claimed a bound
|
|
499
|
+
* delete silently returns 0 — re-measurement showed `changes` is accurate,
|
|
500
|
+
* but the discipline is worth keeping regardless: a delete that reports
|
|
501
|
+
* success and removes nothing is the same failure family as a `matches: 0`
|
|
502
|
+
* on a guessed pattern, and the instrument is what proves it landed.
|
|
503
|
+
*/
|
|
504
|
+
deleteDocumentChunks(docId) {
|
|
505
|
+
const db = this.handle();
|
|
506
|
+
const removed = db
|
|
507
|
+
.prepare("DELETE FROM vec_chunks WHERE rowid BETWEEN CAST(? AS INTEGER) AND " +
|
|
508
|
+
"CAST(? AS INTEGER)")
|
|
509
|
+
.run(chunkRowid(docId, 0), chunkRowid(docId, config_1.CHUNK_ROWID_STRIDE - 1));
|
|
510
|
+
const expected = db
|
|
511
|
+
.prepare("SELECT count(*) AS c FROM chunks WHERE doc_id = ?")
|
|
512
|
+
.get(docId).c;
|
|
513
|
+
if (removed.changes !== expected) {
|
|
514
|
+
throw new DeleteCountError(`Deleting the vectors for document ${docId} removed ` +
|
|
515
|
+
`${removed.changes} rows but the chunk table holds ${expected}. ` +
|
|
516
|
+
`The vector index is out of step with the chunk table.`);
|
|
517
|
+
}
|
|
518
|
+
db.prepare("DELETE FROM chunks WHERE doc_id = ?").run(docId);
|
|
519
|
+
if (this.hasFts5) {
|
|
520
|
+
db.prepare("DELETE FROM fts WHERE rowid BETWEEN CAST(? AS INTEGER) AND " +
|
|
521
|
+
"CAST(? AS INTEGER)").run(chunkRowid(docId, 0), chunkRowid(docId, config_1.CHUNK_ROWID_STRIDE - 1));
|
|
522
|
+
}
|
|
523
|
+
return removed.changes;
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Replace a document and all of its chunks atomically.
|
|
527
|
+
*
|
|
528
|
+
* A vec0 rowid cannot be overwritten — verified, a duplicate throws a UNIQUE
|
|
529
|
+
* constraint and `insert or replace` throws it too — so a replace is always
|
|
530
|
+
* a delete followed by an insert, and both must be in one transaction or a
|
|
531
|
+
* crashed sync leaves an index that is missing a document body while its
|
|
532
|
+
* `documents` row still claims it is complete.
|
|
533
|
+
*/
|
|
534
|
+
replaceDocument(doc, chunks) {
|
|
535
|
+
const db = this.handle();
|
|
536
|
+
if (chunks.length !== doc.chunkCount) {
|
|
537
|
+
throw new Error(`Document ${doc.id} declares ${doc.chunkCount} chunks but received ` +
|
|
538
|
+
`${chunks.length}.`);
|
|
539
|
+
}
|
|
540
|
+
const insertDoc = db.prepare("INSERT INTO documents (id, modified, title, chunk_count, truncated, metadata_at) " +
|
|
541
|
+
"VALUES (?, ?, ?, ?, ?, ?) " +
|
|
542
|
+
"ON CONFLICT (id) DO UPDATE SET modified = excluded.modified, " +
|
|
543
|
+
"title = excluded.title, chunk_count = excluded.chunk_count, " +
|
|
544
|
+
"truncated = excluded.truncated, metadata_at = excluded.metadata_at");
|
|
545
|
+
const insertChunk = db.prepare("INSERT INTO chunks (doc_id, position, kind, text) VALUES (?, ?, ?, ?)");
|
|
546
|
+
const insertVec = db.prepare("INSERT INTO vec_chunks (rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
|
|
547
|
+
const insertFts = this.hasFts5
|
|
548
|
+
? db.prepare("INSERT INTO fts (rowid, text) VALUES (CAST(? AS INTEGER), ?)")
|
|
549
|
+
: undefined;
|
|
550
|
+
const run = db.transaction(() => {
|
|
551
|
+
this.deleteDocumentChunks(doc.id);
|
|
552
|
+
insertDoc.run(doc.id, doc.modified, doc.title, doc.chunkCount, doc.truncated ? 1 : 0, doc.metadataAt);
|
|
553
|
+
chunks.forEach((chunk, position) => {
|
|
554
|
+
if (position >= config_1.CHUNK_ROWID_STRIDE) {
|
|
555
|
+
throw new Error(`Document ${doc.id} has ${chunks.length} chunks, beyond the ` +
|
|
556
|
+
`rowid stride ${config_1.CHUNK_ROWID_STRIDE}.`);
|
|
557
|
+
}
|
|
558
|
+
const rowid = chunkRowid(doc.id, position);
|
|
559
|
+
insertChunk.run(doc.id, position, chunk.kind, chunk.text);
|
|
560
|
+
insertVec.run(rowid, toBlob(chunk.embedding));
|
|
561
|
+
insertFts === null || insertFts === void 0 ? void 0 : insertFts.run(rowid, chunk.text);
|
|
562
|
+
});
|
|
563
|
+
});
|
|
564
|
+
run();
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Replace only the metadata chunk (position 0) and stamp `metadata_at`.
|
|
568
|
+
* Used by the 24 h metadata refresh, which exists because tags,
|
|
569
|
+
* correspondents and document types change without touching `modified`.
|
|
570
|
+
*/
|
|
571
|
+
replaceMetadata(docId, chunk, metadataAt) {
|
|
572
|
+
const db = this.handle();
|
|
573
|
+
if (chunk.kind !== "metadata") {
|
|
574
|
+
throw new Error("replaceMetadata requires a chunk of kind 'metadata'.");
|
|
575
|
+
}
|
|
576
|
+
const rowid = chunkRowid(docId, 0);
|
|
577
|
+
const update = db.prepare("UPDATE documents SET metadata_at = ? WHERE id = ?");
|
|
578
|
+
const insertChunk = db.prepare("INSERT INTO chunks (doc_id, position, kind, text) VALUES (?, 0, ?, ?) " +
|
|
579
|
+
"ON CONFLICT (doc_id, position) DO UPDATE SET kind = excluded.kind, " +
|
|
580
|
+
"text = excluded.text");
|
|
581
|
+
const insertVec = db.prepare("INSERT INTO vec_chunks (rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
|
|
582
|
+
const insertFts = this.hasFts5
|
|
583
|
+
? db.prepare("INSERT INTO fts (rowid, text) VALUES (CAST(? AS INTEGER), ?)")
|
|
584
|
+
: undefined;
|
|
585
|
+
const deleteFts = this.hasFts5
|
|
586
|
+
? db.prepare("DELETE FROM fts WHERE rowid = CAST(? AS INTEGER)")
|
|
587
|
+
: undefined;
|
|
588
|
+
const countVec = db.prepare("SELECT count(*) AS c FROM vec_chunks WHERE rowid = CAST(? AS INTEGER)");
|
|
589
|
+
const run = db.transaction(() => {
|
|
590
|
+
// The DELETE's own row count is NOT asserted: a fresh document may not yet
|
|
591
|
+
// have position 0 in the vector table, and `changes: 0` is legitimate
|
|
592
|
+
// then. What IS asserted is the post-condition -- exactly one vector row
|
|
593
|
+
// for this document's metadata chunk after the delete-then-insert.
|
|
594
|
+
//
|
|
595
|
+
// This assertion is the comment that used to sit here, made real. The
|
|
596
|
+
// comment claimed it and the code said `void del`, which is the
|
|
597
|
+
// comment-vs-code lie class: a reader trusts a guarantee that has no
|
|
598
|
+
// enforcement, and the failure it describes (a duplicated vector row
|
|
599
|
+
// making one document score twice) has no other detector.
|
|
600
|
+
db.prepare("DELETE FROM vec_chunks WHERE rowid = CAST(? AS INTEGER)").run(rowid);
|
|
601
|
+
// The lexical index needs the same delete-then-insert. Without it the OLD
|
|
602
|
+
// metadata text stays in `fts` forever and remains searchable alongside
|
|
603
|
+
// the new text, so one document matches two contradictory strings.
|
|
604
|
+
// Caught by a test, not by review: "old metadata" still hit.
|
|
605
|
+
deleteFts === null || deleteFts === void 0 ? void 0 : deleteFts.run(rowid);
|
|
606
|
+
insertChunk.run(docId, chunk.kind, chunk.text);
|
|
607
|
+
insertVec.run(rowid, toBlob(chunk.embedding));
|
|
608
|
+
insertFts === null || insertFts === void 0 ? void 0 : insertFts.run(rowid, chunk.text);
|
|
609
|
+
update.run(metadataAt, docId);
|
|
610
|
+
const after = countVec.get(rowid).c;
|
|
611
|
+
if (after !== 1) {
|
|
612
|
+
// Thrown inside the transaction, so the whole replace rolls back: a
|
|
613
|
+
// document whose metadata vector is duplicated is better left with its
|
|
614
|
+
// previous metadata than with two contradictory ones.
|
|
615
|
+
throw new Error(`replaceMetadata left ${after} vector rows for document ${docId}; expected exactly 1.`);
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
run();
|
|
619
|
+
}
|
|
620
|
+
/** Delete documents and everything belonging to them, atomically. */
|
|
621
|
+
deleteDocuments(ids) {
|
|
622
|
+
if (ids.length === 0)
|
|
623
|
+
return;
|
|
624
|
+
const db = this.handle();
|
|
625
|
+
const run = db.transaction(() => {
|
|
626
|
+
for (const id of ids)
|
|
627
|
+
this.deleteDocumentChunks(id);
|
|
628
|
+
const drop = db.prepare("DELETE FROM documents WHERE id = ?");
|
|
629
|
+
for (const id of ids)
|
|
630
|
+
drop.run(id);
|
|
631
|
+
});
|
|
632
|
+
run();
|
|
633
|
+
}
|
|
634
|
+
/**
|
|
635
|
+
* The stored chunks of one document, in position order.
|
|
636
|
+
*
|
|
637
|
+
* A read accessor, and it exists because `textHits`/`vectorHits` return only
|
|
638
|
+
* the BEST chunk per document -- correct for ranking, useless for asserting
|
|
639
|
+
* what a document actually contains. Without it a test can only observe the
|
|
640
|
+
* metadata chunk by getting lucky with a query.
|
|
641
|
+
*/
|
|
642
|
+
chunksFor(docId) {
|
|
643
|
+
return this.handle()
|
|
644
|
+
.prepare("SELECT position, kind, text FROM chunks WHERE doc_id = ? ORDER BY position ASC")
|
|
645
|
+
.all(docId);
|
|
646
|
+
}
|
|
647
|
+
/**
|
|
648
|
+
* How many indexed chunks contain each term.
|
|
649
|
+
*
|
|
650
|
+
* Exists so the passage anchor can pick the MOST DISTINCTIVE query term
|
|
651
|
+
* instead of merely the longest. Measured on the live corpus, length is a bad
|
|
652
|
+
* proxy: for "what did the doctor say about the ankle" it chose `doctor`
|
|
653
|
+
* (1,903 chunks) over `ankle` (808) purely because it is one character
|
|
654
|
+
* longer, and for "the practice of a wrathful deity with a lion head" it
|
|
655
|
+
* chose `practice` (36,459 chunks) over `wrathful` (4,657). Rarity agreed
|
|
656
|
+
* with a human read in 3 of 3 cases; length in 0 of 3.
|
|
657
|
+
*
|
|
658
|
+
* One indexed lookup per term, and `toFtsMatch` already caps a query at 32
|
|
659
|
+
* terms, so this cannot become an unbounded scan. A term absent from the
|
|
660
|
+
* index reports 0 rather than throwing -- "not in the corpus" is a legitimate
|
|
661
|
+
* and very informative answer here.
|
|
662
|
+
*/
|
|
663
|
+
termChunkCounts(terms) {
|
|
664
|
+
const out = new Map();
|
|
665
|
+
const db = this.handle();
|
|
666
|
+
if (!this.hasFts5)
|
|
667
|
+
return out;
|
|
668
|
+
const stmt = db.prepare("SELECT count(*) AS c FROM fts WHERE fts MATCH ?");
|
|
669
|
+
for (const term of new Set(terms)) {
|
|
670
|
+
try {
|
|
671
|
+
out.set(term, stmt.get(`"${term.replace(/"/g, '""')}"`).c);
|
|
672
|
+
}
|
|
673
|
+
catch (_a) {
|
|
674
|
+
// A term FTS5 cannot parse is simply unscored; the anchor falls back
|
|
675
|
+
// to length for it rather than failing the whole search.
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
return out;
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* How many indexed documents were cut off at the chunk cap.
|
|
682
|
+
*
|
|
683
|
+
* Exposed because truncation is the one index fact that changes what a caller
|
|
684
|
+
* can conclude from a miss: a truncated document lost its tail, so "no such
|
|
685
|
+
* text" may be a false negative rather than an absence.
|
|
686
|
+
*/
|
|
687
|
+
truncatedCount() {
|
|
688
|
+
return this.handle().prepare("SELECT count(*) AS c FROM documents WHERE truncated = 1").get().c;
|
|
689
|
+
}
|
|
690
|
+
counts() {
|
|
691
|
+
const db = this.handle();
|
|
692
|
+
const one = (sql) => db.prepare(sql).get().c;
|
|
693
|
+
return {
|
|
694
|
+
documents: one("SELECT count(*) AS c FROM documents"),
|
|
695
|
+
chunks: one("SELECT count(*) AS c FROM chunks"),
|
|
696
|
+
vectors: one("SELECT count(*) AS c FROM vec_chunks"),
|
|
697
|
+
ftsRows: this.hasFts5 ? one("SELECT count(*) AS c FROM fts") : null,
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* The best chunk of each document by vector similarity.
|
|
702
|
+
*
|
|
703
|
+
* Per-document dedupe is deliberate: a chunk list lets one long document fill
|
|
704
|
+
* the result set, and a document that matches only by meaning then drops out.
|
|
705
|
+
* The reference implementation does this in Postgres with `DISTINCT ON`; here
|
|
706
|
+
* it is a window function over the kNN result.
|
|
707
|
+
*
|
|
708
|
+
* `score` is `1 − distance` — a similarity. The raw `distance` is never
|
|
709
|
+
* handed to a caller as a score, because on a cosine table 0 means identical
|
|
710
|
+
* and 1 means orthogonal, which reads exactly backwards.
|
|
711
|
+
*/
|
|
712
|
+
vectorHits(embedding, limit) {
|
|
713
|
+
const db = this.handle();
|
|
714
|
+
if (limit <= 0)
|
|
715
|
+
return [];
|
|
716
|
+
// A FIXED over-fetch is not enough, and the reason is arithmetic rather
|
|
717
|
+
// than a matter of degree. This store returns one hit per document, so `k`
|
|
718
|
+
// candidates containing D distinct documents can only ever yield D results.
|
|
719
|
+
// With `max(limit*20, 200)` and a 4,000-chunk document, a caller asking for
|
|
720
|
+
// 10 got **1** -- all 200 (or 800) nearest rows belonged to that document,
|
|
721
|
+
// measured against a corpus of 60 documents. The live corpus has 21
|
|
722
|
+
// documents over 2M chars, so this is a real scale risk, not a corner case.
|
|
723
|
+
//
|
|
724
|
+
// So the budget WIDENS until either `limit` distinct documents are in hand
|
|
725
|
+
// or the pool is exhausted. Doubling keeps the number of round trips
|
|
726
|
+
// logarithmic in the worst case, and VECTOR_CANDIDATE_CEILING bounds it so
|
|
727
|
+
// a corpus with one enormous document cannot make a query unbounded.
|
|
728
|
+
// Clamped from the start: `limit * 20` at MAX_LIMIT 50 is 1000, but a
|
|
729
|
+
// caller is not the only input -- `runSemanticSearch` multiplies by 4
|
|
730
|
+
// before it gets here, so an unclamped initial value could already exceed
|
|
731
|
+
// sqlite-vec's k cap before the widening loop ran once.
|
|
732
|
+
let candidates = Math.min(Math.max(limit * 20, 200), exports.VECTOR_CANDIDATE_CEILING);
|
|
733
|
+
let hits = [];
|
|
734
|
+
for (;;) {
|
|
735
|
+
hits = this.vectorHitsAt(embedding, candidates, limit);
|
|
736
|
+
// `hits.length` is a count of DISTINCT documents (per-document dedupe),
|
|
737
|
+
// so this compares the right quantity against the right target.
|
|
738
|
+
if (hits.length >= limit || candidates >= exports.VECTOR_CANDIDATE_CEILING)
|
|
739
|
+
return hits;
|
|
740
|
+
candidates = Math.min(candidates * 2, exports.VECTOR_CANDIDATE_CEILING);
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
/** One kNN round trip at a given candidate budget. */
|
|
744
|
+
vectorHitsAt(embedding, candidates, limit) {
|
|
745
|
+
const db = this.handle();
|
|
746
|
+
const sql = `
|
|
747
|
+
WITH nearest AS (
|
|
748
|
+
SELECT rowid, distance
|
|
749
|
+
FROM vec_chunks
|
|
750
|
+
WHERE embedding MATCH CAST(? AS BLOB) AND k = CAST(? AS INTEGER)
|
|
751
|
+
), ranked AS (
|
|
752
|
+
SELECT rowid, distance,
|
|
753
|
+
(rowid / ${config_1.CHUNK_ROWID_STRIDE}) AS doc_id,
|
|
754
|
+
ROW_NUMBER() OVER (
|
|
755
|
+
PARTITION BY (rowid / ${config_1.CHUNK_ROWID_STRIDE})
|
|
756
|
+
ORDER BY distance ASC, rowid ASC
|
|
757
|
+
) AS rank_in_doc
|
|
758
|
+
FROM nearest
|
|
759
|
+
)
|
|
760
|
+
SELECT c.doc_id AS documentId, c.position AS position, c.text AS text,
|
|
761
|
+
1.0 - r.distance AS score
|
|
762
|
+
FROM ranked r
|
|
763
|
+
JOIN chunks c
|
|
764
|
+
ON c.doc_id = r.doc_id
|
|
765
|
+
AND c.position = (r.rowid % ${config_1.CHUNK_ROWID_STRIDE})
|
|
766
|
+
WHERE r.rank_in_doc = 1
|
|
767
|
+
ORDER BY r.distance ASC, c.doc_id ASC
|
|
768
|
+
LIMIT CAST(? AS INTEGER)
|
|
769
|
+
`;
|
|
770
|
+
return db
|
|
771
|
+
.prepare(sql)
|
|
772
|
+
.all(toBlob(embedding), candidates, limit);
|
|
773
|
+
}
|
|
774
|
+
/**
|
|
775
|
+
* The best chunk of each document by BM25.
|
|
776
|
+
*
|
|
777
|
+
* FTS5's `bm25()` returns a NEGATIVE score where more negative is better, so
|
|
778
|
+
* it is negated to a positive "higher is better" value. It is NOT comparable
|
|
779
|
+
* to the vector similarity in magnitude — only in order — which is why the
|
|
780
|
+
* ranker fuses the two lists by rank rather than by score.
|
|
781
|
+
*/
|
|
782
|
+
textHits(query, limit) {
|
|
783
|
+
if (!this.hasFts5)
|
|
784
|
+
throw new LexicalUnavailableError();
|
|
785
|
+
const match = toFtsMatch(query);
|
|
786
|
+
if (!match)
|
|
787
|
+
return [];
|
|
788
|
+
const db = this.handle();
|
|
789
|
+
const sql = `
|
|
790
|
+
WITH matched AS (
|
|
791
|
+
SELECT rowid, bm25(fts) AS bm
|
|
792
|
+
FROM fts
|
|
793
|
+
WHERE fts MATCH CAST(? AS TEXT)
|
|
794
|
+
), ranked AS (
|
|
795
|
+
SELECT rowid, bm,
|
|
796
|
+
(rowid / ${config_1.CHUNK_ROWID_STRIDE}) AS doc_id,
|
|
797
|
+
ROW_NUMBER() OVER (
|
|
798
|
+
PARTITION BY (rowid / ${config_1.CHUNK_ROWID_STRIDE})
|
|
799
|
+
ORDER BY bm ASC, rowid ASC
|
|
800
|
+
) AS rank_in_doc
|
|
801
|
+
FROM matched
|
|
802
|
+
)
|
|
803
|
+
SELECT c.doc_id AS documentId, c.position AS position, c.text AS text,
|
|
804
|
+
-r.bm AS score
|
|
805
|
+
FROM ranked r
|
|
806
|
+
JOIN chunks c
|
|
807
|
+
ON c.doc_id = r.doc_id
|
|
808
|
+
AND c.position = (r.rowid % ${config_1.CHUNK_ROWID_STRIDE})
|
|
809
|
+
WHERE r.rank_in_doc = 1
|
|
810
|
+
ORDER BY r.bm ASC, c.doc_id ASC
|
|
811
|
+
LIMIT CAST(? AS INTEGER)
|
|
812
|
+
`;
|
|
813
|
+
return db.prepare(sql).all(match, limit);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
exports.SearchIndex = SearchIndex;
|
|
817
|
+
/**
|
|
818
|
+
* Turn a user's words into an FTS5 MATCH expression.
|
|
819
|
+
*
|
|
820
|
+
* Raw user text is NOT a valid MATCH expression — measured: `c++ (test)` throws
|
|
821
|
+
* `fts5: syntax error near "+"`, `a-b` throws `no such column: b`, and an empty
|
|
822
|
+
* string throws. So the query is tokenised into bare terms and every term is
|
|
823
|
+
* quoted, which makes FTS5 treat it as a literal rather than as query syntax.
|
|
824
|
+
* Terms are OR-ed because a user typing three words wants documents matching
|
|
825
|
+
* any of them ranked by BM25, not only those matching all three.
|
|
826
|
+
*
|
|
827
|
+
* `remove_diacritics 2` means `titulo` matches `Título`; verified both
|
|
828
|
+
* directions on this build.
|
|
829
|
+
*/
|
|
830
|
+
function toFtsMatch(query) {
|
|
831
|
+
const terms = query
|
|
832
|
+
.toLowerCase()
|
|
833
|
+
// \p{M} is REQUIRED, not decorative. A Tibetan syllable is a base letter
|
|
834
|
+
// plus combining marks -- U+0FB3 and U+0F74 are category Mn, not L -- so
|
|
835
|
+
// splitting on [^\p{L}\p{N}_] tears "རླུང" into "ར" and "ང": two fragments,
|
|
836
|
+
// neither of which is the word. Measured on this build. This corpus is
|
|
837
|
+
// heavily Tibetan, so the bug would have made those terms unfindable
|
|
838
|
+
// lexically while looking like an ordinary no-match.
|
|
839
|
+
.split(/[^\p{L}\p{N}\p{M}_]+/u)
|
|
840
|
+
.filter((term) => term.length > 0);
|
|
841
|
+
if (terms.length === 0)
|
|
842
|
+
return null;
|
|
843
|
+
// Cap the expression: a 500-term OR is pathological and slower than the
|
|
844
|
+
// answer is worth. The first terms are the ones the user typed first.
|
|
845
|
+
const capped = terms.slice(0, 32);
|
|
846
|
+
return capped.map((term) => `"${term.replace(/"/g, '""')}"`).join(" OR ");
|
|
847
|
+
}
|