@retiform/paperless-mcp 2.3.5 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +10 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Hit } from "./store";
|
|
2
|
+
export type { Hit };
|
|
3
|
+
export interface Fused {
|
|
4
|
+
documentId: number;
|
|
5
|
+
score: number;
|
|
6
|
+
passage: string;
|
|
7
|
+
matched: "meaning" | "words" | "both";
|
|
8
|
+
bestSimilarity: number | null;
|
|
9
|
+
}
|
|
10
|
+
export declare const DEFAULT_FUSE_K = 10;
|
|
11
|
+
export declare function fuse(vector: Hit[], text: Hit[], k?: number): Fused[];
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Reciprocal rank fusion of a vector (meaning) list and a text (words) list.
|
|
3
|
+
//
|
|
4
|
+
// Ported verbatim from jaimemachado/paperless-mcp (src/search/rank.ts at
|
|
5
|
+
// 50f7beec0d36804cab8fbcaa7b47f85e562e8061). Their only import was `Hit` from
|
|
6
|
+
// ./store; when this file was written store.ts did not exist on this branch, so
|
|
7
|
+
// Hit was declared here to let the module compile and test alone. That
|
|
8
|
+
// declaration is now GONE and store.ts owns Hit -- the merge of the two vector
|
|
9
|
+
// shards put two structurally identical live definitions in one tree, which is
|
|
10
|
+
// a defect whether or not it typechecks. The re-export below is kept so the
|
|
11
|
+
// existing `import { fuse, type Hit } from "./rank"` call sites keep working
|
|
12
|
+
// against the single definition.
|
|
13
|
+
//
|
|
14
|
+
// A document's score is the sum of 1/(k + bestRank) over each list it appears
|
|
15
|
+
// in, using its BEST rank in that list. A document found only by meaning (the
|
|
16
|
+
// vector list) must stay in the results -- that is the whole point of fusing
|
|
17
|
+
// rather than intersecting.
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.DEFAULT_FUSE_K = void 0;
|
|
20
|
+
exports.fuse = fuse;
|
|
21
|
+
// `k` is the RRF dampening constant: it settles how much a document's rank
|
|
22
|
+
// matters relative to how many lists agree about it. A document both arms hold
|
|
23
|
+
// at rank D scores 2/(k+D), while a document only ONE arm found at rank 1
|
|
24
|
+
// scores 1/(k+1). So a dual-arm document outranks a perfect single-arm hit for
|
|
25
|
+
// every D < k+2.
|
|
26
|
+
//
|
|
27
|
+
// At the upstream default of k=60 that crossover sits at D=62 -- meaning any
|
|
28
|
+
// document both arms happened to return in their top 62 outranked a document
|
|
29
|
+
// one arm had ranked FIRST. k=60 is the figure from the original RRF paper,
|
|
30
|
+
// where candidate lists ran to thousands; it was inherited here verbatim from
|
|
31
|
+
// jaimemachado/paperless-mcp and never tuned for a corpus whose arms return at
|
|
32
|
+
// most a few hundred candidates and whose top 10 is what a caller reads.
|
|
33
|
+
//
|
|
34
|
+
// Measured on the live index (eval/bakeoff.mjs, 20 ground-truthed cases,
|
|
35
|
+
// 24 strategies, reproducible run to run): k=60 scores 15/20 at depth 10 while
|
|
36
|
+
// every k of 10 or below scores 18/20. Two cases carry the whole difference --
|
|
37
|
+
// a document ONE arm ranked first fell to rank 11 and to rank 52 under k=60.
|
|
38
|
+
// Which value below 60 is best is inside this fixture's noise (10, 5, 2, 1 and
|
|
39
|
+
// 0 all tie at 18/20), so k=10 is chosen for being the smallest change that
|
|
40
|
+
// removes the pathology while keeping rank order meaningful across the pool
|
|
41
|
+
// rather than collapsing it to bare reciprocal rank. Do not read 18 as "the
|
|
42
|
+
// best available number"; read it as "k no longer fights the ranking".
|
|
43
|
+
exports.DEFAULT_FUSE_K = 10;
|
|
44
|
+
function fuse(vector, text, k = exports.DEFAULT_FUSE_K) {
|
|
45
|
+
const byDocument = new Map();
|
|
46
|
+
function addRanks(hits, side) {
|
|
47
|
+
var _a;
|
|
48
|
+
const bestRank = new Map();
|
|
49
|
+
hits.forEach((hit, index) => {
|
|
50
|
+
const rank = index + 1;
|
|
51
|
+
if (!bestRank.has(hit.documentId))
|
|
52
|
+
bestRank.set(hit.documentId, rank);
|
|
53
|
+
});
|
|
54
|
+
for (const hit of hits) {
|
|
55
|
+
const entry = (_a = byDocument.get(hit.documentId)) !== null && _a !== void 0 ? _a : { score: 0 };
|
|
56
|
+
byDocument.set(hit.documentId, entry);
|
|
57
|
+
if (side === "vector" && !entry.vector)
|
|
58
|
+
entry.vector = hit;
|
|
59
|
+
if (side === "text" && !entry.text)
|
|
60
|
+
entry.text = hit;
|
|
61
|
+
}
|
|
62
|
+
for (const [documentId, rank] of bestRank) {
|
|
63
|
+
const entry = byDocument.get(documentId);
|
|
64
|
+
entry.score += 1 / (k + rank);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
addRanks(vector, "vector");
|
|
68
|
+
addRanks(text, "text");
|
|
69
|
+
return [...byDocument.entries()]
|
|
70
|
+
.map(([documentId, entry]) => {
|
|
71
|
+
var _a;
|
|
72
|
+
const matched = entry.vector && entry.text
|
|
73
|
+
? "both"
|
|
74
|
+
: entry.vector
|
|
75
|
+
? "meaning"
|
|
76
|
+
: "words";
|
|
77
|
+
const best = (_a = entry.vector) !== null && _a !== void 0 ? _a : entry.text;
|
|
78
|
+
return {
|
|
79
|
+
documentId,
|
|
80
|
+
score: entry.score,
|
|
81
|
+
passage: best.text,
|
|
82
|
+
matched,
|
|
83
|
+
bestSimilarity: entry.vector ? entry.vector.score : null,
|
|
84
|
+
};
|
|
85
|
+
})
|
|
86
|
+
.sort((a, b) => b.score - a.score);
|
|
87
|
+
}
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import { CHUNK_ROWID_STRIDE, PrefixMode } from "./config";
|
|
3
|
+
/**
|
|
4
|
+
* The vector + lexical index for the Paperless corpus, in one SQLite file.
|
|
5
|
+
*
|
|
6
|
+
* This is a REWRITE, not a port. The reference implementation
|
|
7
|
+
* (`jaimemachado/paperless-mcp@50f7bee:src/search/store.ts`) is Postgres with
|
|
8
|
+
* pgvector: `public.vector(1024)`, HNSW over `halfvec`, `to_tsvector('simple',
|
|
9
|
+
* public.unaccent(...))`, GIN on a stored `tsvector`, and
|
|
10
|
+
* `SET LOCAL hnsw.iterative_scan`. None of that survives to SQLite, and none of
|
|
11
|
+
* it should — this store exists so the corpus can be searched without a second
|
|
12
|
+
* web app and without a database server.
|
|
13
|
+
*
|
|
14
|
+
* ── Schema ────────────────────────────────────────────────────────────────
|
|
15
|
+
* documents id, modified, title, chunk_count, truncated, metadata_at
|
|
16
|
+
* chunks doc_id, position, kind, text (PK: doc_id, position)
|
|
17
|
+
* vec_chunks embedding float[768] distance_metric=cosine
|
|
18
|
+
* fts FTS5 over chunks.text, contentless, rowid = chunk rowid
|
|
19
|
+
* index_meta key/value: schema_version, dimensions, model, prefix_mode,
|
|
20
|
+
* plus the sync watermark (watermark_count, watermark_modified,
|
|
21
|
+
* watermark_at) — see Watermark below
|
|
22
|
+
*
|
|
23
|
+
* ── The five measured traps, each handled explicitly ──────────────────────
|
|
24
|
+
* Every one of these was reproduced on this box under the gateway child's
|
|
25
|
+
* runtime (node v24.21.0, better-sqlite3 13.2.0, sqlite-vec 0.1.9) before a
|
|
26
|
+
* line of this file was written. Two of the five turned out differently from
|
|
27
|
+
* the recon notes, and the notes are corrected here rather than in a drawer.
|
|
28
|
+
*
|
|
29
|
+
* 1. ROWID PARAMETERS MUST BE CAST. `insert into v(rowid, embedding) values
|
|
30
|
+
* (?, ?)` fails with `SQLITE_ERROR: Only integers are allows for primary
|
|
31
|
+
* key values on v` while both the literal form and `CAST(? AS INTEGER)`
|
|
32
|
+
* succeed. Verified: `k = ?` does NOT need the cast (it worked at
|
|
33
|
+
* `k = ?` bound to 2), so the cast is applied to rowid contexts only, not
|
|
34
|
+
* sprayed everywhere.
|
|
35
|
+
*
|
|
36
|
+
* 2. `changes` IS RELIABLE — the recon note was WRONG about this. The note
|
|
37
|
+
* recorded `delete from v where rowid = ? [5]` returning `changes: 0` and
|
|
38
|
+
* concluded that a bound-rowid delete silently no-ops. Re-measured five
|
|
39
|
+
* ways: the row simply did not exist. A bound delete against an EXISTING
|
|
40
|
+
* rowid returns `changes: 1` for a number, a float, a string, and a bigint
|
|
41
|
+
* alike, and a range delete returns the true count (3 of 3). So `changes` is
|
|
42
|
+
* a real instrument, and the discipline it buys is the right one anyway —
|
|
43
|
+
* ASSERT ON IT, never on the absence of a thrown error — but the mechanism
|
|
44
|
+
* in the note ("vec0 silently ignores bound rowids") does not exist.
|
|
45
|
+
*
|
|
46
|
+
* 3. NO `INSERT OR REPLACE` ON vec0. Verified: a duplicate rowid throws
|
|
47
|
+
* `UNIQUE constraint failed on v primary key`, and `insert or replace`
|
|
48
|
+
* throws the same. A replace is DELETE-then-INSERT inside one transaction.
|
|
49
|
+
* (The cascade/rollback path was verified: a vec0 insert inside an explicit
|
|
50
|
+
* transaction disappears on ROLLBACK.)
|
|
51
|
+
*
|
|
52
|
+
* 4. `rowid IN (…)` COMPOSES WITH `MATCH`. Verified. This store does not
|
|
53
|
+
* currently push visibility filters into the kNN statement, because the
|
|
54
|
+
* caller-side filter belongs to the tool lap, but the mechanism is
|
|
55
|
+
* available and the note is accurate.
|
|
56
|
+
*
|
|
57
|
+
* 5. DEFAULT METRIC IS L2 — `distance_metric=cosine` is declared explicitly.
|
|
58
|
+
* Verified: cosine gives self 0 and orthogonal 0.50098 (= 1 − cos 90°),
|
|
59
|
+
* while a default table gives self 0 and orthogonal 16 for the same unit
|
|
60
|
+
* vectors. Similarity is `1 − distance`, and `vectorHits` returns the
|
|
61
|
+
* similarity, never the raw distance.
|
|
62
|
+
*
|
|
63
|
+
* ── One further finding, which changed the schema ─────────────────────────
|
|
64
|
+
* A rowid of `doc_id * 1001 + position` — the shape the recon notes proposed —
|
|
65
|
+
* COLLIDES as soon as `MAX_CHUNKS` exceeds 1000. With 4001 rows per document
|
|
66
|
+
* the arithmetic produces 15,000 collisions across just six documents:
|
|
67
|
+
* (doc 1, pos 1001) and (doc 2, pos 0) both map to 2002. Since `MAX_CHUNKS`
|
|
68
|
+
* is being raised to 4000, the stride is 8192 instead: a power of two (so the
|
|
69
|
+
* decode is a mask rather than a division that has to be exactly right), and
|
|
70
|
+
* comfortably above the maximum chunk count, with zero collisions verified at
|
|
71
|
+
* 4001 rows per document.
|
|
72
|
+
*/
|
|
73
|
+
export declare const SCHEMA_VERSION = 1;
|
|
74
|
+
export declare const EMBEDDING_DIMENSIONS = 768;
|
|
75
|
+
/**
|
|
76
|
+
* Rowid stride, re-exported from ./config where it is DECLARED.
|
|
77
|
+
*
|
|
78
|
+
* It moved so the configuration could derive `MAX_CHUNKS`'s upper bound from
|
|
79
|
+
* the same number the store encodes with, instead of hard-coding 100000 and
|
|
80
|
+
* deferring the contradiction to `chunkRowid` mid-build. The re-export keeps
|
|
81
|
+
* every existing `import { CHUNK_ROWID_STRIDE } from "./store"` call site
|
|
82
|
+
* working against the single declaration.
|
|
83
|
+
*/
|
|
84
|
+
export { CHUNK_ROWID_STRIDE };
|
|
85
|
+
/** Whether FTS5 is compiled into the bundled SQLite. Measured, not assumed. */
|
|
86
|
+
export declare const HAS_FTS5: boolean;
|
|
87
|
+
/**
|
|
88
|
+
* The hard ceiling on kNN candidates for one query.
|
|
89
|
+
*
|
|
90
|
+
* It is **4096 because sqlite-vec refuses more**, not because that is a good
|
|
91
|
+
* number: `k value in knn query too large, provided 6400 and the limit is 4096`
|
|
92
|
+
* is the measured error, and it fires as a hard SQLITE_ERROR rather than
|
|
93
|
+
* clamping silently. Discovered by writing this widening loop against a ceiling
|
|
94
|
+
* of 20,000 and watching eleven previously-green tests fail -- which is the
|
|
95
|
+
* argument for keeping the number here, derived from the constraint, rather
|
|
96
|
+
* than somewhere in the widening logic.
|
|
97
|
+
*
|
|
98
|
+
* ⚠️ THE CONSEQUENCE, stated because it is a real limit on defect 5's fix: with
|
|
99
|
+
* a 4,000-chunk document occupying the top of the ranking, a query can find at
|
|
100
|
+
* most `4096 - 4000 + 1 = 97` distinct documents no matter how far it widens.
|
|
101
|
+
* That is far more than any caller's `limit` (MAX_LIMIT is 50), so the widening
|
|
102
|
+
* does its job; but it is a ceiling, not an absence of one.
|
|
103
|
+
*/
|
|
104
|
+
export declare const VECTOR_CANDIDATE_CEILING = 4096;
|
|
105
|
+
export interface IndexedDocument {
|
|
106
|
+
id: number;
|
|
107
|
+
modified: string;
|
|
108
|
+
title: string;
|
|
109
|
+
chunkCount: number;
|
|
110
|
+
truncated: boolean;
|
|
111
|
+
metadataAt: string;
|
|
112
|
+
}
|
|
113
|
+
export interface StoredChunk {
|
|
114
|
+
kind: "metadata" | "content";
|
|
115
|
+
text: string;
|
|
116
|
+
embedding: number[];
|
|
117
|
+
}
|
|
118
|
+
export interface Hit {
|
|
119
|
+
documentId: number;
|
|
120
|
+
position: number;
|
|
121
|
+
text: string;
|
|
122
|
+
score: number;
|
|
123
|
+
}
|
|
124
|
+
export interface IndexMeta {
|
|
125
|
+
schemaVersion: number;
|
|
126
|
+
dimensions: number;
|
|
127
|
+
model: string;
|
|
128
|
+
prefixMode: PrefixMode;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* What the corpus looked like when the index was last brought up to date.
|
|
132
|
+
*
|
|
133
|
+
* ── Why these are SEPARATE KEYS and not part of `IndexMeta` ─────────────────
|
|
134
|
+
*
|
|
135
|
+
* `meta()` reads the key/value table and returns a FIXED object built from four
|
|
136
|
+
* named keys. It never enumerates, so any other row is structurally invisible
|
|
137
|
+
* to it — which is exactly why `assertSchemaCompatible` cannot start refusing an
|
|
138
|
+
* index just because these rows exist. A watermark is not schema: it is a
|
|
139
|
+
* measurement of the remote corpus taken at a point in time, and a reader that
|
|
140
|
+
* does not know about it must be able to open the index unchanged.
|
|
141
|
+
*
|
|
142
|
+
* That matters in the direction that actually bites: an index built BEFORE these
|
|
143
|
+
* keys existed must still open, and an index written WITH them must still open
|
|
144
|
+
* under a build that predates them. Both hold because neither side enumerates.
|
|
145
|
+
*
|
|
146
|
+
* ⚠️ Adding keys here is deliberately NOT a `SCHEMA_VERSION` bump. A bump would
|
|
147
|
+
* refuse every existing index and force a rebuild that takes most of an hour, to
|
|
148
|
+
* add three rows nobody has to read.
|
|
149
|
+
*/
|
|
150
|
+
export interface Watermark {
|
|
151
|
+
/** Live document count at the moment the sync finished. */
|
|
152
|
+
count: number;
|
|
153
|
+
/**
|
|
154
|
+
* `max(modified)` over the corpus at that moment, or null on an empty corpus.
|
|
155
|
+
*
|
|
156
|
+
* Stored as the string Paperless sent, not as an epoch, because it is a
|
|
157
|
+
* POINT OF COMPARISON rather than a value anything reads. Comparison is by
|
|
158
|
+
* instant at the call site anyway — Paperless may re-serialise one moment with
|
|
159
|
+
* a different UTC offset, and a string compare would then report a change on
|
|
160
|
+
* every single run.
|
|
161
|
+
*/
|
|
162
|
+
modified: string | null;
|
|
163
|
+
/** ISO timestamp of when that sync finished. Reported, never compared. */
|
|
164
|
+
at: string;
|
|
165
|
+
}
|
|
166
|
+
export interface Counts {
|
|
167
|
+
documents: number;
|
|
168
|
+
chunks: number;
|
|
169
|
+
vectors: number;
|
|
170
|
+
ftsRows: number | null;
|
|
171
|
+
}
|
|
172
|
+
export declare class SchemaIncompatibleError extends Error {
|
|
173
|
+
constructor(message: string);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Thrown when the lexical half is asked for on a build without FTS5.
|
|
177
|
+
*
|
|
178
|
+
* Returning an empty array would be indistinguishable from "the query matched
|
|
179
|
+
* nothing", which would let the ranker believe it had searched lexically and
|
|
180
|
+
* found silence. That is the silent degradation the plan forbids, so the absence
|
|
181
|
+
* is a refusal the caller must handle, not a quiet empty list.
|
|
182
|
+
*/
|
|
183
|
+
export declare class LexicalUnavailableError extends Error {
|
|
184
|
+
constructor();
|
|
185
|
+
}
|
|
186
|
+
export declare class DeleteCountError extends Error {
|
|
187
|
+
constructor(message: string);
|
|
188
|
+
}
|
|
189
|
+
/** Encode a chunk's rowid. Verified collision-free for 8191 chunks per doc. */
|
|
190
|
+
export declare function chunkRowid(docId: number, position: number): number;
|
|
191
|
+
export declare function decodeChunkRowid(rowid: number): {
|
|
192
|
+
docId: number;
|
|
193
|
+
position: number;
|
|
194
|
+
};
|
|
195
|
+
/**
|
|
196
|
+
* A lazily-opened index.
|
|
197
|
+
*
|
|
198
|
+
* Lazy is not tidiness — it is a named risk in the plan. The gateway launches
|
|
199
|
+
* this package's child at startup, and opening a ~1 GB SQLite file (plus
|
|
200
|
+
* loading the vec0 extension, which is a dynamic shared object) at module
|
|
201
|
+
* import time could exceed the gateway's child-startup tolerance. `open()` is
|
|
202
|
+
* therefore explicit, `close()` is explicit, and no code in this module runs
|
|
203
|
+
* on import beyond the FTS5 probe, which is a `:memory:` connection that is
|
|
204
|
+
* closed immediately.
|
|
205
|
+
*/
|
|
206
|
+
export declare class SearchIndex {
|
|
207
|
+
private readonly path;
|
|
208
|
+
private readonly options;
|
|
209
|
+
private db;
|
|
210
|
+
/**
|
|
211
|
+
* `fts5` exists only so the absent-FTS5 path can be exercised on a box that
|
|
212
|
+
* HAS FTS5. Without an injectable override, the degradation branch would be
|
|
213
|
+
* untestable here and the only evidence that it works would be that it
|
|
214
|
+
* compiles — which is not evidence.
|
|
215
|
+
*/
|
|
216
|
+
private readonly hasFts5;
|
|
217
|
+
constructor(path: string, options?: {
|
|
218
|
+
readonly?: boolean;
|
|
219
|
+
fts5?: boolean;
|
|
220
|
+
/**
|
|
221
|
+
* What this process believes the index should be. When supplied, `open()`
|
|
222
|
+
* asserts the stored index agrees and THROWS on a mismatch.
|
|
223
|
+
*
|
|
224
|
+
* Supplied by the query path. The assertion has to live on the open path
|
|
225
|
+
* because that is the one place every reader passes through: `writeMeta`
|
|
226
|
+
* and `assertSchemaCompatible` were both dead in production, so an index
|
|
227
|
+
* built with `e5` and queried with `nomic` returned silently worse
|
|
228
|
+
* results while the KDoc claimed the mismatch "must be refused loudly".
|
|
229
|
+
*/
|
|
230
|
+
expected?: {
|
|
231
|
+
dimensions?: number;
|
|
232
|
+
model?: string;
|
|
233
|
+
prefixMode?: PrefixMode;
|
|
234
|
+
};
|
|
235
|
+
});
|
|
236
|
+
/**
|
|
237
|
+
* Whether the lexical half can actually run in this build.
|
|
238
|
+
*
|
|
239
|
+
* Exposed because the caller's "is anything searchable at all" question is a
|
|
240
|
+
* question about CAPABILITY, not about whether a list came back empty. Those
|
|
241
|
+
* are different facts and conflating them produced a false statement of
|
|
242
|
+
* mechanism: a legitimate no-match with FTS5 present was reported as
|
|
243
|
+
* "no lexical fallback exists".
|
|
244
|
+
*/
|
|
245
|
+
get lexicalAvailable(): boolean;
|
|
246
|
+
get isOpen(): boolean;
|
|
247
|
+
/** Open the database, creating and migrating it when writable. */
|
|
248
|
+
open(): Database.Database;
|
|
249
|
+
/**
|
|
250
|
+
* Whether the index has ever been BUILT — i.e. carries metadata ROWS.
|
|
251
|
+
*
|
|
252
|
+
* Not "does the table exist": `migrate()` creates the table on every open, so
|
|
253
|
+
* a table check is true on a brand-new index and would make
|
|
254
|
+
* `assertSchemaCompatible` throw "no metadata, build it first" on the very
|
|
255
|
+
* run that exists to build it. The distinction that matters is rows, because
|
|
256
|
+
* rows mean vectors with a stated provenance.
|
|
257
|
+
*/
|
|
258
|
+
private hasMetaRows;
|
|
259
|
+
close(): void;
|
|
260
|
+
private handle;
|
|
261
|
+
private migrate;
|
|
262
|
+
/** Read the stored metadata, or undefined on a fresh index. */
|
|
263
|
+
meta(): IndexMeta | undefined;
|
|
264
|
+
/** Write the metadata for a fresh index. Refuses to overwrite silently. */
|
|
265
|
+
writeMeta(meta: IndexMeta): void;
|
|
266
|
+
/**
|
|
267
|
+
* Record what the remote corpus looked like when this sync finished.
|
|
268
|
+
*
|
|
269
|
+
* Written through the same upsert as `writeMeta` and in its own transaction,
|
|
270
|
+
* because the two are separate facts with separate lifetimes: the identity is
|
|
271
|
+
* written BEFORE the work (so the index never holds vectors whose provenance
|
|
272
|
+
* it cannot state) while the watermark can only be written AFTER it. A single
|
|
273
|
+
* transaction spanning both would mean the identity could not be committed
|
|
274
|
+
* until the corpus had been read.
|
|
275
|
+
*/
|
|
276
|
+
writeWatermark(watermark: Watermark): void;
|
|
277
|
+
/**
|
|
278
|
+
* Read the stored watermark, or undefined when this index has never recorded
|
|
279
|
+
* one.
|
|
280
|
+
*
|
|
281
|
+
* Undefined is the honest answer for an index built before the watermark
|
|
282
|
+
* existed, and it must STAY honest: the caller runs the full diff in that
|
|
283
|
+
* case. Defaulting to `{count: 0}` would claim the corpus was empty, and the
|
|
284
|
+
* first sync after an upgrade would then skip on nothing having changed —
|
|
285
|
+
* silently, permanently.
|
|
286
|
+
*/
|
|
287
|
+
watermark(): Watermark | undefined;
|
|
288
|
+
/**
|
|
289
|
+
* Refuse to operate on an index whose schema or dimension we do not know.
|
|
290
|
+
* An unknown schema version is the one condition that must never degrade.
|
|
291
|
+
*/
|
|
292
|
+
assertSchemaCompatible(expected?: {
|
|
293
|
+
dimensions?: number;
|
|
294
|
+
model?: string;
|
|
295
|
+
prefixMode?: PrefixMode;
|
|
296
|
+
}): IndexMeta;
|
|
297
|
+
/** `{ docId → modified }` for every indexed document. */
|
|
298
|
+
indexed(): Map<number, string>;
|
|
299
|
+
/** `{ docId → metadataAt }`, needed by the 24 h metadata refresh. */
|
|
300
|
+
metadataStamps(): Map<number, string>;
|
|
301
|
+
/**
|
|
302
|
+
* Delete every chunk of a document. Returns the number of vector rows removed.
|
|
303
|
+
*
|
|
304
|
+
* `changes` is asserted rather than ignored. The recon note claimed a bound
|
|
305
|
+
* delete silently returns 0 — re-measurement showed `changes` is accurate,
|
|
306
|
+
* but the discipline is worth keeping regardless: a delete that reports
|
|
307
|
+
* success and removes nothing is the same failure family as a `matches: 0`
|
|
308
|
+
* on a guessed pattern, and the instrument is what proves it landed.
|
|
309
|
+
*/
|
|
310
|
+
deleteDocumentChunks(docId: number): number;
|
|
311
|
+
/**
|
|
312
|
+
* Replace a document and all of its chunks atomically.
|
|
313
|
+
*
|
|
314
|
+
* A vec0 rowid cannot be overwritten — verified, a duplicate throws a UNIQUE
|
|
315
|
+
* constraint and `insert or replace` throws it too — so a replace is always
|
|
316
|
+
* a delete followed by an insert, and both must be in one transaction or a
|
|
317
|
+
* crashed sync leaves an index that is missing a document body while its
|
|
318
|
+
* `documents` row still claims it is complete.
|
|
319
|
+
*/
|
|
320
|
+
replaceDocument(doc: IndexedDocument, chunks: StoredChunk[]): void;
|
|
321
|
+
/**
|
|
322
|
+
* Replace only the metadata chunk (position 0) and stamp `metadata_at`.
|
|
323
|
+
* Used by the 24 h metadata refresh, which exists because tags,
|
|
324
|
+
* correspondents and document types change without touching `modified`.
|
|
325
|
+
*/
|
|
326
|
+
replaceMetadata(docId: number, chunk: StoredChunk, metadataAt: string): void;
|
|
327
|
+
/** Delete documents and everything belonging to them, atomically. */
|
|
328
|
+
deleteDocuments(ids: number[]): void;
|
|
329
|
+
/**
|
|
330
|
+
* The stored chunks of one document, in position order.
|
|
331
|
+
*
|
|
332
|
+
* A read accessor, and it exists because `textHits`/`vectorHits` return only
|
|
333
|
+
* the BEST chunk per document -- correct for ranking, useless for asserting
|
|
334
|
+
* what a document actually contains. Without it a test can only observe the
|
|
335
|
+
* metadata chunk by getting lucky with a query.
|
|
336
|
+
*/
|
|
337
|
+
chunksFor(docId: number): {
|
|
338
|
+
position: number;
|
|
339
|
+
kind: "metadata" | "content";
|
|
340
|
+
text: string;
|
|
341
|
+
}[];
|
|
342
|
+
/**
|
|
343
|
+
* How many indexed chunks contain each term.
|
|
344
|
+
*
|
|
345
|
+
* Exists so the passage anchor can pick the MOST DISTINCTIVE query term
|
|
346
|
+
* instead of merely the longest. Measured on the live corpus, length is a bad
|
|
347
|
+
* proxy: for "what did the doctor say about the ankle" it chose `doctor`
|
|
348
|
+
* (1,903 chunks) over `ankle` (808) purely because it is one character
|
|
349
|
+
* longer, and for "the practice of a wrathful deity with a lion head" it
|
|
350
|
+
* chose `practice` (36,459 chunks) over `wrathful` (4,657). Rarity agreed
|
|
351
|
+
* with a human read in 3 of 3 cases; length in 0 of 3.
|
|
352
|
+
*
|
|
353
|
+
* One indexed lookup per term, and `toFtsMatch` already caps a query at 32
|
|
354
|
+
* terms, so this cannot become an unbounded scan. A term absent from the
|
|
355
|
+
* index reports 0 rather than throwing -- "not in the corpus" is a legitimate
|
|
356
|
+
* and very informative answer here.
|
|
357
|
+
*/
|
|
358
|
+
termChunkCounts(terms: string[]): Map<string, number>;
|
|
359
|
+
/**
|
|
360
|
+
* How many indexed documents were cut off at the chunk cap.
|
|
361
|
+
*
|
|
362
|
+
* Exposed because truncation is the one index fact that changes what a caller
|
|
363
|
+
* can conclude from a miss: a truncated document lost its tail, so "no such
|
|
364
|
+
* text" may be a false negative rather than an absence.
|
|
365
|
+
*/
|
|
366
|
+
truncatedCount(): number;
|
|
367
|
+
counts(): Counts;
|
|
368
|
+
/**
|
|
369
|
+
* The best chunk of each document by vector similarity.
|
|
370
|
+
*
|
|
371
|
+
* Per-document dedupe is deliberate: a chunk list lets one long document fill
|
|
372
|
+
* the result set, and a document that matches only by meaning then drops out.
|
|
373
|
+
* The reference implementation does this in Postgres with `DISTINCT ON`; here
|
|
374
|
+
* it is a window function over the kNN result.
|
|
375
|
+
*
|
|
376
|
+
* `score` is `1 − distance` — a similarity. The raw `distance` is never
|
|
377
|
+
* handed to a caller as a score, because on a cosine table 0 means identical
|
|
378
|
+
* and 1 means orthogonal, which reads exactly backwards.
|
|
379
|
+
*/
|
|
380
|
+
vectorHits(embedding: number[], limit: number): Hit[];
|
|
381
|
+
/** One kNN round trip at a given candidate budget. */
|
|
382
|
+
private vectorHitsAt;
|
|
383
|
+
/**
|
|
384
|
+
* The best chunk of each document by BM25.
|
|
385
|
+
*
|
|
386
|
+
* FTS5's `bm25()` returns a NEGATIVE score where more negative is better, so
|
|
387
|
+
* it is negated to a positive "higher is better" value. It is NOT comparable
|
|
388
|
+
* to the vector similarity in magnitude — only in order — which is why the
|
|
389
|
+
* ranker fuses the two lists by rank rather than by score.
|
|
390
|
+
*/
|
|
391
|
+
textHits(query: string, limit: number): Hit[];
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Turn a user's words into an FTS5 MATCH expression.
|
|
395
|
+
*
|
|
396
|
+
* Raw user text is NOT a valid MATCH expression — measured: `c++ (test)` throws
|
|
397
|
+
* `fts5: syntax error near "+"`, `a-b` throws `no such column: b`, and an empty
|
|
398
|
+
* string throws. So the query is tokenised into bare terms and every term is
|
|
399
|
+
* quoted, which makes FTS5 treat it as a literal rather than as query syntax.
|
|
400
|
+
* Terms are OR-ed because a user typing three words wants documents matching
|
|
401
|
+
* any of them ranked by BM25, not only those matching all three.
|
|
402
|
+
*
|
|
403
|
+
* `remove_diacritics 2` means `titulo` matches `Título`; verified both
|
|
404
|
+
* directions on this build.
|
|
405
|
+
*/
|
|
406
|
+
export declare function toFtsMatch(query: string): string | null;
|