@retiform/paperless-mcp 2.3.6 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +301 -4
  2. package/build/api/PaperlessAPI.js +1 -6
  3. package/build/search/build-outcome.d.ts +28 -0
  4. package/build/search/build-outcome.js +31 -0
  5. package/build/search/chunk.d.ts +16 -0
  6. package/build/search/chunk.js +64 -0
  7. package/build/search/config.d.ts +120 -0
  8. package/build/search/config.js +264 -0
  9. package/build/search/embedder.d.ts +346 -0
  10. package/build/search/embedder.js +734 -0
  11. package/build/search/rank.d.ts +11 -0
  12. package/build/search/rank.js +87 -0
  13. package/build/search/store.d.ts +406 -0
  14. package/build/search/store.js +847 -0
  15. package/build/search/sync.d.ts +204 -0
  16. package/build/search/sync.js +819 -0
  17. package/build/search/webhook.d.ts +193 -0
  18. package/build/search/webhook.js +445 -0
  19. package/build/server.d.ts +29 -0
  20. package/build/server.js +77 -0
  21. package/build/tools/documents.d.ts +13 -3
  22. package/build/tools/documents.js +23 -6
  23. package/build/tools/search.d.ts +518 -0
  24. package/build/tools/search.js +1199 -0
  25. package/build/tools/utils/middlewares.d.ts +5 -0
  26. package/build/tools/utils/middlewares.js +282 -9
  27. package/package.json +9 -3
  28. package/paperless-mcp.dxt +0 -0
  29. package/build/api/documentEnhancer.test.d.ts +0 -1
  30. package/build/api/documentEnhancer.test.js +0 -80
  31. package/build/manifest.test.d.ts +0 -1
  32. package/build/manifest.test.js +0 -66
  33. package/build/resources/documents.test.d.ts +0 -1
  34. package/build/resources/documents.test.js +0 -124
  35. package/build/server.test.d.ts +0 -1
  36. package/build/server.test.js +0 -54
  37. package/build/tools/boundedContent.test.d.ts +0 -1
  38. package/build/tools/boundedContent.test.js +0 -565
  39. package/build/tools/documents.test.d.ts +0 -1
  40. package/build/tools/documents.test.js +0 -658
  41. package/build/tools/mail.test.d.ts +0 -1
  42. package/build/tools/mail.test.js +0 -212
  43. package/build/tools/notes.test.d.ts +0 -1
  44. package/build/tools/notes.test.js +0 -177
  45. package/build/tools/schemaCompat.test.d.ts +0 -1
  46. package/build/tools/schemaCompat.test.js +0 -59
  47. package/build/tools/utils/monetary.test.d.ts +0 -1
  48. package/build/tools/utils/monetary.test.js +0 -25
  49. package/build/tools/utils/resourceUri.test.d.ts +0 -1
  50. package/build/tools/utils/resourceUri.test.js +0 -50
  51. package/build/tools/utils/selectFields.test.d.ts +0 -1
  52. package/build/tools/utils/selectFields.test.js +0 -164
@@ -0,0 +1,1199 @@
1
+ "use strict";
2
+ // Semantic search tools over the local vector index.
3
+ //
4
+ // Three tools, and one of them is the whole reason this file exists:
5
+ //
6
+ // semantic_search meaning + words, fused, with a degradation contract
7
+ // embedding_status what the index actually holds
8
+ // sync_embeddings bring the index up to date, explicitly
9
+ //
10
+ // ── The degradation contract ───────────────────────────────────────────────
11
+ // The embedding server being down is EXPECTED, not exceptional. It is a local
12
+ // process, restarted for model changes and absent on a fresh machine. So:
13
+ //
14
+ // * No tool here throws because Ollama is down. A thrown error from an MCP
15
+ // tool reads as "the tool is broken", which is a lie about a tool that is
16
+ // working correctly and reporting a real condition.
17
+ // * Lexical results are STILL RETURNED, with `semantic: "unavailable"`. The
18
+ // lexical half lives in the same SQLite file and does not need the
19
+ // embedder, so an outage degrades the ranking, not the capability.
20
+ // * A degraded call NEVER returns a silent empty list. An empty result and
21
+ // "the embedder is down" are different facts and the caller cannot tell
22
+ // them apart from `passages: []` — which is the exact lie this contract
23
+ // exists to prevent. `semantic` and `degraded_reason` carry it instead.
24
+ //
25
+ // ── Return shape ───────────────────────────────────────────────────────────
26
+ // `passages[]` mirrors `search_passages` exactly — { id, title, score?,
27
+ // highlights?, passage, offset, content_length } — so a client that has learned
28
+ // one has learned the other. The one addition is `matched`, which earns its
29
+ // place because it tells the caller whether to trust the hit or re-query
30
+ // lexically: "words" means the hit is a lexical match the vector side did not
31
+ // find, so a caller looking for a paraphrase should not conclude absence.
32
+ //
33
+ // ── Lazy store open ────────────────────────────────────────────────────────
34
+ // The gateway launches this package's child at startup. Opening a ~1 GB SQLite
35
+ // file and loading the vec0 extension at import time could exceed its child
36
+ // startup tolerance, so nothing here opens the store until the first tool call,
37
+ // and the open is memoised. That is a named risk in the plan, not tidiness.
38
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
39
+ if (k2 === undefined) k2 = k;
40
+ var desc = Object.getOwnPropertyDescriptor(m, k);
41
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
42
+ desc = { enumerable: true, get: function() { return m[k]; } };
43
+ }
44
+ Object.defineProperty(o, k2, desc);
45
+ }) : (function(o, m, k, k2) {
46
+ if (k2 === undefined) k2 = k;
47
+ o[k2] = m[k];
48
+ }));
49
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
50
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
51
+ }) : function(o, v) {
52
+ o["default"] = v;
53
+ });
54
+ var __importStar = (this && this.__importStar) || (function () {
55
+ var ownKeys = function(o) {
56
+ ownKeys = Object.getOwnPropertyNames || function (o) {
57
+ var ar = [];
58
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
59
+ return ar;
60
+ };
61
+ return ownKeys(o);
62
+ };
63
+ return function (mod) {
64
+ if (mod && mod.__esModule) return mod;
65
+ var result = {};
66
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
67
+ __setModuleDefault(result, mod);
68
+ return result;
69
+ };
70
+ })();
71
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
72
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
73
+ return new (P || (P = Promise))(function (resolve, reject) {
74
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
75
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
76
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
77
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
78
+ });
79
+ };
80
+ var __rest = (this && this.__rest) || function (s, e) {
81
+ var t = {};
82
+ for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
83
+ t[p] = s[p];
84
+ if (s != null && typeof Object.getOwnPropertySymbols === "function")
85
+ for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
86
+ if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
87
+ t[p[i]] = s[p[i]];
88
+ }
89
+ return t;
90
+ };
91
+ Object.defineProperty(exports, "__esModule", { value: true });
92
+ exports.METADATA_BATCH = exports.SearchContext = exports.PASSAGE_OVERHEAD_CHARS = exports.MAX_RESULT_CHARS = exports.MAX_LIMIT = exports.DEFAULT_LIMIT = exports.DEFAULT_WINDOW = exports.STRONG_MATCH = void 0;
93
+ exports.budgetedLimit = budgetedLimit;
94
+ exports.buildPassage = buildPassage;
95
+ exports.queryTerms = queryTerms;
96
+ exports.runSemanticSearch = runSemanticSearch;
97
+ exports.registerSearchTools = registerSearchTools;
98
+ exports.buildDocumentFilter = buildDocumentFilter;
99
+ exports.loadVisibleDocuments = loadVisibleDocuments;
100
+ exports.loadDocumentMetadata = loadDocumentMetadata;
101
+ exports.reconstructDocumentText = reconstructDocumentText;
102
+ exports.indexTextLoader = indexTextLoader;
103
+ exports.fitResultToBudget = fitResultToBudget;
104
+ exports.embeddingStatus = embeddingStatus;
105
+ const zod_1 = require("zod");
106
+ const config_1 = require("../search/config");
107
+ const embedder_1 = require("../search/embedder");
108
+ const store_1 = require("../search/store");
109
+ const chunk_1 = require("../search/chunk");
110
+ const rank_1 = require("../search/rank");
111
+ const sync_1 = require("../search/sync");
112
+ const webhook_1 = require("../search/webhook");
113
+ const middlewares_1 = require("./utils/middlewares");
114
+ /** A similarity at or above this is a strong match. Similarity, never distance.
115
+ *
116
+ * ⚠️ NOT USED AS A GATE ANY MORE, and the measurement is why. On this corpus
117
+ * the floor cannot separate a good query from a bad one. Measured on the live
118
+ * index (2,476 docs), best `1 - distance` per query:
119
+ *
120
+ * relevant "kuji nine invisibility…" 0.700 "naga offering…" 0.746
121
+ * "barber pole worm in goats…" 0.761 "channels and winds" 0.766
122
+ * "three humours and prana disorders" 0.731
123
+ * irrelevant "xyzzy plugh frobnicate quux" 0.599 "stock market options" 0.595
124
+ * "quantum chromodynamics…" 0.656 "best pizza dough…" 0.766
125
+ * "how to change a flat tire" 0.730
126
+ *
127
+ * The ranges OVERLAP: two of five irrelevant queries score above the worst
128
+ * relevant one, and no threshold in 0.500–0.800 separates the sets. A floor at
129
+ * 0.45 sits 0.145 BELOW the worst irrelevant query, so it could never fire —
130
+ * `strong_match` was constant `true` whenever the vector arm ran.
131
+ *
132
+ * Kept exported only so existing importers compile; treat any use of it as a
133
+ * claim that needs this card's measurement re-run on the corpus in question.
134
+ */
135
+ exports.STRONG_MATCH = 0.45;
136
+ /** Chars per returned passage, matching search_passages' default. */
137
+ exports.DEFAULT_WINDOW = 800;
138
+ exports.DEFAULT_LIMIT = 10;
139
+ exports.MAX_LIMIT = 50;
140
+ /**
141
+ * The largest JSON body a caller can be handed, in characters.
142
+ *
143
+ * This is not our budget to choose freely: it is the ceiling the CONSUMER
144
+ * applies. Mneme truncates any non-memory tool result at 20,000 characters by
145
+ * taking head and tail, and the agent on the other side cannot detect that it
146
+ * happened -- a passage that was silently cut reads exactly like a short one.
147
+ *
148
+ * `MAX_LIMIT * max window` is 208,485 characters, ten times this number, so
149
+ * until this constant existed the tool's own advertised argument range could
150
+ * produce a result the client would mangle. Bounding it in the schema
151
+ * description would be advice; the enforcement is `budgetedLimit` (so a
152
+ * maximal call does not waste work) plus `fitResultToBudget` (so the returned
153
+ * bytes are measured, not predicted).
154
+ */
155
+ exports.MAX_RESULT_CHARS = 20000;
156
+ /**
157
+ * JSON scaffolding a single passage costs, roughly: id, title, score, matched,
158
+ * offset, content_length, offset_space, quotes and commas.
159
+ *
160
+ * Used only to pick the passage COUNT before any work is done. `PASSAGE_OVERHEAD_CHARS` is deliberately
161
+ * generous: over-estimating costs a passage, under-estimating costs a wasted
162
+ * build that `fitResultToBudget` then discards anyway.
163
+ */
164
+ exports.PASSAGE_OVERHEAD_CHARS = 200;
165
+ /**
166
+ * Fraction of MAX_RESULT_CHARS that `budgetedLimit` will spend on passages,
167
+ * leaving the rest for the envelope (query, counts, retriever list, reasons).
168
+ */
169
+ const RESULT_BUDGET_FRACTION = 0.9;
170
+ /**
171
+ * How many passages of `window` characters can fit inside the consumer's
172
+ * ceiling. Never zero: a call that cannot afford even one passage should still
173
+ * return one and let `fitResultToBudget` trim if the bytes really do not fit.
174
+ */
175
+ function budgetedLimit(limit, window) {
176
+ const ceiling = Math.floor((exports.MAX_RESULT_CHARS * RESULT_BUDGET_FRACTION) / (window + exports.PASSAGE_OVERHEAD_CHARS));
177
+ return Math.max(1, Math.min(limit, exports.MAX_LIMIT, ceiling));
178
+ }
179
+ /**
180
+ * The filter arguments, mirroring `search_passages`.
181
+ *
182
+ * Kept in one object so the two tools cannot drift: if search_passages gains a
183
+ * filter, this is the one place that has to learn it.
184
+ *
185
+ * ── WHERE THEY ARE APPLIED, AND WHAT THAT COSTS ────────────────────────────
186
+ * They are evaluated in JS against each hit's fetched document, NOT pushed into
187
+ * the kNN statement. That is a deliberate trade with a known cost, stated here
188
+ * because the previous comment argued against this approach while describing
189
+ * it as the approach:
190
+ *
191
+ * PRO The store's kNN returns one hit per document, and the documents are
192
+ * fetched anyway to build passages, so the filter is free at the point
193
+ * it is needed and needs no new SQL.
194
+ * CON `rowid IN (…)` is NOT composed into the MATCH statement, so the
195
+ * filtered set can come back SHORT of `limit` with no signal that
196
+ * ranking was trimmed. This is why `filtered_out` and its reason
197
+ * breakdown exist: the shortfall is reported rather than hidden.
198
+ *
199
+ * Pushing the filter into the kNN would remove the CON, at the cost of a
200
+ * per-query SQL shape and of needing each filter to be expressible as a set of
201
+ * document ids before the query runs. That is the right eventual design; it is
202
+ * not this change.
203
+ *
204
+ * ⚠️ EVERY KEY HERE MUST BE HONOURED BY `buildDocumentFilter`. A key that is
205
+ * advertised to the model and silently ignored is worse than an absent one: the
206
+ * caller gets unfiltered results with no error, no warning, and no way to
207
+ * notice. `storage_path` was exactly that until it was implemented.
208
+ */
209
+ const FILTER_SHAPE = {
210
+ correspondent: zod_1.z.number().optional(),
211
+ document_type: zod_1.z.number().optional(),
212
+ tag: zod_1.z.number().optional(),
213
+ storage_path: zod_1.z.number().optional(),
214
+ created__date__gte: zod_1.z.string().optional(),
215
+ created__date__lte: zod_1.z.string().optional(),
216
+ };
217
+ /**
218
+ * Cut a bounded passage around the most DISTINCTIVE place the query terms appear.
219
+ *
220
+ * Mirrors buildSearchPassage's anchoring priority minus the server highlight,
221
+ * which does not exist on this path: the vector side has no highlighted span,
222
+ * and the lexical side's BM25 match gives a chunk rather than an offset. So the
223
+ * anchor is (1) the RAREST matching query term, (2) the head of the document.
224
+ *
225
+ * Rarity, not query order and not length. Query order was the original defect:
226
+ * English queries open with articles, so first-match anchored "the practice of
227
+ * a wrathful deity with a lion head" on "the" at character 3, and containment
228
+ * still passed because "the" really was in the window. Length was my first
229
+ * correction and it is also wrong -- measured live, it picks `doctor` over
230
+ * `ankle` by one character. Rarity picks the term the query is ABOUT.
231
+ * When the caller cannot supply counts, length is the fallback.
232
+ *
233
+ * A window that does not contain the hit is worse than no window, which is why
234
+ * the term search runs against the whole document rather than the chunk.
235
+ */
236
+ function buildPassage(doc, terms, window,
237
+ /**
238
+ * How many indexed chunks contain each term, when the caller can supply it.
239
+ *
240
+ * ⚠️ THIS IS THE CORRECT DISCRIMINATOR AND LENGTH IS NOT. Measured on the
241
+ * live corpus: "what did the doctor say about the ankle" picked `doctor`
242
+ * (1,903 chunks) over `ankle` (808) on length alone -- wrong, and wrong in
243
+ * the way that matters, because the window then lands on a discussion of
244
+ * doctors rather than of the ankle. "the practice of a wrathful deity with a
245
+ * lion head" picked `practice` (36,459) over `wrathful` (4,657). Rarity
246
+ * matched a human read in 3 of 3 probes; length in 0 of 3.
247
+ *
248
+ * Optional so a caller with no index (the search_passages path, whose terms
249
+ * come from the server's own query) still works -- it then falls back to
250
+ * length, which is worse but still better than query order.
251
+ */
252
+ termRarity) {
253
+ var _a;
254
+ const full = (_a = doc.content) !== null && _a !== void 0 ? _a : "";
255
+ const lower = full.toLowerCase();
256
+ let center = -1;
257
+ let found;
258
+ // ANCHOR ON THE MOST SPECIFIC TERM, not the first one in query order.
259
+ //
260
+ // Measured defect this fixes: the query "the practice of a wrathful deity with
261
+ // a lion head" anchored on "the" at character 3, because `indexOf` walks the
262
+ // query left-to-right and English queries open with articles. 11 of 18 windows
263
+ // landed at the document head. Containment still passed -- "the" really is in
264
+ // the window -- which is exactly why no test caught it: the property that
265
+ // holds is the weak one, and the one that matters (a window around the
266
+ // DISTINCTIVE term) was never asserted.
267
+ //
268
+ // Length is the cheap proxy for specificity here, and it is the right one:
269
+ // a longer term is what distinguishes this query from most others. Ties keep
270
+ // query order via the stable sort. If nothing matches, offset 0 and say so --
271
+ // a head window is honest when the query genuinely appears nowhere.
272
+ // Rarity first (rarest in the corpus = most specific to this query), then
273
+ // length as a tie-break, then query order via the stable sort. A term the
274
+ // caller could not score sorts on length alone.
275
+ const rarityOf = (t) => { var _a; return (_a = termRarity === null || termRarity === void 0 ? void 0 : termRarity.get(t.toLowerCase())) !== null && _a !== void 0 ? _a : Number.MAX_SAFE_INTEGER; };
276
+ const candidates = [...terms].sort((a, b) => {
277
+ const ra = rarityOf(a);
278
+ const rb = rarityOf(b);
279
+ if (ra !== rb)
280
+ return ra - rb;
281
+ return b.length - a.length;
282
+ });
283
+ for (const term of candidates) {
284
+ const at = lower.indexOf(term.toLowerCase());
285
+ if (at >= 0) {
286
+ center = at;
287
+ found = term;
288
+ break;
289
+ }
290
+ }
291
+ const resolved = center < 0 ? 0 : center;
292
+ const start = Math.max(0, resolved - Math.floor(window / 2));
293
+ // The end is widened only when doing so is required to CONTAIN the term it
294
+ // is anchoring on. A window that stops one character short of the match it
295
+ // was centred for is the exact defect the containment rule forbids, and with
296
+ // a term longer than `window` the plain `start + window` would do it.
297
+ const end = Math.max(start + window, resolved + (found ? found.length : 0));
298
+ return Object.assign({ passage: full.slice(start, end), offset: start, content_length: full.length }, (found ? { term: found } : {}));
299
+ }
300
+ /** Query terms for passage anchoring, from the same tokenisation the FTS side uses. */
301
+ function queryTerms(query) {
302
+ return query
303
+ .toLowerCase()
304
+ // \p{M} is required for the same reason it is in store.toFtsMatch: the
305
+ // anchoring terms and the lexical terms must tokenise identically, or a
306
+ // passage is anchored on one fragment of a syllable and matched on another.
307
+ .split(/[^\p{L}\p{N}\p{M}_]+/u)
308
+ .filter((term) => term.length > 0);
309
+ }
310
+ function errorMessage(err) {
311
+ return err instanceof Error ? err.message : String(err);
312
+ }
313
+ /**
314
+ * Everything the three tools share, built once and lazily.
315
+ *
316
+ * `store` is `undefined` until the first call. `openFailed` remembers a failure
317
+ * so a broken index does not retry a 1 GB open on every single tool call -- and
318
+ * remembers the MESSAGE, so embedding_status can report why.
319
+ */
320
+ class SearchContext {
321
+ /**
322
+ * `embedderFactory` exists so "no embedder" can be stated rather than
323
+ * inferred. A context whose embedder is merely unset would reach
324
+ * `embedderOrUndefined` and CONSTRUCT a real client — which then makes a
325
+ * network call. That is correct in production and unacceptable in a test, and
326
+ * the difference has to be expressible, so the factory can return undefined
327
+ * and mean it.
328
+ */
329
+ constructor(config, embedderFactory) {
330
+ this.syncStarted = false;
331
+ this.webhookStarted = false;
332
+ this.configOverride = config;
333
+ this.embedderFactory = embedderFactory;
334
+ }
335
+ /** State the embedder explicitly. `null` means "there is deliberately none". */
336
+ setEmbedder(embedder) {
337
+ this.embedder = embedder;
338
+ this.sync = undefined;
339
+ }
340
+ /**
341
+ * Resolve config from the environment at CALL time.
342
+ *
343
+ * Not at import: the gateway sets the child's environment before launch, but
344
+ * tests and the CLI set it after import, and reading once at module load
345
+ * makes the config untestable.
346
+ */
347
+ config() {
348
+ var _a;
349
+ return (_a = this.configOverride) !== null && _a !== void 0 ? _a : (0, config_1.searchConfig)(process.env);
350
+ }
351
+ /** True when the search tools should register at all. */
352
+ get isConfigured() {
353
+ return this.config() !== undefined;
354
+ }
355
+ /** Open the index, memoised. Throws the underlying error on first failure. */
356
+ index() {
357
+ if (this.store)
358
+ return this.store;
359
+ if (this.openFailed) {
360
+ throw new Error(`The search index could not be opened: ${this.openFailed}`);
361
+ }
362
+ const config = this.config();
363
+ if (!config) {
364
+ throw new Error("Semantic search is not configured. Set PAPERLESS_MCP_EMBEDDING_URL " +
365
+ "and the other PAPERLESS_MCP_* search variables to enable it.");
366
+ }
367
+ // `expected` is what makes the deployment policy-adjacent schema guard REAL at query
368
+ // time. Without it, an index built with `e5` and queried with `nomic`
369
+ // returned `semantic: "ok"` and quietly worse results, while
370
+ // `assertPrefixModeMatches`'s own KDoc said the mismatch "must be refused
371
+ // loudly". Same for a same-width model swap and a bumped schema_version.
372
+ const store = new store_1.SearchIndex((0, config_1.indexPath)(config), {
373
+ expected: {
374
+ dimensions: store_1.EMBEDDING_DIMENSIONS,
375
+ model: config.model,
376
+ prefixMode: config.prefixMode,
377
+ },
378
+ });
379
+ try {
380
+ store.open();
381
+ }
382
+ catch (err) {
383
+ this.openFailed = errorMessage(err);
384
+ throw err;
385
+ }
386
+ this.store = store;
387
+ return store;
388
+ }
389
+ embedderOrUndefined() {
390
+ var _a;
391
+ // `null` is memoised and means "deliberately none" — checked explicitly so
392
+ // it never falls through to construction.
393
+ if (this.embedder === null)
394
+ return undefined;
395
+ if (this.embedder)
396
+ return this.embedder;
397
+ const config = this.config();
398
+ if (!config)
399
+ return undefined;
400
+ try {
401
+ const built = this.embedderFactory
402
+ ? this.embedderFactory(config)
403
+ : new embedder_1.OllamaEmbedder(Object.assign({ url: config.embeddingUrl, model: config.model, prefixMode: config.prefixMode,
404
+ // Threaded rather than defaulted here: the value is read and
405
+ // range-checked in config.ts, and a knob that is validated but
406
+ // never passed on is an advertised setting with no effect.
407
+ batchSize: config.syncBatchSize, localOnly: config.embeddingLocalOnly }, (config.embeddingApiKey ? { apiKey: config.embeddingApiKey } : {})));
408
+ this.embedder = built !== null && built !== void 0 ? built : null;
409
+ }
410
+ catch (_b) {
411
+ // A misconfigured endpoint is not an unavailable embedder, but on this
412
+ // path both mean "no vectors this turn", and the lexical half still works.
413
+ this.embedder = null;
414
+ return undefined;
415
+ }
416
+ return (_a = this.embedder) !== null && _a !== void 0 ? _a : undefined;
417
+ }
418
+ syncRunner(api) {
419
+ if (this.sync)
420
+ return this.sync;
421
+ const embedder = this.embedderOrUndefined();
422
+ if (!embedder) {
423
+ throw new Error("Semantic search cannot sync without an embedder. Check " +
424
+ "PAPERLESS_MCP_EMBEDDING_URL.");
425
+ }
426
+ const config = this.config();
427
+ this.sync = new sync_1.IndexSync(Object.assign({ api, store: this.index(), embedder }, (config ? { maxChunks: config.maxChunks } : {})));
428
+ return this.sync;
429
+ }
430
+ /**
431
+ * Start the background sync timer when the deployment asked for one.
432
+ *
433
+ * ⚠️ THIS METHOD IS THE CALLER `IndexSync.start()` NEVER HAD. The interval was
434
+ * read and range-checked in config.ts from the day it was added, and nothing
435
+ * ever acted on it: `start()` existed, was documented, and was invoked from
436
+ * nowhere. A validated setting with no consumer reads as a working feature
437
+ * and is not one.
438
+ *
439
+ * Called from `registerSearchTools`, so the wiring is exercised by every
440
+ * server that configures search at all. A test that constructs the timer
441
+ * directly proves `start()` works and says nothing about whether anything
442
+ * calls it; the caller is what has to be asserted.
443
+ *
444
+ * A background sync that cannot start must not take down tool registration:
445
+ * the observable `sync_embeddings` tool still works, so failure here is
446
+ * logged and swallowed rather than thrown.
447
+ */
448
+ startBackgroundSync(api) {
449
+ if (this.syncStarted)
450
+ return;
451
+ const config = this.config();
452
+ if (!config || config.syncIntervalMs <= 0)
453
+ return;
454
+ try {
455
+ this.syncRunner(api).start(config.syncIntervalMs);
456
+ this.syncStarted = true;
457
+ }
458
+ catch (err) {
459
+ console.error(`[paperless-mcp] background sync not started: ${errorMessage(err)}`);
460
+ }
461
+ }
462
+ /** True when a background sync timer is live. Used by the wiring test. */
463
+ get isBackgroundSyncActive() {
464
+ var _a;
465
+ return ((_a = this.sync) === null || _a === void 0 ? void 0 : _a.isScheduled) === true;
466
+ }
467
+ /**
468
+ * Start the change-notification receiver when the deployment asked for one.
469
+ *
470
+ * ⚠️ THE SAME CALLER PROBLEM AS `startBackgroundSync`, ONE LAYER OUT. A
471
+ * receiver class nobody constructs is a feature nobody has, and the interval
472
+ * in this file is the standing proof: it was read, validated and never acted
473
+ * on. So construction happens here, on the path `registerSearchTools` always
474
+ * walks, rather than behind a bootstrap only a test would call.
475
+ *
476
+ * Started asynchronously and never awaited. Registration is synchronous and
477
+ * happens before the transport connects; a bind that fails must not delay or
478
+ * break the tool surface, because the observable `sync_embeddings` tool still
479
+ * works and the watermark still converges on the next start without it.
480
+ */
481
+ startWebhook(api) {
482
+ var _a;
483
+ if (this.webhookStarted)
484
+ return;
485
+ const config = this.config();
486
+ if (!config || config.webhookPort === 0)
487
+ return;
488
+ const refusal = (0, config_1.webhookRefusal)(config);
489
+ if (refusal) {
490
+ console.error(`[paperless-mcp] webhook not started: ${refusal}`);
491
+ return;
492
+ }
493
+ // The runner is resolved lazily, per event: building it needs an embedder
494
+ // and an open store, both of which are expensive and neither of which
495
+ // should be paid at startup by an install that never receives an event.
496
+ const receiver = new webhook_1.WebhookReceiver({
497
+ secret: (_a = config.webhookSecret) !== null && _a !== void 0 ? _a : "",
498
+ port: config.webhookPort,
499
+ host: config.webhookHost,
500
+ debounceMs: config.webhookDebounceMs,
501
+ runner: () => this.syncRunner(api),
502
+ });
503
+ this.webhook = receiver;
504
+ this.webhookStarted = true;
505
+ void receiver.start().catch((err) => {
506
+ // A port already in use, or a refused bind. Logged and dropped: the
507
+ // receiver is optional infrastructure and must not kill the server.
508
+ console.error(`[paperless-mcp] webhook not started: ${errorMessage(err)}`);
509
+ this.webhook = undefined;
510
+ this.webhookStarted = false;
511
+ });
512
+ }
513
+ /** True when a receiver is bound. Used by the wiring test. */
514
+ get isWebhookListening() {
515
+ var _a;
516
+ return ((_a = this.webhook) === null || _a === void 0 ? void 0 : _a.isListening) === true;
517
+ }
518
+ /** The port the receiver actually bound, or undefined when it is off. */
519
+ get webhookPort() {
520
+ var _a;
521
+ return (_a = this.webhook) === null || _a === void 0 ? void 0 : _a.boundPort;
522
+ }
523
+ close() {
524
+ var _a, _b;
525
+ // Stop the timer and the receiver BEFORE dropping the store, or a stdio
526
+ // child shutting down leaves a scheduled run firing against a closed index.
527
+ //
528
+ // ⚠️ The receiver's close is ASYNC and `close()` is not, so it is started
529
+ // here and not awaited. That is safe rather than sloppy: the receiver's own
530
+ // `close()` clears its debounce timer synchronously first, so no new run can
531
+ // be scheduled from this point on -- only an already-firing one can still be
532
+ // in flight, and that one is bounded by the store's own `isOpen` state.
533
+ (_a = this.sync) === null || _a === void 0 ? void 0 : _a.stop();
534
+ if (this.webhook) {
535
+ void this.webhook.close().catch(() => {
536
+ // Nothing useful to do while shutting down.
537
+ });
538
+ this.webhook = undefined;
539
+ }
540
+ this.webhookStarted = false;
541
+ (_b = this.store) === null || _b === void 0 ? void 0 : _b.close();
542
+ this.store = undefined;
543
+ this.sync = undefined;
544
+ this.syncStarted = false;
545
+ }
546
+ }
547
+ exports.SearchContext = SearchContext;
548
+ /**
549
+ * Fuse the two lists, tolerating an absent lexical half or an absent embedder.
550
+ *
551
+ * This is the function the contract is built on, and it is deliberately one
552
+ * function: the degradation lives in one place so no caller can accidentally
553
+ * reintroduce a silent empty list.
554
+ */
555
+ function runSemanticSearch(context, query, opts) {
556
+ return __awaiter(this, void 0, void 0, function* () {
557
+ const store = context.index();
558
+ // ⚠️ BUDGETED, not merely clamped. `MAX_LIMIT` alone permits
559
+ // `50 * 4000 = 208,485` characters, which is 10x the ceiling the consuming
560
+ // client applies and silently mangles. This is the first of the two
561
+ // enforcements: it stops the work from being done. `fitResultToBudget` is
562
+ // the second and it measures real bytes.
563
+ const limit = budgetedLimit(Math.min(Math.max(opts.limit, 1), exports.MAX_LIMIT), opts.window);
564
+ const terms = queryTerms(query);
565
+ // ── THE STALENESS REPORT, BUILT BEFORE THE SEARCH SO ONE SPREAD SENDS IT ──
566
+ //
567
+ // Computed once and spread into BOTH return paths (the empty-fused early
568
+ // return and the main one). Computing it twice would be two probes for one
569
+ // answer, and computing it only on the main path would silently omit it from
570
+ // exactly the response where a caller is most likely to wonder whether the
571
+ // index is current — an empty result.
572
+ //
573
+ // ⚠️ A FAILED PROBE YIELDS `stale: null`, NEVER `false`. "I could not check"
574
+ // and "I checked and it is current" are different facts, and a search that
575
+ // fabricates the second because Paperless was down would be lying about the
576
+ // one thing this field exists to report. The search itself is unaffected: the
577
+ // degradation contract already covers a dead embedder, and a dead Paperless
578
+ // costs only this annotation.
579
+ const stalenessFields = opts.checkStaleness
580
+ ? yield readStaleness(opts.corpusState, store, opts.now)
581
+ : {};
582
+ let semantic = "ok";
583
+ const reasons = [];
584
+ let vector = [];
585
+ let text = [];
586
+ // ⚠️ THE THREE STATES, kept distinct, because collapsing them produced a FALSE
587
+ // STATEMENT OF MECHANISM. The old guard fired when BOTH lists came back empty
588
+ // and the embedder was down, and announced "no lexical fallback exists" --
589
+ // which was untrue whenever FTS5 was present and simply had no match. A
590
+ // caller then could not tell "nothing in the corpus says zzzznothing" from
591
+ // "search is broken", which is the exact distinction the degradation contract
592
+ // exists to preserve.
593
+ //
594
+ // (a) BOTH RAN, NOTHING MATCHED -> semantic ok|unavailable, empty list,
595
+ // and a field that says so. ALLOWED.
596
+ // (b) LEXICAL RAN, EMBEDDER DOWN -> results + unavailable + the reason.
597
+ // Allowed; already covered above.
598
+ // (c) NEITHER IS CAPABLE -> refuse with an error. There is
599
+ // genuinely nothing to search with, and
600
+ // an empty list would read as "no such
601
+ // document".
602
+ //
603
+ // The predicate for (c) is CAPABILITY, not emptiness: `store.lexicalAvailable`
604
+ // is a property of the build, and the embedder's absence is a property of the
605
+ // configuration. Neither can be inferred from a list length.
606
+ // The lexical half needs no network and no model. It runs whenever the build
607
+ // supports it, and it is the reason a degraded call still returns something.
608
+ let lexicalCapable = store.lexicalAvailable;
609
+ if (lexicalCapable) {
610
+ try {
611
+ // Over-fetch because filters are applied below: fetching exactly `limit`
612
+ // would let one filtered-out document empty the result.
613
+ text = store.textHits(query, limit * 4);
614
+ }
615
+ catch (err) {
616
+ if (!(err instanceof store_1.LexicalUnavailableError))
617
+ throw err;
618
+ // The build CLAIMED FTS5 and the query path disagreed. Treat the
619
+ // capability as false rather than swallowing it: this is the one place
620
+ // the property could be wrong, and state (c) must not be missed because
621
+ // of it.
622
+ lexicalCapable = false;
623
+ reasons.push(`Lexical search failed despite FTS5 being present: ${errorMessage(err)}`);
624
+ }
625
+ }
626
+ else {
627
+ reasons.push("Lexical search is unavailable in this build: SQLite has no FTS5 module.");
628
+ }
629
+ const embedder = context.embedderOrUndefined();
630
+ let vectorCapable = false;
631
+ if (!embedder) {
632
+ semantic = "unavailable";
633
+ reasons.push("The embedding client is not configured.");
634
+ }
635
+ else {
636
+ try {
637
+ const embedding = yield embedder.embedQuery(query);
638
+ vector = store.vectorHits(embedding, limit * 4);
639
+ vectorCapable = true;
640
+ }
641
+ catch (err) {
642
+ if (!(err instanceof embedder_1.EmbeddingUnavailableError))
643
+ throw err;
644
+ // EXPECTED. The lexical list above already holds whatever could be found.
645
+ semantic = "unavailable";
646
+ reasons.push(errorMessage(err));
647
+ }
648
+ }
649
+ // State (c): neither retriever is capable. Refuse loudly.
650
+ if (!lexicalCapable && !vectorCapable) {
651
+ throw new Error(`Semantic search is fully unavailable: neither the vector side nor a ` +
652
+ `lexical fallback can run. ${reasons.join(" ")}`);
653
+ }
654
+ // One reason string, but FIRST-REASON-WINS is now explicit rather than the
655
+ // `??=` accident it was: `??=` let a lexical-unavailable reason mask a later
656
+ // embedder reason, so a call with no FTS5 AND a dead embedder reported only
657
+ // the lexical half and left the reader guessing about the vectors.
658
+ const degradedReason = reasons.length > 0 ? reasons.join(" ") : undefined;
659
+ const fused = (0, rank_1.fuse)(vector, text);
660
+ // Which retrievers actually RAN. Reported rather than inferred: an empty
661
+ // result with both here is "both searched and neither matched", with one is
662
+ // "only that half could run", and the difference is the whole point.
663
+ const retrievers = [];
664
+ if (vectorCapable)
665
+ retrievers.push("vector");
666
+ if (lexicalCapable)
667
+ retrievers.push("lexical");
668
+ if (fused.length === 0) {
669
+ return Object.assign(Object.assign(Object.assign(Object.assign({ query,
670
+ semantic }, (degradedReason ? { degraded_reason: degradedReason } : {})), { strong_match: false, total_hits: 0, filtered_out: 0, retrievers }), stalenessFields), {
671
+ // State (a) on its own terms: nothing matched and the payload says so —
672
+ // `semantic`, `retrievers`, and an empty list together are unambiguous.
673
+ passages: [] });
674
+ }
675
+ // Anchoring and filters both need METADATA. They do NOT need content, and
676
+ // fetching it per hit was the defect: one `GET /documents/{id}/` per hit, up
677
+ // to 80 for a 10-passage call, each returning a full body that was sliced to
678
+ // a window and thrown away.
679
+ const documents = yield opts.loadDocuments(fused.map((hit) => hit.documentId));
680
+ // The text the window is cut from, from the index. `undefined` rather than an
681
+ // empty map when no loader was injected, so the fallback is explicit.
682
+ const indexText = opts.loadText ? opts.loadText(fused) : undefined;
683
+ const passages = [];
684
+ // ONE rarity lookup for the whole request, not one per document. The anchor
685
+ // needs it per document, but the counts are a property of the corpus and the
686
+ // query, not of the document being windowed.
687
+ const rarity = store.termChunkCounts(terms);
688
+ // Post-fetch accounting. Without it `total_hits: 12, passages: []` was a
689
+ // legal, unexplained response: the caller could not tell that its own filter
690
+ // arguments had removed everything.
691
+ let documentUnavailable = 0;
692
+ let metadataFiltered = 0;
693
+ for (const hit of fused) {
694
+ const doc = documents.get(hit.documentId);
695
+ if (!doc) {
696
+ documentUnavailable += 1;
697
+ continue;
698
+ }
699
+ if (opts.filters && !opts.filters(doc)) {
700
+ metadataFiltered += 1;
701
+ continue;
702
+ }
703
+ const indexed = indexText === null || indexText === void 0 ? void 0 : indexText.get(doc.id);
704
+ const fromIndex = indexed !== undefined;
705
+ const anchored = buildPassage({ id: doc.id, title: doc.title, content: fromIndex ? indexed : doc.content }, terms, opts.window, rarity);
706
+ passages.push(Object.assign(Object.assign({ id: doc.id, title: doc.title, score: hit.score, matched: hit.matched, passage: anchored.passage, offset: anchored.offset, content_length: anchored.content_length }, (anchored.term ? { highlights: anchored.term } : {})), { offset_space: fromIndex ? "normalized" : "document" }));
707
+ if (passages.length >= limit)
708
+ break;
709
+ }
710
+ // ⚠️ `strong_match` is now literally "a passage was returned". The 0.45
711
+ // similarity gate it used to consult could never fire on this corpus — see
712
+ // the STRONG_MATCH note — so the old computation reported `true` on every
713
+ // call that produced a hit, regardless of quality. A field that cannot be
714
+ // false is not evidence.
715
+ const strong = passages.length > 0;
716
+ // `filtered_out` counts what the loop dropped, NOT `total_hits - passages`
717
+ // -- those differ when the `limit` break fires with hits still unexamined.
718
+ // The honest decomposition is: fused = shown + dropped + never-reached.
719
+ const filteredOut = documentUnavailable + metadataFiltered;
720
+ return fitResultToBudget(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ query,
721
+ semantic }, (degradedReason ? { degraded_reason: degradedReason } : {})), { strong_match: strong, total_hits: fused.length, filtered_out: filteredOut }), (filteredOut > 0
722
+ ? { filtered_out_reasons: { document_unavailable: documentUnavailable, metadata_filter: metadataFiltered } }
723
+ : {})), { retrievers }), stalenessFields), { passages }));
724
+ });
725
+ }
726
+ /**
727
+ * The staleness annotation, as a spreadable fragment.
728
+ *
729
+ * Returns the fields with `stale: null` when the corpus cannot be read or the
730
+ * index carries no watermark. Neither invents a value.
731
+ *
732
+ * ⚠️ AN INDEX WITH NO WATERMARK IS `stale: null`, NOT `stale: true`. It is
733
+ * genuinely unknown — the index predates the watermark, or has never completed
734
+ * a run — and asserting either answer would be a guess dressed as a measurement.
735
+ * The actionable thing for such a caller is `sync_embeddings`, and
736
+ * `staleness_reason` says exactly that.
737
+ */
738
+ function readStaleness(corpusState, store, now) {
739
+ return __awaiter(this, void 0, void 0, function* () {
740
+ var _a, _b;
741
+ if (!corpusState) {
742
+ return {
743
+ stale: null,
744
+ index_age_s: null,
745
+ live_count: null,
746
+ indexed_count: null,
747
+ staleness_reason: "No corpus probe was available to this call.",
748
+ };
749
+ }
750
+ const live = yield corpusState();
751
+ const watermark = store.watermark();
752
+ const indexedCount = (_a = watermark === null || watermark === void 0 ? void 0 : watermark.count) !== null && _a !== void 0 ? _a : null;
753
+ if (!watermark) {
754
+ return {
755
+ stale: null,
756
+ index_age_s: null,
757
+ live_count: (_b = live === null || live === void 0 ? void 0 : live.count) !== null && _b !== void 0 ? _b : null,
758
+ indexed_count: null,
759
+ staleness_reason: "The index has no recorded watermark (it predates one, or has never " +
760
+ "completed a sync), so staleness cannot be determined. Run " +
761
+ "sync_embeddings.",
762
+ };
763
+ }
764
+ // Age is computable from the stored timestamp alone, so it is reported even
765
+ // when the live probe failed — it is a fact about the index, not about the
766
+ // corpus.
767
+ const syncedAt = new Date(watermark.at).getTime();
768
+ const indexAge = Number.isFinite(syncedAt)
769
+ ? Math.max(0, Math.round(((now !== null && now !== void 0 ? now : Date.now)() - syncedAt) / 1000))
770
+ : null;
771
+ if (!live) {
772
+ return {
773
+ stale: null,
774
+ index_age_s: indexAge,
775
+ live_count: null,
776
+ indexed_count: indexedCount,
777
+ staleness_reason: "The live corpus could not be read, so staleness cannot be " +
778
+ "determined. The search results above are unaffected.",
779
+ };
780
+ }
781
+ return {
782
+ stale: (0, sync_1.corpusChanged)(live, watermark),
783
+ index_age_s: indexAge,
784
+ live_count: live.count,
785
+ indexed_count: indexedCount,
786
+ };
787
+ });
788
+ }
789
+ /** Register the three semantic tools. No-op when search is not configured. */
790
+ function registerSearchTools(server, api, context = new SearchContext()) {
791
+ if (!context.isConfigured)
792
+ return context;
793
+ // Opt-in background sync. No-op unless PAPERLESS_MCP_SYNC_INTERVAL_MS > 0.
794
+ context.startBackgroundSync(api);
795
+ // Opt-in change-notification receiver. No-op unless
796
+ // PAPERLESS_MCP_WEBHOOK_PORT > 0, and refuses without a secret.
797
+ context.startWebhook(api);
798
+ const filterArgs = FILTER_SHAPE;
799
+ server.tool("semantic_search", "Search the document corpus by MEANING as well as by words, using a local vector index fused with the corpus' own BM25 lexical search. Returns bounded passages in the same shape as search_passages (id, title, score, passage, offset, content_length) plus `matched`, which says whether the hit came from meaning, words, or both -- a 'words' hit is a lexical match the vector side did not find, so do not read it as absence of a paraphrase. HONEST DEGRADATION: `semantic` is 'unavailable' and lexical results are still returned when the local embedding server is down; a degraded call never returns a silent empty list. strong_match is true exactly when at least one passage was returned; it is a COARSE 'did-anything-come-back' flag, NOT a quality judgement -- the 0.45 similarity floor it used to consult cannot fire on this corpus (best raw similarities run 0.73-0.79), so judge quality from `matched`, `retrievers` and the passages themselves. Requires the local index to have been built with sync_embeddings.", Object.assign({ query: zod_1.z.string().min(1).describe("What to search for, in natural language."), limit: zod_1.z
800
+ .number()
801
+ .int()
802
+ .min(1)
803
+ .max(exports.MAX_LIMIT)
804
+ .optional()
805
+ .describe(`Maximum passages to return (default ${exports.DEFAULT_LIMIT}, max ${exports.MAX_LIMIT}).`), window: zod_1.z
806
+ .number()
807
+ .int()
808
+ .min(200)
809
+ .max(4000)
810
+ .optional()
811
+ .describe(`Characters per passage (default ${exports.DEFAULT_WINDOW}).`), check_staleness: zod_1.z
812
+ .boolean()
813
+ .optional()
814
+ .describe("Also report whether the index still matches the live corpus: `stale`, `index_age_s`, `live_count`, `indexed_count`. Costs one extra request (~44% on a sub-second call), so it is off by default. `stale: null` means the check could not be made — NOT that the index is current.") }, filterArgs), (0, middlewares_1.withErrorHandling)((args) => __awaiter(this, void 0, void 0, function* () {
815
+ // `check_staleness` is destructured OUT of `filters`, not left in it.
816
+ // `buildDocumentFilter` ignores unknown keys today, so leaving it there
817
+ // would work and would also mean every future filter key has to remember
818
+ // to exclude it — a silent coupling rather than a stated one.
819
+ const { limit, window, check_staleness } = args, filters = __rest(args, ["limit", "window", "check_staleness"]);
820
+ const result = yield runSemanticSearch(context, args.query, Object.assign(Object.assign({ limit: limit !== null && limit !== void 0 ? limit : exports.DEFAULT_LIMIT, window: window !== null && window !== void 0 ? window : exports.DEFAULT_WINDOW,
821
+ // ONE batched metadata request for N documents, content never fetched.
822
+ loadDocuments: (ids) => loadDocumentMetadata(api, ids),
823
+ // The text the window is cut from comes from the index, which already
824
+ // holds it. Reading it here instead of over HTTP is the 41x.
825
+ loadText: indexTextLoader(context.index()), filters: buildDocumentFilter(filters) }, (check_staleness ? { checkStaleness: true } : {})), {
826
+ // Bound late: `api` is this server's client, and the probe must use the
827
+ // same one the rest of the tool does.
828
+ corpusState: () => (0, sync_1.readCorpusState)(api) }));
829
+ return {
830
+ content: [{ type: "text", text: JSON.stringify(result) }],
831
+ };
832
+ })));
833
+ server.tool("embedding_status", "Report what the local semantic-search index actually holds: document and chunk counts, embedding dimensions, the model and prefix mode it was built with, when it last synced, how many documents are pending, how much disk the index uses, and whether any documents were truncated at the chunk cap. Use this before trusting semantic_search, and to tell 'the index is empty' apart from 'nothing matched'.", {}, (0, middlewares_1.withErrorHandling)(() => __awaiter(this, void 0, void 0, function* () {
834
+ const status = yield embeddingStatus(context, api);
835
+ return {
836
+ content: [{ type: "text", text: JSON.stringify(status) }],
837
+ };
838
+ })));
839
+ server.tool("sync_embeddings", "Bring the local semantic-search index up to date with the Paperless corpus. Fetches every document whose `modified` changed since the last run, re-chunks and re-embeds it, removes documents that no longer exist, and refreshes metadata for documents whose tags or correspondent changed without the document being edited. Long-running on a first build (tens of minutes for a large corpus) and it stops with an error if the local embedding server is unavailable, rather than leaving a half-built index that claims to be complete. Run embedding_status afterwards to confirm. force:true re-embeds every document and should only be used when the chunker or the model changed.", {
840
+ force: zod_1.z
841
+ .boolean()
842
+ .optional()
843
+ .describe("Re-embed every document even if `modified` is unchanged. Only needed after a chunker or model change; a normal sync is incremental."),
844
+ }, (0, middlewares_1.withErrorHandling)((args) => __awaiter(this, void 0, void 0, function* () {
845
+ const runner = context.syncRunner(api);
846
+ if (args.force) {
847
+ // ⚠️ PROBE BEFORE DELETE. The old order was delete -> run -> throw on
848
+ // outage, which turned a transient embedder blip into "you no longer
849
+ // have an index" on a rebuild that takes ~58 minutes. The failure the
850
+ // caller can afford is a refusal; the failure it cannot afford is
851
+ // having destroyed the working index to discover the server is down.
852
+ //
853
+ // The probe is a real embedding of a constant string (see
854
+ // `OllamaEmbedder.probe`), not a GET: a server that is up without its
855
+ // model pulled answers GET and then fails on the first real request.
856
+ const embedder = context.embedderOrUndefined();
857
+ if (embedder) {
858
+ if (embedder.probe) {
859
+ yield embedder.probe();
860
+ }
861
+ else {
862
+ // A fake without probe(): still exercise the same path the sync
863
+ // will use, so the ordering guarantee holds for every embedder.
864
+ yield embedder.embedQuery("ping");
865
+ }
866
+ }
867
+ // force means "the stored vectors are not trustworthy", which on this
868
+ // schema means deleting the document rows so the diff sees everything
869
+ // as new. Done through deleteDocuments rather than a file unlink so the
870
+ // FTS and vector tables are emptied in the same transaction.
871
+ const store = context.index();
872
+ store.deleteDocuments([...store.indexed().keys()]);
873
+ }
874
+ // ⚠️ `force` IS PASSED THROUGH, not merely acted on locally. The pre-sync
875
+ // delete below empties the document rows so the diff sees everything as
876
+ // new; the flag ALSO has to reach `run()` because the watermark check
877
+ // there would otherwise skip the run it just prepared for. Both halves are
878
+ // required: without the delete the vectors are stale, and without the flag
879
+ // the run does nothing at all.
880
+ const report = yield runner.run({ force: args.force === true });
881
+ return {
882
+ content: [{ type: "text", text: JSON.stringify(report) }],
883
+ };
884
+ })));
885
+ return context;
886
+ }
887
+ /** A predicate over visible documents, from the search_passages filter args. */
888
+ function buildDocumentFilter(filters) {
889
+ // Every key in FILTER_SHAPE is honoured here. `storage_path` used to be the
890
+ // exception: advertised to the model, absent from this predicate, and
891
+ // therefore silently ignored -- a caller got unfiltered results with no
892
+ // error, no warning, and no way to notice. That is worse than not offering
893
+ // the filter at all, because the schema is a promise.
894
+ const active = filters.correspondent !== undefined ||
895
+ filters.document_type !== undefined ||
896
+ filters.tag !== undefined ||
897
+ filters.storage_path !== undefined ||
898
+ filters.created__date__gte !== undefined ||
899
+ filters.created__date__lte !== undefined;
900
+ if (!active)
901
+ return undefined;
902
+ return (doc) => {
903
+ if (filters.correspondent !== undefined && doc.correspondent !== filters.correspondent) {
904
+ return false;
905
+ }
906
+ if (filters.document_type !== undefined && doc.document_type !== filters.document_type) {
907
+ return false;
908
+ }
909
+ if (filters.tag !== undefined && !doc.tags.includes(filters.tag))
910
+ return false;
911
+ // An unfiled document (storage_path === null) can never match a numeric
912
+ // storage_path, which is correct: the caller asked for a filing location.
913
+ if (filters.storage_path !== undefined && doc.storage_path !== filters.storage_path) {
914
+ return false;
915
+ }
916
+ if (filters.created__date__gte !== undefined && doc.created < filters.created__date__gte) {
917
+ return false;
918
+ }
919
+ if (filters.created__date__lte !== undefined && doc.created > filters.created__date__lte) {
920
+ return false;
921
+ }
922
+ return true;
923
+ };
924
+ }
925
+ /**
926
+ * Fetch the documents behind a set of ids, tolerating individual failures.
927
+ *
928
+ * One unfetchable document must not fail the whole search: the remaining hits
929
+ * are still real, and a search that returns four good passages and one gap is
930
+ * strictly better than an error. The gap is visible because the passage for
931
+ * that id is absent.
932
+ */
933
+ function loadVisibleDocuments(api, ids) {
934
+ return __awaiter(this, void 0, void 0, function* () {
935
+ const documents = new Map();
936
+ const unique = [...new Set(ids)];
937
+ const results = yield Promise.all(unique.map((id) => __awaiter(this, void 0, void 0, function* () {
938
+ try {
939
+ const doc = (yield api.request(`/documents/${id}/`));
940
+ return doc;
941
+ }
942
+ catch (_a) {
943
+ return undefined;
944
+ }
945
+ })));
946
+ for (const doc of results) {
947
+ if (doc && typeof doc.id === "number")
948
+ documents.set(doc.id, doc);
949
+ }
950
+ return documents;
951
+ });
952
+ }
953
+ /**
954
+ * Ids per batched metadata request.
955
+ *
956
+ * A single call can legitimately want metadata for `limit * 4` distinct
957
+ * documents from EACH retriever, so 400 is reachable. One request per document
958
+ * is what this replaces; one request for 400 ids would be a 2 KB URL, so the
959
+ * list is cut into batches rather than trusted to be short.
960
+ */
961
+ exports.METADATA_BATCH = 100;
962
+ /**
963
+ * Fetch metadata for many documents in as few requests as possible.
964
+ *
965
+ * ⚠️ THIS NEVER FETCHES DOCUMENT CONTENT, and that is the point of the whole
966
+ * change. The previous path issued one `GET /documents/{id}/` per hit -- up to
967
+ * 80 of them for a 10-passage call -- and read every full body only to slice a
968
+ * window out of it. Measured on the live box for 40 documents: 2,398 ms and
969
+ * 2,162,306 bytes, against 58 ms and 6,520 bytes for the same metadata in one
970
+ * ranged call. The text the window needs was already in the index.
971
+ *
972
+ * `content` is therefore `null` on every returned document, which the
973
+ * `VisibleDocument` type already allows. Nothing on this path reads it.
974
+ *
975
+ * A batch that fails does NOT fail the search: the affected ids are simply
976
+ * absent from the map, which the caller already reports as
977
+ * `document_unavailable`. That is the same tolerance the per-id version had,
978
+ * kept deliberately so one probe failure cannot turn a search into an error.
979
+ */
980
+ function loadDocumentMetadata(api, ids) {
981
+ return __awaiter(this, void 0, void 0, function* () {
982
+ var _a;
983
+ const documents = new Map();
984
+ const unique = [...new Set(ids)];
985
+ if (unique.length === 0)
986
+ return documents;
987
+ for (let offset = 0; offset < unique.length; offset += exports.METADATA_BATCH) {
988
+ const batch = unique.slice(offset, offset + exports.METADATA_BATCH);
989
+ const path = `/documents/?id__in=${batch.join(",")}` +
990
+ `&fields=id,title,correspondent,document_type,storage_path,tags,created,modified` +
991
+ `&page_size=${batch.length}`;
992
+ try {
993
+ const page = (yield api.request(path));
994
+ for (const doc of (_a = page.results) !== null && _a !== void 0 ? _a : []) {
995
+ if (typeof doc.id === "number") {
996
+ documents.set(doc.id, Object.assign(Object.assign({}, doc), { content: null }));
997
+ }
998
+ }
999
+ }
1000
+ catch (_b) {
1001
+ // Left absent, exactly as the per-id path left a failed fetch absent.
1002
+ }
1003
+ }
1004
+ return documents;
1005
+ });
1006
+ }
1007
+ /**
1008
+ * Rebuild a document's text from the chunks the index already holds.
1009
+ *
1010
+ * The chunker emits `CHUNK_SIZE` characters at a stride of
1011
+ * `CHUNK_SIZE - CHUNK_OVERLAP`, so consecutive chunks share exactly the
1012
+ * trailing `CHUNK_OVERLAP` characters of the previous one. Splicing therefore
1013
+ * has to drop that overlap -- but ONLY when it is really there, because a
1014
+ * document shorter than one chunk is a single chunk and dropping 200 characters
1015
+ * from it would delete text rather than deduplicate it.
1016
+ *
1017
+ * The metadata chunk at position 0 is skipped: it is a synthetic
1018
+ * "Title: … Sender: …" line, not document text, and including it would put
1019
+ * fabricated words in front of a passage and shift every offset by its length.
1020
+ *
1021
+ * ⚠️ The result is NORMALISED text -- `chunkText` collapses whitespace runs
1022
+ * before slicing. Offsets derived from it are labelled `offset_space:
1023
+ * "normalized"` for that reason.
1024
+ */
1025
+ function reconstructDocumentText(chunks) {
1026
+ // ⚠️ PIECES + ONE JOIN, NEVER `built += …`. The obvious implementation
1027
+ // (`built += …; built.endsWith(head)`) is O(n²) on a long document and the
1028
+ // cost is not theoretical: measured on the live index it took **14.7 seconds**
1029
+ // to reconstruct a 3,606-chunk document, because `endsWith` flattens V8's
1030
+ // rope on every iteration while the rope keeps growing. The array-plus-join
1031
+ // form does the same splice in ~2 ms.
1032
+ const pieces = [];
1033
+ let previous;
1034
+ for (const chunk of chunks) {
1035
+ if (chunk.kind !== "content")
1036
+ continue;
1037
+ // The overlap is compared against the PREVIOUS CHUNK, not against the
1038
+ // accumulated text. It is the same 200 characters — the chunker emits
1039
+ // consecutive slices at a stride of CHUNK_SIZE - CHUNK_OVERLAP — but
1040
+ // comparing a short, already-flat string is O(200) rather than O(document).
1041
+ let head = 0;
1042
+ if (previous !== undefined) {
1043
+ const probe = Math.min(chunk_1.CHUNK_OVERLAP, previous.length, chunk.text.length);
1044
+ if (probe > 0 && previous.slice(-probe) === chunk.text.slice(0, probe)) {
1045
+ head = probe;
1046
+ }
1047
+ }
1048
+ pieces.push(chunk.text.slice(head));
1049
+ previous = chunk.text;
1050
+ }
1051
+ return pieces.join("");
1052
+ }
1053
+ /**
1054
+ * The production `loadText`: document text straight out of the open index.
1055
+ *
1056
+ * One prepared SELECT per document, against a local SQLite file, replacing one
1057
+ * HTTPS GET per document that returned the whole body. Measured on the live
1058
+ * index (read + splice): 2 ms for a typical document, 34–42 ms for the largest
1059
+ * (3,606 chunks / 7.2 M characters).
1060
+ *
1061
+ * A document whose chunks cannot be read is simply absent from the map, which
1062
+ * makes the caller fall back to `doc.content` — `null` on the batched-metadata
1063
+ * path, hence an empty passage rather than an error. That is the same
1064
+ * "one unusable document must not fail the search" tolerance the fetch path
1065
+ * had, and `content_length: 0` makes it visible instead of silent.
1066
+ */
1067
+ function indexTextLoader(store) {
1068
+ return (hits) => {
1069
+ const text = new Map();
1070
+ for (const id of new Set(hits.map((hit) => hit.documentId))) {
1071
+ try {
1072
+ text.set(id, reconstructDocumentText(store.chunksFor(id)));
1073
+ }
1074
+ catch (_a) {
1075
+ // Absent on purpose: reported downstream as an empty passage.
1076
+ }
1077
+ }
1078
+ return text;
1079
+ };
1080
+ }
1081
+ /**
1082
+ * Trim a result until its own serialised body fits the consumer's ceiling.
1083
+ *
1084
+ * Measured on the REAL bytes, not predicted from `window * passages`. That
1085
+ * distinction is the whole mechanism: titles, ids and counts vary, and an
1086
+ * arithmetic budget that is right on average is exactly the kind of bound that
1087
+ * fails on the one call that matters. `budgetedLimit` keeps the work small;
1088
+ * this is what makes the guarantee actual.
1089
+ */
1090
+ function fitResultToBudget(result) {
1091
+ if (JSON.stringify(result).length <= exports.MAX_RESULT_CHARS)
1092
+ return result;
1093
+ // ⚠️ A PASSAGE-ONLY TRIM IS NOT A BOUND. Measured: `fitResultToBudget` dropped
1094
+ // passages and never touched the ECHOED QUERY, which has no length limit -- a
1095
+ // 30,000-char query returned 30,128 chars with zero passages, over the ceiling
1096
+ // the function exists to enforce. Dropping every passage cannot fix an
1097
+ // oversize field that is not a passage. Echo less.
1098
+ const echo = typeof result.query === "string" ? result.query : "";
1099
+ const passages = [...result.passages];
1100
+ let dropped = 0;
1101
+ let candidate = result;
1102
+ while (passages.length > 0) {
1103
+ passages.pop();
1104
+ dropped += 1;
1105
+ candidate = Object.assign(Object.assign({}, result), { passages, trimmed: { reason: "result_size", dropped, limit_chars: exports.MAX_RESULT_CHARS } });
1106
+ if (JSON.stringify(candidate).length <= exports.MAX_RESULT_CHARS)
1107
+ return candidate;
1108
+ }
1109
+ // Every passage is gone and it is STILL over budget: the query echo alone is
1110
+ // too large. Truncate the echo rather than return an oversize result, and say
1111
+ // so -- a silently truncated query is indistinguishable from a short one.
1112
+ //
1113
+ // ⚠️ MEASURE AFTER ADDING THE FLAG, NOT BEFORE. The first cut of this computed
1114
+ // the overflow against the payload WITHOUT `query_truncated`, then added that
1115
+ // field -- which is 24 chars -- and returned 20,074 against a 20,000 ceiling.
1116
+ // A bound that its own error message breaks is not a bound, and the test
1117
+ // caught it only because it asserted the serialised length rather than the
1118
+ // passage count. Loop until it actually fits; the flag costs bytes, so assume
1119
+ // it is present from the first attempt.
1120
+ const declared = Object.assign(Object.assign({}, candidate), { trimmed: {
1121
+ reason: "result_size",
1122
+ dropped,
1123
+ limit_chars: exports.MAX_RESULT_CHARS,
1124
+ query_truncated: true,
1125
+ } });
1126
+ let keep = echo.length;
1127
+ for (;;) {
1128
+ const over = JSON.stringify(Object.assign(Object.assign({}, declared), { query: echo.slice(0, keep) })).length - exports.MAX_RESULT_CHARS;
1129
+ if (over <= 0 || keep === 0)
1130
+ break;
1131
+ keep = Math.max(0, keep - over - 1);
1132
+ }
1133
+ return Object.assign(Object.assign({}, declared), { query: echo.slice(0, keep) });
1134
+ }
1135
+ /**
1136
+ * Report the store's own view of itself.
1137
+ *
1138
+ * `pending` is computed against Paperless' document count, so it is `null`
1139
+ * rather than `0` when that count cannot be read: a zero here would say "the
1140
+ * index is current" on the strength of a failed request.
1141
+ */
1142
+ function embeddingStatus(context, api) {
1143
+ return __awaiter(this, void 0, void 0, function* () {
1144
+ var _a, _b, _c;
1145
+ const config = context.config();
1146
+ const path = config ? (0, config_1.indexPath)(config) : "";
1147
+ const base = {
1148
+ documents: 0,
1149
+ chunks: 0,
1150
+ dimensions: 0,
1151
+ model: "",
1152
+ prefix_mode: "",
1153
+ last_sync: null,
1154
+ pending: null,
1155
+ store_bytes: null,
1156
+ truncated_documents: 0,
1157
+ index_path: path,
1158
+ };
1159
+ let store;
1160
+ try {
1161
+ store = context.index();
1162
+ }
1163
+ catch (err) {
1164
+ return Object.assign(Object.assign({}, base), { open_error: errorMessage(err) });
1165
+ }
1166
+ const counts = store.counts();
1167
+ const meta = store.meta();
1168
+ const documents = store.indexed();
1169
+ const stamps = store.metadataStamps();
1170
+ let lastSync = null;
1171
+ for (const stamp of stamps.values()) {
1172
+ if (lastSync === null || stamp > lastSync)
1173
+ lastSync = stamp;
1174
+ }
1175
+ // Truncation is the one index fact that changes what a caller can conclude
1176
+ // from a miss: a truncated document lost its tail, so "not found" may be a
1177
+ // false negative. Counted with one query, not one query per document.
1178
+ const truncated = store.truncatedCount();
1179
+ let storeBytes = null;
1180
+ try {
1181
+ const { statSync } = yield Promise.resolve().then(() => __importStar(require("node:fs")));
1182
+ storeBytes = statSync(path).size;
1183
+ }
1184
+ catch (_d) {
1185
+ // Not an error: a fresh index file may not exist yet.
1186
+ }
1187
+ let pending = null;
1188
+ try {
1189
+ const page = (yield api.request("/documents/?fields=id&page_size=1"));
1190
+ if (typeof page.count === "number") {
1191
+ pending = Math.max(0, page.count - documents.size);
1192
+ }
1193
+ }
1194
+ catch (_e) {
1195
+ // Left null: a failed count is not a count of zero.
1196
+ }
1197
+ return Object.assign(Object.assign({}, base), { documents: counts.documents, chunks: counts.chunks, dimensions: (_a = meta === null || meta === void 0 ? void 0 : meta.dimensions) !== null && _a !== void 0 ? _a : 0, model: (_b = meta === null || meta === void 0 ? void 0 : meta.model) !== null && _b !== void 0 ? _b : "", prefix_mode: (_c = meta === null || meta === void 0 ? void 0 : meta.prefixMode) !== null && _c !== void 0 ? _c : "", last_sync: lastSync, pending, store_bytes: storeBytes, truncated_documents: truncated });
1198
+ });
1199
+ }