@retiform/paperless-mcp 2.3.6 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +301 -4
  2. package/build/api/PaperlessAPI.js +1 -6
  3. package/build/search/build-outcome.d.ts +28 -0
  4. package/build/search/build-outcome.js +31 -0
  5. package/build/search/chunk.d.ts +16 -0
  6. package/build/search/chunk.js +64 -0
  7. package/build/search/config.d.ts +120 -0
  8. package/build/search/config.js +264 -0
  9. package/build/search/embedder.d.ts +346 -0
  10. package/build/search/embedder.js +734 -0
  11. package/build/search/rank.d.ts +11 -0
  12. package/build/search/rank.js +87 -0
  13. package/build/search/store.d.ts +406 -0
  14. package/build/search/store.js +847 -0
  15. package/build/search/sync.d.ts +204 -0
  16. package/build/search/sync.js +819 -0
  17. package/build/search/webhook.d.ts +193 -0
  18. package/build/search/webhook.js +445 -0
  19. package/build/server.d.ts +29 -0
  20. package/build/server.js +77 -0
  21. package/build/tools/documents.d.ts +13 -3
  22. package/build/tools/documents.js +23 -6
  23. package/build/tools/search.d.ts +518 -0
  24. package/build/tools/search.js +1199 -0
  25. package/build/tools/utils/middlewares.d.ts +5 -0
  26. package/build/tools/utils/middlewares.js +282 -9
  27. package/package.json +9 -3
  28. package/paperless-mcp.dxt +0 -0
  29. package/build/api/documentEnhancer.test.d.ts +0 -1
  30. package/build/api/documentEnhancer.test.js +0 -80
  31. package/build/manifest.test.d.ts +0 -1
  32. package/build/manifest.test.js +0 -66
  33. package/build/resources/documents.test.d.ts +0 -1
  34. package/build/resources/documents.test.js +0 -124
  35. package/build/server.test.d.ts +0 -1
  36. package/build/server.test.js +0 -54
  37. package/build/tools/boundedContent.test.d.ts +0 -1
  38. package/build/tools/boundedContent.test.js +0 -565
  39. package/build/tools/documents.test.d.ts +0 -1
  40. package/build/tools/documents.test.js +0 -658
  41. package/build/tools/mail.test.d.ts +0 -1
  42. package/build/tools/mail.test.js +0 -212
  43. package/build/tools/notes.test.d.ts +0 -1
  44. package/build/tools/notes.test.js +0 -177
  45. package/build/tools/schemaCompat.test.d.ts +0 -1
  46. package/build/tools/schemaCompat.test.js +0 -59
  47. package/build/tools/utils/monetary.test.d.ts +0 -1
  48. package/build/tools/utils/monetary.test.js +0 -25
  49. package/build/tools/utils/resourceUri.test.d.ts +0 -1
  50. package/build/tools/utils/resourceUri.test.js +0 -50
  51. package/build/tools/utils/selectFields.test.d.ts +0 -1
  52. package/build/tools/utils/selectFields.test.js +0 -164
@@ -0,0 +1,819 @@
1
+ "use strict";
2
+ // Incremental sync between the Paperless corpus and the local vector index.
3
+ //
4
+ // Algorithm ported from jaimemachado/paperless-mcp (src/search/sync.ts at
5
+ // 50f7beec0d36804cab8fbcaa7b47f85e562e8061). The algorithm is theirs; the
6
+ // bindings are not. Every adaptation is listed here because several of them are
7
+ // corrections rather than translations, and a silent one would be a defect:
8
+ //
9
+ // 1. `../metrics` and `../tracing` are NOT ported. Neither module exists in
10
+ // this fork (the store lap made the same call), and adding OpenTelemetry
11
+ // and prom-client to emit counters nobody reads is not a trade worth
12
+ // making. Console diagnostics replace the two console.error calls the
13
+ // reference already used.
14
+ // 2. `EmbeddingUnavailable` -> `EmbeddingUnavailableError`, and the abort is
15
+ // preserved: an embedding outage stops the run rather than writing an
16
+ // index that is missing documents it will claim to have.
17
+ // 3. The reference's `replaceDocument` call omits `chunkCount`, because its
18
+ // Postgres store derived the count. Ours REQUIRES it and rejects a
19
+ // mismatch against the chunk array, so it is passed explicitly.
20
+ // 4. The metadata-refresh loop in the reference reads `info.metadataAt` off
21
+ // `indexed()`. Our `indexed()` returns `id -> modified` only, so the
22
+ // stamp comes from `metadataStamps()`. Using the reference's shape here
23
+ // would have compared `undefined` on every document and refreshed the
24
+ // entire corpus every single run -- a correct-looking port that redoes
25
+ // 58 minutes of work on each call.
26
+ // 5. `pathAfterApi` is defensive. The reference slices at
27
+ // `indexOf("/api/") + 4`, which silently eats four characters when the
28
+ // marker is absent. Ours returns the path unchanged in that case instead
29
+ // of repairing a URL into a wrong one.
30
+ // 6. `MAX_CHUNKS` is injected rather than assumed, so the sync and the
31
+ // chunker cannot disagree about how much of a document was kept.
32
+ //
33
+ // ── The five load-bearing choices, each verified against our install ────────
34
+ //
35
+ // `modified` IS the incremental key. Verified on this install: the documents
36
+ // table has a `modified` column and `max(modified)` is today. A document
37
+ // edited in Paperless bumps it; a document whose tags changed does NOT.
38
+ //
39
+ // DELETES ONLY RUN WHEN `rows.length === count`. A list whose rows disagree
40
+ // with its own count changed while it was being paged, and the missing rows
41
+ // are indistinguishable from deleted documents. Deleting on a short list
42
+ // would remove live documents from the index, so the delete step is SKIPPED
43
+ // and reported instead. Tested.
44
+ //
45
+ // A 24 HOUR METADATA REFRESH. Tags, correspondents and document types change
46
+ // without touching `modified`, so a document whose sender was corrected
47
+ // would keep the old name in its metadata chunk forever. That chunk is
48
+ // vector- and lexical-indexed, so the stale name stays searchable.
49
+ //
50
+ // ⚠️ THAT REFRESH SURVIVES THE WATERMARK, AND IT IS NOT REDUNDANT WITH IT.
51
+ // Measured against Paperless' own bulk-edit code: `set_correspondent`,
52
+ // `set_document_type`, `set_storage_path` and `add_tag` all go through
53
+ // `qs.update(...)` or a `bulk_create` on the through-table. Neither fires
54
+ // `auto_now`, so none of them bumps `modified`. A corpus-level watermark is
55
+ // therefore structurally blind to an M2M edit — it cannot see one even in
56
+ // principle, let alone in this implementation. Deleting the refresh on the
57
+ // theory that the watermark subsumes it would re-open the stale-metadata hole
58
+ // it was written to close. It is a candidate for WIDENING (a cheaper M2M
59
+ // probe), never for removal.
60
+ //
61
+ // THE WATERMARK. `count` and `max(modified)` are recorded at the end of every
62
+ // completed run, and consulted at the start of the next one as a single
63
+ // request. A run against an unchanged corpus returns in ~218 ms without
64
+ // paging a single document. `force` bypasses it, because `force` is a
65
+ // statement about the stored vectors rather than about the corpus.
66
+ //
67
+ // AN EMBEDDING OUTAGE ABORTS THE WHOLE RUN. The alternative -- skip the
68
+ // document, carry on, report `failed` -- leaves an index that is missing
69
+ // documents while `documents` rows claim they are indexed, which is worse
70
+ // than no index because it looks complete.
71
+ //
72
+ // A RE-ENTRANCY GUARD. Two concurrent syncs against one SQLite file is a
73
+ // failure this arc has already hit once. `run()` returns the in-flight
74
+ // promise rather than starting a second pass.
75
+ //
76
+ // ── Triggering ─────────────────────────────────────────────────────────────
77
+ // On demand is the primary path: the `sync_embeddings` tool. `start()` exists
78
+ // but nothing calls it unless `PAPERLESS_MCP_SYNC_INTERVAL_MS` is set, because
79
+ // a stdio child's process lifetime belongs to the gateway -- a hidden timer
80
+ // refreshes on a schedule nobody can see or schedule around. An observable tool
81
+ // call can be scheduled with the scheduler we already use, where the schedule
82
+ // is legible.
83
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
84
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
85
+ return new (P || (P = Promise))(function (resolve, reject) {
86
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
87
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
88
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
89
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
90
+ });
91
+ };
92
+ Object.defineProperty(exports, "__esModule", { value: true });
93
+ exports.IndexSync = exports.REQUEST_TIMEOUT_MS = exports.WATERMARK_PATH = exports.MAX_LIST_PAGES = exports.DOCUMENT_LIST_PATH = exports.METADATA_REFRESH_MS = void 0;
94
+ exports.corpusChanged = corpusChanged;
95
+ exports.readCorpusState = readCorpusState;
96
+ exports.pathAfterApi = pathAfterApi;
97
+ const chunk_1 = require("./chunk");
98
+ const embedder_1 = require("./embedder");
99
+ const store_1 = require("./store");
100
+ /** How often a document's metadata chunk is rebuilt regardless of `modified`. */
101
+ exports.METADATA_REFRESH_MS = 24 * 60 * 60 * 1000;
102
+ /**
103
+ * Paged in id order, with the fields the diff needs and nothing else.
104
+ *
105
+ * Ordering by id matters: an insert does not move a row, so a document added
106
+ * while the list is being paged lands at the end rather than shuffling the
107
+ * pages already read. A delete DOES move rows, which is why the row count is
108
+ * compared to the reported count below.
109
+ */
110
+ exports.DOCUMENT_LIST_PATH = "/documents/?fields=id,modified&page_size=1000&ordering=id";
111
+ /** A repeated or endless `next` would otherwise keep a run pending forever. */
112
+ exports.MAX_LIST_PAGES = 10000;
113
+ /**
114
+ * One request that answers "how big is the corpus, and when did it last move".
115
+ *
116
+ * Deliberately NOT a walk of the document list. The list is paged at 1000 rows
117
+ * with a full `id,modified` payload per row; on a corpus of a few thousand that
118
+ * is several requests and a few hundred kilobytes to learn two scalars. This is
119
+ * ONE request and about 200 bytes.
120
+ *
121
+ * `ordering=-modified&page_size=1` makes `results[0].modified` the maximum over
122
+ * the whole corpus, which is the value the incremental diff is keyed on.
123
+ */
124
+ exports.WATERMARK_PATH = "/documents/?fields=id,modified&ordering=-modified&page_size=1";
125
+ /**
126
+ * Whether the live corpus differs from the indexed one.
127
+ *
128
+ * ⚠️ COMPARED AS INSTANTS, NOT STRINGS. `modified` arrives as an ISO 8601 string
129
+ * with an offset, and the same moment can be serialised with a different offset
130
+ * (`…+00:00` one day, `…-05:00` the next). A string compare would report a
131
+ * change on every run and defeat the entire point of the watermark, while
132
+ * looking exactly like a working check. The existing document diff already got
133
+ * this right; this is the same treatment applied to the corpus-level value.
134
+ *
135
+ * `count` is checked FIRST and separately. It is the only signal a soft delete
136
+ * produces: Paperless clears `deleted_at` through a queryset `.update()`, which
137
+ * does not fire `auto_now`, so a deleted document never bumps `max(modified)`.
138
+ * The count still drops, and the count is free in the same response.
139
+ */
140
+ function corpusChanged(live, watermark) {
141
+ if (live.count !== watermark.count)
142
+ return true;
143
+ if (live.modified === null)
144
+ return watermark.modified !== null;
145
+ if (watermark.modified === null)
146
+ return true;
147
+ return new Date(live.modified).getTime() > new Date(watermark.modified).getTime();
148
+ }
149
+ /**
150
+ * The corpus's size and newest `modified`, from ONE request.
151
+ *
152
+ * Returns undefined when the request fails. A watermark probe that THROWS would
153
+ * turn a cosmetic optimisation into a hard dependency: the sync would stop
154
+ * working on any Paperless that declines this particular query, when the list
155
+ * walk it already knows how to do would have succeeded. Falling back to the full
156
+ * diff is always correct, merely slower.
157
+ *
158
+ * Module-level rather than a private method because two callers need it — the
159
+ * sync's skip check and the search tool's opt-in staleness report — and two
160
+ * copies of "which fields, which ordering, which page size" is how they drift
161
+ * apart. One request shape, one place.
162
+ */
163
+ function readCorpusState(api, requestTimeoutMs) {
164
+ return __awaiter(this, void 0, void 0, function* () {
165
+ var _a, _b;
166
+ try {
167
+ const page = yield api.request(exports.WATERMARK_PATH, requestTimeoutMs !== undefined
168
+ ? { signal: AbortSignal.timeout(requestTimeoutMs) }
169
+ : {});
170
+ const first = ((_a = page.results) !== null && _a !== void 0 ? _a : [])[0];
171
+ return { count: page.count, modified: (_b = first === null || first === void 0 ? void 0 : first.modified) !== null && _b !== void 0 ? _b : null };
172
+ }
173
+ catch (err) {
174
+ log(`the corpus watermark could not be read: ${errorMessage(err)}`);
175
+ return undefined;
176
+ }
177
+ });
178
+ }
179
+ /** Without a limit, a Paperless that never answers keeps a run pending forever. */
180
+ exports.REQUEST_TIMEOUT_MS = 60000;
181
+ /**
182
+ * Turn an absolute `next` URL into the path this API client accepts.
183
+ *
184
+ * The list answer carries `next` as a full URL (verified: `GET
185
+ * /api/documents/?fields=id,modified&page_size=3&ordering=id` returns an
186
+ * absolute URL), while `PaperlessAPI.request` prepends `/api` itself. So the
187
+ * base is stripped and the query string kept.
188
+ *
189
+ * Deliberately NOT the reference's arithmetic: it slices at
190
+ * `indexOf("/api/") + 4`, which returns a four-character-amputated path when
191
+ * the marker is missing -- a wrong URL that looks like a right one. An absent
192
+ * marker returns the input unchanged, which fails loudly at the request instead.
193
+ */
194
+ function pathAfterApi(url) {
195
+ const { pathname, search } = new URL(url);
196
+ const marker = pathname.indexOf("/api/");
197
+ if (marker < 0)
198
+ return pathname + search;
199
+ return pathname.slice(marker + "/api".length) + search;
200
+ }
201
+ function toNameMap(rows) {
202
+ return new Map(rows.map((row) => [row.id, row.name]));
203
+ }
204
+ function errorMessage(err) {
205
+ return err instanceof Error ? err.message : String(err);
206
+ }
207
+ function withDocumentContext(err, context) {
208
+ if (err && typeof err === "object") {
209
+ Object.assign(err, context);
210
+ }
211
+ return err;
212
+ }
213
+ /**
214
+ * Name the document in one line, so a failure is diagnosable from the log
215
+ * without a second round trip.
216
+ *
217
+ * Id first, because the id is what a person pastes into the Paperless URL; the
218
+ * title second, because it is what makes the line recognisable at a glance. A
219
+ * title is a document HEADING, never body text, so it is safe to log.
220
+ */
221
+ function describeDocument(err, fallbackId) {
222
+ const id = err && typeof err === "object" && "documentId" in err
223
+ ? err.documentId
224
+ : fallbackId;
225
+ const title = err && typeof err === "object" && "documentTitle" in err
226
+ ? err.documentTitle
227
+ : undefined;
228
+ if (!title)
229
+ return `${id} (title unavailable: the document could not be read)`;
230
+ return `${id} \u201c${title}\u201d`;
231
+ }
232
+ function log(line) {
233
+ console.error(`[paperless-mcp] index: ${line}`);
234
+ }
235
+ class IndexSync {
236
+ constructor(opts) {
237
+ var _a, _b, _c, _d;
238
+ /** Whether the run in flight is a targeted one. See `run()`. */
239
+ this.runningTargeted = false;
240
+ this.api = opts.api;
241
+ this.store = opts.store;
242
+ this.embedder = opts.embedder;
243
+ this.maxChunks = (_a = opts.maxChunks) !== null && _a !== void 0 ? _a : chunk_1.MAX_CHUNKS;
244
+ this.now = (_b = opts.now) !== null && _b !== void 0 ? _b : Date.now;
245
+ this.requestTimeoutMs = (_c = opts.requestTimeoutMs) !== null && _c !== void 0 ? _c : exports.REQUEST_TIMEOUT_MS;
246
+ this.maxListPages = (_d = opts.maxListPages) !== null && _d !== void 0 ? _d : exports.MAX_LIST_PAGES;
247
+ }
248
+ /**
249
+ * Run one sync. Re-entrant calls join the run in flight.
250
+ *
251
+ * Two concurrent writers on one SQLite file is a failure this arc has already
252
+ * hit once, and WAL plus `busy_timeout` only makes the second writer wait --
253
+ * it does not make two simultaneous passes correct, because both would diff
254
+ * against the same stale `indexed()` snapshot and the loser's deletes would
255
+ * remove the winner's work.
256
+ */
257
+ run(opts = {}) {
258
+ var _a, _b;
259
+ const targeted = ((_b = (_a = opts.documentIds) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) > 0;
260
+ // ⚠️ A TARGETED RUN AND A FULL RUN ANSWER DIFFERENT QUESTIONS, SO THEY MUST
261
+ // NOT SHARE ONE PROMISE.
262
+ //
263
+ // Joining is right when both callers want the same sweep: two scheduler
264
+ // ticks, or a tool call landing on a run already in flight. It is wrong when
265
+ // a full run joins a targeted one -- the full caller would receive a report
266
+ // covering one document and read it as "the corpus is up to date", which is
267
+ // the exact substitution this whole subsystem exists to avoid.
268
+ //
269
+ // So a full run requested during a targeted run is SEQUENCED behind it
270
+ // rather than joined to it. Same path, one run after the other; no second
271
+ // sync implementation.
272
+ if (!targeted && this.running && this.runningTargeted) {
273
+ return this.running.then(() => this.run(opts), () => this.run(opts));
274
+ }
275
+ if (!this.running) {
276
+ this.runningTargeted = targeted;
277
+ this.running = this.doRun(opts).finally(() => {
278
+ this.running = undefined;
279
+ this.runningTargeted = false;
280
+ });
281
+ }
282
+ return this.running;
283
+ }
284
+ /**
285
+ * Run once and then on an interval. NOT called by this package.
286
+ *
287
+ * A stdio child's lifetime is the gateway's, so a timer here refreshes on a
288
+ * schedule nobody can see. The observable path is the `sync_embeddings` tool,
289
+ * which can be scheduled where the schedule is legible.
290
+ */
291
+ start(intervalMs) {
292
+ var _a, _b;
293
+ if (this.timer)
294
+ return;
295
+ const scheduled = () => this.run().catch((err) => log(`scheduled run failed: ${errorMessage(err)}`));
296
+ void scheduled();
297
+ this.timer = setInterval(scheduled, intervalMs);
298
+ // Do not hold the process open for a refresh nobody asked for.
299
+ (_b = (_a = this.timer).unref) === null || _b === void 0 ? void 0 : _b.call(_a);
300
+ }
301
+ stop() {
302
+ if (this.timer)
303
+ clearInterval(this.timer);
304
+ this.timer = undefined;
305
+ }
306
+ get isRunning() {
307
+ return this.running !== undefined;
308
+ }
309
+ /**
310
+ * True while a background timer is live.
311
+ *
312
+ * Distinct from `isRunning`, which reports a pass in flight: between two
313
+ * scheduled ticks the timer is set and nothing is running. A wiring test that
314
+ * asked `isRunning` would report "no schedule" on a working schedule whenever
315
+ * it happened to look between ticks.
316
+ */
317
+ get isScheduled() {
318
+ return this.timer !== undefined;
319
+ }
320
+ doRun(opts) {
321
+ return __awaiter(this, void 0, void 0, function* () {
322
+ const started = this.now();
323
+ try {
324
+ const report = yield this.syncAll(opts);
325
+ const seconds = ((this.now() - started) / 1000).toFixed(1);
326
+ if (report.skipped) {
327
+ log(`run finished in ${seconds}s: SKIPPED (${report.skipReason})`);
328
+ return report;
329
+ }
330
+ log(`run finished in ${seconds}s: +${report.added} added, ` +
331
+ `~${report.updated} updated, -${report.deleted} deleted, ` +
332
+ `${report.metadataRefreshed} metadata refreshed, ${report.failed} failed, ` +
333
+ `${report.splits} split` +
334
+ (report.deletedSkipped ? " (deletes SKIPPED: list changed mid-run)" : ""));
335
+ return report;
336
+ }
337
+ catch (err) {
338
+ log(`run failed: ${errorMessage(err)}`);
339
+ throw err;
340
+ }
341
+ });
342
+ }
343
+ syncAll(opts) {
344
+ return __awaiter(this, void 0, void 0, function* () {
345
+ var _a;
346
+ const report = {
347
+ added: 0,
348
+ updated: 0,
349
+ deleted: 0,
350
+ metadataRefreshed: 0,
351
+ failed: 0,
352
+ deletedSkipped: false,
353
+ splits: 0,
354
+ skipped: false,
355
+ };
356
+ // ⚠️ THE INDEX'S IDENTITY IS WRITTEN BY THE RUN, NOT BY THE FIXTURE.
357
+ //
358
+ // `writeMeta` and `assertSchemaCompatible` both existed and were both dead:
359
+ // no production call site for either, so `embedding_status` reported
360
+ // `dimensions: 0, model: "", prefix_mode: ""` on a real index, and a
361
+ // prefix-mode mismatch could never be refused. The suite stayed green
362
+ // because the FIXTURES called `writeMeta` themselves -- the tool asserted
363
+ // the function while nothing asserted the wiring, which is the
364
+ // no-op-stub-in-a-test-fake family: a contract leg with no observable
365
+ // output and therefore no test.
366
+ //
367
+ // Written FIRST, before any document work, so the index never holds vectors
368
+ // whose provenance it cannot state. It is also cheap: four upserts.
369
+ this.store.writeMeta({
370
+ schemaVersion: store_1.SCHEMA_VERSION,
371
+ dimensions: this.embedder.dimensions,
372
+ model: this.embedder.model,
373
+ prefixMode: this.embedder.prefixMode,
374
+ });
375
+ // ── TARGETED RUN: THE CALLER NAMED THE DOCUMENTS ────────────────────────
376
+ //
377
+ // This is the webhook's path. A delivery that named ids is a statement
378
+ // about THOSE documents, and re-indexing exactly those is both cheaper and
379
+ // more complete than any corpus-level signal could be.
380
+ //
381
+ // ⚠️ WHY THIS CANNOT BE EXPRESSED AS THE WATERMARK DIFF, which is the whole
382
+ // reason it is a separate branch. A tag, correspondent or document-type
383
+ // edit goes through a queryset update on a through-table: `auto_now` never
384
+ // fires, so `modified` does not move, and the document count is unchanged.
385
+ // Both watermark scalars therefore read "nothing happened" while the
386
+ // document's metadata chunk has genuinely gone stale. The event is the only
387
+ // signal that exists for that change, and it names the document, so there
388
+ // is no reason to go looking for one at corpus scale.
389
+ //
390
+ // The document is re-fetched in full, so content and metadata are both
391
+ // rebuilt. That is deliberate: the payload carries no change type, so
392
+ // deciding between a content reindex and a metadata-only refresh would mean
393
+ // guessing. One detail fetch settles it by observation.
394
+ if (opts.documentIds && opts.documentIds.length > 0) {
395
+ const ids = [...new Set(opts.documentIds)];
396
+ // ⚠️ SNAPSHOT BEFORE THE LOOP, exactly as the full path does. Reading
397
+ // `indexed()` per document would consult it AFTER that document was
398
+ // written, so every document would classify as "updated" and `added`
399
+ // would always be zero -- a count that is wrong in the flattering
400
+ // direction, and wrong for a reason no assertion about the index's final
401
+ // contents could detect.
402
+ const indexedBefore = this.store.indexed();
403
+ let names;
404
+ for (const id of ids) {
405
+ try {
406
+ names !== null && names !== void 0 ? names : (names = yield this.loadNames());
407
+ const context = yield this.syncDocument(id, names);
408
+ report.splits += context.splits;
409
+ if (indexedBefore.has(id))
410
+ report.updated++;
411
+ else
412
+ report.added++;
413
+ }
414
+ catch (err) {
415
+ if (err instanceof embedder_1.EmbeddingUnavailableError) {
416
+ // Same reasoning as the full path: every remaining document would
417
+ // fail identically, so the run stops and the index keeps what it
418
+ // has. The next delivery retries.
419
+ log(`document ${describeDocument(err, id)} failed and the targeted ` +
420
+ `run is ABORTING: ${errorMessage(err)}`);
421
+ throw err;
422
+ }
423
+ // ⚠️ A DOCUMENT THAT NO LONGER EXISTS MUST NOT ABORT THE RUN. A
424
+ // delivery can arrive for a document deleted between the change and
425
+ // the webhook — the event is already in flight and cannot be
426
+ // recalled. Counting it as `failed` and continuing is the whole
427
+ // handling; the deletion itself is the watermark's `count` check,
428
+ // because the sending side fires nothing on delete.
429
+ report.failed++;
430
+ log(`document ${describeDocument(err, id)} failed; CONTINUING (a ` +
431
+ `delivery may name a document deleted since the change): ` +
432
+ `${errorMessage(err)}`);
433
+ }
434
+ }
435
+ // ⚠️ THE WATERMARK IS NOT WRITTEN HERE, AND MUST NOT BE. It states "the
436
+ // corpus is indexed through this point", and a targeted run has looked at
437
+ // a handful of documents rather than the corpus. Recording one would claim
438
+ // a sweep that never happened and stop the next full run from doing it.
439
+ report.skipped = false;
440
+ return report;
441
+ }
442
+ // ── THE WATERMARK, CONSULTED BEFORE ANY PAGING ──────────────────────────
443
+ //
444
+ // One request, two scalars. If the corpus has not moved since the last run
445
+ // there is no diff to compute and the run returns having cost ~218 ms
446
+ // instead of a paged walk of every document id.
447
+ //
448
+ // ⚠️ `force` BYPASSES THIS ENTIRELY. `force` means "the stored vectors are
449
+ // not trustworthy" — it is a statement about the INDEX, not about the
450
+ // corpus. Letting the watermark short-circuit it would make `force` a no-op
451
+ // on the second call, which is a silent lie about what the flag does.
452
+ const live = opts.force
453
+ ? undefined
454
+ : yield readCorpusState(this.api, this.requestTimeoutMs);
455
+ if (live) {
456
+ const watermark = this.store.watermark();
457
+ // No stored watermark means this index predates the watermark, or has
458
+ // never completed a run. Either way there is nothing to compare against
459
+ // and the diff must run: treating an absent watermark as "nothing
460
+ // changed" would skip the first sync after an upgrade and then keep
461
+ // skipping, permanently and silently.
462
+ if (watermark && !corpusChanged(live, watermark)) {
463
+ report.skipped = true;
464
+ report.skipReason =
465
+ `the corpus matches the last run (${live.count} documents, ` +
466
+ `max(modified) ${(_a = live.modified) !== null && _a !== void 0 ? _a : "none"}); the document diff was ` +
467
+ `skipped`;
468
+ // ⚠️ A SKIP SKIPS THE DIFF, NOT THE METADATA REFRESH.
469
+ //
470
+ // This is the whole reason the watermark cannot subsume the refresh: a
471
+ // tag, correspondent or document-type edit goes through a queryset
472
+ // `.update()` on a through-table, `auto_now` never fires, and neither
473
+ // `count` nor `max(modified)` moves. A corpus can be byte-identical in
474
+ // both scalars while a document's metadata chunk is stale.
475
+ //
476
+ // So the refresh runs here too. On a freshly-synced index every stamp
477
+ // is inside the window and this loop makes ZERO requests, which is what
478
+ // keeps the "one request" promise true in the common case. When it does
479
+ // have work to do, that work is real and skipping it would leave a
480
+ // corrected sender's old name searchable forever.
481
+ //
482
+ // ⚠️ The `remote` membership filter the main path uses is NOT available
483
+ // here — there is no list walk — so this iterates the indexed ids
484
+ // directly. A document deleted since the last run could therefore be
485
+ // refreshed once before the next count-changing run removes it. One
486
+ // wasted embedding, self-correcting, and strictly better than a stale
487
+ // metadata chunk living forever.
488
+ yield this.refreshStaleMetadata(report, this.store.indexed(), undefined);
489
+ return report;
490
+ }
491
+ }
492
+ const remote = new Map();
493
+ const { rows, count } = yield this.listAll(exports.DOCUMENT_LIST_PATH);
494
+ for (const row of rows)
495
+ remote.set(row.id, row.modified);
496
+ const indexed = this.store.indexed();
497
+ // ── DELETES, AND WHY THE COUNT IS THE ONLY SIGNAL THAT SEES THEM ────────
498
+ //
499
+ // A soft delete in Paperless sets `deleted_at` through a queryset
500
+ // `.update()`. That bypasses the model layer, so `auto_now` never fires and
501
+ // `modified` does not move — which means `max(modified)` is structurally
502
+ // BLIND to a deletion and can never be the signal that triggers one.
503
+ //
504
+ // `count` is not blind. It drops. And it costs nothing here: it arrived in
505
+ // the same list response this walk already reads.
506
+ //
507
+ // So the watermark comparison above decomposes into three cases, and this
508
+ // is why it checks `count` FIRST rather than treating the two scalars as
509
+ // interchangeable:
510
+ // count down, modified unchanged -> a deletion. The delete step below
511
+ // runs; nothing else has changed.
512
+ // count same, modified newer -> additions or edits. The diff runs.
513
+ // count same, modified same -> nothing at all. The run skips.
514
+ //
515
+ // ⚠️ DO NOT "SIMPLIFY" THIS INTO A SINGLE max(modified) CHECK. It reads like
516
+ // a tidier condition and it silently stops deleting anything, forever.
517
+ //
518
+ // A list whose rows disagree with its own count changed while it was being
519
+ // paged. A missing row is then indistinguishable from a deleted document,
520
+ // so the delete step is skipped rather than trusted.
521
+ const listComplete = rows.length === count;
522
+ report.deletedSkipped = !listComplete;
523
+ if (!listComplete) {
524
+ log(`the list returned ${rows.length} documents but Paperless counts ` +
525
+ `${count}; the delete step is skipped`);
526
+ }
527
+ if (listComplete) {
528
+ const toDelete = [...indexed.keys()].filter((id) => !remote.has(id));
529
+ if (toDelete.length > 0) {
530
+ this.store.deleteDocuments(toDelete);
531
+ report.deleted = toDelete.length;
532
+ log(`deleted ${toDelete.length} documents: ${toDelete.join(",")}`);
533
+ }
534
+ }
535
+ const toSync = [...remote.entries()].filter(([id, modified]) => {
536
+ // `force` re-embeds every document, not merely the changed ones. Bypassing
537
+ // the watermark alone would make `force` a no-op on a corpus whose
538
+ // `modified` values are all unchanged, which is the common case — the flag
539
+ // exists precisely for "the stored vectors are not trustworthy", and a
540
+ // trustworthy-looking `modified` says nothing about that.
541
+ if (opts.force)
542
+ return true;
543
+ const stored = indexed.get(id);
544
+ if (stored === undefined)
545
+ return true;
546
+ // Compared as instants, not strings: Paperless may re-serialise the same
547
+ // moment with a different offset, and a string compare would then reindex
548
+ // the whole corpus on every run.
549
+ return new Date(modified).getTime() !== new Date(stored).getTime();
550
+ });
551
+ let names;
552
+ const reindexed = new Set();
553
+ for (const [id] of toSync) {
554
+ try {
555
+ names !== null && names !== void 0 ? names : (names = yield this.loadNames());
556
+ const context = yield this.syncDocument(id, names);
557
+ reindexed.add(id);
558
+ report.splits += context.splits;
559
+ if (indexed.has(id))
560
+ report.updated++;
561
+ else
562
+ report.added++;
563
+ }
564
+ catch (err) {
565
+ if (err instanceof embedder_1.EmbeddingUnavailableError) {
566
+ // ⚠️ ABORT vs CONTINUE, AND THE LOG SAYS WHICH.
567
+ //
568
+ // An embedding outage (or a text that cannot be made small enough to
569
+ // embed) is not a per-document problem: every remaining document
570
+ // would fail the same way, so the run stops and the index keeps what
571
+ // it has. The line names the document so the next investigation
572
+ // starts from an id instead of from a scan of 2,466 candidates.
573
+ log(`document ${describeDocument(err, id)} failed and the run is ` +
574
+ `ABORTING (this failure would repeat for every remaining ` +
575
+ `document): ${errorMessage(err)}`);
576
+ throw err;
577
+ }
578
+ report.failed++;
579
+ log(`document ${describeDocument(err, id)} failed; CONTINUING with the ` +
580
+ `next document: ${errorMessage(err)}`);
581
+ }
582
+ }
583
+ // The 24 h metadata refresh. Tags, correspondents and document types change
584
+ // without touching `modified`, so without this a corrected sender stays in
585
+ // the metadata chunk -- which is indexed both ways -- forever.
586
+ //
587
+ // Documents just reindexed above are excluded: their metadata chunk was
588
+ // rebuilt as part of the reindex seconds ago.
589
+ const refreshTargets = new Map([...indexed.keys()].filter((id) => !reindexed.has(id)).map((id) => [id, ""]));
590
+ yield this.refreshStaleMetadata(report, refreshTargets, remote);
591
+ // ── THE WATERMARK IS WRITTEN AT THE END, FROM THE LIST WE ACTUALLY READ ──
592
+ //
593
+ // Not from a second probe. The list walk already reported `count`, and
594
+ // `remote` holds every `modified` it returned, so the maximum is arithmetic
595
+ // over data already in hand rather than another request.
596
+ //
597
+ // ⚠️ DELIBERATELY NOT `Math.max(...remote.values())`. Spreading a few
598
+ // thousand values into an argument list is a stack-depth question for no
599
+ // benefit, and `new Date(x).getTime()` is the comparison the rest of this
600
+ // file already uses. Instants, not strings.
601
+ let newest = null;
602
+ let newestAt = -Infinity;
603
+ for (const modified of remote.values()) {
604
+ const at = new Date(modified).getTime();
605
+ if (Number.isFinite(at) && at > newestAt) {
606
+ newestAt = at;
607
+ newest = modified;
608
+ }
609
+ }
610
+ // ⚠️ ONLY WHEN THE LIST WAS COMPLETE. A partial page set means documents
611
+ // were moving while the walk ran, so the maximum it saw is a maximum over a
612
+ // subset — recording it would state "the corpus is up to date through here"
613
+ // on the strength of a list that was known to be short.
614
+ if (listComplete) {
615
+ this.store.writeWatermark({
616
+ count,
617
+ modified: newest,
618
+ at: new Date(this.now()).toISOString(),
619
+ });
620
+ }
621
+ return report;
622
+ });
623
+ }
624
+ /**
625
+ * Rebuild the metadata chunk of every indexed document whose stamp is older
626
+ * than the refresh window.
627
+ *
628
+ * Extracted because TWO paths need it: the full run, and the watermark skip.
629
+ * A skip that returned before this would silently retire the 24 h refresh on
630
+ * any corpus that had stopped changing — and a stopped corpus is exactly the
631
+ * one whose tags and correspondents still get edited by hand, because nobody
632
+ * is uploading to it.
633
+ *
634
+ * `remote` is optional: the full run passes the live id set so a document
635
+ * deleted since the last sync is not refreshed, while the skip path has no
636
+ * list to consult and passes undefined.
637
+ */
638
+ refreshStaleMetadata(report, indexed, remote) {
639
+ return __awaiter(this, void 0, void 0, function* () {
640
+ const stamps = this.store.metadataStamps();
641
+ const refreshBefore = this.now() - exports.METADATA_REFRESH_MS;
642
+ let names;
643
+ for (const id of indexed.keys()) {
644
+ if (remote && !remote.has(id))
645
+ continue;
646
+ const metadataAt = stamps.get(id);
647
+ if (metadataAt && new Date(metadataAt).getTime() >= refreshBefore)
648
+ continue;
649
+ try {
650
+ names !== null && names !== void 0 ? names : (names = yield this.loadNames());
651
+ report.splits += yield this.refreshMetadata(id, names);
652
+ report.metadataRefreshed++;
653
+ }
654
+ catch (err) {
655
+ if (err instanceof embedder_1.EmbeddingUnavailableError) {
656
+ log(`document ${describeDocument(err, id)} metadata failed and the ` +
657
+ `run is ABORTING: ${errorMessage(err)}`);
658
+ throw err;
659
+ }
660
+ report.failed++;
661
+ log(`document ${describeDocument(err, id)} metadata failed; CONTINUING ` +
662
+ `with the next document: ${errorMessage(err)}`);
663
+ }
664
+ }
665
+ });
666
+ }
667
+ /**
668
+ * Page through a list endpoint, following `next`.
669
+ *
670
+ * A repeated `next` would loop forever and an endless one would never finish,
671
+ * so pages already fetched are tracked and the run fails past a page cap
672
+ * rather than hanging with no diagnostic.
673
+ */
674
+ listAll(path) {
675
+ return __awaiter(this, void 0, void 0, function* () {
676
+ var _a;
677
+ const rows = [];
678
+ const seen = new Set();
679
+ let count = 0;
680
+ let next = path;
681
+ while (next) {
682
+ if (seen.has(next)) {
683
+ throw new Error(`Paperless returned the same next page twice: ${next}`);
684
+ }
685
+ if (seen.size >= this.maxListPages) {
686
+ throw new Error(`Paperless returned more than ${this.maxListPages} pages for ${path}`);
687
+ }
688
+ seen.add(next);
689
+ const page = yield this.get(next);
690
+ if (seen.size === 1)
691
+ count = page.count;
692
+ rows.push(...((_a = page.results) !== null && _a !== void 0 ? _a : []));
693
+ next = page.next ? pathAfterApi(page.next) : null;
694
+ }
695
+ return { rows, count };
696
+ });
697
+ }
698
+ /**
699
+ * One request with a timeout.
700
+ *
701
+ * `PaperlessAPI.request` passes `options.signal` through to axios, which is
702
+ * what makes the budget real: before that was wired, this signal was accepted
703
+ * and silently ignored, so a Paperless that accepted the connection and then
704
+ * stalled would hold the sync open forever while the code read as though it
705
+ * had a 60 second limit.
706
+ */
707
+ get(path) {
708
+ return this.api.request(path, {
709
+ signal: AbortSignal.timeout(this.requestTimeoutMs),
710
+ });
711
+ }
712
+ loadNames() {
713
+ return __awaiter(this, void 0, void 0, function* () {
714
+ const [correspondents, documentTypes, tags] = yield Promise.all([
715
+ this.listAll("/correspondents/?page_size=1000"),
716
+ this.listAll("/document_types/?page_size=1000"),
717
+ this.listAll("/tags/?page_size=1000"),
718
+ ]);
719
+ return {
720
+ correspondents: toNameMap(correspondents.rows),
721
+ documentTypes: toNameMap(documentTypes.rows),
722
+ tags: toNameMap(tags.rows),
723
+ };
724
+ });
725
+ }
726
+ metadataTextFor(doc, names) {
727
+ const tags = doc.tags
728
+ .map((id) => names.tags.get(id))
729
+ .filter((name) => !!name);
730
+ return (0, chunk_1.metadataText)({ title: doc.title, created: doc.created }, {
731
+ correspondent: doc.correspondent
732
+ ? names.correspondents.get(doc.correspondent)
733
+ : undefined,
734
+ documentType: doc.document_type
735
+ ? names.documentTypes.get(doc.document_type)
736
+ : undefined,
737
+ tags,
738
+ });
739
+ }
740
+ syncDocument(id, names) {
741
+ return __awaiter(this, void 0, void 0, function* () {
742
+ var _a, _b, _c;
743
+ // A detail fetch can fail on its own, before any embedding happens. The id
744
+ // is still the useful part of the identity, so it is attached here and the
745
+ // title is left absent rather than invented.
746
+ let doc;
747
+ try {
748
+ doc = yield this.get(`/documents/${id}/`);
749
+ }
750
+ catch (err) {
751
+ throw withDocumentContext(err, { documentId: id, documentTitle: "" });
752
+ }
753
+ const metaText = this.metadataTextFor(doc, names);
754
+ const { chunks, truncated } = (0, chunk_1.chunkText)((_a = doc.content) !== null && _a !== void 0 ? _a : "", this.maxChunks);
755
+ const texts = [metaText, ...chunks];
756
+ // ⚠️ THE TITLE IS ATTACHED HERE BECAUSE THIS IS THE ONLY FRAME THAT HAS IT.
757
+ //
758
+ // A bare "document 1944 failed" costs a second round trip to identify, and
759
+ // on a 2,466-document corpus walking candidates by hand is how twenty tool
760
+ // calls get spent on one diagnosis. The title is a document heading rather
761
+ // than body text, so naming it in a log line places no content at risk.
762
+ let embeddings;
763
+ try {
764
+ embeddings = yield this.embedder.embedDocuments(texts);
765
+ }
766
+ catch (err) {
767
+ throw withDocumentContext(err, {
768
+ documentId: id,
769
+ documentTitle: doc.title,
770
+ });
771
+ }
772
+ const splits = (_c = (_b = this.embedder.lastSplits) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0;
773
+ const storedChunks = embeddings.map((embedding, i) => ({
774
+ kind: i === 0 ? "metadata" : "content",
775
+ text: texts[i],
776
+ embedding,
777
+ }));
778
+ this.store.replaceDocument({
779
+ id,
780
+ modified: doc.modified,
781
+ title: doc.title,
782
+ // Our store rejects a chunk count that does not match the array it was
783
+ // handed, which is the assertion that catches a mis-assembled document
784
+ // before it reaches the vector table. The reference store derived this
785
+ // and so never passed it.
786
+ chunkCount: storedChunks.length,
787
+ truncated,
788
+ metadataAt: new Date(this.now()).toISOString(),
789
+ }, storedChunks);
790
+ return { splits };
791
+ });
792
+ }
793
+ refreshMetadata(id, names) {
794
+ return __awaiter(this, void 0, void 0, function* () {
795
+ var _a, _b;
796
+ let doc;
797
+ try {
798
+ doc = yield this.get(`/documents/${id}/`);
799
+ }
800
+ catch (err) {
801
+ throw withDocumentContext(err, { documentId: id, documentTitle: "" });
802
+ }
803
+ const metaText = this.metadataTextFor(doc, names);
804
+ let embedding;
805
+ try {
806
+ [embedding] = yield this.embedder.embedDocuments([metaText]);
807
+ }
808
+ catch (err) {
809
+ throw withDocumentContext(err, {
810
+ documentId: id,
811
+ documentTitle: doc.title,
812
+ });
813
+ }
814
+ this.store.replaceMetadata(id, { kind: "metadata", text: metaText, embedding }, new Date(this.now()).toISOString());
815
+ return (_b = (_a = this.embedder.lastSplits) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
816
+ });
817
+ }
818
+ }
819
+ exports.IndexSync = IndexSync;