@retiform/paperless-mcp 2.3.5 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +10 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,819 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Incremental sync between the Paperless corpus and the local vector index.
|
|
3
|
+
//
|
|
4
|
+
// Algorithm ported from jaimemachado/paperless-mcp (src/search/sync.ts at
|
|
5
|
+
// 50f7beec0d36804cab8fbcaa7b47f85e562e8061). The algorithm is theirs; the
|
|
6
|
+
// bindings are not. Every adaptation is listed here because several of them are
|
|
7
|
+
// corrections rather than translations, and a silent one would be a defect:
|
|
8
|
+
//
|
|
9
|
+
// 1. `../metrics` and `../tracing` are NOT ported. Neither module exists in
|
|
10
|
+
// this fork (the store lap made the same call), and adding OpenTelemetry
|
|
11
|
+
// and prom-client to emit counters nobody reads is not a trade worth
|
|
12
|
+
// making. Console diagnostics replace the two console.error calls the
|
|
13
|
+
// reference already used.
|
|
14
|
+
// 2. `EmbeddingUnavailable` -> `EmbeddingUnavailableError`, and the abort is
|
|
15
|
+
// preserved: an embedding outage stops the run rather than writing an
|
|
16
|
+
// index that is missing documents it will claim to have.
|
|
17
|
+
// 3. The reference's `replaceDocument` call omits `chunkCount`, because its
|
|
18
|
+
// Postgres store derived the count. Ours REQUIRES it and rejects a
|
|
19
|
+
// mismatch against the chunk array, so it is passed explicitly.
|
|
20
|
+
// 4. The metadata-refresh loop in the reference reads `info.metadataAt` off
|
|
21
|
+
// `indexed()`. Our `indexed()` returns `id -> modified` only, so the
|
|
22
|
+
// stamp comes from `metadataStamps()`. Using the reference's shape here
|
|
23
|
+
// would have compared `undefined` on every document and refreshed the
|
|
24
|
+
// entire corpus every single run -- a correct-looking port that redoes
|
|
25
|
+
// 58 minutes of work on each call.
|
|
26
|
+
// 5. `pathAfterApi` is defensive. The reference slices at
|
|
27
|
+
// `indexOf("/api/") + 4`, which silently eats four characters when the
|
|
28
|
+
// marker is absent. Ours returns the path unchanged in that case instead
|
|
29
|
+
// of repairing a URL into a wrong one.
|
|
30
|
+
// 6. `MAX_CHUNKS` is injected rather than assumed, so the sync and the
|
|
31
|
+
// chunker cannot disagree about how much of a document was kept.
|
|
32
|
+
//
|
|
33
|
+
// ── The five load-bearing choices, each verified against our install ────────
|
|
34
|
+
//
|
|
35
|
+
// `modified` IS the incremental key. Verified on this install: the documents
|
|
36
|
+
// table has a `modified` column and `max(modified)` is today. A document
|
|
37
|
+
// edited in Paperless bumps it; a document whose tags changed does NOT.
|
|
38
|
+
//
|
|
39
|
+
// DELETES ONLY RUN WHEN `rows.length === count`. A list whose rows disagree
|
|
40
|
+
// with its own count changed while it was being paged, and the missing rows
|
|
41
|
+
// are indistinguishable from deleted documents. Deleting on a short list
|
|
42
|
+
// would remove live documents from the index, so the delete step is SKIPPED
|
|
43
|
+
// and reported instead. Tested.
|
|
44
|
+
//
|
|
45
|
+
// A 24 HOUR METADATA REFRESH. Tags, correspondents and document types change
|
|
46
|
+
// without touching `modified`, so a document whose sender was corrected
|
|
47
|
+
// would keep the old name in its metadata chunk forever. That chunk is
|
|
48
|
+
// vector- and lexical-indexed, so the stale name stays searchable.
|
|
49
|
+
//
|
|
50
|
+
// ⚠️ THAT REFRESH SURVIVES THE WATERMARK, AND IT IS NOT REDUNDANT WITH IT.
|
|
51
|
+
// Measured against Paperless' own bulk-edit code: `set_correspondent`,
|
|
52
|
+
// `set_document_type`, `set_storage_path` and `add_tag` all go through
|
|
53
|
+
// `qs.update(...)` or a `bulk_create` on the through-table. Neither fires
|
|
54
|
+
// `auto_now`, so none of them bumps `modified`. A corpus-level watermark is
|
|
55
|
+
// therefore structurally blind to an M2M edit — it cannot see one even in
|
|
56
|
+
// principle, let alone in this implementation. Deleting the refresh on the
|
|
57
|
+
// theory that the watermark subsumes it would re-open the stale-metadata hole
|
|
58
|
+
// it was written to close. It is a candidate for WIDENING (a cheaper M2M
|
|
59
|
+
// probe), never for removal.
|
|
60
|
+
//
|
|
61
|
+
// THE WATERMARK. `count` and `max(modified)` are recorded at the end of every
|
|
62
|
+
// completed run, and consulted at the start of the next one as a single
|
|
63
|
+
// request. A run against an unchanged corpus returns in ~218 ms without
|
|
64
|
+
// paging a single document. `force` bypasses it, because `force` is a
|
|
65
|
+
// statement about the stored vectors rather than about the corpus.
|
|
66
|
+
//
|
|
67
|
+
// AN EMBEDDING OUTAGE ABORTS THE WHOLE RUN. The alternative -- skip the
|
|
68
|
+
// document, carry on, report `failed` -- leaves an index that is missing
|
|
69
|
+
// documents while `documents` rows claim they are indexed, which is worse
|
|
70
|
+
// than no index because it looks complete.
|
|
71
|
+
//
|
|
72
|
+
// A RE-ENTRANCY GUARD. Two concurrent syncs against one SQLite file is a
|
|
73
|
+
// failure this arc has already hit once. `run()` returns the in-flight
|
|
74
|
+
// promise rather than starting a second pass.
|
|
75
|
+
//
|
|
76
|
+
// ── Triggering ─────────────────────────────────────────────────────────────
|
|
77
|
+
// On demand is the primary path: the `sync_embeddings` tool. `start()` exists
|
|
78
|
+
// but nothing calls it unless `PAPERLESS_MCP_SYNC_INTERVAL_MS` is set, because
|
|
79
|
+
// a stdio child's process lifetime belongs to the gateway -- a hidden timer
|
|
80
|
+
// refreshes on a schedule nobody can see or schedule around. An observable tool
|
|
81
|
+
// call can be scheduled with the scheduler we already use, where the schedule
|
|
82
|
+
// is legible.
|
|
83
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
84
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
85
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
86
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
87
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
88
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
89
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
90
|
+
});
|
|
91
|
+
};
|
|
92
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
93
|
+
exports.IndexSync = exports.REQUEST_TIMEOUT_MS = exports.WATERMARK_PATH = exports.MAX_LIST_PAGES = exports.DOCUMENT_LIST_PATH = exports.METADATA_REFRESH_MS = void 0;
|
|
94
|
+
exports.corpusChanged = corpusChanged;
|
|
95
|
+
exports.readCorpusState = readCorpusState;
|
|
96
|
+
exports.pathAfterApi = pathAfterApi;
|
|
97
|
+
const chunk_1 = require("./chunk");
|
|
98
|
+
const embedder_1 = require("./embedder");
|
|
99
|
+
const store_1 = require("./store");
|
|
100
|
+
/** How often a document's metadata chunk is rebuilt regardless of `modified`. */
|
|
101
|
+
exports.METADATA_REFRESH_MS = 24 * 60 * 60 * 1000;
|
|
102
|
+
/**
|
|
103
|
+
* Paged in id order, with the fields the diff needs and nothing else.
|
|
104
|
+
*
|
|
105
|
+
* Ordering by id matters: an insert does not move a row, so a document added
|
|
106
|
+
* while the list is being paged lands at the end rather than shuffling the
|
|
107
|
+
* pages already read. A delete DOES move rows, which is why the row count is
|
|
108
|
+
* compared to the reported count below.
|
|
109
|
+
*/
|
|
110
|
+
exports.DOCUMENT_LIST_PATH = "/documents/?fields=id,modified&page_size=1000&ordering=id";
|
|
111
|
+
/** A repeated or endless `next` would otherwise keep a run pending forever. */
|
|
112
|
+
exports.MAX_LIST_PAGES = 10000;
|
|
113
|
+
/**
|
|
114
|
+
* One request that answers "how big is the corpus, and when did it last move".
|
|
115
|
+
*
|
|
116
|
+
* Deliberately NOT a walk of the document list. The list is paged at 1000 rows
|
|
117
|
+
* with a full `id,modified` payload per row; on a corpus of a few thousand that
|
|
118
|
+
* is several requests and a few hundred kilobytes to learn two scalars. This is
|
|
119
|
+
* ONE request and about 200 bytes.
|
|
120
|
+
*
|
|
121
|
+
* `ordering=-modified&page_size=1` makes `results[0].modified` the maximum over
|
|
122
|
+
* the whole corpus, which is the value the incremental diff is keyed on.
|
|
123
|
+
*/
|
|
124
|
+
exports.WATERMARK_PATH = "/documents/?fields=id,modified&ordering=-modified&page_size=1";
|
|
125
|
+
/**
|
|
126
|
+
* Whether the live corpus differs from the indexed one.
|
|
127
|
+
*
|
|
128
|
+
* ⚠️ COMPARED AS INSTANTS, NOT STRINGS. `modified` arrives as an ISO 8601 string
|
|
129
|
+
* with an offset, and the same moment can be serialised with a different offset
|
|
130
|
+
* (`…+00:00` one day, `…-05:00` the next). A string compare would report a
|
|
131
|
+
* change on every run and defeat the entire point of the watermark, while
|
|
132
|
+
* looking exactly like a working check. The existing document diff already got
|
|
133
|
+
* this right; this is the same treatment applied to the corpus-level value.
|
|
134
|
+
*
|
|
135
|
+
* `count` is checked FIRST and separately. It is the only signal a soft delete
|
|
136
|
+
* produces: Paperless clears `deleted_at` through a queryset `.update()`, which
|
|
137
|
+
* does not fire `auto_now`, so a deleted document never bumps `max(modified)`.
|
|
138
|
+
* The count still drops, and the count is free in the same response.
|
|
139
|
+
*/
|
|
140
|
+
function corpusChanged(live, watermark) {
|
|
141
|
+
if (live.count !== watermark.count)
|
|
142
|
+
return true;
|
|
143
|
+
if (live.modified === null)
|
|
144
|
+
return watermark.modified !== null;
|
|
145
|
+
if (watermark.modified === null)
|
|
146
|
+
return true;
|
|
147
|
+
return new Date(live.modified).getTime() > new Date(watermark.modified).getTime();
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The corpus's size and newest `modified`, from ONE request.
|
|
151
|
+
*
|
|
152
|
+
* Returns undefined when the request fails. A watermark probe that THROWS would
|
|
153
|
+
* turn a cosmetic optimisation into a hard dependency: the sync would stop
|
|
154
|
+
* working on any Paperless that declines this particular query, when the list
|
|
155
|
+
* walk it already knows how to do would have succeeded. Falling back to the full
|
|
156
|
+
* diff is always correct, merely slower.
|
|
157
|
+
*
|
|
158
|
+
* Module-level rather than a private method because two callers need it — the
|
|
159
|
+
* sync's skip check and the search tool's opt-in staleness report — and two
|
|
160
|
+
* copies of "which fields, which ordering, which page size" is how they drift
|
|
161
|
+
* apart. One request shape, one place.
|
|
162
|
+
*/
|
|
163
|
+
function readCorpusState(api, requestTimeoutMs) {
|
|
164
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
165
|
+
var _a, _b;
|
|
166
|
+
try {
|
|
167
|
+
const page = yield api.request(exports.WATERMARK_PATH, requestTimeoutMs !== undefined
|
|
168
|
+
? { signal: AbortSignal.timeout(requestTimeoutMs) }
|
|
169
|
+
: {});
|
|
170
|
+
const first = ((_a = page.results) !== null && _a !== void 0 ? _a : [])[0];
|
|
171
|
+
return { count: page.count, modified: (_b = first === null || first === void 0 ? void 0 : first.modified) !== null && _b !== void 0 ? _b : null };
|
|
172
|
+
}
|
|
173
|
+
catch (err) {
|
|
174
|
+
log(`the corpus watermark could not be read: ${errorMessage(err)}`);
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
/** Without a limit, a Paperless that never answers keeps a run pending forever. */
|
|
180
|
+
exports.REQUEST_TIMEOUT_MS = 60000;
|
|
181
|
+
/**
|
|
182
|
+
* Turn an absolute `next` URL into the path this API client accepts.
|
|
183
|
+
*
|
|
184
|
+
* The list answer carries `next` as a full URL (verified: `GET
|
|
185
|
+
* /api/documents/?fields=id,modified&page_size=3&ordering=id` returns an
|
|
186
|
+
* absolute URL), while `PaperlessAPI.request` prepends `/api` itself. So the
|
|
187
|
+
* base is stripped and the query string kept.
|
|
188
|
+
*
|
|
189
|
+
* Deliberately NOT the reference's arithmetic: it slices at
|
|
190
|
+
* `indexOf("/api/") + 4`, which returns a four-character-amputated path when
|
|
191
|
+
* the marker is missing -- a wrong URL that looks like a right one. An absent
|
|
192
|
+
* marker returns the input unchanged, which fails loudly at the request instead.
|
|
193
|
+
*/
|
|
194
|
+
function pathAfterApi(url) {
|
|
195
|
+
const { pathname, search } = new URL(url);
|
|
196
|
+
const marker = pathname.indexOf("/api/");
|
|
197
|
+
if (marker < 0)
|
|
198
|
+
return pathname + search;
|
|
199
|
+
return pathname.slice(marker + "/api".length) + search;
|
|
200
|
+
}
|
|
201
|
+
function toNameMap(rows) {
|
|
202
|
+
return new Map(rows.map((row) => [row.id, row.name]));
|
|
203
|
+
}
|
|
204
|
+
function errorMessage(err) {
|
|
205
|
+
return err instanceof Error ? err.message : String(err);
|
|
206
|
+
}
|
|
207
|
+
function withDocumentContext(err, context) {
|
|
208
|
+
if (err && typeof err === "object") {
|
|
209
|
+
Object.assign(err, context);
|
|
210
|
+
}
|
|
211
|
+
return err;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Name the document in one line, so a failure is diagnosable from the log
|
|
215
|
+
* without a second round trip.
|
|
216
|
+
*
|
|
217
|
+
* Id first, because the id is what a person pastes into the Paperless URL; the
|
|
218
|
+
* title second, because it is what makes the line recognisable at a glance. A
|
|
219
|
+
* title is a document HEADING, never body text, so it is safe to log.
|
|
220
|
+
*/
|
|
221
|
+
function describeDocument(err, fallbackId) {
|
|
222
|
+
const id = err && typeof err === "object" && "documentId" in err
|
|
223
|
+
? err.documentId
|
|
224
|
+
: fallbackId;
|
|
225
|
+
const title = err && typeof err === "object" && "documentTitle" in err
|
|
226
|
+
? err.documentTitle
|
|
227
|
+
: undefined;
|
|
228
|
+
if (!title)
|
|
229
|
+
return `${id} (title unavailable: the document could not be read)`;
|
|
230
|
+
return `${id} \u201c${title}\u201d`;
|
|
231
|
+
}
|
|
232
|
+
function log(line) {
|
|
233
|
+
console.error(`[paperless-mcp] index: ${line}`);
|
|
234
|
+
}
|
|
235
|
+
class IndexSync {
|
|
236
|
+
constructor(opts) {
|
|
237
|
+
var _a, _b, _c, _d;
|
|
238
|
+
/** Whether the run in flight is a targeted one. See `run()`. */
|
|
239
|
+
this.runningTargeted = false;
|
|
240
|
+
this.api = opts.api;
|
|
241
|
+
this.store = opts.store;
|
|
242
|
+
this.embedder = opts.embedder;
|
|
243
|
+
this.maxChunks = (_a = opts.maxChunks) !== null && _a !== void 0 ? _a : chunk_1.MAX_CHUNKS;
|
|
244
|
+
this.now = (_b = opts.now) !== null && _b !== void 0 ? _b : Date.now;
|
|
245
|
+
this.requestTimeoutMs = (_c = opts.requestTimeoutMs) !== null && _c !== void 0 ? _c : exports.REQUEST_TIMEOUT_MS;
|
|
246
|
+
this.maxListPages = (_d = opts.maxListPages) !== null && _d !== void 0 ? _d : exports.MAX_LIST_PAGES;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Run one sync. Re-entrant calls join the run in flight.
|
|
250
|
+
*
|
|
251
|
+
* Two concurrent writers on one SQLite file is a failure this arc has already
|
|
252
|
+
* hit once, and WAL plus `busy_timeout` only makes the second writer wait --
|
|
253
|
+
* it does not make two simultaneous passes correct, because both would diff
|
|
254
|
+
* against the same stale `indexed()` snapshot and the loser's deletes would
|
|
255
|
+
* remove the winner's work.
|
|
256
|
+
*/
|
|
257
|
+
run(opts = {}) {
|
|
258
|
+
var _a, _b;
|
|
259
|
+
const targeted = ((_b = (_a = opts.documentIds) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) > 0;
|
|
260
|
+
// ⚠️ A TARGETED RUN AND A FULL RUN ANSWER DIFFERENT QUESTIONS, SO THEY MUST
|
|
261
|
+
// NOT SHARE ONE PROMISE.
|
|
262
|
+
//
|
|
263
|
+
// Joining is right when both callers want the same sweep: two scheduler
|
|
264
|
+
// ticks, or a tool call landing on a run already in flight. It is wrong when
|
|
265
|
+
// a full run joins a targeted one -- the full caller would receive a report
|
|
266
|
+
// covering one document and read it as "the corpus is up to date", which is
|
|
267
|
+
// the exact substitution this whole subsystem exists to avoid.
|
|
268
|
+
//
|
|
269
|
+
// So a full run requested during a targeted run is SEQUENCED behind it
|
|
270
|
+
// rather than joined to it. Same path, one run after the other; no second
|
|
271
|
+
// sync implementation.
|
|
272
|
+
if (!targeted && this.running && this.runningTargeted) {
|
|
273
|
+
return this.running.then(() => this.run(opts), () => this.run(opts));
|
|
274
|
+
}
|
|
275
|
+
if (!this.running) {
|
|
276
|
+
this.runningTargeted = targeted;
|
|
277
|
+
this.running = this.doRun(opts).finally(() => {
|
|
278
|
+
this.running = undefined;
|
|
279
|
+
this.runningTargeted = false;
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
return this.running;
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Run once and then on an interval. NOT called by this package.
|
|
286
|
+
*
|
|
287
|
+
* A stdio child's lifetime is the gateway's, so a timer here refreshes on a
|
|
288
|
+
* schedule nobody can see. The observable path is the `sync_embeddings` tool,
|
|
289
|
+
* which can be scheduled where the schedule is legible.
|
|
290
|
+
*/
|
|
291
|
+
start(intervalMs) {
|
|
292
|
+
var _a, _b;
|
|
293
|
+
if (this.timer)
|
|
294
|
+
return;
|
|
295
|
+
const scheduled = () => this.run().catch((err) => log(`scheduled run failed: ${errorMessage(err)}`));
|
|
296
|
+
void scheduled();
|
|
297
|
+
this.timer = setInterval(scheduled, intervalMs);
|
|
298
|
+
// Do not hold the process open for a refresh nobody asked for.
|
|
299
|
+
(_b = (_a = this.timer).unref) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
300
|
+
}
|
|
301
|
+
stop() {
|
|
302
|
+
if (this.timer)
|
|
303
|
+
clearInterval(this.timer);
|
|
304
|
+
this.timer = undefined;
|
|
305
|
+
}
|
|
306
|
+
get isRunning() {
|
|
307
|
+
return this.running !== undefined;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* True while a background timer is live.
|
|
311
|
+
*
|
|
312
|
+
* Distinct from `isRunning`, which reports a pass in flight: between two
|
|
313
|
+
* scheduled ticks the timer is set and nothing is running. A wiring test that
|
|
314
|
+
* asked `isRunning` would report "no schedule" on a working schedule whenever
|
|
315
|
+
* it happened to look between ticks.
|
|
316
|
+
*/
|
|
317
|
+
get isScheduled() {
|
|
318
|
+
return this.timer !== undefined;
|
|
319
|
+
}
|
|
320
|
+
doRun(opts) {
|
|
321
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
322
|
+
const started = this.now();
|
|
323
|
+
try {
|
|
324
|
+
const report = yield this.syncAll(opts);
|
|
325
|
+
const seconds = ((this.now() - started) / 1000).toFixed(1);
|
|
326
|
+
if (report.skipped) {
|
|
327
|
+
log(`run finished in ${seconds}s: SKIPPED (${report.skipReason})`);
|
|
328
|
+
return report;
|
|
329
|
+
}
|
|
330
|
+
log(`run finished in ${seconds}s: +${report.added} added, ` +
|
|
331
|
+
`~${report.updated} updated, -${report.deleted} deleted, ` +
|
|
332
|
+
`${report.metadataRefreshed} metadata refreshed, ${report.failed} failed, ` +
|
|
333
|
+
`${report.splits} split` +
|
|
334
|
+
(report.deletedSkipped ? " (deletes SKIPPED: list changed mid-run)" : ""));
|
|
335
|
+
return report;
|
|
336
|
+
}
|
|
337
|
+
catch (err) {
|
|
338
|
+
log(`run failed: ${errorMessage(err)}`);
|
|
339
|
+
throw err;
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
syncAll(opts) {
|
|
344
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
345
|
+
var _a;
|
|
346
|
+
const report = {
|
|
347
|
+
added: 0,
|
|
348
|
+
updated: 0,
|
|
349
|
+
deleted: 0,
|
|
350
|
+
metadataRefreshed: 0,
|
|
351
|
+
failed: 0,
|
|
352
|
+
deletedSkipped: false,
|
|
353
|
+
splits: 0,
|
|
354
|
+
skipped: false,
|
|
355
|
+
};
|
|
356
|
+
// ⚠️ THE INDEX'S IDENTITY IS WRITTEN BY THE RUN, NOT BY THE FIXTURE.
|
|
357
|
+
//
|
|
358
|
+
// `writeMeta` and `assertSchemaCompatible` both existed and were both dead:
|
|
359
|
+
// no production call site for either, so `embedding_status` reported
|
|
360
|
+
// `dimensions: 0, model: "", prefix_mode: ""` on a real index, and a
|
|
361
|
+
// prefix-mode mismatch could never be refused. The suite stayed green
|
|
362
|
+
// because the FIXTURES called `writeMeta` themselves -- the tool asserted
|
|
363
|
+
// the function while nothing asserted the wiring, which is the
|
|
364
|
+
// no-op-stub-in-a-test-fake family: a contract leg with no observable
|
|
365
|
+
// output and therefore no test.
|
|
366
|
+
//
|
|
367
|
+
// Written FIRST, before any document work, so the index never holds vectors
|
|
368
|
+
// whose provenance it cannot state. It is also cheap: four upserts.
|
|
369
|
+
this.store.writeMeta({
|
|
370
|
+
schemaVersion: store_1.SCHEMA_VERSION,
|
|
371
|
+
dimensions: this.embedder.dimensions,
|
|
372
|
+
model: this.embedder.model,
|
|
373
|
+
prefixMode: this.embedder.prefixMode,
|
|
374
|
+
});
|
|
375
|
+
// ── TARGETED RUN: THE CALLER NAMED THE DOCUMENTS ────────────────────────
|
|
376
|
+
//
|
|
377
|
+
// This is the webhook's path. A delivery that named ids is a statement
|
|
378
|
+
// about THOSE documents, and re-indexing exactly those is both cheaper and
|
|
379
|
+
// more complete than any corpus-level signal could be.
|
|
380
|
+
//
|
|
381
|
+
// ⚠️ WHY THIS CANNOT BE EXPRESSED AS THE WATERMARK DIFF, which is the whole
|
|
382
|
+
// reason it is a separate branch. A tag, correspondent or document-type
|
|
383
|
+
// edit goes through a queryset update on a through-table: `auto_now` never
|
|
384
|
+
// fires, so `modified` does not move, and the document count is unchanged.
|
|
385
|
+
// Both watermark scalars therefore read "nothing happened" while the
|
|
386
|
+
// document's metadata chunk has genuinely gone stale. The event is the only
|
|
387
|
+
// signal that exists for that change, and it names the document, so there
|
|
388
|
+
// is no reason to go looking for one at corpus scale.
|
|
389
|
+
//
|
|
390
|
+
// The document is re-fetched in full, so content and metadata are both
|
|
391
|
+
// rebuilt. That is deliberate: the payload carries no change type, so
|
|
392
|
+
// deciding between a content reindex and a metadata-only refresh would mean
|
|
393
|
+
// guessing. One detail fetch settles it by observation.
|
|
394
|
+
if (opts.documentIds && opts.documentIds.length > 0) {
|
|
395
|
+
const ids = [...new Set(opts.documentIds)];
|
|
396
|
+
// ⚠️ SNAPSHOT BEFORE THE LOOP, exactly as the full path does. Reading
|
|
397
|
+
// `indexed()` per document would consult it AFTER that document was
|
|
398
|
+
// written, so every document would classify as "updated" and `added`
|
|
399
|
+
// would always be zero -- a count that is wrong in the flattering
|
|
400
|
+
// direction, and wrong for a reason no assertion about the index's final
|
|
401
|
+
// contents could detect.
|
|
402
|
+
const indexedBefore = this.store.indexed();
|
|
403
|
+
let names;
|
|
404
|
+
for (const id of ids) {
|
|
405
|
+
try {
|
|
406
|
+
names !== null && names !== void 0 ? names : (names = yield this.loadNames());
|
|
407
|
+
const context = yield this.syncDocument(id, names);
|
|
408
|
+
report.splits += context.splits;
|
|
409
|
+
if (indexedBefore.has(id))
|
|
410
|
+
report.updated++;
|
|
411
|
+
else
|
|
412
|
+
report.added++;
|
|
413
|
+
}
|
|
414
|
+
catch (err) {
|
|
415
|
+
if (err instanceof embedder_1.EmbeddingUnavailableError) {
|
|
416
|
+
// Same reasoning as the full path: every remaining document would
|
|
417
|
+
// fail identically, so the run stops and the index keeps what it
|
|
418
|
+
// has. The next delivery retries.
|
|
419
|
+
log(`document ${describeDocument(err, id)} failed and the targeted ` +
|
|
420
|
+
`run is ABORTING: ${errorMessage(err)}`);
|
|
421
|
+
throw err;
|
|
422
|
+
}
|
|
423
|
+
// ⚠️ A DOCUMENT THAT NO LONGER EXISTS MUST NOT ABORT THE RUN. A
|
|
424
|
+
// delivery can arrive for a document deleted between the change and
|
|
425
|
+
// the webhook — the event is already in flight and cannot be
|
|
426
|
+
// recalled. Counting it as `failed` and continuing is the whole
|
|
427
|
+
// handling; the deletion itself is the watermark's `count` check,
|
|
428
|
+
// because the sending side fires nothing on delete.
|
|
429
|
+
report.failed++;
|
|
430
|
+
log(`document ${describeDocument(err, id)} failed; CONTINUING (a ` +
|
|
431
|
+
`delivery may name a document deleted since the change): ` +
|
|
432
|
+
`${errorMessage(err)}`);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
// ⚠️ THE WATERMARK IS NOT WRITTEN HERE, AND MUST NOT BE. It states "the
|
|
436
|
+
// corpus is indexed through this point", and a targeted run has looked at
|
|
437
|
+
// a handful of documents rather than the corpus. Recording one would claim
|
|
438
|
+
// a sweep that never happened and stop the next full run from doing it.
|
|
439
|
+
report.skipped = false;
|
|
440
|
+
return report;
|
|
441
|
+
}
|
|
442
|
+
// ── THE WATERMARK, CONSULTED BEFORE ANY PAGING ──────────────────────────
|
|
443
|
+
//
|
|
444
|
+
// One request, two scalars. If the corpus has not moved since the last run
|
|
445
|
+
// there is no diff to compute and the run returns having cost ~218 ms
|
|
446
|
+
// instead of a paged walk of every document id.
|
|
447
|
+
//
|
|
448
|
+
// ⚠️ `force` BYPASSES THIS ENTIRELY. `force` means "the stored vectors are
|
|
449
|
+
// not trustworthy" — it is a statement about the INDEX, not about the
|
|
450
|
+
// corpus. Letting the watermark short-circuit it would make `force` a no-op
|
|
451
|
+
// on the second call, which is a silent lie about what the flag does.
|
|
452
|
+
const live = opts.force
|
|
453
|
+
? undefined
|
|
454
|
+
: yield readCorpusState(this.api, this.requestTimeoutMs);
|
|
455
|
+
if (live) {
|
|
456
|
+
const watermark = this.store.watermark();
|
|
457
|
+
// No stored watermark means this index predates the watermark, or has
|
|
458
|
+
// never completed a run. Either way there is nothing to compare against
|
|
459
|
+
// and the diff must run: treating an absent watermark as "nothing
|
|
460
|
+
// changed" would skip the first sync after an upgrade and then keep
|
|
461
|
+
// skipping, permanently and silently.
|
|
462
|
+
if (watermark && !corpusChanged(live, watermark)) {
|
|
463
|
+
report.skipped = true;
|
|
464
|
+
report.skipReason =
|
|
465
|
+
`the corpus matches the last run (${live.count} documents, ` +
|
|
466
|
+
`max(modified) ${(_a = live.modified) !== null && _a !== void 0 ? _a : "none"}); the document diff was ` +
|
|
467
|
+
`skipped`;
|
|
468
|
+
// ⚠️ A SKIP SKIPS THE DIFF, NOT THE METADATA REFRESH.
|
|
469
|
+
//
|
|
470
|
+
// This is the whole reason the watermark cannot subsume the refresh: a
|
|
471
|
+
// tag, correspondent or document-type edit goes through a queryset
|
|
472
|
+
// `.update()` on a through-table, `auto_now` never fires, and neither
|
|
473
|
+
// `count` nor `max(modified)` moves. A corpus can be byte-identical in
|
|
474
|
+
// both scalars while a document's metadata chunk is stale.
|
|
475
|
+
//
|
|
476
|
+
// So the refresh runs here too. On a freshly-synced index every stamp
|
|
477
|
+
// is inside the window and this loop makes ZERO requests, which is what
|
|
478
|
+
// keeps the "one request" promise true in the common case. When it does
|
|
479
|
+
// have work to do, that work is real and skipping it would leave a
|
|
480
|
+
// corrected sender's old name searchable forever.
|
|
481
|
+
//
|
|
482
|
+
// ⚠️ The `remote` membership filter the main path uses is NOT available
|
|
483
|
+
// here — there is no list walk — so this iterates the indexed ids
|
|
484
|
+
// directly. A document deleted since the last run could therefore be
|
|
485
|
+
// refreshed once before the next count-changing run removes it. One
|
|
486
|
+
// wasted embedding, self-correcting, and strictly better than a stale
|
|
487
|
+
// metadata chunk living forever.
|
|
488
|
+
yield this.refreshStaleMetadata(report, this.store.indexed(), undefined);
|
|
489
|
+
return report;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
const remote = new Map();
|
|
493
|
+
const { rows, count } = yield this.listAll(exports.DOCUMENT_LIST_PATH);
|
|
494
|
+
for (const row of rows)
|
|
495
|
+
remote.set(row.id, row.modified);
|
|
496
|
+
const indexed = this.store.indexed();
|
|
497
|
+
// ── DELETES, AND WHY THE COUNT IS THE ONLY SIGNAL THAT SEES THEM ────────
|
|
498
|
+
//
|
|
499
|
+
// A soft delete in Paperless sets `deleted_at` through a queryset
|
|
500
|
+
// `.update()`. That bypasses the model layer, so `auto_now` never fires and
|
|
501
|
+
// `modified` does not move — which means `max(modified)` is structurally
|
|
502
|
+
// BLIND to a deletion and can never be the signal that triggers one.
|
|
503
|
+
//
|
|
504
|
+
// `count` is not blind. It drops. And it costs nothing here: it arrived in
|
|
505
|
+
// the same list response this walk already reads.
|
|
506
|
+
//
|
|
507
|
+
// So the watermark comparison above decomposes into three cases, and this
|
|
508
|
+
// is why it checks `count` FIRST rather than treating the two scalars as
|
|
509
|
+
// interchangeable:
|
|
510
|
+
// count down, modified unchanged -> a deletion. The delete step below
|
|
511
|
+
// runs; nothing else has changed.
|
|
512
|
+
// count same, modified newer -> additions or edits. The diff runs.
|
|
513
|
+
// count same, modified same -> nothing at all. The run skips.
|
|
514
|
+
//
|
|
515
|
+
// ⚠️ DO NOT "SIMPLIFY" THIS INTO A SINGLE max(modified) CHECK. It reads like
|
|
516
|
+
// a tidier condition and it silently stops deleting anything, forever.
|
|
517
|
+
//
|
|
518
|
+
// A list whose rows disagree with its own count changed while it was being
|
|
519
|
+
// paged. A missing row is then indistinguishable from a deleted document,
|
|
520
|
+
// so the delete step is skipped rather than trusted.
|
|
521
|
+
const listComplete = rows.length === count;
|
|
522
|
+
report.deletedSkipped = !listComplete;
|
|
523
|
+
if (!listComplete) {
|
|
524
|
+
log(`the list returned ${rows.length} documents but Paperless counts ` +
|
|
525
|
+
`${count}; the delete step is skipped`);
|
|
526
|
+
}
|
|
527
|
+
if (listComplete) {
|
|
528
|
+
const toDelete = [...indexed.keys()].filter((id) => !remote.has(id));
|
|
529
|
+
if (toDelete.length > 0) {
|
|
530
|
+
this.store.deleteDocuments(toDelete);
|
|
531
|
+
report.deleted = toDelete.length;
|
|
532
|
+
log(`deleted ${toDelete.length} documents: ${toDelete.join(",")}`);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
const toSync = [...remote.entries()].filter(([id, modified]) => {
|
|
536
|
+
// `force` re-embeds every document, not merely the changed ones. Bypassing
|
|
537
|
+
// the watermark alone would make `force` a no-op on a corpus whose
|
|
538
|
+
// `modified` values are all unchanged, which is the common case — the flag
|
|
539
|
+
// exists precisely for "the stored vectors are not trustworthy", and a
|
|
540
|
+
// trustworthy-looking `modified` says nothing about that.
|
|
541
|
+
if (opts.force)
|
|
542
|
+
return true;
|
|
543
|
+
const stored = indexed.get(id);
|
|
544
|
+
if (stored === undefined)
|
|
545
|
+
return true;
|
|
546
|
+
// Compared as instants, not strings: Paperless may re-serialise the same
|
|
547
|
+
// moment with a different offset, and a string compare would then reindex
|
|
548
|
+
// the whole corpus on every run.
|
|
549
|
+
return new Date(modified).getTime() !== new Date(stored).getTime();
|
|
550
|
+
});
|
|
551
|
+
let names;
|
|
552
|
+
const reindexed = new Set();
|
|
553
|
+
for (const [id] of toSync) {
|
|
554
|
+
try {
|
|
555
|
+
names !== null && names !== void 0 ? names : (names = yield this.loadNames());
|
|
556
|
+
const context = yield this.syncDocument(id, names);
|
|
557
|
+
reindexed.add(id);
|
|
558
|
+
report.splits += context.splits;
|
|
559
|
+
if (indexed.has(id))
|
|
560
|
+
report.updated++;
|
|
561
|
+
else
|
|
562
|
+
report.added++;
|
|
563
|
+
}
|
|
564
|
+
catch (err) {
|
|
565
|
+
if (err instanceof embedder_1.EmbeddingUnavailableError) {
|
|
566
|
+
// ⚠️ ABORT vs CONTINUE, AND THE LOG SAYS WHICH.
|
|
567
|
+
//
|
|
568
|
+
// An embedding outage (or a text that cannot be made small enough to
|
|
569
|
+
// embed) is not a per-document problem: every remaining document
|
|
570
|
+
// would fail the same way, so the run stops and the index keeps what
|
|
571
|
+
// it has. The line names the document so the next investigation
|
|
572
|
+
// starts from an id instead of from a scan of 2,466 candidates.
|
|
573
|
+
log(`document ${describeDocument(err, id)} failed and the run is ` +
|
|
574
|
+
`ABORTING (this failure would repeat for every remaining ` +
|
|
575
|
+
`document): ${errorMessage(err)}`);
|
|
576
|
+
throw err;
|
|
577
|
+
}
|
|
578
|
+
report.failed++;
|
|
579
|
+
log(`document ${describeDocument(err, id)} failed; CONTINUING with the ` +
|
|
580
|
+
`next document: ${errorMessage(err)}`);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
// The 24 h metadata refresh. Tags, correspondents and document types change
|
|
584
|
+
// without touching `modified`, so without this a corrected sender stays in
|
|
585
|
+
// the metadata chunk -- which is indexed both ways -- forever.
|
|
586
|
+
//
|
|
587
|
+
// Documents just reindexed above are excluded: their metadata chunk was
|
|
588
|
+
// rebuilt as part of the reindex seconds ago.
|
|
589
|
+
const refreshTargets = new Map([...indexed.keys()].filter((id) => !reindexed.has(id)).map((id) => [id, ""]));
|
|
590
|
+
yield this.refreshStaleMetadata(report, refreshTargets, remote);
|
|
591
|
+
// ── THE WATERMARK IS WRITTEN AT THE END, FROM THE LIST WE ACTUALLY READ ──
|
|
592
|
+
//
|
|
593
|
+
// Not from a second probe. The list walk already reported `count`, and
|
|
594
|
+
// `remote` holds every `modified` it returned, so the maximum is arithmetic
|
|
595
|
+
// over data already in hand rather than another request.
|
|
596
|
+
//
|
|
597
|
+
// ⚠️ DELIBERATELY NOT `Math.max(...remote.values())`. Spreading a few
|
|
598
|
+
// thousand values into an argument list is a stack-depth question for no
|
|
599
|
+
// benefit, and `new Date(x).getTime()` is the comparison the rest of this
|
|
600
|
+
// file already uses. Instants, not strings.
|
|
601
|
+
let newest = null;
|
|
602
|
+
let newestAt = -Infinity;
|
|
603
|
+
for (const modified of remote.values()) {
|
|
604
|
+
const at = new Date(modified).getTime();
|
|
605
|
+
if (Number.isFinite(at) && at > newestAt) {
|
|
606
|
+
newestAt = at;
|
|
607
|
+
newest = modified;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
// ⚠️ ONLY WHEN THE LIST WAS COMPLETE. A partial page set means documents
|
|
611
|
+
// were moving while the walk ran, so the maximum it saw is a maximum over a
|
|
612
|
+
// subset — recording it would state "the corpus is up to date through here"
|
|
613
|
+
// on the strength of a list that was known to be short.
|
|
614
|
+
if (listComplete) {
|
|
615
|
+
this.store.writeWatermark({
|
|
616
|
+
count,
|
|
617
|
+
modified: newest,
|
|
618
|
+
at: new Date(this.now()).toISOString(),
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
return report;
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Rebuild the metadata chunk of every indexed document whose stamp is older
|
|
626
|
+
* than the refresh window.
|
|
627
|
+
*
|
|
628
|
+
* Extracted because TWO paths need it: the full run, and the watermark skip.
|
|
629
|
+
* A skip that returned before this would silently retire the 24 h refresh on
|
|
630
|
+
* any corpus that had stopped changing — and a stopped corpus is exactly the
|
|
631
|
+
* one whose tags and correspondents still get edited by hand, because nobody
|
|
632
|
+
* is uploading to it.
|
|
633
|
+
*
|
|
634
|
+
* `remote` is optional: the full run passes the live id set so a document
|
|
635
|
+
* deleted since the last sync is not refreshed, while the skip path has no
|
|
636
|
+
* list to consult and passes undefined.
|
|
637
|
+
*/
|
|
638
|
+
refreshStaleMetadata(report, indexed, remote) {
|
|
639
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
640
|
+
const stamps = this.store.metadataStamps();
|
|
641
|
+
const refreshBefore = this.now() - exports.METADATA_REFRESH_MS;
|
|
642
|
+
let names;
|
|
643
|
+
for (const id of indexed.keys()) {
|
|
644
|
+
if (remote && !remote.has(id))
|
|
645
|
+
continue;
|
|
646
|
+
const metadataAt = stamps.get(id);
|
|
647
|
+
if (metadataAt && new Date(metadataAt).getTime() >= refreshBefore)
|
|
648
|
+
continue;
|
|
649
|
+
try {
|
|
650
|
+
names !== null && names !== void 0 ? names : (names = yield this.loadNames());
|
|
651
|
+
report.splits += yield this.refreshMetadata(id, names);
|
|
652
|
+
report.metadataRefreshed++;
|
|
653
|
+
}
|
|
654
|
+
catch (err) {
|
|
655
|
+
if (err instanceof embedder_1.EmbeddingUnavailableError) {
|
|
656
|
+
log(`document ${describeDocument(err, id)} metadata failed and the ` +
|
|
657
|
+
`run is ABORTING: ${errorMessage(err)}`);
|
|
658
|
+
throw err;
|
|
659
|
+
}
|
|
660
|
+
report.failed++;
|
|
661
|
+
log(`document ${describeDocument(err, id)} metadata failed; CONTINUING ` +
|
|
662
|
+
`with the next document: ${errorMessage(err)}`);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Page through a list endpoint, following `next`.
|
|
669
|
+
*
|
|
670
|
+
* A repeated `next` would loop forever and an endless one would never finish,
|
|
671
|
+
* so pages already fetched are tracked and the run fails past a page cap
|
|
672
|
+
* rather than hanging with no diagnostic.
|
|
673
|
+
*/
|
|
674
|
+
listAll(path) {
|
|
675
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
676
|
+
var _a;
|
|
677
|
+
const rows = [];
|
|
678
|
+
const seen = new Set();
|
|
679
|
+
let count = 0;
|
|
680
|
+
let next = path;
|
|
681
|
+
while (next) {
|
|
682
|
+
if (seen.has(next)) {
|
|
683
|
+
throw new Error(`Paperless returned the same next page twice: ${next}`);
|
|
684
|
+
}
|
|
685
|
+
if (seen.size >= this.maxListPages) {
|
|
686
|
+
throw new Error(`Paperless returned more than ${this.maxListPages} pages for ${path}`);
|
|
687
|
+
}
|
|
688
|
+
seen.add(next);
|
|
689
|
+
const page = yield this.get(next);
|
|
690
|
+
if (seen.size === 1)
|
|
691
|
+
count = page.count;
|
|
692
|
+
rows.push(...((_a = page.results) !== null && _a !== void 0 ? _a : []));
|
|
693
|
+
next = page.next ? pathAfterApi(page.next) : null;
|
|
694
|
+
}
|
|
695
|
+
return { rows, count };
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
/**
|
|
699
|
+
* One request with a timeout.
|
|
700
|
+
*
|
|
701
|
+
* `PaperlessAPI.request` passes `options.signal` through to axios, which is
|
|
702
|
+
* what makes the budget real: before that was wired, this signal was accepted
|
|
703
|
+
* and silently ignored, so a Paperless that accepted the connection and then
|
|
704
|
+
* stalled would hold the sync open forever while the code read as though it
|
|
705
|
+
* had a 60 second limit.
|
|
706
|
+
*/
|
|
707
|
+
get(path) {
|
|
708
|
+
return this.api.request(path, {
|
|
709
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
loadNames() {
|
|
713
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
714
|
+
const [correspondents, documentTypes, tags] = yield Promise.all([
|
|
715
|
+
this.listAll("/correspondents/?page_size=1000"),
|
|
716
|
+
this.listAll("/document_types/?page_size=1000"),
|
|
717
|
+
this.listAll("/tags/?page_size=1000"),
|
|
718
|
+
]);
|
|
719
|
+
return {
|
|
720
|
+
correspondents: toNameMap(correspondents.rows),
|
|
721
|
+
documentTypes: toNameMap(documentTypes.rows),
|
|
722
|
+
tags: toNameMap(tags.rows),
|
|
723
|
+
};
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
metadataTextFor(doc, names) {
|
|
727
|
+
const tags = doc.tags
|
|
728
|
+
.map((id) => names.tags.get(id))
|
|
729
|
+
.filter((name) => !!name);
|
|
730
|
+
return (0, chunk_1.metadataText)({ title: doc.title, created: doc.created }, {
|
|
731
|
+
correspondent: doc.correspondent
|
|
732
|
+
? names.correspondents.get(doc.correspondent)
|
|
733
|
+
: undefined,
|
|
734
|
+
documentType: doc.document_type
|
|
735
|
+
? names.documentTypes.get(doc.document_type)
|
|
736
|
+
: undefined,
|
|
737
|
+
tags,
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
syncDocument(id, names) {
|
|
741
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
742
|
+
var _a, _b, _c;
|
|
743
|
+
// A detail fetch can fail on its own, before any embedding happens. The id
|
|
744
|
+
// is still the useful part of the identity, so it is attached here and the
|
|
745
|
+
// title is left absent rather than invented.
|
|
746
|
+
let doc;
|
|
747
|
+
try {
|
|
748
|
+
doc = yield this.get(`/documents/${id}/`);
|
|
749
|
+
}
|
|
750
|
+
catch (err) {
|
|
751
|
+
throw withDocumentContext(err, { documentId: id, documentTitle: "" });
|
|
752
|
+
}
|
|
753
|
+
const metaText = this.metadataTextFor(doc, names);
|
|
754
|
+
const { chunks, truncated } = (0, chunk_1.chunkText)((_a = doc.content) !== null && _a !== void 0 ? _a : "", this.maxChunks);
|
|
755
|
+
const texts = [metaText, ...chunks];
|
|
756
|
+
// ⚠️ THE TITLE IS ATTACHED HERE BECAUSE THIS IS THE ONLY FRAME THAT HAS IT.
|
|
757
|
+
//
|
|
758
|
+
// A bare "document 1944 failed" costs a second round trip to identify, and
|
|
759
|
+
// on a 2,466-document corpus walking candidates by hand is how twenty tool
|
|
760
|
+
// calls get spent on one diagnosis. The title is a document heading rather
|
|
761
|
+
// than body text, so naming it in a log line places no content at risk.
|
|
762
|
+
let embeddings;
|
|
763
|
+
try {
|
|
764
|
+
embeddings = yield this.embedder.embedDocuments(texts);
|
|
765
|
+
}
|
|
766
|
+
catch (err) {
|
|
767
|
+
throw withDocumentContext(err, {
|
|
768
|
+
documentId: id,
|
|
769
|
+
documentTitle: doc.title,
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
const splits = (_c = (_b = this.embedder.lastSplits) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0;
|
|
773
|
+
const storedChunks = embeddings.map((embedding, i) => ({
|
|
774
|
+
kind: i === 0 ? "metadata" : "content",
|
|
775
|
+
text: texts[i],
|
|
776
|
+
embedding,
|
|
777
|
+
}));
|
|
778
|
+
this.store.replaceDocument({
|
|
779
|
+
id,
|
|
780
|
+
modified: doc.modified,
|
|
781
|
+
title: doc.title,
|
|
782
|
+
// Our store rejects a chunk count that does not match the array it was
|
|
783
|
+
// handed, which is the assertion that catches a mis-assembled document
|
|
784
|
+
// before it reaches the vector table. The reference store derived this
|
|
785
|
+
// and so never passed it.
|
|
786
|
+
chunkCount: storedChunks.length,
|
|
787
|
+
truncated,
|
|
788
|
+
metadataAt: new Date(this.now()).toISOString(),
|
|
789
|
+
}, storedChunks);
|
|
790
|
+
return { splits };
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
refreshMetadata(id, names) {
|
|
794
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
795
|
+
var _a, _b;
|
|
796
|
+
let doc;
|
|
797
|
+
try {
|
|
798
|
+
doc = yield this.get(`/documents/${id}/`);
|
|
799
|
+
}
|
|
800
|
+
catch (err) {
|
|
801
|
+
throw withDocumentContext(err, { documentId: id, documentTitle: "" });
|
|
802
|
+
}
|
|
803
|
+
const metaText = this.metadataTextFor(doc, names);
|
|
804
|
+
let embedding;
|
|
805
|
+
try {
|
|
806
|
+
[embedding] = yield this.embedder.embedDocuments([metaText]);
|
|
807
|
+
}
|
|
808
|
+
catch (err) {
|
|
809
|
+
throw withDocumentContext(err, {
|
|
810
|
+
documentId: id,
|
|
811
|
+
documentTitle: doc.title,
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
this.store.replaceMetadata(id, { kind: "metadata", text: metaText, embedding }, new Date(this.now()).toISOString());
|
|
815
|
+
return (_b = (_a = this.embedder.lastSplits) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
exports.IndexSync = IndexSync;
|