@retiform/paperless-mcp 2.3.6 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +9 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/manifest.test.d.ts +0 -1
- package/build/manifest.test.js +0 -66
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,1199 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Semantic search tools over the local vector index.
|
|
3
|
+
//
|
|
4
|
+
// Three tools, and one of them is the whole reason this file exists:
|
|
5
|
+
//
|
|
6
|
+
// semantic_search meaning + words, fused, with a degradation contract
|
|
7
|
+
// embedding_status what the index actually holds
|
|
8
|
+
// sync_embeddings bring the index up to date, explicitly
|
|
9
|
+
//
|
|
10
|
+
// ── The degradation contract ───────────────────────────────────────────────
|
|
11
|
+
// The embedding server being down is EXPECTED, not exceptional. It is a local
|
|
12
|
+
// process, restarted for model changes and absent on a fresh machine. So:
|
|
13
|
+
//
|
|
14
|
+
// * No tool here throws because Ollama is down. A thrown error from an MCP
|
|
15
|
+
// tool reads as "the tool is broken", which is a lie about a tool that is
|
|
16
|
+
// working correctly and reporting a real condition.
|
|
17
|
+
// * Lexical results are STILL RETURNED, with `semantic: "unavailable"`. The
|
|
18
|
+
// lexical half lives in the same SQLite file and does not need the
|
|
19
|
+
// embedder, so an outage degrades the ranking, not the capability.
|
|
20
|
+
// * A degraded call NEVER returns a silent empty list. An empty result and
|
|
21
|
+
// "the embedder is down" are different facts and the caller cannot tell
|
|
22
|
+
// them apart from `passages: []` — which is the exact lie this contract
|
|
23
|
+
// exists to prevent. `semantic` and `degraded_reason` carry it instead.
|
|
24
|
+
//
|
|
25
|
+
// ── Return shape ───────────────────────────────────────────────────────────
|
|
26
|
+
// `passages[]` mirrors `search_passages` exactly — { id, title, score?,
|
|
27
|
+
// highlights?, passage, offset, content_length } — so a client that has learned
|
|
28
|
+
// one has learned the other. The one addition is `matched`, which earns its
|
|
29
|
+
// place because it tells the caller whether to trust the hit or re-query
|
|
30
|
+
// lexically: "words" means the hit is a lexical match the vector side did not
|
|
31
|
+
// find, so a caller looking for a paraphrase should not conclude absence.
|
|
32
|
+
//
|
|
33
|
+
// ── Lazy store open ────────────────────────────────────────────────────────
|
|
34
|
+
// The gateway launches this package's child at startup. Opening a ~1 GB SQLite
|
|
35
|
+
// file and loading the vec0 extension at import time could exceed its child
|
|
36
|
+
// startup tolerance, so nothing here opens the store until the first tool call,
|
|
37
|
+
// and the open is memoised. That is a named risk in the plan, not tidiness.
|
|
38
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
39
|
+
if (k2 === undefined) k2 = k;
|
|
40
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
41
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
42
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
43
|
+
}
|
|
44
|
+
Object.defineProperty(o, k2, desc);
|
|
45
|
+
}) : (function(o, m, k, k2) {
|
|
46
|
+
if (k2 === undefined) k2 = k;
|
|
47
|
+
o[k2] = m[k];
|
|
48
|
+
}));
|
|
49
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
50
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
51
|
+
}) : function(o, v) {
|
|
52
|
+
o["default"] = v;
|
|
53
|
+
});
|
|
54
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
55
|
+
var ownKeys = function(o) {
|
|
56
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
57
|
+
var ar = [];
|
|
58
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
59
|
+
return ar;
|
|
60
|
+
};
|
|
61
|
+
return ownKeys(o);
|
|
62
|
+
};
|
|
63
|
+
return function (mod) {
|
|
64
|
+
if (mod && mod.__esModule) return mod;
|
|
65
|
+
var result = {};
|
|
66
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
67
|
+
__setModuleDefault(result, mod);
|
|
68
|
+
return result;
|
|
69
|
+
};
|
|
70
|
+
})();
|
|
71
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
72
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
73
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
74
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
75
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
76
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
77
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
78
|
+
});
|
|
79
|
+
};
|
|
80
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
81
|
+
var t = {};
|
|
82
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
83
|
+
t[p] = s[p];
|
|
84
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
85
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
86
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
87
|
+
t[p[i]] = s[p[i]];
|
|
88
|
+
}
|
|
89
|
+
return t;
|
|
90
|
+
};
|
|
91
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
92
|
+
exports.METADATA_BATCH = exports.SearchContext = exports.PASSAGE_OVERHEAD_CHARS = exports.MAX_RESULT_CHARS = exports.MAX_LIMIT = exports.DEFAULT_LIMIT = exports.DEFAULT_WINDOW = exports.STRONG_MATCH = void 0;
|
|
93
|
+
exports.budgetedLimit = budgetedLimit;
|
|
94
|
+
exports.buildPassage = buildPassage;
|
|
95
|
+
exports.queryTerms = queryTerms;
|
|
96
|
+
exports.runSemanticSearch = runSemanticSearch;
|
|
97
|
+
exports.registerSearchTools = registerSearchTools;
|
|
98
|
+
exports.buildDocumentFilter = buildDocumentFilter;
|
|
99
|
+
exports.loadVisibleDocuments = loadVisibleDocuments;
|
|
100
|
+
exports.loadDocumentMetadata = loadDocumentMetadata;
|
|
101
|
+
exports.reconstructDocumentText = reconstructDocumentText;
|
|
102
|
+
exports.indexTextLoader = indexTextLoader;
|
|
103
|
+
exports.fitResultToBudget = fitResultToBudget;
|
|
104
|
+
exports.embeddingStatus = embeddingStatus;
|
|
105
|
+
const zod_1 = require("zod");
|
|
106
|
+
const config_1 = require("../search/config");
|
|
107
|
+
const embedder_1 = require("../search/embedder");
|
|
108
|
+
const store_1 = require("../search/store");
|
|
109
|
+
const chunk_1 = require("../search/chunk");
|
|
110
|
+
const rank_1 = require("../search/rank");
|
|
111
|
+
const sync_1 = require("../search/sync");
|
|
112
|
+
const webhook_1 = require("../search/webhook");
|
|
113
|
+
const middlewares_1 = require("./utils/middlewares");
|
|
114
|
+
/** A similarity at or above this is a strong match. Similarity, never distance.
|
|
115
|
+
*
|
|
116
|
+
* ⚠️ NOT USED AS A GATE ANY MORE, and the measurement is why. On this corpus
|
|
117
|
+
* the floor cannot separate a good query from a bad one. Measured on the live
|
|
118
|
+
* index (2,476 docs), best `1 - distance` per query:
|
|
119
|
+
*
|
|
120
|
+
* relevant "kuji nine invisibility…" 0.700 "naga offering…" 0.746
|
|
121
|
+
* "barber pole worm in goats…" 0.761 "channels and winds" 0.766
|
|
122
|
+
* "three humours and prana disorders" 0.731
|
|
123
|
+
* irrelevant "xyzzy plugh frobnicate quux" 0.599 "stock market options" 0.595
|
|
124
|
+
* "quantum chromodynamics…" 0.656 "best pizza dough…" 0.766
|
|
125
|
+
* "how to change a flat tire" 0.730
|
|
126
|
+
*
|
|
127
|
+
* The ranges OVERLAP: two of five irrelevant queries score above the worst
|
|
128
|
+
* relevant one, and no threshold in 0.500–0.800 separates the sets. A floor at
|
|
129
|
+
* 0.45 sits 0.145 BELOW the worst irrelevant query, so it could never fire —
|
|
130
|
+
* `strong_match` was constant `true` whenever the vector arm ran.
|
|
131
|
+
*
|
|
132
|
+
* Kept exported only so existing importers compile; treat any use of it as a
|
|
133
|
+
* claim that needs this card's measurement re-run on the corpus in question.
|
|
134
|
+
*/
|
|
135
|
+
exports.STRONG_MATCH = 0.45;
|
|
136
|
+
/** Chars per returned passage, matching search_passages' default. */
|
|
137
|
+
exports.DEFAULT_WINDOW = 800;
|
|
138
|
+
exports.DEFAULT_LIMIT = 10;
|
|
139
|
+
exports.MAX_LIMIT = 50;
|
|
140
|
+
/**
|
|
141
|
+
* The largest JSON body a caller can be handed, in characters.
|
|
142
|
+
*
|
|
143
|
+
* This is not our budget to choose freely: it is the ceiling the CONSUMER
|
|
144
|
+
* applies. Mneme truncates any non-memory tool result at 20,000 characters by
|
|
145
|
+
* taking head and tail, and the agent on the other side cannot detect that it
|
|
146
|
+
* happened -- a passage that was silently cut reads exactly like a short one.
|
|
147
|
+
*
|
|
148
|
+
* `MAX_LIMIT * max window` is 208,485 characters, ten times this number, so
|
|
149
|
+
* until this constant existed the tool's own advertised argument range could
|
|
150
|
+
* produce a result the client would mangle. Bounding it in the schema
|
|
151
|
+
* description would be advice; the enforcement is `budgetedLimit` (so a
|
|
152
|
+
* maximal call does not waste work) plus `fitResultToBudget` (so the returned
|
|
153
|
+
* bytes are measured, not predicted).
|
|
154
|
+
*/
|
|
155
|
+
exports.MAX_RESULT_CHARS = 20000;
|
|
156
|
+
/**
|
|
157
|
+
* JSON scaffolding a single passage costs, roughly: id, title, score, matched,
|
|
158
|
+
* offset, content_length, offset_space, quotes and commas.
|
|
159
|
+
*
|
|
160
|
+
* Used only to pick the passage COUNT before any work is done. `PASSAGE_OVERHEAD_CHARS` is deliberately
|
|
161
|
+
* generous: over-estimating costs a passage, under-estimating costs a wasted
|
|
162
|
+
* build that `fitResultToBudget` then discards anyway.
|
|
163
|
+
*/
|
|
164
|
+
exports.PASSAGE_OVERHEAD_CHARS = 200;
|
|
165
|
+
/**
|
|
166
|
+
* Fraction of MAX_RESULT_CHARS that `budgetedLimit` will spend on passages,
|
|
167
|
+
* leaving the rest for the envelope (query, counts, retriever list, reasons).
|
|
168
|
+
*/
|
|
169
|
+
const RESULT_BUDGET_FRACTION = 0.9;
|
|
170
|
+
/**
|
|
171
|
+
* How many passages of `window` characters can fit inside the consumer's
|
|
172
|
+
* ceiling. Never zero: a call that cannot afford even one passage should still
|
|
173
|
+
* return one and let `fitResultToBudget` trim if the bytes really do not fit.
|
|
174
|
+
*/
|
|
175
|
+
function budgetedLimit(limit, window) {
|
|
176
|
+
const ceiling = Math.floor((exports.MAX_RESULT_CHARS * RESULT_BUDGET_FRACTION) / (window + exports.PASSAGE_OVERHEAD_CHARS));
|
|
177
|
+
return Math.max(1, Math.min(limit, exports.MAX_LIMIT, ceiling));
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The filter arguments, mirroring `search_passages`.
|
|
181
|
+
*
|
|
182
|
+
* Kept in one object so the two tools cannot drift: if search_passages gains a
|
|
183
|
+
* filter, this is the one place that has to learn it.
|
|
184
|
+
*
|
|
185
|
+
* ── WHERE THEY ARE APPLIED, AND WHAT THAT COSTS ────────────────────────────
|
|
186
|
+
* They are evaluated in JS against each hit's fetched document, NOT pushed into
|
|
187
|
+
* the kNN statement. That is a deliberate trade with a known cost, stated here
|
|
188
|
+
* because the previous comment argued against this approach while describing
|
|
189
|
+
* it as the approach:
|
|
190
|
+
*
|
|
191
|
+
* PRO The store's kNN returns one hit per document, and the documents are
|
|
192
|
+
* fetched anyway to build passages, so the filter is free at the point
|
|
193
|
+
* it is needed and needs no new SQL.
|
|
194
|
+
* CON `rowid IN (…)` is NOT composed into the MATCH statement, so the
|
|
195
|
+
* filtered set can come back SHORT of `limit` with no signal that
|
|
196
|
+
* ranking was trimmed. This is why `filtered_out` and its reason
|
|
197
|
+
* breakdown exist: the shortfall is reported rather than hidden.
|
|
198
|
+
*
|
|
199
|
+
* Pushing the filter into the kNN would remove the CON, at the cost of a
|
|
200
|
+
* per-query SQL shape and of needing each filter to be expressible as a set of
|
|
201
|
+
* document ids before the query runs. That is the right eventual design; it is
|
|
202
|
+
* not this change.
|
|
203
|
+
*
|
|
204
|
+
* ⚠️ EVERY KEY HERE MUST BE HONOURED BY `buildDocumentFilter`. A key that is
|
|
205
|
+
* advertised to the model and silently ignored is worse than an absent one: the
|
|
206
|
+
* caller gets unfiltered results with no error, no warning, and no way to
|
|
207
|
+
* notice. `storage_path` was exactly that until it was implemented.
|
|
208
|
+
*/
|
|
209
|
+
const FILTER_SHAPE = {
|
|
210
|
+
correspondent: zod_1.z.number().optional(),
|
|
211
|
+
document_type: zod_1.z.number().optional(),
|
|
212
|
+
tag: zod_1.z.number().optional(),
|
|
213
|
+
storage_path: zod_1.z.number().optional(),
|
|
214
|
+
created__date__gte: zod_1.z.string().optional(),
|
|
215
|
+
created__date__lte: zod_1.z.string().optional(),
|
|
216
|
+
};
|
|
217
|
+
/**
|
|
218
|
+
* Cut a bounded passage around the most DISTINCTIVE place the query terms appear.
|
|
219
|
+
*
|
|
220
|
+
* Mirrors buildSearchPassage's anchoring priority minus the server highlight,
|
|
221
|
+
* which does not exist on this path: the vector side has no highlighted span,
|
|
222
|
+
* and the lexical side's BM25 match gives a chunk rather than an offset. So the
|
|
223
|
+
* anchor is (1) the RAREST matching query term, (2) the head of the document.
|
|
224
|
+
*
|
|
225
|
+
* Rarity, not query order and not length. Query order was the original defect:
|
|
226
|
+
* English queries open with articles, so first-match anchored "the practice of
|
|
227
|
+
* a wrathful deity with a lion head" on "the" at character 3, and containment
|
|
228
|
+
* still passed because "the" really was in the window. Length was my first
|
|
229
|
+
* correction and it is also wrong -- measured live, it picks `doctor` over
|
|
230
|
+
* `ankle` by one character. Rarity picks the term the query is ABOUT.
|
|
231
|
+
* When the caller cannot supply counts, length is the fallback.
|
|
232
|
+
*
|
|
233
|
+
* A window that does not contain the hit is worse than no window, which is why
|
|
234
|
+
* the term search runs against the whole document rather than the chunk.
|
|
235
|
+
*/
|
|
236
|
+
function buildPassage(doc, terms, window,
|
|
237
|
+
/**
|
|
238
|
+
* How many indexed chunks contain each term, when the caller can supply it.
|
|
239
|
+
*
|
|
240
|
+
* ⚠️ THIS IS THE CORRECT DISCRIMINATOR AND LENGTH IS NOT. Measured on the
|
|
241
|
+
* live corpus: "what did the doctor say about the ankle" picked `doctor`
|
|
242
|
+
* (1,903 chunks) over `ankle` (808) on length alone -- wrong, and wrong in
|
|
243
|
+
* the way that matters, because the window then lands on a discussion of
|
|
244
|
+
* doctors rather than of the ankle. "the practice of a wrathful deity with a
|
|
245
|
+
* lion head" picked `practice` (36,459) over `wrathful` (4,657). Rarity
|
|
246
|
+
* matched a human read in 3 of 3 probes; length in 0 of 3.
|
|
247
|
+
*
|
|
248
|
+
* Optional so a caller with no index (the search_passages path, whose terms
|
|
249
|
+
* come from the server's own query) still works -- it then falls back to
|
|
250
|
+
* length, which is worse but still better than query order.
|
|
251
|
+
*/
|
|
252
|
+
termRarity) {
|
|
253
|
+
var _a;
|
|
254
|
+
const full = (_a = doc.content) !== null && _a !== void 0 ? _a : "";
|
|
255
|
+
const lower = full.toLowerCase();
|
|
256
|
+
let center = -1;
|
|
257
|
+
let found;
|
|
258
|
+
// ANCHOR ON THE MOST SPECIFIC TERM, not the first one in query order.
|
|
259
|
+
//
|
|
260
|
+
// Measured defect this fixes: the query "the practice of a wrathful deity with
|
|
261
|
+
// a lion head" anchored on "the" at character 3, because `indexOf` walks the
|
|
262
|
+
// query left-to-right and English queries open with articles. 11 of 18 windows
|
|
263
|
+
// landed at the document head. Containment still passed -- "the" really is in
|
|
264
|
+
// the window -- which is exactly why no test caught it: the property that
|
|
265
|
+
// holds is the weak one, and the one that matters (a window around the
|
|
266
|
+
// DISTINCTIVE term) was never asserted.
|
|
267
|
+
//
|
|
268
|
+
// Length is the cheap proxy for specificity here, and it is the right one:
|
|
269
|
+
// a longer term is what distinguishes this query from most others. Ties keep
|
|
270
|
+
// query order via the stable sort. If nothing matches, offset 0 and say so --
|
|
271
|
+
// a head window is honest when the query genuinely appears nowhere.
|
|
272
|
+
// Rarity first (rarest in the corpus = most specific to this query), then
|
|
273
|
+
// length as a tie-break, then query order via the stable sort. A term the
|
|
274
|
+
// caller could not score sorts on length alone.
|
|
275
|
+
const rarityOf = (t) => { var _a; return (_a = termRarity === null || termRarity === void 0 ? void 0 : termRarity.get(t.toLowerCase())) !== null && _a !== void 0 ? _a : Number.MAX_SAFE_INTEGER; };
|
|
276
|
+
const candidates = [...terms].sort((a, b) => {
|
|
277
|
+
const ra = rarityOf(a);
|
|
278
|
+
const rb = rarityOf(b);
|
|
279
|
+
if (ra !== rb)
|
|
280
|
+
return ra - rb;
|
|
281
|
+
return b.length - a.length;
|
|
282
|
+
});
|
|
283
|
+
for (const term of candidates) {
|
|
284
|
+
const at = lower.indexOf(term.toLowerCase());
|
|
285
|
+
if (at >= 0) {
|
|
286
|
+
center = at;
|
|
287
|
+
found = term;
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const resolved = center < 0 ? 0 : center;
|
|
292
|
+
const start = Math.max(0, resolved - Math.floor(window / 2));
|
|
293
|
+
// The end is widened only when doing so is required to CONTAIN the term it
|
|
294
|
+
// is anchoring on. A window that stops one character short of the match it
|
|
295
|
+
// was centred for is the exact defect the containment rule forbids, and with
|
|
296
|
+
// a term longer than `window` the plain `start + window` would do it.
|
|
297
|
+
const end = Math.max(start + window, resolved + (found ? found.length : 0));
|
|
298
|
+
return Object.assign({ passage: full.slice(start, end), offset: start, content_length: full.length }, (found ? { term: found } : {}));
|
|
299
|
+
}
|
|
300
|
+
/** Query terms for passage anchoring, from the same tokenisation the FTS side uses. */
|
|
301
|
+
function queryTerms(query) {
|
|
302
|
+
return query
|
|
303
|
+
.toLowerCase()
|
|
304
|
+
// \p{M} is required for the same reason it is in store.toFtsMatch: the
|
|
305
|
+
// anchoring terms and the lexical terms must tokenise identically, or a
|
|
306
|
+
// passage is anchored on one fragment of a syllable and matched on another.
|
|
307
|
+
.split(/[^\p{L}\p{N}\p{M}_]+/u)
|
|
308
|
+
.filter((term) => term.length > 0);
|
|
309
|
+
}
|
|
310
|
+
function errorMessage(err) {
|
|
311
|
+
return err instanceof Error ? err.message : String(err);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Everything the three tools share, built once and lazily.
|
|
315
|
+
*
|
|
316
|
+
* `store` is `undefined` until the first call. `openFailed` remembers a failure
|
|
317
|
+
* so a broken index does not retry a 1 GB open on every single tool call -- and
|
|
318
|
+
* remembers the MESSAGE, so embedding_status can report why.
|
|
319
|
+
*/
|
|
320
|
+
class SearchContext {
|
|
321
|
+
/**
|
|
322
|
+
* `embedderFactory` exists so "no embedder" can be stated rather than
|
|
323
|
+
* inferred. A context whose embedder is merely unset would reach
|
|
324
|
+
* `embedderOrUndefined` and CONSTRUCT a real client — which then makes a
|
|
325
|
+
* network call. That is correct in production and unacceptable in a test, and
|
|
326
|
+
* the difference has to be expressible, so the factory can return undefined
|
|
327
|
+
* and mean it.
|
|
328
|
+
*/
|
|
329
|
+
constructor(config, embedderFactory) {
|
|
330
|
+
this.syncStarted = false;
|
|
331
|
+
this.webhookStarted = false;
|
|
332
|
+
this.configOverride = config;
|
|
333
|
+
this.embedderFactory = embedderFactory;
|
|
334
|
+
}
|
|
335
|
+
/** State the embedder explicitly. `null` means "there is deliberately none". */
|
|
336
|
+
setEmbedder(embedder) {
|
|
337
|
+
this.embedder = embedder;
|
|
338
|
+
this.sync = undefined;
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Resolve config from the environment at CALL time.
|
|
342
|
+
*
|
|
343
|
+
* Not at import: the gateway sets the child's environment before launch, but
|
|
344
|
+
* tests and the CLI set it after import, and reading once at module load
|
|
345
|
+
* makes the config untestable.
|
|
346
|
+
*/
|
|
347
|
+
config() {
|
|
348
|
+
var _a;
|
|
349
|
+
return (_a = this.configOverride) !== null && _a !== void 0 ? _a : (0, config_1.searchConfig)(process.env);
|
|
350
|
+
}
|
|
351
|
+
/** True when the search tools should register at all. */
|
|
352
|
+
get isConfigured() {
|
|
353
|
+
return this.config() !== undefined;
|
|
354
|
+
}
|
|
355
|
+
/** Open the index, memoised. Throws the underlying error on first failure. */
|
|
356
|
+
index() {
|
|
357
|
+
if (this.store)
|
|
358
|
+
return this.store;
|
|
359
|
+
if (this.openFailed) {
|
|
360
|
+
throw new Error(`The search index could not be opened: ${this.openFailed}`);
|
|
361
|
+
}
|
|
362
|
+
const config = this.config();
|
|
363
|
+
if (!config) {
|
|
364
|
+
throw new Error("Semantic search is not configured. Set PAPERLESS_MCP_EMBEDDING_URL " +
|
|
365
|
+
"and the other PAPERLESS_MCP_* search variables to enable it.");
|
|
366
|
+
}
|
|
367
|
+
// `expected` is what makes the deployment policy-adjacent schema guard REAL at query
|
|
368
|
+
// time. Without it, an index built with `e5` and queried with `nomic`
|
|
369
|
+
// returned `semantic: "ok"` and quietly worse results, while
|
|
370
|
+
// `assertPrefixModeMatches`'s own KDoc said the mismatch "must be refused
|
|
371
|
+
// loudly". Same for a same-width model swap and a bumped schema_version.
|
|
372
|
+
const store = new store_1.SearchIndex((0, config_1.indexPath)(config), {
|
|
373
|
+
expected: {
|
|
374
|
+
dimensions: store_1.EMBEDDING_DIMENSIONS,
|
|
375
|
+
model: config.model,
|
|
376
|
+
prefixMode: config.prefixMode,
|
|
377
|
+
},
|
|
378
|
+
});
|
|
379
|
+
try {
|
|
380
|
+
store.open();
|
|
381
|
+
}
|
|
382
|
+
catch (err) {
|
|
383
|
+
this.openFailed = errorMessage(err);
|
|
384
|
+
throw err;
|
|
385
|
+
}
|
|
386
|
+
this.store = store;
|
|
387
|
+
return store;
|
|
388
|
+
}
|
|
389
|
+
embedderOrUndefined() {
|
|
390
|
+
var _a;
|
|
391
|
+
// `null` is memoised and means "deliberately none" — checked explicitly so
|
|
392
|
+
// it never falls through to construction.
|
|
393
|
+
if (this.embedder === null)
|
|
394
|
+
return undefined;
|
|
395
|
+
if (this.embedder)
|
|
396
|
+
return this.embedder;
|
|
397
|
+
const config = this.config();
|
|
398
|
+
if (!config)
|
|
399
|
+
return undefined;
|
|
400
|
+
try {
|
|
401
|
+
const built = this.embedderFactory
|
|
402
|
+
? this.embedderFactory(config)
|
|
403
|
+
: new embedder_1.OllamaEmbedder(Object.assign({ url: config.embeddingUrl, model: config.model, prefixMode: config.prefixMode,
|
|
404
|
+
// Threaded rather than defaulted here: the value is read and
|
|
405
|
+
// range-checked in config.ts, and a knob that is validated but
|
|
406
|
+
// never passed on is an advertised setting with no effect.
|
|
407
|
+
batchSize: config.syncBatchSize, localOnly: config.embeddingLocalOnly }, (config.embeddingApiKey ? { apiKey: config.embeddingApiKey } : {})));
|
|
408
|
+
this.embedder = built !== null && built !== void 0 ? built : null;
|
|
409
|
+
}
|
|
410
|
+
catch (_b) {
|
|
411
|
+
// A misconfigured endpoint is not an unavailable embedder, but on this
|
|
412
|
+
// path both mean "no vectors this turn", and the lexical half still works.
|
|
413
|
+
this.embedder = null;
|
|
414
|
+
return undefined;
|
|
415
|
+
}
|
|
416
|
+
return (_a = this.embedder) !== null && _a !== void 0 ? _a : undefined;
|
|
417
|
+
}
|
|
418
|
+
syncRunner(api) {
|
|
419
|
+
if (this.sync)
|
|
420
|
+
return this.sync;
|
|
421
|
+
const embedder = this.embedderOrUndefined();
|
|
422
|
+
if (!embedder) {
|
|
423
|
+
throw new Error("Semantic search cannot sync without an embedder. Check " +
|
|
424
|
+
"PAPERLESS_MCP_EMBEDDING_URL.");
|
|
425
|
+
}
|
|
426
|
+
const config = this.config();
|
|
427
|
+
this.sync = new sync_1.IndexSync(Object.assign({ api, store: this.index(), embedder }, (config ? { maxChunks: config.maxChunks } : {})));
|
|
428
|
+
return this.sync;
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Start the background sync timer when the deployment asked for one.
|
|
432
|
+
*
|
|
433
|
+
* ⚠️ THIS METHOD IS THE CALLER `IndexSync.start()` NEVER HAD. The interval was
|
|
434
|
+
* read and range-checked in config.ts from the day it was added, and nothing
|
|
435
|
+
* ever acted on it: `start()` existed, was documented, and was invoked from
|
|
436
|
+
* nowhere. A validated setting with no consumer reads as a working feature
|
|
437
|
+
* and is not one.
|
|
438
|
+
*
|
|
439
|
+
* Called from `registerSearchTools`, so the wiring is exercised by every
|
|
440
|
+
* server that configures search at all. A test that constructs the timer
|
|
441
|
+
* directly proves `start()` works and says nothing about whether anything
|
|
442
|
+
* calls it; the caller is what has to be asserted.
|
|
443
|
+
*
|
|
444
|
+
* A background sync that cannot start must not take down tool registration:
|
|
445
|
+
* the observable `sync_embeddings` tool still works, so failure here is
|
|
446
|
+
* logged and swallowed rather than thrown.
|
|
447
|
+
*/
|
|
448
|
+
startBackgroundSync(api) {
|
|
449
|
+
if (this.syncStarted)
|
|
450
|
+
return;
|
|
451
|
+
const config = this.config();
|
|
452
|
+
if (!config || config.syncIntervalMs <= 0)
|
|
453
|
+
return;
|
|
454
|
+
try {
|
|
455
|
+
this.syncRunner(api).start(config.syncIntervalMs);
|
|
456
|
+
this.syncStarted = true;
|
|
457
|
+
}
|
|
458
|
+
catch (err) {
|
|
459
|
+
console.error(`[paperless-mcp] background sync not started: ${errorMessage(err)}`);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/** True when a background sync timer is live. Used by the wiring test. */
|
|
463
|
+
get isBackgroundSyncActive() {
|
|
464
|
+
var _a;
|
|
465
|
+
return ((_a = this.sync) === null || _a === void 0 ? void 0 : _a.isScheduled) === true;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Start the change-notification receiver when the deployment asked for one.
|
|
469
|
+
*
|
|
470
|
+
* ⚠️ THE SAME CALLER PROBLEM AS `startBackgroundSync`, ONE LAYER OUT. A
|
|
471
|
+
* receiver class nobody constructs is a feature nobody has, and the interval
|
|
472
|
+
* in this file is the standing proof: it was read, validated and never acted
|
|
473
|
+
* on. So construction happens here, on the path `registerSearchTools` always
|
|
474
|
+
* walks, rather than behind a bootstrap only a test would call.
|
|
475
|
+
*
|
|
476
|
+
* Started asynchronously and never awaited. Registration is synchronous and
|
|
477
|
+
* happens before the transport connects; a bind that fails must not delay or
|
|
478
|
+
* break the tool surface, because the observable `sync_embeddings` tool still
|
|
479
|
+
* works and the watermark still converges on the next start without it.
|
|
480
|
+
*/
|
|
481
|
+
startWebhook(api) {
|
|
482
|
+
var _a;
|
|
483
|
+
if (this.webhookStarted)
|
|
484
|
+
return;
|
|
485
|
+
const config = this.config();
|
|
486
|
+
if (!config || config.webhookPort === 0)
|
|
487
|
+
return;
|
|
488
|
+
const refusal = (0, config_1.webhookRefusal)(config);
|
|
489
|
+
if (refusal) {
|
|
490
|
+
console.error(`[paperless-mcp] webhook not started: ${refusal}`);
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
// The runner is resolved lazily, per event: building it needs an embedder
|
|
494
|
+
// and an open store, both of which are expensive and neither of which
|
|
495
|
+
// should be paid at startup by an install that never receives an event.
|
|
496
|
+
const receiver = new webhook_1.WebhookReceiver({
|
|
497
|
+
secret: (_a = config.webhookSecret) !== null && _a !== void 0 ? _a : "",
|
|
498
|
+
port: config.webhookPort,
|
|
499
|
+
host: config.webhookHost,
|
|
500
|
+
debounceMs: config.webhookDebounceMs,
|
|
501
|
+
runner: () => this.syncRunner(api),
|
|
502
|
+
});
|
|
503
|
+
this.webhook = receiver;
|
|
504
|
+
this.webhookStarted = true;
|
|
505
|
+
void receiver.start().catch((err) => {
|
|
506
|
+
// A port already in use, or a refused bind. Logged and dropped: the
|
|
507
|
+
// receiver is optional infrastructure and must not kill the server.
|
|
508
|
+
console.error(`[paperless-mcp] webhook not started: ${errorMessage(err)}`);
|
|
509
|
+
this.webhook = undefined;
|
|
510
|
+
this.webhookStarted = false;
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
/** True when a receiver is bound. Used by the wiring test. */
|
|
514
|
+
get isWebhookListening() {
|
|
515
|
+
var _a;
|
|
516
|
+
return ((_a = this.webhook) === null || _a === void 0 ? void 0 : _a.isListening) === true;
|
|
517
|
+
}
|
|
518
|
+
/** The port the receiver actually bound, or undefined when it is off. */
|
|
519
|
+
get webhookPort() {
|
|
520
|
+
var _a;
|
|
521
|
+
return (_a = this.webhook) === null || _a === void 0 ? void 0 : _a.boundPort;
|
|
522
|
+
}
|
|
523
|
+
close() {
|
|
524
|
+
var _a, _b;
|
|
525
|
+
// Stop the timer and the receiver BEFORE dropping the store, or a stdio
|
|
526
|
+
// child shutting down leaves a scheduled run firing against a closed index.
|
|
527
|
+
//
|
|
528
|
+
// ⚠️ The receiver's close is ASYNC and `close()` is not, so it is started
|
|
529
|
+
// here and not awaited. That is safe rather than sloppy: the receiver's own
|
|
530
|
+
// `close()` clears its debounce timer synchronously first, so no new run can
|
|
531
|
+
// be scheduled from this point on -- only an already-firing one can still be
|
|
532
|
+
// in flight, and that one is bounded by the store's own `isOpen` state.
|
|
533
|
+
(_a = this.sync) === null || _a === void 0 ? void 0 : _a.stop();
|
|
534
|
+
if (this.webhook) {
|
|
535
|
+
void this.webhook.close().catch(() => {
|
|
536
|
+
// Nothing useful to do while shutting down.
|
|
537
|
+
});
|
|
538
|
+
this.webhook = undefined;
|
|
539
|
+
}
|
|
540
|
+
this.webhookStarted = false;
|
|
541
|
+
(_b = this.store) === null || _b === void 0 ? void 0 : _b.close();
|
|
542
|
+
this.store = undefined;
|
|
543
|
+
this.sync = undefined;
|
|
544
|
+
this.syncStarted = false;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
exports.SearchContext = SearchContext;
|
|
548
|
+
/**
|
|
549
|
+
* Fuse the two lists, tolerating an absent lexical half or an absent embedder.
|
|
550
|
+
*
|
|
551
|
+
* This is the function the contract is built on, and it is deliberately one
|
|
552
|
+
* function: the degradation lives in one place so no caller can accidentally
|
|
553
|
+
* reintroduce a silent empty list.
|
|
554
|
+
*/
|
|
555
|
+
function runSemanticSearch(context, query, opts) {
|
|
556
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
557
|
+
const store = context.index();
|
|
558
|
+
// ⚠️ BUDGETED, not merely clamped. `MAX_LIMIT` alone permits
|
|
559
|
+
// `50 * 4000 = 208,485` characters, which is 10x the ceiling the consuming
|
|
560
|
+
// client applies and silently mangles. This is the first of the two
|
|
561
|
+
// enforcements: it stops the work from being done. `fitResultToBudget` is
|
|
562
|
+
// the second and it measures real bytes.
|
|
563
|
+
const limit = budgetedLimit(Math.min(Math.max(opts.limit, 1), exports.MAX_LIMIT), opts.window);
|
|
564
|
+
const terms = queryTerms(query);
|
|
565
|
+
// ── THE STALENESS REPORT, BUILT BEFORE THE SEARCH SO ONE SPREAD SENDS IT ──
|
|
566
|
+
//
|
|
567
|
+
// Computed once and spread into BOTH return paths (the empty-fused early
|
|
568
|
+
// return and the main one). Computing it twice would be two probes for one
|
|
569
|
+
// answer, and computing it only on the main path would silently omit it from
|
|
570
|
+
// exactly the response where a caller is most likely to wonder whether the
|
|
571
|
+
// index is current — an empty result.
|
|
572
|
+
//
|
|
573
|
+
// ⚠️ A FAILED PROBE YIELDS `stale: null`, NEVER `false`. "I could not check"
|
|
574
|
+
// and "I checked and it is current" are different facts, and a search that
|
|
575
|
+
// fabricates the second because Paperless was down would be lying about the
|
|
576
|
+
// one thing this field exists to report. The search itself is unaffected: the
|
|
577
|
+
// degradation contract already covers a dead embedder, and a dead Paperless
|
|
578
|
+
// costs only this annotation.
|
|
579
|
+
const stalenessFields = opts.checkStaleness
|
|
580
|
+
? yield readStaleness(opts.corpusState, store, opts.now)
|
|
581
|
+
: {};
|
|
582
|
+
let semantic = "ok";
|
|
583
|
+
const reasons = [];
|
|
584
|
+
let vector = [];
|
|
585
|
+
let text = [];
|
|
586
|
+
// ⚠️ THE THREE STATES, kept distinct, because collapsing them produced a FALSE
|
|
587
|
+
// STATEMENT OF MECHANISM. The old guard fired when BOTH lists came back empty
|
|
588
|
+
// and the embedder was down, and announced "no lexical fallback exists" --
|
|
589
|
+
// which was untrue whenever FTS5 was present and simply had no match. A
|
|
590
|
+
// caller then could not tell "nothing in the corpus says zzzznothing" from
|
|
591
|
+
// "search is broken", which is the exact distinction the degradation contract
|
|
592
|
+
// exists to preserve.
|
|
593
|
+
//
|
|
594
|
+
// (a) BOTH RAN, NOTHING MATCHED -> semantic ok|unavailable, empty list,
|
|
595
|
+
// and a field that says so. ALLOWED.
|
|
596
|
+
// (b) LEXICAL RAN, EMBEDDER DOWN -> results + unavailable + the reason.
|
|
597
|
+
// Allowed; already covered above.
|
|
598
|
+
// (c) NEITHER IS CAPABLE -> refuse with an error. There is
|
|
599
|
+
// genuinely nothing to search with, and
|
|
600
|
+
// an empty list would read as "no such
|
|
601
|
+
// document".
|
|
602
|
+
//
|
|
603
|
+
// The predicate for (c) is CAPABILITY, not emptiness: `store.lexicalAvailable`
|
|
604
|
+
// is a property of the build, and the embedder's absence is a property of the
|
|
605
|
+
// configuration. Neither can be inferred from a list length.
|
|
606
|
+
// The lexical half needs no network and no model. It runs whenever the build
|
|
607
|
+
// supports it, and it is the reason a degraded call still returns something.
|
|
608
|
+
let lexicalCapable = store.lexicalAvailable;
|
|
609
|
+
if (lexicalCapable) {
|
|
610
|
+
try {
|
|
611
|
+
// Over-fetch because filters are applied below: fetching exactly `limit`
|
|
612
|
+
// would let one filtered-out document empty the result.
|
|
613
|
+
text = store.textHits(query, limit * 4);
|
|
614
|
+
}
|
|
615
|
+
catch (err) {
|
|
616
|
+
if (!(err instanceof store_1.LexicalUnavailableError))
|
|
617
|
+
throw err;
|
|
618
|
+
// The build CLAIMED FTS5 and the query path disagreed. Treat the
|
|
619
|
+
// capability as false rather than swallowing it: this is the one place
|
|
620
|
+
// the property could be wrong, and state (c) must not be missed because
|
|
621
|
+
// of it.
|
|
622
|
+
lexicalCapable = false;
|
|
623
|
+
reasons.push(`Lexical search failed despite FTS5 being present: ${errorMessage(err)}`);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
else {
|
|
627
|
+
reasons.push("Lexical search is unavailable in this build: SQLite has no FTS5 module.");
|
|
628
|
+
}
|
|
629
|
+
const embedder = context.embedderOrUndefined();
|
|
630
|
+
let vectorCapable = false;
|
|
631
|
+
if (!embedder) {
|
|
632
|
+
semantic = "unavailable";
|
|
633
|
+
reasons.push("The embedding client is not configured.");
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
try {
|
|
637
|
+
const embedding = yield embedder.embedQuery(query);
|
|
638
|
+
vector = store.vectorHits(embedding, limit * 4);
|
|
639
|
+
vectorCapable = true;
|
|
640
|
+
}
|
|
641
|
+
catch (err) {
|
|
642
|
+
if (!(err instanceof embedder_1.EmbeddingUnavailableError))
|
|
643
|
+
throw err;
|
|
644
|
+
// EXPECTED. The lexical list above already holds whatever could be found.
|
|
645
|
+
semantic = "unavailable";
|
|
646
|
+
reasons.push(errorMessage(err));
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
// State (c): neither retriever is capable. Refuse loudly.
|
|
650
|
+
if (!lexicalCapable && !vectorCapable) {
|
|
651
|
+
throw new Error(`Semantic search is fully unavailable: neither the vector side nor a ` +
|
|
652
|
+
`lexical fallback can run. ${reasons.join(" ")}`);
|
|
653
|
+
}
|
|
654
|
+
// One reason string, but FIRST-REASON-WINS is now explicit rather than the
|
|
655
|
+
// `??=` accident it was: `??=` let a lexical-unavailable reason mask a later
|
|
656
|
+
// embedder reason, so a call with no FTS5 AND a dead embedder reported only
|
|
657
|
+
// the lexical half and left the reader guessing about the vectors.
|
|
658
|
+
const degradedReason = reasons.length > 0 ? reasons.join(" ") : undefined;
|
|
659
|
+
const fused = (0, rank_1.fuse)(vector, text);
|
|
660
|
+
// Which retrievers actually RAN. Reported rather than inferred: an empty
|
|
661
|
+
// result with both here is "both searched and neither matched", with one is
|
|
662
|
+
// "only that half could run", and the difference is the whole point.
|
|
663
|
+
const retrievers = [];
|
|
664
|
+
if (vectorCapable)
|
|
665
|
+
retrievers.push("vector");
|
|
666
|
+
if (lexicalCapable)
|
|
667
|
+
retrievers.push("lexical");
|
|
668
|
+
if (fused.length === 0) {
|
|
669
|
+
return Object.assign(Object.assign(Object.assign(Object.assign({ query,
|
|
670
|
+
semantic }, (degradedReason ? { degraded_reason: degradedReason } : {})), { strong_match: false, total_hits: 0, filtered_out: 0, retrievers }), stalenessFields), {
|
|
671
|
+
// State (a) on its own terms: nothing matched and the payload says so —
|
|
672
|
+
// `semantic`, `retrievers`, and an empty list together are unambiguous.
|
|
673
|
+
passages: [] });
|
|
674
|
+
}
|
|
675
|
+
// Anchoring and filters both need METADATA. They do NOT need content, and
|
|
676
|
+
// fetching it per hit was the defect: one `GET /documents/{id}/` per hit, up
|
|
677
|
+
// to 80 for a 10-passage call, each returning a full body that was sliced to
|
|
678
|
+
// a window and thrown away.
|
|
679
|
+
const documents = yield opts.loadDocuments(fused.map((hit) => hit.documentId));
|
|
680
|
+
// The text the window is cut from, from the index. `undefined` rather than an
|
|
681
|
+
// empty map when no loader was injected, so the fallback is explicit.
|
|
682
|
+
const indexText = opts.loadText ? opts.loadText(fused) : undefined;
|
|
683
|
+
const passages = [];
|
|
684
|
+
// ONE rarity lookup for the whole request, not one per document. The anchor
|
|
685
|
+
// needs it per document, but the counts are a property of the corpus and the
|
|
686
|
+
// query, not of the document being windowed.
|
|
687
|
+
const rarity = store.termChunkCounts(terms);
|
|
688
|
+
// Post-fetch accounting. Without it `total_hits: 12, passages: []` was a
|
|
689
|
+
// legal, unexplained response: the caller could not tell that its own filter
|
|
690
|
+
// arguments had removed everything.
|
|
691
|
+
let documentUnavailable = 0;
|
|
692
|
+
let metadataFiltered = 0;
|
|
693
|
+
for (const hit of fused) {
|
|
694
|
+
const doc = documents.get(hit.documentId);
|
|
695
|
+
if (!doc) {
|
|
696
|
+
documentUnavailable += 1;
|
|
697
|
+
continue;
|
|
698
|
+
}
|
|
699
|
+
if (opts.filters && !opts.filters(doc)) {
|
|
700
|
+
metadataFiltered += 1;
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
const indexed = indexText === null || indexText === void 0 ? void 0 : indexText.get(doc.id);
|
|
704
|
+
const fromIndex = indexed !== undefined;
|
|
705
|
+
const anchored = buildPassage({ id: doc.id, title: doc.title, content: fromIndex ? indexed : doc.content }, terms, opts.window, rarity);
|
|
706
|
+
passages.push(Object.assign(Object.assign({ id: doc.id, title: doc.title, score: hit.score, matched: hit.matched, passage: anchored.passage, offset: anchored.offset, content_length: anchored.content_length }, (anchored.term ? { highlights: anchored.term } : {})), { offset_space: fromIndex ? "normalized" : "document" }));
|
|
707
|
+
if (passages.length >= limit)
|
|
708
|
+
break;
|
|
709
|
+
}
|
|
710
|
+
// ⚠️ `strong_match` is now literally "a passage was returned". The 0.45
|
|
711
|
+
// similarity gate it used to consult could never fire on this corpus — see
|
|
712
|
+
// the STRONG_MATCH note — so the old computation reported `true` on every
|
|
713
|
+
// call that produced a hit, regardless of quality. A field that cannot be
|
|
714
|
+
// false is not evidence.
|
|
715
|
+
const strong = passages.length > 0;
|
|
716
|
+
// `filtered_out` counts what the loop dropped, NOT `total_hits - passages`
|
|
717
|
+
// -- those differ when the `limit` break fires with hits still unexamined.
|
|
718
|
+
// The honest decomposition is: fused = shown + dropped + never-reached.
|
|
719
|
+
const filteredOut = documentUnavailable + metadataFiltered;
|
|
720
|
+
return fitResultToBudget(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ query,
|
|
721
|
+
semantic }, (degradedReason ? { degraded_reason: degradedReason } : {})), { strong_match: strong, total_hits: fused.length, filtered_out: filteredOut }), (filteredOut > 0
|
|
722
|
+
? { filtered_out_reasons: { document_unavailable: documentUnavailable, metadata_filter: metadataFiltered } }
|
|
723
|
+
: {})), { retrievers }), stalenessFields), { passages }));
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* The staleness annotation, as a spreadable fragment.
|
|
728
|
+
*
|
|
729
|
+
* Returns the fields with `stale: null` when the corpus cannot be read or the
|
|
730
|
+
* index carries no watermark. Neither invents a value.
|
|
731
|
+
*
|
|
732
|
+
* ⚠️ AN INDEX WITH NO WATERMARK IS `stale: null`, NOT `stale: true`. It is
|
|
733
|
+
* genuinely unknown — the index predates the watermark, or has never completed
|
|
734
|
+
* a run — and asserting either answer would be a guess dressed as a measurement.
|
|
735
|
+
* The actionable thing for such a caller is `sync_embeddings`, and
|
|
736
|
+
* `staleness_reason` says exactly that.
|
|
737
|
+
*/
|
|
738
|
+
function readStaleness(corpusState, store, now) {
|
|
739
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
740
|
+
var _a, _b;
|
|
741
|
+
if (!corpusState) {
|
|
742
|
+
return {
|
|
743
|
+
stale: null,
|
|
744
|
+
index_age_s: null,
|
|
745
|
+
live_count: null,
|
|
746
|
+
indexed_count: null,
|
|
747
|
+
staleness_reason: "No corpus probe was available to this call.",
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
const live = yield corpusState();
|
|
751
|
+
const watermark = store.watermark();
|
|
752
|
+
const indexedCount = (_a = watermark === null || watermark === void 0 ? void 0 : watermark.count) !== null && _a !== void 0 ? _a : null;
|
|
753
|
+
if (!watermark) {
|
|
754
|
+
return {
|
|
755
|
+
stale: null,
|
|
756
|
+
index_age_s: null,
|
|
757
|
+
live_count: (_b = live === null || live === void 0 ? void 0 : live.count) !== null && _b !== void 0 ? _b : null,
|
|
758
|
+
indexed_count: null,
|
|
759
|
+
staleness_reason: "The index has no recorded watermark (it predates one, or has never " +
|
|
760
|
+
"completed a sync), so staleness cannot be determined. Run " +
|
|
761
|
+
"sync_embeddings.",
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
// Age is computable from the stored timestamp alone, so it is reported even
|
|
765
|
+
// when the live probe failed — it is a fact about the index, not about the
|
|
766
|
+
// corpus.
|
|
767
|
+
const syncedAt = new Date(watermark.at).getTime();
|
|
768
|
+
const indexAge = Number.isFinite(syncedAt)
|
|
769
|
+
? Math.max(0, Math.round(((now !== null && now !== void 0 ? now : Date.now)() - syncedAt) / 1000))
|
|
770
|
+
: null;
|
|
771
|
+
if (!live) {
|
|
772
|
+
return {
|
|
773
|
+
stale: null,
|
|
774
|
+
index_age_s: indexAge,
|
|
775
|
+
live_count: null,
|
|
776
|
+
indexed_count: indexedCount,
|
|
777
|
+
staleness_reason: "The live corpus could not be read, so staleness cannot be " +
|
|
778
|
+
"determined. The search results above are unaffected.",
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
return {
|
|
782
|
+
stale: (0, sync_1.corpusChanged)(live, watermark),
|
|
783
|
+
index_age_s: indexAge,
|
|
784
|
+
live_count: live.count,
|
|
785
|
+
indexed_count: indexedCount,
|
|
786
|
+
};
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
/** Register the three semantic tools. No-op when search is not configured. */
|
|
790
|
+
function registerSearchTools(server, api, context = new SearchContext()) {
|
|
791
|
+
if (!context.isConfigured)
|
|
792
|
+
return context;
|
|
793
|
+
// Opt-in background sync. No-op unless PAPERLESS_MCP_SYNC_INTERVAL_MS > 0.
|
|
794
|
+
context.startBackgroundSync(api);
|
|
795
|
+
// Opt-in change-notification receiver. No-op unless
|
|
796
|
+
// PAPERLESS_MCP_WEBHOOK_PORT > 0, and refuses without a secret.
|
|
797
|
+
context.startWebhook(api);
|
|
798
|
+
const filterArgs = FILTER_SHAPE;
|
|
799
|
+
server.tool("semantic_search", "Search the document corpus by MEANING as well as by words, using a local vector index fused with the corpus' own BM25 lexical search. Returns bounded passages in the same shape as search_passages (id, title, score, passage, offset, content_length) plus `matched`, which says whether the hit came from meaning, words, or both -- a 'words' hit is a lexical match the vector side did not find, so do not read it as absence of a paraphrase. HONEST DEGRADATION: `semantic` is 'unavailable' and lexical results are still returned when the local embedding server is down; a degraded call never returns a silent empty list. strong_match is true exactly when at least one passage was returned; it is a COARSE 'did-anything-come-back' flag, NOT a quality judgement -- the 0.45 similarity floor it used to consult cannot fire on this corpus (best raw similarities run 0.73-0.79), so judge quality from `matched`, `retrievers` and the passages themselves. Requires the local index to have been built with sync_embeddings.", Object.assign({ query: zod_1.z.string().min(1).describe("What to search for, in natural language."), limit: zod_1.z
|
|
800
|
+
.number()
|
|
801
|
+
.int()
|
|
802
|
+
.min(1)
|
|
803
|
+
.max(exports.MAX_LIMIT)
|
|
804
|
+
.optional()
|
|
805
|
+
.describe(`Maximum passages to return (default ${exports.DEFAULT_LIMIT}, max ${exports.MAX_LIMIT}).`), window: zod_1.z
|
|
806
|
+
.number()
|
|
807
|
+
.int()
|
|
808
|
+
.min(200)
|
|
809
|
+
.max(4000)
|
|
810
|
+
.optional()
|
|
811
|
+
.describe(`Characters per passage (default ${exports.DEFAULT_WINDOW}).`), check_staleness: zod_1.z
|
|
812
|
+
.boolean()
|
|
813
|
+
.optional()
|
|
814
|
+
.describe("Also report whether the index still matches the live corpus: `stale`, `index_age_s`, `live_count`, `indexed_count`. Costs one extra request (~44% on a sub-second call), so it is off by default. `stale: null` means the check could not be made — NOT that the index is current.") }, filterArgs), (0, middlewares_1.withErrorHandling)((args) => __awaiter(this, void 0, void 0, function* () {
|
|
815
|
+
// `check_staleness` is destructured OUT of `filters`, not left in it.
|
|
816
|
+
// `buildDocumentFilter` ignores unknown keys today, so leaving it there
|
|
817
|
+
// would work and would also mean every future filter key has to remember
|
|
818
|
+
// to exclude it — a silent coupling rather than a stated one.
|
|
819
|
+
const { limit, window, check_staleness } = args, filters = __rest(args, ["limit", "window", "check_staleness"]);
|
|
820
|
+
const result = yield runSemanticSearch(context, args.query, Object.assign(Object.assign({ limit: limit !== null && limit !== void 0 ? limit : exports.DEFAULT_LIMIT, window: window !== null && window !== void 0 ? window : exports.DEFAULT_WINDOW,
|
|
821
|
+
// ONE batched metadata request for N documents, content never fetched.
|
|
822
|
+
loadDocuments: (ids) => loadDocumentMetadata(api, ids),
|
|
823
|
+
// The text the window is cut from comes from the index, which already
|
|
824
|
+
// holds it. Reading it here instead of over HTTP is the 41x.
|
|
825
|
+
loadText: indexTextLoader(context.index()), filters: buildDocumentFilter(filters) }, (check_staleness ? { checkStaleness: true } : {})), {
|
|
826
|
+
// Bound late: `api` is this server's client, and the probe must use the
|
|
827
|
+
// same one the rest of the tool does.
|
|
828
|
+
corpusState: () => (0, sync_1.readCorpusState)(api) }));
|
|
829
|
+
return {
|
|
830
|
+
content: [{ type: "text", text: JSON.stringify(result) }],
|
|
831
|
+
};
|
|
832
|
+
})));
|
|
833
|
+
server.tool("embedding_status", "Report what the local semantic-search index actually holds: document and chunk counts, embedding dimensions, the model and prefix mode it was built with, when it last synced, how many documents are pending, how much disk the index uses, and whether any documents were truncated at the chunk cap. Use this before trusting semantic_search, and to tell 'the index is empty' apart from 'nothing matched'.", {}, (0, middlewares_1.withErrorHandling)(() => __awaiter(this, void 0, void 0, function* () {
|
|
834
|
+
const status = yield embeddingStatus(context, api);
|
|
835
|
+
return {
|
|
836
|
+
content: [{ type: "text", text: JSON.stringify(status) }],
|
|
837
|
+
};
|
|
838
|
+
})));
|
|
839
|
+
server.tool("sync_embeddings", "Bring the local semantic-search index up to date with the Paperless corpus. Fetches every document whose `modified` changed since the last run, re-chunks and re-embeds it, removes documents that no longer exist, and refreshes metadata for documents whose tags or correspondent changed without the document being edited. Long-running on a first build (tens of minutes for a large corpus) and it stops with an error if the local embedding server is unavailable, rather than leaving a half-built index that claims to be complete. Run embedding_status afterwards to confirm. force:true re-embeds every document and should only be used when the chunker or the model changed.", {
|
|
840
|
+
force: zod_1.z
|
|
841
|
+
.boolean()
|
|
842
|
+
.optional()
|
|
843
|
+
.describe("Re-embed every document even if `modified` is unchanged. Only needed after a chunker or model change; a normal sync is incremental."),
|
|
844
|
+
}, (0, middlewares_1.withErrorHandling)((args) => __awaiter(this, void 0, void 0, function* () {
|
|
845
|
+
const runner = context.syncRunner(api);
|
|
846
|
+
if (args.force) {
|
|
847
|
+
// ⚠️ PROBE BEFORE DELETE. The old order was delete -> run -> throw on
|
|
848
|
+
// outage, which turned a transient embedder blip into "you no longer
|
|
849
|
+
// have an index" on a rebuild that takes ~58 minutes. The failure the
|
|
850
|
+
// caller can afford is a refusal; the failure it cannot afford is
|
|
851
|
+
// having destroyed the working index to discover the server is down.
|
|
852
|
+
//
|
|
853
|
+
// The probe is a real embedding of a constant string (see
|
|
854
|
+
// `OllamaEmbedder.probe`), not a GET: a server that is up without its
|
|
855
|
+
// model pulled answers GET and then fails on the first real request.
|
|
856
|
+
const embedder = context.embedderOrUndefined();
|
|
857
|
+
if (embedder) {
|
|
858
|
+
if (embedder.probe) {
|
|
859
|
+
yield embedder.probe();
|
|
860
|
+
}
|
|
861
|
+
else {
|
|
862
|
+
// A fake without probe(): still exercise the same path the sync
|
|
863
|
+
// will use, so the ordering guarantee holds for every embedder.
|
|
864
|
+
yield embedder.embedQuery("ping");
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
// force means "the stored vectors are not trustworthy", which on this
|
|
868
|
+
// schema means deleting the document rows so the diff sees everything
|
|
869
|
+
// as new. Done through deleteDocuments rather than a file unlink so the
|
|
870
|
+
// FTS and vector tables are emptied in the same transaction.
|
|
871
|
+
const store = context.index();
|
|
872
|
+
store.deleteDocuments([...store.indexed().keys()]);
|
|
873
|
+
}
|
|
874
|
+
// ⚠️ `force` IS PASSED THROUGH, not merely acted on locally. The pre-sync
|
|
875
|
+
// delete below empties the document rows so the diff sees everything as
|
|
876
|
+
// new; the flag ALSO has to reach `run()` because the watermark check
|
|
877
|
+
// there would otherwise skip the run it just prepared for. Both halves are
|
|
878
|
+
// required: without the delete the vectors are stale, and without the flag
|
|
879
|
+
// the run does nothing at all.
|
|
880
|
+
const report = yield runner.run({ force: args.force === true });
|
|
881
|
+
return {
|
|
882
|
+
content: [{ type: "text", text: JSON.stringify(report) }],
|
|
883
|
+
};
|
|
884
|
+
})));
|
|
885
|
+
return context;
|
|
886
|
+
}
|
|
887
|
+
/** A predicate over visible documents, from the search_passages filter args. */
|
|
888
|
+
function buildDocumentFilter(filters) {
|
|
889
|
+
// Every key in FILTER_SHAPE is honoured here. `storage_path` used to be the
|
|
890
|
+
// exception: advertised to the model, absent from this predicate, and
|
|
891
|
+
// therefore silently ignored -- a caller got unfiltered results with no
|
|
892
|
+
// error, no warning, and no way to notice. That is worse than not offering
|
|
893
|
+
// the filter at all, because the schema is a promise.
|
|
894
|
+
const active = filters.correspondent !== undefined ||
|
|
895
|
+
filters.document_type !== undefined ||
|
|
896
|
+
filters.tag !== undefined ||
|
|
897
|
+
filters.storage_path !== undefined ||
|
|
898
|
+
filters.created__date__gte !== undefined ||
|
|
899
|
+
filters.created__date__lte !== undefined;
|
|
900
|
+
if (!active)
|
|
901
|
+
return undefined;
|
|
902
|
+
return (doc) => {
|
|
903
|
+
if (filters.correspondent !== undefined && doc.correspondent !== filters.correspondent) {
|
|
904
|
+
return false;
|
|
905
|
+
}
|
|
906
|
+
if (filters.document_type !== undefined && doc.document_type !== filters.document_type) {
|
|
907
|
+
return false;
|
|
908
|
+
}
|
|
909
|
+
if (filters.tag !== undefined && !doc.tags.includes(filters.tag))
|
|
910
|
+
return false;
|
|
911
|
+
// An unfiled document (storage_path === null) can never match a numeric
|
|
912
|
+
// storage_path, which is correct: the caller asked for a filing location.
|
|
913
|
+
if (filters.storage_path !== undefined && doc.storage_path !== filters.storage_path) {
|
|
914
|
+
return false;
|
|
915
|
+
}
|
|
916
|
+
if (filters.created__date__gte !== undefined && doc.created < filters.created__date__gte) {
|
|
917
|
+
return false;
|
|
918
|
+
}
|
|
919
|
+
if (filters.created__date__lte !== undefined && doc.created > filters.created__date__lte) {
|
|
920
|
+
return false;
|
|
921
|
+
}
|
|
922
|
+
return true;
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Fetch the documents behind a set of ids, tolerating individual failures.
|
|
927
|
+
*
|
|
928
|
+
* One unfetchable document must not fail the whole search: the remaining hits
|
|
929
|
+
* are still real, and a search that returns four good passages and one gap is
|
|
930
|
+
* strictly better than an error. The gap is visible because the passage for
|
|
931
|
+
* that id is absent.
|
|
932
|
+
*/
|
|
933
|
+
function loadVisibleDocuments(api, ids) {
|
|
934
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
935
|
+
const documents = new Map();
|
|
936
|
+
const unique = [...new Set(ids)];
|
|
937
|
+
const results = yield Promise.all(unique.map((id) => __awaiter(this, void 0, void 0, function* () {
|
|
938
|
+
try {
|
|
939
|
+
const doc = (yield api.request(`/documents/${id}/`));
|
|
940
|
+
return doc;
|
|
941
|
+
}
|
|
942
|
+
catch (_a) {
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
})));
|
|
946
|
+
for (const doc of results) {
|
|
947
|
+
if (doc && typeof doc.id === "number")
|
|
948
|
+
documents.set(doc.id, doc);
|
|
949
|
+
}
|
|
950
|
+
return documents;
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
/**
|
|
954
|
+
* Ids per batched metadata request.
|
|
955
|
+
*
|
|
956
|
+
* A single call can legitimately want metadata for `limit * 4` distinct
|
|
957
|
+
* documents from EACH retriever, so 400 is reachable. One request per document
|
|
958
|
+
* is what this replaces; one request for 400 ids would be a 2 KB URL, so the
|
|
959
|
+
* list is cut into batches rather than trusted to be short.
|
|
960
|
+
*/
|
|
961
|
+
exports.METADATA_BATCH = 100;
|
|
962
|
+
/**
|
|
963
|
+
* Fetch metadata for many documents in as few requests as possible.
|
|
964
|
+
*
|
|
965
|
+
* ⚠️ THIS NEVER FETCHES DOCUMENT CONTENT, and that is the point of the whole
|
|
966
|
+
* change. The previous path issued one `GET /documents/{id}/` per hit -- up to
|
|
967
|
+
* 80 of them for a 10-passage call -- and read every full body only to slice a
|
|
968
|
+
* window out of it. Measured on the live box for 40 documents: 2,398 ms and
|
|
969
|
+
* 2,162,306 bytes, against 58 ms and 6,520 bytes for the same metadata in one
|
|
970
|
+
* ranged call. The text the window needs was already in the index.
|
|
971
|
+
*
|
|
972
|
+
* `content` is therefore `null` on every returned document, which the
|
|
973
|
+
* `VisibleDocument` type already allows. Nothing on this path reads it.
|
|
974
|
+
*
|
|
975
|
+
* A batch that fails does NOT fail the search: the affected ids are simply
|
|
976
|
+
* absent from the map, which the caller already reports as
|
|
977
|
+
* `document_unavailable`. That is the same tolerance the per-id version had,
|
|
978
|
+
* kept deliberately so one probe failure cannot turn a search into an error.
|
|
979
|
+
*/
|
|
980
|
+
function loadDocumentMetadata(api, ids) {
|
|
981
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
982
|
+
var _a;
|
|
983
|
+
const documents = new Map();
|
|
984
|
+
const unique = [...new Set(ids)];
|
|
985
|
+
if (unique.length === 0)
|
|
986
|
+
return documents;
|
|
987
|
+
for (let offset = 0; offset < unique.length; offset += exports.METADATA_BATCH) {
|
|
988
|
+
const batch = unique.slice(offset, offset + exports.METADATA_BATCH);
|
|
989
|
+
const path = `/documents/?id__in=${batch.join(",")}` +
|
|
990
|
+
`&fields=id,title,correspondent,document_type,storage_path,tags,created,modified` +
|
|
991
|
+
`&page_size=${batch.length}`;
|
|
992
|
+
try {
|
|
993
|
+
const page = (yield api.request(path));
|
|
994
|
+
for (const doc of (_a = page.results) !== null && _a !== void 0 ? _a : []) {
|
|
995
|
+
if (typeof doc.id === "number") {
|
|
996
|
+
documents.set(doc.id, Object.assign(Object.assign({}, doc), { content: null }));
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
catch (_b) {
|
|
1001
|
+
// Left absent, exactly as the per-id path left a failed fetch absent.
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
return documents;
|
|
1005
|
+
});
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Rebuild a document's text from the chunks the index already holds.
|
|
1009
|
+
*
|
|
1010
|
+
* The chunker emits `CHUNK_SIZE` characters at a stride of
|
|
1011
|
+
* `CHUNK_SIZE - CHUNK_OVERLAP`, so consecutive chunks share exactly the
|
|
1012
|
+
* trailing `CHUNK_OVERLAP` characters of the previous one. Splicing therefore
|
|
1013
|
+
* has to drop that overlap -- but ONLY when it is really there, because a
|
|
1014
|
+
* document shorter than one chunk is a single chunk and dropping 200 characters
|
|
1015
|
+
* from it would delete text rather than deduplicate it.
|
|
1016
|
+
*
|
|
1017
|
+
* The metadata chunk at position 0 is skipped: it is a synthetic
|
|
1018
|
+
* "Title: … Sender: …" line, not document text, and including it would put
|
|
1019
|
+
* fabricated words in front of a passage and shift every offset by its length.
|
|
1020
|
+
*
|
|
1021
|
+
* ⚠️ The result is NORMALISED text -- `chunkText` collapses whitespace runs
|
|
1022
|
+
* before slicing. Offsets derived from it are labelled `offset_space:
|
|
1023
|
+
* "normalized"` for that reason.
|
|
1024
|
+
*/
|
|
1025
|
+
function reconstructDocumentText(chunks) {
|
|
1026
|
+
// ⚠️ PIECES + ONE JOIN, NEVER `built += …`. The obvious implementation
|
|
1027
|
+
// (`built += …; built.endsWith(head)`) is O(n²) on a long document and the
|
|
1028
|
+
// cost is not theoretical: measured on the live index it took **14.7 seconds**
|
|
1029
|
+
// to reconstruct a 3,606-chunk document, because `endsWith` flattens V8's
|
|
1030
|
+
// rope on every iteration while the rope keeps growing. The array-plus-join
|
|
1031
|
+
// form does the same splice in ~2 ms.
|
|
1032
|
+
const pieces = [];
|
|
1033
|
+
let previous;
|
|
1034
|
+
for (const chunk of chunks) {
|
|
1035
|
+
if (chunk.kind !== "content")
|
|
1036
|
+
continue;
|
|
1037
|
+
// The overlap is compared against the PREVIOUS CHUNK, not against the
|
|
1038
|
+
// accumulated text. It is the same 200 characters — the chunker emits
|
|
1039
|
+
// consecutive slices at a stride of CHUNK_SIZE - CHUNK_OVERLAP — but
|
|
1040
|
+
// comparing a short, already-flat string is O(200) rather than O(document).
|
|
1041
|
+
let head = 0;
|
|
1042
|
+
if (previous !== undefined) {
|
|
1043
|
+
const probe = Math.min(chunk_1.CHUNK_OVERLAP, previous.length, chunk.text.length);
|
|
1044
|
+
if (probe > 0 && previous.slice(-probe) === chunk.text.slice(0, probe)) {
|
|
1045
|
+
head = probe;
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
pieces.push(chunk.text.slice(head));
|
|
1049
|
+
previous = chunk.text;
|
|
1050
|
+
}
|
|
1051
|
+
return pieces.join("");
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* The production `loadText`: document text straight out of the open index.
|
|
1055
|
+
*
|
|
1056
|
+
* One prepared SELECT per document, against a local SQLite file, replacing one
|
|
1057
|
+
* HTTPS GET per document that returned the whole body. Measured on the live
|
|
1058
|
+
* index (read + splice): 2 ms for a typical document, 34–42 ms for the largest
|
|
1059
|
+
* (3,606 chunks / 7.2 M characters).
|
|
1060
|
+
*
|
|
1061
|
+
* A document whose chunks cannot be read is simply absent from the map, which
|
|
1062
|
+
* makes the caller fall back to `doc.content` — `null` on the batched-metadata
|
|
1063
|
+
* path, hence an empty passage rather than an error. That is the same
|
|
1064
|
+
* "one unusable document must not fail the search" tolerance the fetch path
|
|
1065
|
+
* had, and `content_length: 0` makes it visible instead of silent.
|
|
1066
|
+
*/
|
|
1067
|
+
function indexTextLoader(store) {
|
|
1068
|
+
return (hits) => {
|
|
1069
|
+
const text = new Map();
|
|
1070
|
+
for (const id of new Set(hits.map((hit) => hit.documentId))) {
|
|
1071
|
+
try {
|
|
1072
|
+
text.set(id, reconstructDocumentText(store.chunksFor(id)));
|
|
1073
|
+
}
|
|
1074
|
+
catch (_a) {
|
|
1075
|
+
// Absent on purpose: reported downstream as an empty passage.
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
return text;
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
/**
|
|
1082
|
+
* Trim a result until its own serialised body fits the consumer's ceiling.
|
|
1083
|
+
*
|
|
1084
|
+
* Measured on the REAL bytes, not predicted from `window * passages`. That
|
|
1085
|
+
* distinction is the whole mechanism: titles, ids and counts vary, and an
|
|
1086
|
+
* arithmetic budget that is right on average is exactly the kind of bound that
|
|
1087
|
+
* fails on the one call that matters. `budgetedLimit` keeps the work small;
|
|
1088
|
+
* this is what makes the guarantee actual.
|
|
1089
|
+
*/
|
|
1090
|
+
function fitResultToBudget(result) {
|
|
1091
|
+
if (JSON.stringify(result).length <= exports.MAX_RESULT_CHARS)
|
|
1092
|
+
return result;
|
|
1093
|
+
// ⚠️ A PASSAGE-ONLY TRIM IS NOT A BOUND. Measured: `fitResultToBudget` dropped
|
|
1094
|
+
// passages and never touched the ECHOED QUERY, which has no length limit -- a
|
|
1095
|
+
// 30,000-char query returned 30,128 chars with zero passages, over the ceiling
|
|
1096
|
+
// the function exists to enforce. Dropping every passage cannot fix an
|
|
1097
|
+
// oversize field that is not a passage. Echo less.
|
|
1098
|
+
const echo = typeof result.query === "string" ? result.query : "";
|
|
1099
|
+
const passages = [...result.passages];
|
|
1100
|
+
let dropped = 0;
|
|
1101
|
+
let candidate = result;
|
|
1102
|
+
while (passages.length > 0) {
|
|
1103
|
+
passages.pop();
|
|
1104
|
+
dropped += 1;
|
|
1105
|
+
candidate = Object.assign(Object.assign({}, result), { passages, trimmed: { reason: "result_size", dropped, limit_chars: exports.MAX_RESULT_CHARS } });
|
|
1106
|
+
if (JSON.stringify(candidate).length <= exports.MAX_RESULT_CHARS)
|
|
1107
|
+
return candidate;
|
|
1108
|
+
}
|
|
1109
|
+
// Every passage is gone and it is STILL over budget: the query echo alone is
|
|
1110
|
+
// too large. Truncate the echo rather than return an oversize result, and say
|
|
1111
|
+
// so -- a silently truncated query is indistinguishable from a short one.
|
|
1112
|
+
//
|
|
1113
|
+
// ⚠️ MEASURE AFTER ADDING THE FLAG, NOT BEFORE. The first cut of this computed
|
|
1114
|
+
// the overflow against the payload WITHOUT `query_truncated`, then added that
|
|
1115
|
+
// field -- which is 24 chars -- and returned 20,074 against a 20,000 ceiling.
|
|
1116
|
+
// A bound that its own error message breaks is not a bound, and the test
|
|
1117
|
+
// caught it only because it asserted the serialised length rather than the
|
|
1118
|
+
// passage count. Loop until it actually fits; the flag costs bytes, so assume
|
|
1119
|
+
// it is present from the first attempt.
|
|
1120
|
+
const declared = Object.assign(Object.assign({}, candidate), { trimmed: {
|
|
1121
|
+
reason: "result_size",
|
|
1122
|
+
dropped,
|
|
1123
|
+
limit_chars: exports.MAX_RESULT_CHARS,
|
|
1124
|
+
query_truncated: true,
|
|
1125
|
+
} });
|
|
1126
|
+
let keep = echo.length;
|
|
1127
|
+
for (;;) {
|
|
1128
|
+
const over = JSON.stringify(Object.assign(Object.assign({}, declared), { query: echo.slice(0, keep) })).length - exports.MAX_RESULT_CHARS;
|
|
1129
|
+
if (over <= 0 || keep === 0)
|
|
1130
|
+
break;
|
|
1131
|
+
keep = Math.max(0, keep - over - 1);
|
|
1132
|
+
}
|
|
1133
|
+
return Object.assign(Object.assign({}, declared), { query: echo.slice(0, keep) });
|
|
1134
|
+
}
|
|
1135
|
+
/**
|
|
1136
|
+
* Report the store's own view of itself.
|
|
1137
|
+
*
|
|
1138
|
+
* `pending` is computed against Paperless' document count, so it is `null`
|
|
1139
|
+
* rather than `0` when that count cannot be read: a zero here would say "the
|
|
1140
|
+
* index is current" on the strength of a failed request.
|
|
1141
|
+
*/
|
|
1142
|
+
function embeddingStatus(context, api) {
|
|
1143
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1144
|
+
var _a, _b, _c;
|
|
1145
|
+
const config = context.config();
|
|
1146
|
+
const path = config ? (0, config_1.indexPath)(config) : "";
|
|
1147
|
+
const base = {
|
|
1148
|
+
documents: 0,
|
|
1149
|
+
chunks: 0,
|
|
1150
|
+
dimensions: 0,
|
|
1151
|
+
model: "",
|
|
1152
|
+
prefix_mode: "",
|
|
1153
|
+
last_sync: null,
|
|
1154
|
+
pending: null,
|
|
1155
|
+
store_bytes: null,
|
|
1156
|
+
truncated_documents: 0,
|
|
1157
|
+
index_path: path,
|
|
1158
|
+
};
|
|
1159
|
+
let store;
|
|
1160
|
+
try {
|
|
1161
|
+
store = context.index();
|
|
1162
|
+
}
|
|
1163
|
+
catch (err) {
|
|
1164
|
+
return Object.assign(Object.assign({}, base), { open_error: errorMessage(err) });
|
|
1165
|
+
}
|
|
1166
|
+
const counts = store.counts();
|
|
1167
|
+
const meta = store.meta();
|
|
1168
|
+
const documents = store.indexed();
|
|
1169
|
+
const stamps = store.metadataStamps();
|
|
1170
|
+
let lastSync = null;
|
|
1171
|
+
for (const stamp of stamps.values()) {
|
|
1172
|
+
if (lastSync === null || stamp > lastSync)
|
|
1173
|
+
lastSync = stamp;
|
|
1174
|
+
}
|
|
1175
|
+
// Truncation is the one index fact that changes what a caller can conclude
|
|
1176
|
+
// from a miss: a truncated document lost its tail, so "not found" may be a
|
|
1177
|
+
// false negative. Counted with one query, not one query per document.
|
|
1178
|
+
const truncated = store.truncatedCount();
|
|
1179
|
+
let storeBytes = null;
|
|
1180
|
+
try {
|
|
1181
|
+
const { statSync } = yield Promise.resolve().then(() => __importStar(require("node:fs")));
|
|
1182
|
+
storeBytes = statSync(path).size;
|
|
1183
|
+
}
|
|
1184
|
+
catch (_d) {
|
|
1185
|
+
// Not an error: a fresh index file may not exist yet.
|
|
1186
|
+
}
|
|
1187
|
+
let pending = null;
|
|
1188
|
+
try {
|
|
1189
|
+
const page = (yield api.request("/documents/?fields=id&page_size=1"));
|
|
1190
|
+
if (typeof page.count === "number") {
|
|
1191
|
+
pending = Math.max(0, page.count - documents.size);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
catch (_e) {
|
|
1195
|
+
// Left null: a failed count is not a count of zero.
|
|
1196
|
+
}
|
|
1197
|
+
return Object.assign(Object.assign({}, base), { documents: counts.documents, chunks: counts.chunks, dimensions: (_a = meta === null || meta === void 0 ? void 0 : meta.dimensions) !== null && _a !== void 0 ? _a : 0, model: (_b = meta === null || meta === void 0 ? void 0 : meta.model) !== null && _b !== void 0 ? _b : "", prefix_mode: (_c = meta === null || meta === void 0 ? void 0 : meta.prefixMode) !== null && _c !== void 0 ? _c : "", last_sync: lastSync, pending, store_bytes: storeBytes, truncated_documents: truncated });
|
|
1198
|
+
});
|
|
1199
|
+
}
|