@retiform/paperless-mcp 2.3.6 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +9 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/manifest.test.d.ts +0 -1
- package/build/manifest.test.js +0 -66
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,518 @@
|
|
|
1
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import type { PaperlessAPI } from "../api/PaperlessAPI";
|
|
3
|
+
import { type SearchConfig } from "../search/config";
|
|
4
|
+
import { type Embedder } from "../search/embedder";
|
|
5
|
+
import { SearchIndex } from "../search/store";
|
|
6
|
+
import { type Fused } from "../search/rank";
|
|
7
|
+
import { IndexSync, type CorpusState } from "../search/sync";
|
|
8
|
+
/** A similarity at or above this is a strong match. Similarity, never distance.
|
|
9
|
+
*
|
|
10
|
+
* ⚠️ NOT USED AS A GATE ANY MORE, and the measurement is why. On this corpus
|
|
11
|
+
* the floor cannot separate a good query from a bad one. Measured on the live
|
|
12
|
+
* index (2,476 docs), best `1 - distance` per query:
|
|
13
|
+
*
|
|
14
|
+
* relevant "kuji nine invisibility…" 0.700 "naga offering…" 0.746
|
|
15
|
+
* "barber pole worm in goats…" 0.761 "channels and winds" 0.766
|
|
16
|
+
* "three humours and prana disorders" 0.731
|
|
17
|
+
* irrelevant "xyzzy plugh frobnicate quux" 0.599 "stock market options" 0.595
|
|
18
|
+
* "quantum chromodynamics…" 0.656 "best pizza dough…" 0.766
|
|
19
|
+
* "how to change a flat tire" 0.730
|
|
20
|
+
*
|
|
21
|
+
* The ranges OVERLAP: two of five irrelevant queries score above the worst
|
|
22
|
+
* relevant one, and no threshold in 0.500–0.800 separates the sets. A floor at
|
|
23
|
+
* 0.45 sits 0.145 BELOW the worst irrelevant query, so it could never fire —
|
|
24
|
+
* `strong_match` was constant `true` whenever the vector arm ran.
|
|
25
|
+
*
|
|
26
|
+
* Kept exported only so existing importers compile; treat any use of it as a
|
|
27
|
+
* claim that needs this card's measurement re-run on the corpus in question.
|
|
28
|
+
*/
|
|
29
|
+
export declare const STRONG_MATCH = 0.45;
|
|
30
|
+
/** Chars per returned passage, matching search_passages' default. */
|
|
31
|
+
export declare const DEFAULT_WINDOW = 800;
|
|
32
|
+
export declare const DEFAULT_LIMIT = 10;
|
|
33
|
+
export declare const MAX_LIMIT = 50;
|
|
34
|
+
/**
|
|
35
|
+
* The largest JSON body a caller can be handed, in characters.
|
|
36
|
+
*
|
|
37
|
+
* This is not our budget to choose freely: it is the ceiling the CONSUMER
|
|
38
|
+
* applies. Mneme truncates any non-memory tool result at 20,000 characters by
|
|
39
|
+
* taking head and tail, and the agent on the other side cannot detect that it
|
|
40
|
+
* happened -- a passage that was silently cut reads exactly like a short one.
|
|
41
|
+
*
|
|
42
|
+
* `MAX_LIMIT * max window` is 208,485 characters, ten times this number, so
|
|
43
|
+
* until this constant existed the tool's own advertised argument range could
|
|
44
|
+
* produce a result the client would mangle. Bounding it in the schema
|
|
45
|
+
* description would be advice; the enforcement is `budgetedLimit` (so a
|
|
46
|
+
* maximal call does not waste work) plus `fitResultToBudget` (so the returned
|
|
47
|
+
* bytes are measured, not predicted).
|
|
48
|
+
*/
|
|
49
|
+
export declare const MAX_RESULT_CHARS = 20000;
|
|
50
|
+
/**
|
|
51
|
+
* JSON scaffolding a single passage costs, roughly: id, title, score, matched,
|
|
52
|
+
* offset, content_length, offset_space, quotes and commas.
|
|
53
|
+
*
|
|
54
|
+
* Used only to pick the passage COUNT before any work is done. `PASSAGE_OVERHEAD_CHARS` is deliberately
|
|
55
|
+
* generous: over-estimating costs a passage, under-estimating costs a wasted
|
|
56
|
+
* build that `fitResultToBudget` then discards anyway.
|
|
57
|
+
*/
|
|
58
|
+
export declare const PASSAGE_OVERHEAD_CHARS = 200;
|
|
59
|
+
/**
|
|
60
|
+
* How many passages of `window` characters can fit inside the consumer's
|
|
61
|
+
* ceiling. Never zero: a call that cannot afford even one passage should still
|
|
62
|
+
* return one and let `fitResultToBudget` trim if the bytes really do not fit.
|
|
63
|
+
*/
|
|
64
|
+
export declare function budgetedLimit(limit: number, window: number): number;
|
|
65
|
+
export interface VisibleDocument {
|
|
66
|
+
id: number;
|
|
67
|
+
title: string;
|
|
68
|
+
content: string | null;
|
|
69
|
+
correspondent: number | null;
|
|
70
|
+
document_type: number | null;
|
|
71
|
+
/**
|
|
72
|
+
* Paperless' storage path id, or null when the document is unfiled.
|
|
73
|
+
*
|
|
74
|
+
* Nullable because most documents are: on the live corpus it is the exception
|
|
75
|
+
* rather than the rule. It is typed `number | null` rather than optional
|
|
76
|
+
* because `undefined` (field absent) and `null` (explicitly unfiled) filter
|
|
77
|
+
* differently, and `/api/documents/{id}/` always returns the key.
|
|
78
|
+
*/
|
|
79
|
+
storage_path: number | null;
|
|
80
|
+
tags: number[];
|
|
81
|
+
created: string;
|
|
82
|
+
}
|
|
83
|
+
export interface SemanticPassage {
|
|
84
|
+
id: number;
|
|
85
|
+
title: string;
|
|
86
|
+
/**
|
|
87
|
+
* The fusion score, NOT a similarity. See the `score` note on
|
|
88
|
+
* `SemanticSearchResult` — the range is (0, 2/(k+1)] where `k` is
|
|
89
|
+
* `DEFAULT_FUSE_K`, and a value at the top of that range is a perfect fusion
|
|
90
|
+
* on this scale, not a strong similarity. Read `matched` and `highlights`
|
|
91
|
+
* for quality, not this.
|
|
92
|
+
*/
|
|
93
|
+
score?: number;
|
|
94
|
+
/** The query term the passage was anchored on, when one was found. */
|
|
95
|
+
highlights?: string;
|
|
96
|
+
matched: "meaning" | "words" | "both";
|
|
97
|
+
passage: string;
|
|
98
|
+
/**
|
|
99
|
+
* Where `passage` starts, IN THE SPACE NAMED BY `offset_space`.
|
|
100
|
+
*
|
|
101
|
+
* This used to be an offset into the raw document, because the raw document
|
|
102
|
+
* was fetched to build the window. It is not fetched any more, so the space
|
|
103
|
+
* changed and the change is stated rather than silent: an offset the caller
|
|
104
|
+
* cannot interpret is worse than no offset.
|
|
105
|
+
*/
|
|
106
|
+
offset: number;
|
|
107
|
+
/** Length of the text `offset` indexes into. Same space as `offset`. */
|
|
108
|
+
content_length: number;
|
|
109
|
+
/**
|
|
110
|
+
* Which text `offset` and `content_length` are measured against.
|
|
111
|
+
*
|
|
112
|
+
* `"normalized"` — the document as the chunker stored it: whitespace runs
|
|
113
|
+
* collapsed to single spaces, ends trimmed. This is the normal case now, and
|
|
114
|
+
* offsets are only directly comparable with `search_passages` offsets for
|
|
115
|
+
* documents whose text had no such runs.
|
|
116
|
+
* `"document"` — raw document text, the pre-P2 fallback path. Only reachable
|
|
117
|
+
* on a call that supplied no index text loader.
|
|
118
|
+
*/
|
|
119
|
+
offset_space: "normalized" | "document";
|
|
120
|
+
}
|
|
121
|
+
export interface SemanticSearchResult {
|
|
122
|
+
query: string;
|
|
123
|
+
semantic: "ok" | "unavailable";
|
|
124
|
+
degraded_reason?: string;
|
|
125
|
+
/**
|
|
126
|
+
* Whether ANY passage was returned.
|
|
127
|
+
*
|
|
128
|
+
* ⚠️ THIS IS NOT A QUALITY SIGNAL, and it used to pretend to be one. It was
|
|
129
|
+
* computed against `STRONG_MATCH` (a 0.45 similarity floor), which on this
|
|
130
|
+
* corpus cannot fire — measured best similarities for five irrelevant
|
|
131
|
+
* queries were 0.595, 0.599, 0.656, 0.730 and 0.766, all far above it — so
|
|
132
|
+
* the field was a constant `true` on every call where the vector arm ran.
|
|
133
|
+
* A field that cannot be false is not information.
|
|
134
|
+
*
|
|
135
|
+
* It is retained, and REDEFINED, rather than deleted: deleting it changes the
|
|
136
|
+
* response shape for every consumer, and the honest minimum is a field that
|
|
137
|
+
* means something small and says so. It is now exactly equivalent to
|
|
138
|
+
* `passages.length > 0` and is kept only for callers that branch on it.
|
|
139
|
+
*
|
|
140
|
+
* ⚠️ REMOVING IT IS THE RIGHT EVENTUAL FIX. That needs
|
|
141
|
+
* `src/tools/search.test.ts` edited, which was outside this change's file
|
|
142
|
+
* scope, so it is flagged in the report rather than done.
|
|
143
|
+
*
|
|
144
|
+
* Judge result quality from `matched` (per passage) and `retrievers` (which
|
|
145
|
+
* halves ran), never from this.
|
|
146
|
+
*/
|
|
147
|
+
strong_match: boolean;
|
|
148
|
+
/** Pre-filter: how many DISTINCT documents the retrievers returned. */
|
|
149
|
+
total_hits: number;
|
|
150
|
+
/**
|
|
151
|
+
* Post-filter: how many of `total_hits` were dropped before becoming
|
|
152
|
+
* passages.
|
|
153
|
+
*
|
|
154
|
+
* Exists because `total_hits: 12, passages: []` was previously a legal and
|
|
155
|
+
* completely unexplained response. A caller had no way to tell "everything
|
|
156
|
+
* was filtered out by your own filter arguments" from "the ranking found 12
|
|
157
|
+
* documents and every one has since vanished".
|
|
158
|
+
*/
|
|
159
|
+
filtered_out: number;
|
|
160
|
+
/**
|
|
161
|
+
* Why they were dropped, as counts. Absent when nothing was dropped.
|
|
162
|
+
*
|
|
163
|
+
* Split by cause because the two are actionable in different ways: a
|
|
164
|
+
* missing document means Paperless would not serve an id the index holds
|
|
165
|
+
* (deleted mid-flight, permissions), whereas a metadata-filter rejection
|
|
166
|
+
* means the arguments the caller passed did their job.
|
|
167
|
+
*/
|
|
168
|
+
filtered_out_reasons?: {
|
|
169
|
+
document_unavailable: number;
|
|
170
|
+
metadata_filter: number;
|
|
171
|
+
};
|
|
172
|
+
/**
|
|
173
|
+
* Which retrievers actually RAN, versus which merely returned nothing.
|
|
174
|
+
*
|
|
175
|
+
* This is the field that makes the three degradation states legible. An
|
|
176
|
+
* empty `passages` with `retrievers: ["vector", "lexical"]` is the honest
|
|
177
|
+
* answer "both searched and neither matched"; with `["lexical"]` it means
|
|
178
|
+
* only one half could run. Absence of this field would leave the caller
|
|
179
|
+
* inferring capability from a list length, which is how the old guard came
|
|
180
|
+
* to assert a mechanism that had not happened.
|
|
181
|
+
*/
|
|
182
|
+
retrievers: Array<"vector" | "lexical">;
|
|
183
|
+
/**
|
|
184
|
+
* Whether the index still reflects the live corpus. Present ONLY when the
|
|
185
|
+
* caller asked for it with `check_staleness`, because the check costs one
|
|
186
|
+
* extra request and most calls do not need it.
|
|
187
|
+
*
|
|
188
|
+
* ⚠️ `null` MEANS "I COULD NOT CHECK", NEVER "NOT STALE". That distinction is
|
|
189
|
+
* the entire point of making this nullable: a `false` produced because
|
|
190
|
+
* Paperless did not answer would be a fabricated clean bill of health, and
|
|
191
|
+
* fabricating one is the failure mode this whole arc has been written against.
|
|
192
|
+
* A caller that sees `null` knows to retry; a caller that sees `false` knows
|
|
193
|
+
* the index is current.
|
|
194
|
+
*/
|
|
195
|
+
stale?: boolean | null;
|
|
196
|
+
/** Age of the index in seconds, from the recorded sync time. Null when unknown. */
|
|
197
|
+
index_age_s?: number | null;
|
|
198
|
+
/** Live document count, or null when the probe failed. */
|
|
199
|
+
live_count?: number | null;
|
|
200
|
+
/** How many documents the index's own watermark claims. Null when unknown. */
|
|
201
|
+
indexed_count?: number | null;
|
|
202
|
+
/** Why staleness could not be determined, when it could not. */
|
|
203
|
+
staleness_reason?: string;
|
|
204
|
+
/**
|
|
205
|
+
* Set when the response had to be shortened to fit the consumer's ceiling.
|
|
206
|
+
*
|
|
207
|
+
* Present rather than silent for the same reason `filtered_out` is: a caller
|
|
208
|
+
* that asked for 50 passages and got 4 needs to know whether the corpus held
|
|
209
|
+
* 4 or whether 46 were dropped, and the two are indistinguishable from
|
|
210
|
+
* `passages.length`.
|
|
211
|
+
*/
|
|
212
|
+
trimmed?: {
|
|
213
|
+
reason: "result_size";
|
|
214
|
+
dropped: number;
|
|
215
|
+
limit_chars: number;
|
|
216
|
+
/** True when the echoed `query` itself was cut to fit, not just passages. */
|
|
217
|
+
query_truncated?: boolean;
|
|
218
|
+
};
|
|
219
|
+
/**
|
|
220
|
+
* Ranked passages, bounded.
|
|
221
|
+
*
|
|
222
|
+
* ⚠️ ABOUT `score`: it is the reciprocal-rank-fusion SUM, `Σ 1/(k + rank)`,
|
|
223
|
+
* not a similarity, where `k` is `DEFAULT_FUSE_K`. With two retrievers the
|
|
224
|
+
* maximum reachable value is `2 / (k + 1)`. Do NOT hard-code that ceiling:
|
|
225
|
+
* it moves whenever `k` moves (it was 0.033 at the old k=60 and is 0.182 at
|
|
226
|
+
* the shipped k=10), and a caller who assumes a 0..1 similarity scale will
|
|
227
|
+
* read a well-matched passage as terrible either way.
|
|
228
|
+
*
|
|
229
|
+
* It is left un-normalised ON PURPOSE. Normalising against the best score in
|
|
230
|
+
* the same result set would report 1.0 for the head of a set where every hit
|
|
231
|
+
* was bad, which is a worse lie than a small number. A meaningful 0..1
|
|
232
|
+
* score needs a floor calibrated against labelled good and bad results, and
|
|
233
|
+
* that calibration has not been done on this corpus.
|
|
234
|
+
*/
|
|
235
|
+
passages: SemanticPassage[];
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Cut a bounded passage around the most DISTINCTIVE place the query terms appear.
|
|
239
|
+
*
|
|
240
|
+
* Mirrors buildSearchPassage's anchoring priority minus the server highlight,
|
|
241
|
+
* which does not exist on this path: the vector side has no highlighted span,
|
|
242
|
+
* and the lexical side's BM25 match gives a chunk rather than an offset. So the
|
|
243
|
+
* anchor is (1) the RAREST matching query term, (2) the head of the document.
|
|
244
|
+
*
|
|
245
|
+
* Rarity, not query order and not length. Query order was the original defect:
|
|
246
|
+
* English queries open with articles, so first-match anchored "the practice of
|
|
247
|
+
* a wrathful deity with a lion head" on "the" at character 3, and containment
|
|
248
|
+
* still passed because "the" really was in the window. Length was my first
|
|
249
|
+
* correction and it is also wrong -- measured live, it picks `doctor` over
|
|
250
|
+
* `ankle` by one character. Rarity picks the term the query is ABOUT.
|
|
251
|
+
* When the caller cannot supply counts, length is the fallback.
|
|
252
|
+
*
|
|
253
|
+
* A window that does not contain the hit is worse than no window, which is why
|
|
254
|
+
* the term search runs against the whole document rather than the chunk.
|
|
255
|
+
*/
|
|
256
|
+
export declare function buildPassage(doc: {
|
|
257
|
+
id: number;
|
|
258
|
+
title: string;
|
|
259
|
+
content: string | null;
|
|
260
|
+
}, terms: string[], window: number,
|
|
261
|
+
/**
|
|
262
|
+
* How many indexed chunks contain each term, when the caller can supply it.
|
|
263
|
+
*
|
|
264
|
+
* ⚠️ THIS IS THE CORRECT DISCRIMINATOR AND LENGTH IS NOT. Measured on the
|
|
265
|
+
* live corpus: "what did the doctor say about the ankle" picked `doctor`
|
|
266
|
+
* (1,903 chunks) over `ankle` (808) on length alone -- wrong, and wrong in
|
|
267
|
+
* the way that matters, because the window then lands on a discussion of
|
|
268
|
+
* doctors rather than of the ankle. "the practice of a wrathful deity with a
|
|
269
|
+
* lion head" picked `practice` (36,459) over `wrathful` (4,657). Rarity
|
|
270
|
+
* matched a human read in 3 of 3 probes; length in 0 of 3.
|
|
271
|
+
*
|
|
272
|
+
* Optional so a caller with no index (the search_passages path, whose terms
|
|
273
|
+
* come from the server's own query) still works -- it then falls back to
|
|
274
|
+
* length, which is worse but still better than query order.
|
|
275
|
+
*/
|
|
276
|
+
termRarity?: Map<string, number>): {
|
|
277
|
+
passage: string;
|
|
278
|
+
offset: number;
|
|
279
|
+
content_length: number;
|
|
280
|
+
term?: string;
|
|
281
|
+
};
|
|
282
|
+
/** Query terms for passage anchoring, from the same tokenisation the FTS side uses. */
|
|
283
|
+
export declare function queryTerms(query: string): string[];
|
|
284
|
+
/**
|
|
285
|
+
* Everything the three tools share, built once and lazily.
|
|
286
|
+
*
|
|
287
|
+
* `store` is `undefined` until the first call. `openFailed` remembers a failure
|
|
288
|
+
* so a broken index does not retry a 1 GB open on every single tool call -- and
|
|
289
|
+
* remembers the MESSAGE, so embedding_status can report why.
|
|
290
|
+
*/
|
|
291
|
+
export declare class SearchContext {
|
|
292
|
+
private store?;
|
|
293
|
+
private openFailed?;
|
|
294
|
+
private sync?;
|
|
295
|
+
private syncStarted;
|
|
296
|
+
private webhook?;
|
|
297
|
+
private webhookStarted;
|
|
298
|
+
private embedder?;
|
|
299
|
+
private readonly configOverride?;
|
|
300
|
+
private readonly embedderFactory?;
|
|
301
|
+
/**
|
|
302
|
+
* `embedderFactory` exists so "no embedder" can be stated rather than
|
|
303
|
+
* inferred. A context whose embedder is merely unset would reach
|
|
304
|
+
* `embedderOrUndefined` and CONSTRUCT a real client — which then makes a
|
|
305
|
+
* network call. That is correct in production and unacceptable in a test, and
|
|
306
|
+
* the difference has to be expressible, so the factory can return undefined
|
|
307
|
+
* and mean it.
|
|
308
|
+
*/
|
|
309
|
+
constructor(config?: SearchConfig, embedderFactory?: (config: SearchConfig) => Embedder | undefined);
|
|
310
|
+
/** State the embedder explicitly. `null` means "there is deliberately none". */
|
|
311
|
+
setEmbedder(embedder: Embedder | null): void;
|
|
312
|
+
/**
|
|
313
|
+
* Resolve config from the environment at CALL time.
|
|
314
|
+
*
|
|
315
|
+
* Not at import: the gateway sets the child's environment before launch, but
|
|
316
|
+
* tests and the CLI set it after import, and reading once at module load
|
|
317
|
+
* makes the config untestable.
|
|
318
|
+
*/
|
|
319
|
+
config(): SearchConfig | undefined;
|
|
320
|
+
/** True when the search tools should register at all. */
|
|
321
|
+
get isConfigured(): boolean;
|
|
322
|
+
/** Open the index, memoised. Throws the underlying error on first failure. */
|
|
323
|
+
index(): SearchIndex;
|
|
324
|
+
embedderOrUndefined(): Embedder | undefined;
|
|
325
|
+
syncRunner(api: PaperlessAPI): IndexSync;
|
|
326
|
+
/**
|
|
327
|
+
* Start the background sync timer when the deployment asked for one.
|
|
328
|
+
*
|
|
329
|
+
* ⚠️ THIS METHOD IS THE CALLER `IndexSync.start()` NEVER HAD. The interval was
|
|
330
|
+
* read and range-checked in config.ts from the day it was added, and nothing
|
|
331
|
+
* ever acted on it: `start()` existed, was documented, and was invoked from
|
|
332
|
+
* nowhere. A validated setting with no consumer reads as a working feature
|
|
333
|
+
* and is not one.
|
|
334
|
+
*
|
|
335
|
+
* Called from `registerSearchTools`, so the wiring is exercised by every
|
|
336
|
+
* server that configures search at all. A test that constructs the timer
|
|
337
|
+
* directly proves `start()` works and says nothing about whether anything
|
|
338
|
+
* calls it; the caller is what has to be asserted.
|
|
339
|
+
*
|
|
340
|
+
* A background sync that cannot start must not take down tool registration:
|
|
341
|
+
* the observable `sync_embeddings` tool still works, so failure here is
|
|
342
|
+
* logged and swallowed rather than thrown.
|
|
343
|
+
*/
|
|
344
|
+
startBackgroundSync(api: PaperlessAPI): void;
|
|
345
|
+
/** True when a background sync timer is live. Used by the wiring test. */
|
|
346
|
+
get isBackgroundSyncActive(): boolean;
|
|
347
|
+
/**
|
|
348
|
+
* Start the change-notification receiver when the deployment asked for one.
|
|
349
|
+
*
|
|
350
|
+
* ⚠️ THE SAME CALLER PROBLEM AS `startBackgroundSync`, ONE LAYER OUT. A
|
|
351
|
+
* receiver class nobody constructs is a feature nobody has, and the interval
|
|
352
|
+
* in this file is the standing proof: it was read, validated and never acted
|
|
353
|
+
* on. So construction happens here, on the path `registerSearchTools` always
|
|
354
|
+
* walks, rather than behind a bootstrap only a test would call.
|
|
355
|
+
*
|
|
356
|
+
* Started asynchronously and never awaited. Registration is synchronous and
|
|
357
|
+
* happens before the transport connects; a bind that fails must not delay or
|
|
358
|
+
* break the tool surface, because the observable `sync_embeddings` tool still
|
|
359
|
+
* works and the watermark still converges on the next start without it.
|
|
360
|
+
*/
|
|
361
|
+
startWebhook(api: PaperlessAPI): void;
|
|
362
|
+
/** True when a receiver is bound. Used by the wiring test. */
|
|
363
|
+
get isWebhookListening(): boolean;
|
|
364
|
+
/** The port the receiver actually bound, or undefined when it is off. */
|
|
365
|
+
get webhookPort(): number | undefined;
|
|
366
|
+
close(): void;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Fuse the two lists, tolerating an absent lexical half or an absent embedder.
|
|
370
|
+
*
|
|
371
|
+
* This is the function the contract is built on, and it is deliberately one
|
|
372
|
+
* function: the degradation lives in one place so no caller can accidentally
|
|
373
|
+
* reintroduce a silent empty list.
|
|
374
|
+
*/
|
|
375
|
+
export declare function runSemanticSearch(context: SearchContext, query: string, opts: {
|
|
376
|
+
limit: number;
|
|
377
|
+
window: number;
|
|
378
|
+
loadDocuments: (ids: number[]) => Promise<Map<number, VisibleDocument>>;
|
|
379
|
+
/**
|
|
380
|
+
* Document text, read from the index instead of over HTTP.
|
|
381
|
+
*
|
|
382
|
+
* ⚠️ OPTIONAL ONLY FOR TEST INJECTION; production always supplies it. When
|
|
383
|
+
* present, the passage is built from THIS text and the metadata documents
|
|
384
|
+
* never need a `content` field — which is what removes one full-document
|
|
385
|
+
* GET per hit. When absent, the old `doc.content` path is used, and
|
|
386
|
+
* `offset_space` says `"document"` so the caller is not misled about what
|
|
387
|
+
* its offsets index into.
|
|
388
|
+
*/
|
|
389
|
+
loadText?: (hits: Fused[]) => Map<number, string>;
|
|
390
|
+
filters?: (doc: VisibleDocument) => boolean;
|
|
391
|
+
/** Opt in to the staleness report. Costs one extra request when true. */
|
|
392
|
+
checkStaleness?: boolean;
|
|
393
|
+
/**
|
|
394
|
+
* How to read the live corpus. Injected rather than imported so the search
|
|
395
|
+
* path has no opinion about which API client is in use, and so a test can
|
|
396
|
+
* supply a corpus without a network.
|
|
397
|
+
*/
|
|
398
|
+
corpusState?: () => Promise<CorpusState | undefined>;
|
|
399
|
+
/** Injectable clock, so `index_age_s` is testable. */
|
|
400
|
+
now?: () => number;
|
|
401
|
+
}): Promise<SemanticSearchResult>;
|
|
402
|
+
/** Register the three semantic tools. No-op when search is not configured. */
|
|
403
|
+
export declare function registerSearchTools(server: McpServer, api: PaperlessAPI, context?: SearchContext): SearchContext;
|
|
404
|
+
/** A predicate over visible documents, from the search_passages filter args. */
|
|
405
|
+
export declare function buildDocumentFilter(filters: {
|
|
406
|
+
correspondent?: number;
|
|
407
|
+
document_type?: number;
|
|
408
|
+
tag?: number;
|
|
409
|
+
storage_path?: number;
|
|
410
|
+
created__date__gte?: string;
|
|
411
|
+
created__date__lte?: string;
|
|
412
|
+
}): ((doc: VisibleDocument) => boolean) | undefined;
|
|
413
|
+
/**
|
|
414
|
+
* Fetch the documents behind a set of ids, tolerating individual failures.
|
|
415
|
+
*
|
|
416
|
+
* One unfetchable document must not fail the whole search: the remaining hits
|
|
417
|
+
* are still real, and a search that returns four good passages and one gap is
|
|
418
|
+
* strictly better than an error. The gap is visible because the passage for
|
|
419
|
+
* that id is absent.
|
|
420
|
+
*/
|
|
421
|
+
export declare function loadVisibleDocuments(api: PaperlessAPI, ids: number[]): Promise<Map<number, VisibleDocument>>;
|
|
422
|
+
/**
|
|
423
|
+
* Ids per batched metadata request.
|
|
424
|
+
*
|
|
425
|
+
* A single call can legitimately want metadata for `limit * 4` distinct
|
|
426
|
+
* documents from EACH retriever, so 400 is reachable. One request per document
|
|
427
|
+
* is what this replaces; one request for 400 ids would be a 2 KB URL, so the
|
|
428
|
+
* list is cut into batches rather than trusted to be short.
|
|
429
|
+
*/
|
|
430
|
+
export declare const METADATA_BATCH = 100;
|
|
431
|
+
/**
|
|
432
|
+
* Fetch metadata for many documents in as few requests as possible.
|
|
433
|
+
*
|
|
434
|
+
* ⚠️ THIS NEVER FETCHES DOCUMENT CONTENT, and that is the point of the whole
|
|
435
|
+
* change. The previous path issued one `GET /documents/{id}/` per hit -- up to
|
|
436
|
+
* 80 of them for a 10-passage call -- and read every full body only to slice a
|
|
437
|
+
* window out of it. Measured on the live box for 40 documents: 2,398 ms and
|
|
438
|
+
* 2,162,306 bytes, against 58 ms and 6,520 bytes for the same metadata in one
|
|
439
|
+
* ranged call. The text the window needs was already in the index.
|
|
440
|
+
*
|
|
441
|
+
* `content` is therefore `null` on every returned document, which the
|
|
442
|
+
* `VisibleDocument` type already allows. Nothing on this path reads it.
|
|
443
|
+
*
|
|
444
|
+
* A batch that fails does NOT fail the search: the affected ids are simply
|
|
445
|
+
* absent from the map, which the caller already reports as
|
|
446
|
+
* `document_unavailable`. That is the same tolerance the per-id version had,
|
|
447
|
+
* kept deliberately so one probe failure cannot turn a search into an error.
|
|
448
|
+
*/
|
|
449
|
+
export declare function loadDocumentMetadata(api: PaperlessAPI, ids: number[]): Promise<Map<number, VisibleDocument>>;
|
|
450
|
+
/**
|
|
451
|
+
* Rebuild a document's text from the chunks the index already holds.
|
|
452
|
+
*
|
|
453
|
+
* The chunker emits `CHUNK_SIZE` characters at a stride of
|
|
454
|
+
* `CHUNK_SIZE - CHUNK_OVERLAP`, so consecutive chunks share exactly the
|
|
455
|
+
* trailing `CHUNK_OVERLAP` characters of the previous one. Splicing therefore
|
|
456
|
+
* has to drop that overlap -- but ONLY when it is really there, because a
|
|
457
|
+
* document shorter than one chunk is a single chunk and dropping 200 characters
|
|
458
|
+
* from it would delete text rather than deduplicate it.
|
|
459
|
+
*
|
|
460
|
+
* The metadata chunk at position 0 is skipped: it is a synthetic
|
|
461
|
+
* "Title: … Sender: …" line, not document text, and including it would put
|
|
462
|
+
* fabricated words in front of a passage and shift every offset by its length.
|
|
463
|
+
*
|
|
464
|
+
* ⚠️ The result is NORMALISED text -- `chunkText` collapses whitespace runs
|
|
465
|
+
* before slicing. Offsets derived from it are labelled `offset_space:
|
|
466
|
+
* "normalized"` for that reason.
|
|
467
|
+
*/
|
|
468
|
+
export declare function reconstructDocumentText(chunks: {
|
|
469
|
+
position: number;
|
|
470
|
+
kind: "metadata" | "content";
|
|
471
|
+
text: string;
|
|
472
|
+
}[]): string;
|
|
473
|
+
/**
|
|
474
|
+
* The production `loadText`: document text straight out of the open index.
|
|
475
|
+
*
|
|
476
|
+
* One prepared SELECT per document, against a local SQLite file, replacing one
|
|
477
|
+
* HTTPS GET per document that returned the whole body. Measured on the live
|
|
478
|
+
* index (read + splice): 2 ms for a typical document, 34–42 ms for the largest
|
|
479
|
+
* (3,606 chunks / 7.2 M characters).
|
|
480
|
+
*
|
|
481
|
+
* A document whose chunks cannot be read is simply absent from the map, which
|
|
482
|
+
* makes the caller fall back to `doc.content` — `null` on the batched-metadata
|
|
483
|
+
* path, hence an empty passage rather than an error. That is the same
|
|
484
|
+
* "one unusable document must not fail the search" tolerance the fetch path
|
|
485
|
+
* had, and `content_length: 0` makes it visible instead of silent.
|
|
486
|
+
*/
|
|
487
|
+
export declare function indexTextLoader(store: SearchIndex): (hits: Fused[]) => Map<number, string>;
|
|
488
|
+
/**
|
|
489
|
+
* Trim a result until its own serialised body fits the consumer's ceiling.
|
|
490
|
+
*
|
|
491
|
+
* Measured on the REAL bytes, not predicted from `window * passages`. That
|
|
492
|
+
* distinction is the whole mechanism: titles, ids and counts vary, and an
|
|
493
|
+
* arithmetic budget that is right on average is exactly the kind of bound that
|
|
494
|
+
* fails on the one call that matters. `budgetedLimit` keeps the work small;
|
|
495
|
+
* this is what makes the guarantee actual.
|
|
496
|
+
*/
|
|
497
|
+
export declare function fitResultToBudget(result: SemanticSearchResult): SemanticSearchResult;
|
|
498
|
+
export interface EmbeddingStatus {
|
|
499
|
+
documents: number;
|
|
500
|
+
chunks: number;
|
|
501
|
+
dimensions: number;
|
|
502
|
+
model: string;
|
|
503
|
+
prefix_mode: string;
|
|
504
|
+
last_sync: string | null;
|
|
505
|
+
pending: number | null;
|
|
506
|
+
store_bytes: number | null;
|
|
507
|
+
truncated_documents: number;
|
|
508
|
+
index_path: string;
|
|
509
|
+
open_error?: string;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Report the store's own view of itself.
|
|
513
|
+
*
|
|
514
|
+
* `pending` is computed against Paperless' document count, so it is `null`
|
|
515
|
+
* rather than `0` when that count cannot be read: a zero here would say "the
|
|
516
|
+
* index is current" on the strength of a failed request.
|
|
517
|
+
*/
|
|
518
|
+
export declare function embeddingStatus(context: SearchContext, api: PaperlessAPI): Promise<EmbeddingStatus>;
|