@retiform/paperless-mcp 2.3.6 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +9 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/manifest.test.d.ts +0 -1
- package/build/manifest.test.js +0 -66
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_WEBHOOK_DEBOUNCE_MS = exports.DEFAULT_WEBHOOK_HOST = exports.DEFAULT_WEBHOOK_PORT = exports.DEFAULT_SYNC_INTERVAL_MS = exports.DEFAULT_EMBEDDING_LOCAL_ONLY = exports.DEFAULT_SYNC_BATCH_SIZE = exports.DEFAULT_MAX_CHUNKS = exports.DEFAULT_PREFIX_MODE = exports.DEFAULT_EMBEDDING_MODEL = exports.DEFAULT_EMBEDDING_URL = exports.MAX_CHUNKS_LIMIT = exports.CHUNK_ROWID_STRIDE = exports.PREFIX_MODES = void 0;
|
|
4
|
+
exports.defaultIndexDir = defaultIndexDir;
|
|
5
|
+
exports.expandHome = expandHome;
|
|
6
|
+
exports.searchConfig = searchConfig;
|
|
7
|
+
exports.webhookRefusal = webhookRefusal;
|
|
8
|
+
exports.indexPath = indexPath;
|
|
9
|
+
const node_os_1 = require("node:os");
|
|
10
|
+
const node_path_1 = require("node:path");
|
|
11
|
+
/**
|
|
12
|
+
* Semantic-search configuration for the local SQLite index.
|
|
13
|
+
*
|
|
14
|
+
* Deliberate differences from the reference implementation this was ported
|
|
15
|
+
* from (`jaimemachado/paperless-mcp@50f7bee`):
|
|
16
|
+
*
|
|
17
|
+
* - A remote embedding provider IS reachable by configuration, because this
|
|
18
|
+
* package is published for general use and a third-party install must be
|
|
19
|
+
* able to point it at whichever provider it has. What protects a deployment
|
|
20
|
+
* that must not send text off the host is a flag it sets --
|
|
21
|
+
* `PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY=true` -- not a restriction compiled
|
|
22
|
+
* into the library on its behalf.
|
|
23
|
+
* - There is NO `useHttp` gate. That gate makes semantic search dead on
|
|
24
|
+
* arrival for any deployment that launches this package as a stdio child
|
|
25
|
+
* with no CLI arguments, because `--http` is then never true. Nothing about
|
|
26
|
+
* a local sqlite index requires HTTP transport.
|
|
27
|
+
* - There is NO `DATABASE_URL`. The store is a file path.
|
|
28
|
+
* - Every value here has a working default, so "all four or none" would be
|
|
29
|
+
* meaningless. The rule instead is: any search variable present enables the
|
|
30
|
+
* feature; an invalid value refuses loudly rather than silently defaulting.
|
|
31
|
+
*/
|
|
32
|
+
exports.PREFIX_MODES = ["nomic", "none", "e5"];
|
|
33
|
+
/**
|
|
34
|
+
* Rowid stride for the vector index: a chunk's rowid is
|
|
35
|
+
* `docId * CHUNK_ROWID_STRIDE + position`, so a document can hold at most
|
|
36
|
+
* `CHUNK_ROWID_STRIDE - 1` chunks before two documents collide.
|
|
37
|
+
*
|
|
38
|
+
* It lives in config.ts rather than store.ts so the CONFIG derives its own
|
|
39
|
+
* upper bound from the one number the store actually encodes with. A second
|
|
40
|
+
* literal would be a second source of truth, and the cost of the two
|
|
41
|
+
* disagreeing is paid mid-build: `PAPERLESS_MCP_MAX_CHUNKS=100000` was accepted
|
|
42
|
+
* by config and could only ever be rejected by `chunkRowid`, on document 2,000
|
|
43
|
+
* of a 58-minute run.
|
|
44
|
+
*
|
|
45
|
+
* A power of two, so a position decodes with a mask rather than a division that
|
|
46
|
+
* has to be exactly right.
|
|
47
|
+
*/
|
|
48
|
+
exports.CHUNK_ROWID_STRIDE = 8192;
|
|
49
|
+
/** The largest chunk count the stride can encode: positions 0 .. stride-1. */
|
|
50
|
+
exports.MAX_CHUNKS_LIMIT = exports.CHUNK_ROWID_STRIDE - 1;
|
|
51
|
+
exports.DEFAULT_EMBEDDING_URL = "http://127.0.0.1:11434";
|
|
52
|
+
exports.DEFAULT_EMBEDDING_MODEL = "nomic-embed-text";
|
|
53
|
+
exports.DEFAULT_PREFIX_MODE = "nomic";
|
|
54
|
+
exports.DEFAULT_MAX_CHUNKS = 4000;
|
|
55
|
+
exports.DEFAULT_SYNC_BATCH_SIZE = 16;
|
|
56
|
+
/**
|
|
57
|
+
* Off by default. An install that must be able to prove indexed text does not
|
|
58
|
+
* leave the host turns it on; one that is pointing at a hosted provider must
|
|
59
|
+
* not have to fight a policy it never asked for.
|
|
60
|
+
*/
|
|
61
|
+
exports.DEFAULT_EMBEDDING_LOCAL_ONLY = false;
|
|
62
|
+
/** 0 means "no background sync". Opting in is explicit, never a default. */
|
|
63
|
+
exports.DEFAULT_SYNC_INTERVAL_MS = 0;
|
|
64
|
+
/** 0 means "no change-notification receiver". Same reasoning as the interval. */
|
|
65
|
+
exports.DEFAULT_WEBHOOK_PORT = 0;
|
|
66
|
+
/**
|
|
67
|
+
* Loopback, and it has to be said out loud rather than inherited from Node.
|
|
68
|
+
*
|
|
69
|
+
* A listener that defaults to every interface is reachable from anywhere the
|
|
70
|
+
* host is, which for an endpoint that triggers embedding work is a decision the
|
|
71
|
+
* operator should have made. A containerised sender cannot reach loopback and
|
|
72
|
+
* must therefore set this -- which is the intended friction.
|
|
73
|
+
*/
|
|
74
|
+
exports.DEFAULT_WEBHOOK_HOST = "127.0.0.1";
|
|
75
|
+
/** Long enough to absorb a bulk edit's burst, short enough to feel immediate. */
|
|
76
|
+
exports.DEFAULT_WEBHOOK_DEBOUNCE_MS = 2000;
|
|
77
|
+
function defaultIndexDir() {
|
|
78
|
+
return (0, node_path_1.join)((0, node_os_1.homedir)(), ".local", "share", "paperless-mcp", "index");
|
|
79
|
+
}
|
|
80
|
+
const SPECS = {
|
|
81
|
+
embeddingUrl: { preferred: "PAPERLESS_MCP_EMBEDDING_URL", fallback: "EMBEDDING_URL" },
|
|
82
|
+
model: { preferred: "PAPERLESS_MCP_EMBEDDING_MODEL", fallback: "EMBEDDING_MODEL" },
|
|
83
|
+
prefixMode: { preferred: "PAPERLESS_MCP_EMBEDDING_PREFIX_MODE", fallback: "EMBEDDING_PREFIX_MODE" },
|
|
84
|
+
indexDir: { preferred: "PAPERLESS_MCP_INDEX_DIR" },
|
|
85
|
+
maxChunks: { preferred: "PAPERLESS_MCP_MAX_CHUNKS" },
|
|
86
|
+
syncBatchSize: { preferred: "PAPERLESS_MCP_SYNC_BATCH_SIZE" },
|
|
87
|
+
syncIntervalMs: { preferred: "PAPERLESS_MCP_SYNC_INTERVAL_MS", fallback: "SYNC_INTERVAL_MS" },
|
|
88
|
+
embeddingLocalOnly: { preferred: "PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY" },
|
|
89
|
+
embeddingApiKey: { preferred: "PAPERLESS_MCP_EMBEDDING_API_KEY", fallback: "EMBEDDING_API_KEY" },
|
|
90
|
+
webhookPort: { preferred: "PAPERLESS_MCP_WEBHOOK_PORT" },
|
|
91
|
+
webhookSecret: { preferred: "PAPERLESS_MCP_WEBHOOK_SECRET" },
|
|
92
|
+
webhookHost: { preferred: "PAPERLESS_MCP_WEBHOOK_HOST" },
|
|
93
|
+
webhookDebounceMs: { preferred: "PAPERLESS_MCP_WEBHOOK_DEBOUNCE_MS" },
|
|
94
|
+
};
|
|
95
|
+
function readVar(env, spec) {
|
|
96
|
+
var _a;
|
|
97
|
+
const names = spec.fallback ? [spec.preferred, spec.fallback] : [spec.preferred];
|
|
98
|
+
for (const name of names) {
|
|
99
|
+
const value = (_a = env[name]) === null || _a === void 0 ? void 0 : _a.trim();
|
|
100
|
+
if (value)
|
|
101
|
+
return { name, value };
|
|
102
|
+
}
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
function toNumber(where, raw, bounds) {
|
|
106
|
+
const parsed = Number(raw);
|
|
107
|
+
if (!Number.isFinite(parsed)) {
|
|
108
|
+
throw new Error(`${where} must be a number, not "${raw}".`);
|
|
109
|
+
}
|
|
110
|
+
if (bounds.integer && !Number.isInteger(parsed)) {
|
|
111
|
+
throw new Error(`${where} must be a whole number, not "${raw}".`);
|
|
112
|
+
}
|
|
113
|
+
if (parsed < bounds.min || (!bounds.clampMax && parsed > bounds.max)) {
|
|
114
|
+
throw new Error(`${where} must be between ${bounds.min} and ${bounds.max}, not "${raw}".`);
|
|
115
|
+
}
|
|
116
|
+
// Above the ceiling with clamping permitted: the caller asked for more than
|
|
117
|
+
// the store can encode, which is a ceiling rather than an error.
|
|
118
|
+
if (parsed > bounds.max)
|
|
119
|
+
return bounds.max;
|
|
120
|
+
return parsed;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Parse a boolean flag strictly.
|
|
124
|
+
*
|
|
125
|
+
* A flag that silently treats `"yes"`, `"1"` or a typo as false is worse than
|
|
126
|
+
* one that refuses: an operator who set a safety flag and got the default
|
|
127
|
+
* instead has been told the opposite of the truth by a green startup. So the
|
|
128
|
+
* accepted forms are enumerated and everything else throws.
|
|
129
|
+
*/
|
|
130
|
+
function toBoolean(where, raw) {
|
|
131
|
+
const lowered = raw.toLowerCase();
|
|
132
|
+
if (lowered === "true" || lowered === "1")
|
|
133
|
+
return true;
|
|
134
|
+
if (lowered === "false" || lowered === "0")
|
|
135
|
+
return false;
|
|
136
|
+
throw new Error(`${where} must be true or false, not "${raw}".`);
|
|
137
|
+
}
|
|
138
|
+
function toPrefixMode(where, raw) {
|
|
139
|
+
const modes = exports.PREFIX_MODES;
|
|
140
|
+
if (!modes.includes(raw)) {
|
|
141
|
+
throw new Error(`${where} must be one of ${exports.PREFIX_MODES.join(", ")}, not "${raw}".`);
|
|
142
|
+
}
|
|
143
|
+
return raw;
|
|
144
|
+
}
|
|
145
|
+
/** Resolve a leading `~` without pulling in a shell. */
|
|
146
|
+
function expandHome(input) {
|
|
147
|
+
if (input === "~")
|
|
148
|
+
return (0, node_os_1.homedir)();
|
|
149
|
+
if (input.startsWith("~/"))
|
|
150
|
+
return (0, node_path_1.join)((0, node_os_1.homedir)(), input.slice(2));
|
|
151
|
+
return input;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Returns the full configuration, or `undefined` when the feature is off.
|
|
155
|
+
*
|
|
156
|
+
* Off means: none of the search variables are set. Setting any one of them
|
|
157
|
+
* turns the feature on, and every set value is then validated strictly —
|
|
158
|
+
* a bad `EMBEDDING_PREFIX_MODE` throws rather than falling back to `nomic`,
|
|
159
|
+
* because a silent fallback here would silently change every stored vector.
|
|
160
|
+
*/
|
|
161
|
+
function searchConfig(env) {
|
|
162
|
+
var _a, _b, _c, _d;
|
|
163
|
+
const embeddingUrl = readVar(env, SPECS.embeddingUrl);
|
|
164
|
+
const model = readVar(env, SPECS.model);
|
|
165
|
+
const prefixMode = readVar(env, SPECS.prefixMode);
|
|
166
|
+
const indexDir = readVar(env, SPECS.indexDir);
|
|
167
|
+
const maxChunks = readVar(env, SPECS.maxChunks);
|
|
168
|
+
const syncBatchSize = readVar(env, SPECS.syncBatchSize);
|
|
169
|
+
const syncIntervalMs = readVar(env, SPECS.syncIntervalMs);
|
|
170
|
+
const embeddingLocalOnly = readVar(env, SPECS.embeddingLocalOnly);
|
|
171
|
+
const embeddingApiKey = readVar(env, SPECS.embeddingApiKey);
|
|
172
|
+
const webhookPort = readVar(env, SPECS.webhookPort);
|
|
173
|
+
const webhookSecret = readVar(env, SPECS.webhookSecret);
|
|
174
|
+
const webhookHost = readVar(env, SPECS.webhookHost);
|
|
175
|
+
const webhookDebounceMs = readVar(env, SPECS.webhookDebounceMs);
|
|
176
|
+
// ⚠️ `embeddingApiKey` deliberately does NOT appear in this list, and the
|
|
177
|
+
// asymmetry is the point: a credential is a modifier, not a request to turn
|
|
178
|
+
// the feature on. Counting it here would mean an environment that happens to
|
|
179
|
+
// export a provider key silently switches on semantic search for an install
|
|
180
|
+
// that never configured an index.
|
|
181
|
+
const anySet = [
|
|
182
|
+
embeddingUrl,
|
|
183
|
+
model,
|
|
184
|
+
prefixMode,
|
|
185
|
+
indexDir,
|
|
186
|
+
maxChunks,
|
|
187
|
+
syncBatchSize,
|
|
188
|
+
syncIntervalMs,
|
|
189
|
+
embeddingLocalOnly,
|
|
190
|
+
// ⚠️ ONLY the PORT turns the feature on. The secret, the host and the
|
|
191
|
+
// debounce are modifiers: an environment that happens to export a secret,
|
|
192
|
+
// or a bind address, has not asked for a listener. Counting a bind address
|
|
193
|
+
// here would mean a deployment that widened it and then removed the port
|
|
194
|
+
// silently kept a socket open.
|
|
195
|
+
webhookPort,
|
|
196
|
+
].some(Boolean);
|
|
197
|
+
if (!anySet)
|
|
198
|
+
return undefined;
|
|
199
|
+
return {
|
|
200
|
+
embeddingUrl: (_a = embeddingUrl === null || embeddingUrl === void 0 ? void 0 : embeddingUrl.value) !== null && _a !== void 0 ? _a : exports.DEFAULT_EMBEDDING_URL,
|
|
201
|
+
model: (_b = model === null || model === void 0 ? void 0 : model.value) !== null && _b !== void 0 ? _b : exports.DEFAULT_EMBEDDING_MODEL,
|
|
202
|
+
prefixMode: prefixMode
|
|
203
|
+
? toPrefixMode(prefixMode.name, prefixMode.value)
|
|
204
|
+
: exports.DEFAULT_PREFIX_MODE,
|
|
205
|
+
indexDir: expandHome((_c = indexDir === null || indexDir === void 0 ? void 0 : indexDir.value) !== null && _c !== void 0 ? _c : defaultIndexDir()),
|
|
206
|
+
maxChunks: maxChunks
|
|
207
|
+
? toNumber(maxChunks.name, maxChunks.value, {
|
|
208
|
+
min: 1,
|
|
209
|
+
max: exports.MAX_CHUNKS_LIMIT,
|
|
210
|
+
integer: true,
|
|
211
|
+
clampMax: true,
|
|
212
|
+
})
|
|
213
|
+
: exports.DEFAULT_MAX_CHUNKS,
|
|
214
|
+
syncBatchSize: syncBatchSize
|
|
215
|
+
? toNumber(syncBatchSize.name, syncBatchSize.value, { min: 1, max: 256, integer: true })
|
|
216
|
+
: exports.DEFAULT_SYNC_BATCH_SIZE,
|
|
217
|
+
syncIntervalMs: syncIntervalMs
|
|
218
|
+
? toNumber(syncIntervalMs.name, syncIntervalMs.value, { min: 0, max: 86400000, integer: true })
|
|
219
|
+
: exports.DEFAULT_SYNC_INTERVAL_MS,
|
|
220
|
+
embeddingLocalOnly: embeddingLocalOnly
|
|
221
|
+
? toBoolean(embeddingLocalOnly.name, embeddingLocalOnly.value)
|
|
222
|
+
: exports.DEFAULT_EMBEDDING_LOCAL_ONLY,
|
|
223
|
+
embeddingApiKey: embeddingApiKey === null || embeddingApiKey === void 0 ? void 0 : embeddingApiKey.value,
|
|
224
|
+
webhookPort: webhookPort
|
|
225
|
+
? toNumber(webhookPort.name, webhookPort.value, { min: 0, max: 65535, integer: true })
|
|
226
|
+
: exports.DEFAULT_WEBHOOK_PORT,
|
|
227
|
+
webhookSecret: webhookSecret === null || webhookSecret === void 0 ? void 0 : webhookSecret.value,
|
|
228
|
+
webhookHost: (_d = webhookHost === null || webhookHost === void 0 ? void 0 : webhookHost.value) !== null && _d !== void 0 ? _d : exports.DEFAULT_WEBHOOK_HOST,
|
|
229
|
+
webhookDebounceMs: webhookDebounceMs
|
|
230
|
+
? toNumber(webhookDebounceMs.name, webhookDebounceMs.value, {
|
|
231
|
+
min: 0,
|
|
232
|
+
max: 600000,
|
|
233
|
+
integer: true,
|
|
234
|
+
})
|
|
235
|
+
: exports.DEFAULT_WEBHOOK_DEBOUNCE_MS,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Whether the receiver may start, and why not when it may not.
|
|
240
|
+
*
|
|
241
|
+
* ⚠️ A PORT WITHOUT A SECRET IS A CONFIGURATION ERROR, NOT A WARNING. The
|
|
242
|
+
* endpoint causes embedding work; an unauthenticated trigger for that is a
|
|
243
|
+
* resource-abuse vector, and a log line nobody reads is not a mitigation. So
|
|
244
|
+
* this returns a reason rather than a boolean -- the caller decides whether to
|
|
245
|
+
* log and continue (a background sync failing to start must not take down tool
|
|
246
|
+
* registration) or to fail, and either way the reason reaches the operator
|
|
247
|
+
* verbatim instead of being reconstructed.
|
|
248
|
+
*/
|
|
249
|
+
function webhookRefusal(config) {
|
|
250
|
+
if (config.webhookPort === 0)
|
|
251
|
+
return undefined;
|
|
252
|
+
if (!config.webhookSecret) {
|
|
253
|
+
return ("PAPERLESS_MCP_WEBHOOK_PORT is set but PAPERLESS_MCP_WEBHOOK_SECRET is " +
|
|
254
|
+
"not. The receiver refuses to start without a shared secret: it triggers " +
|
|
255
|
+
"embedding work, so an unauthenticated endpoint is a resource-abuse " +
|
|
256
|
+
"vector. Set the secret to a value the Paperless workflow will send, or " +
|
|
257
|
+
"unset the port to keep the receiver off.");
|
|
258
|
+
}
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
/** The index file inside the configured directory. */
|
|
262
|
+
function indexPath(config) {
|
|
263
|
+
return (0, node_path_1.join)(config.indexDir, "index.db");
|
|
264
|
+
}
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An HTTP embeddings client for any OpenAI-compatible `/embeddings` endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Two request/response shapes are spoken, and the caller does not have to say
|
|
5
|
+
* which: Ollama's native `/api/embed` answers `{embeddings: [[...]]}` and its
|
|
6
|
+
* OpenAI-compatible `/v1/embeddings` answers `{data: [{embedding: [...]}]}`.
|
|
7
|
+
* The reply is sniffed. A package that made the user declare which of two
|
|
8
|
+
* well-known JSON shapes it was about to receive would be adding a
|
|
9
|
+
* configuration failure mode in exchange for nothing -- and both shapes were
|
|
10
|
+
* measured against the same server returning identical vectors.
|
|
11
|
+
*
|
|
12
|
+
* ── Where the endpoint comes from ──────────────────────────────────────────
|
|
13
|
+
* The configured URL is used verbatim when it carries a path, so
|
|
14
|
+
* `https://api.openai.com/v1/embeddings` and `http://host:8080/embed` both mean
|
|
15
|
+
* what they say. An origin-only URL falls back to Ollama's default
|
|
16
|
+
* `/api/embed`, which is what makes the shipped default work with no path.
|
|
17
|
+
*
|
|
18
|
+
* ── Local-only mode ───────────────────────────────────────────────────────
|
|
19
|
+
* `localOnly: true` restricts the endpoint to a literal loopback or RFC1918
|
|
20
|
+
* address and refuses DNS names, because a name can resolve anywhere and the
|
|
21
|
+
* process cannot prove where. That is a capability any security-conscious
|
|
22
|
+
* deployment may want, not a policy belonging to one of them: it is off by
|
|
23
|
+
* default, and an install that needs it turns it on.
|
|
24
|
+
*
|
|
25
|
+
* `redirect: "error"` is always on. It is generic hygiene rather than policy:
|
|
26
|
+
* a 307 would let any endpoint -- local or not -- forward a request body to a
|
|
27
|
+
* host the configuration never named.
|
|
28
|
+
*
|
|
29
|
+
* Ported from `jaimemachado/paperless-mcp@50f7bee:src/search/embedder.ts`, with
|
|
30
|
+
* these changes forced by our substrate:
|
|
31
|
+
* 1. The `../metrics` and `../tracing` imports are stripped -- neither module
|
|
32
|
+
* exists in our fork, and porting them would mean adding OpenTelemetry and
|
|
33
|
+
* prom-client for no benefit here.
|
|
34
|
+
* 2. The prefix handling is fixed rather than ported. See `PREFIXES`.
|
|
35
|
+
* 3. The response parser accepts both the native and the OpenAI shape.
|
|
36
|
+
* 4. The endpoint restriction is a configuration value, not a hard gate.
|
|
37
|
+
*/
|
|
38
|
+
import { PrefixMode } from "./config";
|
|
39
|
+
/**
|
|
40
|
+
* Default number of texts per embedding request.
|
|
41
|
+
*
|
|
42
|
+
* Also the module default for `batchSize`, so a caller that omits the option
|
|
43
|
+
* behaves exactly as it did before the option existed.
|
|
44
|
+
*/
|
|
45
|
+
export declare const DEFAULT_BATCH_SIZE = 16;
|
|
46
|
+
export declare class EmbeddingUnavailableError extends Error {
|
|
47
|
+
constructor(message: string);
|
|
48
|
+
}
|
|
49
|
+
/** Thrown when the configured prefix mode cannot be used with the stored index. */
|
|
50
|
+
export declare class PrefixModeMismatchError extends Error {
|
|
51
|
+
constructor(message: string);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Raised when the server rejects a request because the INPUT IS TOO LONG for
|
|
55
|
+
* the model's context window.
|
|
56
|
+
*
|
|
57
|
+
* It extends `EmbeddingUnavailableError` deliberately. A caller that has no
|
|
58
|
+
* specific handling -- a query, or a fragment that is already at the floor --
|
|
59
|
+
* must abort rather than write an index that silently omits a document. The
|
|
60
|
+
* subclass only lets the document path recognise the case and split.
|
|
61
|
+
*
|
|
62
|
+
* ⚠️ THIS IS A TOKEN BUDGET, NOT A CHARACTER BUDGET, AND THE TWO DIVERGE
|
|
63
|
+
* WILDLY. Measured on this corpus: a run of ellipsis (dot leaders in a table of
|
|
64
|
+
* contents) tokenises at roughly 1.5-2 tokens per character, so 2,000
|
|
65
|
+
* characters blows a 2,048-token window, while 2,000 characters of prose uses
|
|
66
|
+
* about 500 and passes with room to spare. `truncate: true` was measured and
|
|
67
|
+
* does NOT fix it. The only fix that works for any chunk is to make the request
|
|
68
|
+
* smaller, which is what the split path does.
|
|
69
|
+
*/
|
|
70
|
+
export declare class EmbeddingContextLengthError extends EmbeddingUnavailableError {
|
|
71
|
+
/** Length in characters of the longest input that was refused. */
|
|
72
|
+
readonly longestChars: number;
|
|
73
|
+
constructor(message: string, longestChars: number);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Is this reply the server saying "your input is too long"?
|
|
77
|
+
*
|
|
78
|
+
* Detected on the SERVER'S OWN TEXT, and only on a 400. Anything else -- a 500,
|
|
79
|
+
* a 401, a 400 whose body says something different -- stays a plain
|
|
80
|
+
* `EmbeddingUnavailableError`, because splitting a request in response to an
|
|
81
|
+
* unrelated 400 would turn one visible failure into a hundred invisible ones.
|
|
82
|
+
*/
|
|
83
|
+
export declare function isContextLengthRejection(status: number, serverText: string): boolean;
|
|
84
|
+
export interface Embedder {
|
|
85
|
+
readonly model: string;
|
|
86
|
+
readonly prefixMode: PrefixMode;
|
|
87
|
+
readonly dimensions: number;
|
|
88
|
+
embedDocuments(texts: string[]): Promise<number[][]>;
|
|
89
|
+
embedQuery(query: string): Promise<number[]>;
|
|
90
|
+
/**
|
|
91
|
+
* Split events observed during the most recent `embedDocuments` call.
|
|
92
|
+
*
|
|
93
|
+
* Declared on the INTERFACE, not only on the implementation, so the sync can
|
|
94
|
+
* report that the fallback fired without knowing which embedder it holds. A
|
|
95
|
+
* fake that omits it is fine -- the field is optional and the sync reads it
|
|
96
|
+
* defensively -- which keeps every existing test double valid.
|
|
97
|
+
*
|
|
98
|
+
* Counts and lengths only. A caller may log this, so it must never be able to
|
|
99
|
+
* log document text through it.
|
|
100
|
+
*/
|
|
101
|
+
readonly lastSplits?: readonly ContextSplitEvent[];
|
|
102
|
+
/**
|
|
103
|
+
* Cheap liveness check that must NOT be assumed equivalent to "constructed".
|
|
104
|
+
*
|
|
105
|
+
* The constructor validates the endpoint, so a constructed embedder proves
|
|
106
|
+
* nothing about whether anything is listening. Callers that are about to do
|
|
107
|
+
* something irreversible -- `sync_embeddings` with `force:true` deletes the
|
|
108
|
+
* entire index before it re-embeds -- must probe first, or an outage on a
|
|
109
|
+
* long rebuild leaves no index at all.
|
|
110
|
+
*
|
|
111
|
+
* Optional so existing fakes stay valid; callers fall back to `embedQuery`.
|
|
112
|
+
*/
|
|
113
|
+
probe?(): Promise<void>;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* The prefix each embedding model family expects.
|
|
117
|
+
*
|
|
118
|
+
* This is the bug fix, not a port. The reference implementation hardcoded the
|
|
119
|
+
* E5 instruction template onto a nomic model. Measured on this box at n=5,
|
|
120
|
+
* nomic-prefixed and plain both put the known-good document at rank 1 five
|
|
121
|
+
* times out of five, while the E5 template managed four of five with a negative
|
|
122
|
+
* margin. That is a signal rather than a benchmark, but it points the same way
|
|
123
|
+
* the model cards do, and the cost of being wrong is silently worse retrieval.
|
|
124
|
+
*
|
|
125
|
+
* `none` is a supported mode because a future model may want no prefix at all,
|
|
126
|
+
* and because refusing to store the choice would make the choice unprovable.
|
|
127
|
+
*/
|
|
128
|
+
export declare const PREFIXES: Record<PrefixMode, {
|
|
129
|
+
document: string;
|
|
130
|
+
query: string;
|
|
131
|
+
}>;
|
|
132
|
+
/**
|
|
133
|
+
* The prefix is baked into every stored vector, so a query vector built with a
|
|
134
|
+
* different prefix is silently worse -- it still returns results, they are just
|
|
135
|
+
* the wrong ones. A mismatch must be refused loudly instead.
|
|
136
|
+
*/
|
|
137
|
+
export declare function assertPrefixModeMatches(configured: PrefixMode, stored: PrefixMode): void;
|
|
138
|
+
/**
|
|
139
|
+
* Validate an embedding endpoint URL.
|
|
140
|
+
*
|
|
141
|
+
* Two checks are always applied, because they are true of every HTTP client
|
|
142
|
+
* rather than of one deployment:
|
|
143
|
+
* - the URL must parse, and
|
|
144
|
+
* - the scheme must be http: or https:. Anything else is not an endpoint this
|
|
145
|
+
* client can speak to, and a `file:` URL reaching an outbound fetch is a
|
|
146
|
+
* defect in any package.
|
|
147
|
+
*
|
|
148
|
+
* `localOnly` adds the third check. It is OFF by default and it exists as a
|
|
149
|
+
* capability, not as an opinion:
|
|
150
|
+
*
|
|
151
|
+
* A DNS NAME IS REFUSED, because a name can resolve anywhere and this process
|
|
152
|
+
* cannot prove where it resolves to at startup -- a `localhost` that works
|
|
153
|
+
* today and a hosts-file edit tomorrow are indistinguishable from inside this
|
|
154
|
+
* function. Requiring a literal address makes the check answerable by reading
|
|
155
|
+
* the configuration.
|
|
156
|
+
*
|
|
157
|
+
* The shape of the local-only check is deliberate, and a naive one would be
|
|
158
|
+
* wrong in two different ways:
|
|
159
|
+
* - Asserting `host === "127.0.0.1"` would reject an Ollama that binds
|
|
160
|
+
* `0.0.0.0` (`OLLAMA_HOST=0.0.0.0`), which is then reached at the LAN
|
|
161
|
+
* address as well as at loopback.
|
|
162
|
+
* - Accepting `0.0.0.0` as a connect target would be wrong the other way: it
|
|
163
|
+
* is the unspecified address, its meaning is platform-dependent, and no
|
|
164
|
+
* legitimate deployment needs to dial it.
|
|
165
|
+
* An allowlist of literal private addresses handles both.
|
|
166
|
+
*/
|
|
167
|
+
export declare function parseEmbeddingEndpoint(rawUrl: string, opts?: {
|
|
168
|
+
localOnly?: boolean;
|
|
169
|
+
}): URL;
|
|
170
|
+
/**
|
|
171
|
+
* Where the request actually goes.
|
|
172
|
+
*
|
|
173
|
+
* A configured path is honoured verbatim: a caller pointing at
|
|
174
|
+
* `https://api.openai.com/v1` gets `https://api.openai.com/v1/embeddings`, and
|
|
175
|
+
* one pointing at `https://host/v1/embeddings` gets exactly that, with no
|
|
176
|
+
* double suffix. An origin with no path falls back to Ollama's default route.
|
|
177
|
+
*
|
|
178
|
+
* Trailing slashes are stripped before the suffix is appended so
|
|
179
|
+
* `http://host:11434/` cannot produce `//api/embed`.
|
|
180
|
+
*/
|
|
181
|
+
export declare function resolveEmbeddingEndpoint(base: URL): string;
|
|
182
|
+
/**
|
|
183
|
+
* Read a vector list out of either reply shape.
|
|
184
|
+
*
|
|
185
|
+
* Returns `undefined` when the body is neither, so the caller can raise one
|
|
186
|
+
* error with one message rather than guessing which shape was attempted.
|
|
187
|
+
*/
|
|
188
|
+
export declare function embeddingsFromBody(body: unknown): number[][] | undefined;
|
|
189
|
+
/**
|
|
190
|
+
* Below this many characters a refused fragment is NOT split further.
|
|
191
|
+
*
|
|
192
|
+
* A fragment this short is refused only if the model window is far smaller than
|
|
193
|
+
* the one this index was built against, and splitting again would produce
|
|
194
|
+
* non-words while looping. Throwing names the fragment length; silently
|
|
195
|
+
* dropping it would leave an index that claims a document it does not hold.
|
|
196
|
+
*/
|
|
197
|
+
export declare const MIN_SPLITTABLE_CHARS = 200;
|
|
198
|
+
/**
|
|
199
|
+
* Hard ceiling on recursion, so a pathological input cannot spin.
|
|
200
|
+
*
|
|
201
|
+
* Halving a 16-text batch down to singles costs 4 levels; halving a 2,000
|
|
202
|
+
* character chunk down to the floor costs 4 more. Twelve is generous and still
|
|
203
|
+
* finite, which is the property that matters.
|
|
204
|
+
*/
|
|
205
|
+
export declare const MAX_SPLIT_DEPTH = 12;
|
|
206
|
+
/** One split event. Counts and lengths only -- never document text. */
|
|
207
|
+
export interface ContextSplitEvent {
|
|
208
|
+
/** `batch` split an over-long request into halves; `text` split one text. */
|
|
209
|
+
readonly reason: "batch" | "text";
|
|
210
|
+
/** Texts in the request that was refused. */
|
|
211
|
+
readonly texts: number;
|
|
212
|
+
/** Characters in the refused text; 0 for a batch split. */
|
|
213
|
+
readonly chars: number;
|
|
214
|
+
readonly depth: number;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Every grapheme-cluster boundary in `text`, as character offsets.
|
|
218
|
+
*
|
|
219
|
+
* ⚠️ THIS IS LOAD-BEARING FOR THIS CORPUS, NOT A REFINEMENT. It is
|
|
220
|
+
* Tibetan-heavy, and a Tibetan syllable stacks combining marks (category Mn) on
|
|
221
|
+
* a base consonant: U+0FB3 U+0F74 is one syllable, two code points, and a slice
|
|
222
|
+
* between them produces a base plus a stranded vowel sign -- text that is not a
|
|
223
|
+
* word, embedded as if it were. `Intl.Segmenter` is present in every Node this
|
|
224
|
+
* package supports (measured on v24.21.0), so the Array.from fallback is
|
|
225
|
+
* belt-and-braces: it keeps surrogate pairs intact, which is the one thing a
|
|
226
|
+
* naive slice gets catastrophically wrong, and leaves combining marks to the
|
|
227
|
+
* segmenter that is actually available.
|
|
228
|
+
*/
|
|
229
|
+
export declare function graphemeBoundaries(text: string): number[];
|
|
230
|
+
/**
|
|
231
|
+
* Cut a text in two so each half can be embedded on its own.
|
|
232
|
+
*
|
|
233
|
+
* A whitespace boundary nearest the midpoint is preferred, because a split
|
|
234
|
+
* between words produces two fragments that still mean something. When the text
|
|
235
|
+
* has no whitespace at all -- a run of dot leaders is exactly that -- the
|
|
236
|
+
* nearest GRAPHEME boundary is used instead. That fallback is what keeps the
|
|
237
|
+
* "never mid-codepoint" rule true for input that offers no word break.
|
|
238
|
+
*
|
|
239
|
+
* Returns undefined when no split makes progress, so the caller throws rather
|
|
240
|
+
* than looping on a text it cannot shorten.
|
|
241
|
+
*/
|
|
242
|
+
export declare function splitForEmbedding(text: string): [string, string] | undefined;
|
|
243
|
+
/**
|
|
244
|
+
* The direction of the mean of two vectors, re-normalised to unit length.
|
|
245
|
+
*
|
|
246
|
+
* A cosine-indexed store compares DIRECTION, so the mean of two halves of one
|
|
247
|
+
* chunk carries the whole of what the store can use while keeping one vector
|
|
248
|
+
* per chunk -- and therefore one row, and an unchanged `chunkCount` contract.
|
|
249
|
+
* The mean is not unit length even when both inputs are, so it is re-normalised.
|
|
250
|
+
*
|
|
251
|
+
* Two nearly opposed halves would average to a vector with no direction. That
|
|
252
|
+
* is reported rather than stored: a zero vector matches nothing and would sit
|
|
253
|
+
* in the index as a chunk that can never be retrieved.
|
|
254
|
+
*/
|
|
255
|
+
export declare function renormalisedMean(a: number[], b: number[]): number[];
|
|
256
|
+
export interface EmbeddingClientOptions {
|
|
257
|
+
/** Base URL, or a full endpoint URL. See `resolveEmbeddingEndpoint`. */
|
|
258
|
+
url: string;
|
|
259
|
+
model: string;
|
|
260
|
+
prefixMode: PrefixMode;
|
|
261
|
+
/** Expected vector width. Defaults to 768. */
|
|
262
|
+
dimensions?: number;
|
|
263
|
+
/**
|
|
264
|
+
* Sent as `Authorization: Bearer <key>` when present.
|
|
265
|
+
*
|
|
266
|
+
* Never logged, never echoed in an error, never included in any object this
|
|
267
|
+
* module returns. A credential that appears in a diagnostic string outlives
|
|
268
|
+
* the request that used it.
|
|
269
|
+
*/
|
|
270
|
+
apiKey?: string;
|
|
271
|
+
/**
|
|
272
|
+
* Restrict the endpoint to a literal loopback or RFC1918 address.
|
|
273
|
+
*
|
|
274
|
+
* Defaults to false. `true` refuses DNS names and public addresses, which is
|
|
275
|
+
* what a deployment that must be able to prove text does not leave the host
|
|
276
|
+
* sets to satisfy itself.
|
|
277
|
+
*/
|
|
278
|
+
localOnly?: boolean;
|
|
279
|
+
/** Texts per request. Defaults to `DEFAULT_BATCH_SIZE`. */
|
|
280
|
+
batchSize?: number;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* An HTTP embeddings client for any OpenAI-compatible `/embeddings` endpoint.
|
|
284
|
+
*
|
|
285
|
+
* The constructor validates the endpoint and the prefix mode, and throws on
|
|
286
|
+
* either failure. Nothing is deferred to first use: a misconfigured index
|
|
287
|
+
* should fail at startup, before it has written a single vector.
|
|
288
|
+
*/
|
|
289
|
+
export declare class OllamaEmbedder implements Embedder {
|
|
290
|
+
#private;
|
|
291
|
+
readonly model: string;
|
|
292
|
+
readonly prefixMode: PrefixMode;
|
|
293
|
+
readonly dimensions: number;
|
|
294
|
+
private readonly endpointUrl;
|
|
295
|
+
private readonly batchSize;
|
|
296
|
+
constructor(opts: EmbeddingClientOptions);
|
|
297
|
+
/**
|
|
298
|
+
* Split events observed during the most recent `embedDocuments` call.
|
|
299
|
+
*
|
|
300
|
+
* Counts and lengths only: a caller may log this, so it must never be able to
|
|
301
|
+
* log document text through it. The sync reads it to report how many chunks
|
|
302
|
+
* needed the fallback, which is the only way to know the split path is
|
|
303
|
+
* actually doing work rather than merely existing.
|
|
304
|
+
*/
|
|
305
|
+
get lastSplits(): readonly ContextSplitEvent[];
|
|
306
|
+
private splits;
|
|
307
|
+
embedDocuments(texts: string[]): Promise<number[][]>;
|
|
308
|
+
/**
|
|
309
|
+
* Embed a batch, halving it on a context-length rejection.
|
|
310
|
+
*
|
|
311
|
+
* The order of the returned vectors is the order of `batch`, which is what
|
|
312
|
+
* makes this a drop-in replacement for the single `request` call it replaces:
|
|
313
|
+
* the caller zips vectors to texts by index and never learns a split happened.
|
|
314
|
+
*
|
|
315
|
+
* A batch of ONE that is still refused moves to `embedOne`, which splits the
|
|
316
|
+
* TEXT. The two levels exist because they fail differently: a multi-text
|
|
317
|
+
* batch can succeed simply by being asked as two requests, while a single
|
|
318
|
+
* over-long text has to be shortened or it will never embed.
|
|
319
|
+
*/
|
|
320
|
+
private embedBatch;
|
|
321
|
+
/**
|
|
322
|
+
* Embed a single text, splitting it when the server says it is too long.
|
|
323
|
+
*
|
|
324
|
+
* The halves are embedded independently and combined into ONE vector, so the
|
|
325
|
+
* chunk stays one row and the sync's `chunkCount` contract -- one vector per
|
|
326
|
+
* chunk -- is untouched. Anything else would change how many rows a document
|
|
327
|
+
* has, and the store rejects a count that disagrees with the array.
|
|
328
|
+
*/
|
|
329
|
+
private embedOne;
|
|
330
|
+
embedQuery(query: string): Promise<number[]>;
|
|
331
|
+
/**
|
|
332
|
+
* Embed one throwaway string and discard it.
|
|
333
|
+
*
|
|
334
|
+
* Deliberately a real embedding rather than a GET to the server's root: a
|
|
335
|
+
* server that is up but has not pulled the model answers GET happily and then
|
|
336
|
+
* fails on the first real request. The whole point of the probe is to fail
|
|
337
|
+
* BEFORE an irreversible step, so it has to exercise the thing that will
|
|
338
|
+
* actually run -- the same model, the same prefix, the same endpoint.
|
|
339
|
+
*
|
|
340
|
+
* The string is a constant, not document text, so nothing real is placed on
|
|
341
|
+
* the wire for a liveness check.
|
|
342
|
+
*/
|
|
343
|
+
probe(): Promise<void>;
|
|
344
|
+
private headers;
|
|
345
|
+
private request;
|
|
346
|
+
}
|