@retiform/paperless-mcp 2.3.6 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +301 -4
- package/build/api/PaperlessAPI.js +1 -6
- package/build/search/build-outcome.d.ts +28 -0
- package/build/search/build-outcome.js +31 -0
- package/build/search/chunk.d.ts +16 -0
- package/build/search/chunk.js +64 -0
- package/build/search/config.d.ts +120 -0
- package/build/search/config.js +264 -0
- package/build/search/embedder.d.ts +346 -0
- package/build/search/embedder.js +734 -0
- package/build/search/rank.d.ts +11 -0
- package/build/search/rank.js +87 -0
- package/build/search/store.d.ts +406 -0
- package/build/search/store.js +847 -0
- package/build/search/sync.d.ts +204 -0
- package/build/search/sync.js +819 -0
- package/build/search/webhook.d.ts +193 -0
- package/build/search/webhook.js +445 -0
- package/build/server.d.ts +29 -0
- package/build/server.js +77 -0
- package/build/tools/documents.d.ts +13 -3
- package/build/tools/documents.js +23 -6
- package/build/tools/search.d.ts +518 -0
- package/build/tools/search.js +1199 -0
- package/build/tools/utils/middlewares.d.ts +5 -0
- package/build/tools/utils/middlewares.js +282 -9
- package/package.json +9 -3
- package/paperless-mcp.dxt +0 -0
- package/build/api/documentEnhancer.test.d.ts +0 -1
- package/build/api/documentEnhancer.test.js +0 -80
- package/build/manifest.test.d.ts +0 -1
- package/build/manifest.test.js +0 -66
- package/build/resources/documents.test.d.ts +0 -1
- package/build/resources/documents.test.js +0 -124
- package/build/server.test.d.ts +0 -1
- package/build/server.test.js +0 -54
- package/build/tools/boundedContent.test.d.ts +0 -1
- package/build/tools/boundedContent.test.js +0 -565
- package/build/tools/documents.test.d.ts +0 -1
- package/build/tools/documents.test.js +0 -658
- package/build/tools/mail.test.d.ts +0 -1
- package/build/tools/mail.test.js +0 -212
- package/build/tools/notes.test.d.ts +0 -1
- package/build/tools/notes.test.js +0 -177
- package/build/tools/schemaCompat.test.d.ts +0 -1
- package/build/tools/schemaCompat.test.js +0 -59
- package/build/tools/utils/monetary.test.d.ts +0 -1
- package/build/tools/utils/monetary.test.js +0 -25
- package/build/tools/utils/resourceUri.test.d.ts +0 -1
- package/build/tools/utils/resourceUri.test.js +0 -50
- package/build/tools/utils/selectFields.test.d.ts +0 -1
- package/build/tools/utils/selectFields.test.js +0 -164
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* An HTTP embeddings client for any OpenAI-compatible `/embeddings` endpoint.
|
|
4
|
+
*
|
|
5
|
+
* Two request/response shapes are spoken, and the caller does not have to say
|
|
6
|
+
* which: Ollama's native `/api/embed` answers `{embeddings: [[...]]}` and its
|
|
7
|
+
* OpenAI-compatible `/v1/embeddings` answers `{data: [{embedding: [...]}]}`.
|
|
8
|
+
* The reply is sniffed. A package that made the user declare which of two
|
|
9
|
+
* well-known JSON shapes it was about to receive would be adding a
|
|
10
|
+
* configuration failure mode in exchange for nothing -- and both shapes were
|
|
11
|
+
* measured against the same server returning identical vectors.
|
|
12
|
+
*
|
|
13
|
+
* ── Where the endpoint comes from ──────────────────────────────────────────
|
|
14
|
+
* The configured URL is used verbatim when it carries a path, so
|
|
15
|
+
* `https://api.openai.com/v1/embeddings` and `http://host:8080/embed` both mean
|
|
16
|
+
* what they say. An origin-only URL falls back to Ollama's default
|
|
17
|
+
* `/api/embed`, which is what makes the shipped default work with no path.
|
|
18
|
+
*
|
|
19
|
+
* ── Local-only mode ───────────────────────────────────────────────────────
|
|
20
|
+
* `localOnly: true` restricts the endpoint to a literal loopback or RFC1918
|
|
21
|
+
* address and refuses DNS names, because a name can resolve anywhere and the
|
|
22
|
+
* process cannot prove where. That is a capability any security-conscious
|
|
23
|
+
* deployment may want, not a policy belonging to one of them: it is off by
|
|
24
|
+
* default, and an install that needs it turns it on.
|
|
25
|
+
*
|
|
26
|
+
* `redirect: "error"` is always on. It is generic hygiene rather than policy:
|
|
27
|
+
* a 307 would let any endpoint -- local or not -- forward a request body to a
|
|
28
|
+
* host the configuration never named.
|
|
29
|
+
*
|
|
30
|
+
* Ported from `jaimemachado/paperless-mcp@50f7bee:src/search/embedder.ts`, with
|
|
31
|
+
* these changes forced by our substrate:
|
|
32
|
+
* 1. The `../metrics` and `../tracing` imports are stripped -- neither module
|
|
33
|
+
* exists in our fork, and porting them would mean adding OpenTelemetry and
|
|
34
|
+
* prom-client for no benefit here.
|
|
35
|
+
* 2. The prefix handling is fixed rather than ported. See `PREFIXES`.
|
|
36
|
+
* 3. The response parser accepts both the native and the OpenAI shape.
|
|
37
|
+
* 4. The endpoint restriction is a configuration value, not a hard gate.
|
|
38
|
+
*/
|
|
39
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
40
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
41
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
42
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
43
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
44
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
45
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
49
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
50
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
51
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
52
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
53
|
+
};
|
|
54
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
55
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
56
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
57
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
58
|
+
};
|
|
59
|
+
var _OllamaEmbedder_apiKey;
|
|
60
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
61
|
+
exports.OllamaEmbedder = exports.MAX_SPLIT_DEPTH = exports.MIN_SPLITTABLE_CHARS = exports.PREFIXES = exports.EmbeddingContextLengthError = exports.PrefixModeMismatchError = exports.EmbeddingUnavailableError = exports.DEFAULT_BATCH_SIZE = void 0;
|
|
62
|
+
exports.isContextLengthRejection = isContextLengthRejection;
|
|
63
|
+
exports.assertPrefixModeMatches = assertPrefixModeMatches;
|
|
64
|
+
exports.parseEmbeddingEndpoint = parseEmbeddingEndpoint;
|
|
65
|
+
exports.resolveEmbeddingEndpoint = resolveEmbeddingEndpoint;
|
|
66
|
+
exports.embeddingsFromBody = embeddingsFromBody;
|
|
67
|
+
exports.graphemeBoundaries = graphemeBoundaries;
|
|
68
|
+
exports.splitForEmbedding = splitForEmbedding;
|
|
69
|
+
exports.renormalisedMean = renormalisedMean;
|
|
70
|
+
/**
|
|
71
|
+
* Default number of texts per embedding request.
|
|
72
|
+
*
|
|
73
|
+
* Also the module default for `batchSize`, so a caller that omits the option
|
|
74
|
+
* behaves exactly as it did before the option existed.
|
|
75
|
+
*/
|
|
76
|
+
exports.DEFAULT_BATCH_SIZE = 16;
|
|
77
|
+
/**
|
|
78
|
+
* The string a liveness probe embeds. A constant, never document text: a probe
|
|
79
|
+
* must not put real material on the wire just to test connectivity.
|
|
80
|
+
*/
|
|
81
|
+
const PROBE_TEXT = "ping";
|
|
82
|
+
const DOCUMENT_TIMEOUT_MS = 300000;
|
|
83
|
+
const QUERY_TIMEOUT_MS = 30000;
|
|
84
|
+
class EmbeddingUnavailableError extends Error {
|
|
85
|
+
constructor(message) {
|
|
86
|
+
super(message);
|
|
87
|
+
this.name = "EmbeddingUnavailableError";
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
exports.EmbeddingUnavailableError = EmbeddingUnavailableError;
|
|
91
|
+
/** Thrown when the configured prefix mode cannot be used with the stored index. */
|
|
92
|
+
class PrefixModeMismatchError extends Error {
|
|
93
|
+
constructor(message) {
|
|
94
|
+
super(message);
|
|
95
|
+
this.name = "PrefixModeMismatchError";
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
exports.PrefixModeMismatchError = PrefixModeMismatchError;
|
|
99
|
+
/**
|
|
100
|
+
* Raised when the server rejects a request because the INPUT IS TOO LONG for
|
|
101
|
+
* the model's context window.
|
|
102
|
+
*
|
|
103
|
+
* It extends `EmbeddingUnavailableError` deliberately. A caller that has no
|
|
104
|
+
* specific handling -- a query, or a fragment that is already at the floor --
|
|
105
|
+
* must abort rather than write an index that silently omits a document. The
|
|
106
|
+
* subclass only lets the document path recognise the case and split.
|
|
107
|
+
*
|
|
108
|
+
* ⚠️ THIS IS A TOKEN BUDGET, NOT A CHARACTER BUDGET, AND THE TWO DIVERGE
|
|
109
|
+
* WILDLY. Measured on this corpus: a run of ellipsis (dot leaders in a table of
|
|
110
|
+
* contents) tokenises at roughly 1.5-2 tokens per character, so 2,000
|
|
111
|
+
* characters blows a 2,048-token window, while 2,000 characters of prose uses
|
|
112
|
+
* about 500 and passes with room to spare. `truncate: true` was measured and
|
|
113
|
+
* does NOT fix it. The only fix that works for any chunk is to make the request
|
|
114
|
+
* smaller, which is what the split path does.
|
|
115
|
+
*/
|
|
116
|
+
class EmbeddingContextLengthError extends EmbeddingUnavailableError {
|
|
117
|
+
constructor(message, longestChars) {
|
|
118
|
+
super(message);
|
|
119
|
+
this.name = "EmbeddingContextLengthError";
|
|
120
|
+
this.longestChars = longestChars;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
exports.EmbeddingContextLengthError = EmbeddingContextLengthError;
|
|
124
|
+
/**
|
|
125
|
+
* Is this reply the server saying "your input is too long"?
|
|
126
|
+
*
|
|
127
|
+
* Detected on the SERVER'S OWN TEXT, and only on a 400. Anything else -- a 500,
|
|
128
|
+
* a 401, a 400 whose body says something different -- stays a plain
|
|
129
|
+
* `EmbeddingUnavailableError`, because splitting a request in response to an
|
|
130
|
+
* unrelated 400 would turn one visible failure into a hundred invisible ones.
|
|
131
|
+
*/
|
|
132
|
+
function isContextLengthRejection(status, serverText) {
|
|
133
|
+
if (status !== 400)
|
|
134
|
+
return false;
|
|
135
|
+
return /input length exceeds the context length/i.test(serverText);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* The prefix each embedding model family expects.
|
|
139
|
+
*
|
|
140
|
+
* This is the bug fix, not a port. The reference implementation hardcoded the
|
|
141
|
+
* E5 instruction template onto a nomic model. Measured on this box at n=5,
|
|
142
|
+
* nomic-prefixed and plain both put the known-good document at rank 1 five
|
|
143
|
+
* times out of five, while the E5 template managed four of five with a negative
|
|
144
|
+
* margin. That is a signal rather than a benchmark, but it points the same way
|
|
145
|
+
* the model cards do, and the cost of being wrong is silently worse retrieval.
|
|
146
|
+
*
|
|
147
|
+
* `none` is a supported mode because a future model may want no prefix at all,
|
|
148
|
+
* and because refusing to store the choice would make the choice unprovable.
|
|
149
|
+
*/
|
|
150
|
+
exports.PREFIXES = {
|
|
151
|
+
nomic: { document: "search_document: ", query: "search_query: " },
|
|
152
|
+
none: { document: "", query: "" },
|
|
153
|
+
e5: { document: "passage: ", query: "query: " },
|
|
154
|
+
};
|
|
155
|
+
/**
|
|
156
|
+
* The prefix is baked into every stored vector, so a query vector built with a
|
|
157
|
+
* different prefix is silently worse -- it still returns results, they are just
|
|
158
|
+
* the wrong ones. A mismatch must be refused loudly instead.
|
|
159
|
+
*/
|
|
160
|
+
function assertPrefixModeMatches(configured, stored) {
|
|
161
|
+
if (configured !== stored) {
|
|
162
|
+
throw new PrefixModeMismatchError(`The index was built with EMBEDDING_PREFIX_MODE="${stored}" but this ` +
|
|
163
|
+
`process is configured for "${configured}". The prefix is stored ` +
|
|
164
|
+
`inside every vector, so querying across a mismatch returns quietly ` +
|
|
165
|
+
`worse results. Re-run the sync with the stored mode, or rebuild the ` +
|
|
166
|
+
`index with the configured one.`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1"]);
|
|
170
|
+
function ipv4ToOctets(host) {
|
|
171
|
+
const match = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
|
|
172
|
+
if (!match)
|
|
173
|
+
return undefined;
|
|
174
|
+
const octets = match.slice(1).map((part) => Number(part));
|
|
175
|
+
// Reject 999.1.1.1 and leading-zero ambiguity alike: every octet must be a
|
|
176
|
+
// canonical 0-255 decimal. Anything else is not a literal we can reason about.
|
|
177
|
+
if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
return octets;
|
|
181
|
+
}
|
|
182
|
+
function isPrivateIpv4(octets) {
|
|
183
|
+
const [a, b] = octets;
|
|
184
|
+
if (a === 10)
|
|
185
|
+
return true;
|
|
186
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
187
|
+
return true;
|
|
188
|
+
if (a === 192 && b === 168)
|
|
189
|
+
return true;
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
/** Strip the brackets a WHATWG URL keeps around an IPv6 literal. */
|
|
193
|
+
function bareHost(hostname) {
|
|
194
|
+
return hostname.startsWith("[") && hostname.endsWith("]")
|
|
195
|
+
? hostname.slice(1, -1)
|
|
196
|
+
: hostname;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Validate an embedding endpoint URL.
|
|
200
|
+
*
|
|
201
|
+
* Two checks are always applied, because they are true of every HTTP client
|
|
202
|
+
* rather than of one deployment:
|
|
203
|
+
* - the URL must parse, and
|
|
204
|
+
* - the scheme must be http: or https:. Anything else is not an endpoint this
|
|
205
|
+
* client can speak to, and a `file:` URL reaching an outbound fetch is a
|
|
206
|
+
* defect in any package.
|
|
207
|
+
*
|
|
208
|
+
* `localOnly` adds the third check. It is OFF by default and it exists as a
|
|
209
|
+
* capability, not as an opinion:
|
|
210
|
+
*
|
|
211
|
+
* A DNS NAME IS REFUSED, because a name can resolve anywhere and this process
|
|
212
|
+
* cannot prove where it resolves to at startup -- a `localhost` that works
|
|
213
|
+
* today and a hosts-file edit tomorrow are indistinguishable from inside this
|
|
214
|
+
* function. Requiring a literal address makes the check answerable by reading
|
|
215
|
+
* the configuration.
|
|
216
|
+
*
|
|
217
|
+
* The shape of the local-only check is deliberate, and a naive one would be
|
|
218
|
+
* wrong in two different ways:
|
|
219
|
+
* - Asserting `host === "127.0.0.1"` would reject an Ollama that binds
|
|
220
|
+
* `0.0.0.0` (`OLLAMA_HOST=0.0.0.0`), which is then reached at the LAN
|
|
221
|
+
* address as well as at loopback.
|
|
222
|
+
* - Accepting `0.0.0.0` as a connect target would be wrong the other way: it
|
|
223
|
+
* is the unspecified address, its meaning is platform-dependent, and no
|
|
224
|
+
* legitimate deployment needs to dial it.
|
|
225
|
+
* An allowlist of literal private addresses handles both.
|
|
226
|
+
*/
|
|
227
|
+
function parseEmbeddingEndpoint(rawUrl, opts = {}) {
|
|
228
|
+
let url;
|
|
229
|
+
try {
|
|
230
|
+
url = new URL(rawUrl);
|
|
231
|
+
}
|
|
232
|
+
catch (_a) {
|
|
233
|
+
throw new Error(`EMBEDDING_URL is not a valid URL: "${rawUrl}". ` +
|
|
234
|
+
`Example: http://127.0.0.1:11434`);
|
|
235
|
+
}
|
|
236
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
237
|
+
throw new Error(`EMBEDDING_URL must use http: or https:, not "${url.protocol}".`);
|
|
238
|
+
}
|
|
239
|
+
const host = bareHost(url.hostname).toLowerCase();
|
|
240
|
+
if (!host) {
|
|
241
|
+
throw new Error(`EMBEDDING_URL has no hostname: "${rawUrl}".`);
|
|
242
|
+
}
|
|
243
|
+
if (!opts.localOnly)
|
|
244
|
+
return url;
|
|
245
|
+
if (LOOPBACK_HOSTS.has(host))
|
|
246
|
+
return url;
|
|
247
|
+
const octets = ipv4ToOctets(host);
|
|
248
|
+
if (!octets) {
|
|
249
|
+
throw new Error(`EMBEDDING_URL host "${host}" is not a literal loopback or RFC1918 ` +
|
|
250
|
+
`address. Local-only embedding is enabled, and a DNS name is refused ` +
|
|
251
|
+
`under it deliberately: a name can resolve anywhere, and this process ` +
|
|
252
|
+
`cannot prove where. Use 127.0.0.1, ::1, 10.x.x.x, 172.16-31.x.x or ` +
|
|
253
|
+
`192.168.x.x, or unset the local-only flag to reach a named endpoint.`);
|
|
254
|
+
}
|
|
255
|
+
if (host === "0.0.0.0") {
|
|
256
|
+
throw new Error(`EMBEDDING_URL host "0.0.0.0" is the unspecified address, not a ` +
|
|
257
|
+
`destination. A server may BIND 0.0.0.0, but you CONNECT to ` +
|
|
258
|
+
`127.0.0.1 -- use http://127.0.0.1:11434.`);
|
|
259
|
+
}
|
|
260
|
+
if (!isPrivateIpv4(octets)) {
|
|
261
|
+
throw new Error(`EMBEDDING_URL host "${host}" is a public address, and local-only ` +
|
|
262
|
+
`embedding is enabled. Use a loopback or RFC1918 address.`);
|
|
263
|
+
}
|
|
264
|
+
return url;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Where the request actually goes.
|
|
268
|
+
*
|
|
269
|
+
* A configured path is honoured verbatim: a caller pointing at
|
|
270
|
+
* `https://api.openai.com/v1` gets `https://api.openai.com/v1/embeddings`, and
|
|
271
|
+
* one pointing at `https://host/v1/embeddings` gets exactly that, with no
|
|
272
|
+
* double suffix. An origin with no path falls back to Ollama's default route.
|
|
273
|
+
*
|
|
274
|
+
* Trailing slashes are stripped before the suffix is appended so
|
|
275
|
+
* `http://host:11434/` cannot produce `//api/embed`.
|
|
276
|
+
*/
|
|
277
|
+
function resolveEmbeddingEndpoint(base) {
|
|
278
|
+
const path = base.pathname.replace(/\/+$/, "");
|
|
279
|
+
if (path === "")
|
|
280
|
+
return `${base.origin}/api/embed`;
|
|
281
|
+
if (/\/embeddings$/.test(path))
|
|
282
|
+
return `${base.origin}${path}`;
|
|
283
|
+
return `${base.origin}${path}/embeddings`;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Read a vector list out of either reply shape.
|
|
287
|
+
*
|
|
288
|
+
* Returns `undefined` when the body is neither, so the caller can raise one
|
|
289
|
+
* error with one message rather than guessing which shape was attempted.
|
|
290
|
+
*/
|
|
291
|
+
function embeddingsFromBody(body) {
|
|
292
|
+
if (!body || typeof body !== "object")
|
|
293
|
+
return undefined;
|
|
294
|
+
const native = body.embeddings;
|
|
295
|
+
if (Array.isArray(native))
|
|
296
|
+
return native;
|
|
297
|
+
const data = body.data;
|
|
298
|
+
if (Array.isArray(data)) {
|
|
299
|
+
const vectors = data.map((entry) => entry && typeof entry === "object"
|
|
300
|
+
? entry.embedding
|
|
301
|
+
: undefined);
|
|
302
|
+
// A `data` array that does not carry embeddings on every entry is not a
|
|
303
|
+
// usable reply; saying so is better than returning undefined vectors.
|
|
304
|
+
if (vectors.every((vector) => Array.isArray(vector))) {
|
|
305
|
+
return vectors;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return undefined;
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Below this many characters a refused fragment is NOT split further.
|
|
312
|
+
*
|
|
313
|
+
* A fragment this short is refused only if the model window is far smaller than
|
|
314
|
+
* the one this index was built against, and splitting again would produce
|
|
315
|
+
* non-words while looping. Throwing names the fragment length; silently
|
|
316
|
+
* dropping it would leave an index that claims a document it does not hold.
|
|
317
|
+
*/
|
|
318
|
+
exports.MIN_SPLITTABLE_CHARS = 200;
|
|
319
|
+
/**
|
|
320
|
+
* Hard ceiling on recursion, so a pathological input cannot spin.
|
|
321
|
+
*
|
|
322
|
+
* Halving a 16-text batch down to singles costs 4 levels; halving a 2,000
|
|
323
|
+
* character chunk down to the floor costs 4 more. Twelve is generous and still
|
|
324
|
+
* finite, which is the property that matters.
|
|
325
|
+
*/
|
|
326
|
+
exports.MAX_SPLIT_DEPTH = 12;
|
|
327
|
+
/**
|
|
328
|
+
* Every grapheme-cluster boundary in `text`, as character offsets.
|
|
329
|
+
*
|
|
330
|
+
* ⚠️ THIS IS LOAD-BEARING FOR THIS CORPUS, NOT A REFINEMENT. It is
|
|
331
|
+
* Tibetan-heavy, and a Tibetan syllable stacks combining marks (category Mn) on
|
|
332
|
+
* a base consonant: U+0FB3 U+0F74 is one syllable, two code points, and a slice
|
|
333
|
+
* between them produces a base plus a stranded vowel sign -- text that is not a
|
|
334
|
+
* word, embedded as if it were. `Intl.Segmenter` is present in every Node this
|
|
335
|
+
* package supports (measured on v24.21.0), so the Array.from fallback is
|
|
336
|
+
* belt-and-braces: it keeps surrogate pairs intact, which is the one thing a
|
|
337
|
+
* naive slice gets catastrophically wrong, and leaves combining marks to the
|
|
338
|
+
* segmenter that is actually available.
|
|
339
|
+
*/
|
|
340
|
+
function graphemeBoundaries(text) {
|
|
341
|
+
const Ctor = Intl
|
|
342
|
+
.Segmenter;
|
|
343
|
+
const bounds = [0];
|
|
344
|
+
if (!Ctor) {
|
|
345
|
+
let offset = 0;
|
|
346
|
+
for (const codePoint of Array.from(text)) {
|
|
347
|
+
offset += codePoint.length;
|
|
348
|
+
bounds.push(offset);
|
|
349
|
+
}
|
|
350
|
+
return bounds;
|
|
351
|
+
}
|
|
352
|
+
const segmenter = new Ctor("en", { granularity: "grapheme" });
|
|
353
|
+
let offset = 0;
|
|
354
|
+
for (const part of segmenter.segment(text)) {
|
|
355
|
+
offset += part.segment.length;
|
|
356
|
+
bounds.push(offset);
|
|
357
|
+
}
|
|
358
|
+
return bounds;
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Cut a text in two so each half can be embedded on its own.
|
|
362
|
+
*
|
|
363
|
+
* A whitespace boundary nearest the midpoint is preferred, because a split
|
|
364
|
+
* between words produces two fragments that still mean something. When the text
|
|
365
|
+
* has no whitespace at all -- a run of dot leaders is exactly that -- the
|
|
366
|
+
* nearest GRAPHEME boundary is used instead. That fallback is what keeps the
|
|
367
|
+
* "never mid-codepoint" rule true for input that offers no word break.
|
|
368
|
+
*
|
|
369
|
+
* Returns undefined when no split makes progress, so the caller throws rather
|
|
370
|
+
* than looping on a text it cannot shorten.
|
|
371
|
+
*/
|
|
372
|
+
function splitForEmbedding(text) {
|
|
373
|
+
const target = Math.floor(text.length / 2);
|
|
374
|
+
for (let delta = 0; delta <= text.length; delta += 1) {
|
|
375
|
+
for (const index of delta === 0 ? [target] : [target - delta, target + delta]) {
|
|
376
|
+
if (index <= 0 || index >= text.length)
|
|
377
|
+
continue;
|
|
378
|
+
if (!/\s/.test(text[index]))
|
|
379
|
+
continue;
|
|
380
|
+
const left = text.slice(0, index).trim();
|
|
381
|
+
const right = text.slice(index).trim();
|
|
382
|
+
if (left.length > 0 && right.length > 0)
|
|
383
|
+
return [left, right];
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
const bounds = graphemeBoundaries(text);
|
|
387
|
+
let best;
|
|
388
|
+
let bestDistance = Number.POSITIVE_INFINITY;
|
|
389
|
+
for (const index of bounds) {
|
|
390
|
+
if (index <= 0 || index >= text.length)
|
|
391
|
+
continue;
|
|
392
|
+
const left = text.slice(0, index);
|
|
393
|
+
const right = text.slice(index);
|
|
394
|
+
if (left.trim().length === 0 || right.trim().length === 0)
|
|
395
|
+
continue;
|
|
396
|
+
const distance = Math.abs(index - target);
|
|
397
|
+
if (distance < bestDistance) {
|
|
398
|
+
bestDistance = distance;
|
|
399
|
+
best = [left, right];
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return best;
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* The direction of the mean of two vectors, re-normalised to unit length.
|
|
406
|
+
*
|
|
407
|
+
* A cosine-indexed store compares DIRECTION, so the mean of two halves of one
|
|
408
|
+
* chunk carries the whole of what the store can use while keeping one vector
|
|
409
|
+
* per chunk -- and therefore one row, and an unchanged `chunkCount` contract.
|
|
410
|
+
* The mean is not unit length even when both inputs are, so it is re-normalised.
|
|
411
|
+
*
|
|
412
|
+
* Two nearly opposed halves would average to a vector with no direction. That
|
|
413
|
+
* is reported rather than stored: a zero vector matches nothing and would sit
|
|
414
|
+
* in the index as a chunk that can never be retrieved.
|
|
415
|
+
*/
|
|
416
|
+
function renormalisedMean(a, b) {
|
|
417
|
+
if (a.length !== b.length) {
|
|
418
|
+
throw new EmbeddingUnavailableError(`Cannot combine vectors of different widths (${a.length} and ${b.length}).`);
|
|
419
|
+
}
|
|
420
|
+
const mean = a.map((value, index) => (value + b[index]) / 2);
|
|
421
|
+
let sumOfSquares = 0;
|
|
422
|
+
for (const value of mean)
|
|
423
|
+
sumOfSquares += value * value;
|
|
424
|
+
const norm = Math.sqrt(sumOfSquares);
|
|
425
|
+
if (!Number.isFinite(norm) || norm === 0) {
|
|
426
|
+
throw new EmbeddingUnavailableError("The two halves of an over-long chunk averaged to a vector with no " +
|
|
427
|
+
"direction, so it could not be stored. This means the halves carry " +
|
|
428
|
+
"opposed content; the chunk needs splitting at a different point.");
|
|
429
|
+
}
|
|
430
|
+
return mean.map((value) => value / norm);
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* An HTTP embeddings client for any OpenAI-compatible `/embeddings` endpoint.
|
|
434
|
+
*
|
|
435
|
+
* The constructor validates the endpoint and the prefix mode, and throws on
|
|
436
|
+
* either failure. Nothing is deferred to first use: a misconfigured index
|
|
437
|
+
* should fail at startup, before it has written a single vector.
|
|
438
|
+
*/
|
|
439
|
+
class OllamaEmbedder {
|
|
440
|
+
constructor(opts) {
|
|
441
|
+
var _a, _b;
|
|
442
|
+
/**
|
|
443
|
+
* Declared with `#`, not `private`.
|
|
444
|
+
*
|
|
445
|
+
* `private` is erased at compile time: the field is an own ENUMERABLE
|
|
446
|
+
* property at runtime, so `JSON.stringify(embedder)` carried the credential
|
|
447
|
+
* and the KDoc's promise that it never appears in a returned object was
|
|
448
|
+
* false. A true private field compiles to a WeakMap slot and is invisible to
|
|
449
|
+
* `Object.keys`, spread, and JSON.
|
|
450
|
+
*/
|
|
451
|
+
_OllamaEmbedder_apiKey.set(this, void 0);
|
|
452
|
+
this.splits = [];
|
|
453
|
+
const base = parseEmbeddingEndpoint(opts.url, { localOnly: opts.localOnly });
|
|
454
|
+
this.endpointUrl = resolveEmbeddingEndpoint(base);
|
|
455
|
+
if (!opts.model.trim()) {
|
|
456
|
+
throw new Error("EMBEDDING_MODEL must not be empty.");
|
|
457
|
+
}
|
|
458
|
+
if (!(opts.prefixMode in exports.PREFIXES)) {
|
|
459
|
+
throw new Error(`Unknown embedding prefix mode "${opts.prefixMode}".`);
|
|
460
|
+
}
|
|
461
|
+
const batchSize = (_a = opts.batchSize) !== null && _a !== void 0 ? _a : exports.DEFAULT_BATCH_SIZE;
|
|
462
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
463
|
+
throw new Error(`Embedding batch size must be a positive whole number, not "${opts.batchSize}".`);
|
|
464
|
+
}
|
|
465
|
+
this.model = opts.model;
|
|
466
|
+
this.prefixMode = opts.prefixMode;
|
|
467
|
+
this.dimensions = (_b = opts.dimensions) !== null && _b !== void 0 ? _b : 768;
|
|
468
|
+
__classPrivateFieldSet(this, _OllamaEmbedder_apiKey, opts.apiKey, "f");
|
|
469
|
+
this.batchSize = batchSize;
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Split events observed during the most recent `embedDocuments` call.
|
|
473
|
+
*
|
|
474
|
+
* Counts and lengths only: a caller may log this, so it must never be able to
|
|
475
|
+
* log document text through it. The sync reads it to report how many chunks
|
|
476
|
+
* needed the fallback, which is the only way to know the split path is
|
|
477
|
+
* actually doing work rather than merely existing.
|
|
478
|
+
*/
|
|
479
|
+
get lastSplits() {
|
|
480
|
+
return this.splits;
|
|
481
|
+
}
|
|
482
|
+
embedDocuments(texts) {
|
|
483
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
484
|
+
this.splits = [];
|
|
485
|
+
const prefix = exports.PREFIXES[this.prefixMode].document;
|
|
486
|
+
const embeddings = [];
|
|
487
|
+
for (let i = 0; i < texts.length; i += this.batchSize) {
|
|
488
|
+
// ⚠️ THE BATCH CARRIES UNPREFIXED TEXT, AND THE PREFIX IS ADDED AT THE
|
|
489
|
+
// LEAF. Prefixing here and splitting later looks equivalent and is not:
|
|
490
|
+
// a split cuts the string, so only the half that kept the head would
|
|
491
|
+
// carry the prefix -- and for nomic the prefix is a MEANING carrier, not
|
|
492
|
+
// decoration. Bare fragments embed into a different region of the space
|
|
493
|
+
// than the queries do, and nothing about the result would look wrong.
|
|
494
|
+
const batch = texts.slice(i, i + this.batchSize);
|
|
495
|
+
embeddings.push(...(yield this.embedBatch(batch, prefix, 0)));
|
|
496
|
+
}
|
|
497
|
+
return embeddings;
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Embed a batch, halving it on a context-length rejection.
|
|
502
|
+
*
|
|
503
|
+
* The order of the returned vectors is the order of `batch`, which is what
|
|
504
|
+
* makes this a drop-in replacement for the single `request` call it replaces:
|
|
505
|
+
* the caller zips vectors to texts by index and never learns a split happened.
|
|
506
|
+
*
|
|
507
|
+
* A batch of ONE that is still refused moves to `embedOne`, which splits the
|
|
508
|
+
* TEXT. The two levels exist because they fail differently: a multi-text
|
|
509
|
+
* batch can succeed simply by being asked as two requests, while a single
|
|
510
|
+
* over-long text has to be shortened or it will never embed.
|
|
511
|
+
*/
|
|
512
|
+
embedBatch(batch, prefix, depth) {
|
|
513
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
514
|
+
if (batch.length === 0)
|
|
515
|
+
return [];
|
|
516
|
+
if (depth > exports.MAX_SPLIT_DEPTH) {
|
|
517
|
+
throw new EmbeddingUnavailableError(`The embedding server rejected this input at split depth ${exports.MAX_SPLIT_DEPTH}. ` +
|
|
518
|
+
`Refusing to recurse further rather than loop.`);
|
|
519
|
+
}
|
|
520
|
+
try {
|
|
521
|
+
return yield this.request(batch.map((text) => prefix + text), DOCUMENT_TIMEOUT_MS);
|
|
522
|
+
}
|
|
523
|
+
catch (err) {
|
|
524
|
+
if (!(err instanceof EmbeddingContextLengthError))
|
|
525
|
+
throw err;
|
|
526
|
+
if (batch.length > 1) {
|
|
527
|
+
// Halve by TEXT COUNT. Each half keeps its relative order, so the
|
|
528
|
+
// concatenation of the two answers is the answer for the whole batch.
|
|
529
|
+
const middle = Math.floor(batch.length / 2);
|
|
530
|
+
this.splits.push({
|
|
531
|
+
reason: "batch",
|
|
532
|
+
texts: batch.length,
|
|
533
|
+
chars: 0,
|
|
534
|
+
depth,
|
|
535
|
+
});
|
|
536
|
+
const left = yield this.embedBatch(batch.slice(0, middle), prefix, depth + 1);
|
|
537
|
+
const right = yield this.embedBatch(batch.slice(middle), prefix, depth + 1);
|
|
538
|
+
return [...left, ...right];
|
|
539
|
+
}
|
|
540
|
+
return [yield this.embedOne(batch[0], prefix, depth)];
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* Embed a single text, splitting it when the server says it is too long.
|
|
546
|
+
*
|
|
547
|
+
* The halves are embedded independently and combined into ONE vector, so the
|
|
548
|
+
* chunk stays one row and the sync's `chunkCount` contract -- one vector per
|
|
549
|
+
* chunk -- is untouched. Anything else would change how many rows a document
|
|
550
|
+
* has, and the store rejects a count that disagrees with the array.
|
|
551
|
+
*/
|
|
552
|
+
embedOne(text, prefix, depth) {
|
|
553
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
554
|
+
try {
|
|
555
|
+
const [embedding] = yield this.request([prefix + text], DOCUMENT_TIMEOUT_MS);
|
|
556
|
+
return embedding;
|
|
557
|
+
}
|
|
558
|
+
catch (err) {
|
|
559
|
+
if (!(err instanceof EmbeddingContextLengthError))
|
|
560
|
+
throw err;
|
|
561
|
+
// ⚠️ THE FLOOR. Below this, splitting produces non-words and the only
|
|
562
|
+
// honest move is to fail loudly with the fragment's length. Silently
|
|
563
|
+
// dropping the chunk would leave an index claiming a document it holds
|
|
564
|
+
// only part of -- the exact failure the abort-on-outage rule exists to
|
|
565
|
+
// prevent, arriving through a different door.
|
|
566
|
+
if (text.length < exports.MIN_SPLITTABLE_CHARS) {
|
|
567
|
+
throw new EmbeddingUnavailableError(`An input of ${text.length} characters was rejected as too long even ` +
|
|
568
|
+
`after splitting down to the ${exports.MIN_SPLITTABLE_CHARS}-character ` +
|
|
569
|
+
`floor. The embedding model's context window is smaller than this ` +
|
|
570
|
+
`index was built against. No content was dropped: the document ` +
|
|
571
|
+
`fails whole.`);
|
|
572
|
+
}
|
|
573
|
+
if (depth > exports.MAX_SPLIT_DEPTH) {
|
|
574
|
+
throw new EmbeddingUnavailableError(`An input of ${text.length} characters was still rejected at split ` +
|
|
575
|
+
`depth ${exports.MAX_SPLIT_DEPTH}. Refusing to recurse further rather ` +
|
|
576
|
+
`than loop.`);
|
|
577
|
+
}
|
|
578
|
+
const halves = splitForEmbedding(text);
|
|
579
|
+
if (!halves) {
|
|
580
|
+
throw new EmbeddingUnavailableError(`An input of ${text.length} characters was rejected as too long, and ` +
|
|
581
|
+
`it has no boundary to split at. No content was dropped: the ` +
|
|
582
|
+
`document fails whole.`);
|
|
583
|
+
}
|
|
584
|
+
this.splits.push({
|
|
585
|
+
reason: "text",
|
|
586
|
+
texts: 1,
|
|
587
|
+
chars: text.length,
|
|
588
|
+
depth,
|
|
589
|
+
});
|
|
590
|
+
// The halves are bare text; each gets the prefix on its own request, so
|
|
591
|
+
// both fragments are embedded the same way a whole chunk would be.
|
|
592
|
+
const left = yield this.embedOne(halves[0], prefix, depth + 1);
|
|
593
|
+
const right = yield this.embedOne(halves[1], prefix, depth + 1);
|
|
594
|
+
return renormalisedMean(left, right);
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
embedQuery(query) {
|
|
599
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
600
|
+
const prefix = exports.PREFIXES[this.prefixMode].query;
|
|
601
|
+
const [embedding] = yield this.request([prefix + query], QUERY_TIMEOUT_MS);
|
|
602
|
+
return embedding;
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Embed one throwaway string and discard it.
|
|
607
|
+
*
|
|
608
|
+
* Deliberately a real embedding rather than a GET to the server's root: a
|
|
609
|
+
* server that is up but has not pulled the model answers GET happily and then
|
|
610
|
+
* fails on the first real request. The whole point of the probe is to fail
|
|
611
|
+
* BEFORE an irreversible step, so it has to exercise the thing that will
|
|
612
|
+
* actually run -- the same model, the same prefix, the same endpoint.
|
|
613
|
+
*
|
|
614
|
+
* The string is a constant, not document text, so nothing real is placed on
|
|
615
|
+
* the wire for a liveness check.
|
|
616
|
+
*/
|
|
617
|
+
probe() {
|
|
618
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
619
|
+
yield this.request([PROBE_TEXT], QUERY_TIMEOUT_MS);
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
headers() {
|
|
623
|
+
const headers = {
|
|
624
|
+
"Content-Type": "application/json",
|
|
625
|
+
};
|
|
626
|
+
if (__classPrivateFieldGet(this, _OllamaEmbedder_apiKey, "f"))
|
|
627
|
+
headers.Authorization = `Bearer ${__classPrivateFieldGet(this, _OllamaEmbedder_apiKey, "f")}`;
|
|
628
|
+
return headers;
|
|
629
|
+
}
|
|
630
|
+
request(input, timeoutMs) {
|
|
631
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
632
|
+
if (input.length === 0)
|
|
633
|
+
return [];
|
|
634
|
+
let response;
|
|
635
|
+
try {
|
|
636
|
+
// `redirect: "error"` is always on. A 307 answer would otherwise let a
|
|
637
|
+
// compromised endpoint forward the body -- which is document text -- to
|
|
638
|
+
// any host it likes, with the configured endpoint still reading as
|
|
639
|
+
// satisfied. Checked: this is the only outbound `fetch` in non-test
|
|
640
|
+
// source, so one option closes the only such path.
|
|
641
|
+
response = yield fetch(this.endpointUrl, {
|
|
642
|
+
method: "POST",
|
|
643
|
+
headers: this.headers(),
|
|
644
|
+
body: JSON.stringify({ model: this.model, input }),
|
|
645
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
646
|
+
redirect: "error",
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
catch (err) {
|
|
650
|
+
// A refused redirect surfaces here as a TypeError from undici. It is
|
|
651
|
+
// reported as unavailable rather than as its own class because the
|
|
652
|
+
// caller's correct response is the same -- stop, do not write vectors --
|
|
653
|
+
// and the message carries the real cause.
|
|
654
|
+
//
|
|
655
|
+
// ⚠️ THE ENDPOINT IS NAMED BUT THE KEY NEVER IS. `err` is included
|
|
656
|
+
// because it is what makes a connection failure diagnosable, and it does
|
|
657
|
+
// not carry request headers.
|
|
658
|
+
throw new EmbeddingUnavailableError(`The embedding server did not answer at ${this.endpointUrl}: ${err}`);
|
|
659
|
+
}
|
|
660
|
+
if (!response.ok) {
|
|
661
|
+
// ⚠️ CARRY THE SERVER'S OWN EXPLANATION.
|
|
662
|
+
//
|
|
663
|
+
// A bare "status 400" is undiagnosable: Ollama answers a rejected input
|
|
664
|
+
// with a JSON `{"error": "..."}` naming the actual cause, and discarding
|
|
665
|
+
// it means the only way to find out what the server objected to is to
|
|
666
|
+
// re-probe it by hand. That is exactly what happened once already, and it
|
|
667
|
+
// cost more time than this branch.
|
|
668
|
+
//
|
|
669
|
+
// The body is truncated and whitespace-collapsed: an error page can be a
|
|
670
|
+
// full HTML document, and an unbounded string in an exception message is
|
|
671
|
+
// its own defect. This is the SERVER's text, never a request header, so
|
|
672
|
+
// it cannot carry the credential.
|
|
673
|
+
//
|
|
674
|
+
// ⚠️ It CAN in principle carry document text, if a provider chooses to
|
|
675
|
+
// echo the offending input back. That is a real trade against
|
|
676
|
+
// diagnosability and it is bounded deliberately: 300 characters, one
|
|
677
|
+
// line, and always the server's words rather than ours. A provider that
|
|
678
|
+
// echoes embeddings input in its errors is worth knowing about anyway.
|
|
679
|
+
let detail = "";
|
|
680
|
+
let serverText = "";
|
|
681
|
+
try {
|
|
682
|
+
serverText = (yield response.text()).replace(/\s+/g, " ").trim();
|
|
683
|
+
if (serverText)
|
|
684
|
+
detail = `: ${serverText.slice(0, 300)}`;
|
|
685
|
+
}
|
|
686
|
+
catch (_a) {
|
|
687
|
+
// Reading the error body is a diagnostic courtesy; its failure must not
|
|
688
|
+
// replace the status error with a read error.
|
|
689
|
+
}
|
|
690
|
+
// ⭐ ONE 400 IS NOT LIKE THE OTHERS. A context-length rejection is about
|
|
691
|
+
// the SIZE of this request, not about the server, so the caller can make
|
|
692
|
+
// the request smaller and succeed. Every other status -- including a 400
|
|
693
|
+
// that says something else -- stays an outage, because retrying a real
|
|
694
|
+
// rejection in halves turns one visible failure into many invisible ones.
|
|
695
|
+
//
|
|
696
|
+
// The longest input is measured HERE rather than passed in: this is the
|
|
697
|
+
// only frame that knows the exact batch that was refused, and a length
|
|
698
|
+
// carried down from the caller would be a second source of truth about
|
|
699
|
+
// the same request.
|
|
700
|
+
if (isContextLengthRejection(response.status, serverText)) {
|
|
701
|
+
const longest = input.reduce((max, text) => Math.max(max, text.length), 0);
|
|
702
|
+
throw new EmbeddingContextLengthError(`The embedding server rejected an over-long input with status ` +
|
|
703
|
+
`${response.status} at ${this.endpointUrl}${detail}`, longest);
|
|
704
|
+
}
|
|
705
|
+
throw new EmbeddingUnavailableError(`The embedding server answered with status ${response.status} at ${this.endpointUrl}${detail}`);
|
|
706
|
+
}
|
|
707
|
+
let body;
|
|
708
|
+
try {
|
|
709
|
+
body = yield response.json();
|
|
710
|
+
}
|
|
711
|
+
catch (err) {
|
|
712
|
+
throw new EmbeddingUnavailableError(`The embedding server answered with a body that is not JSON: ${err}`);
|
|
713
|
+
}
|
|
714
|
+
const embeddings = embeddingsFromBody(body);
|
|
715
|
+
if (!embeddings || embeddings.length !== input.length) {
|
|
716
|
+
throw new EmbeddingUnavailableError("The embedding server answered with an unrecognised body, or with a " +
|
|
717
|
+
"mismatched embeddings array. Expected either `{embeddings: [[...]]}` " +
|
|
718
|
+
"(Ollama native) or `{data: [{embedding: [...]}]}` " +
|
|
719
|
+
"(OpenAI-compatible), with one vector per input " +
|
|
720
|
+
`(${input.length} sent).`);
|
|
721
|
+
}
|
|
722
|
+
for (const embedding of embeddings) {
|
|
723
|
+
if (!Array.isArray(embedding) || embedding.length !== this.dimensions) {
|
|
724
|
+
throw new EmbeddingUnavailableError(`The embedding server returned a vector of the wrong width; ` +
|
|
725
|
+
`expected ${this.dimensions} dimensions. Check that ` +
|
|
726
|
+
`EMBEDDING_MODEL matches the index.`);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return embeddings;
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
exports.OllamaEmbedder = OllamaEmbedder;
|
|
734
|
+
_OllamaEmbedder_apiKey = new WeakMap();
|