@delali/narsil 0.3.1-next.96 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-ZGMR46O3.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-KPMIHLOV.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-3MYKLHRO.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-R7P64REL.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-WMYC5ZK6.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-3A3WDNHO.mjs → snapshot-J57MZK6L.mjs} +8 -6
- package/dist/browser/{write-JODZHKIF.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-H75XXPMY.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-2BY6K52M.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-GMY7F2VA.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-VVFLEGZI.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-LJBJQPB5.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-O5WPRAVZ.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-TCYUAUHF.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-2VBNYYDY.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CbZmXUp6.d.ts → engine-BGCEm9E_.d.ts} +48 -18
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-UFIQJTSG.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-XII3ANGW.mjs → snapshot-HOGXNO5W.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-6M3VG4TU.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from '../adapters-
|
|
1
|
+
import { E as EmbeddingAdapter } from '../adapters-BQFQ4PRz.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* How {@link createOpenAIEmbedding} reaches an embeddings endpoint.
|
|
@@ -29,7 +29,7 @@ interface OpenAIEmbeddingConfig {
|
|
|
29
29
|
*
|
|
30
30
|
* @param config - Endpoint, credentials, model, and vector length.
|
|
31
31
|
* @returns An adapter you pass as `embedding`, or register by name with
|
|
32
|
-
* {@link
|
|
32
|
+
* {@link AdapterRegistrationOperations.registerEmbeddingAdapter}.
|
|
33
33
|
* @throws A `NarsilError` with `EMBEDDING_CONFIG_INVALID` when the base URL,
|
|
34
34
|
* the model, or the dimension count is missing or out of range.
|
|
35
35
|
*
|
|
@@ -1,64 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-K7D4ZTSW.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from '../chunk-KV3AZFT3.mjs';
|
|
2
3
|
|
|
3
4
|
// src/embeddings/constants.ts
|
|
4
|
-
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
5
|
-
var MAX_BACKOFF_MS = 3e4;
|
|
6
|
-
var BASE_BACKOFF_MS = 1e3;
|
|
7
|
-
var MAX_JITTER_MS = 1e3;
|
|
8
5
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
9
6
|
var MAX_INPUTS_PER_REQUEST = 2048;
|
|
10
7
|
|
|
11
8
|
// src/embeddings/openai.ts
|
|
12
|
-
function isRetryableStatus(status) {
|
|
13
|
-
return RETRYABLE_STATUS_CODES.has(status);
|
|
14
|
-
}
|
|
15
|
-
function computeBackoffMs(attempt, retryAfterSeconds) {
|
|
16
|
-
if (retryAfterSeconds !== null && retryAfterSeconds > 0) {
|
|
17
|
-
return Math.min(retryAfterSeconds * 1e3, MAX_BACKOFF_MS);
|
|
18
|
-
}
|
|
19
|
-
const exponential = Math.min(BASE_BACKOFF_MS * 2 ** attempt, MAX_BACKOFF_MS);
|
|
20
|
-
const jitter = Math.random() * MAX_JITTER_MS;
|
|
21
|
-
return exponential + jitter;
|
|
22
|
-
}
|
|
23
|
-
function parseRetryAfterHeader(headers) {
|
|
24
|
-
const raw = headers.get("retry-after");
|
|
25
|
-
if (raw === null) return null;
|
|
26
|
-
const seconds = Number(raw);
|
|
27
|
-
if (Number.isFinite(seconds) && seconds > 0) return seconds;
|
|
28
|
-
return null;
|
|
29
|
-
}
|
|
30
|
-
async function resolveApiKey(apiKey) {
|
|
31
|
-
const resolved = typeof apiKey === "function" ? await apiKey() : apiKey;
|
|
32
|
-
if (typeof resolved !== "string" || resolved.trim().length === 0) {
|
|
33
|
-
throw new NarsilError(
|
|
34
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
35
|
-
"OpenAI embedding adapter apiKey resolved to an empty or non-string value"
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
return resolved;
|
|
39
|
-
}
|
|
40
|
-
function buildSignal(callerSignal, timeoutMs) {
|
|
41
|
-
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
42
|
-
if (callerSignal === void 0) return timeoutSignal;
|
|
43
|
-
return AbortSignal.any([callerSignal, timeoutSignal]);
|
|
44
|
-
}
|
|
45
|
-
async function sleep(ms, signal) {
|
|
46
|
-
return new Promise((resolve, reject) => {
|
|
47
|
-
if (signal?.aborted) {
|
|
48
|
-
reject(signal.reason);
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
const onAbort = () => {
|
|
52
|
-
clearTimeout(timer);
|
|
53
|
-
reject(signal?.reason);
|
|
54
|
-
};
|
|
55
|
-
const timer = setTimeout(() => {
|
|
56
|
-
signal?.removeEventListener("abort", onAbort);
|
|
57
|
-
resolve();
|
|
58
|
-
}, ms);
|
|
59
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
9
|
function parseEmbeddingItems(data) {
|
|
63
10
|
if (!Array.isArray(data)) {
|
|
64
11
|
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, 'OpenAI response missing "data" array');
|
|
@@ -85,80 +32,6 @@ function parseEmbeddingItems(data) {
|
|
|
85
32
|
}
|
|
86
33
|
return items;
|
|
87
34
|
}
|
|
88
|
-
async function parseErrorBody(response) {
|
|
89
|
-
try {
|
|
90
|
-
const body = await response.json();
|
|
91
|
-
const errorType = body?.error?.type ?? "unknown";
|
|
92
|
-
const errorMessage = body?.error?.message ?? `HTTP ${response.status}`;
|
|
93
|
-
return { type: errorType, message: errorMessage };
|
|
94
|
-
} catch {
|
|
95
|
-
return { type: "unknown", message: `HTTP ${response.status}` };
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
async function executeWithRetry(url, requestBody, apiKey, maxRetries, signal) {
|
|
99
|
-
let lastError = null;
|
|
100
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
101
|
-
let response;
|
|
102
|
-
try {
|
|
103
|
-
response = await fetch(url, {
|
|
104
|
-
method: "POST",
|
|
105
|
-
headers: {
|
|
106
|
-
Authorization: `Bearer ${apiKey}`,
|
|
107
|
-
"Content-Type": "application/json"
|
|
108
|
-
},
|
|
109
|
-
body: requestBody,
|
|
110
|
-
signal
|
|
111
|
-
});
|
|
112
|
-
} catch (err) {
|
|
113
|
-
if (err instanceof DOMException || err instanceof Error && err.name === "AbortError") {
|
|
114
|
-
throw err;
|
|
115
|
-
}
|
|
116
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
117
|
-
if (attempt < maxRetries) {
|
|
118
|
-
await sleep(computeBackoffMs(attempt, null), signal);
|
|
119
|
-
continue;
|
|
120
|
-
}
|
|
121
|
-
throw new NarsilError(
|
|
122
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
123
|
-
`OpenAI request failed after ${maxRetries + 1} attempts: network error`,
|
|
124
|
-
{
|
|
125
|
-
cause: lastError.message
|
|
126
|
-
}
|
|
127
|
-
);
|
|
128
|
-
}
|
|
129
|
-
if (response.ok) {
|
|
130
|
-
let parsed;
|
|
131
|
-
try {
|
|
132
|
-
parsed = await response.json();
|
|
133
|
-
} catch {
|
|
134
|
-
lastError = new Error("invalid JSON in a 2xx response body");
|
|
135
|
-
if (attempt < maxRetries) {
|
|
136
|
-
await sleep(computeBackoffMs(attempt, null), signal);
|
|
137
|
-
continue;
|
|
138
|
-
}
|
|
139
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned invalid JSON in response body");
|
|
140
|
-
}
|
|
141
|
-
if (parsed === null || typeof parsed !== "object") {
|
|
142
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned non-object JSON response");
|
|
143
|
-
}
|
|
144
|
-
return parsed;
|
|
145
|
-
}
|
|
146
|
-
const retryAfter = parseRetryAfterHeader(response.headers);
|
|
147
|
-
const errorInfo = await parseErrorBody(response);
|
|
148
|
-
if (!isRetryableStatus(response.status) || attempt >= maxRetries) {
|
|
149
|
-
throw new NarsilError(
|
|
150
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
151
|
-
`OpenAI API error (${response.status}): [${errorInfo.type}] ${errorInfo.message}`,
|
|
152
|
-
{ status: response.status, errorType: errorInfo.type }
|
|
153
|
-
);
|
|
154
|
-
}
|
|
155
|
-
lastError = new Error(`HTTP ${response.status}: ${errorInfo.type}`);
|
|
156
|
-
await sleep(computeBackoffMs(attempt, retryAfter), signal);
|
|
157
|
-
}
|
|
158
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI request failed: exhausted all retries", {
|
|
159
|
-
cause: lastError?.message
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
35
|
function createOpenAIEmbedding(config) {
|
|
163
36
|
if (!config.baseUrl || config.baseUrl.trim().length === 0) {
|
|
164
37
|
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "OpenAI embedding adapter requires a non-empty baseUrl");
|
|
@@ -203,10 +76,18 @@ function createOpenAIEmbedding(config) {
|
|
|
203
76
|
return dimensionCount;
|
|
204
77
|
},
|
|
205
78
|
async embed(input, _purpose, signal) {
|
|
206
|
-
const apiKey = await resolveApiKey(config.apiKey);
|
|
79
|
+
const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
|
|
207
80
|
const body = JSON.stringify({ input: [input], model, dimensions: dimensionCount });
|
|
208
81
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
209
|
-
const result = await
|
|
82
|
+
const result = await postJsonWithRetry({
|
|
83
|
+
url,
|
|
84
|
+
body,
|
|
85
|
+
apiKey,
|
|
86
|
+
maxRetries,
|
|
87
|
+
signal: combinedSignal,
|
|
88
|
+
service: "OpenAI",
|
|
89
|
+
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
90
|
+
});
|
|
210
91
|
const items = parseEmbeddingItems(result.data);
|
|
211
92
|
if (items.length === 0) {
|
|
212
93
|
throw new NarsilError(
|
|
@@ -218,11 +99,19 @@ function createOpenAIEmbedding(config) {
|
|
|
218
99
|
},
|
|
219
100
|
async embedBatch(inputs, _purpose, signal) {
|
|
220
101
|
if (inputs.length === 0) return [];
|
|
221
|
-
const apiKey = await resolveApiKey(config.apiKey);
|
|
102
|
+
const apiKey = await resolveApiKey(config.apiKey, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
|
|
222
103
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
223
104
|
const executeChunk = async (chunk) => {
|
|
224
105
|
const body = JSON.stringify({ input: chunk, model, dimensions: dimensionCount });
|
|
225
|
-
const result = await
|
|
106
|
+
const result = await postJsonWithRetry({
|
|
107
|
+
url,
|
|
108
|
+
body,
|
|
109
|
+
apiKey,
|
|
110
|
+
maxRetries,
|
|
111
|
+
signal: combinedSignal,
|
|
112
|
+
service: "OpenAI",
|
|
113
|
+
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
114
|
+
});
|
|
226
115
|
const items = parseEmbeddingItems(result.data);
|
|
227
116
|
if (items.length !== chunk.length) {
|
|
228
117
|
throw new NarsilError(
|
|
@@ -1,8 +1,47 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from './adapters-
|
|
2
|
-
import { B as BatchResult,
|
|
3
|
-
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { B as BatchResult, m as IndexInfo, I as IndexStats, k as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, P as PreflightResult, h as SuggestParams, j as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, l as NarsilEventMap } from './memory-CUEDwr-F.js';
|
|
3
|
+
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-4W_7tk1f.js';
|
|
4
4
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Registers the adapters that indexes and queries reach by name, which is
|
|
8
|
+
* what lets a binding outlive a restart and reach a worker thread or another
|
|
9
|
+
* node.
|
|
10
|
+
*
|
|
11
|
+
* @public
|
|
12
|
+
*/
|
|
13
|
+
interface AdapterRegistrationOperations {
|
|
14
|
+
/**
|
|
15
|
+
* Registers an embedding adapter under a name that an index config can
|
|
16
|
+
* give and that durability metadata keeps. Registering a name again binds
|
|
17
|
+
* every index that gives that name to the new adapter.
|
|
18
|
+
*
|
|
19
|
+
* @param name - The name that an index config gives in `embedding.adapter`.
|
|
20
|
+
* @param adapter - The adapter that embeds documents and queries.
|
|
21
|
+
*/
|
|
22
|
+
registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
|
|
23
|
+
/**
|
|
24
|
+
* Registers a rerank adapter under a name that a query gives in
|
|
25
|
+
* `rerank.adapter`. Registering a name again sends every later query that
|
|
26
|
+
* gives that name to the new adapter. The engine throws
|
|
27
|
+
* `RERANK_CONFIG_INVALID` for an empty name and for an adapter without a
|
|
28
|
+
* `rerank` function.
|
|
29
|
+
*
|
|
30
|
+
* @param name - The name that a query gives in `rerank.adapter`.
|
|
31
|
+
* @param adapter - The adapter that scores the top hits of a query.
|
|
32
|
+
*/
|
|
33
|
+
registerRerankAdapter(name: string, adapter: RerankAdapter): void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The bytes that {@link Narsil.restore} and {@link Narsil.importDocumentExport}
|
|
38
|
+
* accept, as one array, a stream of chunks, or an async iterable of chunks,
|
|
39
|
+
* such as a Node.js file stream.
|
|
40
|
+
*
|
|
41
|
+
* @public
|
|
42
|
+
*/
|
|
43
|
+
type ByteSource = Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
44
|
+
|
|
6
45
|
/**
|
|
7
46
|
* Opens and closes durable indexes without deleting them.
|
|
8
47
|
*
|
|
@@ -165,25 +204,19 @@ interface DocumentWriteOperations {
|
|
|
165
204
|
waitForWrites(indexName: string): Promise<void>;
|
|
166
205
|
}
|
|
167
206
|
|
|
168
|
-
/**
|
|
169
|
-
* The bytes that {@link Narsil.restore} and {@link Narsil.importDocumentExport}
|
|
170
|
-
* accept, as one array, a stream of chunks, or an async iterable of chunks,
|
|
171
|
-
* such as a Node.js file stream.
|
|
172
|
-
*
|
|
173
|
-
* @public
|
|
174
|
-
*/
|
|
175
|
-
type ByteSource = Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
176
207
|
/**
|
|
177
208
|
* The search engine, and everything you do with it.
|
|
178
209
|
*
|
|
179
210
|
* One instance holds every index you create, so an application usually keeps a
|
|
180
211
|
* single engine for its lifetime and reaches each index by name. Build one
|
|
181
|
-
* with {@link createNarsil}. Its write, open and close,
|
|
182
|
-
*
|
|
212
|
+
* with {@link createNarsil}. Its write, open and close, synonym, and adapter
|
|
213
|
+
* registration methods come from {@link DocumentWriteOperations},
|
|
214
|
+
* {@link IndexLifecycleOperations}, {@link SynonymOperations}, and
|
|
215
|
+
* {@link AdapterRegistrationOperations}.
|
|
183
216
|
*
|
|
184
217
|
* @public
|
|
185
218
|
*/
|
|
186
|
-
interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, SynonymOperations {
|
|
219
|
+
interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, SynonymOperations, AdapterRegistrationOperations {
|
|
187
220
|
/**
|
|
188
221
|
* Creates an index you can insert documents into and query.
|
|
189
222
|
*
|
|
@@ -196,9 +229,6 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, Syno
|
|
|
196
229
|
* and scoring settings.
|
|
197
230
|
*/
|
|
198
231
|
createIndex(name: string, config: IndexConfig): Promise<void>;
|
|
199
|
-
/** Registers an adapter under a name that index configs can reference and
|
|
200
|
-
* durability metadata can persist; re-registering rebinds referencing indexes. */
|
|
201
|
-
registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
|
|
202
232
|
/**
|
|
203
233
|
* Removes an index and everything it holds, including whatever persistence
|
|
204
234
|
* wrote for it. The documents do not come back.
|
|
@@ -544,4 +574,4 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, Syno
|
|
|
544
574
|
shutdown(): Promise<void>;
|
|
545
575
|
}
|
|
546
576
|
|
|
547
|
-
export type { ByteSource as B, DocumentWriteOperations as D, IndexLifecycleOperations as I, Narsil as N, SynonymOperations as S };
|
|
577
|
+
export type { AdapterRegistrationOperations as A, ByteSource as B, DocumentWriteOperations as D, IndexLifecycleOperations as I, Narsil as N, SynonymOperations as S };
|
package/dist/index.browser.mjs
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-
|
|
2
|
-
export {
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-NAFT3K6X.mjs';
|
|
2
|
+
export { isSimdAvailable } from './browser/chunk-QWES44AW.mjs';
|
|
3
|
+
import './browser/chunk-JPTQZWJQ.mjs';
|
|
4
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './browser/chunk-RELZRJNR.mjs';
|
package/dist/index.d.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldOptionChain,
|
|
1
|
+
import { b as StopWordOverride, C as CustomTokenizer } from './schema-4W_7tk1f.js';
|
|
2
|
+
export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldOptionChain, c as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, d as ScoringMode, T as TextFieldOption, V as VectorIndexConfig, e as VectorQuantizationMode, f as VectorStorageMode, W as WriteOptions } from './schema-4W_7tk1f.js';
|
|
3
3
|
import { P as PartialWordSearch } from './partial-words-DTIHO8G4.js';
|
|
4
4
|
import { P as PatternSearch } from './pattern-Cledccbl.js';
|
|
5
5
|
import { S as SynonymSearch } from './synonyms-CpOAHPa-.js';
|
|
6
6
|
export { O as OneWaySynonymRule, a as SynonymRule, T as TwoWaySynonymRule } from './synonyms-CpOAHPa-.js';
|
|
7
7
|
import { L as LanguageModule } from './language-DB8KG1z5.js';
|
|
8
8
|
export { T as TokenizerConfig } from './language-DB8KG1z5.js';
|
|
9
|
-
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, n as ComparisonFilter, D as DocumentImportResult, o as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, p as FacetResult, q as FieldFilter, r as FilterExpression, G as GeoFilter, s as GeoPolygonFilter, t as GeoRadiusFilter, u as GroupConfig, v as GroupReducer, w as GroupResult, H as HighlightConfig, x as HighlightMatch, y as Hit, z as HybridConfig,
|
|
10
|
-
import { N as NarsilConfig } from './config-
|
|
11
|
-
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-
|
|
12
|
-
import { N as Narsil } from './engine-
|
|
13
|
-
export { B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-
|
|
14
|
-
export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-
|
|
9
|
+
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, n as ComparisonFilter, D as DocumentImportResult, o as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, p as FacetResult, q as FieldFilter, r as FilterExpression, G as GeoFilter, s as GeoPolygonFilter, t as GeoRadiusFilter, u as GroupConfig, v as GroupReducer, w as GroupResult, H as HighlightConfig, x as HighlightMatch, y as Hit, z as HybridConfig, m as IndexInfo, I as IndexStats, L as ListParams, g as ListResult, J as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, f as NarsilEventHandlers, l as NarsilEventMap, k as PartitionStatsResult, P as PreflightResult, K as PresenceFilter, O as ProcessMemoryReport, R as QueryCoverage, Q as QueryParams, e as QueryResult, T as RerankConfig, U as ScoreComponents, W as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, X as SortField, Y as SortSpec, Z as StringFilter, h as SuggestParams, j as SuggestResult, _ as TermMatchPolicy, V as VectorMaintenanceResult, $ as VectorQueryConfig, a0 as WorkerCopyReport, i as isNarsilError } from './memory-CUEDwr-F.js';
|
|
10
|
+
import { N as NarsilConfig } from './config-P__Diiy4.js';
|
|
11
|
+
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-P__Diiy4.js';
|
|
12
|
+
import { N as Narsil } from './engine-BGCEm9E_.js';
|
|
13
|
+
export { A as AdapterRegistrationOperations, B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-BGCEm9E_.js';
|
|
14
|
+
export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Registers a tokeniser under a name that an index config can then reference.
|
package/dist/index.mjs
CHANGED
|
@@ -1,32 +1,33 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
|
|
5
|
-
export { registerPartialWordSearch, registerPatternSearch } from './chunk-CHOGQSHK.mjs';
|
|
6
|
-
import './chunk-WLOF6HSP.mjs';
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-VWULPWRJ.mjs';
|
|
2
|
+
import './chunk-NKRBEGJZ.mjs';
|
|
3
|
+
import './chunk-R7EEKAJE.mjs';
|
|
4
|
+
import './chunk-AY3AQ3UH.mjs';
|
|
7
5
|
import './chunk-AV4IJKD2.mjs';
|
|
8
|
-
|
|
9
|
-
export {
|
|
6
|
+
export { registerSynonyms } from './chunk-OWNZYMRE.mjs';
|
|
7
|
+
export { registerPartialWordSearch, registerPatternSearch } from './chunk-K25LZA54.mjs';
|
|
8
|
+
export { getLanguage, registerLanguage } from './chunk-JFRNNBOM.mjs';
|
|
9
|
+
import './chunk-HJSN7UYG.mjs';
|
|
10
|
+
import './chunk-LXWDQLTW.mjs';
|
|
10
11
|
import './chunk-TGP5M5R7.mjs';
|
|
11
12
|
import './chunk-U2BKYEC7.mjs';
|
|
12
13
|
export { isSimdAvailable } from './chunk-KOKM4PQN.mjs';
|
|
13
14
|
import './chunk-QY7GC2MW.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
15
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
16
|
+
import './chunk-ASY5PHYI.mjs';
|
|
16
17
|
import './chunk-WQBCBKY3.mjs';
|
|
17
|
-
import './chunk-
|
|
18
|
-
import './chunk-
|
|
19
|
-
import './chunk-
|
|
18
|
+
import './chunk-MHOSCQCQ.mjs';
|
|
19
|
+
import './chunk-LXPIXDUJ.mjs';
|
|
20
|
+
import './chunk-E74YLAIG.mjs';
|
|
20
21
|
import './chunk-KEHK2O6P.mjs';
|
|
21
22
|
import './chunk-GES7U2PM.mjs';
|
|
22
|
-
import './chunk-
|
|
23
|
-
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-
|
|
23
|
+
import './chunk-OOEXLUGR.mjs';
|
|
24
|
+
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-RONEOEON.mjs';
|
|
24
25
|
import './chunk-X5CV2E2B.mjs';
|
|
25
26
|
import './chunk-7X5JST5L.mjs';
|
|
26
|
-
export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-
|
|
27
|
+
export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-LU2CE5AE.mjs';
|
|
27
28
|
import './chunk-R6A773WK.mjs';
|
|
28
29
|
import './chunk-YTJCGKM4.mjs';
|
|
29
30
|
import './chunk-R6PR4WYZ.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
+
import './chunk-PPY2GSIB.mjs';
|
|
31
32
|
import './chunk-L5ORED7V.mjs';
|
|
32
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
33
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as AnyDocument, S as SchemaDefinition,
|
|
1
|
+
import { A as AnyDocument, S as SchemaDefinition, d as ScoringMode } from './schema-4W_7tk1f.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Every code a {@link NarsilError} carries, grouped by the part of the engine
|
|
@@ -51,6 +51,8 @@ declare const ErrorCodes: {
|
|
|
51
51
|
readonly EMBEDDING_DIMENSION_MISMATCH: "EMBEDDING_DIMENSION_MISMATCH";
|
|
52
52
|
readonly EMBEDDING_NO_SOURCE: "EMBEDDING_NO_SOURCE";
|
|
53
53
|
readonly EMBEDDING_CONFIG_INVALID: "EMBEDDING_CONFIG_INVALID";
|
|
54
|
+
readonly RERANK_FAILED: "RERANK_FAILED";
|
|
55
|
+
readonly RERANK_CONFIG_INVALID: "RERANK_CONFIG_INVALID";
|
|
54
56
|
readonly DOC_MISSING_REQUIRED_FIELD: "DOC_MISSING_REQUIRED_FIELD";
|
|
55
57
|
readonly CONFIG_INVALID: "CONFIG_INVALID";
|
|
56
58
|
readonly QUERY_ROUTING_FAILED: "QUERY_ROUTING_FAILED";
|
|
@@ -890,6 +892,49 @@ type FilterExpression = {
|
|
|
890
892
|
not?: FilterExpression;
|
|
891
893
|
};
|
|
892
894
|
|
|
895
|
+
/**
|
|
896
|
+
* How a query reorders its top hits with a rerank adapter.
|
|
897
|
+
*
|
|
898
|
+
* The engine ranks the query as it would without this setting, passes the
|
|
899
|
+
* top `window` hits to the adapter, and orders them by the adapter's scores.
|
|
900
|
+
* The hits past the window follow in their first ranking, with their scores
|
|
901
|
+
* scaled below the lowest reranked score, so every page of one query comes
|
|
902
|
+
* from the same order.
|
|
903
|
+
*
|
|
904
|
+
* @public
|
|
905
|
+
*/
|
|
906
|
+
interface RerankConfig {
|
|
907
|
+
/**
|
|
908
|
+
* The engine scores the hits with the rerank adapter registered under this
|
|
909
|
+
* key. It throws `RERANK_CONFIG_INVALID` for a key that it has not
|
|
910
|
+
* registered.
|
|
911
|
+
*/
|
|
912
|
+
adapter: string;
|
|
913
|
+
/**
|
|
914
|
+
* The engine sends the adapter the values of these text fields, in this
|
|
915
|
+
* order and joined with newlines, as the text of each hit. Each member of a
|
|
916
|
+
* list field takes a line of its own. The engine throws
|
|
917
|
+
* `RERANK_CONFIG_INVALID` for an empty list, for a field that the schema
|
|
918
|
+
* does not declare as text, and for a field that the list gives twice.
|
|
919
|
+
*/
|
|
920
|
+
fields: string[];
|
|
921
|
+
/**
|
|
922
|
+
* The adapter compares each hit with this text. Where you leave it out, the
|
|
923
|
+
* engine sends the query's `term`, or else its `vector.text`, and it throws
|
|
924
|
+
* `RERANK_CONFIG_INVALID` where the query carries neither.
|
|
925
|
+
*/
|
|
926
|
+
query?: string;
|
|
927
|
+
/**
|
|
928
|
+
* The engine reranks this many of the top hits, from 1 to the engine's
|
|
929
|
+
* `maxRerankWindow`. Where you leave the window out, the engine reranks the
|
|
930
|
+
* smallest of 100 and `maxRerankWindow`, and no more than the `hybrid.depth`
|
|
931
|
+
* of a hybrid query that sets one. A hybrid query that sets `hybrid.depth`
|
|
932
|
+
* reranks its fused list at that depth, so the engine throws
|
|
933
|
+
* `RERANK_CONFIG_INVALID` when you set a window above that depth.
|
|
934
|
+
*/
|
|
935
|
+
window?: number;
|
|
936
|
+
}
|
|
937
|
+
|
|
893
938
|
/**
|
|
894
939
|
* Vector-search inputs passed under `QueryParams.vector`.
|
|
895
940
|
*
|
|
@@ -983,6 +1028,19 @@ interface HybridConfig {
|
|
|
983
1028
|
k?: number;
|
|
984
1029
|
/** The engine uses this weight of the vector score, from 0 to 1, under `linear` alone, and 0.5 where you leave it out. It throws `CONFIG_INVALID` for a value outside that range. */
|
|
985
1030
|
alpha?: number;
|
|
1031
|
+
/**
|
|
1032
|
+
* Set a depth to get the same fused order on every page of one query, and
|
|
1033
|
+
* a cursor that resumes it. The engine then fetches this many results from
|
|
1034
|
+
* the keyword search and from the vector search, fuses them, and pages
|
|
1035
|
+
* through the top `depth` of the fused list. The last page stops at this
|
|
1036
|
+
* depth, so it can hold fewer hits than the limit. Set a whole number from 1
|
|
1037
|
+
* to 10,000. The engine throws `CONFIG_INVALID` for any other value, and
|
|
1038
|
+
* `SEARCH_RESULT_WINDOW_EXCEEDED` for a page that starts at this depth or
|
|
1039
|
+
* beyond. Where you leave the depth out, each search fetches only enough
|
|
1040
|
+
* results to fill the page, so the fused order can change from one page to
|
|
1041
|
+
* the next, and a cluster refuses a cursor with `SEARCH_INVALID_CURSOR`.
|
|
1042
|
+
*/
|
|
1043
|
+
depth?: number;
|
|
986
1044
|
}
|
|
987
1045
|
|
|
988
1046
|
/**
|
|
@@ -1231,6 +1289,16 @@ interface QueryParams {
|
|
|
1231
1289
|
vector?: VectorQueryConfig;
|
|
1232
1290
|
/** These settings control how the keyword and vector rankings merge in `hybrid` mode. */
|
|
1233
1291
|
hybrid?: HybridConfig;
|
|
1292
|
+
/**
|
|
1293
|
+
* These settings reorder the top hits with a rerank adapter, such as a
|
|
1294
|
+
* cross-encoder, after the keyword, vector, or hybrid ranking. The count
|
|
1295
|
+
* matches the count of the same query without `rerank`, while the facets
|
|
1296
|
+
* and the groups come from the first ranking. The engine throws
|
|
1297
|
+
* `SEARCH_INVALID_MODE` for a query that also sets a sort, and
|
|
1298
|
+
* `RERANK_FAILED` where the adapter fails or outlasts the engine's
|
|
1299
|
+
* `rerankTimeout`.
|
|
1300
|
+
*/
|
|
1301
|
+
rerank?: RerankConfig;
|
|
1234
1302
|
/**
|
|
1235
1303
|
* Setting this returns relevance scores on a sorted query. A query that
|
|
1236
1304
|
* names a sort ranks by sort values alone and computes no scores, so its
|
|
@@ -1631,4 +1699,4 @@ interface MemoryStats {
|
|
|
1631
1699
|
}>;
|
|
1632
1700
|
}
|
|
1633
1701
|
|
|
1634
|
-
export { type
|
|
1702
|
+
export { type VectorQueryConfig as $, type ArrayFilter as A, type BatchResult as B, type ClientErrorCode as C, type DocumentImportResult as D, type ErrorCode as E, type FacetConfig as F, type GeoFilter as G, type HighlightConfig as H, type IndexStats as I, type ListedDocument as J, type PresenceFilter as K, type ListParams as L, type MemoryStats as M, NarsilError as N, type ProcessMemoryReport as O, type PreflightResult as P, type QueryParams as Q, type QueryCoverage as R, type ServerErrorCode as S, type RerankConfig as T, type ScoreComponents as U, type VectorMaintenanceResult as V, type SearchMode as W, type SortField as X, type SortSpec as Y, type StringFilter as Z, type TermMatchPolicy as _, ClientErrorCodes as a, type WorkerCopyReport as a0, ErrorCodes as b, type NarsilErrorCode as c, ServerErrorCodes as d, type QueryResult as e, type NarsilEventHandlers as f, type ListResult as g, type SuggestParams as h, isNarsilError as i, type SuggestResult as j, type PartitionStatsResult as k, type NarsilEventMap as l, type IndexInfo as m, type ComparisonFilter as n, type DocumentProjection as o, type FacetResult as p, type FieldFilter as q, type FilterExpression as r, type GeoPolygonFilter as s, type GeoRadiusFilter as t, type GroupConfig as u, type GroupReducer as v, type GroupResult as w, type HighlightMatch as x, type Hit as y, type HybridConfig as z };
|
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, WORKER_COPY_MIN_VECTORS, allLiveDocIds, recalibrateFromStore, signatureOf, adoptGraph, calibrateQuantizer, fieldHandlesOf, BUILD_CHUNK_SIZE, yieldToEventLoop, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-
|
|
2
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
3
|
-
import { createVectorStore } from './chunk-
|
|
4
|
-
export { openSharedVectorField } from './chunk-
|
|
5
|
-
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, releaseLocksHeldBy } from './chunk-
|
|
6
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
7
|
-
import './chunk-37CD3OKS.mjs';
|
|
1
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, WORKER_COPY_MIN_VECTORS, allLiveDocIds, recalibrateFromStore, signatureOf, adoptGraph, calibrateQuantizer, fieldHandlesOf, BUILD_CHUNK_SIZE, yieldToEventLoop, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-COVCQ73F.mjs';
|
|
2
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-WCJKUGHB.mjs';
|
|
3
|
+
import { createVectorStore } from './chunk-UCDF35PU.mjs';
|
|
4
|
+
export { openSharedVectorField } from './chunk-UCDF35PU.mjs';
|
|
5
|
+
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, releaseLocksHeldBy } from './chunk-HXJBSHXF.mjs';
|
|
6
|
+
export { releaseLocksHeldBy } from './chunk-HXJBSHXF.mjs';
|
|
8
7
|
import { noteFallback } from './chunk-DJHHF2L5.mjs';
|
|
8
|
+
import './chunk-37CD3OKS.mjs';
|
|
9
9
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
10
|
-
import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-
|
|
10
|
+
import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-AY3AQ3UH.mjs';
|
|
11
11
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-LXWDQLTW.mjs';
|
|
13
13
|
import './chunk-KOKM4PQN.mjs';
|
|
14
14
|
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-QY7GC2MW.mjs';
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-
|
|
15
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
16
|
+
import './chunk-E74YLAIG.mjs';
|
|
17
17
|
import './chunk-KEHK2O6P.mjs';
|
|
18
18
|
import './chunk-R6A773WK.mjs';
|
|
19
19
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
20
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
20
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
21
21
|
|
|
22
22
|
// src/core/heap.ts
|
|
23
23
|
function createMaxHeap(compare) {
|
|
@@ -275,7 +275,7 @@ function createSharedVectorSearcher(options) {
|
|
|
275
275
|
return { results: scan.results, matched: scan.matched, matchedExact: true, resolvedEveryHit: true };
|
|
276
276
|
}
|
|
277
277
|
function search2(query, k, searchOptions, filter) {
|
|
278
|
-
const { metric, minSimilarity, efSearch, oversample } = searchOptions;
|
|
278
|
+
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
279
279
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
280
280
|
const liveDocuments = flags.liveDocuments();
|
|
281
281
|
if (liveDocuments <= 0) return empty;
|
|
@@ -303,7 +303,12 @@ function createSharedVectorSearcher(options) {
|
|
|
303
303
|
}
|
|
304
304
|
let resolvedEveryHit = true;
|
|
305
305
|
const graphResults = [];
|
|
306
|
-
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
306
|
+
for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
|
|
307
|
+
filter,
|
|
308
|
+
efSearch,
|
|
309
|
+
oversample,
|
|
310
|
+
rescoreWindow
|
|
311
|
+
})) {
|
|
307
312
|
const docId = view.docIdOf(hit.ord);
|
|
308
313
|
if (docId === void 0) {
|
|
309
314
|
resolvedEveryHit = false;
|
|
@@ -403,7 +408,8 @@ function graphSearchOptionsOf(options) {
|
|
|
403
408
|
return {
|
|
404
409
|
...options.filter !== void 0 ? { filter: options.filter } : {},
|
|
405
410
|
...options.efSearch !== void 0 ? { efSearch: options.efSearch } : {},
|
|
406
|
-
...options.oversample !== void 0 ? { oversample: options.oversample } : {}
|
|
411
|
+
...options.oversample !== void 0 ? { oversample: options.oversample } : {},
|
|
412
|
+
...options.rescoreWindow !== void 0 ? { rescoreWindow: options.rescoreWindow } : {}
|
|
407
413
|
};
|
|
408
414
|
}
|
|
409
415
|
async function createVectorSearchPool(requestedCount) {
|
|
@@ -1565,7 +1571,7 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
1565
1571
|
);
|
|
1566
1572
|
}
|
|
1567
1573
|
const currentLiveSize = liveSize(state);
|
|
1568
|
-
const { metric, minSimilarity, efSearch, oversample } = options;
|
|
1574
|
+
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
|
|
1569
1575
|
if (currentLiveSize === 0) return { results: [], matched: 0, matchedExact: true };
|
|
1570
1576
|
if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
|
|
1571
1577
|
if (k <= 0) return fetchedOutcome(state, [], minSimilarity, filter);
|
|
@@ -1584,7 +1590,7 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
1584
1590
|
return scannedOutcome(scan);
|
|
1585
1591
|
}
|
|
1586
1592
|
}
|
|
1587
|
-
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
1593
|
+
const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
1588
1594
|
return fetchedOutcome(
|
|
1589
1595
|
state,
|
|
1590
1596
|
withBufferedVectors(state, query, k, options, filter, hnswResults),
|
|
@@ -1771,7 +1777,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
1771
1777
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
1772
1778
|
filter,
|
|
1773
1779
|
efSearch: options.efSearch,
|
|
1774
|
-
oversample: options.oversample
|
|
1780
|
+
oversample: options.oversample,
|
|
1781
|
+
rescoreWindow: options.rescoreWindow
|
|
1775
1782
|
});
|
|
1776
1783
|
if (confined && state.revision !== filterRevision) {
|
|
1777
1784
|
filter = filterForOptions(state, options);
|