@delali/narsil 0.2.4-next.59 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-75XIEF6H.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from '../chunk-BWJVNZPR.mjs';
|
|
3
|
+
|
|
4
|
+
// src/rerankers/constants.ts
|
|
5
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
6
|
+
var DEFAULT_MAX_RETRIES = 3;
|
|
7
|
+
var DEFAULT_MAX_DOCUMENTS_PER_REQUEST = 1e3;
|
|
8
|
+
|
|
9
|
+
// src/rerankers/http.ts
|
|
10
|
+
var SERVICE = "Rerank service";
|
|
11
|
+
function invalid(message) {
|
|
12
|
+
return new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `HTTP rerank adapter ${message}`);
|
|
13
|
+
}
|
|
14
|
+
function requireConfig(config) {
|
|
15
|
+
if (typeof config?.baseUrl !== "string" || config.baseUrl.trim().length === 0) {
|
|
16
|
+
throw invalid("requires a non-empty baseUrl");
|
|
17
|
+
}
|
|
18
|
+
let protocol;
|
|
19
|
+
try {
|
|
20
|
+
protocol = new URL(config.baseUrl).protocol;
|
|
21
|
+
} catch {
|
|
22
|
+
throw invalid("requires baseUrl to be an absolute URL");
|
|
23
|
+
}
|
|
24
|
+
if (protocol !== "https:" && protocol !== "http:") throw invalid("requires an http or https baseUrl");
|
|
25
|
+
if (typeof config.model !== "string" || config.model.trim().length === 0) throw invalid("requires a non-empty model");
|
|
26
|
+
if (config.timeout !== void 0 && (!Number.isFinite(config.timeout) || config.timeout <= 0)) {
|
|
27
|
+
throw invalid("timeout must be a positive number");
|
|
28
|
+
}
|
|
29
|
+
if (config.maxRetries !== void 0 && (!Number.isInteger(config.maxRetries) || config.maxRetries < 0)) {
|
|
30
|
+
throw invalid("maxRetries must be a non-negative integer");
|
|
31
|
+
}
|
|
32
|
+
const perRequest = config.maxDocumentsPerRequest;
|
|
33
|
+
if (perRequest !== void 0 && (!Number.isInteger(perRequest) || perRequest < 1)) {
|
|
34
|
+
throw invalid("maxDocumentsPerRequest must be a positive integer");
|
|
35
|
+
}
|
|
36
|
+
const apiKey = config.apiKey;
|
|
37
|
+
if (apiKey !== void 0 && typeof apiKey !== "string" && typeof apiKey !== "function") {
|
|
38
|
+
throw invalid("apiKey must be a string or a function returning a string");
|
|
39
|
+
}
|
|
40
|
+
if (typeof apiKey === "string" && apiKey.trim().length === 0) throw invalid("requires a non-empty apiKey");
|
|
41
|
+
}
|
|
42
|
+
function failed(message, details) {
|
|
43
|
+
return new NarsilError(ErrorCodes.RERANK_FAILED, `${SERVICE} ${message}`, details);
|
|
44
|
+
}
|
|
45
|
+
function scoresFromResults(results, count) {
|
|
46
|
+
if (!Array.isArray(results)) throw failed('response is missing a "results" array');
|
|
47
|
+
if (results.length !== count) {
|
|
48
|
+
throw failed(`returned ${results.length} results for ${count} documents`, {
|
|
49
|
+
expected: count,
|
|
50
|
+
actual: results.length
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
const scores = new Array(count);
|
|
54
|
+
const seen = new Uint8Array(count);
|
|
55
|
+
for (let i = 0; i < results.length; i++) {
|
|
56
|
+
const item = results[i];
|
|
57
|
+
const index = typeof item === "object" && item !== null ? item.index : void 0;
|
|
58
|
+
const score = typeof item === "object" && item !== null ? item.relevance_score : void 0;
|
|
59
|
+
if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= count || seen[index] === 1) {
|
|
60
|
+
throw failed(`result at position ${i} carries no index of a document it was sent, or repeats one`);
|
|
61
|
+
}
|
|
62
|
+
if (typeof score !== "number" || !Number.isFinite(score)) {
|
|
63
|
+
throw failed(`result at position ${i} carries no finite "relevance_score"`);
|
|
64
|
+
}
|
|
65
|
+
seen[index] = 1;
|
|
66
|
+
scores[index] = score;
|
|
67
|
+
}
|
|
68
|
+
return scores;
|
|
69
|
+
}
|
|
70
|
+
function createHttpReranker(config) {
|
|
71
|
+
requireConfig(config);
|
|
72
|
+
const url = `${config.baseUrl.replace(/\/+$/, "")}/rerank`;
|
|
73
|
+
const model = config.model;
|
|
74
|
+
const apiKeySource = config.apiKey;
|
|
75
|
+
const timeoutMs = config.timeout ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
76
|
+
const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
77
|
+
const perRequest = config.maxDocumentsPerRequest ?? DEFAULT_MAX_DOCUMENTS_PER_REQUEST;
|
|
78
|
+
return {
|
|
79
|
+
async rerank(query, documents, signal) {
|
|
80
|
+
if (documents.length === 0) return [];
|
|
81
|
+
const apiKey = apiKeySource === void 0 ? null : await resolveApiKey(apiKeySource, "HTTP rerank adapter", ErrorCodes.RERANK_CONFIG_INVALID);
|
|
82
|
+
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
83
|
+
const scores = [];
|
|
84
|
+
for (let start = 0; start < documents.length; start += perRequest) {
|
|
85
|
+
const chunk = documents.slice(start, start + perRequest);
|
|
86
|
+
const response = await postJsonWithRetry({
|
|
87
|
+
url,
|
|
88
|
+
body: JSON.stringify({ model, query, documents: chunk, top_n: chunk.length }),
|
|
89
|
+
apiKey,
|
|
90
|
+
maxRetries,
|
|
91
|
+
signal: combinedSignal,
|
|
92
|
+
service: SERVICE,
|
|
93
|
+
failureCode: ErrorCodes.RERANK_FAILED
|
|
94
|
+
});
|
|
95
|
+
for (const score of scoresFromResults(response.results, chunk.length)) scores.push(score);
|
|
96
|
+
}
|
|
97
|
+
return scores;
|
|
98
|
+
},
|
|
99
|
+
async shutdown() {
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export { createHttpReranker };
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// src/narsil/restore-queue.ts
|
|
2
|
+
function restoreDepsOf(core, host) {
|
|
3
|
+
return {
|
|
4
|
+
...host,
|
|
5
|
+
executor: core.executor,
|
|
6
|
+
indexRegistry: core.indexRegistry,
|
|
7
|
+
durability: core.durability,
|
|
8
|
+
filesystemDurability: core.filesystemDurability,
|
|
9
|
+
embeddingAdapters: core.embeddingAdapters,
|
|
10
|
+
markAnalysisStale: core.analysisRebuild.markStale,
|
|
11
|
+
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
12
|
+
abandonAnalysisRebuild: core.analysisRebuild.abandon,
|
|
13
|
+
indexState: core.indexState,
|
|
14
|
+
copies: core.orchestrator,
|
|
15
|
+
forgetWatermark: core.watermarkNotifier.forget
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
|
+
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
|
+
const restored = previous.then(async () => {
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-QGZPODM5.mjs');
|
|
22
|
+
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
|
+
});
|
|
24
|
+
const settled = restored.then(
|
|
25
|
+
() => void 0,
|
|
26
|
+
() => void 0
|
|
27
|
+
);
|
|
28
|
+
restoresInFlight.set(indexName, settled);
|
|
29
|
+
void settled.then(() => {
|
|
30
|
+
if (restoresInFlight.get(indexName) === settled) restoresInFlight.delete(indexName);
|
|
31
|
+
});
|
|
32
|
+
return restored;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { queueRestore };
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-YEEQHT4V.mjs';
|
|
2
|
+
export { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
|
+
import { rerankOnHost } from './chunk-2RIRGOUL.mjs';
|
|
4
|
+
export { rerankOnHost } from './chunk-2RIRGOUL.mjs';
|
|
5
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-EWZLIYHV.mjs';
|
|
6
|
+
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
7
|
+
import './chunk-LXWDQLTW.mjs';
|
|
8
|
+
import { HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|
|
9
|
+
import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
|
|
10
|
+
|
|
11
|
+
// src/engine/query/hybrid-depth.ts
|
|
12
|
+
function countOfUnion(text, vectorScored) {
|
|
13
|
+
const textIds = /* @__PURE__ */ new Set();
|
|
14
|
+
for (const doc of text.scored) textIds.add(doc.docId);
|
|
15
|
+
let vectorOnly = 0;
|
|
16
|
+
for (const doc of vectorScored) if (!textIds.has(doc.docId)) vectorOnly++;
|
|
17
|
+
if (text.scored.length === text.totalMatched) return text.totalMatched + vectorOnly;
|
|
18
|
+
return Math.max(text.totalMatched, text.scored.length + vectorOnly);
|
|
19
|
+
}
|
|
20
|
+
async function fusedAtDepth(params, context, depth, legs) {
|
|
21
|
+
const { textLeg, vectorLeg } = legs;
|
|
22
|
+
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, searchAfter: _after, pinned: _pinned, ...rest } = params;
|
|
23
|
+
const fusion = legs.resolveHybridFusion(params.hybrid);
|
|
24
|
+
const textOnlyParams = { ...rest, limit: depth, offset: 0 };
|
|
25
|
+
const filtered = legs.filteredSearchOf(params, context);
|
|
26
|
+
if (filtered !== void 0 && legs.filterSubsetsMatchNothing(filtered.first)) {
|
|
27
|
+
return { scored: [], totalMatched: 0 };
|
|
28
|
+
}
|
|
29
|
+
if (!vectorConfig || !vectorConfig.value) {
|
|
30
|
+
const textOnly = await textLeg(textOnlyParams, context);
|
|
31
|
+
return { scored: textOnly.scored, totalMatched: textOnly.totalMatched, facets: textOnly.facets };
|
|
32
|
+
}
|
|
33
|
+
const vectorPending = vectorLeg(
|
|
34
|
+
vectorConfig,
|
|
35
|
+
new Float32Array(vectorConfig.value),
|
|
36
|
+
context,
|
|
37
|
+
filtered,
|
|
38
|
+
depth,
|
|
39
|
+
HYBRID_RESCORE_WINDOW
|
|
40
|
+
);
|
|
41
|
+
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
42
|
+
const vectorScored = vectorOutcome.scored;
|
|
43
|
+
const textScored = textFanOutResult.scored.slice(0, depth);
|
|
44
|
+
let fusedScored;
|
|
45
|
+
if (fusion.strategy === "rrf") {
|
|
46
|
+
fusedScored = legs.reciprocalRankFusion([textScored, vectorScored], { k: fusion.k });
|
|
47
|
+
} else {
|
|
48
|
+
fusedScored = legs.linearCombination(textScored, vectorScored, { alpha: fusion.alpha });
|
|
49
|
+
}
|
|
50
|
+
const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
|
|
51
|
+
const threshold = params.minScore ?? 0;
|
|
52
|
+
const filtersByScore = threshold > 0;
|
|
53
|
+
if (filtersByScore) fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
|
|
54
|
+
return {
|
|
55
|
+
scored: fusedScored.length > depth ? fusedScored.slice(0, depth) : fusedScored,
|
|
56
|
+
totalMatched: filtersByScore ? fusedScored.length : countOfUnion(textFanOutResult, vectorScored),
|
|
57
|
+
matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch && (!filtersByScore || textScored.length === textFanOutResult.totalMatched),
|
|
58
|
+
facets: textFanOutResult.facets
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
async function searchFixedDepthHybrid(params, context, { cursorDepth, offset, limit }, host) {
|
|
62
|
+
const depth = fixedHybridDepthOf(params.hybrid) ?? 0;
|
|
63
|
+
requireWithinHybridDepth(cursorDepth, offset, depth);
|
|
64
|
+
const result = await fusedAtDepth(params, context, depth, host);
|
|
65
|
+
const placesPins = context.partitionIds === void 0;
|
|
66
|
+
const paginate = (placed, _limit, _offset, _binding, _cursor, _sort, pinnedIds) => {
|
|
67
|
+
const pins = placesPins ? pinnedIds : void 0;
|
|
68
|
+
const organic = pins === void 0 ? placed : result.scored.map((doc) => ({ id: doc.docId }));
|
|
69
|
+
const { page, cursor } = pageByPosition(placed, organic, {
|
|
70
|
+
start: cursorDepth + offset,
|
|
71
|
+
limit,
|
|
72
|
+
pinnedIds: pins,
|
|
73
|
+
binding: context.cursorBinding,
|
|
74
|
+
encodeCursor: host.encodePageCursor
|
|
75
|
+
});
|
|
76
|
+
return { paginated: page, nextCursor: cursor };
|
|
77
|
+
};
|
|
78
|
+
return { ...result, paginate };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/engine/query/rerank.ts
|
|
82
|
+
async function executeRerankedQuery(request, context, firstStage, host) {
|
|
83
|
+
const startTime = host.now();
|
|
84
|
+
const runner = context.rerank;
|
|
85
|
+
if (runner === void 0) {
|
|
86
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "This search path holds no rerank adapters");
|
|
87
|
+
}
|
|
88
|
+
const { manager } = context;
|
|
89
|
+
const result = await runRerankedQuery(request, {
|
|
90
|
+
host,
|
|
91
|
+
schema: context.config.schema,
|
|
92
|
+
maxWindow: runner.maxWindow,
|
|
93
|
+
binding: context.cursorBinding,
|
|
94
|
+
clampsPins: true,
|
|
95
|
+
fusesRankings: (params) => params.vector !== void 0 && (params.mode === "hybrid" || params.term !== void 0 && params.term.trim().length > 0),
|
|
96
|
+
firstStage: (params) => firstStage(params, context),
|
|
97
|
+
async texts(docIds, fields) {
|
|
98
|
+
return docIds.map((docId) => rerankTextOf(manager.fieldValues(docId, fields)));
|
|
99
|
+
},
|
|
100
|
+
async storedPins(docIds) {
|
|
101
|
+
return new Set(docIds.filter((docId) => manager.getRef(docId) !== void 0));
|
|
102
|
+
},
|
|
103
|
+
score: (adapter, query, documents) => runner.score(adapter, query, documents),
|
|
104
|
+
async fillDocuments(hits, groups, projection) {
|
|
105
|
+
const read = (docId) => projection.kind === "none" ? {} : manager.get(docId, projection) ?? {};
|
|
106
|
+
for (const hit of hits) hit.document = read(hit.id);
|
|
107
|
+
for (const group of groups ?? []) {
|
|
108
|
+
for (const hit of group.hits) hit.document = read(hit.id);
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
highlight(page, projection) {
|
|
112
|
+
const fields = request.highlight?.fields ?? [];
|
|
113
|
+
const dropped = fields.some((field) => !host.projectionKeepsField(projection, field));
|
|
114
|
+
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
115
|
+
host.applyHighlights(
|
|
116
|
+
page,
|
|
117
|
+
request,
|
|
118
|
+
context.language,
|
|
119
|
+
manager.analysis,
|
|
120
|
+
manager.schema,
|
|
121
|
+
readStoredDocument,
|
|
122
|
+
manager.synonyms
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
result.elapsed = host.now() - startTime;
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// src/rerank/runner.ts
|
|
131
|
+
function rerankRunnerFor(core) {
|
|
132
|
+
return {
|
|
133
|
+
maxWindow: core.rerankSettings.maxWindow,
|
|
134
|
+
score: (adapter, query, documents) => rerankOnHost(core, adapter, query, documents)
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// src/rerank/settings.ts
|
|
139
|
+
function wholeNumberIn(value, low, high) {
|
|
140
|
+
return typeof value === "number" && Number.isInteger(value) && value >= low && value <= high;
|
|
141
|
+
}
|
|
142
|
+
function resolveRerankSettings(config) {
|
|
143
|
+
const timeout = config?.rerankTimeout;
|
|
144
|
+
const maxWindow = config?.maxRerankWindow;
|
|
145
|
+
if (timeout !== void 0 && !wholeNumberIn(timeout, MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS)) {
|
|
146
|
+
throw new NarsilError(
|
|
147
|
+
ErrorCodes.CONFIG_INVALID,
|
|
148
|
+
`rerankTimeout must be a whole number of milliseconds from ${MIN_RERANK_TIMEOUT_MS} to ${MAX_RERANK_TIMEOUT_MS}`,
|
|
149
|
+
{ rerankTimeout: timeout }
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
if (maxWindow !== void 0 && !wholeNumberIn(maxWindow, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW)) {
|
|
153
|
+
throw new NarsilError(
|
|
154
|
+
ErrorCodes.CONFIG_INVALID,
|
|
155
|
+
`maxRerankWindow must be a whole number from ${MIN_RERANK_WINDOW} to ${MAX_RERANK_WINDOW}`,
|
|
156
|
+
{
|
|
157
|
+
maxRerankWindow: maxWindow
|
|
158
|
+
}
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
timeoutMs: timeout ?? DEFAULT_RERANK_TIMEOUT_MS,
|
|
163
|
+
maxWindow: maxWindow ?? DEFAULT_MAX_RERANK_WINDOW
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
async function shutdownRerankAdapters(adapters) {
|
|
167
|
+
const distinct = new Set(adapters.values());
|
|
168
|
+
adapters.clear();
|
|
169
|
+
for (const adapter of distinct) {
|
|
170
|
+
try {
|
|
171
|
+
await adapter.shutdown?.();
|
|
172
|
+
} catch {
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export { executeRerankedQuery, rerankRunnerFor, resolveRerankSettings, searchFixedDepthHybrid, shutdownRerankAdapters };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from './adapters-
|
|
1
|
+
import { E as EmbeddingAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* A document as the engine stores it: any set of fields, with an optional `id`.
|
|
@@ -13,20 +14,69 @@ type AnyDocument = Record<string, unknown> & {
|
|
|
13
14
|
id?: string;
|
|
14
15
|
};
|
|
15
16
|
/**
|
|
16
|
-
*
|
|
17
|
+
* Each of these options can follow the base type of a `string` or `string[]`
|
|
18
|
+
* field after a colon, as in `string:sortable:pattern`.
|
|
17
19
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
* The engine can sort by a field whose type includes `sortable`. On a single
|
|
21
|
+
* `string` field with that option, the engine can also test a text range with
|
|
22
|
+
* `gt`, `gte`, `lt`, `lte`, or `between`. The engine keeps a pattern index for
|
|
23
|
+
* a field whose type includes `pattern`. It answers `contains`, `eq`, `ne`,
|
|
24
|
+
* `in`, `nin`, `startsWith`, and `endsWith` on that field from the index,
|
|
25
|
+
* comparing case folds for any of those tests that sets `caseFold`. The engine
|
|
26
|
+
* raises `DOC_VALIDATION_FAILED` for a value longer than
|
|
27
|
+
* {@link IndexConfig.patternValueLimit} in such a field. For a query with
|
|
28
|
+
* {@link QueryParams.partialWords} set, the engine also matches each query
|
|
29
|
+
* word inside the longer words of a field whose type includes `partial`. It
|
|
30
|
+
* raises `CONFIG_INVALID` for such a field in an index whose `surfaceForms`
|
|
31
|
+
* setting is false, because it matches inside the original spellings of
|
|
32
|
+
* words, which it records only while that setting is on.
|
|
22
33
|
*
|
|
23
|
-
*
|
|
24
|
-
|
|
25
|
-
|
|
34
|
+
* @public
|
|
35
|
+
*/
|
|
36
|
+
type TextFieldOption = 'sortable' | 'pattern' | 'partial';
|
|
37
|
+
/**
|
|
38
|
+
* This union contains every string of options that can follow a base type,
|
|
39
|
+
* with each option after a colon. The options can come in any order, although
|
|
40
|
+
* each one can appear only once.
|
|
26
41
|
*
|
|
27
42
|
* @public
|
|
28
43
|
*/
|
|
29
|
-
type
|
|
44
|
+
type FieldOptionChain<Remaining extends string> = {
|
|
45
|
+
[Option in Remaining]: `:${Option}` | `:${Option}${FieldOptionChain<Exclude<Remaining, Option>>}`;
|
|
46
|
+
}[Remaining];
|
|
47
|
+
/**
|
|
48
|
+
* The engine indexes each field of a schema according to the field's type,
|
|
49
|
+
* which is one of these.
|
|
50
|
+
*
|
|
51
|
+
* The engine splits a `string` value into words, so it can find the document
|
|
52
|
+
* by any word in the value. It can filter and sort by a `number`, `boolean`,
|
|
53
|
+
* or `enum` field. A `geopoint` value is a latitude and longitude pair. A
|
|
54
|
+
* `vector[N]` value is an embedding with N dimensions. For each list form, the
|
|
55
|
+
* engine indexes every element of the list.
|
|
56
|
+
*
|
|
57
|
+
* The engine stores and compares a `verbatim` value exactly as written, as one
|
|
58
|
+
* piece of text. Because the engine keeps that value whole, it leaves the
|
|
59
|
+
* field out of keyword search, although it can test the whole value in a
|
|
60
|
+
* filter. The engine keeps a pattern index for every `verbatim` field, so it
|
|
61
|
+
* can also test such a field with `contains` and `caseFold`.
|
|
62
|
+
*
|
|
63
|
+
* Add the {@link TextFieldOption} options to a `string` or `string[]` type, or
|
|
64
|
+
* `sortable` alone to a `verbatim` or `verbatim[]` type, as in
|
|
65
|
+
* `string:sortable:pattern` or `verbatim[]:sortable`. The engine stores the
|
|
66
|
+
* options in the order `sortable`, `pattern`, `partial`, whatever their order
|
|
67
|
+
* in the type name. It raises `SCHEMA_INVALID_TYPE` for any other
|
|
68
|
+
* option, such as `number:sortable` or `verbatim:pattern`, and for an option
|
|
69
|
+
* written twice.
|
|
70
|
+
*
|
|
71
|
+
* To sort by a text field, the engine keeps up to the first 512 code points of
|
|
72
|
+
* every document's value in memory, against 8 bytes per document for a number
|
|
73
|
+
* field. For that reason, the engine sorts by a text field only where the
|
|
74
|
+
* field's type includes `sortable`, while it raises `SEARCH_INVALID_FIELD`
|
|
75
|
+
* for a sort on any other text field.
|
|
76
|
+
*
|
|
77
|
+
* @public
|
|
78
|
+
*/
|
|
79
|
+
type FieldType = 'string' | `string${FieldOptionChain<TextFieldOption>}` | 'string[]' | `string[]${FieldOptionChain<TextFieldOption>}` | 'verbatim' | 'verbatim:sortable' | 'verbatim[]' | 'verbatim[]:sortable' | 'number' | 'boolean' | 'enum' | 'geopoint' | `vector[${number}]` | 'number[]' | 'boolean[]' | 'enum[]';
|
|
30
80
|
/**
|
|
31
81
|
* The field layout of an index, written as a plain object.
|
|
32
82
|
*
|
|
@@ -173,6 +223,26 @@ interface IndexConfig {
|
|
|
173
223
|
* insert throughput and one entry per changed spelling.
|
|
174
224
|
*/
|
|
175
225
|
surfaceForms?: boolean;
|
|
226
|
+
/**
|
|
227
|
+
* The engine raises `DOC_VALIDATION_FAILED` for a document whose value in a
|
|
228
|
+
* `verbatim` field, or in a field whose type includes `pattern`, is longer
|
|
229
|
+
* than this many code points. It tests each element of a list on its own.
|
|
230
|
+
* Set a whole number from 1 to 65,536, or leave this unset for a limit of
|
|
231
|
+
* 8,192. For any other value, {@link Narsil.createIndex} throws
|
|
232
|
+
* `CONFIG_INVALID`.
|
|
233
|
+
*/
|
|
234
|
+
patternValueLimit?: number;
|
|
235
|
+
/**
|
|
236
|
+
* A search for any word in a rule of this set also matches the synonyms
|
|
237
|
+
* that the rule gives that word, and a document that holds a synonym scores
|
|
238
|
+
* as though it held the word itself. The engine analyses each word with the
|
|
239
|
+
* index's language, tokeniser, and stop words, so a word must analyse to
|
|
240
|
+
* exactly one token. {@link Narsil.createIndex} throws `CONFIG_INVALID` for
|
|
241
|
+
* a stop word, a phrase, a word that holds a control character, or a set
|
|
242
|
+
* beyond 100,000 rules, 500,000 words, or 4,194,304 bytes of words in UTF-8.
|
|
243
|
+
* {@link SynonymOperations.updateSynonyms} replaces the set without a rebuild.
|
|
244
|
+
*/
|
|
245
|
+
synonyms?: SynonymRule[];
|
|
176
246
|
}
|
|
177
247
|
/**
|
|
178
248
|
* BM25 tuning for an index.
|
|
@@ -286,4 +356,4 @@ interface InsertOptions extends WriteOptions {
|
|
|
286
356
|
skipClone?: boolean;
|
|
287
357
|
}
|
|
288
358
|
|
|
289
|
-
export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E,
|
|
359
|
+
export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldOptionChain as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, TextFieldOption as T, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b, FieldType as c, ScoringMode as d, VectorQuantizationMode as e, VectorStorageMode as f };
|
package/dist/server.d.ts
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './
|
|
2
|
-
import {
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY,
|
|
4
|
-
import {
|
|
5
|
-
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-
|
|
6
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-
|
|
7
|
-
import './schema-
|
|
8
|
-
import './adapters-
|
|
9
|
-
import './
|
|
10
|
-
import './config-
|
|
11
|
-
import './
|
|
12
|
-
import './
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-DqOIg5xa.js';
|
|
2
|
+
import { c as ServerOptions, N as NarsilServer } from './capabilities-DiGlB6Sr.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-DiGlB6Sr.js';
|
|
4
|
+
import { N as Narsil } from './engine-rC0vjjY-.js';
|
|
5
|
+
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-CwPTz-tP.js';
|
|
6
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-CwPTz-tP.js';
|
|
7
|
+
import './schema-4W_7tk1f.js';
|
|
8
|
+
import './adapters-BQFQ4PRz.js';
|
|
9
|
+
import './synonyms-CpOAHPa-.js';
|
|
10
|
+
import './config-DffQlmRq.js';
|
|
11
|
+
import './config-Cjvzbs_j.js';
|
|
12
|
+
import './index-Bkdt8nDw.js';
|
|
13
|
+
import './types-Cz5okLml.js';
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* What a running task reports back while it works, and how it learns that a
|