@delali/narsil 0.3.0 → 0.3.1-next.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-ByQRMOx-.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-H6SGzbPH.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,104 @@
1
+ import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-75XIEF6H.mjs';
2
+ import { ErrorCodes, NarsilError } from '../chunk-BWJVNZPR.mjs';
3
+
4
+ // src/rerankers/constants.ts
5
+ var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
6
+ var DEFAULT_MAX_RETRIES = 3;
7
+ var DEFAULT_MAX_DOCUMENTS_PER_REQUEST = 1e3;
8
+
9
+ // src/rerankers/http.ts
10
+ var SERVICE = "Rerank service";
11
+ function invalid(message) {
12
+ return new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `HTTP rerank adapter ${message}`);
13
+ }
14
+ function requireConfig(config) {
15
+ if (typeof config?.baseUrl !== "string" || config.baseUrl.trim().length === 0) {
16
+ throw invalid("requires a non-empty baseUrl");
17
+ }
18
+ let protocol;
19
+ try {
20
+ protocol = new URL(config.baseUrl).protocol;
21
+ } catch {
22
+ throw invalid("requires baseUrl to be an absolute URL");
23
+ }
24
+ if (protocol !== "https:" && protocol !== "http:") throw invalid("requires an http or https baseUrl");
25
+ if (typeof config.model !== "string" || config.model.trim().length === 0) throw invalid("requires a non-empty model");
26
+ if (config.timeout !== void 0 && (!Number.isFinite(config.timeout) || config.timeout <= 0)) {
27
+ throw invalid("timeout must be a positive number");
28
+ }
29
+ if (config.maxRetries !== void 0 && (!Number.isInteger(config.maxRetries) || config.maxRetries < 0)) {
30
+ throw invalid("maxRetries must be a non-negative integer");
31
+ }
32
+ const perRequest = config.maxDocumentsPerRequest;
33
+ if (perRequest !== void 0 && (!Number.isInteger(perRequest) || perRequest < 1)) {
34
+ throw invalid("maxDocumentsPerRequest must be a positive integer");
35
+ }
36
+ const apiKey = config.apiKey;
37
+ if (apiKey !== void 0 && typeof apiKey !== "string" && typeof apiKey !== "function") {
38
+ throw invalid("apiKey must be a string or a function returning a string");
39
+ }
40
+ if (typeof apiKey === "string" && apiKey.trim().length === 0) throw invalid("requires a non-empty apiKey");
41
+ }
42
+ function failed(message, details) {
43
+ return new NarsilError(ErrorCodes.RERANK_FAILED, `${SERVICE} ${message}`, details);
44
+ }
45
+ function scoresFromResults(results, count) {
46
+ if (!Array.isArray(results)) throw failed('response is missing a "results" array');
47
+ if (results.length !== count) {
48
+ throw failed(`returned ${results.length} results for ${count} documents`, {
49
+ expected: count,
50
+ actual: results.length
51
+ });
52
+ }
53
+ const scores = new Array(count);
54
+ const seen = new Uint8Array(count);
55
+ for (let i = 0; i < results.length; i++) {
56
+ const item = results[i];
57
+ const index = typeof item === "object" && item !== null ? item.index : void 0;
58
+ const score = typeof item === "object" && item !== null ? item.relevance_score : void 0;
59
+ if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= count || seen[index] === 1) {
60
+ throw failed(`result at position ${i} carries no index of a document it was sent, or repeats one`);
61
+ }
62
+ if (typeof score !== "number" || !Number.isFinite(score)) {
63
+ throw failed(`result at position ${i} carries no finite "relevance_score"`);
64
+ }
65
+ seen[index] = 1;
66
+ scores[index] = score;
67
+ }
68
+ return scores;
69
+ }
70
+ function createHttpReranker(config) {
71
+ requireConfig(config);
72
+ const url = `${config.baseUrl.replace(/\/+$/, "")}/rerank`;
73
+ const model = config.model;
74
+ const apiKeySource = config.apiKey;
75
+ const timeoutMs = config.timeout ?? DEFAULT_REQUEST_TIMEOUT_MS;
76
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
77
+ const perRequest = config.maxDocumentsPerRequest ?? DEFAULT_MAX_DOCUMENTS_PER_REQUEST;
78
+ return {
79
+ async rerank(query, documents, signal) {
80
+ if (documents.length === 0) return [];
81
+ const apiKey = apiKeySource === void 0 ? null : await resolveApiKey(apiKeySource, "HTTP rerank adapter", ErrorCodes.RERANK_CONFIG_INVALID);
82
+ const combinedSignal = buildSignal(signal, timeoutMs);
83
+ const scores = [];
84
+ for (let start = 0; start < documents.length; start += perRequest) {
85
+ const chunk = documents.slice(start, start + perRequest);
86
+ const response = await postJsonWithRetry({
87
+ url,
88
+ body: JSON.stringify({ model, query, documents: chunk, top_n: chunk.length }),
89
+ apiKey,
90
+ maxRetries,
91
+ signal: combinedSignal,
92
+ service: SERVICE,
93
+ failureCode: ErrorCodes.RERANK_FAILED
94
+ });
95
+ for (const score of scoresFromResults(response.results, chunk.length)) scores.push(score);
96
+ }
97
+ return scores;
98
+ },
99
+ async shutdown() {
100
+ }
101
+ };
102
+ }
103
+
104
+ export { createHttpReranker };
@@ -0,0 +1,35 @@
1
+ // src/narsil/restore-queue.ts
2
+ function restoreDepsOf(core, host) {
3
+ return {
4
+ ...host,
5
+ executor: core.executor,
6
+ indexRegistry: core.indexRegistry,
7
+ durability: core.durability,
8
+ filesystemDurability: core.filesystemDurability,
9
+ embeddingAdapters: core.embeddingAdapters,
10
+ markAnalysisStale: core.analysisRebuild.markStale,
11
+ clearAnalysisStale: core.analysisRebuild.clearStale,
12
+ abandonAnalysisRebuild: core.analysisRebuild.abandon,
13
+ indexState: core.indexState,
14
+ copies: core.orchestrator,
15
+ forgetWatermark: core.watermarkNotifier.forget
16
+ };
17
+ }
18
+ function queueRestore(core, host, restoresInFlight, indexName, data) {
19
+ const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
+ const restored = previous.then(async () => {
21
+ const { restoreFromSnapshot } = await import('./snapshot-QGZPODM5.mjs');
22
+ await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
+ });
24
+ const settled = restored.then(
25
+ () => void 0,
26
+ () => void 0
27
+ );
28
+ restoresInFlight.set(indexName, settled);
29
+ void settled.then(() => {
30
+ if (restoresInFlight.get(indexName) === settled) restoresInFlight.delete(indexName);
31
+ });
32
+ return restored;
33
+ }
34
+
35
+ export { queueRestore };
@@ -0,0 +1,177 @@
1
+ import { runRerankedQuery, rerankTextOf } from './chunk-YEEQHT4V.mjs';
2
+ export { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
3
+ import { rerankOnHost } from './chunk-2RIRGOUL.mjs';
4
+ export { rerankOnHost } from './chunk-2RIRGOUL.mjs';
5
+ import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-EWZLIYHV.mjs';
6
+ import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
7
+ import './chunk-LXWDQLTW.mjs';
8
+ import { HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
9
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
10
+
11
+ // src/engine/query/hybrid-depth.ts
12
+ function countOfUnion(text, vectorScored) {
13
+ const textIds = /* @__PURE__ */ new Set();
14
+ for (const doc of text.scored) textIds.add(doc.docId);
15
+ let vectorOnly = 0;
16
+ for (const doc of vectorScored) if (!textIds.has(doc.docId)) vectorOnly++;
17
+ if (text.scored.length === text.totalMatched) return text.totalMatched + vectorOnly;
18
+ return Math.max(text.totalMatched, text.scored.length + vectorOnly);
19
+ }
20
+ async function fusedAtDepth(params, context, depth, legs) {
21
+ const { textLeg, vectorLeg } = legs;
22
+ const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, searchAfter: _after, pinned: _pinned, ...rest } = params;
23
+ const fusion = legs.resolveHybridFusion(params.hybrid);
24
+ const textOnlyParams = { ...rest, limit: depth, offset: 0 };
25
+ const filtered = legs.filteredSearchOf(params, context);
26
+ if (filtered !== void 0 && legs.filterSubsetsMatchNothing(filtered.first)) {
27
+ return { scored: [], totalMatched: 0 };
28
+ }
29
+ if (!vectorConfig || !vectorConfig.value) {
30
+ const textOnly = await textLeg(textOnlyParams, context);
31
+ return { scored: textOnly.scored, totalMatched: textOnly.totalMatched, facets: textOnly.facets };
32
+ }
33
+ const vectorPending = vectorLeg(
34
+ vectorConfig,
35
+ new Float32Array(vectorConfig.value),
36
+ context,
37
+ filtered,
38
+ depth,
39
+ HYBRID_RESCORE_WINDOW
40
+ );
41
+ const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
42
+ const vectorScored = vectorOutcome.scored;
43
+ const textScored = textFanOutResult.scored.slice(0, depth);
44
+ let fusedScored;
45
+ if (fusion.strategy === "rrf") {
46
+ fusedScored = legs.reciprocalRankFusion([textScored, vectorScored], { k: fusion.k });
47
+ } else {
48
+ fusedScored = legs.linearCombination(textScored, vectorScored, { alpha: fusion.alpha });
49
+ }
50
+ const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
51
+ const threshold = params.minScore ?? 0;
52
+ const filtersByScore = threshold > 0;
53
+ if (filtersByScore) fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
54
+ return {
55
+ scored: fusedScored.length > depth ? fusedScored.slice(0, depth) : fusedScored,
56
+ totalMatched: filtersByScore ? fusedScored.length : countOfUnion(textFanOutResult, vectorScored),
57
+ matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch && (!filtersByScore || textScored.length === textFanOutResult.totalMatched),
58
+ facets: textFanOutResult.facets
59
+ };
60
+ }
61
+ async function searchFixedDepthHybrid(params, context, { cursorDepth, offset, limit }, host) {
62
+ const depth = fixedHybridDepthOf(params.hybrid) ?? 0;
63
+ requireWithinHybridDepth(cursorDepth, offset, depth);
64
+ const result = await fusedAtDepth(params, context, depth, host);
65
+ const placesPins = context.partitionIds === void 0;
66
+ const paginate = (placed, _limit, _offset, _binding, _cursor, _sort, pinnedIds) => {
67
+ const pins = placesPins ? pinnedIds : void 0;
68
+ const organic = pins === void 0 ? placed : result.scored.map((doc) => ({ id: doc.docId }));
69
+ const { page, cursor } = pageByPosition(placed, organic, {
70
+ start: cursorDepth + offset,
71
+ limit,
72
+ pinnedIds: pins,
73
+ binding: context.cursorBinding,
74
+ encodeCursor: host.encodePageCursor
75
+ });
76
+ return { paginated: page, nextCursor: cursor };
77
+ };
78
+ return { ...result, paginate };
79
+ }
80
+
81
+ // src/engine/query/rerank.ts
82
+ async function executeRerankedQuery(request, context, firstStage, host) {
83
+ const startTime = host.now();
84
+ const runner = context.rerank;
85
+ if (runner === void 0) {
86
+ throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "This search path holds no rerank adapters");
87
+ }
88
+ const { manager } = context;
89
+ const result = await runRerankedQuery(request, {
90
+ host,
91
+ schema: context.config.schema,
92
+ maxWindow: runner.maxWindow,
93
+ binding: context.cursorBinding,
94
+ clampsPins: true,
95
+ fusesRankings: (params) => params.vector !== void 0 && (params.mode === "hybrid" || params.term !== void 0 && params.term.trim().length > 0),
96
+ firstStage: (params) => firstStage(params, context),
97
+ async texts(docIds, fields) {
98
+ return docIds.map((docId) => rerankTextOf(manager.fieldValues(docId, fields)));
99
+ },
100
+ async storedPins(docIds) {
101
+ return new Set(docIds.filter((docId) => manager.getRef(docId) !== void 0));
102
+ },
103
+ score: (adapter, query, documents) => runner.score(adapter, query, documents),
104
+ async fillDocuments(hits, groups, projection) {
105
+ const read = (docId) => projection.kind === "none" ? {} : manager.get(docId, projection) ?? {};
106
+ for (const hit of hits) hit.document = read(hit.id);
107
+ for (const group of groups ?? []) {
108
+ for (const hit of group.hits) hit.document = read(hit.id);
109
+ }
110
+ },
111
+ highlight(page, projection) {
112
+ const fields = request.highlight?.fields ?? [];
113
+ const dropped = fields.some((field) => !host.projectionKeepsField(projection, field));
114
+ const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
115
+ host.applyHighlights(
116
+ page,
117
+ request,
118
+ context.language,
119
+ manager.analysis,
120
+ manager.schema,
121
+ readStoredDocument,
122
+ manager.synonyms
123
+ );
124
+ }
125
+ });
126
+ result.elapsed = host.now() - startTime;
127
+ return result;
128
+ }
129
+
130
+ // src/rerank/runner.ts
131
+ function rerankRunnerFor(core) {
132
+ return {
133
+ maxWindow: core.rerankSettings.maxWindow,
134
+ score: (adapter, query, documents) => rerankOnHost(core, adapter, query, documents)
135
+ };
136
+ }
137
+
138
+ // src/rerank/settings.ts
139
+ function wholeNumberIn(value, low, high) {
140
+ return typeof value === "number" && Number.isInteger(value) && value >= low && value <= high;
141
+ }
142
+ function resolveRerankSettings(config) {
143
+ const timeout = config?.rerankTimeout;
144
+ const maxWindow = config?.maxRerankWindow;
145
+ if (timeout !== void 0 && !wholeNumberIn(timeout, MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS)) {
146
+ throw new NarsilError(
147
+ ErrorCodes.CONFIG_INVALID,
148
+ `rerankTimeout must be a whole number of milliseconds from ${MIN_RERANK_TIMEOUT_MS} to ${MAX_RERANK_TIMEOUT_MS}`,
149
+ { rerankTimeout: timeout }
150
+ );
151
+ }
152
+ if (maxWindow !== void 0 && !wholeNumberIn(maxWindow, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW)) {
153
+ throw new NarsilError(
154
+ ErrorCodes.CONFIG_INVALID,
155
+ `maxRerankWindow must be a whole number from ${MIN_RERANK_WINDOW} to ${MAX_RERANK_WINDOW}`,
156
+ {
157
+ maxRerankWindow: maxWindow
158
+ }
159
+ );
160
+ }
161
+ return {
162
+ timeoutMs: timeout ?? DEFAULT_RERANK_TIMEOUT_MS,
163
+ maxWindow: maxWindow ?? DEFAULT_MAX_RERANK_WINDOW
164
+ };
165
+ }
166
+ async function shutdownRerankAdapters(adapters) {
167
+ const distinct = new Set(adapters.values());
168
+ adapters.clear();
169
+ for (const adapter of distinct) {
170
+ try {
171
+ await adapter.shutdown?.();
172
+ } catch {
173
+ }
174
+ }
175
+ }
176
+
177
+ export { executeRerankedQuery, rerankRunnerFor, resolveRerankSettings, searchFixedDepthHybrid, shutdownRerankAdapters };
@@ -1,4 +1,5 @@
1
- import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
1
+ import { E as EmbeddingAdapter } from './adapters-BQFQ4PRz.js';
2
+ import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
2
3
 
3
4
  /**
4
5
  * A document as the engine stores it: any set of fields, with an optional `id`.
@@ -13,20 +14,69 @@ type AnyDocument = Record<string, unknown> & {
13
14
  id?: string;
14
15
  };
15
16
  /**
16
- * The type a schema field declares, which sets how the engine indexes it.
17
+ * Each of these options can follow the base type of a `string` or `string[]`
18
+ * field after a colon, as in `string:sortable:pattern`.
17
19
  *
18
- * `string` is analysed and searchable, `number`, `boolean`, and `enum` are
19
- * filterable and sortable, `geopoint` accepts a latitude and longitude pair,
20
- * and `vector[N]` holds an N-dimensional embedding. Each array form indexes
21
- * every element of the field.
20
+ * The engine can sort by a field whose type includes `sortable`. On a single
21
+ * `string` field with that option, the engine can also test a text range with
22
+ * `gt`, `gte`, `lt`, `lte`, or `between`. The engine keeps a pattern index for
23
+ * a field whose type includes `pattern`. It answers `contains`, `eq`, `ne`,
24
+ * `in`, `nin`, `startsWith`, and `endsWith` on that field from the index,
25
+ * comparing case folds for any of those tests that sets `caseFold`. The engine
26
+ * raises `DOC_VALIDATION_FAILED` for a value longer than
27
+ * {@link IndexConfig.patternValueLimit} in such a field. For a query with
28
+ * {@link QueryParams.partialWords} set, the engine also matches each query
29
+ * word inside the longer words of a field whose type includes `partial`. It
30
+ * raises `CONFIG_INVALID` for such a field in an index whose `surfaceForms`
31
+ * setting is false, because it matches inside the original spellings of
32
+ * words, which it records only while that setting is on.
22
33
  *
23
- * `string:sortable` is a `string` in every respect, and a sort may name it.
24
- * Ordering text costs far more memory per document than ordering a number, so
25
- * a sort naming a plain `string` field raises `SEARCH_INVALID_FIELD`.
34
+ * @public
35
+ */
36
+ type TextFieldOption = 'sortable' | 'pattern' | 'partial';
37
+ /**
38
+ * This union contains every string of options that can follow a base type,
39
+ * with each option after a colon. The options can come in any order, although
40
+ * each one can appear only once.
26
41
  *
27
42
  * @public
28
43
  */
29
- type FieldType = 'string' | 'string:sortable' | 'number' | 'boolean' | 'enum' | 'geopoint' | `vector[${number}]` | 'string[]' | 'number[]' | 'boolean[]' | 'enum[]';
44
+ type FieldOptionChain<Remaining extends string> = {
45
+ [Option in Remaining]: `:${Option}` | `:${Option}${FieldOptionChain<Exclude<Remaining, Option>>}`;
46
+ }[Remaining];
47
+ /**
48
+ * The engine indexes each field of a schema according to the field's type,
49
+ * which is one of these.
50
+ *
51
+ * The engine splits a `string` value into words, so it can find the document
52
+ * by any word in the value. It can filter and sort by a `number`, `boolean`,
53
+ * or `enum` field. A `geopoint` value is a latitude and longitude pair. A
54
+ * `vector[N]` value is an embedding with N dimensions. For each list form, the
55
+ * engine indexes every element of the list.
56
+ *
57
+ * The engine stores and compares a `verbatim` value exactly as written, as one
58
+ * piece of text. Because the engine keeps that value whole, it leaves the
59
+ * field out of keyword search, although it can test the whole value in a
60
+ * filter. The engine keeps a pattern index for every `verbatim` field, so it
61
+ * can also test such a field with `contains` and `caseFold`.
62
+ *
63
+ * Add the {@link TextFieldOption} options to a `string` or `string[]` type, or
64
+ * `sortable` alone to a `verbatim` or `verbatim[]` type, as in
65
+ * `string:sortable:pattern` or `verbatim[]:sortable`. The engine stores the
66
+ * options in the order `sortable`, `pattern`, `partial`, whatever their order
67
+ * in the type name. It raises `SCHEMA_INVALID_TYPE` for any other
68
+ * option, such as `number:sortable` or `verbatim:pattern`, and for an option
69
+ * written twice.
70
+ *
71
+ * To sort by a text field, the engine keeps up to the first 512 code points of
72
+ * every document's value in memory, against 8 bytes per document for a number
73
+ * field. For that reason, the engine sorts by a text field only where the
74
+ * field's type includes `sortable`, while it raises `SEARCH_INVALID_FIELD`
75
+ * for a sort on any other text field.
76
+ *
77
+ * @public
78
+ */
79
+ type FieldType = 'string' | `string${FieldOptionChain<TextFieldOption>}` | 'string[]' | `string[]${FieldOptionChain<TextFieldOption>}` | 'verbatim' | 'verbatim:sortable' | 'verbatim[]' | 'verbatim[]:sortable' | 'number' | 'boolean' | 'enum' | 'geopoint' | `vector[${number}]` | 'number[]' | 'boolean[]' | 'enum[]';
30
80
  /**
31
81
  * The field layout of an index, written as a plain object.
32
82
  *
@@ -173,6 +223,26 @@ interface IndexConfig {
173
223
  * insert throughput and one entry per changed spelling.
174
224
  */
175
225
  surfaceForms?: boolean;
226
+ /**
227
+ * The engine raises `DOC_VALIDATION_FAILED` for a document whose value in a
228
+ * `verbatim` field, or in a field whose type includes `pattern`, is longer
229
+ * than this many code points. It tests each element of a list on its own.
230
+ * Set a whole number from 1 to 65,536, or leave this unset for a limit of
231
+ * 8,192. For any other value, {@link Narsil.createIndex} throws
232
+ * `CONFIG_INVALID`.
233
+ */
234
+ patternValueLimit?: number;
235
+ /**
236
+ * A search for any word in a rule of this set also matches the synonyms
237
+ * that the rule gives that word, and a document that holds a synonym scores
238
+ * as though it held the word itself. The engine analyses each word with the
239
+ * index's language, tokeniser, and stop words, so a word must analyse to
240
+ * exactly one token. {@link Narsil.createIndex} throws `CONFIG_INVALID` for
241
+ * a stop word, a phrase, a word that holds a control character, or a set
242
+ * beyond 100,000 rules, 500,000 words, or 4,194,304 bytes of words in UTF-8.
243
+ * {@link SynonymOperations.updateSynonyms} replaces the set without a rebuild.
244
+ */
245
+ synonyms?: SynonymRule[];
176
246
  }
177
247
  /**
178
248
  * BM25 tuning for an index.
@@ -286,4 +356,4 @@ interface InsertOptions extends WriteOptions {
286
356
  skipClone?: boolean;
287
357
  }
288
358
 
289
- export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldType as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b, ScoringMode as c, VectorQuantizationMode as d, VectorStorageMode as e };
359
+ export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldOptionChain as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, TextFieldOption as T, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b, FieldType as c, ScoringMode as d, VectorQuantizationMode as e, VectorStorageMode as f };
package/dist/server.d.ts CHANGED
@@ -1,15 +1,16 @@
1
- export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-yFaVn2xZ.js';
2
- import { b as ServerOptions, N as NarsilServer } from './capabilities-DSsOGz6F.js';
3
- export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, c as RequestContext, d as RequestDenial, S as SERVER_CAPABILITIES, e as ServerLimits, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-DSsOGz6F.js';
4
- import { a as Narsil } from './engine-CIOyFozK.js';
5
- import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-CH2Dod8w.js';
6
- export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-CH2Dod8w.js';
7
- import './schema-d5Afu03A.js';
8
- import './adapters-79OIa4xm.js';
9
- import './config-BhldZgaK.js';
10
- import './config-DmXULPa-.js';
11
- import './types-DAK7mnUZ.js';
12
- import './types-D5OwhtsQ.js';
1
+ export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-DqOIg5xa.js';
2
+ import { c as ServerOptions, N as NarsilServer } from './capabilities-ByQRMOx-.js';
3
+ export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-ByQRMOx-.js';
4
+ import { N as Narsil } from './engine-rC0vjjY-.js';
5
+ import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-CwPTz-tP.js';
6
+ export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-CwPTz-tP.js';
7
+ import './schema-4W_7tk1f.js';
8
+ import './adapters-BQFQ4PRz.js';
9
+ import './synonyms-CpOAHPa-.js';
10
+ import './config-H6SGzbPH.js';
11
+ import './config-Cjvzbs_j.js';
12
+ import './index-Bkdt8nDw.js';
13
+ import './types-Cz5okLml.js';
13
14
 
14
15
  /**
15
16
  * What a running task reports back while it works, and how it learns that a