@delali/narsil 0.3.0 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,208 @@
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-EJDUFWSO.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-KPJMDPHG.mjs';
3
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-RIJSLY2I.mjs';
4
+ import './chunk-BDOSMEEP.mjs';
5
+ import './chunk-JPTQZWJQ.mjs';
6
+ import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-RHG4O77A.mjs';
7
+ import { decode } from '@msgpack/msgpack';
8
+
9
+ // src/engine/export/json-line.ts
10
+ function refuse(path, kind, docId) {
11
+ throw new NarsilError(
12
+ ErrorCodes.DOC_VALIDATION_FAILED,
13
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
14
+ { docId, field: path }
15
+ );
16
+ }
17
+ function encodeNumber(value, path, target) {
18
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
19
+ if (!target.tagged) refuse(path, String(value), target.docId);
20
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
21
+ }
22
+ function encodeDate(value, path, target) {
23
+ const milliseconds = value.getTime();
24
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
25
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
26
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
27
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
28
+ }
29
+ function encodeBytes(value, path, target) {
30
+ if (!target.tagged) refuse(path, "bytes", target.docId);
31
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
32
+ }
33
+ function encodeList(value, path, target) {
34
+ target.parts.push("[");
35
+ for (let at = 0; at < value.length; at++) {
36
+ if (at > 0) target.parts.push(",");
37
+ encodeValue(value[at], `${path}[${at}]`, target);
38
+ }
39
+ target.parts.push("]");
40
+ }
41
+ function encodeMap(value, path, target) {
42
+ const proto = Object.getPrototypeOf(value);
43
+ if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
44
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
45
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
46
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
47
+ for (let at = 0; at < entries.length; at++) {
48
+ const [key, entry] = entries[at];
49
+ if (at > 0) target.parts.push(",");
50
+ target.parts.push(JSON.stringify(key), ":");
51
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
52
+ }
53
+ target.parts.push(wrapped ? "}}" : "}");
54
+ }
55
+ function encodeValue(value, path, target) {
56
+ if (value === null) {
57
+ target.parts.push("null");
58
+ return;
59
+ }
60
+ switch (typeof value) {
61
+ case "string":
62
+ target.parts.push(JSON.stringify(value));
63
+ return;
64
+ case "boolean":
65
+ target.parts.push(value ? "true" : "false");
66
+ return;
67
+ case "number":
68
+ target.parts.push(encodeNumber(value, path, target));
69
+ return;
70
+ case "object":
71
+ break;
72
+ default:
73
+ refuse(path, `a ${typeof value}`, target.docId);
74
+ }
75
+ if (Array.isArray(value)) encodeList(value, path, target);
76
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
77
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
78
+ else encodeMap(value, path, target);
79
+ }
80
+ function encodeExactJson(value, docId) {
81
+ const target = { docId, tagged: false, parts: [] };
82
+ encodeValue(value, "", target);
83
+ return target.parts.join("");
84
+ }
85
+ function encodeDocumentJson(document, docId) {
86
+ const target = { docId, tagged: true, parts: [] };
87
+ encodeValue(document, "", target);
88
+ return target.parts.join("");
89
+ }
90
+
91
+ // src/engine/export/write.ts
92
+ var VECTOR_FILE_ENVELOPE_VERSION = 2;
93
+ var FLOAT32_BYTES = 4;
94
+ async function locateVectors(checkpoint) {
95
+ const located = /* @__PURE__ */ new Map();
96
+ for (const field of checkpoint.manifest.vectors) {
97
+ const byDocId = /* @__PURE__ */ new Map();
98
+ for (const ref of field.files) {
99
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
100
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
101
+ what: `vector file "${ref.key}"`,
102
+ allowsCompression: false
103
+ });
104
+ const decoded = decodeVectorFilePayload(decode(payload));
105
+ for (let position = 0; position < decoded.docIds.length; position++) {
106
+ if (isDead(ref.dead, position)) continue;
107
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
108
+ }
109
+ }
110
+ located.set(field.fieldPath, byDocId);
111
+ }
112
+ return located;
113
+ }
114
+ async function readVector(checkpoint, location) {
115
+ const width = location.dimension * FLOAT32_BYTES;
116
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
117
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
118
+ }
119
+ async function readSegment(checkpoint, ref) {
120
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
121
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
122
+ what: `segment "${ref.key}"`,
123
+ allowsCompression: true
124
+ });
125
+ return decodeKeywordSegment(payload, false);
126
+ }
127
+ var decoder = new TextDecoder();
128
+ function idAt(contents, ordinal) {
129
+ const ids = contents.arrays.idTable;
130
+ return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
131
+ }
132
+ async function winnersOf(checkpoint, partition) {
133
+ const winners = /* @__PURE__ */ new Map();
134
+ for (let at = 0; at < partition.segments.length; at++) {
135
+ const contents = await readSegment(checkpoint, partition.segments[at]);
136
+ for (const docId of contents.tombstones) winners.delete(docId);
137
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
138
+ }
139
+ return winners;
140
+ }
141
+ async function* documentLines(checkpoint) {
142
+ const vectors = await locateVectors(checkpoint);
143
+ for (const partition of checkpoint.manifest.partitions) {
144
+ const winners = await winnersOf(checkpoint, partition);
145
+ for (let at = 0; at < partition.segments.length; at++) {
146
+ const contents = await readSegment(checkpoint, partition.segments[at]);
147
+ const table = contents.arrays.documentTable;
148
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
149
+ const docId = idAt(contents, ordinal);
150
+ if (winners.get(docId) !== at) continue;
151
+ const stored = decodeStoredDocument(
152
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
153
+ table.discardedFields
154
+ );
155
+ const document = { id: docId, ...stored };
156
+ document.id = docId;
157
+ for (const [fieldPath, located] of vectors) {
158
+ const location = located.get(docId);
159
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
160
+ }
161
+ yield `${encodeDocumentJson(document, docId)}
162
+ `;
163
+ }
164
+ }
165
+ }
166
+ }
167
+ async function* exportLines(header, checkpoint) {
168
+ yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
169
+ `;
170
+ yield* documentLines(checkpoint);
171
+ }
172
+ function documentExportStream(header, checkpoint) {
173
+ const lines = exportLines(header, checkpoint);
174
+ const encoder = new TextEncoder();
175
+ let released = false;
176
+ const release = () => {
177
+ if (released) return;
178
+ released = true;
179
+ checkpoint.release();
180
+ };
181
+ return new ReadableStream({
182
+ async pull(controller) {
183
+ try {
184
+ let text = "";
185
+ while (text.length < EXPORT_CHUNK_BYTES) {
186
+ const step = await lines.next();
187
+ if (step.done) {
188
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
189
+ release();
190
+ controller.close();
191
+ return;
192
+ }
193
+ text += step.value;
194
+ }
195
+ controller.enqueue(encoder.encode(text));
196
+ } catch (error) {
197
+ release();
198
+ controller.error(error);
199
+ }
200
+ },
201
+ async cancel() {
202
+ release();
203
+ await lines.return(void 0);
204
+ }
205
+ });
206
+ }
207
+
208
+ export { documentExportStream };
@@ -1,6 +1,6 @@
1
- import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { C as ClusterNamespace } from './config-BhldZgaK.js';
3
- import { h as TaskStore } from './tasks-CH2Dod8w.js';
1
+ import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
+ import { C as ClusterNamespace } from './config-DffQlmRq.js';
3
+ import { h as TaskStore } from './tasks-CwPTz-tP.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -70,18 +70,41 @@ interface CorsOptions {
70
70
  interface ServerLimits {
71
71
  /** Cap for JSON request bodies (single doc, batch, search). Excess → 413. */
72
72
  maxBodyBytes?: number;
73
- /** Cap for the NDJSON import stream and binary restore body. Excess → 413. */
73
+ /** Cap for the NDJSON import stream. Excess → 413. */
74
74
  maxImportBytes?: number;
75
+ /** The server accepts a snapshot archive for `POST /indexes/{name}/restore`
76
+ * and a document export for `POST /indexes/{name}/import` of up to this many
77
+ * bytes, and it answers a larger upload with 413. The server writes each
78
+ * upload to a temporary file as the upload arrives, so it needs the same
79
+ * memory for an upload of any size. The default is 64 GiB, and 0 accepts an
80
+ * upload of any size. */
81
+ maxRestoreBytes?: number;
75
82
  /** Cap for a single NDJSON line, so one unterminated line cannot exhaust memory. Excess → 413. */
76
83
  maxLineBytes?: number;
77
84
  /** Documents handed to the engine per batch during NDJSON import; the loop yields between batches. */
78
85
  importBatchSize?: number;
79
86
  /** Maximum requests executing engine work at once, counted across every request thread; excess is shed with 429. Omit or 0 to disable. */
80
87
  maxConcurrentRequests?: number;
88
+ /** The server holds request bodies up to this fraction of the memory that
89
+ * the host allows the process, counting them across every request thread
90
+ * from the first byte of a body until its request finishes. The server
91
+ * answers 429 to a body that can take the total past this fraction before it
92
+ * buffers any more of that body, unless no other body is in flight. The
93
+ * fraction is 0.05 by default, and 0 removes the bound. */
94
+ maxBodyBytesInFlightFraction?: number;
81
95
  /** Ceiling for a search's `limit`, `offset`, and `group.maxPerGroup`, so one
82
96
  * request cannot ask for an unbounded result set. Excess → 400. Defaults to
83
97
  * 10000, matching the cluster query result window. */
84
98
  maxResultWindow?: number;
99
+ /**
100
+ * The server answers 400 for a search whose `rerank.window` is above this
101
+ * number, so that one remote request can send at most this many documents
102
+ * to a rerank model. A search that leaves `rerank.window` out reranks at
103
+ * most this many hits. Set it from 1 to the engine's `maxRerankWindow`,
104
+ * which the server checks when it starts. It defaults to 100, or to the
105
+ * engine's `maxRerankWindow` where that is lower.
106
+ */
107
+ maxRerankWindow?: number;
85
108
  /** Ceiling for the number of document ids in one multi-get request, so one
86
109
  * request cannot pull an unbounded number of documents. Excess → 400.
87
110
  * Defaults to 10000. */
@@ -133,6 +156,13 @@ interface ServerOptions {
133
156
  * needs query-time or ingest-time embedding names a server-registered adapter.
134
157
  */
135
158
  embeddingAdapters?: Record<string, EmbeddingAdapter>;
159
+ /**
160
+ * The server registers these rerank adapters on its engine under their
161
+ * keys, so that a JSON query can give a key in `rerank.adapter`. A rerank
162
+ * adapter is an object with methods, which JSON cannot carry, so a query
163
+ * reaches it by its key.
164
+ */
165
+ rerankAdapters?: Record<string, RerankAdapter>;
136
166
  /** These ceilings apply before the server hands any work to the engine. */
137
167
  limits?: ServerLimits;
138
168
  /**
@@ -157,12 +187,27 @@ interface ServerOptions {
157
187
  * is controlled elsewhere.
158
188
  */
159
189
  allowInsecure?: boolean;
190
+ /**
191
+ * The server accepts a `regex` test from any client while this is true,
192
+ * which is the default. While it is false, the server answers 400 with
193
+ * `SEARCH_INVALID_FILTER` to every search, preflight, and document list
194
+ * request whose filter holds a `regex` test, and the error message tells the
195
+ * client that this option refused the test. The other pattern tests, such as
196
+ * `contains` and `wildcard`, stay available.
197
+ *
198
+ * The engine limits the work of every pattern test through its
199
+ * `patternWorkCap` setting, whatever this option holds. Set this option to
200
+ * false on a public endpoint that should accept no regular expressions at
201
+ * all.
202
+ */
203
+ allowRegex?: boolean;
160
204
  /**
161
205
  * The cluster-facing side of the node this server fronts, which is
162
206
  * `node.cluster` on a {@link ClusterNode}. With it set, `/readyz` answers
163
207
  * 200 only while the node reports `SERVING`, `/cluster` reports the
164
- * topology, and `/indexes/:name/cluster` reports one index's allocation.
165
- * Without it, both cluster routes answer 501.
208
+ * topology, `/indexes/:name/cluster` reports one index's allocation, and
209
+ * `PUT /indexes/:name/replication-factor` changes one index's replication
210
+ * factor. Without it, all three cluster routes answer 501.
166
211
  */
167
212
  cluster?: ClusterNamespace;
168
213
  }
@@ -255,6 +300,22 @@ declare const REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
255
300
  * @public
256
301
  */
257
302
  declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
303
+ /**
304
+ * Marks a server that answers `GET /indexes/{name}/export` with a stream of a
305
+ * document export and imports an export sent to `POST /indexes/{name}/import`
306
+ * in a task.
307
+ *
308
+ * @public
309
+ */
310
+ declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
311
+ /**
312
+ * Marks a server that reads the synonym set of an index at
313
+ * `GET /indexes/{name}/synonyms` and replaces it at
314
+ * `POST /indexes/{name}/synonyms`.
315
+ *
316
+ * @public
317
+ */
318
+ declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
258
319
  /**
259
320
  * Every capability this server announces at `/capabilities`.
260
321
  *
@@ -266,4 +327,4 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
266
327
  */
267
328
  declare const SERVER_CAPABILITIES: readonly string[];
268
329
 
269
- export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, TASK_FILTER_CAPABILITY as a, type ServerOptions as b, type RequestContext as c, type RequestDenial as d, type ServerLimits as e };
330
+ export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
@@ -0,0 +1,113 @@
1
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
2
+
3
+ // src/rerank/call.ts
4
+ function unregistered(adapters, name) {
5
+ const registered = [...adapters.keys()].sort();
6
+ return new NarsilError(
7
+ ErrorCodes.RERANK_CONFIG_INVALID,
8
+ `The query gives rerank adapter "${name}", which this engine has not registered. Register it with registerRerankAdapter or the rerankAdapters setting`,
9
+ { adapter: name, registered }
10
+ );
11
+ }
12
+ function scoresOf(raw, expected, name) {
13
+ const list = Array.isArray(raw) ? raw : ArrayBuffer.isView(raw) && !(raw instanceof DataView) ? Array.from(raw) : null;
14
+ if (list === null) {
15
+ throw new NarsilError(
16
+ ErrorCodes.RERANK_FAILED,
17
+ `Rerank adapter "${name}" returned ${typeof raw}, expected a list of scores`,
18
+ {
19
+ adapter: name
20
+ }
21
+ );
22
+ }
23
+ if (list.length !== expected) {
24
+ throw new NarsilError(
25
+ ErrorCodes.RERANK_FAILED,
26
+ `Rerank adapter "${name}" returned ${list.length} scores for ${expected} documents`,
27
+ { adapter: name, expected, actual: list.length }
28
+ );
29
+ }
30
+ const scores = new Array(expected);
31
+ for (let i = 0; i < expected; i++) {
32
+ const score = list[i];
33
+ if (typeof score !== "number" || !Number.isFinite(score)) {
34
+ throw new NarsilError(
35
+ ErrorCodes.RERANK_FAILED,
36
+ `Rerank adapter "${name}" returned a score that is not a finite number at position ${i}`,
37
+ { adapter: name, position: i }
38
+ );
39
+ }
40
+ scores[i] = score;
41
+ }
42
+ return scores;
43
+ }
44
+ function requireRerankAdapter(adapters, name) {
45
+ const adapter = adapters.get(name);
46
+ if (adapter === void 0) throw unregistered(adapters, name);
47
+ return adapter;
48
+ }
49
+ function timedOut(name, timeoutMs) {
50
+ return new NarsilError(
51
+ ErrorCodes.RERANK_FAILED,
52
+ `Rerank adapter "${name}" did not answer within ${timeoutMs} ms, so the engine cancelled the call`,
53
+ { adapter: name, timeoutMs }
54
+ );
55
+ }
56
+ function cancelled(name) {
57
+ return new NarsilError(ErrorCodes.RERANK_FAILED, `The engine cancelled the call to rerank adapter "${name}"`, {
58
+ adapter: name
59
+ });
60
+ }
61
+ function wrapped(err, name) {
62
+ if (err instanceof NarsilError && err.code === ErrorCodes.RERANK_FAILED) return err;
63
+ const message = err instanceof Error ? err.message : String(err);
64
+ return new NarsilError(ErrorCodes.RERANK_FAILED, `Rerank adapter "${name}" failed: ${message}`, {
65
+ adapter: name,
66
+ cause: message,
67
+ ...err instanceof NarsilError ? { causeCode: err.code } : {}
68
+ });
69
+ }
70
+ async function callRerankAdapter(adapters, name, query, documents, options) {
71
+ const adapter = requireRerankAdapter(adapters, name);
72
+ if (documents.length === 0) return [];
73
+ const outer = options.signal;
74
+ if (outer?.aborted) throw cancelled(name);
75
+ const controller = new AbortController();
76
+ let timer;
77
+ let detach;
78
+ const stopped = new Promise((_resolve, reject) => {
79
+ timer = setTimeout(() => {
80
+ controller.abort();
81
+ reject(timedOut(name, options.timeoutMs));
82
+ }, options.timeoutMs);
83
+ if (outer !== void 0) {
84
+ const onAbort = () => {
85
+ controller.abort();
86
+ reject(cancelled(name));
87
+ };
88
+ outer.addEventListener("abort", onAbort, { once: true });
89
+ detach = () => outer.removeEventListener("abort", onAbort);
90
+ }
91
+ });
92
+ let raw;
93
+ try {
94
+ raw = await Promise.race([
95
+ Promise.resolve().then(() => adapter.rerank(query, documents, controller.signal)),
96
+ stopped
97
+ ]);
98
+ } catch (err) {
99
+ throw wrapped(err, name);
100
+ } finally {
101
+ clearTimeout(timer);
102
+ detach?.();
103
+ }
104
+ return scoresOf(raw, documents.length, name);
105
+ }
106
+ function rerankOnHost(host, name, query, documents) {
107
+ return callRerankAdapter(host.rerankAdapters, name, query, documents, {
108
+ timeoutMs: host.rerankSettings.timeoutMs,
109
+ signal: host.abortController.signal
110
+ });
111
+ }
112
+
113
+ export { rerankOnHost };
@@ -0,0 +1,87 @@
1
+ // src/vector/osq/record.ts
2
+ var OSQ_RECORD_TRAILER_BYTES = 16;
3
+ var TRAILER_LOWER = 0;
4
+ var TRAILER_UPPER = 4;
5
+ var TRAILER_CORRECTION = 8;
6
+ var TRAILER_SUM = 12;
7
+ var QUERY_PLANES_FOR_NARROW_CODES = 4;
8
+ var LEVELS_IN_A_TWO_BIT_BYTE = 4;
9
+ function osqCodeBytes(dimension, bits) {
10
+ return Math.ceil(dimension * bits / 8);
11
+ }
12
+ function osqRecordBytes(dimension, bits) {
13
+ return osqCodeBytes(dimension, bits) + OSQ_RECORD_TRAILER_BYTES;
14
+ }
15
+ function packLevels(levels, bits, target, offset) {
16
+ const dimension = levels.length;
17
+ if (bits === 8) {
18
+ target.set(levels, offset);
19
+ return;
20
+ }
21
+ const perByte = 8 / bits;
22
+ target.fill(0, offset, offset + osqCodeBytes(dimension, bits));
23
+ for (let i = 0; i < dimension; i++) {
24
+ const value = levels[i];
25
+ if (value === 0) continue;
26
+ target[offset + Math.floor(i / perByte)] |= value << i % perByte * bits;
27
+ }
28
+ }
29
+ function unpackLevels(bytes, offset, dimension, bits) {
30
+ const levels = new Uint8Array(dimension);
31
+ if (bits === 8) {
32
+ levels.set(bytes.subarray(offset, offset + dimension));
33
+ return levels;
34
+ }
35
+ const perByte = 8 / bits;
36
+ const mask = (1 << bits) - 1;
37
+ for (let i = 0; i < dimension; i++) {
38
+ levels[i] = bytes[offset + Math.floor(i / perByte)] >> i % perByte * bits & mask;
39
+ }
40
+ return levels;
41
+ }
42
+ function osqStagedQueryBytes(dimension, documentBits) {
43
+ if (documentBits === 8) return dimension;
44
+ if (documentBits === 4) return osqCodeBytes(dimension, 4);
45
+ return QUERY_PLANES_FOR_NARROW_CODES * osqCodeBytes(dimension, documentBits);
46
+ }
47
+ function stageQueryLevels(levels, documentBits, target) {
48
+ if (documentBits === 8 || documentBits === 4) {
49
+ packLevels(levels, documentBits, target, 0);
50
+ return;
51
+ }
52
+ const planeBytes = osqCodeBytes(levels.length, documentBits);
53
+ target.fill(0, 0, QUERY_PLANES_FOR_NARROW_CODES * planeBytes);
54
+ if (documentBits === 2) {
55
+ for (let i = 0; i < levels.length; i++) {
56
+ target[i % LEVELS_IN_A_TWO_BIT_BYTE * planeBytes + Math.floor(i / LEVELS_IN_A_TWO_BIT_BYTE)] = levels[i];
57
+ }
58
+ return;
59
+ }
60
+ for (let i = 0; i < levels.length; i++) {
61
+ const value = levels[i];
62
+ if (value === 0) continue;
63
+ const byte = i >> 3;
64
+ const bit = 1 << (i & 7);
65
+ for (let plane = 0; plane < QUERY_PLANES_FOR_NARROW_CODES; plane++) {
66
+ if (value >> plane & 1) target[plane * planeBytes + byte] |= bit;
67
+ }
68
+ }
69
+ }
70
+ function writeRecord(target, offset, code, bits, trailer = new DataView(target.buffer, target.byteOffset, target.byteLength)) {
71
+ packLevels(code.levels, bits, target, offset);
72
+ const at = offset + osqCodeBytes(code.levels.length, bits);
73
+ trailer.setFloat32(at + TRAILER_LOWER, code.lower, true);
74
+ trailer.setFloat32(at + TRAILER_UPPER, code.upper, true);
75
+ trailer.setFloat32(at + TRAILER_CORRECTION, code.correction, true);
76
+ trailer.setUint32(at + TRAILER_SUM, code.sum, true);
77
+ }
78
+ function readTrailer(trailer, codeOffset, codeBytes, into) {
79
+ const at = codeOffset + codeBytes;
80
+ into.lower = trailer.getFloat32(at + TRAILER_LOWER, true);
81
+ into.upper = trailer.getFloat32(at + TRAILER_UPPER, true);
82
+ into.correction = trailer.getFloat32(at + TRAILER_CORRECTION, true);
83
+ into.sum = trailer.getUint32(at + TRAILER_SUM, true);
84
+ return into;
85
+ }
86
+
87
+ export { osqCodeBytes, osqRecordBytes, osqStagedQueryBytes, readTrailer, stageQueryLevels, unpackLevels, writeRecord };
@@ -0,0 +1,7 @@
1
+ // src/distribution/coordinator/constants.ts
2
+ var MAX_NODE_ID_LENGTH = 255;
3
+ var MAX_WATCHERS = 64;
4
+ var NODE_HEARTBEAT_TTL_MS = 3e4;
5
+ var CONTROLLER_LEASE_KEY = "_narsil/controller";
6
+
7
+ export { CONTROLLER_LEASE_KEY, MAX_NODE_ID_LENGTH, MAX_WATCHERS, NODE_HEARTBEAT_TTL_MS };