@delali/narsil 0.3.0 → 0.3.1-next.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-ByQRMOx-.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-H6SGzbPH.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-EJDUFWSO.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-KPJMDPHG.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-RIJSLY2I.mjs';
|
|
4
|
+
import './chunk-BDOSMEEP.mjs';
|
|
5
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
6
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-RHG4O77A.mjs';
|
|
7
|
+
import { decode } from '@msgpack/msgpack';
|
|
8
|
+
|
|
9
|
+
// src/engine/export/json-line.ts
|
|
10
|
+
function refuse(path, kind, docId) {
|
|
11
|
+
throw new NarsilError(
|
|
12
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
13
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
14
|
+
{ docId, field: path }
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
function encodeNumber(value, path, target) {
|
|
18
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
19
|
+
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
20
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
21
|
+
}
|
|
22
|
+
function encodeDate(value, path, target) {
|
|
23
|
+
const milliseconds = value.getTime();
|
|
24
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
25
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
26
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
27
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
28
|
+
}
|
|
29
|
+
function encodeBytes(value, path, target) {
|
|
30
|
+
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
31
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
32
|
+
}
|
|
33
|
+
function encodeList(value, path, target) {
|
|
34
|
+
target.parts.push("[");
|
|
35
|
+
for (let at = 0; at < value.length; at++) {
|
|
36
|
+
if (at > 0) target.parts.push(",");
|
|
37
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
38
|
+
}
|
|
39
|
+
target.parts.push("]");
|
|
40
|
+
}
|
|
41
|
+
function encodeMap(value, path, target) {
|
|
42
|
+
const proto = Object.getPrototypeOf(value);
|
|
43
|
+
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
44
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
45
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
46
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
47
|
+
for (let at = 0; at < entries.length; at++) {
|
|
48
|
+
const [key, entry] = entries[at];
|
|
49
|
+
if (at > 0) target.parts.push(",");
|
|
50
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
51
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
52
|
+
}
|
|
53
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
54
|
+
}
|
|
55
|
+
function encodeValue(value, path, target) {
|
|
56
|
+
if (value === null) {
|
|
57
|
+
target.parts.push("null");
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
switch (typeof value) {
|
|
61
|
+
case "string":
|
|
62
|
+
target.parts.push(JSON.stringify(value));
|
|
63
|
+
return;
|
|
64
|
+
case "boolean":
|
|
65
|
+
target.parts.push(value ? "true" : "false");
|
|
66
|
+
return;
|
|
67
|
+
case "number":
|
|
68
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
69
|
+
return;
|
|
70
|
+
case "object":
|
|
71
|
+
break;
|
|
72
|
+
default:
|
|
73
|
+
refuse(path, `a ${typeof value}`, target.docId);
|
|
74
|
+
}
|
|
75
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
76
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
77
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
78
|
+
else encodeMap(value, path, target);
|
|
79
|
+
}
|
|
80
|
+
function encodeExactJson(value, docId) {
|
|
81
|
+
const target = { docId, tagged: false, parts: [] };
|
|
82
|
+
encodeValue(value, "", target);
|
|
83
|
+
return target.parts.join("");
|
|
84
|
+
}
|
|
85
|
+
function encodeDocumentJson(document, docId) {
|
|
86
|
+
const target = { docId, tagged: true, parts: [] };
|
|
87
|
+
encodeValue(document, "", target);
|
|
88
|
+
return target.parts.join("");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// src/engine/export/write.ts
|
|
92
|
+
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
93
|
+
var FLOAT32_BYTES = 4;
|
|
94
|
+
async function locateVectors(checkpoint) {
|
|
95
|
+
const located = /* @__PURE__ */ new Map();
|
|
96
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
97
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
98
|
+
for (const ref of field.files) {
|
|
99
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
100
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
101
|
+
what: `vector file "${ref.key}"`,
|
|
102
|
+
allowsCompression: false
|
|
103
|
+
});
|
|
104
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
105
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
106
|
+
if (isDead(ref.dead, position)) continue;
|
|
107
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
located.set(field.fieldPath, byDocId);
|
|
111
|
+
}
|
|
112
|
+
return located;
|
|
113
|
+
}
|
|
114
|
+
async function readVector(checkpoint, location) {
|
|
115
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
116
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
117
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
118
|
+
}
|
|
119
|
+
async function readSegment(checkpoint, ref) {
|
|
120
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
121
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
122
|
+
what: `segment "${ref.key}"`,
|
|
123
|
+
allowsCompression: true
|
|
124
|
+
});
|
|
125
|
+
return decodeKeywordSegment(payload, false);
|
|
126
|
+
}
|
|
127
|
+
var decoder = new TextDecoder();
|
|
128
|
+
function idAt(contents, ordinal) {
|
|
129
|
+
const ids = contents.arrays.idTable;
|
|
130
|
+
return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
131
|
+
}
|
|
132
|
+
async function winnersOf(checkpoint, partition) {
|
|
133
|
+
const winners = /* @__PURE__ */ new Map();
|
|
134
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
135
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
136
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
137
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
138
|
+
}
|
|
139
|
+
return winners;
|
|
140
|
+
}
|
|
141
|
+
async function* documentLines(checkpoint) {
|
|
142
|
+
const vectors = await locateVectors(checkpoint);
|
|
143
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
144
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
145
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
146
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
147
|
+
const table = contents.arrays.documentTable;
|
|
148
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
149
|
+
const docId = idAt(contents, ordinal);
|
|
150
|
+
if (winners.get(docId) !== at) continue;
|
|
151
|
+
const stored = decodeStoredDocument(
|
|
152
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
153
|
+
table.discardedFields
|
|
154
|
+
);
|
|
155
|
+
const document = { id: docId, ...stored };
|
|
156
|
+
document.id = docId;
|
|
157
|
+
for (const [fieldPath, located] of vectors) {
|
|
158
|
+
const location = located.get(docId);
|
|
159
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
160
|
+
}
|
|
161
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
162
|
+
`;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async function* exportLines(header, checkpoint) {
|
|
168
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
169
|
+
`;
|
|
170
|
+
yield* documentLines(checkpoint);
|
|
171
|
+
}
|
|
172
|
+
function documentExportStream(header, checkpoint) {
|
|
173
|
+
const lines = exportLines(header, checkpoint);
|
|
174
|
+
const encoder = new TextEncoder();
|
|
175
|
+
let released = false;
|
|
176
|
+
const release = () => {
|
|
177
|
+
if (released) return;
|
|
178
|
+
released = true;
|
|
179
|
+
checkpoint.release();
|
|
180
|
+
};
|
|
181
|
+
return new ReadableStream({
|
|
182
|
+
async pull(controller) {
|
|
183
|
+
try {
|
|
184
|
+
let text = "";
|
|
185
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
186
|
+
const step = await lines.next();
|
|
187
|
+
if (step.done) {
|
|
188
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
189
|
+
release();
|
|
190
|
+
controller.close();
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
text += step.value;
|
|
194
|
+
}
|
|
195
|
+
controller.enqueue(encoder.encode(text));
|
|
196
|
+
} catch (error) {
|
|
197
|
+
release();
|
|
198
|
+
controller.error(error);
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
async cancel() {
|
|
202
|
+
release();
|
|
203
|
+
await lines.return(void 0);
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export { documentExportStream };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from './adapters-
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { C as ClusterNamespace } from './config-H6SGzbPH.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-CwPTz-tP.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -70,18 +70,41 @@ interface CorsOptions {
|
|
|
70
70
|
interface ServerLimits {
|
|
71
71
|
/** Cap for JSON request bodies (single doc, batch, search). Excess → 413. */
|
|
72
72
|
maxBodyBytes?: number;
|
|
73
|
-
/** Cap for the NDJSON import stream
|
|
73
|
+
/** Cap for the NDJSON import stream. Excess → 413. */
|
|
74
74
|
maxImportBytes?: number;
|
|
75
|
+
/** The server accepts a snapshot archive for `POST /indexes/{name}/restore`
|
|
76
|
+
* and a document export for `POST /indexes/{name}/import` of up to this many
|
|
77
|
+
* bytes, and it answers a larger upload with 413. The server writes each
|
|
78
|
+
* upload to a temporary file as the upload arrives, so it needs the same
|
|
79
|
+
* memory for an upload of any size. The default is 64 GiB, and 0 accepts an
|
|
80
|
+
* upload of any size. */
|
|
81
|
+
maxRestoreBytes?: number;
|
|
75
82
|
/** Cap for a single NDJSON line, so one unterminated line cannot exhaust memory. Excess → 413. */
|
|
76
83
|
maxLineBytes?: number;
|
|
77
84
|
/** Documents handed to the engine per batch during NDJSON import; the loop yields between batches. */
|
|
78
85
|
importBatchSize?: number;
|
|
79
86
|
/** Maximum requests executing engine work at once, counted across every request thread; excess is shed with 429. Omit or 0 to disable. */
|
|
80
87
|
maxConcurrentRequests?: number;
|
|
88
|
+
/** The server holds request bodies up to this fraction of the memory that
|
|
89
|
+
* the host allows the process, counting them across every request thread
|
|
90
|
+
* from the first byte of a body until its request finishes. The server
|
|
91
|
+
* answers 429 to a body that can take the total past this fraction before it
|
|
92
|
+
* buffers any more of that body, unless no other body is in flight. The
|
|
93
|
+
* fraction is 0.05 by default, and 0 removes the bound. */
|
|
94
|
+
maxBodyBytesInFlightFraction?: number;
|
|
81
95
|
/** Ceiling for a search's `limit`, `offset`, and `group.maxPerGroup`, so one
|
|
82
96
|
* request cannot ask for an unbounded result set. Excess → 400. Defaults to
|
|
83
97
|
* 10000, matching the cluster query result window. */
|
|
84
98
|
maxResultWindow?: number;
|
|
99
|
+
/**
|
|
100
|
+
* The server answers 400 for a search whose `rerank.window` is above this
|
|
101
|
+
* number, so that one remote request can send at most this many documents
|
|
102
|
+
* to a rerank model. A search that leaves `rerank.window` out reranks at
|
|
103
|
+
* most this many hits. Set it from 1 to the engine's `maxRerankWindow`,
|
|
104
|
+
* which the server checks when it starts. It defaults to 100, or to the
|
|
105
|
+
* engine's `maxRerankWindow` where that is lower.
|
|
106
|
+
*/
|
|
107
|
+
maxRerankWindow?: number;
|
|
85
108
|
/** Ceiling for the number of document ids in one multi-get request, so one
|
|
86
109
|
* request cannot pull an unbounded number of documents. Excess → 400.
|
|
87
110
|
* Defaults to 10000. */
|
|
@@ -133,6 +156,13 @@ interface ServerOptions {
|
|
|
133
156
|
* needs query-time or ingest-time embedding names a server-registered adapter.
|
|
134
157
|
*/
|
|
135
158
|
embeddingAdapters?: Record<string, EmbeddingAdapter>;
|
|
159
|
+
/**
|
|
160
|
+
* The server registers these rerank adapters on its engine under their
|
|
161
|
+
* keys, so that a JSON query can give a key in `rerank.adapter`. A rerank
|
|
162
|
+
* adapter is an object with methods, which JSON cannot carry, so a query
|
|
163
|
+
* reaches it by its key.
|
|
164
|
+
*/
|
|
165
|
+
rerankAdapters?: Record<string, RerankAdapter>;
|
|
136
166
|
/** These ceilings apply before the server hands any work to the engine. */
|
|
137
167
|
limits?: ServerLimits;
|
|
138
168
|
/**
|
|
@@ -157,12 +187,27 @@ interface ServerOptions {
|
|
|
157
187
|
* is controlled elsewhere.
|
|
158
188
|
*/
|
|
159
189
|
allowInsecure?: boolean;
|
|
190
|
+
/**
|
|
191
|
+
* The server accepts a `regex` test from any client while this is true,
|
|
192
|
+
* which is the default. While it is false, the server answers 400 with
|
|
193
|
+
* `SEARCH_INVALID_FILTER` to every search, preflight, and document list
|
|
194
|
+
* request whose filter holds a `regex` test, and the error message tells the
|
|
195
|
+
* client that this option refused the test. The other pattern tests, such as
|
|
196
|
+
* `contains` and `wildcard`, stay available.
|
|
197
|
+
*
|
|
198
|
+
* The engine limits the work of every pattern test through its
|
|
199
|
+
* `patternWorkCap` setting, whatever this option holds. Set this option to
|
|
200
|
+
* false on a public endpoint that should accept no regular expressions at
|
|
201
|
+
* all.
|
|
202
|
+
*/
|
|
203
|
+
allowRegex?: boolean;
|
|
160
204
|
/**
|
|
161
205
|
* The cluster-facing side of the node this server fronts, which is
|
|
162
206
|
* `node.cluster` on a {@link ClusterNode}. With it set, `/readyz` answers
|
|
163
207
|
* 200 only while the node reports `SERVING`, `/cluster` reports the
|
|
164
|
-
* topology,
|
|
165
|
-
*
|
|
208
|
+
* topology, `/indexes/:name/cluster` reports one index's allocation, and
|
|
209
|
+
* `PUT /indexes/:name/replication-factor` changes one index's replication
|
|
210
|
+
* factor. Without it, all three cluster routes answer 501.
|
|
166
211
|
*/
|
|
167
212
|
cluster?: ClusterNamespace;
|
|
168
213
|
}
|
|
@@ -255,6 +300,22 @@ declare const REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
|
255
300
|
* @public
|
|
256
301
|
*/
|
|
257
302
|
declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
303
|
+
/**
|
|
304
|
+
* Marks a server that answers `GET /indexes/{name}/export` with a stream of a
|
|
305
|
+
* document export and imports an export sent to `POST /indexes/{name}/import`
|
|
306
|
+
* in a task.
|
|
307
|
+
*
|
|
308
|
+
* @public
|
|
309
|
+
*/
|
|
310
|
+
declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
311
|
+
/**
|
|
312
|
+
* Marks a server that reads the synonym set of an index at
|
|
313
|
+
* `GET /indexes/{name}/synonyms` and replaces it at
|
|
314
|
+
* `POST /indexes/{name}/synonyms`.
|
|
315
|
+
*
|
|
316
|
+
* @public
|
|
317
|
+
*/
|
|
318
|
+
declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
258
319
|
/**
|
|
259
320
|
* Every capability this server announces at `/capabilities`.
|
|
260
321
|
*
|
|
@@ -266,4 +327,4 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
|
266
327
|
*/
|
|
267
328
|
declare const SERVER_CAPABILITIES: readonly string[];
|
|
268
329
|
|
|
269
|
-
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T,
|
|
330
|
+
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
|
|
2
|
+
|
|
3
|
+
// src/rerank/call.ts
|
|
4
|
+
function unregistered(adapters, name) {
|
|
5
|
+
const registered = [...adapters.keys()].sort();
|
|
6
|
+
return new NarsilError(
|
|
7
|
+
ErrorCodes.RERANK_CONFIG_INVALID,
|
|
8
|
+
`The query gives rerank adapter "${name}", which this engine has not registered. Register it with registerRerankAdapter or the rerankAdapters setting`,
|
|
9
|
+
{ adapter: name, registered }
|
|
10
|
+
);
|
|
11
|
+
}
|
|
12
|
+
function scoresOf(raw, expected, name) {
|
|
13
|
+
const list = Array.isArray(raw) ? raw : ArrayBuffer.isView(raw) && !(raw instanceof DataView) ? Array.from(raw) : null;
|
|
14
|
+
if (list === null) {
|
|
15
|
+
throw new NarsilError(
|
|
16
|
+
ErrorCodes.RERANK_FAILED,
|
|
17
|
+
`Rerank adapter "${name}" returned ${typeof raw}, expected a list of scores`,
|
|
18
|
+
{
|
|
19
|
+
adapter: name
|
|
20
|
+
}
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
if (list.length !== expected) {
|
|
24
|
+
throw new NarsilError(
|
|
25
|
+
ErrorCodes.RERANK_FAILED,
|
|
26
|
+
`Rerank adapter "${name}" returned ${list.length} scores for ${expected} documents`,
|
|
27
|
+
{ adapter: name, expected, actual: list.length }
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
const scores = new Array(expected);
|
|
31
|
+
for (let i = 0; i < expected; i++) {
|
|
32
|
+
const score = list[i];
|
|
33
|
+
if (typeof score !== "number" || !Number.isFinite(score)) {
|
|
34
|
+
throw new NarsilError(
|
|
35
|
+
ErrorCodes.RERANK_FAILED,
|
|
36
|
+
`Rerank adapter "${name}" returned a score that is not a finite number at position ${i}`,
|
|
37
|
+
{ adapter: name, position: i }
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
scores[i] = score;
|
|
41
|
+
}
|
|
42
|
+
return scores;
|
|
43
|
+
}
|
|
44
|
+
function requireRerankAdapter(adapters, name) {
|
|
45
|
+
const adapter = adapters.get(name);
|
|
46
|
+
if (adapter === void 0) throw unregistered(adapters, name);
|
|
47
|
+
return adapter;
|
|
48
|
+
}
|
|
49
|
+
function timedOut(name, timeoutMs) {
|
|
50
|
+
return new NarsilError(
|
|
51
|
+
ErrorCodes.RERANK_FAILED,
|
|
52
|
+
`Rerank adapter "${name}" did not answer within ${timeoutMs} ms, so the engine cancelled the call`,
|
|
53
|
+
{ adapter: name, timeoutMs }
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
function cancelled(name) {
|
|
57
|
+
return new NarsilError(ErrorCodes.RERANK_FAILED, `The engine cancelled the call to rerank adapter "${name}"`, {
|
|
58
|
+
adapter: name
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
function wrapped(err, name) {
|
|
62
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.RERANK_FAILED) return err;
|
|
63
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
64
|
+
return new NarsilError(ErrorCodes.RERANK_FAILED, `Rerank adapter "${name}" failed: ${message}`, {
|
|
65
|
+
adapter: name,
|
|
66
|
+
cause: message,
|
|
67
|
+
...err instanceof NarsilError ? { causeCode: err.code } : {}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
async function callRerankAdapter(adapters, name, query, documents, options) {
|
|
71
|
+
const adapter = requireRerankAdapter(adapters, name);
|
|
72
|
+
if (documents.length === 0) return [];
|
|
73
|
+
const outer = options.signal;
|
|
74
|
+
if (outer?.aborted) throw cancelled(name);
|
|
75
|
+
const controller = new AbortController();
|
|
76
|
+
let timer;
|
|
77
|
+
let detach;
|
|
78
|
+
const stopped = new Promise((_resolve, reject) => {
|
|
79
|
+
timer = setTimeout(() => {
|
|
80
|
+
controller.abort();
|
|
81
|
+
reject(timedOut(name, options.timeoutMs));
|
|
82
|
+
}, options.timeoutMs);
|
|
83
|
+
if (outer !== void 0) {
|
|
84
|
+
const onAbort = () => {
|
|
85
|
+
controller.abort();
|
|
86
|
+
reject(cancelled(name));
|
|
87
|
+
};
|
|
88
|
+
outer.addEventListener("abort", onAbort, { once: true });
|
|
89
|
+
detach = () => outer.removeEventListener("abort", onAbort);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
let raw;
|
|
93
|
+
try {
|
|
94
|
+
raw = await Promise.race([
|
|
95
|
+
Promise.resolve().then(() => adapter.rerank(query, documents, controller.signal)),
|
|
96
|
+
stopped
|
|
97
|
+
]);
|
|
98
|
+
} catch (err) {
|
|
99
|
+
throw wrapped(err, name);
|
|
100
|
+
} finally {
|
|
101
|
+
clearTimeout(timer);
|
|
102
|
+
detach?.();
|
|
103
|
+
}
|
|
104
|
+
return scoresOf(raw, documents.length, name);
|
|
105
|
+
}
|
|
106
|
+
function rerankOnHost(host, name, query, documents) {
|
|
107
|
+
return callRerankAdapter(host.rerankAdapters, name, query, documents, {
|
|
108
|
+
timeoutMs: host.rerankSettings.timeoutMs,
|
|
109
|
+
signal: host.abortController.signal
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export { rerankOnHost };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// src/vector/osq/record.ts
|
|
2
|
+
var OSQ_RECORD_TRAILER_BYTES = 16;
|
|
3
|
+
var TRAILER_LOWER = 0;
|
|
4
|
+
var TRAILER_UPPER = 4;
|
|
5
|
+
var TRAILER_CORRECTION = 8;
|
|
6
|
+
var TRAILER_SUM = 12;
|
|
7
|
+
var QUERY_PLANES_FOR_NARROW_CODES = 4;
|
|
8
|
+
var LEVELS_IN_A_TWO_BIT_BYTE = 4;
|
|
9
|
+
function osqCodeBytes(dimension, bits) {
|
|
10
|
+
return Math.ceil(dimension * bits / 8);
|
|
11
|
+
}
|
|
12
|
+
function osqRecordBytes(dimension, bits) {
|
|
13
|
+
return osqCodeBytes(dimension, bits) + OSQ_RECORD_TRAILER_BYTES;
|
|
14
|
+
}
|
|
15
|
+
function packLevels(levels, bits, target, offset) {
|
|
16
|
+
const dimension = levels.length;
|
|
17
|
+
if (bits === 8) {
|
|
18
|
+
target.set(levels, offset);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const perByte = 8 / bits;
|
|
22
|
+
target.fill(0, offset, offset + osqCodeBytes(dimension, bits));
|
|
23
|
+
for (let i = 0; i < dimension; i++) {
|
|
24
|
+
const value = levels[i];
|
|
25
|
+
if (value === 0) continue;
|
|
26
|
+
target[offset + Math.floor(i / perByte)] |= value << i % perByte * bits;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function unpackLevels(bytes, offset, dimension, bits) {
|
|
30
|
+
const levels = new Uint8Array(dimension);
|
|
31
|
+
if (bits === 8) {
|
|
32
|
+
levels.set(bytes.subarray(offset, offset + dimension));
|
|
33
|
+
return levels;
|
|
34
|
+
}
|
|
35
|
+
const perByte = 8 / bits;
|
|
36
|
+
const mask = (1 << bits) - 1;
|
|
37
|
+
for (let i = 0; i < dimension; i++) {
|
|
38
|
+
levels[i] = bytes[offset + Math.floor(i / perByte)] >> i % perByte * bits & mask;
|
|
39
|
+
}
|
|
40
|
+
return levels;
|
|
41
|
+
}
|
|
42
|
+
function osqStagedQueryBytes(dimension, documentBits) {
|
|
43
|
+
if (documentBits === 8) return dimension;
|
|
44
|
+
if (documentBits === 4) return osqCodeBytes(dimension, 4);
|
|
45
|
+
return QUERY_PLANES_FOR_NARROW_CODES * osqCodeBytes(dimension, documentBits);
|
|
46
|
+
}
|
|
47
|
+
function stageQueryLevels(levels, documentBits, target) {
|
|
48
|
+
if (documentBits === 8 || documentBits === 4) {
|
|
49
|
+
packLevels(levels, documentBits, target, 0);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const planeBytes = osqCodeBytes(levels.length, documentBits);
|
|
53
|
+
target.fill(0, 0, QUERY_PLANES_FOR_NARROW_CODES * planeBytes);
|
|
54
|
+
if (documentBits === 2) {
|
|
55
|
+
for (let i = 0; i < levels.length; i++) {
|
|
56
|
+
target[i % LEVELS_IN_A_TWO_BIT_BYTE * planeBytes + Math.floor(i / LEVELS_IN_A_TWO_BIT_BYTE)] = levels[i];
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
for (let i = 0; i < levels.length; i++) {
|
|
61
|
+
const value = levels[i];
|
|
62
|
+
if (value === 0) continue;
|
|
63
|
+
const byte = i >> 3;
|
|
64
|
+
const bit = 1 << (i & 7);
|
|
65
|
+
for (let plane = 0; plane < QUERY_PLANES_FOR_NARROW_CODES; plane++) {
|
|
66
|
+
if (value >> plane & 1) target[plane * planeBytes + byte] |= bit;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
function writeRecord(target, offset, code, bits, trailer = new DataView(target.buffer, target.byteOffset, target.byteLength)) {
|
|
71
|
+
packLevels(code.levels, bits, target, offset);
|
|
72
|
+
const at = offset + osqCodeBytes(code.levels.length, bits);
|
|
73
|
+
trailer.setFloat32(at + TRAILER_LOWER, code.lower, true);
|
|
74
|
+
trailer.setFloat32(at + TRAILER_UPPER, code.upper, true);
|
|
75
|
+
trailer.setFloat32(at + TRAILER_CORRECTION, code.correction, true);
|
|
76
|
+
trailer.setUint32(at + TRAILER_SUM, code.sum, true);
|
|
77
|
+
}
|
|
78
|
+
function readTrailer(trailer, codeOffset, codeBytes, into) {
|
|
79
|
+
const at = codeOffset + codeBytes;
|
|
80
|
+
into.lower = trailer.getFloat32(at + TRAILER_LOWER, true);
|
|
81
|
+
into.upper = trailer.getFloat32(at + TRAILER_UPPER, true);
|
|
82
|
+
into.correction = trailer.getFloat32(at + TRAILER_CORRECTION, true);
|
|
83
|
+
into.sum = trailer.getUint32(at + TRAILER_SUM, true);
|
|
84
|
+
return into;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export { osqCodeBytes, osqRecordBytes, osqStagedQueryBytes, readTrailer, stageQueryLevels, unpackLevels, writeRecord };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// src/distribution/coordinator/constants.ts
|
|
2
|
+
var MAX_NODE_ID_LENGTH = 255;
|
|
3
|
+
var MAX_WATCHERS = 64;
|
|
4
|
+
var NODE_HEARTBEAT_TTL_MS = 3e4;
|
|
5
|
+
var CONTROLLER_LEASE_KEY = "_narsil/controller";
|
|
6
|
+
|
|
7
|
+
export { CONTROLLER_LEASE_KEY, MAX_NODE_ID_LENGTH, MAX_WATCHERS, NODE_HEARTBEAT_TTL_MS };
|