@delali/narsil 0.3.1-next.131 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-DXAmysqJ.d.ts} +10 -3
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-OGLV4I3L.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-DApBeLbk.d.ts} +17 -2
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +11 -1
- package/dist/{config-D1cDv3sE.d.ts → config-B0H-vp4U.d.ts} +4 -0
- package/dist/{config-CmYJO34X.d.ts → config-Brysl9NO.d.ts} +1 -1
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +21 -5
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CuLMbH1-.d.ts} +63 -6
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +11 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +51 -19
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-DjGjK70Y.d.ts} +22 -6
- package/dist/{thread-SKO7CN3R.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +101 -6
- package/package.json +7 -7
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
2
|
+
|
|
3
|
+
// src/persistence/archive/byte-reader.ts
|
|
4
|
+
async function* chunksOf(source) {
|
|
5
|
+
if (source instanceof Uint8Array) {
|
|
6
|
+
yield source;
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
if (typeof source.getReader === "function") {
|
|
10
|
+
const reader = source.getReader();
|
|
11
|
+
try {
|
|
12
|
+
for (; ; ) {
|
|
13
|
+
const step = await reader.read();
|
|
14
|
+
if (step.done) return;
|
|
15
|
+
yield step.value;
|
|
16
|
+
}
|
|
17
|
+
} finally {
|
|
18
|
+
reader.releaseLock();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
yield* source;
|
|
22
|
+
}
|
|
23
|
+
function createByteReader(source) {
|
|
24
|
+
const iterator = chunksOf(source);
|
|
25
|
+
const pending = [];
|
|
26
|
+
let pendingBytes = 0;
|
|
27
|
+
let ended = false;
|
|
28
|
+
async function fill(length) {
|
|
29
|
+
while (pendingBytes < length && !ended) {
|
|
30
|
+
const step = await iterator.next();
|
|
31
|
+
if (step.done) {
|
|
32
|
+
ended = true;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
if (step.value.byteLength === 0) continue;
|
|
36
|
+
pending.push(step.value);
|
|
37
|
+
pendingBytes += step.value.byteLength;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function take(length) {
|
|
41
|
+
const out = new Uint8Array(length);
|
|
42
|
+
let filled = 0;
|
|
43
|
+
while (filled < length) {
|
|
44
|
+
const head = pending[0];
|
|
45
|
+
const needed = length - filled;
|
|
46
|
+
if (head.byteLength <= needed) {
|
|
47
|
+
out.set(head, filled);
|
|
48
|
+
filled += head.byteLength;
|
|
49
|
+
pending.shift();
|
|
50
|
+
} else {
|
|
51
|
+
out.set(head.subarray(0, needed), filled);
|
|
52
|
+
pending[0] = head.subarray(needed);
|
|
53
|
+
filled += needed;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
pendingBytes -= length;
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
async readExactly(length) {
|
|
61
|
+
await fill(length);
|
|
62
|
+
if (pendingBytes === 0 && length > 0) return null;
|
|
63
|
+
if (pendingBytes < length) return take(pendingBytes);
|
|
64
|
+
return take(length);
|
|
65
|
+
},
|
|
66
|
+
async peek(length) {
|
|
67
|
+
await fill(length);
|
|
68
|
+
const available = Math.min(length, pendingBytes);
|
|
69
|
+
const copy = take(available);
|
|
70
|
+
pending.unshift(copy);
|
|
71
|
+
pendingBytes += copy.byteLength;
|
|
72
|
+
return copy;
|
|
73
|
+
},
|
|
74
|
+
async close() {
|
|
75
|
+
await iterator.return(void 0);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// src/engine/snapshot/restore-config.ts
|
|
81
|
+
function rejectSnapshot(message) {
|
|
82
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
|
|
83
|
+
}
|
|
84
|
+
function isPlainObject(value) {
|
|
85
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
86
|
+
}
|
|
87
|
+
function restoredBm25(raw, reject) {
|
|
88
|
+
if (raw === void 0) {
|
|
89
|
+
return void 0;
|
|
90
|
+
}
|
|
91
|
+
if (!isPlainObject(raw)) {
|
|
92
|
+
reject("bm25 must be an object");
|
|
93
|
+
}
|
|
94
|
+
const fields = restoredBm25Fields(raw.fields, reject);
|
|
95
|
+
return {
|
|
96
|
+
...restoredBm25Pair(raw, "bm25", reject),
|
|
97
|
+
...fields !== void 0 ? { fields } : {}
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function restoredBm25Pair(raw, label, reject) {
|
|
101
|
+
const { k1, b } = raw;
|
|
102
|
+
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
103
|
+
reject(`${label} k1 must be a finite number`);
|
|
104
|
+
}
|
|
105
|
+
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
106
|
+
reject(`${label} b must be a finite number`);
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
...k1 !== void 0 ? { k1 } : {},
|
|
110
|
+
...b !== void 0 ? { b } : {}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function restoredBm25Fields(raw, reject) {
|
|
114
|
+
if (raw === void 0) return void 0;
|
|
115
|
+
if (!isPlainObject(raw)) reject("bm25 fields must be an object");
|
|
116
|
+
const source = raw;
|
|
117
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
118
|
+
for (const name of Object.keys(source)) {
|
|
119
|
+
const pair = source[name];
|
|
120
|
+
if (!isPlainObject(pair)) reject("bm25 fields must map each field to an object");
|
|
121
|
+
fields[name] = restoredBm25Pair(pair, "bm25 field", reject);
|
|
122
|
+
}
|
|
123
|
+
return fields;
|
|
124
|
+
}
|
|
125
|
+
function restoredStopWords(name, list, reject) {
|
|
126
|
+
if (name !== void 0 && typeof name !== "string") {
|
|
127
|
+
reject("stopWords must be a name");
|
|
128
|
+
}
|
|
129
|
+
if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
|
|
130
|
+
reject("stopWordList must be a list of words");
|
|
131
|
+
}
|
|
132
|
+
if (typeof name === "string" && Array.isArray(list)) {
|
|
133
|
+
reject("stopWords and stopWordList cannot both be present");
|
|
134
|
+
}
|
|
135
|
+
if (typeof name === "string") {
|
|
136
|
+
return name;
|
|
137
|
+
}
|
|
138
|
+
if (Array.isArray(list)) {
|
|
139
|
+
return new Set(list);
|
|
140
|
+
}
|
|
141
|
+
return void 0;
|
|
142
|
+
}
|
|
143
|
+
function restoredPartitionConfig(raw, reject = rejectSnapshot) {
|
|
144
|
+
if (raw === void 0) {
|
|
145
|
+
return void 0;
|
|
146
|
+
}
|
|
147
|
+
if (!isPlainObject(raw)) {
|
|
148
|
+
reject("partitionConfig must be an object");
|
|
149
|
+
}
|
|
150
|
+
const { maxDocsPerPartition, maxPartitions, watermark } = raw;
|
|
151
|
+
if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
152
|
+
reject("partitionConfig maxDocsPerPartition must be a positive integer");
|
|
153
|
+
}
|
|
154
|
+
if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
155
|
+
reject("partitionConfig maxPartitions must be a positive integer");
|
|
156
|
+
}
|
|
157
|
+
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
158
|
+
reject("partitionConfig watermark must be above 0 and at most 1");
|
|
159
|
+
}
|
|
160
|
+
return {
|
|
161
|
+
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
162
|
+
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
163
|
+
...typeof watermark === "number" ? { watermark } : {}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
function restoredScoring(raw, reject) {
|
|
167
|
+
if (raw === void 0) {
|
|
168
|
+
return void 0;
|
|
169
|
+
}
|
|
170
|
+
if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
|
|
171
|
+
reject("defaultScoring must be 'local', 'dfs', or 'broadcast'");
|
|
172
|
+
}
|
|
173
|
+
return raw;
|
|
174
|
+
}
|
|
175
|
+
function restoredBoolean(raw, field, reject) {
|
|
176
|
+
if (raw === void 0) {
|
|
177
|
+
return void 0;
|
|
178
|
+
}
|
|
179
|
+
if (typeof raw !== "boolean") {
|
|
180
|
+
reject(`${field} must be a boolean`);
|
|
181
|
+
}
|
|
182
|
+
return raw;
|
|
183
|
+
}
|
|
184
|
+
function restoredRequired(raw, reject) {
|
|
185
|
+
if (raw === void 0) {
|
|
186
|
+
return void 0;
|
|
187
|
+
}
|
|
188
|
+
if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
|
|
189
|
+
reject("required must be a list of field paths");
|
|
190
|
+
}
|
|
191
|
+
return raw;
|
|
192
|
+
}
|
|
193
|
+
function restoredVectorPromotion(raw, reject) {
|
|
194
|
+
if (raw === void 0) {
|
|
195
|
+
return void 0;
|
|
196
|
+
}
|
|
197
|
+
if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
|
|
198
|
+
reject("vectorPromotion must be an object");
|
|
199
|
+
}
|
|
200
|
+
return raw;
|
|
201
|
+
}
|
|
202
|
+
function restoredEmbedding(raw, reject = rejectSnapshot) {
|
|
203
|
+
if (raw === void 0) {
|
|
204
|
+
return void 0;
|
|
205
|
+
}
|
|
206
|
+
if (!isPlainObject(raw)) {
|
|
207
|
+
reject("embedding must be an object");
|
|
208
|
+
}
|
|
209
|
+
const { fields, adapter } = raw;
|
|
210
|
+
if (adapter !== void 0 && typeof adapter !== "string") {
|
|
211
|
+
reject("embedding adapter must be a name");
|
|
212
|
+
}
|
|
213
|
+
if (!isPlainObject(fields)) {
|
|
214
|
+
reject("embedding fields must be an object");
|
|
215
|
+
}
|
|
216
|
+
for (const value of Object.values(fields)) {
|
|
217
|
+
const isPath = typeof value === "string";
|
|
218
|
+
const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
|
|
219
|
+
if (!isPath && !isPathList) {
|
|
220
|
+
reject("embedding fields must map vector fields to source paths");
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
fields,
|
|
225
|
+
...adapter !== void 0 ? { adapter } : {}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function readSharedConfigFields(source, reject) {
|
|
229
|
+
if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
|
|
230
|
+
reject("tokenizer must be a name");
|
|
231
|
+
}
|
|
232
|
+
const stopWords = restoredStopWords(source.stopWords, source.stopWordList, reject);
|
|
233
|
+
const bm25 = restoredBm25(source.bm25, reject);
|
|
234
|
+
const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject);
|
|
235
|
+
const defaultScoring = restoredScoring(source.defaultScoring, reject);
|
|
236
|
+
const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject);
|
|
237
|
+
const strict = restoredBoolean(source.strict, "strict", reject);
|
|
238
|
+
const required = restoredRequired(source.required, reject);
|
|
239
|
+
const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject);
|
|
240
|
+
if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
|
|
241
|
+
reject("patternValueLimit must be a number");
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
|
|
245
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
246
|
+
...bm25 !== void 0 ? { bm25 } : {},
|
|
247
|
+
...surfaceForms !== void 0 ? { surfaceForms } : {},
|
|
248
|
+
...defaultScoring !== void 0 ? { defaultScoring } : {},
|
|
249
|
+
...trackPositions !== void 0 ? { trackPositions } : {},
|
|
250
|
+
...strict !== void 0 ? { strict } : {},
|
|
251
|
+
...required !== void 0 ? { required } : {},
|
|
252
|
+
...vectorPromotion !== void 0 ? { vectorPromotion } : {},
|
|
253
|
+
...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
function restoredConfigFields(envelope, reject = rejectSnapshot) {
|
|
257
|
+
const shared = readSharedConfigFields(envelope, reject);
|
|
258
|
+
const partitions = restoredPartitionConfig(envelope.partitionConfig, reject);
|
|
259
|
+
return {
|
|
260
|
+
...shared,
|
|
261
|
+
...partitions !== void 0 ? { partitions } : {}
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export { chunksOf, createByteReader, rejectSnapshot, restoredConfigFields, restoredEmbedding };
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
import { chunksOf, restoredConfigFields, restoredEmbedding } from './chunk-QHNNULPW.mjs';
|
|
2
|
+
import { flattenSchema, requireConfigurationShape, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-ZAIZQOTM.mjs';
|
|
3
|
+
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-MZI3TEOW.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
5
|
+
|
|
6
|
+
// src/engine/export/lines.ts
|
|
7
|
+
var NEWLINE = 10;
|
|
8
|
+
function joined(parts, total) {
|
|
9
|
+
if (parts.length === 1) return parts[0];
|
|
10
|
+
const bytes = new Uint8Array(total);
|
|
11
|
+
let offset = 0;
|
|
12
|
+
for (const part of parts) {
|
|
13
|
+
bytes.set(part, offset);
|
|
14
|
+
offset += part.byteLength;
|
|
15
|
+
}
|
|
16
|
+
return bytes;
|
|
17
|
+
}
|
|
18
|
+
async function* exportLines(source, maxLineBytes) {
|
|
19
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
20
|
+
let parts = [];
|
|
21
|
+
let pending = 0;
|
|
22
|
+
let overflowing = false;
|
|
23
|
+
let line = 0;
|
|
24
|
+
const finishLine = () => {
|
|
25
|
+
line += 1;
|
|
26
|
+
if (overflowing) {
|
|
27
|
+
overflowing = false;
|
|
28
|
+
parts = [];
|
|
29
|
+
pending = 0;
|
|
30
|
+
return { line, refused: "too-long" };
|
|
31
|
+
}
|
|
32
|
+
const bytes = joined(parts, pending);
|
|
33
|
+
parts = [];
|
|
34
|
+
pending = 0;
|
|
35
|
+
try {
|
|
36
|
+
return { line, text: decoder.decode(bytes) };
|
|
37
|
+
} catch {
|
|
38
|
+
return { line, refused: "not-utf8" };
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
for await (const chunk of chunksOf(source)) {
|
|
42
|
+
let start = 0;
|
|
43
|
+
for (; ; ) {
|
|
44
|
+
const newline = chunk.indexOf(NEWLINE, start);
|
|
45
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
46
|
+
const piece = chunk.subarray(start, end);
|
|
47
|
+
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
48
|
+
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
49
|
+
pending += piece.byteLength;
|
|
50
|
+
} else {
|
|
51
|
+
overflowing = true;
|
|
52
|
+
parts = [];
|
|
53
|
+
}
|
|
54
|
+
if (newline < 0) break;
|
|
55
|
+
yield finishLine();
|
|
56
|
+
start = newline + 1;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (pending > 0 || overflowing) yield finishLine();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// src/serialization/base64.ts
|
|
63
|
+
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
64
|
+
var PAD = 61;
|
|
65
|
+
var BYTES_PER_GROUP = 3;
|
|
66
|
+
var CHARACTERS_PER_GROUP = 4;
|
|
67
|
+
var ASCII_LIMIT = 128;
|
|
68
|
+
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
69
|
+
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
70
|
+
for (let at = 0; at < ALPHABET.length; at++) {
|
|
71
|
+
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
72
|
+
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
73
|
+
}
|
|
74
|
+
var asciiDecoder = new TextDecoder();
|
|
75
|
+
function encodeBase64(bytes) {
|
|
76
|
+
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
77
|
+
let write = 0;
|
|
78
|
+
let read = 0;
|
|
79
|
+
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
80
|
+
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
81
|
+
out[write++] = ENCODE[group >>> 18];
|
|
82
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
83
|
+
out[write++] = ENCODE[group >>> 6 & 63];
|
|
84
|
+
out[write++] = ENCODE[group & 63];
|
|
85
|
+
}
|
|
86
|
+
const left = bytes.byteLength - read;
|
|
87
|
+
if (left > 0) {
|
|
88
|
+
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
89
|
+
out[write++] = ENCODE[group >>> 18];
|
|
90
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
91
|
+
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
92
|
+
out[write++] = PAD;
|
|
93
|
+
}
|
|
94
|
+
return asciiDecoder.decode(out);
|
|
95
|
+
}
|
|
96
|
+
function sextetAt(text, at) {
|
|
97
|
+
const code = text.charCodeAt(at);
|
|
98
|
+
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
99
|
+
}
|
|
100
|
+
function decodeExactBase64(text) {
|
|
101
|
+
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
102
|
+
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
103
|
+
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
104
|
+
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
105
|
+
let write = 0;
|
|
106
|
+
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
107
|
+
const a2 = sextetAt(text, at);
|
|
108
|
+
const b2 = sextetAt(text, at + 1);
|
|
109
|
+
const c2 = sextetAt(text, at + 2);
|
|
110
|
+
const d = sextetAt(text, at + 3);
|
|
111
|
+
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
112
|
+
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
113
|
+
out[write++] = group >>> 16;
|
|
114
|
+
out[write++] = group >>> 8 & 255;
|
|
115
|
+
out[write++] = group & 255;
|
|
116
|
+
}
|
|
117
|
+
if (padding === 0) return out;
|
|
118
|
+
const a = sextetAt(text, fullEnd);
|
|
119
|
+
const b = sextetAt(text, fullEnd + 1);
|
|
120
|
+
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
121
|
+
if ((a | b | c) < 0) return null;
|
|
122
|
+
if (padding === 2 && (b & 15) !== 0) return null;
|
|
123
|
+
if (padding === 1 && (c & 3) !== 0) return null;
|
|
124
|
+
out[write++] = a << 2 | b >>> 4;
|
|
125
|
+
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// src/engine/export/tags.ts
|
|
130
|
+
var TAG_PREFIX = "$";
|
|
131
|
+
var DATE_TAG = "$date";
|
|
132
|
+
var BYTES_TAG = "$bytes";
|
|
133
|
+
var NUMBER_TAG = "$number";
|
|
134
|
+
var OBJECT_TAG = "$object";
|
|
135
|
+
var MILLISECONDS_PER_SECOND = 1e3;
|
|
136
|
+
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
137
|
+
var MAX_NANOSECONDS = 999999999;
|
|
138
|
+
var MAX_DATE_MILLISECONDS = 864e13;
|
|
139
|
+
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
140
|
+
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
141
|
+
["NaN", Number.NaN],
|
|
142
|
+
["Infinity", Number.POSITIVE_INFINITY],
|
|
143
|
+
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
144
|
+
]);
|
|
145
|
+
function refuse(path, reason) {
|
|
146
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
147
|
+
field: path
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
function isRecord(value) {
|
|
151
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
152
|
+
}
|
|
153
|
+
function startsWithTagPrefix(key) {
|
|
154
|
+
return key.startsWith(TAG_PREFIX);
|
|
155
|
+
}
|
|
156
|
+
function soleTagKey(record) {
|
|
157
|
+
let sole = null;
|
|
158
|
+
for (const key of Object.keys(record)) {
|
|
159
|
+
if (sole !== null) return null;
|
|
160
|
+
sole = key;
|
|
161
|
+
}
|
|
162
|
+
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
163
|
+
}
|
|
164
|
+
function dateOf(value, path) {
|
|
165
|
+
if (!Array.isArray(value) || value.length !== 2)
|
|
166
|
+
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
167
|
+
const seconds = value[0];
|
|
168
|
+
const nanoseconds = value[1];
|
|
169
|
+
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
170
|
+
refuse(
|
|
171
|
+
path,
|
|
172
|
+
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
176
|
+
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
177
|
+
}
|
|
178
|
+
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
179
|
+
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
180
|
+
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
181
|
+
return new Date(milliseconds);
|
|
182
|
+
}
|
|
183
|
+
function bytesOf(value, path) {
|
|
184
|
+
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
185
|
+
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
186
|
+
return bytes;
|
|
187
|
+
}
|
|
188
|
+
function numberOf(value, path) {
|
|
189
|
+
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
190
|
+
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
191
|
+
return number;
|
|
192
|
+
}
|
|
193
|
+
function requireDepth(path, depth) {
|
|
194
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
195
|
+
}
|
|
196
|
+
function untagMap(record, path, depth) {
|
|
197
|
+
requireDepth(path, depth);
|
|
198
|
+
for (const key of Object.keys(record)) {
|
|
199
|
+
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
200
|
+
}
|
|
201
|
+
return record;
|
|
202
|
+
}
|
|
203
|
+
function untagList(list, path, depth) {
|
|
204
|
+
requireDepth(path, depth);
|
|
205
|
+
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
206
|
+
return list;
|
|
207
|
+
}
|
|
208
|
+
function untagTag(tag, value, path, depth) {
|
|
209
|
+
switch (tag) {
|
|
210
|
+
case DATE_TAG:
|
|
211
|
+
return dateOf(value, path);
|
|
212
|
+
case BYTES_TAG:
|
|
213
|
+
return bytesOf(value, path);
|
|
214
|
+
case NUMBER_TAG:
|
|
215
|
+
return numberOf(value, path);
|
|
216
|
+
case OBJECT_TAG:
|
|
217
|
+
if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
218
|
+
return untagMap(value, path, depth);
|
|
219
|
+
default:
|
|
220
|
+
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
function untag(value, path, depth) {
|
|
224
|
+
if (typeof value !== "object" || value === null) return value;
|
|
225
|
+
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
226
|
+
const record = value;
|
|
227
|
+
const tag = soleTagKey(record);
|
|
228
|
+
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
229
|
+
}
|
|
230
|
+
function untagDocumentLine(text, parsed) {
|
|
231
|
+
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
232
|
+
return untagMap(parsed, "", 0);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// src/engine/export/import.ts
|
|
236
|
+
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
237
|
+
var DOCUMENT_EXPORT_VERSION = 1;
|
|
238
|
+
function invalidHeader(message) {
|
|
239
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
240
|
+
}
|
|
241
|
+
function isRecord2(value) {
|
|
242
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
243
|
+
}
|
|
244
|
+
function configOfHeader(line) {
|
|
245
|
+
if (line === void 0) invalidHeader("is missing");
|
|
246
|
+
if (!("text" in line)) {
|
|
247
|
+
invalidHeader(
|
|
248
|
+
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
let raw;
|
|
252
|
+
try {
|
|
253
|
+
raw = JSON.parse(line.text);
|
|
254
|
+
} catch {
|
|
255
|
+
invalidHeader("is not JSON");
|
|
256
|
+
}
|
|
257
|
+
if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
258
|
+
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
259
|
+
}
|
|
260
|
+
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
261
|
+
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
262
|
+
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
263
|
+
return {
|
|
264
|
+
schema: configuration.schema,
|
|
265
|
+
language: configuration.language,
|
|
266
|
+
...fields,
|
|
267
|
+
...embedding === void 0 ? {} : {
|
|
268
|
+
embedding: {
|
|
269
|
+
fields: embedding.fields,
|
|
270
|
+
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
async function prepareIndex(engine, indexName, config) {
|
|
276
|
+
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
277
|
+
await engine.createIndex(indexName, config);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
281
|
+
const exported = flattenSchema(config.schema);
|
|
282
|
+
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
283
|
+
for (const field of fields) {
|
|
284
|
+
if (existing[field] === exported[field]) continue;
|
|
285
|
+
throw new NarsilError(
|
|
286
|
+
ErrorCodes.CONFIG_INVALID,
|
|
287
|
+
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
288
|
+
{ indexName, field }
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function highestReportedAt(refused) {
|
|
293
|
+
let highest = 0;
|
|
294
|
+
for (let at = 1; at < refused.length; at++) {
|
|
295
|
+
if (refused[at].line > refused[highest].line) highest = at;
|
|
296
|
+
}
|
|
297
|
+
return highest;
|
|
298
|
+
}
|
|
299
|
+
function refuse2(state, line, code, message) {
|
|
300
|
+
state.result.refusedCount += 1;
|
|
301
|
+
const refused = state.result.refused;
|
|
302
|
+
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
303
|
+
refused.push({ line, code, message });
|
|
304
|
+
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
if (line >= refused[state.highestReported].line) return;
|
|
308
|
+
refused[state.highestReported] = { line, code, message };
|
|
309
|
+
state.highestReported = highestReportedAt(refused);
|
|
310
|
+
}
|
|
311
|
+
async function flush(state) {
|
|
312
|
+
if (state.batch.length === 0) return;
|
|
313
|
+
const batch = state.batch;
|
|
314
|
+
state.batch = [];
|
|
315
|
+
state.batchIds = /* @__PURE__ */ new Set();
|
|
316
|
+
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
317
|
+
const inserts = [];
|
|
318
|
+
const updates = [];
|
|
319
|
+
for (const entry of batch) {
|
|
320
|
+
if (await state.engine.has(state.indexName, entry.docId))
|
|
321
|
+
updates.push({ docId: entry.docId, document: entry.document });
|
|
322
|
+
else inserts.push(entry.document);
|
|
323
|
+
}
|
|
324
|
+
for (const outcome of [
|
|
325
|
+
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
326
|
+
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
327
|
+
]) {
|
|
328
|
+
if (outcome === null) continue;
|
|
329
|
+
state.result.imported += outcome.succeeded.length;
|
|
330
|
+
for (const failure of outcome.failed) {
|
|
331
|
+
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
332
|
+
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
async function takeLine(state, entry) {
|
|
337
|
+
if (!("text" in entry)) {
|
|
338
|
+
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
339
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
if (entry.text.length === 0) return;
|
|
343
|
+
let document;
|
|
344
|
+
try {
|
|
345
|
+
document = JSON.parse(entry.text);
|
|
346
|
+
} catch {
|
|
347
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
351
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
untagDocumentLine(entry.text, document);
|
|
356
|
+
} catch (error) {
|
|
357
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
358
|
+
refuse2(state, entry.line, error.code, error.message);
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
if (state.batchIds.has(document.id)) await flush(state);
|
|
362
|
+
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
363
|
+
state.batchIds.add(document.id);
|
|
364
|
+
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
365
|
+
}
|
|
366
|
+
async function readDocumentExport(engine, indexName, source) {
|
|
367
|
+
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
368
|
+
const header = await lines.next();
|
|
369
|
+
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
370
|
+
const state = {
|
|
371
|
+
engine,
|
|
372
|
+
indexName,
|
|
373
|
+
batch: [],
|
|
374
|
+
batchIds: /* @__PURE__ */ new Set(),
|
|
375
|
+
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
376
|
+
highestReported: 0
|
|
377
|
+
};
|
|
378
|
+
for await (const entry of lines) await takeLine(state, entry);
|
|
379
|
+
await flush(state);
|
|
380
|
+
state.result.refused.sort((a, b) => a.line - b.line);
|
|
381
|
+
return state.result;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, encodeBase64, readDocumentExport, startsWithTagPrefix };
|