@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,387 +0,0 @@
1
- import { chunksOf } from './chunk-CZQ33DLW.mjs';
2
- import { restoredConfigFields, restoredEmbedding } from './chunk-TCWQJL7O.mjs';
3
- import { requireConfigurationShape } from './chunk-W2XDK66Z.mjs';
4
- import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
5
- import { flattenSchema } from './chunk-UBXJQJII.mjs';
6
- import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
7
- import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
8
-
9
- // src/engine/export/lines.ts
10
- var NEWLINE = 10;
11
- function joined(parts, total) {
12
- if (parts.length === 1) return parts[0];
13
- const bytes = new Uint8Array(total);
14
- let offset = 0;
15
- for (const part of parts) {
16
- bytes.set(part, offset);
17
- offset += part.byteLength;
18
- }
19
- return bytes;
20
- }
21
- async function* exportLines(source, maxLineBytes) {
22
- const decoder = new TextDecoder("utf-8", { fatal: true });
23
- let parts = [];
24
- let pending = 0;
25
- let overflowing = false;
26
- let line = 0;
27
- const finishLine = () => {
28
- line += 1;
29
- if (overflowing) {
30
- overflowing = false;
31
- parts = [];
32
- pending = 0;
33
- return { line, refused: "too-long" };
34
- }
35
- const bytes = joined(parts, pending);
36
- parts = [];
37
- pending = 0;
38
- try {
39
- return { line, text: decoder.decode(bytes) };
40
- } catch {
41
- return { line, refused: "not-utf8" };
42
- }
43
- };
44
- for await (const chunk of chunksOf(source)) {
45
- let start = 0;
46
- for (; ; ) {
47
- const newline = chunk.indexOf(NEWLINE, start);
48
- const end = newline < 0 ? chunk.byteLength : newline;
49
- const piece = chunk.subarray(start, end);
50
- if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
51
- if (piece.byteLength > 0) parts.push(piece.slice());
52
- pending += piece.byteLength;
53
- } else {
54
- overflowing = true;
55
- parts = [];
56
- }
57
- if (newline < 0) break;
58
- yield finishLine();
59
- start = newline + 1;
60
- }
61
- }
62
- if (pending > 0 || overflowing) yield finishLine();
63
- }
64
-
65
- // src/serialization/base64.ts
66
- var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
67
- var PAD = 61;
68
- var BYTES_PER_GROUP = 3;
69
- var CHARACTERS_PER_GROUP = 4;
70
- var ASCII_LIMIT = 128;
71
- var ENCODE = new Uint8Array(ALPHABET.length);
72
- var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
73
- for (let at = 0; at < ALPHABET.length; at++) {
74
- ENCODE[at] = ALPHABET.charCodeAt(at);
75
- DECODE[ALPHABET.charCodeAt(at)] = at;
76
- }
77
- var asciiDecoder = new TextDecoder();
78
- function encodeBase64(bytes) {
79
- const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
80
- let write = 0;
81
- let read = 0;
82
- for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
83
- const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
84
- out[write++] = ENCODE[group >>> 18];
85
- out[write++] = ENCODE[group >>> 12 & 63];
86
- out[write++] = ENCODE[group >>> 6 & 63];
87
- out[write++] = ENCODE[group & 63];
88
- }
89
- const left = bytes.byteLength - read;
90
- if (left > 0) {
91
- const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
92
- out[write++] = ENCODE[group >>> 18];
93
- out[write++] = ENCODE[group >>> 12 & 63];
94
- out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
95
- out[write++] = PAD;
96
- }
97
- return asciiDecoder.decode(out);
98
- }
99
- function sextetAt(text, at) {
100
- const code = text.charCodeAt(at);
101
- return code < ASCII_LIMIT ? DECODE[code] : -1;
102
- }
103
- function decodeExactBase64(text) {
104
- if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
105
- const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
106
- const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
107
- const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
108
- let write = 0;
109
- for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
110
- const a2 = sextetAt(text, at);
111
- const b2 = sextetAt(text, at + 1);
112
- const c2 = sextetAt(text, at + 2);
113
- const d = sextetAt(text, at + 3);
114
- if ((a2 | b2 | c2 | d) < 0) return null;
115
- const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
116
- out[write++] = group >>> 16;
117
- out[write++] = group >>> 8 & 255;
118
- out[write++] = group & 255;
119
- }
120
- if (padding === 0) return out;
121
- const a = sextetAt(text, fullEnd);
122
- const b = sextetAt(text, fullEnd + 1);
123
- const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
124
- if ((a | b | c) < 0) return null;
125
- if (padding === 2 && (b & 15) !== 0) return null;
126
- if (padding === 1 && (c & 3) !== 0) return null;
127
- out[write++] = a << 2 | b >>> 4;
128
- if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
129
- return out;
130
- }
131
-
132
- // src/engine/export/tags.ts
133
- var TAG_PREFIX = "$";
134
- var DATE_TAG = "$date";
135
- var BYTES_TAG = "$bytes";
136
- var NUMBER_TAG = "$number";
137
- var OBJECT_TAG = "$object";
138
- var MILLISECONDS_PER_SECOND = 1e3;
139
- var NANOSECONDS_PER_MILLISECOND = 1e6;
140
- var MAX_NANOSECONDS = 999999999;
141
- var MAX_DATE_MILLISECONDS = 864e13;
142
- var ESCAPED_TAG_PREFIX = "\\u0024";
143
- var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
144
- ["NaN", Number.NaN],
145
- ["Infinity", Number.POSITIVE_INFINITY],
146
- ["-Infinity", Number.NEGATIVE_INFINITY]
147
- ]);
148
- function refuse(path, reason) {
149
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
150
- field: path
151
- });
152
- }
153
- function isRecord(value) {
154
- return typeof value === "object" && value !== null && !Array.isArray(value);
155
- }
156
- function startsWithTagPrefix(key) {
157
- return key.startsWith(TAG_PREFIX);
158
- }
159
- function soleTagKey(record) {
160
- let sole = null;
161
- for (const key of Object.keys(record)) {
162
- if (sole !== null) return null;
163
- sole = key;
164
- }
165
- return sole !== null && startsWithTagPrefix(sole) ? sole : null;
166
- }
167
- function dateOf(value, path) {
168
- if (!Array.isArray(value) || value.length !== 2)
169
- refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
170
- const seconds = value[0];
171
- const nanoseconds = value[1];
172
- if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
173
- refuse(
174
- path,
175
- `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
176
- );
177
- }
178
- if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
179
- refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
180
- }
181
- const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
182
- if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
183
- refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
184
- return new Date(milliseconds);
185
- }
186
- function bytesOf(value, path) {
187
- const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
188
- if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
189
- return bytes;
190
- }
191
- function numberOf(value, path) {
192
- const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
193
- if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
194
- return number;
195
- }
196
- function requireDepth(path, depth) {
197
- if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
198
- }
199
- function untagMap(record, path, depth) {
200
- requireDepth(path, depth);
201
- for (const key of Object.keys(record)) {
202
- record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
203
- }
204
- return record;
205
- }
206
- function untagList(list, path, depth) {
207
- requireDepth(path, depth);
208
- for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
209
- return list;
210
- }
211
- function untagTag(tag, value, path, depth) {
212
- switch (tag) {
213
- case DATE_TAG:
214
- return dateOf(value, path);
215
- case BYTES_TAG:
216
- return bytesOf(value, path);
217
- case NUMBER_TAG:
218
- return numberOf(value, path);
219
- case OBJECT_TAG:
220
- if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
221
- return untagMap(value, path, depth);
222
- default:
223
- return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
224
- }
225
- }
226
- function untag(value, path, depth) {
227
- if (typeof value !== "object" || value === null) return value;
228
- if (Array.isArray(value)) return untagList(value, path, depth);
229
- const record = value;
230
- const tag = soleTagKey(record);
231
- return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
232
- }
233
- function untagDocumentLine(text, parsed) {
234
- if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
235
- return untagMap(parsed, "", 0);
236
- }
237
-
238
- // src/engine/export/import.ts
239
- var DOCUMENT_EXPORT_FORMAT = "narsil-export";
240
- var DOCUMENT_EXPORT_VERSION = 1;
241
- function invalidHeader(message) {
242
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
243
- }
244
- function isRecord2(value) {
245
- return typeof value === "object" && value !== null && !Array.isArray(value);
246
- }
247
- function configOfHeader(line) {
248
- if (line === void 0) invalidHeader("is missing");
249
- if (!("text" in line)) {
250
- invalidHeader(
251
- line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
252
- );
253
- }
254
- let raw;
255
- try {
256
- raw = JSON.parse(line.text);
257
- } catch {
258
- invalidHeader("is not JSON");
259
- }
260
- if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
261
- invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
262
- }
263
- const configuration = requireConfigurationShape(raw.index, "document export header");
264
- const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
265
- const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
266
- return {
267
- schema: configuration.schema,
268
- language: configuration.language,
269
- ...fields,
270
- ...embedding === void 0 ? {} : {
271
- embedding: {
272
- fields: embedding.fields,
273
- ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
274
- }
275
- }
276
- };
277
- }
278
- async function prepareIndex(engine, indexName, config) {
279
- if (!engine.listIndexes().some((index) => index.name === indexName)) {
280
- await engine.createIndex(indexName, config);
281
- return;
282
- }
283
- const existing = flattenSchema(engine.getStats(indexName).schema);
284
- const exported = flattenSchema(config.schema);
285
- const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
286
- for (const field of fields) {
287
- if (existing[field] === exported[field]) continue;
288
- throw new NarsilError(
289
- ErrorCodes.CONFIG_INVALID,
290
- `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
291
- { indexName, field }
292
- );
293
- }
294
- }
295
- function highestReportedAt(refused) {
296
- let highest = 0;
297
- for (let at = 1; at < refused.length; at++) {
298
- if (refused[at].line > refused[highest].line) highest = at;
299
- }
300
- return highest;
301
- }
302
- function refuse2(state, line, code, message) {
303
- state.result.refusedCount += 1;
304
- const refused = state.result.refused;
305
- if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
306
- refused.push({ line, code, message });
307
- if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
308
- return;
309
- }
310
- if (line >= refused[state.highestReported].line) return;
311
- refused[state.highestReported] = { line, code, message };
312
- state.highestReported = highestReportedAt(refused);
313
- }
314
- async function flush(state) {
315
- if (state.batch.length === 0) return;
316
- const batch = state.batch;
317
- state.batch = [];
318
- state.batchIds = /* @__PURE__ */ new Set();
319
- const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
320
- const inserts = [];
321
- const updates = [];
322
- for (const entry of batch) {
323
- if (await state.engine.has(state.indexName, entry.docId))
324
- updates.push({ docId: entry.docId, document: entry.document });
325
- else inserts.push(entry.document);
326
- }
327
- for (const outcome of [
328
- inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
329
- updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
330
- ]) {
331
- if (outcome === null) continue;
332
- state.result.imported += outcome.succeeded.length;
333
- for (const failure of outcome.failed) {
334
- const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
335
- refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
336
- }
337
- }
338
- }
339
- async function takeLine(state, entry) {
340
- if (!("text" in entry)) {
341
- const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
342
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
343
- return;
344
- }
345
- if (entry.text.length === 0) return;
346
- let document;
347
- try {
348
- document = JSON.parse(entry.text);
349
- } catch {
350
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
351
- return;
352
- }
353
- if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
354
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
355
- return;
356
- }
357
- try {
358
- untagDocumentLine(entry.text, document);
359
- } catch (error) {
360
- if (!(error instanceof NarsilError)) throw error;
361
- refuse2(state, entry.line, error.code, error.message);
362
- return;
363
- }
364
- if (state.batchIds.has(document.id)) await flush(state);
365
- state.batch.push({ line: entry.line, docId: document.id, document });
366
- state.batchIds.add(document.id);
367
- if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
368
- }
369
- async function readDocumentExport(engine, indexName, source) {
370
- const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
371
- const header = await lines.next();
372
- await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
373
- const state = {
374
- engine,
375
- indexName,
376
- batch: [],
377
- batchIds: /* @__PURE__ */ new Set(),
378
- result: { imported: 0, refusedCount: 0, refused: [] },
379
- highestReported: 0
380
- };
381
- for await (const entry of lines) await takeLine(state, entry);
382
- await flush(state);
383
- state.result.refused.sort((a, b) => a.line - b.line);
384
- return state.result;
385
- }
386
-
387
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, encodeBase64, readDocumentExport, startsWithTagPrefix };
@@ -351,21 +351,6 @@ var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
351
351
  var DEFAULT_EF_SEARCH = 50;
352
352
  var NEAREST_LINK_FALLBACK_HOSTS = 4;
353
353
 
354
- // src/workers/thread-slot.ts
355
- var goneThreadSlots = /* @__PURE__ */ new Set();
356
- function threadSlotOfWorker(workerId) {
357
- return workerId + 1;
358
- }
359
- function noteThreadSlotLive(threadSlot) {
360
- goneThreadSlots.delete(threadSlot);
361
- }
362
- function noteThreadSlotGone(threadSlot) {
363
- goneThreadSlots.add(threadSlot);
364
- }
365
- function threadSlotGone(threadSlot) {
366
- return goneThreadSlots.has(threadSlot);
367
- }
368
-
369
354
  // src/vector/constants.ts
370
355
  var WASM_PAGE_BYTES = 65536;
371
356
  var MAX_WASM_PAGES = 65536;
@@ -399,6 +384,21 @@ var OSQ_REFINE_ROUNDS = 5;
399
384
  var OSQ_REFINE_TOLERANCE = 1e-8;
400
385
  var MIN_OVERSAMPLE = 1;
401
386
 
387
+ // src/workers/thread-slot.ts
388
+ var goneThreadSlots = /* @__PURE__ */ new Set();
389
+ function threadSlotOfWorker(workerId) {
390
+ return workerId + 1;
391
+ }
392
+ function noteThreadSlotLive(threadSlot) {
393
+ goneThreadSlots.delete(threadSlot);
394
+ }
395
+ function noteThreadSlotGone(threadSlot) {
396
+ goneThreadSlots.add(threadSlot);
397
+ }
398
+ function threadSlotGone(threadSlot) {
399
+ return goneThreadSlots.has(threadSlot);
400
+ }
401
+
402
402
  // src/workers/constants.ts
403
403
  var WORKER_YOUNG_GENERATION_MB = 24;
404
404
  var MIN_WORKER_COUNT = 2;