@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import './chunk-
|
|
5
|
-
import {
|
|
6
|
-
import './chunk-
|
|
7
|
-
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-
|
|
8
|
-
import {
|
|
9
|
-
import './chunk-
|
|
10
|
-
import { MAX_FILE_BYTES } from './chunk-4MFH226Q.mjs';
|
|
1
|
+
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-KTOK275N.mjs';
|
|
2
|
+
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-ZEV26AQ6.mjs';
|
|
3
|
+
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-MVM33ZUL.mjs';
|
|
4
|
+
import './chunk-TCLEOZ4P.mjs';
|
|
5
|
+
import { vectorsPerFile } from './chunk-6RHTAVZX.mjs';
|
|
6
|
+
import './chunk-HKX2G6WH.mjs';
|
|
7
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
8
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-R4WUDZQV.mjs';
|
|
9
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-5DB2VDGI.mjs';
|
|
11
10
|
import './chunk-JPTQZWJQ.mjs';
|
|
11
|
+
import './chunk-26A5JFT2.mjs';
|
|
12
|
+
import './chunk-OIZS5KIX.mjs';
|
|
13
|
+
import { MAX_FILE_BYTES } from './chunk-D33A5VMU.mjs';
|
|
12
14
|
import './chunk-MZI3TEOW.mjs';
|
|
13
15
|
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
14
16
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -70,10 +72,11 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
70
72
|
codes: null,
|
|
71
73
|
vectors: vectorsToBytes(packed)
|
|
72
74
|
};
|
|
73
|
-
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
|
|
75
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
|
|
74
76
|
}
|
|
75
77
|
return {
|
|
76
78
|
unchanged: false,
|
|
79
|
+
storesHeads: false,
|
|
77
80
|
kept: [],
|
|
78
81
|
keptFiles: [],
|
|
79
82
|
newFiles: files.length,
|
|
@@ -165,6 +168,7 @@ async function buildArchiveFromMemory(input) {
|
|
|
165
168
|
|
|
166
169
|
// src/persistence/archive/constants.ts
|
|
167
170
|
var MAX_PAX_HEADER_BYTES = 65536;
|
|
171
|
+
var ARCHIVE_FILE_CHUNK_BYTES = 4194304;
|
|
168
172
|
|
|
169
173
|
// src/persistence/archive/tar.ts
|
|
170
174
|
var TAR_BLOCK_BYTES = 512;
|
|
@@ -349,17 +353,32 @@ async function readBlock(reader) {
|
|
|
349
353
|
if (block.byteLength < TAR_BLOCK_BYTES) invalidArchive("the archive ends inside an entry header");
|
|
350
354
|
return block;
|
|
351
355
|
}
|
|
356
|
+
async function skipPadding(reader, size) {
|
|
357
|
+
const remainder = size % TAR_BLOCK_BYTES;
|
|
358
|
+
if (remainder === 0) return;
|
|
359
|
+
const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
|
|
360
|
+
if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
|
|
361
|
+
invalidArchive("the archive ends inside the padding of an entry");
|
|
362
|
+
}
|
|
363
|
+
}
|
|
352
364
|
async function readPadded(reader, size) {
|
|
353
365
|
const body = await reader.readExactly(size) ?? new Uint8Array(0);
|
|
354
366
|
if (body.byteLength < size) invalidArchive("the archive ends inside an entry");
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
367
|
+
await skipPadding(reader, size);
|
|
368
|
+
return body;
|
|
369
|
+
}
|
|
370
|
+
function entryBody(reader, size) {
|
|
371
|
+
let remaining = size;
|
|
372
|
+
async function* chunks() {
|
|
373
|
+
while (remaining > 0) {
|
|
374
|
+
const length = Math.min(ARCHIVE_FILE_CHUNK_BYTES, remaining);
|
|
375
|
+
const chunk = await reader.readExactly(length);
|
|
376
|
+
if (chunk === null || chunk.byteLength < length) invalidArchive("the archive ends inside an entry");
|
|
377
|
+
remaining -= length;
|
|
378
|
+
yield chunk;
|
|
360
379
|
}
|
|
361
380
|
}
|
|
362
|
-
return
|
|
381
|
+
return chunks();
|
|
363
382
|
}
|
|
364
383
|
async function nextEntryHeader(reader) {
|
|
365
384
|
const block = await readBlock(reader);
|
|
@@ -419,12 +438,16 @@ async function readSnapshotArchive(reader, handlers) {
|
|
|
419
438
|
if (header.size !== ref.bytes) {
|
|
420
439
|
invalidArchive(`the entry "${expected}" holds ${header.size} bytes where the manifest lists ${ref.bytes}`);
|
|
421
440
|
}
|
|
422
|
-
const
|
|
423
|
-
if (headerChecksumOf(
|
|
441
|
+
const start = await reader.peek(Math.min(HEADER_SIZE, header.size));
|
|
442
|
+
if (headerChecksumOf(start) !== ref.checksum) {
|
|
424
443
|
invalidArchive(`the entry "${expected}" carries a checksum other than the manifest lists`, { entry: expected });
|
|
425
444
|
}
|
|
426
445
|
seen.add(expected);
|
|
427
|
-
|
|
446
|
+
const entry = entryBody(reader, header.size);
|
|
447
|
+
await handlers.file(expected, ref, entry);
|
|
448
|
+
let unread = await entry.next();
|
|
449
|
+
while (unread.done !== true) unread = await entry.next();
|
|
450
|
+
await skipPadding(reader, header.size);
|
|
428
451
|
}
|
|
429
452
|
const trailing = await nextEntryHeader(reader);
|
|
430
453
|
if (trailing !== null) {
|
|
@@ -440,12 +463,20 @@ function entryChunks(name, bytes) {
|
|
|
440
463
|
(chunk) => chunk.byteLength > 0
|
|
441
464
|
);
|
|
442
465
|
}
|
|
466
|
+
async function* fileEntryChunks(source, name, ref) {
|
|
467
|
+
yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
|
|
468
|
+
for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
|
|
469
|
+
yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
|
|
470
|
+
}
|
|
471
|
+
const padding = tarPadding(ref.bytes);
|
|
472
|
+
if (padding.byteLength > 0) yield padding;
|
|
473
|
+
}
|
|
443
474
|
async function* archiveChunks(source) {
|
|
444
475
|
yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
|
|
445
476
|
yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
|
|
446
477
|
const prefix = `${source.sourceIndexName}/`;
|
|
447
478
|
for (const ref of manifestFileRefs(source.manifest)) {
|
|
448
|
-
yield*
|
|
479
|
+
yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
|
|
449
480
|
}
|
|
450
481
|
yield tarEnd();
|
|
451
482
|
}
|
|
@@ -479,6 +510,27 @@ function archiveStream(source) {
|
|
|
479
510
|
});
|
|
480
511
|
}
|
|
481
512
|
|
|
513
|
+
// src/persistence/archive/file-entry.ts
|
|
514
|
+
async function writeArchiveFile(store, key, chunks) {
|
|
515
|
+
const { directory } = store;
|
|
516
|
+
if (directory === void 0) {
|
|
517
|
+
const held = [];
|
|
518
|
+
for await (const chunk of chunks) held.push(chunk);
|
|
519
|
+
await store.write(key, held);
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
const receiving = `${key}.receiving`;
|
|
523
|
+
await directory.remove(receiving);
|
|
524
|
+
const handle = await directory.appendHandle(receiving);
|
|
525
|
+
try {
|
|
526
|
+
for await (const chunk of chunks) await handle.append(chunk);
|
|
527
|
+
await handle.sync();
|
|
528
|
+
} finally {
|
|
529
|
+
await handle.close();
|
|
530
|
+
}
|
|
531
|
+
await directory.move(receiving, key);
|
|
532
|
+
}
|
|
533
|
+
|
|
482
534
|
// src/persistence/archive/validate.ts
|
|
483
535
|
function requireDimension(key, found, expected) {
|
|
484
536
|
if (expected === found) return;
|
|
@@ -502,6 +554,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
|
|
|
502
554
|
async function validateStagedCheckpoint(store, manifest) {
|
|
503
555
|
for (const partition of manifest.partitions) {
|
|
504
556
|
for (const ref of partition.segments) {
|
|
557
|
+
const { directory } = store;
|
|
558
|
+
if (directory !== void 0) {
|
|
559
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
|
|
560
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
561
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
505
565
|
const contents = await readSegmentFile(store, ref);
|
|
506
566
|
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
507
567
|
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -557,6 +617,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
557
617
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
558
618
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
559
619
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
620
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
560
621
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
561
622
|
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
562
623
|
}
|
|
@@ -702,7 +763,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
|
702
763
|
}
|
|
703
764
|
await staging.finish();
|
|
704
765
|
const relocate = await staging.relocation().catch(() => null);
|
|
705
|
-
if (relocate
|
|
766
|
+
if (relocate === null) return;
|
|
767
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
768
|
+
const { manager } = restored.detached;
|
|
769
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
770
|
+
const partition = manager.getPartition(partitionId);
|
|
771
|
+
if (!isCompositePartition(partition)) continue;
|
|
772
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
773
|
+
}
|
|
706
774
|
}
|
|
707
775
|
function activate(indexName, restored, entry, deps) {
|
|
708
776
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -825,6 +893,9 @@ function partitionCountOf(manifest) {
|
|
|
825
893
|
function missingPart(what) {
|
|
826
894
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
827
895
|
}
|
|
896
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
897
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
898
|
+
}
|
|
828
899
|
async function stageArchive(indexName, reader, staging, deps) {
|
|
829
900
|
const staged = { prepared: null, manifest: null };
|
|
830
901
|
await readSnapshotArchive(reader, {
|
|
@@ -835,17 +906,29 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
835
906
|
if (staged.prepared === null) missingPart("configuration");
|
|
836
907
|
requireSameSchema(staged.prepared, manifest2);
|
|
837
908
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
838
|
-
|
|
909
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
910
|
+
requireRestoreMemory(
|
|
911
|
+
indexName,
|
|
912
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
913
|
+
replacesIndex
|
|
914
|
+
);
|
|
839
915
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
840
916
|
},
|
|
841
|
-
file: async (entryName, _ref,
|
|
917
|
+
file: async (entryName, _ref, chunks) => {
|
|
842
918
|
if (staged.manifest === null) missingPart("manifest");
|
|
843
|
-
await staging
|
|
919
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
844
920
|
}
|
|
845
921
|
});
|
|
846
922
|
const { prepared, manifest } = staged;
|
|
847
923
|
if (prepared === null) missingPart("configuration");
|
|
848
924
|
if (manifest === null) missingPart("manifest");
|
|
925
|
+
const { directory } = staging;
|
|
926
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
927
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-BUFNZMEU.mjs');
|
|
928
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
929
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
930
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
931
|
+
}
|
|
849
932
|
await validateStagedCheckpoint(staging, manifest);
|
|
850
933
|
return { prepared, manifest };
|
|
851
934
|
}
|
|
@@ -1,100 +1,20 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import './chunk-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-VFYMPCOV.mjs';
|
|
2
|
+
import './chunk-ZEV26AQ6.mjs';
|
|
3
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-CCOAOO73.mjs';
|
|
4
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-TCLEOZ4P.mjs';
|
|
5
|
+
import './chunk-WCFCHIBB.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-5DB2VDGI.mjs';
|
|
8
8
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
|
+
import './chunk-26A5JFT2.mjs';
|
|
10
|
+
import './chunk-D33A5VMU.mjs';
|
|
9
11
|
import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
|
|
10
|
-
import
|
|
12
|
+
import './chunk-35B42UFM.mjs';
|
|
11
13
|
import { decode } from '@msgpack/msgpack';
|
|
12
14
|
|
|
13
|
-
// src/engine/export/json-line.ts
|
|
14
|
-
function refuse(path, kind, docId) {
|
|
15
|
-
throw new NarsilError(
|
|
16
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
17
|
-
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
18
|
-
{ docId, field: path }
|
|
19
|
-
);
|
|
20
|
-
}
|
|
21
|
-
function encodeNumber(value, path, target) {
|
|
22
|
-
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
23
|
-
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
24
|
-
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
25
|
-
}
|
|
26
|
-
function encodeDate(value, path, target) {
|
|
27
|
-
const milliseconds = value.getTime();
|
|
28
|
-
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
29
|
-
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
30
|
-
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
31
|
-
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
32
|
-
}
|
|
33
|
-
function encodeBytes(value, path, target) {
|
|
34
|
-
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
35
|
-
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
36
|
-
}
|
|
37
|
-
function encodeList(value, path, target) {
|
|
38
|
-
target.parts.push("[");
|
|
39
|
-
for (let at = 0; at < value.length; at++) {
|
|
40
|
-
if (at > 0) target.parts.push(",");
|
|
41
|
-
encodeValue(value[at], `${path}[${at}]`, target);
|
|
42
|
-
}
|
|
43
|
-
target.parts.push("]");
|
|
44
|
-
}
|
|
45
|
-
function encodeMap(value, path, target) {
|
|
46
|
-
const proto = Object.getPrototypeOf(value);
|
|
47
|
-
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
48
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
49
|
-
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
50
|
-
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
51
|
-
for (let at = 0; at < entries.length; at++) {
|
|
52
|
-
const [key, entry] = entries[at];
|
|
53
|
-
if (at > 0) target.parts.push(",");
|
|
54
|
-
target.parts.push(JSON.stringify(key), ":");
|
|
55
|
-
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
56
|
-
}
|
|
57
|
-
target.parts.push(wrapped ? "}}" : "}");
|
|
58
|
-
}
|
|
59
|
-
function encodeValue(value, path, target) {
|
|
60
|
-
if (value === null) {
|
|
61
|
-
target.parts.push("null");
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
switch (typeof value) {
|
|
65
|
-
case "string":
|
|
66
|
-
target.parts.push(JSON.stringify(value));
|
|
67
|
-
return;
|
|
68
|
-
case "boolean":
|
|
69
|
-
target.parts.push(value ? "true" : "false");
|
|
70
|
-
return;
|
|
71
|
-
case "number":
|
|
72
|
-
target.parts.push(encodeNumber(value, path, target));
|
|
73
|
-
return;
|
|
74
|
-
case "object":
|
|
75
|
-
break;
|
|
76
|
-
default:
|
|
77
|
-
refuse(path, `a ${typeof value}`, target.docId);
|
|
78
|
-
}
|
|
79
|
-
if (Array.isArray(value)) encodeList(value, path, target);
|
|
80
|
-
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
81
|
-
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
82
|
-
else encodeMap(value, path, target);
|
|
83
|
-
}
|
|
84
|
-
function encodeExactJson(value, docId) {
|
|
85
|
-
const target = { docId, tagged: false, parts: [] };
|
|
86
|
-
encodeValue(value, "", target);
|
|
87
|
-
return target.parts.join("");
|
|
88
|
-
}
|
|
89
|
-
function encodeDocumentJson(document, docId) {
|
|
90
|
-
const target = { docId, tagged: true, parts: [] };
|
|
91
|
-
encodeValue(document, "", target);
|
|
92
|
-
return target.parts.join("");
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// src/engine/export/write.ts
|
|
96
15
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
97
16
|
var FLOAT32_BYTES = 4;
|
|
17
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
98
18
|
async function locateVectors(checkpoint) {
|
|
99
19
|
const located = /* @__PURE__ */ new Map();
|
|
100
20
|
for (const field of checkpoint.manifest.vectors) {
|
|
@@ -142,7 +62,59 @@ async function winnersOf(checkpoint, partition) {
|
|
|
142
62
|
}
|
|
143
63
|
return winners;
|
|
144
64
|
}
|
|
65
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
66
|
+
const contents = await readSegment(checkpoint, ref);
|
|
67
|
+
const table = contents.arrays.documentTable;
|
|
68
|
+
const ids = /* @__PURE__ */ new Set();
|
|
69
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
70
|
+
return {
|
|
71
|
+
count: contents.arrays.documentCount,
|
|
72
|
+
schema: contents.schema,
|
|
73
|
+
tombstones: new Set(contents.tombstones),
|
|
74
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
75
|
+
holds: (docId) => ids.has(docId),
|
|
76
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
77
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
78
|
+
table.discardedFields
|
|
79
|
+
),
|
|
80
|
+
close: () => void 0
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
84
|
+
const located = await locateVectors(checkpoint);
|
|
85
|
+
return {
|
|
86
|
+
async vectorOf(fieldPath, docId) {
|
|
87
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
88
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
89
|
+
},
|
|
90
|
+
close: () => located.clear()
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
94
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-BUFNZMEU.mjs');
|
|
95
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
96
|
+
try {
|
|
97
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
98
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
99
|
+
}
|
|
100
|
+
} finally {
|
|
101
|
+
vectors.close();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
105
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
106
|
+
for (const ref of partition.segments) {
|
|
107
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
108
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
145
113
|
async function* documentLines(checkpoint) {
|
|
114
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
115
|
+
yield* pagedDocumentLines(checkpoint);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
146
118
|
const vectors = await locateVectors(checkpoint);
|
|
147
119
|
for (const partition of checkpoint.manifest.partitions) {
|
|
148
120
|
const winners = await winnersOf(checkpoint, partition);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { E as ErrorDetail, i as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CIG5yDyg.js';
|
|
3
|
+
import { E as ErrorDetail, i as TaskStore } from './tasks-CjRJRGTb.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KGOXHLLZ.mjs';
|
|
2
2
|
import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { untagDocumentLine } from './chunk-BOJZGUS6.mjs';
|
|
2
|
+
import { chunksOf } from './chunk-CZQ33DLW.mjs';
|
|
3
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-L76W3IZJ.mjs';
|
|
4
|
+
import { requireConfigurationShape } from './chunk-IO5F7ZBE.mjs';
|
|
5
|
+
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
|
|
6
|
+
import { flattenSchema } from './chunk-KGOXHLLZ.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
8
|
+
|
|
9
|
+
// src/engine/export/lines.ts
|
|
10
|
+
var NEWLINE = 10;
|
|
11
|
+
function joined(parts, total) {
|
|
12
|
+
if (parts.length === 1) return parts[0];
|
|
13
|
+
const bytes = new Uint8Array(total);
|
|
14
|
+
let offset = 0;
|
|
15
|
+
for (const part of parts) {
|
|
16
|
+
bytes.set(part, offset);
|
|
17
|
+
offset += part.byteLength;
|
|
18
|
+
}
|
|
19
|
+
return bytes;
|
|
20
|
+
}
|
|
21
|
+
async function* exportLines(source, maxLineBytes) {
|
|
22
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
23
|
+
let parts = [];
|
|
24
|
+
let pending = 0;
|
|
25
|
+
let overflowing = false;
|
|
26
|
+
let line = 0;
|
|
27
|
+
const finishLine = () => {
|
|
28
|
+
line += 1;
|
|
29
|
+
if (overflowing) {
|
|
30
|
+
overflowing = false;
|
|
31
|
+
parts = [];
|
|
32
|
+
pending = 0;
|
|
33
|
+
return { line, refused: "too-long" };
|
|
34
|
+
}
|
|
35
|
+
const bytes = joined(parts, pending);
|
|
36
|
+
parts = [];
|
|
37
|
+
pending = 0;
|
|
38
|
+
try {
|
|
39
|
+
return { line, text: decoder.decode(bytes) };
|
|
40
|
+
} catch {
|
|
41
|
+
return { line, refused: "not-utf8" };
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
for await (const chunk of chunksOf(source)) {
|
|
45
|
+
let start = 0;
|
|
46
|
+
for (; ; ) {
|
|
47
|
+
const newline = chunk.indexOf(NEWLINE, start);
|
|
48
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
49
|
+
const piece = chunk.subarray(start, end);
|
|
50
|
+
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
51
|
+
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
52
|
+
pending += piece.byteLength;
|
|
53
|
+
} else {
|
|
54
|
+
overflowing = true;
|
|
55
|
+
parts = [];
|
|
56
|
+
}
|
|
57
|
+
if (newline < 0) break;
|
|
58
|
+
yield finishLine();
|
|
59
|
+
start = newline + 1;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (pending > 0 || overflowing) yield finishLine();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// src/engine/export/import.ts
|
|
66
|
+
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
67
|
+
var DOCUMENT_EXPORT_VERSION = 1;
|
|
68
|
+
function invalidHeader(message) {
|
|
69
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
70
|
+
}
|
|
71
|
+
function isRecord(value) {
|
|
72
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
73
|
+
}
|
|
74
|
+
function configOfHeader(line) {
|
|
75
|
+
if (line === void 0) invalidHeader("is missing");
|
|
76
|
+
if (!("text" in line)) {
|
|
77
|
+
invalidHeader(
|
|
78
|
+
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
let raw;
|
|
82
|
+
try {
|
|
83
|
+
raw = JSON.parse(line.text);
|
|
84
|
+
} catch {
|
|
85
|
+
invalidHeader("is not JSON");
|
|
86
|
+
}
|
|
87
|
+
if (!isRecord(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
88
|
+
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
89
|
+
}
|
|
90
|
+
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
91
|
+
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
92
|
+
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
93
|
+
return {
|
|
94
|
+
schema: configuration.schema,
|
|
95
|
+
language: configuration.language,
|
|
96
|
+
...fields,
|
|
97
|
+
...embedding === void 0 ? {} : {
|
|
98
|
+
embedding: {
|
|
99
|
+
fields: embedding.fields,
|
|
100
|
+
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
async function prepareIndex(engine, indexName, config) {
|
|
106
|
+
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
107
|
+
await engine.createIndex(indexName, config);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
111
|
+
const exported = flattenSchema(config.schema);
|
|
112
|
+
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
113
|
+
for (const field of fields) {
|
|
114
|
+
if (existing[field] === exported[field]) continue;
|
|
115
|
+
throw new NarsilError(
|
|
116
|
+
ErrorCodes.CONFIG_INVALID,
|
|
117
|
+
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
118
|
+
{ indexName, field }
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function highestReportedAt(refused) {
|
|
123
|
+
let highest = 0;
|
|
124
|
+
for (let at = 1; at < refused.length; at++) {
|
|
125
|
+
if (refused[at].line > refused[highest].line) highest = at;
|
|
126
|
+
}
|
|
127
|
+
return highest;
|
|
128
|
+
}
|
|
129
|
+
function refuse(state, line, code, message) {
|
|
130
|
+
state.result.refusedCount += 1;
|
|
131
|
+
const refused = state.result.refused;
|
|
132
|
+
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
133
|
+
refused.push({ line, code, message });
|
|
134
|
+
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (line >= refused[state.highestReported].line) return;
|
|
138
|
+
refused[state.highestReported] = { line, code, message };
|
|
139
|
+
state.highestReported = highestReportedAt(refused);
|
|
140
|
+
}
|
|
141
|
+
async function flush(state) {
|
|
142
|
+
if (state.batch.length === 0) return;
|
|
143
|
+
const batch = state.batch;
|
|
144
|
+
state.batch = [];
|
|
145
|
+
state.batchIds = /* @__PURE__ */ new Set();
|
|
146
|
+
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
147
|
+
const inserts = [];
|
|
148
|
+
const updates = [];
|
|
149
|
+
for (const entry of batch) {
|
|
150
|
+
if (await state.engine.has(state.indexName, entry.docId))
|
|
151
|
+
updates.push({ docId: entry.docId, document: entry.document });
|
|
152
|
+
else inserts.push(entry.document);
|
|
153
|
+
}
|
|
154
|
+
for (const outcome of [
|
|
155
|
+
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
156
|
+
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
157
|
+
]) {
|
|
158
|
+
if (outcome === null) continue;
|
|
159
|
+
state.result.imported += outcome.succeeded.length;
|
|
160
|
+
for (const failure of outcome.failed) {
|
|
161
|
+
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
162
|
+
refuse(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
async function takeLine(state, entry) {
|
|
167
|
+
if (!("text" in entry)) {
|
|
168
|
+
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
169
|
+
refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (entry.text.length === 0) return;
|
|
173
|
+
let document;
|
|
174
|
+
try {
|
|
175
|
+
document = JSON.parse(entry.text);
|
|
176
|
+
} catch {
|
|
177
|
+
refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (!isRecord(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
181
|
+
refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
try {
|
|
185
|
+
untagDocumentLine(entry.text, document);
|
|
186
|
+
} catch (error) {
|
|
187
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
188
|
+
refuse(state, entry.line, error.code, error.message);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
if (state.batchIds.has(document.id)) await flush(state);
|
|
192
|
+
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
193
|
+
state.batchIds.add(document.id);
|
|
194
|
+
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
195
|
+
}
|
|
196
|
+
async function readDocumentExport(engine, indexName, source) {
|
|
197
|
+
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
198
|
+
const header = await lines.next();
|
|
199
|
+
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
200
|
+
const state = {
|
|
201
|
+
engine,
|
|
202
|
+
indexName,
|
|
203
|
+
batch: [],
|
|
204
|
+
batchIds: /* @__PURE__ */ new Set(),
|
|
205
|
+
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
206
|
+
highestReported: 0
|
|
207
|
+
};
|
|
208
|
+
for await (const entry of lines) await takeLine(state, entry);
|
|
209
|
+
await flush(state);
|
|
210
|
+
state.result.refused.sort((a, b) => a.line - b.line);
|
|
211
|
+
return state.result;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport };
|