@delali/narsil 0.3.1-next.90 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
- package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
- package/dist/chunk-2GZGBOQI.mjs +426 -0
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/chunk-7KC56574.mjs +507 -0
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
- package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
- package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
- package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/chunk-UAUZGXJP.mjs +3591 -0
- package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
- package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/chunk-ZHS44YKF.mjs +35 -0
- package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +16 -2
- package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +5 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +3 -3
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4620 -4293
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +22 -18
- package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
- package/dist/react.d.ts +8 -7
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +38 -11
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-EGESD26S.mjs +0 -18
- package/dist/snapshot-22CUEYYW.mjs +0 -469
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint,
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors,
|
|
5
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION,
|
|
6
|
-
import { NarsilError, ErrorCodes, MAX_FILE_BYTES
|
|
1
|
+
import { vectorsPerFile } from './chunk-5S74HQRT.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-4JTBL63P.mjs';
|
|
3
|
+
import './chunk-PP6N5CCS.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, readHeapStatistics, readResidentMemory, prepareStoredSynonyms, rejectSnapshot, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-HC6DMR4T.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
8
8
|
|
|
9
9
|
// src/persistence/archive/memory-store.ts
|
|
@@ -498,6 +498,136 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
498
498
|
}
|
|
499
499
|
}
|
|
500
500
|
|
|
501
|
+
// src/engine/snapshot/constants.ts
|
|
502
|
+
var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
|
|
503
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
|
|
504
|
+
var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
|
|
505
|
+
var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
|
|
506
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
|
|
507
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
|
|
508
|
+
var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
|
|
509
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
|
|
510
|
+
var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
|
|
511
|
+
var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
|
|
512
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
|
|
513
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
|
|
514
|
+
|
|
515
|
+
// src/engine/snapshot/memory-budget.ts
|
|
516
|
+
var MEGABYTE_BYTES = 1048576;
|
|
517
|
+
var ARCHIVE_RATES = {
|
|
518
|
+
heap: {
|
|
519
|
+
perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
|
|
520
|
+
perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
|
|
521
|
+
perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
522
|
+
},
|
|
523
|
+
external: {
|
|
524
|
+
perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
525
|
+
perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
|
|
526
|
+
perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
527
|
+
}
|
|
528
|
+
};
|
|
529
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
530
|
+
heap: {
|
|
531
|
+
perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
532
|
+
perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
533
|
+
perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
534
|
+
},
|
|
535
|
+
external: {
|
|
536
|
+
perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
537
|
+
perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
538
|
+
perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
542
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
543
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
544
|
+
return bytes;
|
|
545
|
+
}
|
|
546
|
+
function footprintAt(rates, storedBytes, vectors) {
|
|
547
|
+
return {
|
|
548
|
+
heapBytes: bytesAt(rates.heap, storedBytes, vectors),
|
|
549
|
+
externalBytes: bytesAt(rates.external, storedBytes, vectors)
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
function deadCountOf(dead) {
|
|
553
|
+
if (dead === null) return 0;
|
|
554
|
+
let count = 0;
|
|
555
|
+
for (const byte of dead) {
|
|
556
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
557
|
+
}
|
|
558
|
+
return count;
|
|
559
|
+
}
|
|
560
|
+
function archiveFootprint(prepared, manifest) {
|
|
561
|
+
let segmentBytes = 0;
|
|
562
|
+
for (const partition of manifest.partitions) {
|
|
563
|
+
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
564
|
+
}
|
|
565
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
566
|
+
const vectors = [];
|
|
567
|
+
for (const field of manifest.vectors) {
|
|
568
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
569
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
570
|
+
}
|
|
571
|
+
return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
|
|
572
|
+
}
|
|
573
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
|
|
574
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
575
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
576
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
|
|
577
|
+
}
|
|
578
|
+
function inMegabytes(bytes) {
|
|
579
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
580
|
+
}
|
|
581
|
+
function refuse(indexName, budget, neededBytes, availableBytes) {
|
|
582
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
583
|
+
throw new NarsilError(
|
|
584
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
585
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
|
|
586
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
function requireRestoreMemory(indexName, footprint) {
|
|
590
|
+
const heap = readHeapStatistics();
|
|
591
|
+
if (heap !== null) {
|
|
592
|
+
const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
|
|
593
|
+
if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
|
|
594
|
+
}
|
|
595
|
+
const resident = readResidentMemory();
|
|
596
|
+
if (resident !== null) {
|
|
597
|
+
const neededBytes = footprint.heapBytes + footprint.externalBytes;
|
|
598
|
+
const availableBytes = resident.limitBytes - resident.residentBytes;
|
|
599
|
+
if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// src/workers/detached-index.ts
|
|
604
|
+
function attachablePolicy(shared) {
|
|
605
|
+
let attached = false;
|
|
606
|
+
const policy = {
|
|
607
|
+
get enabled() {
|
|
608
|
+
return attached && shared.enabled;
|
|
609
|
+
},
|
|
610
|
+
get count() {
|
|
611
|
+
return shared.count;
|
|
612
|
+
},
|
|
613
|
+
get host() {
|
|
614
|
+
return shared.host;
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
return {
|
|
618
|
+
policy,
|
|
619
|
+
attach() {
|
|
620
|
+
attached = true;
|
|
621
|
+
}
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
625
|
+
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
626
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
627
|
+
entry.manager.synonyms = synonyms;
|
|
628
|
+
return { indexName, manager: entry.manager, entry, attach };
|
|
629
|
+
}
|
|
630
|
+
|
|
501
631
|
// src/engine/snapshot/restore-setup.ts
|
|
502
632
|
function prepareRestoredIndex(configuration, deps, reject) {
|
|
503
633
|
const configFields = restoredConfigFields(configuration, reject);
|
|
@@ -534,15 +664,120 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
534
664
|
}
|
|
535
665
|
}
|
|
536
666
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
537
|
-
|
|
667
|
+
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
668
|
+
return { config, language, embeddingAdapter, adapterName, storedRevision, synonyms };
|
|
538
669
|
}
|
|
539
|
-
|
|
670
|
+
function restoredSynonyms(raw, config, language, reject) {
|
|
671
|
+
if (raw === void 0 || raw === null) return null;
|
|
672
|
+
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
673
|
+
}
|
|
674
|
+
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
540
675
|
const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
|
|
541
676
|
if (loadingVectorCode !== null) await loadingVectorCode;
|
|
542
|
-
|
|
543
|
-
|
|
677
|
+
const detached = createDetachedIndex(
|
|
678
|
+
indexName,
|
|
679
|
+
prepared.config,
|
|
680
|
+
prepared.language,
|
|
681
|
+
prepared.synonyms?.lookup ?? null,
|
|
682
|
+
deps.executor
|
|
683
|
+
);
|
|
684
|
+
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
685
|
+
return detached;
|
|
686
|
+
}
|
|
687
|
+
function restoredRegistryEntry(prepared, manager, deps) {
|
|
688
|
+
return {
|
|
689
|
+
config: prepared.config,
|
|
690
|
+
language: prepared.language,
|
|
691
|
+
embeddingAdapter: prepared.embeddingAdapter,
|
|
692
|
+
embeddingAdapterName: prepared.adapterName,
|
|
693
|
+
vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
|
|
694
|
+
indexUuid: null,
|
|
695
|
+
heldPartitions: null,
|
|
696
|
+
documentCount: manager.countDocuments(),
|
|
697
|
+
partitionCount: manager.partitionCount,
|
|
698
|
+
analysisRevision: prepared.storedRevision,
|
|
699
|
+
synonyms: prepared.synonyms?.encoded ?? null
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
// src/engine/snapshot/replace.ts
|
|
704
|
+
function releaseTimedOut(indexName, timeoutMs) {
|
|
705
|
+
return new NarsilError(
|
|
706
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
707
|
+
`The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
|
|
708
|
+
{ indexName, timeoutMs }
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
async function withinLimit(indexName, release, work, deps) {
|
|
712
|
+
release.settled = work.catch(() => {
|
|
713
|
+
release.reloads = true;
|
|
714
|
+
});
|
|
715
|
+
let timer;
|
|
716
|
+
const limit = new Promise((_, reject) => {
|
|
717
|
+
timer = setTimeout(() => {
|
|
718
|
+
release.timedOut = true;
|
|
719
|
+
reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
|
|
720
|
+
}, deps.releaseTimeoutMs);
|
|
721
|
+
});
|
|
722
|
+
try {
|
|
723
|
+
await Promise.race([work, limit]);
|
|
724
|
+
} finally {
|
|
725
|
+
clearTimeout(timer);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
function vectorIndexesOf(indexName, deps) {
|
|
729
|
+
const manager = deps.executor.getManager(indexName);
|
|
730
|
+
return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
|
|
731
|
+
}
|
|
732
|
+
async function holdVectorCopies(vectorIndexes, release) {
|
|
733
|
+
for (const vectorIndex of vectorIndexes) {
|
|
734
|
+
if (release.held.includes(vectorIndex)) continue;
|
|
735
|
+
release.held.push(vectorIndex);
|
|
736
|
+
await vectorIndex.holdWorkerCopies();
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
async function releaseCopies(indexName, release, deps) {
|
|
740
|
+
release.reloads = await deps.copies.releaseIndexCopies(indexName);
|
|
741
|
+
await holdVectorCopies(vectorIndexesOf(indexName, deps), release);
|
|
742
|
+
}
|
|
743
|
+
async function endRelease(indexName, release, deps) {
|
|
744
|
+
await release.settled;
|
|
745
|
+
for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
|
|
746
|
+
await deps.copies.endCopyRelease(indexName, release.reloads);
|
|
747
|
+
}
|
|
748
|
+
function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
749
|
+
const durable = deps.durability?.manager;
|
|
750
|
+
if (durable === void 0) return [];
|
|
751
|
+
return Array.from({ length: partitionCount }, (_, partitionId) => ({
|
|
752
|
+
partitionId,
|
|
753
|
+
lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
|
|
754
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
755
|
+
}));
|
|
756
|
+
}
|
|
757
|
+
async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
758
|
+
const durable = deps.durability?.manager;
|
|
759
|
+
const staging = restored.staging;
|
|
760
|
+
if (durable === void 0 || staging === null) return;
|
|
761
|
+
const previous = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
762
|
+
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
763
|
+
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
764
|
+
try {
|
|
765
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
766
|
+
await staging.commit();
|
|
767
|
+
} catch (error) {
|
|
768
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
|
|
769
|
+
throw error;
|
|
544
770
|
}
|
|
771
|
+
await staging.finish();
|
|
772
|
+
const relocate = await staging.relocation().catch(() => null);
|
|
773
|
+
if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
774
|
+
}
|
|
775
|
+
function activate(indexName, restored, entry, deps) {
|
|
776
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
777
|
+
deps.indexRegistry.set(indexName, entry);
|
|
778
|
+
restored.detached.attach();
|
|
545
779
|
deps.clearAnalysisStale(indexName);
|
|
780
|
+
const { prepared } = restored;
|
|
546
781
|
if (prepared.storedRevision !== prepared.language.revision) {
|
|
547
782
|
deps.markAnalysisStale({
|
|
548
783
|
indexName,
|
|
@@ -551,42 +786,68 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
551
786
|
currentRevision: prepared.language.revision
|
|
552
787
|
});
|
|
553
788
|
}
|
|
554
|
-
deps.
|
|
555
|
-
deps.
|
|
556
|
-
config: prepared.config,
|
|
557
|
-
language: prepared.language,
|
|
558
|
-
embeddingAdapter: prepared.embeddingAdapter,
|
|
559
|
-
embeddingAdapterName: prepared.adapterName,
|
|
560
|
-
vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
|
|
561
|
-
indexUuid: null,
|
|
562
|
-
heldPartitions: null,
|
|
563
|
-
documentCount: 0,
|
|
564
|
-
partitionCount,
|
|
565
|
-
analysisRevision: prepared.storedRevision
|
|
566
|
-
});
|
|
567
|
-
await deps.indexState.registerOpen(indexName);
|
|
568
|
-
const release = await deps.indexState.acquire(indexName, false);
|
|
569
|
-
const manager = deps.requireManager(indexName);
|
|
570
|
-
while (manager.partitionCount < partitionCount) manager.addPartition();
|
|
571
|
-
return { manager, release };
|
|
789
|
+
if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
|
|
790
|
+
deps.forgetWatermark(indexName);
|
|
572
791
|
}
|
|
573
|
-
async function
|
|
792
|
+
async function replaceExisting(indexName, restored, entry, deps) {
|
|
793
|
+
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
574
794
|
try {
|
|
575
|
-
|
|
576
|
-
|
|
795
|
+
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
796
|
+
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
797
|
+
await deps.abandonAnalysisRebuild(indexName);
|
|
798
|
+
const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
|
|
799
|
+
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf(indexName, deps);
|
|
800
|
+
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
801
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
802
|
+
await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
|
|
803
|
+
activate(indexName, restored, entry, deps);
|
|
804
|
+
});
|
|
805
|
+
} finally {
|
|
806
|
+
const ending = endRelease(indexName, release, deps).catch(() => void 0);
|
|
807
|
+
if (!release.timedOut) await ending;
|
|
577
808
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
809
|
+
}
|
|
810
|
+
async function createFresh(indexName, restored, entry, deps) {
|
|
811
|
+
deps.copies.desyncIndex(indexName);
|
|
812
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
813
|
+
deps.indexRegistry.set(indexName, entry);
|
|
814
|
+
try {
|
|
815
|
+
await deps.indexState.registerReplacing(indexName, async () => {
|
|
816
|
+
await commitToStorage(indexName, restored, 0, deps);
|
|
817
|
+
activate(indexName, restored, entry, deps);
|
|
818
|
+
});
|
|
819
|
+
} catch (error) {
|
|
820
|
+
if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
|
|
821
|
+
throw error;
|
|
822
|
+
} finally {
|
|
823
|
+
await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
|
|
582
824
|
}
|
|
583
825
|
}
|
|
584
|
-
function
|
|
585
|
-
const
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
826
|
+
async function stageMetadata(indexName, staging, entry) {
|
|
827
|
+
const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
|
|
828
|
+
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
829
|
+
}
|
|
830
|
+
async function installRestoredIndex(indexName, restored, deps) {
|
|
831
|
+
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
832
|
+
let replaced;
|
|
833
|
+
try {
|
|
834
|
+
if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
|
|
835
|
+
replaced = deps.indexRegistry.get(indexName);
|
|
836
|
+
if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
|
|
837
|
+
else await createFresh(indexName, restored, entry, deps);
|
|
838
|
+
} catch (error) {
|
|
839
|
+
if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
|
|
840
|
+
else deps.executor.discardDetachedIndex(restored.detached);
|
|
841
|
+
throw error;
|
|
842
|
+
}
|
|
843
|
+
await restored.staging?.removeRetired().catch(() => void 0);
|
|
844
|
+
const durable = deps.durability?.manager;
|
|
845
|
+
if (restored.staging === null && durable !== void 0) {
|
|
846
|
+
await durable.persistMetadata(indexName);
|
|
847
|
+
await durable.checkpoint(indexName);
|
|
589
848
|
}
|
|
849
|
+
if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
|
|
850
|
+
await deps.announceIndexCreated(indexName, restored.prepared.config);
|
|
590
851
|
}
|
|
591
852
|
|
|
592
853
|
// src/engine/snapshot/archive-restore.ts
|
|
@@ -637,6 +898,9 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
637
898
|
manifest: async (manifest2, sourceName) => {
|
|
638
899
|
if (staged.prepared === null) missingPart("configuration");
|
|
639
900
|
requireSameSchema(staged.prepared, manifest2);
|
|
901
|
+
if (deps.indexRegistry.has(indexName)) {
|
|
902
|
+
requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
|
|
903
|
+
}
|
|
640
904
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
641
905
|
},
|
|
642
906
|
file: async (entryName, _ref, bytes) => {
|
|
@@ -650,44 +914,30 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
650
914
|
await validateStagedCheckpoint(staging, manifest);
|
|
651
915
|
return { prepared, manifest };
|
|
652
916
|
}
|
|
653
|
-
async function
|
|
917
|
+
async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
654
918
|
const { prepared, manifest } = staged;
|
|
655
|
-
|
|
656
|
-
const
|
|
919
|
+
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
920
|
+
const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
|
|
657
921
|
try {
|
|
658
|
-
if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
|
|
659
|
-
const store = staging === null ? memory : await staging.commit();
|
|
660
|
-
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
661
922
|
await loadSegmentedCheckpoint(
|
|
662
923
|
store,
|
|
663
924
|
indexName,
|
|
664
925
|
manifest,
|
|
665
|
-
{ manager:
|
|
926
|
+
{ manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
|
|
666
927
|
true
|
|
667
928
|
);
|
|
668
|
-
recordRestoredCounts(indexName, created.manager, deps);
|
|
669
|
-
if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
|
|
670
|
-
durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
|
|
671
|
-
await durable.persistMetadata(indexName);
|
|
672
|
-
} else if (durable !== void 0) {
|
|
673
|
-
await durable.persistMetadata(indexName);
|
|
674
|
-
await durable.checkpoint(indexName);
|
|
675
|
-
}
|
|
676
|
-
await deps.announceIndexCreated(indexName, prepared.config);
|
|
677
|
-
return created;
|
|
678
929
|
} catch (error) {
|
|
679
|
-
|
|
680
|
-
await abandonRestoredIndex(indexName, deps);
|
|
930
|
+
deps.executor.discardDetachedIndex(detached);
|
|
681
931
|
throw error;
|
|
682
932
|
}
|
|
933
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
683
934
|
}
|
|
684
935
|
async function restoreArchive(indexName, reader, deps) {
|
|
685
936
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
686
|
-
const
|
|
937
|
+
const store = staging?.store ?? memoryCheckpointStore();
|
|
687
938
|
try {
|
|
688
|
-
const staged = await stageArchive(indexName, reader,
|
|
689
|
-
|
|
690
|
-
created.release();
|
|
939
|
+
const staged = await stageArchive(indexName, reader, store, deps);
|
|
940
|
+
await installRestoredIndex(indexName, await loadStagedIndex(indexName, staged, staging, store, deps), deps);
|
|
691
941
|
} finally {
|
|
692
942
|
await staging?.discard().catch(() => void 0);
|
|
693
943
|
await reader.close();
|
|
@@ -740,16 +990,20 @@ function vectorsOfSnapshot(envelope) {
|
|
|
740
990
|
}
|
|
741
991
|
return byDocument;
|
|
742
992
|
}
|
|
993
|
+
function footprintOf(envelope, prepared) {
|
|
994
|
+
let partitionBytes = 0;
|
|
995
|
+
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
996
|
+
const vectorCounts = /* @__PURE__ */ new Map();
|
|
997
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
998
|
+
let count = 0;
|
|
999
|
+
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
1000
|
+
vectorCounts.set(fieldPath, count);
|
|
1001
|
+
}
|
|
1002
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
|
|
1003
|
+
}
|
|
743
1004
|
function documentsOfPartition(envelope, payload) {
|
|
744
1005
|
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
745
1006
|
}
|
|
746
|
-
function withVectors(fields, vectors) {
|
|
747
|
-
if (vectors === void 0 || vectors.size === 0) return fields;
|
|
748
|
-
const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
|
|
749
|
-
const document = nested ? structuredClone(fields) : { ...fields };
|
|
750
|
-
for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
|
|
751
|
-
return document;
|
|
752
|
-
}
|
|
753
1007
|
function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
754
1008
|
manager.insert(docId, fields, { skipClone: true });
|
|
755
1009
|
if (vectors === void 0) return;
|
|
@@ -760,51 +1014,62 @@ function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
|
760
1014
|
throw err;
|
|
761
1015
|
}
|
|
762
1016
|
}
|
|
763
|
-
|
|
764
|
-
if (deps.durability === null) {
|
|
765
|
-
for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
|
|
766
|
-
return;
|
|
767
|
-
}
|
|
768
|
-
const mutations = chunk.map(([docId, fields]) => ({
|
|
769
|
-
docId,
|
|
770
|
-
document: withVectors(fields, vectors.get(docId)),
|
|
771
|
-
apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
|
|
772
|
-
}));
|
|
773
|
-
for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
|
|
774
|
-
if (!outcome.ok) throw outcome.error;
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
|
|
1017
|
+
function insertSnapshotDocuments(envelope, manager) {
|
|
778
1018
|
const vectors = vectorsOfSnapshot(envelope);
|
|
779
1019
|
for (const payload of envelope.partitions ?? []) {
|
|
780
|
-
const
|
|
781
|
-
(
|
|
782
|
-
);
|
|
783
|
-
for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
|
|
784
|
-
await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
|
|
1020
|
+
for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
|
|
1021
|
+
insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
|
|
785
1022
|
}
|
|
786
1023
|
}
|
|
787
1024
|
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
788
1025
|
}
|
|
1026
|
+
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
1027
|
+
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
1028
|
+
const archive = await buildArchiveFromMemory({
|
|
1029
|
+
indexName,
|
|
1030
|
+
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
1031
|
+
manager: built.manager,
|
|
1032
|
+
vectorIndexes: built.manager.getVectorIndexes(),
|
|
1033
|
+
appliedSeqNo: () => 0,
|
|
1034
|
+
partitionId: null
|
|
1035
|
+
});
|
|
1036
|
+
try {
|
|
1037
|
+
for (const ref of manifestFileRefs(archive.manifest))
|
|
1038
|
+
await staging.store.write(ref.key, [await archive.readFile(ref)]);
|
|
1039
|
+
} finally {
|
|
1040
|
+
archive.release();
|
|
1041
|
+
}
|
|
1042
|
+
await validateStagedCheckpoint(staging.store, archive.manifest);
|
|
1043
|
+
return archive.manifest;
|
|
1044
|
+
}
|
|
1045
|
+
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
1046
|
+
const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
|
|
1047
|
+
let manifest;
|
|
1048
|
+
try {
|
|
1049
|
+
insertSnapshotDocuments(envelope, built.manager);
|
|
1050
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
1051
|
+
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
1052
|
+
} catch (err) {
|
|
1053
|
+
deps.executor.discardDetachedIndex(built);
|
|
1054
|
+
throw err;
|
|
1055
|
+
}
|
|
1056
|
+
deps.executor.discardDetachedIndex(built);
|
|
1057
|
+
return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
|
|
1058
|
+
}
|
|
789
1059
|
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
790
1060
|
const envelope = await decodeEarlierSnapshot(data);
|
|
791
1061
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
1062
|
+
if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
|
|
792
1063
|
const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
|
|
793
|
-
const
|
|
794
|
-
const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
|
|
1064
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
795
1065
|
try {
|
|
796
|
-
await
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
}
|
|
802
|
-
await deps.announceIndexCreated(indexName, prepared.config);
|
|
803
|
-
} catch (err) {
|
|
804
|
-
await abandonRestoredIndex(indexName, deps);
|
|
805
|
-
throw err;
|
|
1066
|
+
await installRestoredIndex(
|
|
1067
|
+
indexName,
|
|
1068
|
+
await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
|
|
1069
|
+
deps
|
|
1070
|
+
);
|
|
806
1071
|
} finally {
|
|
807
|
-
|
|
1072
|
+
await staging?.discard().catch(() => void 0);
|
|
808
1073
|
}
|
|
809
1074
|
}
|
|
810
1075
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
4
|
-
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-PP6N5CCS.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-HC6DMR4T.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WKPJ4E73.mjs';
|
|
4
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
7
7
|
// src/engine/export/json-line.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-3kBsQ0Pi.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-Bj9WBlFG.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -284,6 +284,14 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
|
284
284
|
* @public
|
|
285
285
|
*/
|
|
286
286
|
declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
287
|
+
/**
|
|
288
|
+
* Marks a server that reads the synonym set of an index at
|
|
289
|
+
* `GET /indexes/{name}/synonyms` and replaces it at
|
|
290
|
+
* `POST /indexes/{name}/synonyms`.
|
|
291
|
+
*
|
|
292
|
+
* @public
|
|
293
|
+
*/
|
|
294
|
+
declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
287
295
|
/**
|
|
288
296
|
* Every capability this server announces at `/capabilities`.
|
|
289
297
|
*
|
|
@@ -295,4 +303,4 @@ declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
|
295
303
|
*/
|
|
296
304
|
declare const SERVER_CAPABILITIES: readonly string[];
|
|
297
305
|
|
|
298
|
-
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T,
|
|
306
|
+
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
|