@delali/narsil 0.3.1-next.91 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-4JTBL63P.mjs} +4 -4
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-HC6DMR4T.mjs} +238 -114
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-PPTZ565G.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-xsJfdFoQ.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-A4G7JSYO.mjs → chunk-C45FUYPI.mjs} +98 -49
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-67Z2V6TC.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-3TTYK5A7.mjs → chunk-EX62BQT6.mjs} +9 -9
- package/dist/{chunk-3RDS4UHC.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-7OQHSDK6.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-QWWCLRXX.mjs} +6 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-UAUZGXJP.mjs} +40 -21
- package/dist/{chunk-EONIHG3V.mjs → chunk-WBS3EYC4.mjs} +7 -7
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-CgayPw2R.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-3kBsQ0Pi.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-B-RnO6dr.d.ts} +1 -1
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-Y55SCPFC.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-DSUfY2F0.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-wuXcYOD1.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-Bj9WBlFG.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-BMNV7P2H.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint,
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig,
|
|
5
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION,
|
|
6
|
-
import { NarsilError, ErrorCodes, MAX_FILE_BYTES
|
|
1
|
+
import { vectorsPerFile } from './chunk-5S74HQRT.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-4JTBL63P.mjs';
|
|
3
|
+
import './chunk-PP6N5CCS.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, readHeapStatistics, readResidentMemory, prepareStoredSynonyms, rejectSnapshot, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-HC6DMR4T.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
8
8
|
|
|
9
9
|
// src/persistence/archive/memory-store.ts
|
|
@@ -498,6 +498,136 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
498
498
|
}
|
|
499
499
|
}
|
|
500
500
|
|
|
501
|
+
// src/engine/snapshot/constants.ts
|
|
502
|
+
var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
|
|
503
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
|
|
504
|
+
var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
|
|
505
|
+
var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
|
|
506
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
|
|
507
|
+
var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
|
|
508
|
+
var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
|
|
509
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
|
|
510
|
+
var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
|
|
511
|
+
var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
|
|
512
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
|
|
513
|
+
var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
|
|
514
|
+
|
|
515
|
+
// src/engine/snapshot/memory-budget.ts
|
|
516
|
+
var MEGABYTE_BYTES = 1048576;
|
|
517
|
+
var ARCHIVE_RATES = {
|
|
518
|
+
heap: {
|
|
519
|
+
perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
|
|
520
|
+
perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
|
|
521
|
+
perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
522
|
+
},
|
|
523
|
+
external: {
|
|
524
|
+
perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
525
|
+
perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
|
|
526
|
+
perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
527
|
+
}
|
|
528
|
+
};
|
|
529
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
530
|
+
heap: {
|
|
531
|
+
perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
532
|
+
perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
533
|
+
perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
534
|
+
},
|
|
535
|
+
external: {
|
|
536
|
+
perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
537
|
+
perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
538
|
+
perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
542
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
543
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
544
|
+
return bytes;
|
|
545
|
+
}
|
|
546
|
+
function footprintAt(rates, storedBytes, vectors) {
|
|
547
|
+
return {
|
|
548
|
+
heapBytes: bytesAt(rates.heap, storedBytes, vectors),
|
|
549
|
+
externalBytes: bytesAt(rates.external, storedBytes, vectors)
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
function deadCountOf(dead) {
|
|
553
|
+
if (dead === null) return 0;
|
|
554
|
+
let count = 0;
|
|
555
|
+
for (const byte of dead) {
|
|
556
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
557
|
+
}
|
|
558
|
+
return count;
|
|
559
|
+
}
|
|
560
|
+
function archiveFootprint(prepared, manifest) {
|
|
561
|
+
let segmentBytes = 0;
|
|
562
|
+
for (const partition of manifest.partitions) {
|
|
563
|
+
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
564
|
+
}
|
|
565
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
566
|
+
const vectors = [];
|
|
567
|
+
for (const field of manifest.vectors) {
|
|
568
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
569
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
570
|
+
}
|
|
571
|
+
return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
|
|
572
|
+
}
|
|
573
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
|
|
574
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
575
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
576
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
|
|
577
|
+
}
|
|
578
|
+
function inMegabytes(bytes) {
|
|
579
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
580
|
+
}
|
|
581
|
+
function refuse(indexName, budget, neededBytes, availableBytes) {
|
|
582
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
583
|
+
throw new NarsilError(
|
|
584
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
585
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
|
|
586
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
function requireRestoreMemory(indexName, footprint) {
|
|
590
|
+
const heap = readHeapStatistics();
|
|
591
|
+
if (heap !== null) {
|
|
592
|
+
const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
|
|
593
|
+
if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
|
|
594
|
+
}
|
|
595
|
+
const resident = readResidentMemory();
|
|
596
|
+
if (resident !== null) {
|
|
597
|
+
const neededBytes = footprint.heapBytes + footprint.externalBytes;
|
|
598
|
+
const availableBytes = resident.limitBytes - resident.residentBytes;
|
|
599
|
+
if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// src/workers/detached-index.ts
|
|
604
|
+
function attachablePolicy(shared) {
|
|
605
|
+
let attached = false;
|
|
606
|
+
const policy = {
|
|
607
|
+
get enabled() {
|
|
608
|
+
return attached && shared.enabled;
|
|
609
|
+
},
|
|
610
|
+
get count() {
|
|
611
|
+
return shared.count;
|
|
612
|
+
},
|
|
613
|
+
get host() {
|
|
614
|
+
return shared.host;
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
return {
|
|
618
|
+
policy,
|
|
619
|
+
attach() {
|
|
620
|
+
attached = true;
|
|
621
|
+
}
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
625
|
+
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
626
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
627
|
+
entry.manager.synonyms = synonyms;
|
|
628
|
+
return { indexName, manager: entry.manager, entry, attach };
|
|
629
|
+
}
|
|
630
|
+
|
|
501
631
|
// src/engine/snapshot/restore-setup.ts
|
|
502
632
|
function prepareRestoredIndex(configuration, deps, reject) {
|
|
503
633
|
const configFields = restoredConfigFields(configuration, reject);
|
|
@@ -541,23 +671,21 @@ function restoredSynonyms(raw, config, language, reject) {
|
|
|
541
671
|
if (raw === void 0 || raw === null) return null;
|
|
542
672
|
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
543
673
|
}
|
|
544
|
-
async function
|
|
674
|
+
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
545
675
|
const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
|
|
546
676
|
if (loadingVectorCode !== null) await loadingVectorCode;
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
deps.
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
deps.executor.createIndex(indexName, prepared.config, prepared.language, false, prepared.synonyms?.lookup ?? null);
|
|
560
|
-
deps.indexRegistry.set(indexName, {
|
|
677
|
+
const detached = createDetachedIndex(
|
|
678
|
+
indexName,
|
|
679
|
+
prepared.config,
|
|
680
|
+
prepared.language,
|
|
681
|
+
prepared.synonyms?.lookup ?? null,
|
|
682
|
+
deps.executor
|
|
683
|
+
);
|
|
684
|
+
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
685
|
+
return detached;
|
|
686
|
+
}
|
|
687
|
+
function restoredRegistryEntry(prepared, manager, deps) {
|
|
688
|
+
return {
|
|
561
689
|
config: prepared.config,
|
|
562
690
|
language: prepared.language,
|
|
563
691
|
embeddingAdapter: prepared.embeddingAdapter,
|
|
@@ -565,34 +693,161 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
565
693
|
vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
|
|
566
694
|
indexUuid: null,
|
|
567
695
|
heldPartitions: null,
|
|
568
|
-
documentCount:
|
|
569
|
-
partitionCount,
|
|
696
|
+
documentCount: manager.countDocuments(),
|
|
697
|
+
partitionCount: manager.partitionCount,
|
|
570
698
|
analysisRevision: prepared.storedRevision,
|
|
571
699
|
synonyms: prepared.synonyms?.encoded ?? null
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
// src/engine/snapshot/replace.ts
|
|
704
|
+
function releaseTimedOut(indexName, timeoutMs) {
|
|
705
|
+
return new NarsilError(
|
|
706
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
707
|
+
`The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
|
|
708
|
+
{ indexName, timeoutMs }
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
async function withinLimit(indexName, release, work, deps) {
|
|
712
|
+
release.settled = work.catch(() => {
|
|
713
|
+
release.reloads = true;
|
|
714
|
+
});
|
|
715
|
+
let timer;
|
|
716
|
+
const limit = new Promise((_, reject) => {
|
|
717
|
+
timer = setTimeout(() => {
|
|
718
|
+
release.timedOut = true;
|
|
719
|
+
reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
|
|
720
|
+
}, deps.releaseTimeoutMs);
|
|
572
721
|
});
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
722
|
+
try {
|
|
723
|
+
await Promise.race([work, limit]);
|
|
724
|
+
} finally {
|
|
725
|
+
clearTimeout(timer);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
function vectorIndexesOf(indexName, deps) {
|
|
729
|
+
const manager = deps.executor.getManager(indexName);
|
|
730
|
+
return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
|
|
731
|
+
}
|
|
732
|
+
async function holdVectorCopies(vectorIndexes, release) {
|
|
733
|
+
for (const vectorIndex of vectorIndexes) {
|
|
734
|
+
if (release.held.includes(vectorIndex)) continue;
|
|
735
|
+
release.held.push(vectorIndex);
|
|
736
|
+
await vectorIndex.holdWorkerCopies();
|
|
737
|
+
}
|
|
578
738
|
}
|
|
579
|
-
async function
|
|
739
|
+
async function releaseCopies(indexName, release, deps) {
|
|
740
|
+
release.reloads = await deps.copies.releaseIndexCopies(indexName);
|
|
741
|
+
await holdVectorCopies(vectorIndexesOf(indexName, deps), release);
|
|
742
|
+
}
|
|
743
|
+
async function endRelease(indexName, release, deps) {
|
|
744
|
+
await release.settled;
|
|
745
|
+
for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
|
|
746
|
+
await deps.copies.endCopyRelease(indexName, release.reloads);
|
|
747
|
+
}
|
|
748
|
+
function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
749
|
+
const durable = deps.durability?.manager;
|
|
750
|
+
if (durable === void 0) return [];
|
|
751
|
+
return Array.from({ length: partitionCount }, (_, partitionId) => ({
|
|
752
|
+
partitionId,
|
|
753
|
+
lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
|
|
754
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
755
|
+
}));
|
|
756
|
+
}
|
|
757
|
+
async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
758
|
+
const durable = deps.durability?.manager;
|
|
759
|
+
const staging = restored.staging;
|
|
760
|
+
if (durable === void 0 || staging === null) return;
|
|
761
|
+
const previous = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
762
|
+
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
763
|
+
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
764
|
+
try {
|
|
765
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
766
|
+
await staging.commit();
|
|
767
|
+
} catch (error) {
|
|
768
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
|
|
769
|
+
throw error;
|
|
770
|
+
}
|
|
771
|
+
await staging.finish();
|
|
772
|
+
const relocate = await staging.relocation().catch(() => null);
|
|
773
|
+
if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
774
|
+
}
|
|
775
|
+
function activate(indexName, restored, entry, deps) {
|
|
776
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
777
|
+
deps.indexRegistry.set(indexName, entry);
|
|
778
|
+
restored.detached.attach();
|
|
779
|
+
deps.clearAnalysisStale(indexName);
|
|
780
|
+
const { prepared } = restored;
|
|
781
|
+
if (prepared.storedRevision !== prepared.language.revision) {
|
|
782
|
+
deps.markAnalysisStale({
|
|
783
|
+
indexName,
|
|
784
|
+
language: prepared.language.name,
|
|
785
|
+
storedRevision: prepared.storedRevision,
|
|
786
|
+
currentRevision: prepared.language.revision
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
|
|
790
|
+
deps.forgetWatermark(indexName);
|
|
791
|
+
}
|
|
792
|
+
async function replaceExisting(indexName, restored, entry, deps) {
|
|
793
|
+
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
580
794
|
try {
|
|
581
|
-
|
|
582
|
-
|
|
795
|
+
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
796
|
+
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
797
|
+
await deps.abandonAnalysisRebuild(indexName);
|
|
798
|
+
const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
|
|
799
|
+
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf(indexName, deps);
|
|
800
|
+
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
801
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
802
|
+
await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
|
|
803
|
+
activate(indexName, restored, entry, deps);
|
|
804
|
+
});
|
|
805
|
+
} finally {
|
|
806
|
+
const ending = endRelease(indexName, release, deps).catch(() => void 0);
|
|
807
|
+
if (!release.timedOut) await ending;
|
|
583
808
|
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
809
|
+
}
|
|
810
|
+
async function createFresh(indexName, restored, entry, deps) {
|
|
811
|
+
deps.copies.desyncIndex(indexName);
|
|
812
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
813
|
+
deps.indexRegistry.set(indexName, entry);
|
|
814
|
+
try {
|
|
815
|
+
await deps.indexState.registerReplacing(indexName, async () => {
|
|
816
|
+
await commitToStorage(indexName, restored, 0, deps);
|
|
817
|
+
activate(indexName, restored, entry, deps);
|
|
818
|
+
});
|
|
819
|
+
} catch (error) {
|
|
820
|
+
if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
|
|
821
|
+
throw error;
|
|
822
|
+
} finally {
|
|
823
|
+
await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
|
|
588
824
|
}
|
|
589
825
|
}
|
|
590
|
-
function
|
|
591
|
-
const
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
826
|
+
async function stageMetadata(indexName, staging, entry) {
|
|
827
|
+
const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
|
|
828
|
+
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
829
|
+
}
|
|
830
|
+
async function installRestoredIndex(indexName, restored, deps) {
|
|
831
|
+
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
832
|
+
let replaced;
|
|
833
|
+
try {
|
|
834
|
+
if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
|
|
835
|
+
replaced = deps.indexRegistry.get(indexName);
|
|
836
|
+
if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
|
|
837
|
+
else await createFresh(indexName, restored, entry, deps);
|
|
838
|
+
} catch (error) {
|
|
839
|
+
if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
|
|
840
|
+
else deps.executor.discardDetachedIndex(restored.detached);
|
|
841
|
+
throw error;
|
|
842
|
+
}
|
|
843
|
+
await restored.staging?.removeRetired().catch(() => void 0);
|
|
844
|
+
const durable = deps.durability?.manager;
|
|
845
|
+
if (restored.staging === null && durable !== void 0) {
|
|
846
|
+
await durable.persistMetadata(indexName);
|
|
847
|
+
await durable.checkpoint(indexName);
|
|
595
848
|
}
|
|
849
|
+
if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
|
|
850
|
+
await deps.announceIndexCreated(indexName, restored.prepared.config);
|
|
596
851
|
}
|
|
597
852
|
|
|
598
853
|
// src/engine/snapshot/archive-restore.ts
|
|
@@ -643,6 +898,9 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
643
898
|
manifest: async (manifest2, sourceName) => {
|
|
644
899
|
if (staged.prepared === null) missingPart("configuration");
|
|
645
900
|
requireSameSchema(staged.prepared, manifest2);
|
|
901
|
+
if (deps.indexRegistry.has(indexName)) {
|
|
902
|
+
requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
|
|
903
|
+
}
|
|
646
904
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
647
905
|
},
|
|
648
906
|
file: async (entryName, _ref, bytes) => {
|
|
@@ -656,44 +914,30 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
656
914
|
await validateStagedCheckpoint(staging, manifest);
|
|
657
915
|
return { prepared, manifest };
|
|
658
916
|
}
|
|
659
|
-
async function
|
|
917
|
+
async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
660
918
|
const { prepared, manifest } = staged;
|
|
661
|
-
|
|
662
|
-
const
|
|
919
|
+
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
920
|
+
const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
|
|
663
921
|
try {
|
|
664
|
-
if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
|
|
665
|
-
const store = staging === null ? memory : await staging.commit();
|
|
666
|
-
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
667
922
|
await loadSegmentedCheckpoint(
|
|
668
923
|
store,
|
|
669
924
|
indexName,
|
|
670
925
|
manifest,
|
|
671
|
-
{ manager:
|
|
926
|
+
{ manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
|
|
672
927
|
true
|
|
673
928
|
);
|
|
674
|
-
recordRestoredCounts(indexName, created.manager, deps);
|
|
675
|
-
if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
|
|
676
|
-
durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
|
|
677
|
-
await durable.persistMetadata(indexName);
|
|
678
|
-
} else if (durable !== void 0) {
|
|
679
|
-
await durable.persistMetadata(indexName);
|
|
680
|
-
await durable.checkpoint(indexName);
|
|
681
|
-
}
|
|
682
|
-
await deps.announceIndexCreated(indexName, prepared.config);
|
|
683
|
-
return created;
|
|
684
929
|
} catch (error) {
|
|
685
|
-
|
|
686
|
-
await abandonRestoredIndex(indexName, deps);
|
|
930
|
+
deps.executor.discardDetachedIndex(detached);
|
|
687
931
|
throw error;
|
|
688
932
|
}
|
|
933
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
689
934
|
}
|
|
690
935
|
async function restoreArchive(indexName, reader, deps) {
|
|
691
936
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
692
|
-
const
|
|
937
|
+
const store = staging?.store ?? memoryCheckpointStore();
|
|
693
938
|
try {
|
|
694
|
-
const staged = await stageArchive(indexName, reader,
|
|
695
|
-
|
|
696
|
-
created.release();
|
|
939
|
+
const staged = await stageArchive(indexName, reader, store, deps);
|
|
940
|
+
await installRestoredIndex(indexName, await loadStagedIndex(indexName, staged, staging, store, deps), deps);
|
|
697
941
|
} finally {
|
|
698
942
|
await staging?.discard().catch(() => void 0);
|
|
699
943
|
await reader.close();
|
|
@@ -746,16 +990,20 @@ function vectorsOfSnapshot(envelope) {
|
|
|
746
990
|
}
|
|
747
991
|
return byDocument;
|
|
748
992
|
}
|
|
993
|
+
function footprintOf(envelope, prepared) {
|
|
994
|
+
let partitionBytes = 0;
|
|
995
|
+
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
996
|
+
const vectorCounts = /* @__PURE__ */ new Map();
|
|
997
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
998
|
+
let count = 0;
|
|
999
|
+
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
1000
|
+
vectorCounts.set(fieldPath, count);
|
|
1001
|
+
}
|
|
1002
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
|
|
1003
|
+
}
|
|
749
1004
|
function documentsOfPartition(envelope, payload) {
|
|
750
1005
|
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
751
1006
|
}
|
|
752
|
-
function withVectors(fields, vectors) {
|
|
753
|
-
if (vectors === void 0 || vectors.size === 0) return fields;
|
|
754
|
-
const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
|
|
755
|
-
const document = nested ? structuredClone(fields) : { ...fields };
|
|
756
|
-
for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
|
|
757
|
-
return document;
|
|
758
|
-
}
|
|
759
1007
|
function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
760
1008
|
manager.insert(docId, fields, { skipClone: true });
|
|
761
1009
|
if (vectors === void 0) return;
|
|
@@ -766,51 +1014,62 @@ function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
|
766
1014
|
throw err;
|
|
767
1015
|
}
|
|
768
1016
|
}
|
|
769
|
-
|
|
770
|
-
if (deps.durability === null) {
|
|
771
|
-
for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
|
|
772
|
-
return;
|
|
773
|
-
}
|
|
774
|
-
const mutations = chunk.map(([docId, fields]) => ({
|
|
775
|
-
docId,
|
|
776
|
-
document: withVectors(fields, vectors.get(docId)),
|
|
777
|
-
apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
|
|
778
|
-
}));
|
|
779
|
-
for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
|
|
780
|
-
if (!outcome.ok) throw outcome.error;
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
|
|
1017
|
+
function insertSnapshotDocuments(envelope, manager) {
|
|
784
1018
|
const vectors = vectorsOfSnapshot(envelope);
|
|
785
1019
|
for (const payload of envelope.partitions ?? []) {
|
|
786
|
-
const
|
|
787
|
-
(
|
|
788
|
-
);
|
|
789
|
-
for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
|
|
790
|
-
await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
|
|
1020
|
+
for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
|
|
1021
|
+
insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
|
|
791
1022
|
}
|
|
792
1023
|
}
|
|
793
1024
|
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
794
1025
|
}
|
|
1026
|
+
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
1027
|
+
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
1028
|
+
const archive = await buildArchiveFromMemory({
|
|
1029
|
+
indexName,
|
|
1030
|
+
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
1031
|
+
manager: built.manager,
|
|
1032
|
+
vectorIndexes: built.manager.getVectorIndexes(),
|
|
1033
|
+
appliedSeqNo: () => 0,
|
|
1034
|
+
partitionId: null
|
|
1035
|
+
});
|
|
1036
|
+
try {
|
|
1037
|
+
for (const ref of manifestFileRefs(archive.manifest))
|
|
1038
|
+
await staging.store.write(ref.key, [await archive.readFile(ref)]);
|
|
1039
|
+
} finally {
|
|
1040
|
+
archive.release();
|
|
1041
|
+
}
|
|
1042
|
+
await validateStagedCheckpoint(staging.store, archive.manifest);
|
|
1043
|
+
return archive.manifest;
|
|
1044
|
+
}
|
|
1045
|
+
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
1046
|
+
const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
|
|
1047
|
+
let manifest;
|
|
1048
|
+
try {
|
|
1049
|
+
insertSnapshotDocuments(envelope, built.manager);
|
|
1050
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
1051
|
+
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
1052
|
+
} catch (err) {
|
|
1053
|
+
deps.executor.discardDetachedIndex(built);
|
|
1054
|
+
throw err;
|
|
1055
|
+
}
|
|
1056
|
+
deps.executor.discardDetachedIndex(built);
|
|
1057
|
+
return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
|
|
1058
|
+
}
|
|
795
1059
|
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
796
1060
|
const envelope = await decodeEarlierSnapshot(data);
|
|
797
1061
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
1062
|
+
if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
|
|
798
1063
|
const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
|
|
799
|
-
const
|
|
800
|
-
const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
|
|
1064
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
801
1065
|
try {
|
|
802
|
-
await
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
}
|
|
808
|
-
await deps.announceIndexCreated(indexName, prepared.config);
|
|
809
|
-
} catch (err) {
|
|
810
|
-
await abandonRestoredIndex(indexName, deps);
|
|
811
|
-
throw err;
|
|
1066
|
+
await installRestoredIndex(
|
|
1067
|
+
indexName,
|
|
1068
|
+
await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
|
|
1069
|
+
deps
|
|
1070
|
+
);
|
|
812
1071
|
} finally {
|
|
813
|
-
|
|
1072
|
+
await staging?.discard().catch(() => void 0);
|
|
814
1073
|
}
|
|
815
1074
|
}
|
|
816
1075
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
4
|
-
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-PP6N5CCS.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-HC6DMR4T.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WKPJ4E73.mjs';
|
|
4
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
7
7
|
// src/engine/export/json-line.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-3kBsQ0Pi.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-Bj9WBlFG.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-7KC56574.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
7
7
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
|
|
2
|
-
import { isWordIndexedBase, parseFieldType } from './chunk-
|
|
2
|
+
import { isWordIndexedBase, parseFieldType } from './chunk-BB7MASRQ.mjs';
|
|
3
3
|
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
4
|
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/schema/validator/schema.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { createNarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/tokenizer/constants.ts
|
|
5
5
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|