@delali/narsil 0.3.1-next.180 → 0.3.1-next.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
- package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
- package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
- package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
- package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
- package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
- package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
- package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
- package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
- package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
- package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
- package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
- package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
- package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +391 -138
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +1 -1
- package/dist/index.react-native.mjs +1 -1
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
- package/dist/query-suggestions/merge-worker.mjs +4 -4
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +2 -2
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
- package/dist/react-native/partial-words.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +1 -1
- package/dist/react-native/sync.mjs +1 -1
- package/dist/react-native/synonyms.mjs +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +16 -16
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +1 -1
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +1 -1
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/openapi.json +32 -0
- package/package.json +7 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
2
|
+
import { C as ClusterNamespace } from './config-D9WR3MhO.js';
|
|
3
3
|
import { l as RecordedQuery } from './query-suggestions-CBYkrQ5T.js';
|
|
4
|
-
import { E as ErrorDetail, j as TaskStore } from './tasks-
|
|
4
|
+
import { E as ErrorDetail, j as TaskStore } from './tasks-X2IJQc8J.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-
|
|
1
|
+
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/frozen/share.ts
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION,
|
|
1
|
+
import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, resumePagedFile, suspendPagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-TTTBE4TB.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
4
|
-
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-
|
|
5
|
-
import { DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-
|
|
4
|
+
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-PK3BGGHO.mjs';
|
|
5
|
+
import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-MSIN4NM3.mjs';
|
|
6
6
|
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
7
7
|
import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
|
|
8
|
-
import {
|
|
9
|
-
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-
|
|
8
|
+
import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
|
|
9
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-CWXR7OP4.mjs';
|
|
10
10
|
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
11
11
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
12
12
|
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-X5NHWOXT.mjs';
|
|
@@ -3575,6 +3575,9 @@ async function mergePagedWindow(input) {
|
|
|
3575
3575
|
for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
|
|
3576
3576
|
}
|
|
3577
3577
|
}
|
|
3578
|
+
for (const docId of input.replacedAfterWindow ?? []) {
|
|
3579
|
+
for (const source of sources) source.segment.tombstoneDocument(docId);
|
|
3580
|
+
}
|
|
3578
3581
|
const tombstones = /* @__PURE__ */ new Set();
|
|
3579
3582
|
if (!input.dropsEveryTombstone) {
|
|
3580
3583
|
for (const source of sources) {
|
|
@@ -3605,6 +3608,61 @@ async function mergePagedWindow(input) {
|
|
|
3605
3608
|
}
|
|
3606
3609
|
}
|
|
3607
3610
|
|
|
3611
|
+
// src/persistence/paged/segment-scan.ts
|
|
3612
|
+
async function openPagedSegmentScan(directory, ref) {
|
|
3613
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3614
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
3615
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
3616
|
+
const { paged } = segment;
|
|
3617
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
3618
|
+
const ids = segment.arrays.idTable;
|
|
3619
|
+
const tombstones = new Set(snapshot.tombstones);
|
|
3620
|
+
return {
|
|
3621
|
+
count: snapshot.documentCount,
|
|
3622
|
+
schema: snapshot.schema,
|
|
3623
|
+
tombstones,
|
|
3624
|
+
idAt: (ordinal) => ids.idAt(ordinal),
|
|
3625
|
+
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
3626
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
3627
|
+
close: () => {
|
|
3628
|
+
paged.close();
|
|
3629
|
+
cache.clear();
|
|
3630
|
+
}
|
|
3631
|
+
};
|
|
3632
|
+
}
|
|
3633
|
+
async function checkPagedSegmentFully(directory, ref) {
|
|
3634
|
+
const listing = await pagedListingOf(directory, ref);
|
|
3635
|
+
await requirePagedPayloadChecksum(directory, listing);
|
|
3636
|
+
const snapshot = openPagedSnapshot(listing);
|
|
3637
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
3638
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
3639
|
+
const { paged } = segment;
|
|
3640
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
3641
|
+
try {
|
|
3642
|
+
await checkPagedContents(segment, cache);
|
|
3643
|
+
return snapshot.schema;
|
|
3644
|
+
} finally {
|
|
3645
|
+
paged.close();
|
|
3646
|
+
cache.clear();
|
|
3647
|
+
}
|
|
3648
|
+
}
|
|
3649
|
+
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
3650
|
+
let bytes = 0;
|
|
3651
|
+
for (const partition of manifest.partitions) {
|
|
3652
|
+
for (const ref of partition.segments) {
|
|
3653
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
3654
|
+
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
3655
|
+
}
|
|
3656
|
+
}
|
|
3657
|
+
return bytes;
|
|
3658
|
+
}
|
|
3659
|
+
function countsAsLatestHolder(scans, at, docId) {
|
|
3660
|
+
for (let later = at + 1; later < scans.length; later++) {
|
|
3661
|
+
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
3662
|
+
}
|
|
3663
|
+
return true;
|
|
3664
|
+
}
|
|
3665
|
+
|
|
3608
3666
|
// src/persistence/durability/partition-checkpoint.ts
|
|
3609
3667
|
function checkpointLastSeqNo(checkpoint, partitionId) {
|
|
3610
3668
|
for (const c of checkpoint) {
|
|
@@ -3623,6 +3681,25 @@ function checkpointPrimaryTerm(checkpoint, partitionId) {
|
|
|
3623
3681
|
return 0;
|
|
3624
3682
|
}
|
|
3625
3683
|
|
|
3684
|
+
// src/persistence/durability/stored-bytes.ts
|
|
3685
|
+
var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
|
|
3686
|
+
function savedBytesOfManifest(manifest) {
|
|
3687
|
+
let bytes = 0;
|
|
3688
|
+
const partitions = [];
|
|
3689
|
+
for (const partition of manifest.partitions) {
|
|
3690
|
+
let partitionBytes = 0;
|
|
3691
|
+
for (const segment of partition.segments) partitionBytes += segment.bytes;
|
|
3692
|
+
partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
|
|
3693
|
+
bytes += partitionBytes;
|
|
3694
|
+
}
|
|
3695
|
+
let vectorBytes = 0;
|
|
3696
|
+
for (const field of manifest.vectors) {
|
|
3697
|
+
for (const file of field.files) vectorBytes += file.bytes;
|
|
3698
|
+
for (const part of field.graphParts) vectorBytes += part.bytes;
|
|
3699
|
+
}
|
|
3700
|
+
return { bytes: bytes + vectorBytes, vectorBytes, partitions };
|
|
3701
|
+
}
|
|
3702
|
+
|
|
3626
3703
|
// src/persistence/durability/commit-marker.ts
|
|
3627
3704
|
var COMMIT_MARKER_SLOT_SIZE = 36;
|
|
3628
3705
|
var COMMIT_MARKER_SLOT_COUNT = 2;
|
|
@@ -4457,49 +4534,34 @@ function retainedTombstones(frozen, contents) {
|
|
|
4457
4534
|
}
|
|
4458
4535
|
return [...retained];
|
|
4459
4536
|
}
|
|
4460
|
-
async function
|
|
4461
|
-
|
|
4462
|
-
if (segments.length <= compactionThreshold) {
|
|
4463
|
-
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4464
|
-
}
|
|
4465
|
-
const paged = input.paged === true;
|
|
4466
|
-
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
4467
|
-
if (window === null) {
|
|
4468
|
-
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4469
|
-
}
|
|
4470
|
-
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
4471
|
-
const replacing = (written2) => ({
|
|
4472
|
-
segments: [
|
|
4473
|
-
...segments.slice(0, window.start),
|
|
4474
|
-
...written2.segments,
|
|
4475
|
-
...segments.slice(window.start + window.windowLength)
|
|
4476
|
-
],
|
|
4477
|
-
nextSegmentId: written2.nextSegmentId
|
|
4478
|
-
});
|
|
4479
|
-
if (paged) {
|
|
4537
|
+
async function mergeSegmentWindow(input, window) {
|
|
4538
|
+
if (input.paged === true) {
|
|
4480
4539
|
const { directory } = input.store;
|
|
4481
|
-
if (directory === void 0) return
|
|
4482
|
-
return
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
);
|
|
4540
|
+
if (directory === void 0) return null;
|
|
4541
|
+
return mergePagedWindow({
|
|
4542
|
+
store: input.store,
|
|
4543
|
+
directory,
|
|
4544
|
+
indexName: input.indexName,
|
|
4545
|
+
partitionId: input.partitionId,
|
|
4546
|
+
schema: input.schema,
|
|
4547
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
4548
|
+
textAnalysis: input.textAnalysis,
|
|
4549
|
+
nextSegmentId: input.nextSegmentId,
|
|
4550
|
+
refs: window.refs,
|
|
4551
|
+
dropsEveryTombstone: window.startsTheList,
|
|
4552
|
+
...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
|
|
4553
|
+
});
|
|
4496
4554
|
}
|
|
4497
4555
|
const contents = [];
|
|
4498
|
-
for (const ref of
|
|
4556
|
+
for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
|
|
4499
4557
|
const frozen = contents.map(frozenOf);
|
|
4500
4558
|
applyTimeOrder(frozen, contents);
|
|
4559
|
+
for (const docId of window.replacedAfterWindow ?? []) {
|
|
4560
|
+
for (const segment of frozen) segment.tombstoneDocument(docId);
|
|
4561
|
+
}
|
|
4501
4562
|
const writeRecords = writeRecordsOf2(frozen, contents);
|
|
4502
|
-
const
|
|
4563
|
+
const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
|
|
4564
|
+
return writeSegmentsOf({
|
|
4503
4565
|
store: input.store,
|
|
4504
4566
|
indexName: input.indexName,
|
|
4505
4567
|
partitionId: input.partitionId,
|
|
@@ -4508,11 +4570,133 @@ async function compactPartitionSegments(input) {
|
|
|
4508
4570
|
partialWordAnalysis: input.partialWordAnalysis,
|
|
4509
4571
|
textAnalysis: input.textAnalysis,
|
|
4510
4572
|
selections: frozen.map((segment) => ({ segment })),
|
|
4511
|
-
tombstones: retainedTombstones(frozen, contents),
|
|
4573
|
+
tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
|
|
4512
4574
|
mergesIntoOneFile: true,
|
|
4513
4575
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
4514
4576
|
});
|
|
4515
|
-
|
|
4577
|
+
}
|
|
4578
|
+
async function compactPartitionSegments(input) {
|
|
4579
|
+
const { segments, compactionThreshold } = input;
|
|
4580
|
+
if (segments.length <= compactionThreshold) {
|
|
4581
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4582
|
+
}
|
|
4583
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
|
|
4584
|
+
if (window === null) {
|
|
4585
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4586
|
+
}
|
|
4587
|
+
const written = await mergeSegmentWindow(input, {
|
|
4588
|
+
refs: segments.slice(window.start, window.start + window.windowLength),
|
|
4589
|
+
startsTheList: window.start === 0
|
|
4590
|
+
});
|
|
4591
|
+
if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
|
|
4592
|
+
return {
|
|
4593
|
+
segments: [
|
|
4594
|
+
...segments.slice(0, window.start),
|
|
4595
|
+
...written.segments,
|
|
4596
|
+
...segments.slice(window.start + window.windowLength)
|
|
4597
|
+
],
|
|
4598
|
+
nextSegmentId: written.nextSegmentId
|
|
4599
|
+
};
|
|
4600
|
+
}
|
|
4601
|
+
|
|
4602
|
+
// src/persistence/durability/segment/removed-documents.ts
|
|
4603
|
+
var decoder2 = new TextDecoder();
|
|
4604
|
+
async function readSegmentIds(store, ref) {
|
|
4605
|
+
const { directory } = store;
|
|
4606
|
+
if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
|
|
4607
|
+
const scan = await openPagedSegmentScan(directory, ref);
|
|
4608
|
+
try {
|
|
4609
|
+
const ids2 = new Array(scan.count);
|
|
4610
|
+
for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
|
|
4611
|
+
return { ids: ids2, tombstones: [...scan.tombstones] };
|
|
4612
|
+
} finally {
|
|
4613
|
+
scan.close();
|
|
4614
|
+
}
|
|
4615
|
+
}
|
|
4616
|
+
const contents = await readSegmentFile(store, ref);
|
|
4617
|
+
const table = contents.arrays.idTable;
|
|
4618
|
+
const ids = new Array(contents.arrays.documentCount);
|
|
4619
|
+
for (let ordinal = 0; ordinal < ids.length; ordinal++) {
|
|
4620
|
+
ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
|
|
4621
|
+
}
|
|
4622
|
+
return { ids, tombstones: contents.tombstones };
|
|
4623
|
+
}
|
|
4624
|
+
async function documentsReplacedLater(store, segments) {
|
|
4625
|
+
const later = /* @__PURE__ */ new Set();
|
|
4626
|
+
const replaced = new Array(segments.length);
|
|
4627
|
+
for (let at = segments.length - 1; at >= 0; at--) {
|
|
4628
|
+
const { ids, tombstones } = await readSegmentIds(store, segments[at]);
|
|
4629
|
+
const found = [];
|
|
4630
|
+
for (const docId of ids) if (later.has(docId)) found.push(docId);
|
|
4631
|
+
replaced[at] = found;
|
|
4632
|
+
if (at === 0) break;
|
|
4633
|
+
for (const docId of ids) later.add(docId);
|
|
4634
|
+
for (const docId of tombstones) later.add(docId);
|
|
4635
|
+
}
|
|
4636
|
+
return replaced;
|
|
4637
|
+
}
|
|
4638
|
+
function exceedsRemovedShare(removed, held) {
|
|
4639
|
+
return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
|
|
4640
|
+
}
|
|
4641
|
+
function freedBytesOf(ref, replacedCount) {
|
|
4642
|
+
return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
|
|
4643
|
+
}
|
|
4644
|
+
function chooseRewrites(segments, replaced, budget) {
|
|
4645
|
+
let held = 0;
|
|
4646
|
+
let removed = 0;
|
|
4647
|
+
for (let at = 0; at < segments.length; at++) {
|
|
4648
|
+
held += segments[at].docCount;
|
|
4649
|
+
removed += replaced[at].length;
|
|
4650
|
+
}
|
|
4651
|
+
const candidates = [];
|
|
4652
|
+
for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
|
|
4653
|
+
candidates.sort(
|
|
4654
|
+
(left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
|
|
4655
|
+
);
|
|
4656
|
+
const chosen = [];
|
|
4657
|
+
for (const at of candidates) {
|
|
4658
|
+
if (!exceedsRemovedShare(removed, held)) break;
|
|
4659
|
+
const bytes = segments[at].bytes;
|
|
4660
|
+
if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
|
|
4661
|
+
chosen.push(at);
|
|
4662
|
+
budget.spentBytes += bytes;
|
|
4663
|
+
held -= replaced[at].length;
|
|
4664
|
+
removed -= replaced[at].length;
|
|
4665
|
+
}
|
|
4666
|
+
return chosen.sort((left, right) => left - right);
|
|
4667
|
+
}
|
|
4668
|
+
function holdsSomething(ref) {
|
|
4669
|
+
return ref.docCount > 0 || ref.tombstoneCount > 0;
|
|
4670
|
+
}
|
|
4671
|
+
async function rewriteRemovedDocuments(input) {
|
|
4672
|
+
const { segments, liveDocumentCount } = input;
|
|
4673
|
+
const unchanged = { segments, nextSegmentId: input.nextSegmentId };
|
|
4674
|
+
if (liveDocumentCount === void 0) return unchanged;
|
|
4675
|
+
let held = 0;
|
|
4676
|
+
for (const ref of segments) held += ref.docCount;
|
|
4677
|
+
if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
|
|
4678
|
+
const replaced = await documentsReplacedLater(input.store, segments);
|
|
4679
|
+
const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
|
|
4680
|
+
if (chosen.size === 0) return unchanged;
|
|
4681
|
+
const rewritten = [];
|
|
4682
|
+
let nextSegmentId = input.nextSegmentId;
|
|
4683
|
+
for (let at = 0; at < segments.length; at++) {
|
|
4684
|
+
if (!chosen.has(at)) {
|
|
4685
|
+
rewritten.push(segments[at]);
|
|
4686
|
+
continue;
|
|
4687
|
+
}
|
|
4688
|
+
const written = await mergeSegmentWindow(
|
|
4689
|
+
{ ...input, nextSegmentId },
|
|
4690
|
+
{ refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
|
|
4691
|
+
);
|
|
4692
|
+
if (written === null) {
|
|
4693
|
+
rewritten.push(segments[at]);
|
|
4694
|
+
continue;
|
|
4695
|
+
}
|
|
4696
|
+
nextSegmentId = written.nextSegmentId;
|
|
4697
|
+
rewritten.push(...written.segments.filter(holdsSomething));
|
|
4698
|
+
}
|
|
4699
|
+
return { segments: rewritten, nextSegmentId };
|
|
4516
4700
|
}
|
|
4517
4701
|
|
|
4518
4702
|
// src/persistence/durability/segment/write.ts
|
|
@@ -4637,8 +4821,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
4637
4821
|
compactionThreshold: context.compactionThreshold,
|
|
4638
4822
|
paged: context.input.metadata.textStorage === "disk"
|
|
4639
4823
|
});
|
|
4640
|
-
|
|
4641
|
-
|
|
4824
|
+
const rewritten = await rewriteRemovedDocuments({
|
|
4825
|
+
store: context.input.store,
|
|
4826
|
+
indexName: context.indexName,
|
|
4827
|
+
partitionId,
|
|
4828
|
+
schema: context.input.metadata.schema,
|
|
4829
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
4830
|
+
textAnalysis: context.textAnalysis,
|
|
4831
|
+
segments: compacted.segments,
|
|
4832
|
+
nextSegmentId: compacted.nextSegmentId,
|
|
4833
|
+
paged: context.input.metadata.textStorage === "disk",
|
|
4834
|
+
liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
|
|
4835
|
+
budget: context.rewriteBudget
|
|
4836
|
+
});
|
|
4837
|
+
segments = rewritten.segments;
|
|
4838
|
+
nextSegmentId = rewritten.nextSegmentId;
|
|
4642
4839
|
return { partitionId, nextSegmentId, segments };
|
|
4643
4840
|
}
|
|
4644
4841
|
function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
|
|
@@ -4663,7 +4860,11 @@ async function writeCheckpointSegments(input) {
|
|
|
4663
4860
|
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
4664
4861
|
textAnalysis,
|
|
4665
4862
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
4666
|
-
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
4863
|
+
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
|
|
4864
|
+
rewriteBudget: {
|
|
4865
|
+
spentBytes: 0,
|
|
4866
|
+
limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
|
|
4867
|
+
}
|
|
4667
4868
|
};
|
|
4668
4869
|
const checkpointByPartition = /* @__PURE__ */ new Map();
|
|
4669
4870
|
for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
|
|
@@ -4710,9 +4911,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
|
4710
4911
|
};
|
|
4711
4912
|
const packed = await encodeSegmentManifest(manifest);
|
|
4712
4913
|
await store.write(manifestKey(indexName), packed.chunks);
|
|
4713
|
-
return {
|
|
4914
|
+
return { manifest, saved: savedBytesOfManifest(manifest) };
|
|
4714
4915
|
}
|
|
4715
|
-
async function
|
|
4916
|
+
async function checkpointGarbage(store, indexName, manifest) {
|
|
4716
4917
|
const referenced = manifestReferencedKeys(manifest);
|
|
4717
4918
|
const stored = await store.list(segmentsPrefix(indexName));
|
|
4718
4919
|
return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
|
|
@@ -4723,4 +4924,4 @@ async function removeCheckpointGarbage(store, garbage) {
|
|
|
4723
4924
|
}
|
|
4724
4925
|
}
|
|
4725
4926
|
|
|
4726
|
-
export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry,
|
|
4927
|
+
export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, residentSectionsOf, savedBytesOfManifest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-
|
|
1
|
+
import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
import { ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/engine/bm25-update.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
2
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
2
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-MSIN4NM3.mjs';
|
|
3
3
|
import { nodeThreadId } from './chunk-DG6WIEB3.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
5
5
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-
|
|
1
|
+
import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-FIXUVFUC.mjs';
|
|
2
2
|
import { ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
import { rm } from 'fs/promises';
|
|
4
4
|
import path from 'path';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-
|
|
1
|
+
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
import { flattenSchema } from './chunk-X5NHWOXT.mjs';
|
|
3
3
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-
|
|
1
|
+
import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-TTTBE4TB.mjs';
|
|
2
2
|
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, requireBoundedUpperLists, osqRecordBytes, osqBitsOf, countDead, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, createFixedBuffer } from './chunk-TNEU32EN.mjs';
|
|
3
|
-
import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-
|
|
4
|
-
import { yieldToEventLoop, compareScores } from './chunk-
|
|
3
|
+
import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-C77OBNPP.mjs';
|
|
4
|
+
import { yieldToEventLoop, compareScores } from './chunk-CWXR7OP4.mjs';
|
|
5
5
|
import { decode, decodeMulti } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { signalOfSearch, requireSearchActive } from './chunk-DG6WIEB3.mjs';
|
|
7
7
|
import { MIN_M, MAX_M, MAX_LAYER_CAP } from './chunk-ZXX5GZEX.mjs';
|
|
@@ -12680,7 +12680,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
|
|
|
12680
12680
|
|
|
12681
12681
|
// src/core/segment-codec-code.ts
|
|
12682
12682
|
var segmentCodec = codeLoader(async () => {
|
|
12683
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-
|
|
12683
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-ACALKS2C.mjs')]);
|
|
12684
12684
|
code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
|
|
12685
12685
|
return code7;
|
|
12686
12686
|
}, "segment codec");
|
|
@@ -12688,7 +12688,7 @@ var loadSegmentCodec = segmentCodec.load;
|
|
|
12688
12688
|
|
|
12689
12689
|
// src/query-suggestions/store-code.ts
|
|
12690
12690
|
var suggestionStore = codeLoader(async () => {
|
|
12691
|
-
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-
|
|
12691
|
+
const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-CWHHEDRK.mjs')]);
|
|
12692
12692
|
code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
|
|
12693
12693
|
return code7;
|
|
12694
12694
|
}, "query suggestions store");
|
|
@@ -12769,7 +12769,7 @@ var storageCode = codeLoader(async () => {
|
|
|
12769
12769
|
loadSegmentMerge(),
|
|
12770
12770
|
loadSuggestionStore(),
|
|
12771
12771
|
loadVectorCore(),
|
|
12772
|
-
import('./storage-leaf-
|
|
12772
|
+
import('./storage-leaf-OJ55R4C5.mjs')
|
|
12773
12773
|
]);
|
|
12774
12774
|
code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
|
|
12775
12775
|
return code7;
|
|
@@ -19104,8 +19104,7 @@ function assembleFrozenSegment(segmentId, source2, documentSource, sharedSnapsho
|
|
|
19104
19104
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
19105
19105
|
return removals.spellingsOf(ordinal);
|
|
19106
19106
|
},
|
|
19107
|
-
prepareTombstone(docId, removedSpellings, analysis) {
|
|
19108
|
-
const ordinal = docStore.getInternalId(docId);
|
|
19107
|
+
prepareTombstone(docId, removedSpellings, analysis, ordinal = docStore.getInternalId(docId)) {
|
|
19109
19108
|
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
19110
19109
|
const publishTombstone = tombstones.prepare(ordinal);
|
|
19111
19110
|
const strings4 = analysis === void 0 ? null : prepareFrozenStrings(
|
|
@@ -21493,7 +21492,7 @@ function hasLanguage(name) {
|
|
|
21493
21492
|
|
|
21494
21493
|
// src/narsil/restore-queue-code.ts
|
|
21495
21494
|
var restoreQueue = codeLoader(async () => {
|
|
21496
|
-
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-
|
|
21495
|
+
const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-LF4DTVJC.mjs')]);
|
|
21497
21496
|
code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
|
|
21498
21497
|
return code7;
|
|
21499
21498
|
}, "restore");
|
|
@@ -22068,7 +22067,7 @@ var loadEmbedCode = embedCode.load;
|
|
|
22068
22067
|
|
|
22069
22068
|
// src/engine/optional-writes-code.ts
|
|
22070
22069
|
var optionalWrites = codeLoader(async () => {
|
|
22071
|
-
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-
|
|
22070
|
+
const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-QE7QUI45.mjs')]);
|
|
22072
22071
|
code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
|
|
22073
22072
|
return code7;
|
|
22074
22073
|
}, "write");
|
|
@@ -24291,7 +24290,8 @@ var DURABILITY_CONFIG_KEYS = {
|
|
|
24291
24290
|
segmentMaxBytes: true,
|
|
24292
24291
|
checkpointIntervalMs: true,
|
|
24293
24292
|
checkpointMutationThreshold: true,
|
|
24294
|
-
compactionThreshold: true
|
|
24293
|
+
compactionThreshold: true,
|
|
24294
|
+
rewriteBytesPerCheckpoint: true
|
|
24295
24295
|
};
|
|
24296
24296
|
var WORKER_CONFIG_KEYS = {
|
|
24297
24297
|
enabled: true,
|
|
@@ -24702,7 +24702,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
24702
24702
|
|
|
24703
24703
|
// src/engine/durability-config-code.ts
|
|
24704
24704
|
var durabilityConfig = codeLoader(
|
|
24705
|
-
() => import('./durability-config-
|
|
24705
|
+
() => import('./durability-config-NWU2G2SJ.mjs').then((code7) => {
|
|
24706
24706
|
code7.bindDurabilityConfigHost(optionalCodeHost());
|
|
24707
24707
|
return code7;
|
|
24708
24708
|
}),
|
|
@@ -25275,7 +25275,7 @@ var workerCopies = codeLoader(async () => {
|
|
|
25275
25275
|
const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
|
|
25276
25276
|
loadSegmentCodec(),
|
|
25277
25277
|
loadSegmentMerge(),
|
|
25278
|
-
import('./worker-copies-
|
|
25278
|
+
import('./worker-copies-373P6IYZ.mjs')
|
|
25279
25279
|
]);
|
|
25280
25280
|
code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
|
|
25281
25281
|
return code7;
|
|
@@ -25521,12 +25521,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
25521
25521
|
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
25522
25522
|
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
25523
25523
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
25524
|
-
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
25525
|
-
const entry = indexRegistry.get(indexName);
|
|
25526
|
-
if (entry === void 0) return;
|
|
25527
|
-
entry.documentCount = documentCount;
|
|
25528
|
-
entry.partitionCount = partitionCount;
|
|
25529
|
-
},
|
|
25530
25524
|
engineCore: () => engineCore
|
|
25531
25525
|
});
|
|
25532
25526
|
const invalidationCode = config?.invalidation === void 0 ? null : durabilityWiring ?? await loadStorageCode();
|
|
@@ -29366,6 +29360,8 @@ function createNarsilFromCore(core2, config) {
|
|
|
29366
29360
|
documentCount: manager?.countDocuments() ?? entry.documentCount,
|
|
29367
29361
|
partitionCount: manager?.partitionCount ?? entry.partitionCount,
|
|
29368
29362
|
estimatedMemoryBytes: manager?.estimateMemoryBytes() ?? 0,
|
|
29363
|
+
savedBytes: entry.storedSizes?.savedBytes,
|
|
29364
|
+
logBytes: entry.storedSizes?.logBytes,
|
|
29369
29365
|
language: entry.language.name,
|
|
29370
29366
|
schema: entry.config.schema,
|
|
29371
29367
|
heldSearchStates: manager?.heldSearchStates ?? 0
|
|
@@ -29373,8 +29369,10 @@ function createNarsilFromCore(core2, config) {
|
|
|
29373
29369
|
},
|
|
29374
29370
|
getPartitionStats(indexName) {
|
|
29375
29371
|
guardShutdown();
|
|
29376
|
-
requireIndex(indexName);
|
|
29377
|
-
|
|
29372
|
+
const entry = requireIndex(indexName);
|
|
29373
|
+
const stats = executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
29374
|
+
entry.storedSizes?.addTo(stats);
|
|
29375
|
+
return stats;
|
|
29378
29376
|
},
|
|
29379
29377
|
insert(indexName, document, docId, options) {
|
|
29380
29378
|
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
@@ -29595,7 +29593,7 @@ var suggestionsCode = codeLoader(async () => {
|
|
|
29595
29593
|
const [msgpack, suggestionStore2, code7] = await Promise.all([
|
|
29596
29594
|
loadMessagePack(),
|
|
29597
29595
|
loadSuggestionStore(),
|
|
29598
|
-
import('./suggestions-leaf-
|
|
29596
|
+
import('./suggestions-leaf-ZTU3ZTWJ.mjs')
|
|
29599
29597
|
]);
|
|
29600
29598
|
code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
|
|
29601
29599
|
return code7;
|
|
@@ -30749,7 +30747,7 @@ var segmentMerge = codeLoader(async () => {
|
|
|
30749
30747
|
const [msgpack, codec, code7] = await Promise.all([
|
|
30750
30748
|
loadMessagePack(),
|
|
30751
30749
|
loadSegmentCodec(),
|
|
30752
|
-
import('./segment-merge-leaf-
|
|
30750
|
+
import('./segment-merge-leaf-SKMPASFK.mjs')
|
|
30753
30751
|
]);
|
|
30754
30752
|
code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
|
|
30755
30753
|
return code7;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { getIndexMetadata } from './chunk-
|
|
2
|
-
import { fnv1a } from './chunk-
|
|
1
|
+
import { getIndexMetadata } from './chunk-F2UOTRY7.mjs';
|
|
2
|
+
import { fnv1a } from './chunk-CWXR7OP4.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/distribution/cluster-node/routable-allocation.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-X4UMDSB4.mjs';
|
|
2
2
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { storedStateOf } from './chunk-
|
|
4
|
-
import { getLanguage, generateId, decodeSynonymRules } from './chunk-
|
|
3
|
+
import { storedStateOf } from './chunk-C77OBNPP.mjs';
|
|
4
|
+
import { getLanguage, generateId, decodeSynonymRules } from './chunk-CWXR7OP4.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
7
7
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-JP77OWMI.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { encodeJson, SYNC_CLOSE_SERVER_ERROR, SYNC_SUBPROTOCOL, encodeScore } from './chunk-FETZYCH7.mjs';
|
|
2
|
-
import { answerRefusalOf } from './chunk-
|
|
3
|
-
import { archiveRange } from './chunk-
|
|
4
|
-
import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-
|
|
2
|
+
import { answerRefusalOf } from './chunk-GGG5R4CU.mjs';
|
|
3
|
+
import { archiveRange } from './chunk-NPXORGMK.mjs';
|
|
4
|
+
import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-CWXR7OP4.mjs';
|
|
5
5
|
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
|
|
6
6
|
import { MAX_FILTER_DEPTH } from './chunk-YM2HI5N5.mjs';
|
|
7
7
|
import { bindSignalStop } from './chunk-DG6WIEB3.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readRecentMemoryFigures, readMemoryFigures } from './chunk-
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
import { collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { openCountsFile } from './chunk-
|
|
2
|
-
import { SCRAMBLED_ID_BYTES } from './chunk-
|
|
1
|
+
import { openCountsFile } from './chunk-MCAJOLVK.mjs';
|
|
2
|
+
import { SCRAMBLED_ID_BYTES } from './chunk-C77OBNPP.mjs';
|
|
3
3
|
export { encode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
function bindSuggestionsHost(_host, _msgpack, _suggestionStore) {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-
|
|
2
|
-
import { archiveLayout } from './chunk-
|
|
3
|
-
import { manifestFileRefs } from './chunk-
|
|
4
|
-
import { DIRECTORY_IDENTITY_CHECK_INTERVAL_MS } from './chunk-
|
|
1
|
+
import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-SA7U67XF.mjs';
|
|
2
|
+
import { archiveLayout } from './chunk-NPXORGMK.mjs';
|
|
3
|
+
import { manifestFileRefs } from './chunk-PK3BGGHO.mjs';
|
|
4
|
+
import { DIRECTORY_IDENTITY_CHECK_INTERVAL_MS } from './chunk-MSIN4NM3.mjs';
|
|
5
5
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
6
|
-
import { openChecksummedEnvelope } from './chunk-
|
|
7
|
-
import { withOpenIndex, emitEngineEvent } from './chunk-
|
|
6
|
+
import { openChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
|
|
7
|
+
import { withOpenIndex, emitEngineEvent } from './chunk-CWXR7OP4.mjs';
|
|
8
8
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
9
9
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
10
10
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|