@delali/narsil 0.3.1-next.87 → 0.3.1-next.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
- package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
- package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
- package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
- package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
- package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
- package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
- package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
- package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
- package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
- package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
- package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
- package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
- package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
- package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
- package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
- package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
- package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
- package/dist/distribution/transport/grpc.mjs +45 -7
- package/dist/distribution.mjs +14 -27
- package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -5
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react.mjs +4 -2
- package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
- package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
- package/dist/workers/entry.mjs +3 -3
- package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
- package/package.json +13 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-R6BMMQXH.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-A3UB2MXL.mjs';
|
|
3
3
|
import { requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
5
|
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -40,12 +40,12 @@ function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
-
function applyDeleteEntry(entry, manager, vecIndexes) {
|
|
43
|
+
function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
44
44
|
removeDocumentVectors(entry.documentId, vecIndexes);
|
|
45
45
|
if (!manager.has(entry.documentId)) {
|
|
46
46
|
return;
|
|
47
47
|
}
|
|
48
|
-
manager.remove(entry.documentId);
|
|
48
|
+
manager.remove(entry.documentId, removedSpellings);
|
|
49
49
|
}
|
|
50
50
|
function restoreVectorFields(document, vectorFieldPaths) {
|
|
51
51
|
for (const fieldPath of vectorFieldPaths) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-MWJIAREY.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment,
|
|
6
|
-
import { fnv1a } from './chunk-
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-R6BMMQXH.mjs';
|
|
5
|
+
import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-EMXAYPTQ.mjs';
|
|
6
|
+
import { fnv1a } from './chunk-X4SEXL4U.mjs';
|
|
7
7
|
import { getLanguage } from './chunk-PR4O7CAP.mjs';
|
|
8
8
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
9
9
|
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-CVQWQVFX.mjs';
|
|
@@ -1385,7 +1385,7 @@ function noSpellingsOf(analysis) {
|
|
|
1385
1385
|
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
1386
1386
|
}
|
|
1387
1387
|
function contentsOfGroup(group, request, tombstones) {
|
|
1388
|
-
const { schema, partialWordAnalysis } = request;
|
|
1388
|
+
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
1389
1389
|
if (group === null) {
|
|
1390
1390
|
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
1391
1391
|
return {
|
|
@@ -1394,7 +1394,7 @@ function contentsOfGroup(group, request, tombstones) {
|
|
|
1394
1394
|
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
1395
1395
|
};
|
|
1396
1396
|
}
|
|
1397
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
|
|
1397
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
1398
1398
|
return {
|
|
1399
1399
|
schema,
|
|
1400
1400
|
tombstones,
|
|
@@ -1529,6 +1529,7 @@ async function compactPartitionSegments(input) {
|
|
|
1529
1529
|
nextSegmentId: input.nextSegmentId,
|
|
1530
1530
|
schema: input.schema,
|
|
1531
1531
|
partialWordAnalysis: input.partialWordAnalysis,
|
|
1532
|
+
textAnalysis: input.textAnalysis,
|
|
1532
1533
|
selections: frozen.map((segment) => ({ segment })),
|
|
1533
1534
|
tombstones: retainedTombstones(frozen, contents),
|
|
1534
1535
|
mergesIntoOneFile: true
|
|
@@ -1544,9 +1545,7 @@ async function compactPartitionSegments(input) {
|
|
|
1544
1545
|
}
|
|
1545
1546
|
|
|
1546
1547
|
// src/persistence/durability/segment/write.ts
|
|
1547
|
-
function
|
|
1548
|
-
const fields = partialFieldsOf(config.schema);
|
|
1549
|
-
if (fields.length === 0) return null;
|
|
1548
|
+
function textAnalysisOf(config) {
|
|
1550
1549
|
let resolved = null;
|
|
1551
1550
|
const resolve = () => {
|
|
1552
1551
|
resolved ??= {
|
|
@@ -1556,7 +1555,6 @@ function partialWordAnalysisOf(config) {
|
|
|
1556
1555
|
return resolved;
|
|
1557
1556
|
};
|
|
1558
1557
|
return {
|
|
1559
|
-
fields,
|
|
1560
1558
|
get language() {
|
|
1561
1559
|
return resolve().language;
|
|
1562
1560
|
},
|
|
@@ -1565,6 +1563,19 @@ function partialWordAnalysisOf(config) {
|
|
|
1565
1563
|
}
|
|
1566
1564
|
};
|
|
1567
1565
|
}
|
|
1566
|
+
function partialWordAnalysisOf(config, text) {
|
|
1567
|
+
const fields = partialFieldsOf(config.schema);
|
|
1568
|
+
if (fields.length === 0) return null;
|
|
1569
|
+
return {
|
|
1570
|
+
fields,
|
|
1571
|
+
get language() {
|
|
1572
|
+
return text.language;
|
|
1573
|
+
},
|
|
1574
|
+
get options() {
|
|
1575
|
+
return text.options;
|
|
1576
|
+
}
|
|
1577
|
+
};
|
|
1578
|
+
}
|
|
1568
1579
|
function resolveCompactionThreshold(configured) {
|
|
1569
1580
|
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
1570
1581
|
return DEFAULT_COMPACTION_THRESHOLD;
|
|
@@ -1585,6 +1596,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
1585
1596
|
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
1586
1597
|
schema: context.input.metadata.schema,
|
|
1587
1598
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
1599
|
+
textAnalysis: context.textAnalysis,
|
|
1588
1600
|
selections: whole.selections,
|
|
1589
1601
|
tombstones: []
|
|
1590
1602
|
});
|
|
@@ -1608,6 +1620,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1608
1620
|
nextSegmentId,
|
|
1609
1621
|
schema: context.input.metadata.schema,
|
|
1610
1622
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
1623
|
+
textAnalysis: context.textAnalysis,
|
|
1611
1624
|
selections: built.selections,
|
|
1612
1625
|
tombstones: built.tombstones
|
|
1613
1626
|
});
|
|
@@ -1627,6 +1640,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1627
1640
|
partitionId,
|
|
1628
1641
|
schema: context.input.metadata.schema,
|
|
1629
1642
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
1643
|
+
textAnalysis: context.textAnalysis,
|
|
1630
1644
|
segments,
|
|
1631
1645
|
nextSegmentId,
|
|
1632
1646
|
compactionThreshold: context.compactionThreshold
|
|
@@ -1649,11 +1663,13 @@ function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkp
|
|
|
1649
1663
|
}
|
|
1650
1664
|
async function writeCheckpointSegments(input) {
|
|
1651
1665
|
const config = reconstructSchemaFromMetadata(input.metadata);
|
|
1666
|
+
const textAnalysis = textAnalysisOf(config);
|
|
1652
1667
|
const context = {
|
|
1653
1668
|
input,
|
|
1654
1669
|
indexName: input.metadata.indexName,
|
|
1655
1670
|
config,
|
|
1656
|
-
partialWordAnalysis: partialWordAnalysisOf(config),
|
|
1671
|
+
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
1672
|
+
textAnalysis,
|
|
1657
1673
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
1658
1674
|
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
1659
1675
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createDirectExecutor, generateId, createPartitionRouter, readHeapStatistics, createPartitionIndex, mergeFrozenSegments, createSharedFrozenSegment, isCompositePartition, freezeSegmentShared, resolvePartitionInsertOptions, encodeSegmentState, createFrozenSegment } from './chunk-
|
|
2
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
3
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, fnv1a } from './chunk-
|
|
1
|
+
import { createDirectExecutor, generateId, createPartitionRouter, readHeapStatistics, createPartitionIndex, mergeFrozenSegments, createSharedFrozenSegment, isCompositePartition, freezeSegmentShared, resolvePartitionInsertOptions, encodeSegmentState, createFrozenSegment } from './chunk-EMXAYPTQ.mjs';
|
|
2
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-DCK4P5BI.mjs';
|
|
3
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, fnv1a } from './chunk-X4SEXL4U.mjs';
|
|
4
4
|
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-J2BPXTD7.mjs';
|
|
5
5
|
import { getLanguage } from './chunk-PR4O7CAP.mjs';
|
|
6
6
|
import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RVM7AODE.mjs';
|
|
@@ -8,7 +8,7 @@ import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
|
8
8
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
|
|
9
9
|
import { requireSearchBackendSettings } from './chunk-QZN3MIJ3.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-T6CNO72J.mjs';
|
|
11
|
-
import { getNestedValue } from './chunk-
|
|
11
|
+
import { getNestedValue } from './chunk-A3UB2MXL.mjs';
|
|
12
12
|
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-CVQWQVFX.mjs';
|
|
13
13
|
import { isSingleStringFieldType } from './chunk-BVOJIYRV.mjs';
|
|
14
14
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
@@ -1628,7 +1628,15 @@ function checkHeapAfterRecovery(core) {
|
|
|
1628
1628
|
// src/core/partition/frozen/share.ts
|
|
1629
1629
|
function shareFrozenSegment(segment, analysis) {
|
|
1630
1630
|
if (segment.sharedSnapshot !== null) {
|
|
1631
|
-
|
|
1631
|
+
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
1632
|
+
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
1633
|
+
return { snapshot: segment.sharedSnapshot, tombstonedDocIds };
|
|
1634
|
+
}
|
|
1635
|
+
return {
|
|
1636
|
+
snapshot: segment.sharedSnapshot,
|
|
1637
|
+
tombstonedDocIds,
|
|
1638
|
+
survivingSurfaceForms: segment.surfaces.reader.serialize()
|
|
1639
|
+
};
|
|
1632
1640
|
}
|
|
1633
1641
|
const payload = encodeSegmentState(segment);
|
|
1634
1642
|
const documents = payload.docIds.map((docId) => {
|
|
@@ -1646,6 +1654,35 @@ function shareFrozenSegment(segment, analysis) {
|
|
|
1646
1654
|
|
|
1647
1655
|
// src/engine/worker-resync.ts
|
|
1648
1656
|
var copyLoadSequence = 0;
|
|
1657
|
+
function emptySerializablePartition(manager, partitionId) {
|
|
1658
|
+
return {
|
|
1659
|
+
indexName: manager.indexName,
|
|
1660
|
+
partitionId,
|
|
1661
|
+
totalPartitions: manager.partitionCount,
|
|
1662
|
+
language: manager.language.name,
|
|
1663
|
+
schema: {},
|
|
1664
|
+
docCount: 0,
|
|
1665
|
+
avgDocLength: 0,
|
|
1666
|
+
documents: {},
|
|
1667
|
+
invertedIndex: {},
|
|
1668
|
+
fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
|
|
1669
|
+
statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
function captureLiveTail(partition) {
|
|
1673
|
+
if (partition.count() === 0) return null;
|
|
1674
|
+
const payload = partition.encodeSegment();
|
|
1675
|
+
const documents = payload.docIds.map((docId) => {
|
|
1676
|
+
const document = partition.getRef(docId);
|
|
1677
|
+
if (document === void 0) {
|
|
1678
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while copying a partition`, {
|
|
1679
|
+
docId
|
|
1680
|
+
});
|
|
1681
|
+
}
|
|
1682
|
+
return document;
|
|
1683
|
+
});
|
|
1684
|
+
return { payload, documents };
|
|
1685
|
+
}
|
|
1649
1686
|
function captureComposite(manager, partitionId) {
|
|
1650
1687
|
const partition = manager.getPartition(partitionId);
|
|
1651
1688
|
if (!isCompositePartition(partition)) return null;
|
|
@@ -1656,13 +1693,7 @@ function captureComposite(manager, partitionId) {
|
|
|
1656
1693
|
if (shared === null) return null;
|
|
1657
1694
|
frozen.push(shared);
|
|
1658
1695
|
}
|
|
1659
|
-
|
|
1660
|
-
manager.indexName,
|
|
1661
|
-
manager.partitionCount,
|
|
1662
|
-
manager.language.name,
|
|
1663
|
-
manager.schema
|
|
1664
|
-
);
|
|
1665
|
-
return { partitionId, live, frozen };
|
|
1696
|
+
return { partitionId, live: captureLiveTail(partition.live), frozen };
|
|
1666
1697
|
}
|
|
1667
1698
|
function capturePartitions(manager) {
|
|
1668
1699
|
const captured = [];
|
|
@@ -1670,30 +1701,45 @@ function capturePartitions(manager) {
|
|
|
1670
1701
|
captured.push(
|
|
1671
1702
|
captureComposite(manager, partitionId) ?? {
|
|
1672
1703
|
partitionId,
|
|
1673
|
-
live: manager.
|
|
1704
|
+
live: captureLiveTail(manager.getPartition(partitionId)),
|
|
1674
1705
|
frozen: []
|
|
1675
1706
|
}
|
|
1676
1707
|
);
|
|
1677
1708
|
}
|
|
1678
1709
|
return captured;
|
|
1679
1710
|
}
|
|
1680
|
-
async function sendPartition(
|
|
1711
|
+
async function sendPartition(manager, executors, captured) {
|
|
1712
|
+
const { indexName } = manager;
|
|
1681
1713
|
await Promise.all(
|
|
1682
1714
|
executors.map(
|
|
1683
1715
|
(workerExecutor) => workerExecutor.execute({
|
|
1684
1716
|
type: "deserialize",
|
|
1685
1717
|
indexName,
|
|
1686
1718
|
partitionId: captured.partitionId,
|
|
1687
|
-
data: captured.
|
|
1719
|
+
data: emptySerializablePartition(manager, captured.partitionId),
|
|
1688
1720
|
requestId: `resync-${indexName}-${captured.partitionId}`
|
|
1689
1721
|
})
|
|
1690
1722
|
)
|
|
1691
1723
|
);
|
|
1724
|
+
const live = captured.live;
|
|
1725
|
+
if (live !== null) {
|
|
1726
|
+
await Promise.all(
|
|
1727
|
+
executors.map(
|
|
1728
|
+
(workerExecutor) => workerExecutor.execute({
|
|
1729
|
+
type: "mergeSegments",
|
|
1730
|
+
indexName,
|
|
1731
|
+
segments: [{ partitionId: captured.partitionId, payload: live.payload, documents: live.documents }],
|
|
1732
|
+
requestId: `resync-live-${indexName}-${captured.partitionId}`
|
|
1733
|
+
})
|
|
1734
|
+
)
|
|
1735
|
+
);
|
|
1736
|
+
}
|
|
1692
1737
|
if (captured.frozen.length === 0) return;
|
|
1693
1738
|
const segments = captured.frozen.map((shared) => ({
|
|
1694
1739
|
partitionId: captured.partitionId,
|
|
1695
1740
|
snapshot: shared.snapshot,
|
|
1696
|
-
tombstonedDocIds: shared.tombstonedDocIds
|
|
1741
|
+
tombstonedDocIds: shared.tombstonedDocIds,
|
|
1742
|
+
survivingSurfaceForms: shared.survivingSurfaceForms
|
|
1697
1743
|
}));
|
|
1698
1744
|
await Promise.all(
|
|
1699
1745
|
executors.map(
|
|
@@ -1739,7 +1785,7 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
1739
1785
|
)
|
|
1740
1786
|
);
|
|
1741
1787
|
for (const partition of captured) {
|
|
1742
|
-
await sendPartition(
|
|
1788
|
+
await sendPartition(manager, executors, partition);
|
|
1743
1789
|
}
|
|
1744
1790
|
return copyLoad;
|
|
1745
1791
|
}
|
|
@@ -2337,21 +2383,24 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
2337
2383
|
const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
|
|
2338
2384
|
let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
|
|
2339
2385
|
if (snapshot === null && mainCanMerge && composite !== null) {
|
|
2340
|
-
snapshot = mergeFrozenSegments(
|
|
2386
|
+
snapshot = mergeFrozenSegments(
|
|
2387
|
+
composite.frozenSegmentsById(segmentIds),
|
|
2388
|
+
manager.partialWordAnalysis,
|
|
2389
|
+
manager.textAnalysis
|
|
2390
|
+
);
|
|
2341
2391
|
}
|
|
2342
2392
|
if (snapshot === null) {
|
|
2343
2393
|
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
2344
2394
|
return false;
|
|
2345
2395
|
}
|
|
2346
|
-
|
|
2347
|
-
composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot));
|
|
2348
|
-
}
|
|
2396
|
+
const removedSpellings = mainCanMerge && composite !== null ? composite.swapFrozenSegments(segmentIds, createSharedFrozenSegment(snapshot)) : void 0;
|
|
2349
2397
|
await replicateToWorkers(state, {
|
|
2350
2398
|
type: "swapSegments",
|
|
2351
2399
|
indexName,
|
|
2352
2400
|
partitionId,
|
|
2353
2401
|
dropSegmentIds: [...segmentIds],
|
|
2354
2402
|
snapshot,
|
|
2403
|
+
removedSpellings,
|
|
2355
2404
|
requestId: createRequestId()
|
|
2356
2405
|
});
|
|
2357
2406
|
return true;
|
|
@@ -3871,7 +3920,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
3871
3920
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
3872
3921
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
3873
3922
|
}
|
|
3874
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
3923
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-ZLRQU6FG.mjs');
|
|
3875
3924
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
3876
3925
|
const executor = createDirectExecutor({
|
|
3877
3926
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -5588,16 +5637,18 @@ async function prepareRemoval(ctx, indexName, docId) {
|
|
|
5588
5637
|
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
5589
5638
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
5590
5639
|
}
|
|
5591
|
-
const
|
|
5640
|
+
const manager = ctx.requireManager(indexName);
|
|
5641
|
+
const restoreRef = ctx.durability && !rebalancing ? manager.getRef(docId) : void 0;
|
|
5592
5642
|
return {
|
|
5593
5643
|
docId,
|
|
5594
5644
|
document: null,
|
|
5595
5645
|
restoreDoc: restoreRef ? structuredClone(restoreRef) : void 0,
|
|
5646
|
+
removedSpellings: manager.removedSpellingsOf(docId),
|
|
5596
5647
|
buffered: false
|
|
5597
5648
|
};
|
|
5598
5649
|
}
|
|
5599
5650
|
function applyOfRemoval(ctx, indexName, prepared) {
|
|
5600
|
-
const { docId } = prepared;
|
|
5651
|
+
const { docId, removedSpellings } = prepared;
|
|
5601
5652
|
return async () => {
|
|
5602
5653
|
if (ctx.isRebalancing(indexName)) {
|
|
5603
5654
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
@@ -5610,11 +5661,12 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
5610
5661
|
prepared.buffered = true;
|
|
5611
5662
|
return;
|
|
5612
5663
|
}
|
|
5613
|
-
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
|
|
5664
|
+
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
|
|
5614
5665
|
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
5615
5666
|
};
|
|
5616
5667
|
}
|
|
5617
|
-
async function settleRemoval(ctx, indexName,
|
|
5668
|
+
async function settleRemoval(ctx, indexName, prepared) {
|
|
5669
|
+
const { docId, removedSpellings } = prepared;
|
|
5618
5670
|
try {
|
|
5619
5671
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
5620
5672
|
} catch (err) {
|
|
@@ -5624,7 +5676,8 @@ async function settleRemoval(ctx, indexName, docId) {
|
|
|
5624
5676
|
type: "remove",
|
|
5625
5677
|
indexName,
|
|
5626
5678
|
docId,
|
|
5627
|
-
requestId: `replicate-remove-${docId}
|
|
5679
|
+
requestId: `replicate-remove-${docId}`,
|
|
5680
|
+
removedSpellings
|
|
5628
5681
|
});
|
|
5629
5682
|
}
|
|
5630
5683
|
async function removeDocument(ctx, indexName, docId, options) {
|
|
@@ -5642,7 +5695,7 @@ async function removeDocument(ctx, indexName, docId, options) {
|
|
|
5642
5695
|
} else {
|
|
5643
5696
|
await apply();
|
|
5644
5697
|
}
|
|
5645
|
-
if (!prepared.buffered) await settleRemoval(ctx, indexName,
|
|
5698
|
+
if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
|
|
5646
5699
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
5647
5700
|
}
|
|
5648
5701
|
async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
@@ -5678,7 +5731,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
5678
5731
|
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
5679
5732
|
continue;
|
|
5680
5733
|
}
|
|
5681
|
-
if (!removal.buffered) await settleRemoval(ctx, indexName, removal
|
|
5734
|
+
if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
|
|
5682
5735
|
succeeded.push(removal.docId);
|
|
5683
5736
|
}
|
|
5684
5737
|
if (chunkEnd < docIds.length) {
|
|
@@ -5726,7 +5779,8 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
5726
5779
|
indexName,
|
|
5727
5780
|
docId: prepared.docId,
|
|
5728
5781
|
document: prepared.partitionDoc,
|
|
5729
|
-
requestId: `replicate-update-${prepared.docId}
|
|
5782
|
+
requestId: `replicate-update-${prepared.docId}`,
|
|
5783
|
+
removedSpellings: prepared.removedSpellings
|
|
5730
5784
|
});
|
|
5731
5785
|
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
5732
5786
|
for (const [fieldPath, vec] of prepared.extractedVectors) {
|
|
@@ -5793,12 +5847,13 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
5793
5847
|
oldDocument,
|
|
5794
5848
|
rollbackDoc,
|
|
5795
5849
|
extractedVectors,
|
|
5850
|
+
removedSpellings: manager.removedSpellingsOf(docId),
|
|
5796
5851
|
buffered: false
|
|
5797
5852
|
};
|
|
5798
5853
|
}
|
|
5799
5854
|
function applyOfUpdate(ctx, indexName, prepared) {
|
|
5800
5855
|
const manager = ctx.requireManager(indexName);
|
|
5801
|
-
const { docId, document, partitionDoc, rollbackDoc, extractedVectors } = prepared;
|
|
5856
|
+
const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
|
|
5802
5857
|
return async () => {
|
|
5803
5858
|
if (ctx.isRebalancing(indexName)) {
|
|
5804
5859
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
@@ -5811,7 +5866,14 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
5811
5866
|
prepared.buffered = true;
|
|
5812
5867
|
return;
|
|
5813
5868
|
}
|
|
5814
|
-
await ctx.executor.execute({
|
|
5869
|
+
await ctx.executor.execute({
|
|
5870
|
+
type: "update",
|
|
5871
|
+
indexName,
|
|
5872
|
+
docId,
|
|
5873
|
+
document: partitionDoc,
|
|
5874
|
+
requestId: docId,
|
|
5875
|
+
removedSpellings
|
|
5876
|
+
});
|
|
5815
5877
|
try {
|
|
5816
5878
|
updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
5817
5879
|
} catch (err) {
|
|
@@ -5996,9 +6058,10 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
5996
6058
|
vectorFieldPaths
|
|
5997
6059
|
);
|
|
5998
6060
|
const overwrote = manager.has(entry.docId);
|
|
6061
|
+
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
5999
6062
|
if (overwrote) {
|
|
6000
6063
|
removeDocumentVectors(entry.docId, vecIndexes);
|
|
6001
|
-
manager.remove(entry.docId);
|
|
6064
|
+
manager.remove(entry.docId, removedSpellings);
|
|
6002
6065
|
}
|
|
6003
6066
|
manager.insert(entry.docId, partitionDoc);
|
|
6004
6067
|
if (extractedVectors.size > 0) {
|
|
@@ -6012,17 +6075,20 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
6012
6075
|
} catch (err) {
|
|
6013
6076
|
warnHookError("afterInsert", err);
|
|
6014
6077
|
}
|
|
6015
|
-
|
|
6016
|
-
type: overwrote ? "update" : "insert",
|
|
6078
|
+
const replicated = {
|
|
6017
6079
|
indexName,
|
|
6018
6080
|
docId: entry.docId,
|
|
6019
6081
|
document: entry.document,
|
|
6020
6082
|
requestId: `replicate-insert-${entry.docId}`
|
|
6021
|
-
}
|
|
6083
|
+
};
|
|
6084
|
+
await ctx.orchestrator.replicateToWorkers(
|
|
6085
|
+
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
6086
|
+
);
|
|
6022
6087
|
return;
|
|
6023
6088
|
}
|
|
6024
6089
|
if (entry.action === "remove") {
|
|
6025
|
-
manager.
|
|
6090
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
6091
|
+
manager.remove(entry.docId, removedSpellings);
|
|
6026
6092
|
removeDocumentVectors(entry.docId, vecIndexes);
|
|
6027
6093
|
try {
|
|
6028
6094
|
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
@@ -6033,15 +6099,17 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
6033
6099
|
type: "remove",
|
|
6034
6100
|
indexName,
|
|
6035
6101
|
docId: entry.docId,
|
|
6036
|
-
requestId: `replicate-remove-${entry.docId}
|
|
6102
|
+
requestId: `replicate-remove-${entry.docId}`,
|
|
6103
|
+
removedSpellings
|
|
6037
6104
|
});
|
|
6038
6105
|
return;
|
|
6039
6106
|
}
|
|
6040
6107
|
if (entry.action === "update" && entry.document) {
|
|
6041
6108
|
const oldDocument = manager.get(entry.docId);
|
|
6042
6109
|
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
|
|
6110
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
6043
6111
|
if (manager.has(entry.docId)) {
|
|
6044
|
-
manager.update(entry.docId, updateDoc);
|
|
6112
|
+
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
6045
6113
|
} else {
|
|
6046
6114
|
manager.insert(entry.docId, updateDoc);
|
|
6047
6115
|
}
|
|
@@ -6078,7 +6146,8 @@ async function replayEntry(manager, indexName, entry, ctx) {
|
|
|
6078
6146
|
indexName,
|
|
6079
6147
|
docId: entry.docId,
|
|
6080
6148
|
document: entry.document,
|
|
6081
|
-
requestId: `replicate-update-${entry.docId}
|
|
6149
|
+
requestId: `replicate-update-${entry.docId}`,
|
|
6150
|
+
removedSpellings
|
|
6082
6151
|
});
|
|
6083
6152
|
}
|
|
6084
6153
|
}
|
|
@@ -6727,7 +6796,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6727
6796
|
};
|
|
6728
6797
|
}
|
|
6729
6798
|
async function restoreIndex(indexName, data) {
|
|
6730
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
6799
|
+
const { restoreFromSnapshot } = await import('./snapshot-22CUEYYW.mjs');
|
|
6731
6800
|
await restoreFromSnapshot(indexName, data, {
|
|
6732
6801
|
executor: core.executor,
|
|
6733
6802
|
indexRegistry: core.indexRegistry,
|
|
@@ -6747,7 +6816,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6747
6816
|
return {
|
|
6748
6817
|
snapshot(indexName) {
|
|
6749
6818
|
return withOpenIndex(indexName, async () => {
|
|
6750
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6819
|
+
const { createSnapshot } = await import('./snapshot-22CUEYYW.mjs');
|
|
6751
6820
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6752
6821
|
});
|
|
6753
6822
|
},
|
|
@@ -6759,8 +6828,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6759
6828
|
exportDocuments(indexName) {
|
|
6760
6829
|
return withOpenIndex(indexName, async () => {
|
|
6761
6830
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6762
|
-
import('./snapshot-
|
|
6763
|
-
import('./write-
|
|
6831
|
+
import('./snapshot-22CUEYYW.mjs'),
|
|
6832
|
+
import('./write-PXDKPG2R.mjs')
|
|
6764
6833
|
]);
|
|
6765
6834
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6766
6835
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -7244,4 +7313,4 @@ async function createNarsil(config) {
|
|
|
7244
7313
|
// src/index.ts
|
|
7245
7314
|
var VERSION = "0.1.0";
|
|
7246
7315
|
|
|
7247
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareDocumentVectors, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
|
|
7316
|
+
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareDocumentVectors, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
1
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-CL4OM7GB.mjs';
|
|
2
2
|
import { vectorsPerFile } from './chunk-MXS5HWIH.mjs';
|
|
3
3
|
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-YCJLVNWR.mjs';
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-QAMUC2BJ.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-R6BMMQXH.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
8
8
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-
|
|
2
|
-
import { manifestKey, segmentsPrefix } from './chunk-
|
|
3
|
-
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-
|
|
1
|
+
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-CL4OM7GB.mjs';
|
|
2
|
+
import { manifestKey, segmentsPrefix } from './chunk-QAMUC2BJ.mjs';
|
|
3
|
+
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-R6BMMQXH.mjs';
|
|
4
4
|
import { VALID_HNSW_METRICS } from './chunk-RVM7AODE.mjs';
|
|
5
5
|
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-CVQWQVFX.mjs';
|
|
6
6
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
1
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-QAMUC2BJ.mjs';
|
|
2
2
|
import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|