@delali/narsil 0.3.1-next.91 → 0.3.1-next.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-4JTBL63P.mjs} +4 -4
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-HC6DMR4T.mjs} +238 -114
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-PP6N5CCS.mjs} +2 -2
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-PPTZ565G.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-2ZAA7UDH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-xsJfdFoQ.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
- package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-A4G7JSYO.mjs → chunk-C45FUYPI.mjs} +98 -49
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-67Z2V6TC.mjs → chunk-EWBXO45L.mjs} +3 -3
- package/dist/{chunk-3TTYK5A7.mjs → chunk-EX62BQT6.mjs} +9 -9
- package/dist/{chunk-3RDS4UHC.mjs → chunk-EZD4MWOU.mjs} +4 -4
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-7OQHSDK6.mjs → chunk-J53IHLG7.mjs} +7 -7
- package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-QWWCLRXX.mjs} +6 -6
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-UAUZGXJP.mjs} +40 -21
- package/dist/{chunk-EONIHG3V.mjs → chunk-WBS3EYC4.mjs} +7 -7
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-CgayPw2R.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-3kBsQ0Pi.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-B-RnO6dr.d.ts} +1 -1
- package/dist/direct-executor-IJTRU3YN.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-Y55SCPFC.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-DSUfY2F0.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-wuXcYOD1.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-4SG45BLF.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-XBKE7PUG.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-Bj9WBlFG.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-BMNV7P2H.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-QM3VPX4Z.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-
|
|
4
|
-
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-
|
|
3
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-SBCEHIPL.mjs';
|
|
4
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-4ZJWRETC.mjs';
|
|
5
5
|
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FGMGUDHX.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-
|
|
7
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-BB7MASRQ.mjs';
|
|
8
|
+
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-7KC56574.mjs';
|
|
9
9
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
13
13
|
|
|
14
14
|
// src/core/partition/partial-words/platform.ts
|
|
15
15
|
function platformPartialWordSearch() {
|
|
@@ -423,6 +423,7 @@ function createDocumentStore() {
|
|
|
423
423
|
const reverseMap = [];
|
|
424
424
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
425
425
|
let nextInternalId = 0;
|
|
426
|
+
let numberingResets = 0;
|
|
426
427
|
let sortedIds = null;
|
|
427
428
|
let storedContentBytes = 0;
|
|
428
429
|
function replaceStored(docId, document, fieldLengths) {
|
|
@@ -461,6 +462,7 @@ function createDocumentStore() {
|
|
|
461
462
|
forwardMap.clear();
|
|
462
463
|
reverseMap.length = 0;
|
|
463
464
|
nextInternalId = 0;
|
|
465
|
+
numberingResets++;
|
|
464
466
|
sortedIds = null;
|
|
465
467
|
}
|
|
466
468
|
const idResolver = {
|
|
@@ -569,6 +571,9 @@ function createDocumentStore() {
|
|
|
569
571
|
internalIdCapacity() {
|
|
570
572
|
return nextInternalId;
|
|
571
573
|
},
|
|
574
|
+
numberingGeneration() {
|
|
575
|
+
return numberingResets;
|
|
576
|
+
},
|
|
572
577
|
contentBytes() {
|
|
573
578
|
return storedContentBytes;
|
|
574
579
|
},
|
|
@@ -4905,7 +4910,7 @@ function estimatePartitionBytes(state) {
|
|
|
4905
4910
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
4906
4911
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
4907
4912
|
bytes += state.partialWords.bytes();
|
|
4908
|
-
return bytes + sortColumnBytes(state);
|
|
4913
|
+
return bytes + sortColumnBytes(state) + (state.vectorOrdinalMaps?.bytes ?? 0);
|
|
4909
4914
|
}
|
|
4910
4915
|
|
|
4911
4916
|
// src/core/partition/merge.ts
|
|
@@ -6329,7 +6334,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6329
6334
|
lastSchemaRef: null,
|
|
6330
6335
|
trackPositions,
|
|
6331
6336
|
sortColumns: null,
|
|
6332
|
-
scoreBuffer: null
|
|
6337
|
+
scoreBuffer: null,
|
|
6338
|
+
vectorOrdinalMaps: null
|
|
6333
6339
|
};
|
|
6334
6340
|
function clearAll() {
|
|
6335
6341
|
state.invertedIdx.clear();
|
|
@@ -6351,6 +6357,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6351
6357
|
state.lastSchemaRef = null;
|
|
6352
6358
|
state.sortColumns = null;
|
|
6353
6359
|
state.scoreBuffer = null;
|
|
6360
|
+
state.vectorOrdinalMaps = null;
|
|
6354
6361
|
}
|
|
6355
6362
|
const partition = {
|
|
6356
6363
|
get partitionId() {
|
|
@@ -6505,6 +6512,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6505
6512
|
estimateMemoryBytes() {
|
|
6506
6513
|
return estimatePartitionBytes(state);
|
|
6507
6514
|
},
|
|
6515
|
+
readStates() {
|
|
6516
|
+
return [state];
|
|
6517
|
+
},
|
|
6508
6518
|
searchFulltext(params) {
|
|
6509
6519
|
return searchFulltext(state, params);
|
|
6510
6520
|
},
|
|
@@ -6664,7 +6674,7 @@ function frozenSegmentBytes(segment) {
|
|
|
6664
6674
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
6665
6675
|
}
|
|
6666
6676
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
6667
|
-
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
6677
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
6668
6678
|
}
|
|
6669
6679
|
|
|
6670
6680
|
// src/core/partition/composite/compaction.ts
|
|
@@ -6737,6 +6747,24 @@ function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
|
6737
6747
|
}
|
|
6738
6748
|
return live.fieldValues(docId, fieldPaths);
|
|
6739
6749
|
}
|
|
6750
|
+
function* compositeDocIds(frozen, live) {
|
|
6751
|
+
for (const segment of frozen) {
|
|
6752
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
6753
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
6754
|
+
if (docId !== void 0) yield docId;
|
|
6755
|
+
}
|
|
6756
|
+
}
|
|
6757
|
+
yield* live.docIds();
|
|
6758
|
+
}
|
|
6759
|
+
function compositeSortedDocIds(frozen, live) {
|
|
6760
|
+
const merged = [];
|
|
6761
|
+
for (const segment of frozen) {
|
|
6762
|
+
merged.push(...segment.docStore.sortedDocIds());
|
|
6763
|
+
}
|
|
6764
|
+
merged.push(...live.sortedDocIds());
|
|
6765
|
+
merged.sort(compareCodePoints);
|
|
6766
|
+
return merged;
|
|
6767
|
+
}
|
|
6740
6768
|
|
|
6741
6769
|
// src/core/partition/composite/filters.ts
|
|
6742
6770
|
var WORD_BITS = 32;
|
|
@@ -7412,23 +7440,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7412
7440
|
}
|
|
7413
7441
|
return total;
|
|
7414
7442
|
},
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
7418
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
7419
|
-
if (docId !== void 0) yield docId;
|
|
7420
|
-
}
|
|
7421
|
-
}
|
|
7422
|
-
yield* live.docIds();
|
|
7443
|
+
docIds() {
|
|
7444
|
+
return compositeDocIds(frozen, live);
|
|
7423
7445
|
},
|
|
7424
7446
|
sortedDocIds() {
|
|
7425
|
-
|
|
7426
|
-
for (const segment of frozen) {
|
|
7427
|
-
merged.push(...segment.docStore.sortedDocIds());
|
|
7428
|
-
}
|
|
7429
|
-
merged.push(...live.sortedDocIds());
|
|
7430
|
-
merged.sort(compareCodePoints);
|
|
7431
|
-
return merged;
|
|
7447
|
+
return compositeSortedDocIds(frozen, live);
|
|
7432
7448
|
},
|
|
7433
7449
|
releaseSortedDocIds() {
|
|
7434
7450
|
for (const segment of frozen) {
|
|
@@ -7442,6 +7458,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7442
7458
|
total += frozenSegmentBytes(segment);
|
|
7443
7459
|
}
|
|
7444
7460
|
return total;
|
|
7461
|
+
},
|
|
7462
|
+
readStates() {
|
|
7463
|
+
return subs();
|
|
7445
7464
|
}
|
|
7446
7465
|
};
|
|
7447
7466
|
}
|
|
@@ -8436,4 +8455,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8436
8455
|
return count;
|
|
8437
8456
|
}
|
|
8438
8457
|
|
|
8439
|
-
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
8458
|
+
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
2
2
|
import { existsSync } from 'fs';
|
|
3
3
|
import { createRequire } from 'module';
|
|
4
4
|
import { join, dirname } from 'path';
|
|
5
5
|
|
|
6
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
6
|
+
var NATIVE_CORE_ABI_VERSION = 4;
|
|
7
7
|
var BINARY_FILE_NAME = "narsil-core.node";
|
|
8
8
|
var MUSL_DIRECTORY = "musl";
|
|
9
9
|
var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
|
|
2
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
2
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-EX62BQT6.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
5
5
|
import { openSync, close, readFileSync } from 'fs';
|
|
6
6
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
7
7
|
import { endianness } from 'os';
|
|
@@ -628,4 +628,4 @@ async function listRecursive(dir, base, pathMod, fs) {
|
|
|
628
628
|
return results;
|
|
629
629
|
}
|
|
630
630
|
|
|
631
|
-
export { createDurableDirectory };
|
|
631
|
+
export { createDurableDirectory, getFs };
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-EZD4MWOU.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-OHP7QR33.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
6
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
7
|
-
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-
|
|
8
|
-
import { generateId, getLanguage } from './chunk-
|
|
9
|
-
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-
|
|
10
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
11
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-C45FUYPI.mjs';
|
|
5
|
+
import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
6
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-UAUZGXJP.mjs';
|
|
7
|
+
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-CHOGQSHK.mjs';
|
|
8
|
+
import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
|
|
9
|
+
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-4ZJWRETC.mjs';
|
|
10
|
+
import { resolveIndexAnalysis } from './chunk-2GZGBOQI.mjs';
|
|
11
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
12
12
|
import { encode, decode } from '@msgpack/msgpack';
|
|
13
13
|
|
|
14
14
|
// src/persistence/durability/constants.ts
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
3
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-C45FUYPI.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-SBCEHIPL.mjs';
|
|
3
|
+
import { requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
5
|
+
import { ErrorCodes, NarsilError } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { decode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
function validateReplicationEntry(entry, localPrimaryTerm, log) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
2
2
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/vector-index/dead-bits.ts
|
|
6
6
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
|
|
2
|
-
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-
|
|
3
|
-
import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
|
|
2
|
+
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-WLOF6HSP.mjs';
|
|
4
3
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
|
|
4
|
+
import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
|
|
5
5
|
import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
8
8
|
import { encode, decode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/distribution/cluster/constants.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
3
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
6
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-7VALMHZF.mjs';
|
|
2
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-WBS3EYC4.mjs';
|
|
3
|
+
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-F52QGNZ2.mjs';
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-EX62BQT6.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-C45FUYPI.mjs';
|
|
6
|
+
import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
8
8
|
|
|
9
9
|
// src/persistence/archive/memory-store.ts
|
|
10
10
|
function joined(chunks) {
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-VITS4ZR5.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// src/runtime/simd-support.ts
|
|
2
|
+
var SIMD_PROBE_MODULE = new Uint8Array([
|
|
3
|
+
0,
|
|
4
|
+
97,
|
|
5
|
+
115,
|
|
6
|
+
109,
|
|
7
|
+
1,
|
|
8
|
+
0,
|
|
9
|
+
0,
|
|
10
|
+
0,
|
|
11
|
+
1,
|
|
12
|
+
5,
|
|
13
|
+
1,
|
|
14
|
+
96,
|
|
15
|
+
0,
|
|
16
|
+
1,
|
|
17
|
+
123,
|
|
18
|
+
3,
|
|
19
|
+
2,
|
|
20
|
+
1,
|
|
21
|
+
0,
|
|
22
|
+
10,
|
|
23
|
+
10,
|
|
24
|
+
1,
|
|
25
|
+
8,
|
|
26
|
+
0,
|
|
27
|
+
65,
|
|
28
|
+
0,
|
|
29
|
+
253,
|
|
30
|
+
15,
|
|
31
|
+
253,
|
|
32
|
+
98,
|
|
33
|
+
11
|
|
34
|
+
]);
|
|
35
|
+
function runtimeStartsSimdModule() {
|
|
36
|
+
try {
|
|
37
|
+
if (typeof WebAssembly === "undefined") return false;
|
|
38
|
+
new WebAssembly.Instance(new WebAssembly.Module(SIMD_PROBE_MODULE));
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
var simdAvailable = runtimeStartsSimdModule();
|
|
45
|
+
function isSimdAvailable() {
|
|
46
|
+
return simdAvailable;
|
|
47
|
+
}
|
|
48
|
+
function markSimdUnavailable() {
|
|
49
|
+
simdAvailable = false;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export { isSimdAvailable, markSimdUnavailable };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
2
2
|
import { tabulatedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
3
|
-
import { parsedTypeOf } from './chunk-
|
|
3
|
+
import { parsedTypeOf } from './chunk-BB7MASRQ.mjs';
|
|
4
4
|
import { codePointCount, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
|
|
7
7
|
// src/core/bitset.ts
|
|
8
8
|
function createBitSet(capacity) {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-
|
|
2
|
-
import { manifestKey, segmentsPrefix } from './chunk-
|
|
3
|
-
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-
|
|
4
|
-
import { VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-WBS3EYC4.mjs';
|
|
2
|
+
import { manifestKey, segmentsPrefix } from './chunk-EX62BQT6.mjs';
|
|
3
|
+
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-C45FUYPI.mjs';
|
|
4
|
+
import { VALID_HNSW_METRICS } from './chunk-RDJKYKMQ.mjs';
|
|
5
|
+
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-4ZJWRETC.mjs';
|
|
6
6
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
8
8
|
import { decode, encode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/serialization/stored-vector-values.ts
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// src/vector/constants.ts
|
|
2
|
+
var WASM_PAGE_BYTES = 65536;
|
|
3
|
+
var MAX_WASM_PAGES = 65536;
|
|
4
|
+
var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
5
|
+
var VECTOR_BLOCK_INITIAL_SLOTS = 256;
|
|
6
|
+
var DISK_STORAGE_BLOCK_BYTES = 67108864;
|
|
7
|
+
var MAX_VECTOR_ORDINALS = 16777216;
|
|
8
|
+
var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
|
|
9
|
+
var VECTOR_SCRATCH_SLOTS = 64;
|
|
10
|
+
var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
|
|
11
|
+
var MAX_DOC_ID_TABLE_BYTES = 1073741824;
|
|
12
|
+
var MAX_WORKER_COPIES = 64;
|
|
13
|
+
var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
|
|
14
|
+
var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
|
|
15
|
+
var VECTOR_INSERT_TIMEOUT_MS = 6e5;
|
|
16
|
+
var NATIVE_ATTACH_RETRY_MS = 1e3;
|
|
17
|
+
var MAX_RENUMBERED_SEARCH_ATTEMPTS = 3;
|
|
18
|
+
var LOCK_SPIN_ITERATIONS = 128;
|
|
19
|
+
var LOCK_WAIT_SLICE_MS = 20;
|
|
20
|
+
var GRAPH_LOCK_WAIT_SLICE_MS = 1;
|
|
21
|
+
var GRAPH_BUILD_DISPATCH_CHUNK = 64;
|
|
22
|
+
var GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER = 2;
|
|
23
|
+
|
|
24
|
+
// src/vector/osq/constants.ts
|
|
25
|
+
var OSQ_GRID = { 1: 0.798, 2: 1.493, 4: 2.514, 8: 3.922 };
|
|
26
|
+
var OSQ_STEPS = { 1: 1, 2: 3, 4: 15, 8: 255 };
|
|
27
|
+
var OSQ_QUERY_BITS = { 1: 4, 2: 4, 4: 4, 8: 8 };
|
|
28
|
+
var OSQ_LAMBDA = 0.1;
|
|
29
|
+
var OSQ_REFINE_ROUNDS = 5;
|
|
30
|
+
var OSQ_REFINE_TOLERANCE = 1e-8;
|
|
31
|
+
var MIN_OVERSAMPLE = 1;
|
|
32
|
+
|
|
33
|
+
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_SLICE_MS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_TABLE_BYTES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, OSQ_GRID, OSQ_LAMBDA, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_STEPS, SCRATCH_SLOTS_PER_THREAD_POOL, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES };
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex, applyPartitionFiltersBitset } from './chunk-CHOGQSHK.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Q5DC6WAB.mjs';
|
|
3
|
-
import { MIN_OVERSAMPLE } from './chunk-
|
|
4
|
-
import { partialFieldsOf, flattenSchema } from './chunk-
|
|
3
|
+
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QY7GC2MW.mjs';
|
|
4
|
+
import { partialFieldsOf, flattenSchema } from './chunk-4ZJWRETC.mjs';
|
|
5
5
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import { tokenize } from './chunk-
|
|
7
|
+
import { isWordIndexedFieldType } from './chunk-BB7MASRQ.mjs';
|
|
8
|
+
import { tokenize } from './chunk-7KC56574.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -321,17 +321,6 @@ function searchOptionsFor(manager, patternWork) {
|
|
|
321
321
|
patternWork
|
|
322
322
|
};
|
|
323
323
|
}
|
|
324
|
-
function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
|
|
325
|
-
const filterDocIds = /* @__PURE__ */ new Set();
|
|
326
|
-
if (!params.filters) return filterDocIds;
|
|
327
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
328
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
|
|
329
|
-
for (const docId of partitionFiltered) {
|
|
330
|
-
filterDocIds.add(docId);
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
return filterDocIds;
|
|
334
|
-
}
|
|
335
324
|
function vectorResultsToScored(results) {
|
|
336
325
|
return results.map((r) => ({
|
|
337
326
|
docId: r.docId,
|
|
@@ -1193,6 +1182,47 @@ function minMaxNormalize(docs) {
|
|
|
1193
1182
|
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
1194
1183
|
}
|
|
1195
1184
|
|
|
1185
|
+
// src/engine/query/filter-subsets.ts
|
|
1186
|
+
function collectFilterSubsets(manager, filters, schema, partitionIds, patternWork) {
|
|
1187
|
+
const subsets = [];
|
|
1188
|
+
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
1189
|
+
for (const state of partition.readStates()) {
|
|
1190
|
+
subsets.push({
|
|
1191
|
+
owner: state,
|
|
1192
|
+
bits: applyPartitionFiltersBitset(state, filters, schema, patternWork),
|
|
1193
|
+
numberingGeneration: state.docStore.numberingGeneration()
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
return subsets;
|
|
1198
|
+
}
|
|
1199
|
+
function filterSubsetsMatchNothing(subsets) {
|
|
1200
|
+
for (const subset of subsets) {
|
|
1201
|
+
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
1202
|
+
if (subset.bits[wordIndex] !== 0) return false;
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
return true;
|
|
1206
|
+
}
|
|
1207
|
+
function filterSubsetsRenumbered(subsets) {
|
|
1208
|
+
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
1209
|
+
}
|
|
1210
|
+
async function searchCurrentSubsets(fieldName, first, collectAgain, search) {
|
|
1211
|
+
let subsets = first;
|
|
1212
|
+
for (let attempt = 1; ; attempt++) {
|
|
1213
|
+
const answer = await search(subsets);
|
|
1214
|
+
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
1215
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
1216
|
+
throw new NarsilError(
|
|
1217
|
+
ErrorCodes.WORKER_BUSY,
|
|
1218
|
+
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
1219
|
+
{ fieldName, attempts: attempt }
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
subsets = collectAgain();
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1196
1226
|
// src/engine/query/vector.ts
|
|
1197
1227
|
function oversampleOf(vectorConfig) {
|
|
1198
1228
|
const oversample = vectorConfig.oversample;
|
|
@@ -1217,8 +1247,36 @@ function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
|
1217
1247
|
}
|
|
1218
1248
|
return Math.min(reach + 1, RESULT_WINDOW);
|
|
1219
1249
|
}
|
|
1250
|
+
function filteredSearchOf(params, context) {
|
|
1251
|
+
const filters = params.filters;
|
|
1252
|
+
if (!filters) return void 0;
|
|
1253
|
+
const { manager, config, partitionIds, patternWork } = context;
|
|
1254
|
+
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
1255
|
+
return { first: collectAgain(), collectAgain };
|
|
1256
|
+
}
|
|
1257
|
+
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered) {
|
|
1258
|
+
const options = {
|
|
1259
|
+
metric: vectorConfig.metric ?? "cosine",
|
|
1260
|
+
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1261
|
+
efSearch: vectorConfig.efSearch,
|
|
1262
|
+
oversample: oversampleOf(vectorConfig)
|
|
1263
|
+
};
|
|
1264
|
+
if (filtered !== void 0) {
|
|
1265
|
+
return searchCurrentSubsets(
|
|
1266
|
+
vecIndex.fieldName,
|
|
1267
|
+
filtered.first,
|
|
1268
|
+
filtered.collectAgain,
|
|
1269
|
+
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
const filterPartitions = partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
1273
|
+
return vecIndex.searchParallel(
|
|
1274
|
+
queryVector,
|
|
1275
|
+
k,
|
|
1276
|
+
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
1277
|
+
);
|
|
1278
|
+
}
|
|
1220
1279
|
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1221
|
-
const { manager, config, partitionIds } = context;
|
|
1222
1280
|
const vectorConfig = params.vector;
|
|
1223
1281
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1224
1282
|
return { scored: [], totalMatched: 0 };
|
|
@@ -1227,26 +1285,13 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
1227
1285
|
if (!vecIndex) {
|
|
1228
1286
|
return { scored: [], totalMatched: 0 };
|
|
1229
1287
|
}
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
|
|
1234
|
-
if (filterDocIds.size === 0) {
|
|
1235
|
-
return { scored: [], totalMatched: 0 };
|
|
1236
|
-
}
|
|
1237
|
-
} else {
|
|
1238
|
-
filterPartitions = partitionsForVectorSearch(manager, vecIndex, partitionIds);
|
|
1288
|
+
const filtered = filteredSearchOf(params, context);
|
|
1289
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1290
|
+
return { scored: [], totalMatched: 0 };
|
|
1239
1291
|
}
|
|
1240
1292
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
1241
1293
|
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1242
|
-
const outcome = await vecIndex
|
|
1243
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1244
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1245
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1246
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1247
|
-
efSearch: vectorConfig.efSearch,
|
|
1248
|
-
oversample: oversampleOf(vectorConfig)
|
|
1249
|
-
});
|
|
1294
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered);
|
|
1250
1295
|
const scored = vectorResultsToScored(outcome.results);
|
|
1251
1296
|
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1252
1297
|
}
|
|
@@ -1272,34 +1317,21 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1272
1317
|
searchOptionsFor(manager, context.patternWork)
|
|
1273
1318
|
);
|
|
1274
1319
|
}
|
|
1275
|
-
async function vectorLeg(vectorConfig, queryVector, context,
|
|
1276
|
-
const { manager } = context;
|
|
1320
|
+
async function vectorLeg(vectorConfig, queryVector, context, filtered, k) {
|
|
1277
1321
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1278
1322
|
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1279
|
-
const
|
|
1280
|
-
const outcome = await vecIndex.searchParallel(queryVector, k, {
|
|
1281
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1282
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1283
|
-
...filterDocIds !== void 0 ? { filterDocIds } : {},
|
|
1284
|
-
...filterPartitions !== void 0 ? { filterPartitions } : {},
|
|
1285
|
-
efSearch: vectorConfig.efSearch,
|
|
1286
|
-
oversample: oversampleOf(vectorConfig)
|
|
1287
|
-
});
|
|
1323
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered);
|
|
1288
1324
|
return {
|
|
1289
1325
|
scored: vectorResultsToScored(outcome.results),
|
|
1290
1326
|
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
1291
1327
|
};
|
|
1292
1328
|
}
|
|
1293
1329
|
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1294
|
-
const { manager, config } = context;
|
|
1295
1330
|
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
1296
1331
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
1297
|
-
|
|
1298
|
-
if (
|
|
1299
|
-
|
|
1300
|
-
if (filterDocIds.size === 0) {
|
|
1301
|
-
return { scored: [], totalMatched: 0 };
|
|
1302
|
-
}
|
|
1332
|
+
const filtered = filteredSearchOf(params, context);
|
|
1333
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1334
|
+
return { scored: [], totalMatched: 0 };
|
|
1303
1335
|
}
|
|
1304
1336
|
if (!vectorConfig || !vectorConfig.value) {
|
|
1305
1337
|
const textOnly = await textLeg(textOnlyParams, context);
|
|
@@ -1309,7 +1341,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
1309
1341
|
vectorConfig,
|
|
1310
1342
|
new Float32Array(vectorConfig.value),
|
|
1311
1343
|
context,
|
|
1312
|
-
|
|
1344
|
+
filtered,
|
|
1313
1345
|
vectorFetchDepth(limit, offset, cursorDepth)
|
|
1314
1346
|
);
|
|
1315
1347
|
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QY7GC2MW.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
3
3
|
|
|
4
4
|
// src/serialization/constants.ts
|
|
5
5
|
var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
2
|
-
import { loadNativeCore } from './chunk-
|
|
2
|
+
import { loadNativeCore } from './chunk-ES33L3HO.mjs';
|
|
3
3
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
4
4
|
import { MAX_SORT_FIELDS } from './chunk-FGMGUDHX.mjs';
|
|
5
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
7
7
|
import { tmpdir } from 'os';
|
|
8
8
|
import path from 'path';
|
|
@@ -137,6 +137,7 @@ var STATUS_BY_CODE = {
|
|
|
137
137
|
DOC_ALREADY_EXISTS: 409,
|
|
138
138
|
PARTITION_REBALANCING_BACKPRESSURE: 409,
|
|
139
139
|
PARTITION_CAPACITY_EXCEEDED: 409,
|
|
140
|
+
RESTORE_MEMORY_EXCEEDED: 507,
|
|
140
141
|
WORKER_BUSY: 429,
|
|
141
142
|
PARTITION_CORRUPTED: 503,
|
|
142
143
|
WORKER_CRASHED: 503,
|