@delali/narsil 0.3.1-next.129 → 0.3.1-next.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-5D6NJD6C.mjs → chunk-2Y3CO4EQ.mjs} +342 -201
- package/dist/browser/{chunk-WLD6DDIX.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-YAGXZU4W.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-WZBTVTSU.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-XTFJZHQP.mjs → chunk-NSCSR6MA.mjs} +15 -5
- package/dist/browser/{chunk-3NVZD52E.mjs → chunk-UDGB2MWT.mjs} +5 -28
- package/dist/browser/{chunk-II5POT55.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-HCCC6BJB.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-S2VDU2JE.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-BZ7WWEC4.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-PL4TTFJE.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-RTUCHI2L.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-IMU7G5NU.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-BTjv-tK8.d.ts → capabilities-B6WxrWxt.d.ts} +3 -10
- package/dist/{chunk-3XHFVYIQ.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-37RTD6KD.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-K76F5YYX.mjs → chunk-2MKX2SCO.mjs} +219 -58
- package/dist/{chunk-AN3CUMPZ.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-OPO5SE4J.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-PTIJAZWL.mjs → chunk-4GDLFDPZ.mjs} +1 -1
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-TY64A3ZW.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-3LJVM6XM.mjs → chunk-5W7AEVTH.mjs} +2 -2
- package/dist/{chunk-PCRWCOZC.mjs → chunk-6M6SDH3S.mjs} +43 -23
- package/dist/{chunk-MGD7XETB.mjs → chunk-6SSVIW7Y.mjs} +19 -3
- package/dist/{chunk-5LC5VMDQ.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-4XEVROSP.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-WI5Q3L43.mjs → chunk-CZ7ISLXA.mjs} +2 -0
- package/dist/{chunk-RC27KF7F.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-F4UKMTKM.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-CTF6WFTX.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-DX2FI4LU.mjs → chunk-FQBFZAWG.mjs} +3 -1
- package/dist/{chunk-XTER67YM.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-WQYB4TNR.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-2HQQ7Y5D.mjs → chunk-JT3YZGCE.mjs} +2 -2
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-WUIDPBSY.mjs → chunk-K77FJXKS.mjs} +17 -7
- package/dist/{chunk-CSSCLRGY.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-44TB6AIZ.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-EZIGAOUJ.mjs → chunk-OGLV4I3L.mjs} +6 -5
- package/dist/{chunk-FRSWYMXH.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-AFOZR4ZQ.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/{chunk-TB7YMDD5.mjs → chunk-RCMNL7RI.mjs} +650 -100
- package/dist/{chunk-Y4NOHICF.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-RYH3PASD.mjs → chunk-RWESFSJI.mjs} +21 -11
- package/dist/{chunk-QQS4QZQA.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-K73PZTBR.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-43Q7HAI2.mjs → chunk-T4OFG4IQ.mjs} +4 -4
- package/dist/{chunk-2YOAGAGF.mjs → chunk-TGHNXN7W.mjs} +59 -30
- package/dist/{chunk-J4LRE5Q7.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-XRXBCOA4.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-Y4US5EGH.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-5VSTAJET.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-M6MBSVIJ.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-BOcveV7A.d.ts → client-gNAfaHcE.d.ts} +3 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +2 -0
- package/dist/{config-CPmIY3aH.d.ts → config-CmYJO34X.d.ts} +3 -3
- package/dist/{events-pM0lna7M.d.ts → config-D1cDv3sE.d.ts} +431 -3
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -8
- package/dist/distribution.mjs +85 -45
- package/dist/{durability-wiring-6R26AJDI.mjs → durability-wiring-S7OX2MAI.mjs} +27 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-JjRAXpT3.d.ts → engine-m_Ddr1bz.d.ts} +2 -2
- package/dist/{index-DkQ2DlgR.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -18
- package/dist/{long-query-thread-6VLA6YJI.mjs → long-query-thread-BSL25PD6.mjs} +17 -15
- package/dist/{module-S4WGMFNW.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -21
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +2 -0
- package/dist/{rerank-V4J4MIWR.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-PUGWCPCX.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-TBFBQQQO.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-DGAP_8HJ.d.ts → schema-BTXbHCqA.d.ts} +8 -5
- package/dist/server.d.ts +198 -11
- package/dist/server.mjs +30 -19
- package/dist/{snapshot-PUVSMV7E.mjs → snapshot-DOQLGGEX.mjs} +44 -29
- package/dist/synonyms.mjs +4 -3
- package/dist/tasks-Co8HfRD7.d.ts +704 -0
- package/dist/{thread-TA7QQBWP.mjs → thread-SKO7CN3R.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-TGKYT3C4.mjs → write-ZKR7SNRT.mjs} +22 -20
- package/openapi.json +5215 -0
- package/package.json +16 -8
- package/dist/config-C3YXk5Rn.d.ts +0 -412
- package/dist/direct-executor-ULA6V5HC.mjs +0 -26
- package/dist/tasks-txGvNz6h.d.ts +0 -302
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-AIIJDWMG.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
3
3
|
|
|
4
4
|
// src/vector/vector-index/dead-bits.ts
|
|
5
5
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-2Y3CO4EQ.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
2
|
-
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
3
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-M5UAAXGU.mjs';
|
|
2
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-2Y3CO4EQ.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-D7WQF6DV.mjs';
|
|
4
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-AIIJDWMG.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
6
6
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/engine/durability-metadata.ts
|
|
@@ -30,6 +30,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
30
30
|
if (config.analysisRevision !== void 0) {
|
|
31
31
|
metadata.analysisRevision = config.analysisRevision;
|
|
32
32
|
}
|
|
33
|
+
if (config.arrayPositionGap !== void 0) {
|
|
34
|
+
metadata.arrayPositionGap = config.arrayPositionGap;
|
|
35
|
+
}
|
|
33
36
|
if (config.tokenizer !== void 0) {
|
|
34
37
|
metadata.tokenizer = config.tokenizer;
|
|
35
38
|
}
|
|
@@ -89,6 +92,7 @@ function metadataConfigOf(entry) {
|
|
|
89
92
|
...embedding !== void 0 ? { embedding } : {},
|
|
90
93
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
91
94
|
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
95
|
+
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
92
96
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
93
97
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
94
98
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -321,6 +325,9 @@ function metadataToWire(meta) {
|
|
|
321
325
|
if (meta.analysisRevision !== void 0) {
|
|
322
326
|
wire.analysis_revision = meta.analysisRevision;
|
|
323
327
|
}
|
|
328
|
+
if (meta.arrayPositionGap !== void 0) {
|
|
329
|
+
wire.array_position_gap = meta.arrayPositionGap;
|
|
330
|
+
}
|
|
324
331
|
if (meta.tokenizer !== void 0) {
|
|
325
332
|
wire.tokenizer = meta.tokenizer;
|
|
326
333
|
}
|
|
@@ -413,6 +420,9 @@ function wireToMetadata(raw) {
|
|
|
413
420
|
if (typeof raw.analysis_revision === "string") {
|
|
414
421
|
meta.analysisRevision = raw.analysis_revision;
|
|
415
422
|
}
|
|
423
|
+
if (typeof raw.array_position_gap === "number") {
|
|
424
|
+
meta.arrayPositionGap = raw.array_position_gap;
|
|
425
|
+
}
|
|
416
426
|
if (typeof raw.tokenizer === "string") {
|
|
417
427
|
meta.tokenizer = raw.tokenizer;
|
|
418
428
|
}
|
|
@@ -32,6 +32,8 @@ var ErrorCodes = {
|
|
|
32
32
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
33
33
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
34
34
|
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
35
|
+
SEARCH_INVALID_PHRASE: "SEARCH_INVALID_PHRASE",
|
|
36
|
+
SEARCH_PHRASE_NOT_READY: "SEARCH_PHRASE_NOT_READY",
|
|
35
37
|
BM25_NOT_READY: "BM25_NOT_READY",
|
|
36
38
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
37
39
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
@@ -171,6 +173,8 @@ var PREFIX_MAX_EXPANSIONS = 50;
|
|
|
171
173
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
172
174
|
var MAX_TOLERANCE = 10;
|
|
173
175
|
var MAX_PREFIX_LENGTH = 1024;
|
|
176
|
+
var MAX_PHRASE_WORDS = 32;
|
|
177
|
+
var MAX_QUERY_PHRASES = 32;
|
|
174
178
|
var DEFAULT_MAX_PER_GROUP = 1;
|
|
175
179
|
var DEFAULT_RANK_CONSTANT = 60;
|
|
176
180
|
var MIN_RANK_CONSTANT = 1;
|
|
@@ -181,31 +185,4 @@ var MIN_HYBRID_DEPTH = 1;
|
|
|
181
185
|
var MAX_HYBRID_DEPTH = 1e4;
|
|
182
186
|
var HYBRID_RESCORE_WINDOW = 10;
|
|
183
187
|
|
|
184
|
-
|
|
185
|
-
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
186
|
-
var BATCH_CHUNK_SIZE = 1e3;
|
|
187
|
-
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
188
|
-
var DEFAULT_OFFSET = 0;
|
|
189
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
190
|
-
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
191
|
-
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
192
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
193
|
-
var MAX_DOC_ID_LENGTH = 512;
|
|
194
|
-
var EXPORT_CHUNK_BYTES = 1048576;
|
|
195
|
-
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
196
|
-
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
197
|
-
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
198
|
-
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
199
|
-
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
200
|
-
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
201
|
-
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
202
|
-
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
203
|
-
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
204
|
-
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
205
|
-
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
206
|
-
var MAX_SUGGEST_LIMIT = 100;
|
|
207
|
-
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
208
|
-
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
209
|
-
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
210
|
-
|
|
211
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, ClientErrorCodes, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, ErrorCodes, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NarsilError, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW, STATISTICS_BROADCAST_INTERVAL_MS, ServerErrorCodes, createNarsilError, isNarsilError };
|
|
188
|
+
export { BM25_DISPLAYED_NAME_LENGTH, ClientErrorCodes, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, ErrorCodes, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, NarsilError, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW, ServerErrorCodes, createNarsilError, isNarsilError };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-
|
|
2
|
-
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-
|
|
3
|
-
import {
|
|
1
|
+
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-D7WQF6DV.mjs';
|
|
2
|
+
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-AIIJDWMG.mjs';
|
|
3
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
4
5
|
|
|
5
6
|
// src/vector/shared-buffers/growable.ts
|
|
6
7
|
function sharedMemoryAvailable() {
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { DEFAULT_PAGE_SIZE } from './chunk-UDGB2MWT.mjs';
|
|
2
|
+
|
|
3
|
+
// src/engine/constants.ts
|
|
4
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
5
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
6
|
+
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
|
+
var DEFAULT_OFFSET = 0;
|
|
8
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
9
|
+
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
10
|
+
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
11
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
12
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
13
|
+
var EXPORT_CHUNK_BYTES = 1048576;
|
|
14
|
+
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
15
|
+
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
16
|
+
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
17
|
+
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
18
|
+
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
19
|
+
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
20
|
+
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
21
|
+
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
22
|
+
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
23
|
+
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
24
|
+
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
25
|
+
var MAX_SUGGEST_LIMIT = 100;
|
|
26
|
+
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
27
|
+
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
28
|
+
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
29
|
+
|
|
30
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS };
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
5
|
-
import
|
|
6
|
-
import {
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-NSCSR6MA.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-M5UAAXGU.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-2Y3CO4EQ.mjs';
|
|
5
|
+
import './chunk-26A5JFT2.mjs';
|
|
6
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-D7WQF6DV.mjs';
|
|
7
|
+
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-AIIJDWMG.mjs';
|
|
7
8
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
-
import {
|
|
9
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-VEGHFP2P.mjs';
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
9
11
|
import { decode } from '@msgpack/msgpack';
|
|
10
12
|
|
|
11
13
|
// src/persistence/durability/node-fs.browser.ts
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-
|
|
3
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
4
|
-
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-
|
|
5
|
-
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-
|
|
6
|
-
import
|
|
2
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-VBBLTD7O.mjs';
|
|
3
|
+
export { releaseLocksHeldBy } from './chunk-VBBLTD7O.mjs';
|
|
4
|
+
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-D7WQF6DV.mjs';
|
|
5
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-AIIJDWMG.mjs';
|
|
6
|
+
import './chunk-VEGHFP2P.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-UDGB2MWT.mjs';
|
|
7
8
|
|
|
8
9
|
// src/core/heap.ts
|
|
9
10
|
function createMaxHeap(compare) {
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
import { createBitSet, bitsetIsEmpty, GrowableUint32, PHRASE_ROWS_START, bitsetHas, bitsetSet, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-26A5JFT2.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-UDGB2MWT.mjs';
|
|
3
|
+
|
|
4
|
+
// src/core/partition/phrase-matching.ts
|
|
5
|
+
var UINT32_BITS = 32;
|
|
6
|
+
var WorkTally = class {
|
|
7
|
+
constructor(meter) {
|
|
8
|
+
this.meter = meter;
|
|
9
|
+
}
|
|
10
|
+
pending = 0;
|
|
11
|
+
add(units) {
|
|
12
|
+
this.pending += units;
|
|
13
|
+
if (this.meter !== void 0 && this.pending >= this.meter.flushUnits) this.flush();
|
|
14
|
+
}
|
|
15
|
+
tickClock(units) {
|
|
16
|
+
this.meter?.clock?.tick(units);
|
|
17
|
+
}
|
|
18
|
+
flush() {
|
|
19
|
+
const units = this.pending;
|
|
20
|
+
this.pending = 0;
|
|
21
|
+
this.meter?.add(units);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
function placeRowsByDigit(docIds, from, to, shift, starts, tally) {
|
|
25
|
+
const count = from.length;
|
|
26
|
+
const mask = starts.length - 1;
|
|
27
|
+
starts.fill(0);
|
|
28
|
+
for (let row = 0; row < count; row++) starts[docIds[row] >>> shift & mask]++;
|
|
29
|
+
let next = 0;
|
|
30
|
+
for (let digit = 0; digit < starts.length; digit++) {
|
|
31
|
+
const held = starts[digit];
|
|
32
|
+
starts[digit] = next;
|
|
33
|
+
next += held;
|
|
34
|
+
}
|
|
35
|
+
for (let begin = 0; begin < count; begin += PHRASE_SORT_CLOCK_ROWS) {
|
|
36
|
+
const end = Math.min(count, begin + PHRASE_SORT_CLOCK_ROWS);
|
|
37
|
+
for (let at = begin; at < end; at++) {
|
|
38
|
+
const row = from[at];
|
|
39
|
+
to[starts[docIds[row] >>> shift & mask]++] = row;
|
|
40
|
+
}
|
|
41
|
+
tally.tickClock(end - begin);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function sortedRowOrder(docIds, count, tally) {
|
|
45
|
+
let order = new Uint32Array(count);
|
|
46
|
+
for (let row = 0; row < count; row++) order[row] = row;
|
|
47
|
+
let spare = new Uint32Array(count);
|
|
48
|
+
const starts = new Uint32Array(1 << PHRASE_SORT_RADIX_BITS);
|
|
49
|
+
for (let shift = 0; shift < UINT32_BITS; shift += PHRASE_SORT_RADIX_BITS) {
|
|
50
|
+
placeRowsByDigit(docIds, order, spare, shift, starts, tally);
|
|
51
|
+
const placed = spare;
|
|
52
|
+
spare = order;
|
|
53
|
+
order = placed;
|
|
54
|
+
}
|
|
55
|
+
return order;
|
|
56
|
+
}
|
|
57
|
+
function tokenRowsOf(list, tally) {
|
|
58
|
+
const store = list.positionStore ?? null;
|
|
59
|
+
if (store === null) return null;
|
|
60
|
+
const count = list.length;
|
|
61
|
+
const docIds = list.docIds;
|
|
62
|
+
if (list.ordered) {
|
|
63
|
+
return { list, keys: docIds, order: null, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
64
|
+
}
|
|
65
|
+
tally.add(count);
|
|
66
|
+
const order = sortedRowOrder(docIds, count, tally);
|
|
67
|
+
const keys = new Uint32Array(count);
|
|
68
|
+
for (let row = 0; row < count; row++) keys[row] = docIds[order[row]];
|
|
69
|
+
return { list, keys, order, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
70
|
+
}
|
|
71
|
+
function resolvePhrase(index, phrase, sharedRows, tally) {
|
|
72
|
+
const entryRows = [];
|
|
73
|
+
const rows = [];
|
|
74
|
+
for (const token of phrase.tokens) {
|
|
75
|
+
let found = sharedRows.get(token);
|
|
76
|
+
if (found === void 0) {
|
|
77
|
+
const list = index.lookupReadOnce?.(token) ?? index.lookup(token);
|
|
78
|
+
if (list === void 0) return null;
|
|
79
|
+
const built = tokenRowsOf(list, tally);
|
|
80
|
+
if (built === null) return null;
|
|
81
|
+
found = built;
|
|
82
|
+
sharedRows.set(token, found);
|
|
83
|
+
}
|
|
84
|
+
if (!rows.includes(found)) rows.push(found);
|
|
85
|
+
entryRows.push(found);
|
|
86
|
+
}
|
|
87
|
+
let leadEntry = 0;
|
|
88
|
+
for (let entry = 1; entry < entryRows.length; entry++) {
|
|
89
|
+
if (entryRows[entry].count < entryRows[leadEntry].count) leadEntry = entry;
|
|
90
|
+
}
|
|
91
|
+
return { phrase, rows, entryRows, lead: entryRows[leadEntry], leadOffset: phrase.offsets[leadEntry] };
|
|
92
|
+
}
|
|
93
|
+
function firstRowFor(rows, doc) {
|
|
94
|
+
const { keys, count } = rows;
|
|
95
|
+
let low = rows.cursor < count && keys[rows.cursor] <= doc ? rows.cursor : 0;
|
|
96
|
+
let high = low;
|
|
97
|
+
let step = 1;
|
|
98
|
+
while (high < count && keys[high] < doc) {
|
|
99
|
+
low = high + 1;
|
|
100
|
+
high += step;
|
|
101
|
+
step *= 2;
|
|
102
|
+
}
|
|
103
|
+
if (high > count) high = count;
|
|
104
|
+
while (low < high) {
|
|
105
|
+
const middle = low + high >>> 1;
|
|
106
|
+
if (keys[middle] < doc) low = middle + 1;
|
|
107
|
+
else high = middle;
|
|
108
|
+
}
|
|
109
|
+
rows.cursor = low;
|
|
110
|
+
return low;
|
|
111
|
+
}
|
|
112
|
+
function positionRowFor(rows, doc, field) {
|
|
113
|
+
const { keys, order, fields, count } = rows;
|
|
114
|
+
for (let at = firstRowFor(rows, doc); at < count && keys[at] === doc; at++) {
|
|
115
|
+
const row = order === null ? at : order[at];
|
|
116
|
+
if (fields[row] === field) return row;
|
|
117
|
+
}
|
|
118
|
+
return -1;
|
|
119
|
+
}
|
|
120
|
+
function countPhraseStarts(resolved, leadRow, cursors, ends, tally) {
|
|
121
|
+
const { entryRows, lead, leadOffset, phrase } = resolved;
|
|
122
|
+
const offsets = phrase.offsets;
|
|
123
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
124
|
+
const rows = entryRows[entry];
|
|
125
|
+
const start = rows.list.positionStartAt?.(rows.positionRow) ?? 0;
|
|
126
|
+
cursors[entry] = start;
|
|
127
|
+
ends[entry] = start + (rows.list.positionCountAt?.(rows.positionRow) ?? 0);
|
|
128
|
+
}
|
|
129
|
+
const leadStore = lead.store;
|
|
130
|
+
const leadStart = lead.list.positionStartAt?.(leadRow) ?? 0;
|
|
131
|
+
const leadEnd = leadStart + (lead.list.positionCountAt?.(leadRow) ?? 0);
|
|
132
|
+
let starts = 0;
|
|
133
|
+
for (let at = leadStart; at < leadEnd; at++) {
|
|
134
|
+
const anchor = leadStore[at] - leadOffset;
|
|
135
|
+
if (anchor < 0) continue;
|
|
136
|
+
let holdsEveryEntry = true;
|
|
137
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
138
|
+
const store = entryRows[entry].store;
|
|
139
|
+
const wanted = anchor + offsets[entry];
|
|
140
|
+
let cursor = cursors[entry];
|
|
141
|
+
const end = ends[entry];
|
|
142
|
+
let compared = 1;
|
|
143
|
+
while (cursor < end && store[cursor] < wanted) {
|
|
144
|
+
cursor++;
|
|
145
|
+
compared++;
|
|
146
|
+
}
|
|
147
|
+
cursors[entry] = cursor;
|
|
148
|
+
tally.add(compared);
|
|
149
|
+
if (cursor === end) {
|
|
150
|
+
return starts;
|
|
151
|
+
}
|
|
152
|
+
if (store[cursor] !== wanted) {
|
|
153
|
+
holdsEveryEntry = false;
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (holdsEveryEntry) starts++;
|
|
158
|
+
}
|
|
159
|
+
return starts;
|
|
160
|
+
}
|
|
161
|
+
function collectPhraseRows(resolved, searchable, allowed, capacity, keepRows, tally) {
|
|
162
|
+
const hits = createBitSet(capacity);
|
|
163
|
+
const collected = {
|
|
164
|
+
phrase: resolved.phrase,
|
|
165
|
+
lists: resolved.entryRows.map((rows2) => rows2.list),
|
|
166
|
+
docIds: new GrowableUint32(PHRASE_ROWS_START),
|
|
167
|
+
fieldIndices: new GrowableUint32(PHRASE_ROWS_START),
|
|
168
|
+
frequencies: new GrowableUint32(PHRASE_ROWS_START)
|
|
169
|
+
};
|
|
170
|
+
const { lead, rows } = resolved;
|
|
171
|
+
const leadList = lead.list;
|
|
172
|
+
const docIds = leadList.docIds;
|
|
173
|
+
const fields = leadList.fieldNameIndices;
|
|
174
|
+
const deleted = leadList.deletedDocs;
|
|
175
|
+
const hasDeleted = deleted.size > 0;
|
|
176
|
+
const cursors = new Int32Array(resolved.entryRows.length);
|
|
177
|
+
const ends = new Int32Array(resolved.entryRows.length);
|
|
178
|
+
for (const each of rows) each.cursor = 0;
|
|
179
|
+
for (let leadRow = 0; leadRow < lead.count; leadRow++) {
|
|
180
|
+
tally.add(1);
|
|
181
|
+
const doc = docIds[leadRow];
|
|
182
|
+
const field = fields[leadRow];
|
|
183
|
+
if (searchable[field] === 0) continue;
|
|
184
|
+
if (hasDeleted && deleted.has(doc)) continue;
|
|
185
|
+
if (allowed !== void 0 && !bitsetHas(allowed, doc)) continue;
|
|
186
|
+
let holdsEveryToken = true;
|
|
187
|
+
for (const each of rows) {
|
|
188
|
+
if (each === lead) {
|
|
189
|
+
each.positionRow = leadRow;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const row = positionRowFor(each, doc, field);
|
|
193
|
+
if (row === -1) {
|
|
194
|
+
holdsEveryToken = false;
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
each.positionRow = row;
|
|
198
|
+
}
|
|
199
|
+
if (!holdsEveryToken) continue;
|
|
200
|
+
const starts = countPhraseStarts(resolved, leadRow, cursors, ends, tally);
|
|
201
|
+
if (starts === 0) continue;
|
|
202
|
+
bitsetSet(hits, doc);
|
|
203
|
+
if (keepRows) {
|
|
204
|
+
collected.docIds.push(doc);
|
|
205
|
+
collected.fieldIndices.push(field);
|
|
206
|
+
collected.frequencies.push(starts);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return { hits, rows: collected };
|
|
210
|
+
}
|
|
211
|
+
function matchPhrases(index, phrases, searchable, filterBitset, capacity, meter, keepRows) {
|
|
212
|
+
const tally = new WorkTally(meter);
|
|
213
|
+
const sharedRows = /* @__PURE__ */ new Map();
|
|
214
|
+
const resolved = [];
|
|
215
|
+
for (const phrase of phrases) {
|
|
216
|
+
const found = resolvePhrase(index, phrase, sharedRows, tally);
|
|
217
|
+
if (found === null) {
|
|
218
|
+
tally.flush();
|
|
219
|
+
return { matched: createBitSet(capacity), rows: [] };
|
|
220
|
+
}
|
|
221
|
+
resolved.push(found);
|
|
222
|
+
}
|
|
223
|
+
resolved.sort((first, second) => first.lead.count - second.lead.count);
|
|
224
|
+
const rows = [];
|
|
225
|
+
let allowed = filterBitset;
|
|
226
|
+
for (const each of resolved) {
|
|
227
|
+
const collected = collectPhraseRows(each, searchable, allowed, capacity, keepRows, tally);
|
|
228
|
+
allowed = collected.hits;
|
|
229
|
+
rows.push(collected.rows);
|
|
230
|
+
if (bitsetIsEmpty(allowed)) break;
|
|
231
|
+
}
|
|
232
|
+
tally.flush();
|
|
233
|
+
return { matched: allowed ?? createBitSet(capacity), rows };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// src/search/phrase-analysis.ts
|
|
237
|
+
function phraseKeyOf(tokens, offsets) {
|
|
238
|
+
const slots = [];
|
|
239
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
240
|
+
const gap = index === 0 ? 0 : offsets[index] - offsets[index - 1] - 1;
|
|
241
|
+
for (let missing = 0; missing < gap; missing++) slots.push("?");
|
|
242
|
+
slots.push(tokens[index]);
|
|
243
|
+
}
|
|
244
|
+
return `"${slots.join(" ")}"`;
|
|
245
|
+
}
|
|
246
|
+
function identityOf(tokens, offsets) {
|
|
247
|
+
return JSON.stringify([tokens, offsets]);
|
|
248
|
+
}
|
|
249
|
+
function analyseQueryPhrases(parts, analyseText) {
|
|
250
|
+
const wordParts = [];
|
|
251
|
+
const phrases = [];
|
|
252
|
+
const seen = /* @__PURE__ */ new Set();
|
|
253
|
+
let keptPhraseCount = 0;
|
|
254
|
+
let largestPhraseSize = 0;
|
|
255
|
+
let largestTokenCount = 0;
|
|
256
|
+
let wordsAfterLastPhrase = "";
|
|
257
|
+
for (let index = 0; index < parts.length; index++) {
|
|
258
|
+
const part = parts[index];
|
|
259
|
+
if (index % 2 === 0) {
|
|
260
|
+
wordParts.push(part);
|
|
261
|
+
wordsAfterLastPhrase = `${wordsAfterLastPhrase} ${part}`;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (part.trim().length === 0) continue;
|
|
265
|
+
const entries = analyseText(part);
|
|
266
|
+
if (entries.length === 0) continue;
|
|
267
|
+
const ordered = [...entries].sort((first, second) => first.position - second.position);
|
|
268
|
+
const firstPosition = ordered[0].position;
|
|
269
|
+
const size = ordered[ordered.length - 1].position - firstPosition + 1;
|
|
270
|
+
keptPhraseCount++;
|
|
271
|
+
wordsAfterLastPhrase = "";
|
|
272
|
+
if (size > largestPhraseSize) largestPhraseSize = size;
|
|
273
|
+
if (ordered.length > largestTokenCount) largestTokenCount = ordered.length;
|
|
274
|
+
const tokens = ordered.map((entry) => entry.token);
|
|
275
|
+
const offsets = ordered.map((entry) => entry.position - firstPosition);
|
|
276
|
+
const identity = identityOf(tokens, offsets);
|
|
277
|
+
if (seen.has(identity)) continue;
|
|
278
|
+
seen.add(identity);
|
|
279
|
+
phrases.push({ key: phraseKeyOf(tokens, offsets), tokens, offsets });
|
|
280
|
+
}
|
|
281
|
+
const endsWithPhrase = keptPhraseCount > 0 && (wordsAfterLastPhrase.trim().length === 0 || analyseText(wordsAfterLastPhrase).length === 0);
|
|
282
|
+
return {
|
|
283
|
+
words: wordParts.join(" "),
|
|
284
|
+
phrases,
|
|
285
|
+
keptPhraseCount,
|
|
286
|
+
largestPhraseSize,
|
|
287
|
+
largestTokenCount,
|
|
288
|
+
endsWithPhrase
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function withWordsOnly(params, phrases) {
|
|
292
|
+
return { ...params, term: phrases.words, prefix: phrases.endsWithPhrase ? false : params.prefix };
|
|
293
|
+
}
|
|
294
|
+
function refusePhraseUntilRebuilt(indexName, rebuildMode) {
|
|
295
|
+
const remedy = rebuildMode === "manual" ? "This engine leaves the rebuild to its caller, so somebody has to start it with rebuildAnalysis" : "Try the query again once the rebuild finishes";
|
|
296
|
+
throw new NarsilError(
|
|
297
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
298
|
+
`Index "${indexName}" holds array fields whose terms need a rebuild, so it cannot match a phrase yet. ${remedy}`,
|
|
299
|
+
{ indexName, rebuild: rebuildMode }
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
function requirePhrasesWithinLimits(phrases, trackPositions) {
|
|
303
|
+
if (phrases.keptPhraseCount === 0) return;
|
|
304
|
+
if (phrases.keptPhraseCount > MAX_QUERY_PHRASES) {
|
|
305
|
+
throw new NarsilError(
|
|
306
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
307
|
+
`A query takes at most ${MAX_QUERY_PHRASES} phrases, while this query holds ${phrases.keptPhraseCount}`,
|
|
308
|
+
{ limit: MAX_QUERY_PHRASES, phrases: phrases.keptPhraseCount }
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
if (phrases.largestPhraseSize > MAX_PHRASE_WORDS || phrases.largestTokenCount > MAX_PHRASE_WORDS) {
|
|
312
|
+
throw new NarsilError(
|
|
313
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
314
|
+
`A phrase takes at most ${MAX_PHRASE_WORDS} words, counting the stop words between its first and last word, while this query holds one of ${Math.max(phrases.largestPhraseSize, phrases.largestTokenCount)}`,
|
|
315
|
+
{ limit: MAX_PHRASE_WORDS, words: Math.max(phrases.largestPhraseSize, phrases.largestTokenCount) }
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
if (!trackPositions) {
|
|
319
|
+
throw new NarsilError(
|
|
320
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
321
|
+
"This index stores no word positions because its trackPositions setting is off, so it cannot match a phrase. Remove the quotes, or create the index with trackPositions on",
|
|
322
|
+
{ trackPositions: false }
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export { analyseQueryPhrases, matchPhrases, refusePhraseUntilRebuilt, requirePhrasesWithinLimits, withWordsOnly };
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
-
import {
|
|
2
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-VEGHFP2P.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-UDGB2MWT.mjs';
|
|
3
4
|
|
|
4
5
|
// src/search/hybrid-depth.ts
|
|
5
6
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { collectAndReturnPages, vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
5
|
-
import
|
|
6
|
-
import {
|
|
1
|
+
import { collectAndReturnPages, vectorsPerFile } from './chunk-VBBLTD7O.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-NSCSR6MA.mjs';
|
|
3
|
+
import './chunk-M5UAAXGU.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2Y3CO4EQ.mjs';
|
|
5
|
+
import './chunk-26A5JFT2.mjs';
|
|
6
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-D7WQF6DV.mjs';
|
|
7
|
+
import { MAX_FILE_BYTES } from './chunk-AIIJDWMG.mjs';
|
|
7
8
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
-
import
|
|
9
|
+
import './chunk-VEGHFP2P.mjs';
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
9
11
|
import { decode } from '@msgpack/msgpack';
|
|
10
12
|
|
|
11
13
|
// src/persistence/archive/memory-store.ts
|
|
@@ -733,7 +735,15 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
733
735
|
}
|
|
734
736
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
735
737
|
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
736
|
-
return {
|
|
738
|
+
return {
|
|
739
|
+
config,
|
|
740
|
+
language,
|
|
741
|
+
embeddingAdapter,
|
|
742
|
+
adapterName,
|
|
743
|
+
storedRevision,
|
|
744
|
+
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
745
|
+
synonyms
|
|
746
|
+
};
|
|
737
747
|
}
|
|
738
748
|
function restoredSynonyms(raw, config, language, reject) {
|
|
739
749
|
if (raw === void 0 || raw === null) return null;
|
|
@@ -764,6 +774,7 @@ function restoredRegistryEntry(prepared, manager, deps) {
|
|
|
764
774
|
documentCount: manager.countDocuments(),
|
|
765
775
|
partitionCount: manager.partitionCount,
|
|
766
776
|
analysisRevision: prepared.storedRevision,
|
|
777
|
+
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
767
778
|
synonyms: prepared.synonyms?.encoded ?? null
|
|
768
779
|
};
|
|
769
780
|
}
|
|
@@ -846,7 +857,7 @@ function activate(indexName, restored, entry, deps) {
|
|
|
846
857
|
restored.detached.attach();
|
|
847
858
|
deps.clearAnalysisStale(indexName);
|
|
848
859
|
const { prepared } = restored;
|
|
849
|
-
if (prepared.storedRevision !== prepared.language.revision) {
|
|
860
|
+
if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
|
|
850
861
|
deps.markAnalysisStale({
|
|
851
862
|
indexName,
|
|
852
863
|
language: prepared.language.name,
|
|
@@ -1131,7 +1142,11 @@ async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
|
1131
1142
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
1132
1143
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
1133
1144
|
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
1134
|
-
const analysedNow = {
|
|
1145
|
+
const analysedNow = {
|
|
1146
|
+
...prepared,
|
|
1147
|
+
storedRevision: prepared.language.revision,
|
|
1148
|
+
storedArrayPositionGap: ARRAY_POSITION_GAP
|
|
1149
|
+
};
|
|
1135
1150
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
1136
1151
|
try {
|
|
1137
1152
|
await installRestoredIndex(
|