@delali/narsil 0.3.1-next.151 → 0.3.1-next.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
- package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
- package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
- package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
- package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
- package/dist/browser/chunk-3I27KXQD.mjs +34 -0
- package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
- package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
- package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
- package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
- package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
- package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
- package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
- package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
- package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
- package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
- package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
- package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
- package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
- package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
- package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
- package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
- package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
- package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
- package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
- package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
- package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
- package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
- package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
- package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
- package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
- package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
- package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
- package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
- package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
- package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
- package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
- package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
- package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
- package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
- package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
- package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
- package/dist/chunk-MATMVH65.mjs +9 -0
- package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
- package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
- package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
- package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
- package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
- package/dist/chunk-VD7J552I.mjs +28 -0
- package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
- package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
- package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
- package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
- package/dist/distribution/coordinator/etcd.d.ts +9 -0
- package/dist/distribution/coordinator/etcd.mjs +267 -106
- package/dist/distribution.mjs +23 -21
- package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
- package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +1 -1
- package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
- package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
- package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
- package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
- package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
- package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
- package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
- package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
- package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
- package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
- package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
- package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
- package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
- package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
- package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
- package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
- package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
- package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
- package/dist/server.mjs +5 -5
- package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
- package/package.json +7 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-
|
|
1
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-RBEHLIKX.mjs';
|
|
2
2
|
import './chunk-VP6XRMWR.mjs';
|
|
3
|
-
import './chunk-
|
|
3
|
+
import './chunk-2PNYKG2D.mjs';
|
|
4
4
|
import './chunk-6F6USYII.mjs';
|
|
5
|
-
import './chunk-
|
|
5
|
+
import './chunk-UGVUW27S.mjs';
|
|
6
6
|
import './chunk-3HV7M5OJ.mjs';
|
|
7
7
|
import './chunk-KAKQ74LS.mjs';
|
|
8
8
|
import './chunk-HJSN7UYG.mjs';
|
|
@@ -15,7 +15,7 @@ import './chunk-2HJWQPR6.mjs';
|
|
|
15
15
|
import './chunk-X5CV2E2B.mjs';
|
|
16
16
|
import './chunk-TGP5M5R7.mjs';
|
|
17
17
|
import './chunk-U2BKYEC7.mjs';
|
|
18
|
-
import './chunk-
|
|
18
|
+
import './chunk-DGRM4EZZ.mjs';
|
|
19
19
|
import './chunk-QHPYS427.mjs';
|
|
20
20
|
import './chunk-QYRNLLLI.mjs';
|
|
21
21
|
import './chunk-ZXNQEW43.mjs';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-NITCPDFL.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
2
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-2A2IPRC3.mjs';
|
|
2
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-KWYQ6KX4.mjs';
|
|
3
3
|
import './chunk-MPK76GVS.mjs';
|
|
4
4
|
import './chunk-JPTQZWJQ.mjs';
|
|
5
5
|
import './chunk-26A5JFT2.mjs';
|
package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-VBX2TI5V.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { extractVectorFieldsFromSchema, readHeapStatistics, readResidentMemory } from './chunk-
|
|
1
|
+
import { extractVectorFieldsFromSchema, readHeapStatistics, readResidentMemory } from './chunk-KWYQ6KX4.mjs';
|
|
2
2
|
import { collectAndReturnPages } from './chunk-OIZS5KIX.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
4
4
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// src/vector/vector-index/constants.ts
|
|
2
|
+
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
3
|
+
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
4
|
+
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
5
|
+
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
6
|
+
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
7
|
+
var BUILD_CHUNK_SIZE = 100;
|
|
8
|
+
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
9
|
+
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
10
|
+
var SEARCH_REFILL_RESERVE = 64;
|
|
11
|
+
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
12
|
+
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
13
|
+
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
14
|
+
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
15
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
16
|
+
var REBUILD_REMOVED_RATIO = 0.2;
|
|
17
|
+
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
18
|
+
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
19
|
+
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
20
|
+
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
21
|
+
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
22
|
+
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
23
|
+
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
24
|
+
var OSQ4_MIN_DIMENSION = 384;
|
|
25
|
+
var DEFAULT_VECTOR_METRIC = "cosine";
|
|
26
|
+
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
27
|
+
|
|
28
|
+
// src/vector/vector-index/default-quantization.ts
|
|
29
|
+
function defaultQuantizationFor(dimension) {
|
|
30
|
+
if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
|
|
31
|
+
return "osq8";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, BUILD_TURN_HANDOVER_AFTER_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, REBUILD_REMOVED_RATIO, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_STRING_HEADER_BYTES, WORKER_COPY_MIN_VECTORS, defaultQuantizationFor };
|
|
@@ -326,6 +326,20 @@ function decodeVectorGraphPartPayload(raw) {
|
|
|
326
326
|
function sameGraphHeader(a, b) {
|
|
327
327
|
return a.entryPoint === b.entryPoint && a.maxLayer === b.maxLayer && a.m === b.m && a.efConstruction === b.efConstruction && a.metric === b.metric && a.shuffledBuildCount === b.shuffledBuildCount && a.calibratedMeanLength === b.calibratedMeanLength;
|
|
328
328
|
}
|
|
329
|
+
function requireOneGraphPerVector(part, p) {
|
|
330
|
+
if (part.graphs.length < 2) return;
|
|
331
|
+
const held = new Uint8Array(VECTOR_GRAPH_PART_VECTORS);
|
|
332
|
+
for (const graph of part.graphs) {
|
|
333
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
334
|
+
if (graph.levels[at] === 0) continue;
|
|
335
|
+
if (held[at] === 1) {
|
|
336
|
+
const vector = p * VECTOR_GRAPH_PART_VECTORS + at;
|
|
337
|
+
invalid2("vector graph part", `part ${p} gives vector ${vector} a node in two graphs`, { part: p, vector });
|
|
338
|
+
}
|
|
339
|
+
held[at] = 1;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
329
343
|
function assembleGraphParts(parts) {
|
|
330
344
|
const name = "vector graph part";
|
|
331
345
|
const first = parts[0];
|
|
@@ -346,6 +360,7 @@ function assembleGraphParts(parts) {
|
|
|
346
360
|
if (!fits) invalid2(name, `part ${p} gives graph ${g} ${levels} bytes of levels`, { part: p, levels });
|
|
347
361
|
partLists[g].push(graph);
|
|
348
362
|
}
|
|
363
|
+
requireOneGraphPerVector(part, p);
|
|
349
364
|
}
|
|
350
365
|
const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
|
|
351
366
|
return { fieldName: first.fieldName, graphs };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND } from './chunk-
|
|
1
|
+
import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND } from './chunk-HT7J73QL.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/engine/export/json-line.ts
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-KWYQ6KX4.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -606,7 +606,11 @@ function decodeKeywordSegment(payload, shared) {
|
|
|
606
606
|
}
|
|
607
607
|
const raw = requireRecord(decoded, "payload");
|
|
608
608
|
if (raw.v !== KEYWORD_SEGMENT_PAYLOAD_VERSION) {
|
|
609
|
-
|
|
609
|
+
throw new NarsilError(
|
|
610
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
611
|
+
`This keyword segment payload was written in layout ${String(raw.v)}, and this build reads layout ${KEYWORD_SEGMENT_PAYLOAD_VERSION} alone`,
|
|
612
|
+
{ version: raw.v }
|
|
613
|
+
);
|
|
610
614
|
}
|
|
611
615
|
const schema = decodeSchema(raw.schema);
|
|
612
616
|
const tombstones = requireStrings(raw.tombstones, "tombstones");
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { REBUILD_REMOVED_RATIO, BUILD_TURN_HANDOVER_AFTER_MS, DRIFT_CHECK_GROWTH_FACTOR, VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE, VECTOR_FILE_FIXED_OVERHEAD_BYTES, VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT, VECTOR_FILE_STRING_HEADER_BYTES } from './chunk-3I27KXQD.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-TQ5UMFG5.mjs';
|
|
3
|
+
import { osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-55WYRLDZ.mjs';
|
|
3
4
|
import { MAX_FILE_BYTES } from './chunk-IAXZAVYL.mjs';
|
|
4
5
|
import { MAX_DOC_ID_LENGTH } from './chunk-4RD3AFSB.mjs';
|
|
5
6
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
@@ -71,32 +72,6 @@ function ordinalFilterForSubsets(subsets, space) {
|
|
|
71
72
|
return filter;
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
// src/vector/vector-index/constants.ts
|
|
75
|
-
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
76
|
-
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
77
|
-
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
78
|
-
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
79
|
-
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
80
|
-
var BUILD_CHUNK_SIZE = 100;
|
|
81
|
-
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS = 256;
|
|
82
|
-
var SEARCH_SCAN_CHUNK_VECTORS = 4096;
|
|
83
|
-
var SEARCH_REFILL_RESERVE = 64;
|
|
84
|
-
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES = 1;
|
|
85
|
-
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
86
|
-
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
87
|
-
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
88
|
-
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
89
|
-
var REBUILD_REMOVED_RATIO = 0.2;
|
|
90
|
-
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
91
|
-
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
92
|
-
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
93
|
-
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
94
|
-
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
95
|
-
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
96
|
-
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
97
|
-
var OSQ4_MIN_DIMENSION = 384;
|
|
98
|
-
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
99
|
-
|
|
100
75
|
// src/vector/vector-index/shared.ts
|
|
101
76
|
var VECTOR_WORKER_COPIES_ALLOWED = { enabled: true };
|
|
102
77
|
function liveSize(state) {
|
|
@@ -675,4 +650,4 @@ function recordCheckpoint(state, plan, written) {
|
|
|
675
650
|
state.savedSignature = plan.signature;
|
|
676
651
|
}
|
|
677
652
|
|
|
678
|
-
export {
|
|
653
|
+
export { VECTOR_WORKER_COPIES_ALLOWED, adoptGraph, adoptLoadedCalibration, allLiveDocIds, assignStorePartitions, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, handOverBuildTurns, liveOrdinals, liveSize, noBuildHistory, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
import { gunzip, gzip } from './chunk-JNOAKDUM.mjs';
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-
|
|
3
|
-
import { restoreVectorFields } from './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-QJCO3TJU.mjs';
|
|
3
|
+
import { restoreVectorFields } from './chunk-B3JM6Z74.mjs';
|
|
4
|
+
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor } from './chunk-3I27KXQD.mjs';
|
|
5
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-55WYRLDZ.mjs';
|
|
6
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-KWYQ6KX4.mjs';
|
|
6
7
|
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-IAXZAVYL.mjs';
|
|
7
8
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
8
9
|
import { decode, encode } from '@msgpack/msgpack';
|
|
9
10
|
|
|
10
11
|
// src/engine/durability-metadata.ts
|
|
12
|
+
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
13
|
+
function vectorFieldsOf(schema, promotion) {
|
|
14
|
+
const fields = {};
|
|
15
|
+
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
16
|
+
const match = VECTOR_TYPE.exec(type);
|
|
17
|
+
if (match === null) continue;
|
|
18
|
+
const dimension = Number(match[1]);
|
|
19
|
+
fields[fieldPath] = {
|
|
20
|
+
dimension,
|
|
21
|
+
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
22
|
+
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
return Object.keys(fields).length === 0 ? null : fields;
|
|
26
|
+
}
|
|
11
27
|
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
12
28
|
const metadata = {
|
|
13
29
|
indexName,
|
|
@@ -23,6 +39,10 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
23
39
|
engineVersion: VERSION
|
|
24
40
|
};
|
|
25
41
|
metadata.documentCount = documentCount ?? config.documentCount;
|
|
42
|
+
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
43
|
+
if (vectorFields !== null) {
|
|
44
|
+
metadata.vectorFields = vectorFields;
|
|
45
|
+
}
|
|
26
46
|
if (config.embedding) {
|
|
27
47
|
metadata.embedding = config.embedding;
|
|
28
48
|
}
|
|
@@ -1358,7 +1378,7 @@ function holdsEveryHead(ref) {
|
|
|
1358
1378
|
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
1359
1379
|
const { pathOf } = store;
|
|
1360
1380
|
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
1361
|
-
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-
|
|
1381
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-YECAK4MN.mjs');
|
|
1362
1382
|
await Promise.all([
|
|
1363
1383
|
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
1364
1384
|
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
@@ -1379,8 +1399,8 @@ async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
|
1379
1399
|
resolvePartition
|
|
1380
1400
|
});
|
|
1381
1401
|
if (field === null) return false;
|
|
1382
|
-
vectorIndex.openColdField(field, async (beginRestore, stillWanted,
|
|
1383
|
-
await waitBeforeColdLoad(
|
|
1402
|
+
vectorIndex.openColdField(field, async (beginRestore, stillWanted, loadingWanted) => {
|
|
1403
|
+
await waitBeforeColdLoad(loadingWanted);
|
|
1384
1404
|
if (!stillWanted()) return;
|
|
1385
1405
|
const graph = await readGraphParts(store, ref);
|
|
1386
1406
|
if (!stillWanted()) return;
|
|
@@ -1720,18 +1740,19 @@ function frameRecord(entry) {
|
|
|
1720
1740
|
}
|
|
1721
1741
|
function checkSegmentHeader(data) {
|
|
1722
1742
|
if (data.length < SEGMENT_HEADER_SIZE) {
|
|
1723
|
-
return { ok: false, reason: "segment shorter than the 8-byte header" };
|
|
1743
|
+
return { ok: false, reason: "segment shorter than the 8-byte header", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
1724
1744
|
}
|
|
1725
1745
|
for (let i = 0; i < WAL_MAGIC.length; i += 1) {
|
|
1726
1746
|
if (data[i] !== WAL_MAGIC[i]) {
|
|
1727
|
-
return { ok: false, reason: "missing NRSW magic" };
|
|
1747
|
+
return { ok: false, reason: "missing NRSW magic", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
1728
1748
|
}
|
|
1729
1749
|
}
|
|
1730
1750
|
const formatVersion = data[4];
|
|
1731
1751
|
if (formatVersion > WAL_FORMAT_VERSION) {
|
|
1732
1752
|
return {
|
|
1733
1753
|
ok: false,
|
|
1734
|
-
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}
|
|
1754
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
|
|
1755
|
+
code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
|
|
1735
1756
|
};
|
|
1736
1757
|
}
|
|
1737
1758
|
return { ok: true };
|
|
@@ -1750,7 +1771,9 @@ function corrupt(message, details) {
|
|
|
1750
1771
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
1751
1772
|
const header = checkSegmentHeader(data);
|
|
1752
1773
|
if (!header.ok) {
|
|
1753
|
-
|
|
1774
|
+
throw new NarsilError(header.code, `WAL segment header invalid in durable region: ${header.reason}`, {
|
|
1775
|
+
reason: header.reason
|
|
1776
|
+
});
|
|
1754
1777
|
}
|
|
1755
1778
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
1756
1779
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -1898,14 +1921,10 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
1898
1921
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
1899
1922
|
const header = checkSegmentHeader(bytes);
|
|
1900
1923
|
if (!header.ok) {
|
|
1901
|
-
throw new NarsilError(
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
key,
|
|
1906
|
-
reason: header.reason
|
|
1907
|
-
}
|
|
1908
|
-
);
|
|
1924
|
+
throw new NarsilError(header.code, `Sealed WAL segment header invalid: ${header.reason}`, {
|
|
1925
|
+
key,
|
|
1926
|
+
reason: header.reason
|
|
1927
|
+
});
|
|
1909
1928
|
}
|
|
1910
1929
|
const durableEntries2 = readDurableRegion(bytes, bytes.length);
|
|
1911
1930
|
for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
|
|
@@ -2119,7 +2138,7 @@ async function compactPartitionSegments(input) {
|
|
|
2119
2138
|
if (paged) {
|
|
2120
2139
|
const { directory } = input.store;
|
|
2121
2140
|
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
2122
|
-
const { mergePagedWindow } = await import('./module-
|
|
2141
|
+
const { mergePagedWindow } = await import('./module-KAMRAJLE.mjs');
|
|
2123
2142
|
return replacing(
|
|
2124
2143
|
await mergePagedWindow({
|
|
2125
2144
|
store: input.store,
|
|
@@ -2207,7 +2226,7 @@ function pagedSettingsOf(context) {
|
|
|
2207
2226
|
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
2208
2227
|
const { directory } = context.input.store;
|
|
2209
2228
|
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
2210
|
-
const { pagedTombstonesFor } = await import('./module-
|
|
2229
|
+
const { pagedTombstonesFor } = await import('./module-KAMRAJLE.mjs');
|
|
2211
2230
|
return pagedTombstonesFor(
|
|
2212
2231
|
directory,
|
|
2213
2232
|
older,
|
|
@@ -2408,7 +2427,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
2408
2427
|
}
|
|
2409
2428
|
}
|
|
2410
2429
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
2411
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-
|
|
2430
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-KAMRAJLE.mjs');
|
|
2412
2431
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
2413
2432
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
2414
2433
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
@@ -2422,7 +2441,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
2422
2441
|
const { directory } = store;
|
|
2423
2442
|
for (const ref of partition.segments) {
|
|
2424
2443
|
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
2425
|
-
const { isPagedSegmentFile } = await import('./module-
|
|
2444
|
+
const { isPagedSegmentFile } = await import('./module-KAMRAJLE.mjs');
|
|
2426
2445
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
2427
2446
|
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
2428
2447
|
await yieldToEventLoop();
|
|
@@ -2478,7 +2497,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2478
2497
|
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
2479
2498
|
);
|
|
2480
2499
|
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
2481
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-
|
|
2500
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-KAMRAJLE.mjs');
|
|
2482
2501
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2483
2502
|
}
|
|
2484
2503
|
for (const partition of manifest.partitions) {
|
|
@@ -15353,7 +15353,7 @@ async function attachSegments(entry, action) {
|
|
|
15353
15353
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
15354
15354
|
const surfaceForms = segment.survivingSurfaceForms;
|
|
15355
15355
|
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
15356
|
-
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-
|
|
15356
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-KAMRAJLE.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
15357
15357
|
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
15358
15358
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
15359
15359
|
}
|
|
@@ -15408,7 +15408,7 @@ function runSegmentAction(entry, action) {
|
|
|
15408
15408
|
case "attachSegments":
|
|
15409
15409
|
return attachSegments(entry, action);
|
|
15410
15410
|
case "adoptDiskSegments":
|
|
15411
|
-
return import('./module-
|
|
15411
|
+
return import('./module-KAMRAJLE.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
15412
15412
|
case "compactSegments": {
|
|
15413
15413
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
15414
15414
|
return mergeFrozenSegments(
|
|
@@ -15456,7 +15456,7 @@ function requireLoadedVectorCode() {
|
|
|
15456
15456
|
function loadVectorCode() {
|
|
15457
15457
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15458
15458
|
if (loading2 === null) {
|
|
15459
|
-
loading2 = import('./module-
|
|
15459
|
+
loading2 = import('./module-XQLEOYNM.mjs').then(
|
|
15460
15460
|
(code) => {
|
|
15461
15461
|
loaded2 = code;
|
|
15462
15462
|
return code;
|
|
@@ -15872,7 +15872,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
15872
15872
|
// src/partitioning/rewrite-spill-of.ts
|
|
15873
15873
|
async function rewriteSpillOf(storage, manager, target) {
|
|
15874
15874
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
15875
|
-
const { openRewriteSpill } = await import('./module-
|
|
15875
|
+
const { openRewriteSpill } = await import('./module-KAMRAJLE.mjs');
|
|
15876
15876
|
return openRewriteSpill(storage, manager, target);
|
|
15877
15877
|
}
|
|
15878
15878
|
|
|
@@ -16565,7 +16565,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16565
16565
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
16566
16566
|
if (existing !== void 0) {
|
|
16567
16567
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
16568
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
16568
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-2V2HZ2NY.mjs');
|
|
16569
16569
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
16570
16570
|
}
|
|
16571
16571
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -19389,7 +19389,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19389
19389
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19390
19390
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19391
19391
|
}
|
|
19392
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19392
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-LM7WFL6X.mjs');
|
|
19393
19393
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19394
19394
|
const executor = createDirectExecutor({
|
|
19395
19395
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19574,7 +19574,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19574
19574
|
requireManager,
|
|
19575
19575
|
resumeChange: async (indexName) => {
|
|
19576
19576
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
19577
|
-
await (await import('./analysis-change-entry-
|
|
19577
|
+
await (await import('./analysis-change-entry-JL5U2N3Q.mjs')).resumeChangeOf(engineCore, indexName);
|
|
19578
19578
|
},
|
|
19579
19579
|
onClose: hooks?.onIndexClose,
|
|
19580
19580
|
onOpen: (indexName) => {
|
|
@@ -19658,7 +19658,7 @@ async function startEngineCore(core, config) {
|
|
|
19658
19658
|
checkHeapAfterRecovery(core);
|
|
19659
19659
|
}
|
|
19660
19660
|
if (core.invalidation) await core.invalidation.start();
|
|
19661
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
19661
|
+
if (core.durability) await (await import('./analysis-change-entry-JL5U2N3Q.mjs')).resumeEveryChange(core);
|
|
19662
19662
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
19663
19663
|
} catch (err) {
|
|
19664
19664
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -20337,7 +20337,7 @@ async function dropEngineIndex(core, name) {
|
|
|
20337
20337
|
const entry = core.requireIndex(name);
|
|
20338
20338
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
20339
20339
|
const abandonChange = async () => {
|
|
20340
|
-
await (await import('./analysis-change-entry-
|
|
20340
|
+
await (await import('./analysis-change-entry-JL5U2N3Q.mjs')).abandonChangeOf(core, name);
|
|
20341
20341
|
};
|
|
20342
20342
|
if (changeRecorded()) await abandonChange();
|
|
20343
20343
|
await core.indexState.drop(name, async () => {
|
|
@@ -22096,14 +22096,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22096
22096
|
return {
|
|
22097
22097
|
snapshot(indexName) {
|
|
22098
22098
|
return withOpenIndex2(indexName, async () => {
|
|
22099
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22099
|
+
const { createSnapshot } = await import('./snapshot-UGZVMP4H.mjs');
|
|
22100
22100
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22101
22101
|
});
|
|
22102
22102
|
},
|
|
22103
22103
|
async restore(indexName, data) {
|
|
22104
22104
|
core.guardShutdown();
|
|
22105
22105
|
validateIndexName(indexName);
|
|
22106
|
-
const restored = import('./restore-queue-
|
|
22106
|
+
const restored = import('./restore-queue-UQ6AAUVX.mjs').then(
|
|
22107
22107
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22108
22108
|
);
|
|
22109
22109
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22112,8 +22112,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22112
22112
|
exportDocuments(indexName) {
|
|
22113
22113
|
return withOpenIndex2(indexName, async () => {
|
|
22114
22114
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22115
|
-
import('./snapshot-
|
|
22116
|
-
import('./write-
|
|
22115
|
+
import('./snapshot-UGZVMP4H.mjs'),
|
|
22116
|
+
import('./write-I5P5MW4O.mjs')
|
|
22117
22117
|
]);
|
|
22118
22118
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22119
22119
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22121,7 +22121,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
22121
22121
|
},
|
|
22122
22122
|
importDocumentExport(indexName, data) {
|
|
22123
22123
|
core.guardShutdown();
|
|
22124
|
-
const imported = import('./import-
|
|
22124
|
+
const imported = import('./import-QOY6ZH6X.mjs').then(
|
|
22125
22125
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
22126
22126
|
);
|
|
22127
22127
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -24134,7 +24134,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24134
24134
|
const entry = requireIndex(indexName);
|
|
24135
24135
|
await withOpenIndexWrite(indexName, async () => {
|
|
24136
24136
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
24137
|
-
return (await import('./analysis-change-
|
|
24137
|
+
return (await import('./analysis-change-2V2HZ2NY.mjs')).resumeAnalysisChange(core, indexName);
|
|
24138
24138
|
});
|
|
24139
24139
|
},
|
|
24140
24140
|
async changeAnalysis(indexName, change) {
|
|
@@ -24142,7 +24142,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24142
24142
|
requireIndex(indexName);
|
|
24143
24143
|
const changed = withOpenIndex2(
|
|
24144
24144
|
indexName,
|
|
24145
|
-
async () => (await import('./analysis-change-
|
|
24145
|
+
async () => (await import('./analysis-change-2V2HZ2NY.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
24146
24146
|
);
|
|
24147
24147
|
await emitWriteEventAfter(core, indexName, changed);
|
|
24148
24148
|
},
|
|
@@ -24180,7 +24180,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24180
24180
|
}
|
|
24181
24181
|
rebalancesStarting.set(indexName, started);
|
|
24182
24182
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
24183
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
24183
|
+
const { executeRebalance } = await import('./analysis-change-2V2HZ2NY.mjs');
|
|
24184
24184
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
24185
24185
|
releaseWrites();
|
|
24186
24186
|
return running;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { chunksOf, restoredConfigFields, restoredEmbedding } from './chunk-NVEJJP4R.mjs';
|
|
2
|
-
import { untagDocumentLine } from './chunk-
|
|
3
|
-
import { flattenSchema, requireConfigurationShape } from './chunk-
|
|
2
|
+
import { untagDocumentLine } from './chunk-HT7J73QL.mjs';
|
|
3
|
+
import { flattenSchema, requireConfigurationShape } from './chunk-KWYQ6KX4.mjs';
|
|
4
4
|
import { turnsAreBudgeted, turnIsSpent, yieldTurn } from './chunk-IAXZAVYL.mjs';
|
|
5
5
|
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-4RD3AFSB.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-
|
|
2
|
-
import { crc32, openChecksummedEnvelope, packChecksummedEnvelope, selectionKeeps, HEADER_SIZE, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
1
|
+
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-B3JM6Z74.mjs';
|
|
2
|
+
import { crc32, openChecksummedEnvelope, packChecksummedEnvelope, selectionKeeps, HEADER_SIZE, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents } from './chunk-KWYQ6KX4.mjs';
|
|
3
3
|
import { fnv1a } from './chunk-MPK76GVS.mjs';
|
|
4
4
|
import { MAX_FILE_BYTES } from './chunk-IAXZAVYL.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
@@ -106,7 +106,11 @@ function decodePagedSegmentHead(head, payloadLength, what) {
|
|
|
106
106
|
if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
|
|
107
107
|
const entry = raw;
|
|
108
108
|
if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
|
|
109
|
-
|
|
109
|
+
throw new NarsilError(
|
|
110
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
111
|
+
`The head of ${what} was written in layout ${String(entry.v)}, and this build reads layout ${PAGED_SEGMENT_HEAD_VERSION} alone`,
|
|
112
|
+
{ version: entry.v }
|
|
113
|
+
);
|
|
110
114
|
}
|
|
111
115
|
if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
|
|
112
116
|
if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
|
|
@@ -907,7 +911,7 @@ async function writePagedGroup(state, group) {
|
|
|
907
911
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
908
912
|
const id = state.nextSegmentId;
|
|
909
913
|
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
910
|
-
const { writePagedSegment } = await import('./module-
|
|
914
|
+
const { writePagedSegment } = await import('./module-KAMRAJLE.mjs');
|
|
911
915
|
const written = await writePagedSegment({
|
|
912
916
|
directory,
|
|
913
917
|
key,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, unpackLevels, readTrailer, stageQueryLevels, writeRecord } from './chunk-
|
|
1
|
+
import { osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, unpackLevels, readTrailer, stageQueryLevels, writeRecord } from './chunk-55WYRLDZ.mjs';
|
|
2
2
|
import { markSimdUnavailable, adoptSharedTurns, OSQ_STEPS, DEFAULT_EF_SEARCH, OSQ_GRID, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, OSQ_QUERY_BITS, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_PLACEMENT_QUERY_BITS, compareCodePoints, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, OSQ_LAMBDA, threadSlotGone, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS } from './chunk-IAXZAVYL.mjs';
|
|
3
3
|
import { NATIVE_THREAD_SEARCH_SLOTS, VECTOR_SCRATCH_SLOTS, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, WASM_PAGE_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_BLOCK_INITIAL_SLOTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_DEADLINE_MS } from './chunk-TY7L6OXO.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-2V2HZ2NY.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-2V2HZ2NY.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { encodeDocumentJson } from './chunk-
|
|
2
|
-
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, decodeSections, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-
|
|
3
|
-
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-
|
|
1
|
+
import { encodeDocumentJson } from './chunk-7ZJTRUVW.mjs';
|
|
2
|
+
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, decodeSections, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-QJCO3TJU.mjs';
|
|
3
|
+
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-B3JM6Z74.mjs';
|
|
4
4
|
import { createVectorFileReader } from './chunk-LURXVCHG.mjs';
|
|
5
|
-
import { HEADER_SIZE, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, isCompositePartition, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, buildPartialWordTableFrom, mergeFieldIndexes, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, resolvePartitionInsertOptions, flattenSchema, createPartitionIndex, createCompositePartition, decodeSurfaceTable, readHeader, VECTOR_PATTERN, mergeDocFrequencies, parseVectorDimension, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, scanPostingBlocks, encodeDocument, holdsADiscardedField, endOfValue } from './chunk-
|
|
5
|
+
import { HEADER_SIZE, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, isCompositePartition, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, buildPartialWordTableFrom, mergeFieldIndexes, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, resolvePartitionInsertOptions, flattenSchema, createPartitionIndex, createCompositePartition, decodeSurfaceTable, readHeader, VECTOR_PATTERN, mergeDocFrequencies, parseVectorDimension, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, scanPostingBlocks, encodeDocument, holdsADiscardedField, endOfValue } from './chunk-KWYQ6KX4.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-IAXZAVYL.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
8
8
|
import { decode, encode } from '@msgpack/msgpack';
|
|
@@ -1306,7 +1306,7 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
1306
1306
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
1307
1307
|
if (dimension === null) return null;
|
|
1308
1308
|
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
1309
|
-
const { openColdVectorField } = await import('./cold-module-
|
|
1309
|
+
const { openColdVectorField } = await import('./cold-module-YECAK4MN.mjs');
|
|
1310
1310
|
return openColdVectorField({
|
|
1311
1311
|
fieldPath: field.fieldPath,
|
|
1312
1312
|
dimension,
|