@delali/narsil 0.3.1-next.83 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-67MQF5CV.mjs} +215 -48
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-CUP2E4OH.mjs → chunk-RCSXJCI5.mjs} +373 -199
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +22 -7
- package/dist/index.browser.mjs +1040 -304
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig,
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-RCSXJCI5.mjs';
|
|
5
5
|
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
6
|
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
7
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
8
|
-
import { releaseLocksHeldBy } from './chunk-
|
|
9
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
8
|
+
import { releaseLocksHeldBy } from './chunk-NIWDQTZB.mjs';
|
|
9
|
+
import { requireSearchBackendSettings } from './chunk-NFF5TDRG.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
|
|
11
11
|
import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
|
|
12
12
|
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
|
|
13
13
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
14
14
|
import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
16
|
-
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-
|
|
16
|
+
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-YRX6XMOQ.mjs';
|
|
17
17
|
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
18
18
|
import { openSync, close, readFileSync } from 'fs';
|
|
19
19
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
@@ -160,6 +160,8 @@ function yieldToEventLoop() {
|
|
|
160
160
|
// src/partitioning/constants.ts
|
|
161
161
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
162
162
|
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
163
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
164
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
163
165
|
|
|
164
166
|
// src/partitioning/rebalancer.ts
|
|
165
167
|
function createRebalancer() {
|
|
@@ -452,6 +454,74 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
452
454
|
}
|
|
453
455
|
}
|
|
454
456
|
|
|
457
|
+
// src/partitioning/text-rebuild.ts
|
|
458
|
+
function interrupted(pass) {
|
|
459
|
+
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
460
|
+
}
|
|
461
|
+
async function buildPartition(pass, source, partitionId) {
|
|
462
|
+
const { manager, options } = pass;
|
|
463
|
+
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
464
|
+
const docIds = [...source.docIds()];
|
|
465
|
+
target.beginBatch();
|
|
466
|
+
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
467
|
+
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
468
|
+
for (let i = start; i < end; i++) {
|
|
469
|
+
const document = manager.getRef(docIds[i]);
|
|
470
|
+
if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
|
|
471
|
+
}
|
|
472
|
+
await yieldToEventLoop();
|
|
473
|
+
if (interrupted(pass)) return null;
|
|
474
|
+
}
|
|
475
|
+
target.endBatch();
|
|
476
|
+
if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
|
|
477
|
+
return target;
|
|
478
|
+
}
|
|
479
|
+
function applyWrittenDocuments(pass, built) {
|
|
480
|
+
const { manager, options } = pass;
|
|
481
|
+
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
482
|
+
for (const partition of built) {
|
|
483
|
+
if (!partition.has(docId)) continue;
|
|
484
|
+
partition.remove(docId, manager.schema, manager.language, options);
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
488
|
+
const document = manager.getRef(docId);
|
|
489
|
+
if (partitionId === void 0 || document === void 0) continue;
|
|
490
|
+
built[partitionId].insert(docId, document, manager.schema, manager.language, options);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
async function rebuildOnce(pass) {
|
|
494
|
+
const partitionCount = pass.manager.partitionCount;
|
|
495
|
+
const built = [];
|
|
496
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
497
|
+
const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
|
|
498
|
+
if (partition === null) return false;
|
|
499
|
+
built.push(partition);
|
|
500
|
+
pass.hooks.onPartitionBuilt(built.length);
|
|
501
|
+
}
|
|
502
|
+
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
503
|
+
applyWrittenDocuments(pass, built);
|
|
504
|
+
await yieldToEventLoop();
|
|
505
|
+
if (interrupted(pass)) return false;
|
|
506
|
+
}
|
|
507
|
+
applyWrittenDocuments(pass, built);
|
|
508
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
509
|
+
pass.hooks.onSwitch();
|
|
510
|
+
return true;
|
|
511
|
+
}
|
|
512
|
+
async function rebuildTextIndexes(manager, hooks) {
|
|
513
|
+
const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
|
|
514
|
+
while (hooks.isCurrent()) {
|
|
515
|
+
const log = manager.openWriteLog();
|
|
516
|
+
try {
|
|
517
|
+
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
518
|
+
} finally {
|
|
519
|
+
log.close();
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return false;
|
|
523
|
+
}
|
|
524
|
+
|
|
455
525
|
// src/engine/events.ts
|
|
456
526
|
function eventHandlersOf(events) {
|
|
457
527
|
const handlers = /* @__PURE__ */ new Map();
|
|
@@ -474,11 +544,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
|
|
|
474
544
|
}
|
|
475
545
|
|
|
476
546
|
// src/engine/analysis-rebuild.ts
|
|
477
|
-
function yieldToEventLoop2() {
|
|
478
|
-
return new Promise((resolve) => {
|
|
479
|
-
setTimeout(resolve, 0);
|
|
480
|
-
});
|
|
481
|
-
}
|
|
482
547
|
function toError(value) {
|
|
483
548
|
return value instanceof Error ? value : new Error(String(value));
|
|
484
549
|
}
|
|
@@ -489,9 +554,13 @@ function wireAnalysisRebuild(wiring) {
|
|
|
489
554
|
resyncIndex: wiring.resyncIndex,
|
|
490
555
|
async persistAnalysisRevision(indexName) {
|
|
491
556
|
const durability = wiring.durabilityManager;
|
|
492
|
-
if (durability === null)
|
|
493
|
-
|
|
557
|
+
if (durability === null) {
|
|
558
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
494
561
|
await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
|
|
562
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
563
|
+
await durability.persistMetadata(indexName);
|
|
495
564
|
},
|
|
496
565
|
emit(payload) {
|
|
497
566
|
emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
|
|
@@ -502,9 +571,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
502
571
|
const stale = /* @__PURE__ */ new Map();
|
|
503
572
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
504
573
|
let queue = Promise.resolve();
|
|
574
|
+
let active = null;
|
|
505
575
|
async function rebuildNow(indexName, entry) {
|
|
506
576
|
const manager = deps.getManager(indexName);
|
|
507
|
-
if (manager === void 0) {
|
|
577
|
+
if (manager === void 0 || entry.abandoned) {
|
|
508
578
|
entry.run = null;
|
|
509
579
|
return;
|
|
510
580
|
}
|
|
@@ -512,12 +582,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
512
582
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
513
583
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
514
584
|
try {
|
|
515
|
-
|
|
516
|
-
manager.
|
|
517
|
-
|
|
518
|
-
|
|
585
|
+
const rebuilt = await rebuildTextIndexes(manager, {
|
|
586
|
+
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
587
|
+
onPartitionBuilt(partitionsBuilt) {
|
|
588
|
+
entry.partitionsRebuilt = partitionsBuilt;
|
|
589
|
+
},
|
|
590
|
+
onSwitch() {
|
|
591
|
+
stale.delete(indexName);
|
|
592
|
+
}
|
|
593
|
+
});
|
|
594
|
+
if (!rebuilt) {
|
|
595
|
+
entry.run = null;
|
|
596
|
+
if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
|
|
597
|
+
return;
|
|
519
598
|
}
|
|
520
|
-
stale.delete(indexName);
|
|
521
599
|
await deps.persistAnalysisRevision(indexName);
|
|
522
600
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
523
601
|
deps.emit({
|
|
@@ -528,7 +606,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
528
606
|
});
|
|
529
607
|
} catch (err) {
|
|
530
608
|
const error = toError(err);
|
|
531
|
-
stale.set(indexName, entry);
|
|
532
609
|
entry.run = null;
|
|
533
610
|
await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
534
611
|
deps.emit({
|
|
@@ -541,11 +618,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
541
618
|
throw error;
|
|
542
619
|
}
|
|
543
620
|
}
|
|
621
|
+
function runTracked(indexName, entry) {
|
|
622
|
+
const done = rebuildNow(indexName, entry);
|
|
623
|
+
const tracked = { indexName, entry, done };
|
|
624
|
+
active = tracked;
|
|
625
|
+
return done.finally(() => {
|
|
626
|
+
if (active === tracked) active = null;
|
|
627
|
+
});
|
|
628
|
+
}
|
|
544
629
|
function enqueue(indexName, entry) {
|
|
545
630
|
if (entry.run !== null) {
|
|
546
631
|
return entry.run;
|
|
547
632
|
}
|
|
548
|
-
|
|
633
|
+
entry.abandoned = false;
|
|
634
|
+
const run = queue.then(() => runTracked(indexName, entry));
|
|
549
635
|
entry.run = run;
|
|
550
636
|
queue = run.then(
|
|
551
637
|
() => void 0,
|
|
@@ -580,7 +666,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
580
666
|
if (existing !== void 0 && existing.run !== null) {
|
|
581
667
|
return;
|
|
582
668
|
}
|
|
583
|
-
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
|
|
669
|
+
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
|
|
584
670
|
},
|
|
585
671
|
clearStale(indexName) {
|
|
586
672
|
stale.delete(indexName);
|
|
@@ -591,6 +677,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
591
677
|
isRunning(indexName) {
|
|
592
678
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
593
679
|
},
|
|
680
|
+
async abandon(indexName) {
|
|
681
|
+
const entry = stale.get(indexName);
|
|
682
|
+
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
683
|
+
const running = active;
|
|
684
|
+
if (running === null || running.indexName !== indexName) return;
|
|
685
|
+
running.entry.abandoned = true;
|
|
686
|
+
await running.done.catch(() => void 0);
|
|
687
|
+
},
|
|
594
688
|
review(indexName) {
|
|
595
689
|
const entry = stale.get(indexName);
|
|
596
690
|
return entry === void 0 ? Promise.resolve() : review(indexName, entry);
|
|
@@ -1400,7 +1494,7 @@ function buildEntry(fields) {
|
|
|
1400
1494
|
}
|
|
1401
1495
|
|
|
1402
1496
|
// src/serialization/chunk-checksum.ts
|
|
1403
|
-
function
|
|
1497
|
+
function yieldToEventLoop2() {
|
|
1404
1498
|
if (typeof setImmediate === "function") {
|
|
1405
1499
|
return new Promise((resolve) => setImmediate(resolve));
|
|
1406
1500
|
}
|
|
@@ -1412,7 +1506,7 @@ async function advanceChecksum(start, payload) {
|
|
|
1412
1506
|
const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
|
|
1413
1507
|
state = crc32Update(state, payload.subarray(offset, end));
|
|
1414
1508
|
if (end < payload.length) {
|
|
1415
|
-
await
|
|
1509
|
+
await yieldToEventLoop2();
|
|
1416
1510
|
}
|
|
1417
1511
|
}
|
|
1418
1512
|
return state;
|
|
@@ -5639,7 +5733,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
5639
5733
|
manager.endBatchRemove();
|
|
5640
5734
|
}
|
|
5641
5735
|
}
|
|
5642
|
-
function
|
|
5736
|
+
function yieldToEventLoop3() {
|
|
5643
5737
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
5644
5738
|
}
|
|
5645
5739
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
@@ -5650,7 +5744,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
5650
5744
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
5651
5745
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
5652
5746
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
5653
|
-
await
|
|
5747
|
+
await yieldToEventLoop3();
|
|
5654
5748
|
}
|
|
5655
5749
|
}
|
|
5656
5750
|
|
|
@@ -8078,7 +8172,7 @@ function metadataConfigOf(entry) {
|
|
|
8078
8172
|
b: entry.config.bm25?.b ?? 0.75,
|
|
8079
8173
|
...embedding !== void 0 ? { embedding } : {},
|
|
8080
8174
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
8081
|
-
analysisRevision: entry.
|
|
8175
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
8082
8176
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
8083
8177
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
8084
8178
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -8166,7 +8260,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
8166
8260
|
indexUuid: metadata.indexUuid ?? null,
|
|
8167
8261
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
8168
8262
|
documentCount: metadata.documentCount ?? 0,
|
|
8169
|
-
partitionCount: metadata.partitionCount
|
|
8263
|
+
partitionCount: metadata.partitionCount,
|
|
8264
|
+
analysisRevision: metadata.analysisRevision ?? null
|
|
8170
8265
|
});
|
|
8171
8266
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
8172
8267
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -8289,8 +8384,14 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
8289
8384
|
return entry;
|
|
8290
8385
|
}
|
|
8291
8386
|
}
|
|
8387
|
+
function transitionPending(indexName, entry) {
|
|
8388
|
+
if (entries.get(indexName) !== entry) return true;
|
|
8389
|
+
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
8390
|
+
return entry.state === "closing" && entry.closePromise !== null;
|
|
8391
|
+
}
|
|
8292
8392
|
async function reopen(indexName, explicit) {
|
|
8293
|
-
|
|
8393
|
+
let entry = await awaitPendingTransitions(indexName);
|
|
8394
|
+
while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
|
|
8294
8395
|
if (entry.state === "open") return;
|
|
8295
8396
|
if (explicit) {
|
|
8296
8397
|
entry.reopenFailures = 0;
|
|
@@ -8879,6 +8980,7 @@ function shareFrozenSegment(segment, analysis) {
|
|
|
8879
8980
|
}
|
|
8880
8981
|
|
|
8881
8982
|
// src/engine/worker-resync.ts
|
|
8983
|
+
var copyLoadSequence = 0;
|
|
8882
8984
|
function captureComposite(manager, partitionId) {
|
|
8883
8985
|
const partition = manager.getPartition(partitionId);
|
|
8884
8986
|
if (!isCompositePartition(partition)) return null;
|
|
@@ -8939,8 +9041,8 @@ async function sendPartition(indexName, executors, captured) {
|
|
|
8939
9041
|
)
|
|
8940
9042
|
);
|
|
8941
9043
|
}
|
|
8942
|
-
|
|
8943
|
-
|
|
9044
|
+
function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
|
|
9045
|
+
return loadIndexOntoWorkers(
|
|
8944
9046
|
indexName,
|
|
8945
9047
|
pool.getAllExecutors(),
|
|
8946
9048
|
config,
|
|
@@ -8950,6 +9052,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
|
|
|
8950
9052
|
);
|
|
8951
9053
|
}
|
|
8952
9054
|
async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
|
|
9055
|
+
copyLoadSequence += 1;
|
|
9056
|
+
const copyLoad = `${indexName}#${copyLoadSequence}`;
|
|
8953
9057
|
const captured = capturePartitions(manager);
|
|
8954
9058
|
await Promise.allSettled(
|
|
8955
9059
|
executors.map(
|
|
@@ -8964,13 +9068,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
8964
9068
|
indexName,
|
|
8965
9069
|
config,
|
|
8966
9070
|
requestId: `resync-create-${indexName}`,
|
|
8967
|
-
analysisStale
|
|
9071
|
+
analysisStale,
|
|
9072
|
+
copyLoad
|
|
8968
9073
|
})
|
|
8969
9074
|
)
|
|
8970
9075
|
);
|
|
8971
9076
|
for (const partition of captured) {
|
|
8972
9077
|
await sendPartition(indexName, executors, partition);
|
|
8973
9078
|
}
|
|
9079
|
+
return copyLoad;
|
|
8974
9080
|
}
|
|
8975
9081
|
|
|
8976
9082
|
// src/engine/orchestration/constants.ts
|
|
@@ -9198,6 +9304,14 @@ function enqueueReplication(state, indexName, action) {
|
|
|
9198
9304
|
queue.inFlight.push(run);
|
|
9199
9305
|
queue.tail = run;
|
|
9200
9306
|
}
|
|
9307
|
+
function markCopiesLoaded(state, indexName, copyLoad) {
|
|
9308
|
+
enqueueReplication(state, indexName, {
|
|
9309
|
+
type: "copyLoaded",
|
|
9310
|
+
indexName,
|
|
9311
|
+
copyLoad,
|
|
9312
|
+
requestId: `copy-loaded-${copyLoad}`
|
|
9313
|
+
});
|
|
9314
|
+
}
|
|
9201
9315
|
async function replicateToWorkers(state, action) {
|
|
9202
9316
|
if (!("indexName" in action)) {
|
|
9203
9317
|
if (state.workerPool === null) return;
|
|
@@ -10163,8 +10277,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
10163
10277
|
if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
|
|
10164
10278
|
const buffered = [];
|
|
10165
10279
|
state.copyLoadBuffers.set(indexName, buffered);
|
|
10280
|
+
let copyLoad;
|
|
10166
10281
|
try {
|
|
10167
|
-
await loadIndexOntoWorkers(
|
|
10282
|
+
copyLoad = await loadIndexOntoWorkers(
|
|
10168
10283
|
indexName,
|
|
10169
10284
|
[replacement.executor],
|
|
10170
10285
|
entry.config,
|
|
@@ -10177,6 +10292,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
10177
10292
|
state.copyLoadBuffers.delete(indexName);
|
|
10178
10293
|
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
10179
10294
|
}
|
|
10295
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10180
10296
|
refreshVectorCopies(state, indexName);
|
|
10181
10297
|
}
|
|
10182
10298
|
async function replaceWorker(state, pool, workerId) {
|
|
@@ -10353,7 +10469,13 @@ async function loadCopies(state, indexName, reason) {
|
|
|
10353
10469
|
state.copyLoadBuffers.set(indexName, buffered);
|
|
10354
10470
|
try {
|
|
10355
10471
|
freezeLiveTailsBeforeCopiesLoad(manager);
|
|
10356
|
-
|
|
10472
|
+
const copyLoad = await transferIndexToPool(
|
|
10473
|
+
indexName,
|
|
10474
|
+
pool,
|
|
10475
|
+
entry.config,
|
|
10476
|
+
manager,
|
|
10477
|
+
state.callbacks?.isAnalysisStale?.(indexName)
|
|
10478
|
+
);
|
|
10357
10479
|
if (state.desyncedIndexes.has(indexName)) {
|
|
10358
10480
|
discardLoadedCopies(state, pool, indexName);
|
|
10359
10481
|
return;
|
|
@@ -10362,6 +10484,7 @@ async function loadCopies(state, indexName, reason) {
|
|
|
10362
10484
|
state.lastAccessAt.set(indexName, Date.now());
|
|
10363
10485
|
state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
|
|
10364
10486
|
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
10487
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10365
10488
|
} catch (err) {
|
|
10366
10489
|
if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
|
|
10367
10490
|
throw err;
|
|
@@ -10705,7 +10828,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
10705
10828
|
idleSweep: null
|
|
10706
10829
|
};
|
|
10707
10830
|
startIdleSweep(state);
|
|
10831
|
+
function settlingCopyWork(indexName) {
|
|
10832
|
+
return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
|
|
10833
|
+
}
|
|
10708
10834
|
async function resyncIndex(indexName, wasScaledOut) {
|
|
10835
|
+
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
10836
|
+
await pending;
|
|
10837
|
+
}
|
|
10709
10838
|
state.desyncedIndexes.delete(indexName);
|
|
10710
10839
|
if (!wasScaledOut) return;
|
|
10711
10840
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
@@ -10763,8 +10892,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
10763
10892
|
const manager = executor.getManager(indexName);
|
|
10764
10893
|
if (pool !== null && entry !== void 0 && manager !== void 0) {
|
|
10765
10894
|
try {
|
|
10766
|
-
|
|
10895
|
+
const copyLoad = await transferIndexToPool(
|
|
10896
|
+
indexName,
|
|
10897
|
+
pool,
|
|
10898
|
+
entry.config,
|
|
10899
|
+
manager,
|
|
10900
|
+
callbacks?.isAnalysisStale?.(indexName)
|
|
10901
|
+
);
|
|
10767
10902
|
state.scaledOutIndexes.add(indexName);
|
|
10903
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10768
10904
|
refreshVectorCopies(state, indexName);
|
|
10769
10905
|
} catch (repairError) {
|
|
10770
10906
|
throw new NarsilError(
|
|
@@ -11057,6 +11193,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
11057
11193
|
getManager: (indexName) => executor.getManager(indexName),
|
|
11058
11194
|
desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
|
|
11059
11195
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
11196
|
+
recordCurrentAnalysis: (indexName) => {
|
|
11197
|
+
const entry = indexRegistry.get(indexName);
|
|
11198
|
+
if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
|
|
11199
|
+
},
|
|
11060
11200
|
durabilityManager: durability?.manager ?? null
|
|
11061
11201
|
});
|
|
11062
11202
|
const indexState = wireIndexState({
|
|
@@ -11231,11 +11371,16 @@ function refuseCode(entry) {
|
|
|
11231
11371
|
}
|
|
11232
11372
|
function indexConfigurationOf(entry) {
|
|
11233
11373
|
refuseCode(entry);
|
|
11234
|
-
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
|
|
11374
|
+
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
|
|
11235
11375
|
}
|
|
11236
11376
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
11237
11377
|
const adapter = config.embedding?.adapter;
|
|
11238
|
-
const payload = configurationPayloadOf(
|
|
11378
|
+
const payload = configurationPayloadOf(
|
|
11379
|
+
config,
|
|
11380
|
+
language,
|
|
11381
|
+
typeof adapter === "string" ? adapter : null,
|
|
11382
|
+
language.revision
|
|
11383
|
+
);
|
|
11239
11384
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
11240
11385
|
const bytes = encode(payload).byteLength;
|
|
11241
11386
|
if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
@@ -11245,12 +11390,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
11245
11390
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
11246
11391
|
);
|
|
11247
11392
|
}
|
|
11248
|
-
function configurationPayloadOf(config, language, adapterName) {
|
|
11393
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
11249
11394
|
const bm25 = config.bm25;
|
|
11250
11395
|
return {
|
|
11251
11396
|
schema: config.schema,
|
|
11252
11397
|
language: language.name,
|
|
11253
|
-
analysisRevision:
|
|
11398
|
+
...analysisRevision !== null ? { analysisRevision } : {},
|
|
11254
11399
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
11255
11400
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
11256
11401
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -11409,7 +11554,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
11409
11554
|
indexUuid: indexUuid ?? null,
|
|
11410
11555
|
heldPartitions: null,
|
|
11411
11556
|
documentCount: 0,
|
|
11412
|
-
partitionCount: indexConfig.partitions?.maxPartitions ?? 1
|
|
11557
|
+
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
11558
|
+
analysisRevision: language.revision
|
|
11413
11559
|
});
|
|
11414
11560
|
await core.indexState.registerOpen(name);
|
|
11415
11561
|
if (core.durability) {
|
|
@@ -11421,6 +11567,7 @@ async function dropEngineIndex(core, name) {
|
|
|
11421
11567
|
core.guardShutdown();
|
|
11422
11568
|
const entry = core.requireIndex(name);
|
|
11423
11569
|
await core.indexState.drop(name, async () => {
|
|
11570
|
+
await core.analysisRebuild.abandon(name);
|
|
11424
11571
|
await core.orchestrator.closeIndex(name);
|
|
11425
11572
|
if (core.durability) {
|
|
11426
11573
|
try {
|
|
@@ -14196,7 +14343,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
14196
14343
|
indexUuid: null,
|
|
14197
14344
|
heldPartitions: null,
|
|
14198
14345
|
documentCount: 0,
|
|
14199
|
-
partitionCount
|
|
14346
|
+
partitionCount,
|
|
14347
|
+
analysisRevision: prepared.storedRevision
|
|
14200
14348
|
});
|
|
14201
14349
|
await deps.indexState.registerOpen(indexName);
|
|
14202
14350
|
const release = await deps.indexState.acquire(indexName, false);
|
|
@@ -14581,6 +14729,15 @@ async function enterRead(core, indexName) {
|
|
|
14581
14729
|
core.guardShutdown();
|
|
14582
14730
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
14583
14731
|
}
|
|
14732
|
+
async function readUnderOneAnalysis(core, indexName, read) {
|
|
14733
|
+
let analysisStale = core.analysisRebuild.isStale(indexName);
|
|
14734
|
+
let result = await read();
|
|
14735
|
+
while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
|
|
14736
|
+
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
14737
|
+
result = await read();
|
|
14738
|
+
}
|
|
14739
|
+
return { result, analysisStale };
|
|
14740
|
+
}
|
|
14584
14741
|
function scopedParams(params, options) {
|
|
14585
14742
|
if (options?.globalStats === void 0) {
|
|
14586
14743
|
return params;
|
|
@@ -14613,7 +14770,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14613
14770
|
entry.embeddingAdapterName
|
|
14614
14771
|
);
|
|
14615
14772
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
14616
|
-
const
|
|
14773
|
+
const context = {
|
|
14617
14774
|
manager,
|
|
14618
14775
|
language: entry.language,
|
|
14619
14776
|
config: entry.config,
|
|
@@ -14623,7 +14780,15 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14623
14780
|
partitionIds: options?.partitionIds,
|
|
14624
14781
|
cursorBinding: queryBindingOf(params),
|
|
14625
14782
|
patternWorkCap: core.patternWorkCap
|
|
14626
|
-
}
|
|
14783
|
+
};
|
|
14784
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
14785
|
+
core,
|
|
14786
|
+
indexName,
|
|
14787
|
+
() => executeQuery(resolvedParams, context)
|
|
14788
|
+
);
|
|
14789
|
+
if (analysisStale) {
|
|
14790
|
+
result.analysisStale = true;
|
|
14791
|
+
}
|
|
14627
14792
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
14628
14793
|
const observed = { ...result, hits: [...result.hits] };
|
|
14629
14794
|
try {
|
|
@@ -14632,9 +14797,6 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14632
14797
|
console.warn("afterSearch plugin hook error:", err);
|
|
14633
14798
|
}
|
|
14634
14799
|
}
|
|
14635
|
-
if (core.analysisRebuild.isStale(indexName)) {
|
|
14636
|
-
result.analysisStale = true;
|
|
14637
|
-
}
|
|
14638
14800
|
return result;
|
|
14639
14801
|
} finally {
|
|
14640
14802
|
release();
|
|
@@ -14655,7 +14817,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
14655
14817
|
entry.embeddingAdapterName
|
|
14656
14818
|
);
|
|
14657
14819
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
14658
|
-
const
|
|
14820
|
+
const context = {
|
|
14659
14821
|
manager,
|
|
14660
14822
|
language: entry.language,
|
|
14661
14823
|
config: entry.config,
|
|
@@ -14665,8 +14827,13 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
14665
14827
|
partitionIds: options?.partitionIds,
|
|
14666
14828
|
cursorBinding: queryBindingOf(params),
|
|
14667
14829
|
patternWorkCap: core.patternWorkCap
|
|
14668
|
-
}
|
|
14669
|
-
|
|
14830
|
+
};
|
|
14831
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
14832
|
+
core,
|
|
14833
|
+
indexName,
|
|
14834
|
+
() => executePreflight(resolvedParams, context)
|
|
14835
|
+
);
|
|
14836
|
+
if (analysisStale) {
|
|
14670
14837
|
result.analysisStale = true;
|
|
14671
14838
|
}
|
|
14672
14839
|
return result;
|
|
@@ -18,7 +18,7 @@ function stopUsingTheNativeSearchCore(error) {
|
|
|
18
18
|
}
|
|
19
19
|
function noteFallback(operation) {
|
|
20
20
|
}
|
|
21
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
21
|
+
var NATIVE_CORE_ABI_VERSION = 3;
|
|
22
22
|
var BINARY_FILE_NAME = "narsil-core.node";
|
|
23
23
|
var MUSL_DIRECTORY = "musl";
|
|
24
24
|
var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
|