@delali/narsil 0.3.1-next.83 → 0.3.1-next.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-CUP2E4OH.mjs → chunk-IA67SOJO.mjs} +102 -106
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-XIQZKKTK.mjs} +177 -39
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-6WWFY5EI.mjs} +1 -1
- package/dist/distribution.mjs +19 -4
- package/dist/index.browser.mjs +277 -143
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/workers/entry.mjs +1 -1
- package/package.json +7 -7
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig,
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-IA67SOJO.mjs';
|
|
5
5
|
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
6
|
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
7
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
@@ -160,6 +160,8 @@ function yieldToEventLoop() {
|
|
|
160
160
|
// src/partitioning/constants.ts
|
|
161
161
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
162
162
|
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
163
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
164
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
163
165
|
|
|
164
166
|
// src/partitioning/rebalancer.ts
|
|
165
167
|
function createRebalancer() {
|
|
@@ -452,6 +454,74 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
452
454
|
}
|
|
453
455
|
}
|
|
454
456
|
|
|
457
|
+
// src/partitioning/text-rebuild.ts
|
|
458
|
+
function interrupted(pass) {
|
|
459
|
+
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
460
|
+
}
|
|
461
|
+
async function buildPartition(pass, source, partitionId) {
|
|
462
|
+
const { manager, options } = pass;
|
|
463
|
+
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
464
|
+
const docIds = [...source.docIds()];
|
|
465
|
+
target.beginBatch();
|
|
466
|
+
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
467
|
+
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
468
|
+
for (let i = start; i < end; i++) {
|
|
469
|
+
const document = manager.getRef(docIds[i]);
|
|
470
|
+
if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
|
|
471
|
+
}
|
|
472
|
+
await yieldToEventLoop();
|
|
473
|
+
if (interrupted(pass)) return null;
|
|
474
|
+
}
|
|
475
|
+
target.endBatch();
|
|
476
|
+
if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
|
|
477
|
+
return target;
|
|
478
|
+
}
|
|
479
|
+
function applyWrittenDocuments(pass, built) {
|
|
480
|
+
const { manager, options } = pass;
|
|
481
|
+
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
482
|
+
for (const partition of built) {
|
|
483
|
+
if (!partition.has(docId)) continue;
|
|
484
|
+
partition.remove(docId, manager.schema, manager.language, options);
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
const partitionId = manager.partitionIdOf(docId);
|
|
488
|
+
const document = manager.getRef(docId);
|
|
489
|
+
if (partitionId === void 0 || document === void 0) continue;
|
|
490
|
+
built[partitionId].insert(docId, document, manager.schema, manager.language, options);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
async function rebuildOnce(pass) {
|
|
494
|
+
const partitionCount = pass.manager.partitionCount;
|
|
495
|
+
const built = [];
|
|
496
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
497
|
+
const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
|
|
498
|
+
if (partition === null) return false;
|
|
499
|
+
built.push(partition);
|
|
500
|
+
pass.hooks.onPartitionBuilt(built.length);
|
|
501
|
+
}
|
|
502
|
+
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
503
|
+
applyWrittenDocuments(pass, built);
|
|
504
|
+
await yieldToEventLoop();
|
|
505
|
+
if (interrupted(pass)) return false;
|
|
506
|
+
}
|
|
507
|
+
applyWrittenDocuments(pass, built);
|
|
508
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
509
|
+
pass.hooks.onSwitch();
|
|
510
|
+
return true;
|
|
511
|
+
}
|
|
512
|
+
async function rebuildTextIndexes(manager, hooks) {
|
|
513
|
+
const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
|
|
514
|
+
while (hooks.isCurrent()) {
|
|
515
|
+
const log = manager.openWriteLog();
|
|
516
|
+
try {
|
|
517
|
+
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
518
|
+
} finally {
|
|
519
|
+
log.close();
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return false;
|
|
523
|
+
}
|
|
524
|
+
|
|
455
525
|
// src/engine/events.ts
|
|
456
526
|
function eventHandlersOf(events) {
|
|
457
527
|
const handlers = /* @__PURE__ */ new Map();
|
|
@@ -474,11 +544,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
|
|
|
474
544
|
}
|
|
475
545
|
|
|
476
546
|
// src/engine/analysis-rebuild.ts
|
|
477
|
-
function yieldToEventLoop2() {
|
|
478
|
-
return new Promise((resolve) => {
|
|
479
|
-
setTimeout(resolve, 0);
|
|
480
|
-
});
|
|
481
|
-
}
|
|
482
547
|
function toError(value) {
|
|
483
548
|
return value instanceof Error ? value : new Error(String(value));
|
|
484
549
|
}
|
|
@@ -489,9 +554,13 @@ function wireAnalysisRebuild(wiring) {
|
|
|
489
554
|
resyncIndex: wiring.resyncIndex,
|
|
490
555
|
async persistAnalysisRevision(indexName) {
|
|
491
556
|
const durability = wiring.durabilityManager;
|
|
492
|
-
if (durability === null)
|
|
493
|
-
|
|
557
|
+
if (durability === null) {
|
|
558
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
494
561
|
await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
|
|
562
|
+
wiring.recordCurrentAnalysis(indexName);
|
|
563
|
+
await durability.persistMetadata(indexName);
|
|
495
564
|
},
|
|
496
565
|
emit(payload) {
|
|
497
566
|
emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
|
|
@@ -502,9 +571,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
502
571
|
const stale = /* @__PURE__ */ new Map();
|
|
503
572
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
504
573
|
let queue = Promise.resolve();
|
|
574
|
+
let active = null;
|
|
505
575
|
async function rebuildNow(indexName, entry) {
|
|
506
576
|
const manager = deps.getManager(indexName);
|
|
507
|
-
if (manager === void 0) {
|
|
577
|
+
if (manager === void 0 || entry.abandoned) {
|
|
508
578
|
entry.run = null;
|
|
509
579
|
return;
|
|
510
580
|
}
|
|
@@ -512,12 +582,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
512
582
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
513
583
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
514
584
|
try {
|
|
515
|
-
|
|
516
|
-
manager.
|
|
517
|
-
|
|
518
|
-
|
|
585
|
+
const rebuilt = await rebuildTextIndexes(manager, {
|
|
586
|
+
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
587
|
+
onPartitionBuilt(partitionsBuilt) {
|
|
588
|
+
entry.partitionsRebuilt = partitionsBuilt;
|
|
589
|
+
},
|
|
590
|
+
onSwitch() {
|
|
591
|
+
stale.delete(indexName);
|
|
592
|
+
}
|
|
593
|
+
});
|
|
594
|
+
if (!rebuilt) {
|
|
595
|
+
entry.run = null;
|
|
596
|
+
if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
|
|
597
|
+
return;
|
|
519
598
|
}
|
|
520
|
-
stale.delete(indexName);
|
|
521
599
|
await deps.persistAnalysisRevision(indexName);
|
|
522
600
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
523
601
|
deps.emit({
|
|
@@ -528,7 +606,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
528
606
|
});
|
|
529
607
|
} catch (err) {
|
|
530
608
|
const error = toError(err);
|
|
531
|
-
stale.set(indexName, entry);
|
|
532
609
|
entry.run = null;
|
|
533
610
|
await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
534
611
|
deps.emit({
|
|
@@ -541,11 +618,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
541
618
|
throw error;
|
|
542
619
|
}
|
|
543
620
|
}
|
|
621
|
+
function runTracked(indexName, entry) {
|
|
622
|
+
const done = rebuildNow(indexName, entry);
|
|
623
|
+
const tracked = { indexName, entry, done };
|
|
624
|
+
active = tracked;
|
|
625
|
+
return done.finally(() => {
|
|
626
|
+
if (active === tracked) active = null;
|
|
627
|
+
});
|
|
628
|
+
}
|
|
544
629
|
function enqueue(indexName, entry) {
|
|
545
630
|
if (entry.run !== null) {
|
|
546
631
|
return entry.run;
|
|
547
632
|
}
|
|
548
|
-
|
|
633
|
+
entry.abandoned = false;
|
|
634
|
+
const run = queue.then(() => runTracked(indexName, entry));
|
|
549
635
|
entry.run = run;
|
|
550
636
|
queue = run.then(
|
|
551
637
|
() => void 0,
|
|
@@ -580,7 +666,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
580
666
|
if (existing !== void 0 && existing.run !== null) {
|
|
581
667
|
return;
|
|
582
668
|
}
|
|
583
|
-
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
|
|
669
|
+
stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
|
|
584
670
|
},
|
|
585
671
|
clearStale(indexName) {
|
|
586
672
|
stale.delete(indexName);
|
|
@@ -591,6 +677,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
591
677
|
isRunning(indexName) {
|
|
592
678
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
593
679
|
},
|
|
680
|
+
async abandon(indexName) {
|
|
681
|
+
const entry = stale.get(indexName);
|
|
682
|
+
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
683
|
+
const running = active;
|
|
684
|
+
if (running === null || running.indexName !== indexName) return;
|
|
685
|
+
running.entry.abandoned = true;
|
|
686
|
+
await running.done.catch(() => void 0);
|
|
687
|
+
},
|
|
594
688
|
review(indexName) {
|
|
595
689
|
const entry = stale.get(indexName);
|
|
596
690
|
return entry === void 0 ? Promise.resolve() : review(indexName, entry);
|
|
@@ -1400,7 +1494,7 @@ function buildEntry(fields) {
|
|
|
1400
1494
|
}
|
|
1401
1495
|
|
|
1402
1496
|
// src/serialization/chunk-checksum.ts
|
|
1403
|
-
function
|
|
1497
|
+
function yieldToEventLoop2() {
|
|
1404
1498
|
if (typeof setImmediate === "function") {
|
|
1405
1499
|
return new Promise((resolve) => setImmediate(resolve));
|
|
1406
1500
|
}
|
|
@@ -1412,7 +1506,7 @@ async function advanceChecksum(start, payload) {
|
|
|
1412
1506
|
const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
|
|
1413
1507
|
state = crc32Update(state, payload.subarray(offset, end));
|
|
1414
1508
|
if (end < payload.length) {
|
|
1415
|
-
await
|
|
1509
|
+
await yieldToEventLoop2();
|
|
1416
1510
|
}
|
|
1417
1511
|
}
|
|
1418
1512
|
return state;
|
|
@@ -5639,7 +5733,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
5639
5733
|
manager.endBatchRemove();
|
|
5640
5734
|
}
|
|
5641
5735
|
}
|
|
5642
|
-
function
|
|
5736
|
+
function yieldToEventLoop3() {
|
|
5643
5737
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
5644
5738
|
}
|
|
5645
5739
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
@@ -5650,7 +5744,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
5650
5744
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
5651
5745
|
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
5652
5746
|
if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
|
|
5653
|
-
await
|
|
5747
|
+
await yieldToEventLoop3();
|
|
5654
5748
|
}
|
|
5655
5749
|
}
|
|
5656
5750
|
|
|
@@ -8078,7 +8172,7 @@ function metadataConfigOf(entry) {
|
|
|
8078
8172
|
b: entry.config.bm25?.b ?? 0.75,
|
|
8079
8173
|
...embedding !== void 0 ? { embedding } : {},
|
|
8080
8174
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
8081
|
-
analysisRevision: entry.
|
|
8175
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
8082
8176
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
8083
8177
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
8084
8178
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -8166,7 +8260,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
8166
8260
|
indexUuid: metadata.indexUuid ?? null,
|
|
8167
8261
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
8168
8262
|
documentCount: metadata.documentCount ?? 0,
|
|
8169
|
-
partitionCount: metadata.partitionCount
|
|
8263
|
+
partitionCount: metadata.partitionCount,
|
|
8264
|
+
analysisRevision: metadata.analysisRevision ?? null
|
|
8170
8265
|
});
|
|
8171
8266
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
8172
8267
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -8289,8 +8384,14 @@ function createIndexStateCoordinator(config, callbacks) {
|
|
|
8289
8384
|
return entry;
|
|
8290
8385
|
}
|
|
8291
8386
|
}
|
|
8387
|
+
function transitionPending(indexName, entry) {
|
|
8388
|
+
if (entries.get(indexName) !== entry) return true;
|
|
8389
|
+
if (entry.state === "dropping" && entry.dropPromise !== null) return true;
|
|
8390
|
+
return entry.state === "closing" && entry.closePromise !== null;
|
|
8391
|
+
}
|
|
8292
8392
|
async function reopen(indexName, explicit) {
|
|
8293
|
-
|
|
8393
|
+
let entry = await awaitPendingTransitions(indexName);
|
|
8394
|
+
while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
|
|
8294
8395
|
if (entry.state === "open") return;
|
|
8295
8396
|
if (explicit) {
|
|
8296
8397
|
entry.reopenFailures = 0;
|
|
@@ -10705,7 +10806,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
10705
10806
|
idleSweep: null
|
|
10706
10807
|
};
|
|
10707
10808
|
startIdleSweep(state);
|
|
10809
|
+
function settlingCopyWork(indexName) {
|
|
10810
|
+
return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
|
|
10811
|
+
}
|
|
10708
10812
|
async function resyncIndex(indexName, wasScaledOut) {
|
|
10813
|
+
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
10814
|
+
await pending;
|
|
10815
|
+
}
|
|
10709
10816
|
state.desyncedIndexes.delete(indexName);
|
|
10710
10817
|
if (!wasScaledOut) return;
|
|
10711
10818
|
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
@@ -11057,6 +11164,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
11057
11164
|
getManager: (indexName) => executor.getManager(indexName),
|
|
11058
11165
|
desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
|
|
11059
11166
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
11167
|
+
recordCurrentAnalysis: (indexName) => {
|
|
11168
|
+
const entry = indexRegistry.get(indexName);
|
|
11169
|
+
if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
|
|
11170
|
+
},
|
|
11060
11171
|
durabilityManager: durability?.manager ?? null
|
|
11061
11172
|
});
|
|
11062
11173
|
const indexState = wireIndexState({
|
|
@@ -11231,11 +11342,16 @@ function refuseCode(entry) {
|
|
|
11231
11342
|
}
|
|
11232
11343
|
function indexConfigurationOf(entry) {
|
|
11233
11344
|
refuseCode(entry);
|
|
11234
|
-
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
|
|
11345
|
+
return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
|
|
11235
11346
|
}
|
|
11236
11347
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
11237
11348
|
const adapter = config.embedding?.adapter;
|
|
11238
|
-
const payload = configurationPayloadOf(
|
|
11349
|
+
const payload = configurationPayloadOf(
|
|
11350
|
+
config,
|
|
11351
|
+
language,
|
|
11352
|
+
typeof adapter === "string" ? adapter : null,
|
|
11353
|
+
language.revision
|
|
11354
|
+
);
|
|
11239
11355
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
11240
11356
|
const bytes = encode(payload).byteLength;
|
|
11241
11357
|
if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
@@ -11245,12 +11361,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
11245
11361
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
11246
11362
|
);
|
|
11247
11363
|
}
|
|
11248
|
-
function configurationPayloadOf(config, language, adapterName) {
|
|
11364
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
11249
11365
|
const bm25 = config.bm25;
|
|
11250
11366
|
return {
|
|
11251
11367
|
schema: config.schema,
|
|
11252
11368
|
language: language.name,
|
|
11253
|
-
analysisRevision:
|
|
11369
|
+
...analysisRevision !== null ? { analysisRevision } : {},
|
|
11254
11370
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
11255
11371
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
11256
11372
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -11409,7 +11525,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
11409
11525
|
indexUuid: indexUuid ?? null,
|
|
11410
11526
|
heldPartitions: null,
|
|
11411
11527
|
documentCount: 0,
|
|
11412
|
-
partitionCount: indexConfig.partitions?.maxPartitions ?? 1
|
|
11528
|
+
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
11529
|
+
analysisRevision: language.revision
|
|
11413
11530
|
});
|
|
11414
11531
|
await core.indexState.registerOpen(name);
|
|
11415
11532
|
if (core.durability) {
|
|
@@ -11421,6 +11538,7 @@ async function dropEngineIndex(core, name) {
|
|
|
11421
11538
|
core.guardShutdown();
|
|
11422
11539
|
const entry = core.requireIndex(name);
|
|
11423
11540
|
await core.indexState.drop(name, async () => {
|
|
11541
|
+
await core.analysisRebuild.abandon(name);
|
|
11424
11542
|
await core.orchestrator.closeIndex(name);
|
|
11425
11543
|
if (core.durability) {
|
|
11426
11544
|
try {
|
|
@@ -14196,7 +14314,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
14196
14314
|
indexUuid: null,
|
|
14197
14315
|
heldPartitions: null,
|
|
14198
14316
|
documentCount: 0,
|
|
14199
|
-
partitionCount
|
|
14317
|
+
partitionCount,
|
|
14318
|
+
analysisRevision: prepared.storedRevision
|
|
14200
14319
|
});
|
|
14201
14320
|
await deps.indexState.registerOpen(indexName);
|
|
14202
14321
|
const release = await deps.indexState.acquire(indexName, false);
|
|
@@ -14581,6 +14700,15 @@ async function enterRead(core, indexName) {
|
|
|
14581
14700
|
core.guardShutdown();
|
|
14582
14701
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
14583
14702
|
}
|
|
14703
|
+
async function readUnderOneAnalysis(core, indexName, read) {
|
|
14704
|
+
let analysisStale = core.analysisRebuild.isStale(indexName);
|
|
14705
|
+
let result = await read();
|
|
14706
|
+
while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
|
|
14707
|
+
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
14708
|
+
result = await read();
|
|
14709
|
+
}
|
|
14710
|
+
return { result, analysisStale };
|
|
14711
|
+
}
|
|
14584
14712
|
function scopedParams(params, options) {
|
|
14585
14713
|
if (options?.globalStats === void 0) {
|
|
14586
14714
|
return params;
|
|
@@ -14613,7 +14741,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14613
14741
|
entry.embeddingAdapterName
|
|
14614
14742
|
);
|
|
14615
14743
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
14616
|
-
const
|
|
14744
|
+
const context = {
|
|
14617
14745
|
manager,
|
|
14618
14746
|
language: entry.language,
|
|
14619
14747
|
config: entry.config,
|
|
@@ -14623,7 +14751,15 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14623
14751
|
partitionIds: options?.partitionIds,
|
|
14624
14752
|
cursorBinding: queryBindingOf(params),
|
|
14625
14753
|
patternWorkCap: core.patternWorkCap
|
|
14626
|
-
}
|
|
14754
|
+
};
|
|
14755
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
14756
|
+
core,
|
|
14757
|
+
indexName,
|
|
14758
|
+
() => executeQuery(resolvedParams, context)
|
|
14759
|
+
);
|
|
14760
|
+
if (analysisStale) {
|
|
14761
|
+
result.analysisStale = true;
|
|
14762
|
+
}
|
|
14627
14763
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
14628
14764
|
const observed = { ...result, hits: [...result.hits] };
|
|
14629
14765
|
try {
|
|
@@ -14632,9 +14768,6 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
14632
14768
|
console.warn("afterSearch plugin hook error:", err);
|
|
14633
14769
|
}
|
|
14634
14770
|
}
|
|
14635
|
-
if (core.analysisRebuild.isStale(indexName)) {
|
|
14636
|
-
result.analysisStale = true;
|
|
14637
|
-
}
|
|
14638
14771
|
return result;
|
|
14639
14772
|
} finally {
|
|
14640
14773
|
release();
|
|
@@ -14655,7 +14788,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
14655
14788
|
entry.embeddingAdapterName
|
|
14656
14789
|
);
|
|
14657
14790
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
14658
|
-
const
|
|
14791
|
+
const context = {
|
|
14659
14792
|
manager,
|
|
14660
14793
|
language: entry.language,
|
|
14661
14794
|
config: entry.config,
|
|
@@ -14665,8 +14798,13 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
14665
14798
|
partitionIds: options?.partitionIds,
|
|
14666
14799
|
cursorBinding: queryBindingOf(params),
|
|
14667
14800
|
patternWorkCap: core.patternWorkCap
|
|
14668
|
-
}
|
|
14669
|
-
|
|
14801
|
+
};
|
|
14802
|
+
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
14803
|
+
core,
|
|
14804
|
+
indexName,
|
|
14805
|
+
() => executePreflight(resolvedParams, context)
|
|
14806
|
+
);
|
|
14807
|
+
if (analysisStale) {
|
|
14670
14808
|
result.analysisStale = true;
|
|
14671
14809
|
}
|
|
14672
14810
|
return result;
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
2
|
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf,
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-XIQZKKTK.mjs';
|
|
4
4
|
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
|
|
5
5
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
6
6
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
|
|
7
|
-
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
7
|
+
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-IA67SOJO.mjs';
|
|
8
8
|
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
|
|
9
9
|
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
10
10
|
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
@@ -5623,6 +5623,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
5623
5623
|
const entry = core.requireIndex(indexName);
|
|
5624
5624
|
const manager = core.requireManager(indexName);
|
|
5625
5625
|
let captured = 0;
|
|
5626
|
+
const config = await encodeIndexConfiguration(entry);
|
|
5626
5627
|
const checkpoint = await buildArchiveFromMemory({
|
|
5627
5628
|
indexName,
|
|
5628
5629
|
metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
|
|
@@ -5634,7 +5635,6 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
5634
5635
|
},
|
|
5635
5636
|
partitionId
|
|
5636
5637
|
});
|
|
5637
|
-
const config = await encodeIndexConfiguration(entry);
|
|
5638
5638
|
return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
|
|
5639
5639
|
} finally {
|
|
5640
5640
|
release();
|
|
@@ -5660,6 +5660,7 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
|
|
|
5660
5660
|
async function receiveArchive(indexName, partitionId, source, schema) {
|
|
5661
5661
|
const store = memoryCheckpointStore();
|
|
5662
5662
|
let language = "";
|
|
5663
|
+
let analysisRevision = null;
|
|
5663
5664
|
let received = null;
|
|
5664
5665
|
await readSnapshotArchive(createByteReader(source), {
|
|
5665
5666
|
config: async (bytes) => {
|
|
@@ -5671,6 +5672,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
5671
5672
|
const raw = decode(payload);
|
|
5672
5673
|
if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
|
|
5673
5674
|
language = raw.language;
|
|
5675
|
+
analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
|
|
5674
5676
|
},
|
|
5675
5677
|
manifest: async (manifest, sourceName) => {
|
|
5676
5678
|
const listed = manifest.partitions.map((partition) => partition.partitionId);
|
|
@@ -5697,7 +5699,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
5697
5699
|
file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
|
|
5698
5700
|
});
|
|
5699
5701
|
if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
|
|
5700
|
-
return { language, manifest: received, store };
|
|
5702
|
+
return { language, analysisRevision, manifest: received, store };
|
|
5701
5703
|
}
|
|
5702
5704
|
function emptyPartition(manager, partitionId) {
|
|
5703
5705
|
return {
|
|
@@ -5749,6 +5751,18 @@ async function replacePartition(manager, partitionId, archive) {
|
|
|
5749
5751
|
vectorIndex.scheduleBuild();
|
|
5750
5752
|
}
|
|
5751
5753
|
}
|
|
5754
|
+
function recordReceivedAnalysis(core, indexName, receivedRevision) {
|
|
5755
|
+
const entry = core.requireIndex(indexName);
|
|
5756
|
+
const current = entry.language.revision;
|
|
5757
|
+
if (receivedRevision === current) return;
|
|
5758
|
+
entry.analysisRevision = entry.analysisRevision === current || entry.analysisRevision === receivedRevision ? receivedRevision : null;
|
|
5759
|
+
core.analysisRebuild.markStale({
|
|
5760
|
+
indexName,
|
|
5761
|
+
language: entry.language.name,
|
|
5762
|
+
storedRevision: entry.analysisRevision,
|
|
5763
|
+
currentRevision: current
|
|
5764
|
+
});
|
|
5765
|
+
}
|
|
5752
5766
|
async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
|
|
5753
5767
|
core.guardShutdown();
|
|
5754
5768
|
requireRestoreTarget(indexName, partitionId, partitionCount);
|
|
@@ -5775,6 +5789,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
5775
5789
|
const wasScaledOut = core.orchestrator.desyncIndex(indexName);
|
|
5776
5790
|
try {
|
|
5777
5791
|
await replacePartition(core.requireManager(indexName), partitionId, archive);
|
|
5792
|
+
recordReceivedAnalysis(core, indexName, archive.analysisRevision);
|
|
5778
5793
|
} finally {
|
|
5779
5794
|
await core.orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
5780
5795
|
}
|