@delali/narsil 0.3.1-next.84 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XIQZKKTK.mjs → chunk-67MQF5CV.mjs} +39 -10
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-IA67SOJO.mjs → chunk-RCSXJCI5.mjs} +271 -93
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6WWFY5EI.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +5 -5
- package/dist/index.browser.mjs +763 -161
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-RCSXJCI5.mjs';
|
|
5
5
|
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
6
|
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
7
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
8
|
-
import { releaseLocksHeldBy } from './chunk-
|
|
9
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
8
|
+
import { releaseLocksHeldBy } from './chunk-NIWDQTZB.mjs';
|
|
9
|
+
import { requireSearchBackendSettings } from './chunk-NFF5TDRG.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
|
|
11
11
|
import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
|
|
12
12
|
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
|
|
13
13
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
14
14
|
import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
16
|
-
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-
|
|
16
|
+
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-YRX6XMOQ.mjs';
|
|
17
17
|
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
18
18
|
import { openSync, close, readFileSync } from 'fs';
|
|
19
19
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
@@ -8980,6 +8980,7 @@ function shareFrozenSegment(segment, analysis) {
|
|
|
8980
8980
|
}
|
|
8981
8981
|
|
|
8982
8982
|
// src/engine/worker-resync.ts
|
|
8983
|
+
var copyLoadSequence = 0;
|
|
8983
8984
|
function captureComposite(manager, partitionId) {
|
|
8984
8985
|
const partition = manager.getPartition(partitionId);
|
|
8985
8986
|
if (!isCompositePartition(partition)) return null;
|
|
@@ -9040,8 +9041,8 @@ async function sendPartition(indexName, executors, captured) {
|
|
|
9040
9041
|
)
|
|
9041
9042
|
);
|
|
9042
9043
|
}
|
|
9043
|
-
|
|
9044
|
-
|
|
9044
|
+
function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
|
|
9045
|
+
return loadIndexOntoWorkers(
|
|
9045
9046
|
indexName,
|
|
9046
9047
|
pool.getAllExecutors(),
|
|
9047
9048
|
config,
|
|
@@ -9051,6 +9052,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
|
|
|
9051
9052
|
);
|
|
9052
9053
|
}
|
|
9053
9054
|
async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
|
|
9055
|
+
copyLoadSequence += 1;
|
|
9056
|
+
const copyLoad = `${indexName}#${copyLoadSequence}`;
|
|
9054
9057
|
const captured = capturePartitions(manager);
|
|
9055
9058
|
await Promise.allSettled(
|
|
9056
9059
|
executors.map(
|
|
@@ -9065,13 +9068,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
9065
9068
|
indexName,
|
|
9066
9069
|
config,
|
|
9067
9070
|
requestId: `resync-create-${indexName}`,
|
|
9068
|
-
analysisStale
|
|
9071
|
+
analysisStale,
|
|
9072
|
+
copyLoad
|
|
9069
9073
|
})
|
|
9070
9074
|
)
|
|
9071
9075
|
);
|
|
9072
9076
|
for (const partition of captured) {
|
|
9073
9077
|
await sendPartition(indexName, executors, partition);
|
|
9074
9078
|
}
|
|
9079
|
+
return copyLoad;
|
|
9075
9080
|
}
|
|
9076
9081
|
|
|
9077
9082
|
// src/engine/orchestration/constants.ts
|
|
@@ -9299,6 +9304,14 @@ function enqueueReplication(state, indexName, action) {
|
|
|
9299
9304
|
queue.inFlight.push(run);
|
|
9300
9305
|
queue.tail = run;
|
|
9301
9306
|
}
|
|
9307
|
+
function markCopiesLoaded(state, indexName, copyLoad) {
|
|
9308
|
+
enqueueReplication(state, indexName, {
|
|
9309
|
+
type: "copyLoaded",
|
|
9310
|
+
indexName,
|
|
9311
|
+
copyLoad,
|
|
9312
|
+
requestId: `copy-loaded-${copyLoad}`
|
|
9313
|
+
});
|
|
9314
|
+
}
|
|
9302
9315
|
async function replicateToWorkers(state, action) {
|
|
9303
9316
|
if (!("indexName" in action)) {
|
|
9304
9317
|
if (state.workerPool === null) return;
|
|
@@ -10264,8 +10277,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
10264
10277
|
if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
|
|
10265
10278
|
const buffered = [];
|
|
10266
10279
|
state.copyLoadBuffers.set(indexName, buffered);
|
|
10280
|
+
let copyLoad;
|
|
10267
10281
|
try {
|
|
10268
|
-
await loadIndexOntoWorkers(
|
|
10282
|
+
copyLoad = await loadIndexOntoWorkers(
|
|
10269
10283
|
indexName,
|
|
10270
10284
|
[replacement.executor],
|
|
10271
10285
|
entry.config,
|
|
@@ -10278,6 +10292,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
10278
10292
|
state.copyLoadBuffers.delete(indexName);
|
|
10279
10293
|
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
10280
10294
|
}
|
|
10295
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10281
10296
|
refreshVectorCopies(state, indexName);
|
|
10282
10297
|
}
|
|
10283
10298
|
async function replaceWorker(state, pool, workerId) {
|
|
@@ -10454,7 +10469,13 @@ async function loadCopies(state, indexName, reason) {
|
|
|
10454
10469
|
state.copyLoadBuffers.set(indexName, buffered);
|
|
10455
10470
|
try {
|
|
10456
10471
|
freezeLiveTailsBeforeCopiesLoad(manager);
|
|
10457
|
-
|
|
10472
|
+
const copyLoad = await transferIndexToPool(
|
|
10473
|
+
indexName,
|
|
10474
|
+
pool,
|
|
10475
|
+
entry.config,
|
|
10476
|
+
manager,
|
|
10477
|
+
state.callbacks?.isAnalysisStale?.(indexName)
|
|
10478
|
+
);
|
|
10458
10479
|
if (state.desyncedIndexes.has(indexName)) {
|
|
10459
10480
|
discardLoadedCopies(state, pool, indexName);
|
|
10460
10481
|
return;
|
|
@@ -10463,6 +10484,7 @@ async function loadCopies(state, indexName, reason) {
|
|
|
10463
10484
|
state.lastAccessAt.set(indexName, Date.now());
|
|
10464
10485
|
state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
|
|
10465
10486
|
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
10487
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10466
10488
|
} catch (err) {
|
|
10467
10489
|
if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
|
|
10468
10490
|
throw err;
|
|
@@ -10870,8 +10892,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
10870
10892
|
const manager = executor.getManager(indexName);
|
|
10871
10893
|
if (pool !== null && entry !== void 0 && manager !== void 0) {
|
|
10872
10894
|
try {
|
|
10873
|
-
|
|
10895
|
+
const copyLoad = await transferIndexToPool(
|
|
10896
|
+
indexName,
|
|
10897
|
+
pool,
|
|
10898
|
+
entry.config,
|
|
10899
|
+
manager,
|
|
10900
|
+
callbacks?.isAnalysisStale?.(indexName)
|
|
10901
|
+
);
|
|
10874
10902
|
state.scaledOutIndexes.add(indexName);
|
|
10903
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
10875
10904
|
refreshVectorCopies(state, indexName);
|
|
10876
10905
|
} catch (repairError) {
|
|
10877
10906
|
throw new NarsilError(
|
|
@@ -18,7 +18,7 @@ function stopUsingTheNativeSearchCore(error) {
|
|
|
18
18
|
}
|
|
19
19
|
function noteFallback(operation) {
|
|
20
20
|
}
|
|
21
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
21
|
+
var NATIVE_CORE_ABI_VERSION = 3;
|
|
22
22
|
var BINARY_FILE_NAME = "narsil-core.node";
|
|
23
23
|
var MUSL_DIRECTORY = "musl";
|
|
24
24
|
var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
|