@delali/narsil 0.3.1-next.84 → 0.3.1-next.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,19 @@
1
1
  import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-IA67SOJO.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-XPK3GB2P.mjs';
5
5
  import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
7
7
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
8
- import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
9
- import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
8
+ import { releaseLocksHeldBy } from './chunk-B3F7LZXM.mjs';
9
+ import { requireSearchBackendSettings } from './chunk-NFF5TDRG.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
11
11
  import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
12
12
  import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
13
13
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
14
14
  import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
15
15
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
16
- import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
16
+ import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-YRX6XMOQ.mjs';
17
17
  import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
18
18
  import { openSync, close, readFileSync } from 'fs';
19
19
  import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
@@ -8980,6 +8980,7 @@ function shareFrozenSegment(segment, analysis) {
8980
8980
  }
8981
8981
 
8982
8982
  // src/engine/worker-resync.ts
8983
+ var copyLoadSequence = 0;
8983
8984
  function captureComposite(manager, partitionId) {
8984
8985
  const partition = manager.getPartition(partitionId);
8985
8986
  if (!isCompositePartition(partition)) return null;
@@ -9040,8 +9041,8 @@ async function sendPartition(indexName, executors, captured) {
9040
9041
  )
9041
9042
  );
9042
9043
  }
9043
- async function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
9044
- await loadIndexOntoWorkers(
9044
+ function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
9045
+ return loadIndexOntoWorkers(
9045
9046
  indexName,
9046
9047
  pool.getAllExecutors(),
9047
9048
  config,
@@ -9051,6 +9052,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
9051
9052
  );
9052
9053
  }
9053
9054
  async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
9055
+ copyLoadSequence += 1;
9056
+ const copyLoad = `${indexName}#${copyLoadSequence}`;
9054
9057
  const captured = capturePartitions(manager);
9055
9058
  await Promise.allSettled(
9056
9059
  executors.map(
@@ -9065,13 +9068,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
9065
9068
  indexName,
9066
9069
  config,
9067
9070
  requestId: `resync-create-${indexName}`,
9068
- analysisStale
9071
+ analysisStale,
9072
+ copyLoad
9069
9073
  })
9070
9074
  )
9071
9075
  );
9072
9076
  for (const partition of captured) {
9073
9077
  await sendPartition(indexName, executors, partition);
9074
9078
  }
9079
+ return copyLoad;
9075
9080
  }
9076
9081
 
9077
9082
  // src/engine/orchestration/constants.ts
@@ -9299,6 +9304,14 @@ function enqueueReplication(state, indexName, action) {
9299
9304
  queue.inFlight.push(run);
9300
9305
  queue.tail = run;
9301
9306
  }
9307
+ function markCopiesLoaded(state, indexName, copyLoad) {
9308
+ enqueueReplication(state, indexName, {
9309
+ type: "copyLoaded",
9310
+ indexName,
9311
+ copyLoad,
9312
+ requestId: `copy-loaded-${copyLoad}`
9313
+ });
9314
+ }
9302
9315
  async function replicateToWorkers(state, action) {
9303
9316
  if (!("indexName" in action)) {
9304
9317
  if (state.workerPool === null) return;
@@ -10264,8 +10277,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
10264
10277
  if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
10265
10278
  const buffered = [];
10266
10279
  state.copyLoadBuffers.set(indexName, buffered);
10280
+ let copyLoad;
10267
10281
  try {
10268
- await loadIndexOntoWorkers(
10282
+ copyLoad = await loadIndexOntoWorkers(
10269
10283
  indexName,
10270
10284
  [replacement.executor],
10271
10285
  entry.config,
@@ -10278,6 +10292,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
10278
10292
  state.copyLoadBuffers.delete(indexName);
10279
10293
  for (const action of buffered) enqueueReplication(state, indexName, action);
10280
10294
  }
10295
+ markCopiesLoaded(state, indexName, copyLoad);
10281
10296
  refreshVectorCopies(state, indexName);
10282
10297
  }
10283
10298
  async function replaceWorker(state, pool, workerId) {
@@ -10454,7 +10469,13 @@ async function loadCopies(state, indexName, reason) {
10454
10469
  state.copyLoadBuffers.set(indexName, buffered);
10455
10470
  try {
10456
10471
  freezeLiveTailsBeforeCopiesLoad(manager);
10457
- await transferIndexToPool(indexName, pool, entry.config, manager, state.callbacks?.isAnalysisStale?.(indexName));
10472
+ const copyLoad = await transferIndexToPool(
10473
+ indexName,
10474
+ pool,
10475
+ entry.config,
10476
+ manager,
10477
+ state.callbacks?.isAnalysisStale?.(indexName)
10478
+ );
10458
10479
  if (state.desyncedIndexes.has(indexName)) {
10459
10480
  discardLoadedCopies(state, pool, indexName);
10460
10481
  return;
@@ -10463,6 +10484,7 @@ async function loadCopies(state, indexName, reason) {
10463
10484
  state.lastAccessAt.set(indexName, Date.now());
10464
10485
  state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
10465
10486
  for (const action of buffered) enqueueReplication(state, indexName, action);
10487
+ markCopiesLoaded(state, indexName, copyLoad);
10466
10488
  } catch (err) {
10467
10489
  if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
10468
10490
  throw err;
@@ -10870,8 +10892,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10870
10892
  const manager = executor.getManager(indexName);
10871
10893
  if (pool !== null && entry !== void 0 && manager !== void 0) {
10872
10894
  try {
10873
- await transferIndexToPool(indexName, pool, entry.config, manager, callbacks?.isAnalysisStale?.(indexName));
10895
+ const copyLoad = await transferIndexToPool(
10896
+ indexName,
10897
+ pool,
10898
+ entry.config,
10899
+ manager,
10900
+ callbacks?.isAnalysisStale?.(indexName)
10901
+ );
10874
10902
  state.scaledOutIndexes.add(indexName);
10903
+ markCopiesLoaded(state, indexName, copyLoad);
10875
10904
  refreshVectorCopies(state, indexName);
10876
10905
  } catch (repairError) {
10877
10906
  throw new NarsilError(
@@ -18,7 +18,7 @@ function stopUsingTheNativeSearchCore(error) {
18
18
  }
19
19
  function noteFallback(operation) {
20
20
  }
21
- var NATIVE_CORE_ABI_VERSION = 2;
21
+ var NATIVE_CORE_ABI_VERSION = 3;
22
22
  var BINARY_FILE_NAME = "narsil-core.node";
23
23
  var MUSL_DIRECTORY = "musl";
24
24
  var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
@@ -1,4 +1,4 @@
1
- import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-B3UOQJD3.mjs';
1
+ import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-NFF5TDRG.mjs';
2
2
  import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
3
3
  import { MAX_SORT_FIELDS } from './chunk-KJ7LB3TR.mjs';
4
4
  import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
@@ -251,6 +251,7 @@ var MIN_UNBOUNDED_SERVER_LIMIT = 0;
251
251
  var RESPONSE_STREAM_THRESHOLD_BYTES = 64 * 1024;
252
252
  var MAX_INDEX_TOUCH_INTERVAL_MS = 1e3;
253
253
  var INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT = 4;
254
+ var REQUEST_THREAD_DEPARTURE_DRAIN_MS = 100;
254
255
  var DEFAULT_MAX_RETAINED_TASKS = 1e3;
255
256
  var DEFAULT_TASK_PAGE_SIZE = 20;
256
257
  var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
@@ -913,8 +914,8 @@ function sinkWritingAfterStatus(res, headers) {
913
914
  res.writeHeader(key, value);
914
915
  return sink;
915
916
  },
916
- end(body) {
917
- res.end(body);
917
+ end(body, closeConnection) {
918
+ res.end(body, closeConnection);
918
919
  return sink;
919
920
  },
920
921
  write(chunk) {
@@ -981,7 +982,7 @@ function sendDenial(res, authorization) {
981
982
  sendError(res, authorization.denial.status, authorization.denial.code, authorization.denial.message);
982
983
  }
983
984
  function createRouteRunner(deps) {
984
- const { authorize, gate, corsHeaders } = deps;
985
+ const { authorize, gate, corsHeaders, track } = deps;
985
986
  return (handler, opts) => {
986
987
  const paramCount = opts.paramCount ?? 0;
987
988
  const needsBody = opts.needsBody ?? false;
@@ -1026,6 +1027,7 @@ function createRouteRunner(deps) {
1026
1027
  );
1027
1028
  return;
1028
1029
  }
1030
+ const release = track?.(res, abort);
1029
1031
  const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
1030
1032
  const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
1031
1033
  const spoolPromise = opts.spoolsBody === true ? spoolBody(
@@ -1034,7 +1036,7 @@ function createRouteRunner(deps) {
1034
1036
  abort,
1035
1037
  hookPromise.then((authorization) => authorization.allowed)
1036
1038
  ).catch(() => null) : Promise.resolve(null);
1037
- Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
1039
+ const running = Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
1038
1040
  const discardSpool = () => {
1039
1041
  void spooledBody?.remove().catch(() => void 0);
1040
1042
  };
@@ -1064,6 +1066,7 @@ function createRouteRunner(deps) {
1064
1066
  }
1065
1067
  }).catch(() => {
1066
1068
  });
1069
+ if (release !== void 0) void running.then(release);
1067
1070
  };
1068
1071
  };
1069
1072
  }
@@ -1195,4 +1198,26 @@ function childAppOf(app) {
1195
1198
  return candidate;
1196
1199
  }
1197
1200
 
1198
- export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
1201
+ // src/server/request-threads/departure.ts
1202
+ var DEPARTURE_WORD = 0;
1203
+ var AWAITING_DEPARTURE = 0;
1204
+ var DEPARTURE_CONFIRMED = 1;
1205
+ function createDepartureWord() {
1206
+ return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
1207
+ }
1208
+ function confirmDeparture(word) {
1209
+ Atomics.store(word, DEPARTURE_WORD, DEPARTURE_CONFIRMED);
1210
+ Atomics.notify(word, DEPARTURE_WORD);
1211
+ }
1212
+ function departureConfirmed(word) {
1213
+ return Atomics.load(word, DEPARTURE_WORD) === DEPARTURE_CONFIRMED;
1214
+ }
1215
+ function awaitDepartureConfirmation(word) {
1216
+ while (!departureConfirmed(word)) Atomics.wait(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
1217
+ }
1218
+ async function departureConfirmation(word) {
1219
+ const waiting = Atomics.waitAsync(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
1220
+ if (waiting.async) await waiting.value;
1221
+ }
1222
+
1223
+ export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };