@delali/narsil 0.3.1-next.83 → 0.3.1-next.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-B3UOQJD3.mjs';
1
+ import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-NFF5TDRG.mjs';
2
2
  import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
3
3
  import { MAX_SORT_FIELDS } from './chunk-KJ7LB3TR.mjs';
4
4
  import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
@@ -251,6 +251,7 @@ var MIN_UNBOUNDED_SERVER_LIMIT = 0;
251
251
  var RESPONSE_STREAM_THRESHOLD_BYTES = 64 * 1024;
252
252
  var MAX_INDEX_TOUCH_INTERVAL_MS = 1e3;
253
253
  var INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT = 4;
254
+ var REQUEST_THREAD_DEPARTURE_DRAIN_MS = 100;
254
255
  var DEFAULT_MAX_RETAINED_TASKS = 1e3;
255
256
  var DEFAULT_TASK_PAGE_SIZE = 20;
256
257
  var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
@@ -913,8 +914,8 @@ function sinkWritingAfterStatus(res, headers) {
913
914
  res.writeHeader(key, value);
914
915
  return sink;
915
916
  },
916
- end(body) {
917
- res.end(body);
917
+ end(body, closeConnection) {
918
+ res.end(body, closeConnection);
918
919
  return sink;
919
920
  },
920
921
  write(chunk) {
@@ -981,7 +982,7 @@ function sendDenial(res, authorization) {
981
982
  sendError(res, authorization.denial.status, authorization.denial.code, authorization.denial.message);
982
983
  }
983
984
  function createRouteRunner(deps) {
984
- const { authorize, gate, corsHeaders } = deps;
985
+ const { authorize, gate, corsHeaders, track } = deps;
985
986
  return (handler, opts) => {
986
987
  const paramCount = opts.paramCount ?? 0;
987
988
  const needsBody = opts.needsBody ?? false;
@@ -1026,6 +1027,7 @@ function createRouteRunner(deps) {
1026
1027
  );
1027
1028
  return;
1028
1029
  }
1030
+ const release = track?.(res, abort);
1029
1031
  const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
1030
1032
  const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
1031
1033
  const spoolPromise = opts.spoolsBody === true ? spoolBody(
@@ -1034,7 +1036,7 @@ function createRouteRunner(deps) {
1034
1036
  abort,
1035
1037
  hookPromise.then((authorization) => authorization.allowed)
1036
1038
  ).catch(() => null) : Promise.resolve(null);
1037
- Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
1039
+ const running = Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
1038
1040
  const discardSpool = () => {
1039
1041
  void spooledBody?.remove().catch(() => void 0);
1040
1042
  };
@@ -1064,6 +1066,7 @@ function createRouteRunner(deps) {
1064
1066
  }
1065
1067
  }).catch(() => {
1066
1068
  });
1069
+ if (release !== void 0) void running.then(release);
1067
1070
  };
1068
1071
  };
1069
1072
  }
@@ -1195,4 +1198,26 @@ function childAppOf(app) {
1195
1198
  return candidate;
1196
1199
  }
1197
1200
 
1198
- export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
1201
+ // src/server/request-threads/departure.ts
1202
+ var DEPARTURE_WORD = 0;
1203
+ var AWAITING_DEPARTURE = 0;
1204
+ var DEPARTURE_CONFIRMED = 1;
1205
+ function createDepartureWord() {
1206
+ return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
1207
+ }
1208
+ function confirmDeparture(word) {
1209
+ Atomics.store(word, DEPARTURE_WORD, DEPARTURE_CONFIRMED);
1210
+ Atomics.notify(word, DEPARTURE_WORD);
1211
+ }
1212
+ function departureConfirmed(word) {
1213
+ return Atomics.load(word, DEPARTURE_WORD) === DEPARTURE_CONFIRMED;
1214
+ }
1215
+ function awaitDepartureConfirmation(word) {
1216
+ while (!departureConfirmed(word)) Atomics.wait(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
1217
+ }
1218
+ async function departureConfirmation(word) {
1219
+ const waiting = Atomics.waitAsync(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
1220
+ if (waiting.async) await waiting.value;
1221
+ }
1222
+
1223
+ export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
@@ -9,6 +9,7 @@ var KNOWN_ACTION_TYPES = [
9
9
  "has",
10
10
  "count",
11
11
  "createIndex",
12
+ "copyLoaded",
12
13
  "loadVectorCopy",
13
14
  "dropVectorCopy",
14
15
  "insertVectorOrdinals",
@@ -1,11 +1,11 @@
1
- export { createDirectExecutor } from './chunk-CUP2E4OH.mjs';
1
+ export { createDirectExecutor } from './chunk-RCSXJCI5.mjs';
2
2
  import './chunk-XYDP7CCJ.mjs';
3
3
  import './chunk-OLXM65YH.mjs';
4
4
  import './chunk-ZCKPRCYI.mjs';
5
5
  import './chunk-TGP5M5R7.mjs';
6
- import './chunk-RZXNWQSM.mjs';
6
+ import './chunk-NIWDQTZB.mjs';
7
7
  import './chunk-ZJ4CSAB2.mjs';
8
- import './chunk-B3UOQJD3.mjs';
8
+ import './chunk-NFF5TDRG.mjs';
9
9
  import './chunk-SREPGC6O.mjs';
10
10
  import './chunk-WQBCBKY3.mjs';
11
11
  import './chunk-4VBG6PEH.mjs';
@@ -1,17 +1,17 @@
1
1
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
2
2
  import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
3
- import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-7JZZ3HNV.mjs';
3
+ import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-67MQF5CV.mjs';
4
4
  import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
5
5
  import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
6
6
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
7
- import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-CUP2E4OH.mjs';
7
+ import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-RCSXJCI5.mjs';
8
8
  import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
9
9
  import { getLanguage } from './chunk-OLXM65YH.mjs';
10
10
  import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
11
11
  import './chunk-TGP5M5R7.mjs';
12
- import './chunk-RZXNWQSM.mjs';
12
+ import './chunk-NIWDQTZB.mjs';
13
13
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
14
- import './chunk-B3UOQJD3.mjs';
14
+ import './chunk-NFF5TDRG.mjs';
15
15
  import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
16
16
  import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
17
17
  import './chunk-4VBG6PEH.mjs';
@@ -23,7 +23,7 @@ import './chunk-U2BKYEC7.mjs';
23
23
  import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
24
24
  import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
25
25
  import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
26
- import './chunk-POWOTUKJ.mjs';
26
+ import './chunk-YRX6XMOQ.mjs';
27
27
  import { ErrorCodes, NarsilError, describeError } from './chunk-TPJU7NDJ.mjs';
28
28
  import { decode, encode } from '@msgpack/msgpack';
29
29
  import { randomUUID } from 'crypto';
@@ -5623,6 +5623,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5623
5623
  const entry = core.requireIndex(indexName);
5624
5624
  const manager = core.requireManager(indexName);
5625
5625
  let captured = 0;
5626
+ const config = await encodeIndexConfiguration(entry);
5626
5627
  const checkpoint = await buildArchiveFromMemory({
5627
5628
  indexName,
5628
5629
  metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
@@ -5634,7 +5635,6 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5634
5635
  },
5635
5636
  partitionId
5636
5637
  });
5637
- const config = await encodeIndexConfiguration(entry);
5638
5638
  return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
5639
5639
  } finally {
5640
5640
  release();
@@ -5660,6 +5660,7 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
5660
5660
  async function receiveArchive(indexName, partitionId, source, schema) {
5661
5661
  const store = memoryCheckpointStore();
5662
5662
  let language = "";
5663
+ let analysisRevision = null;
5663
5664
  let received = null;
5664
5665
  await readSnapshotArchive(createByteReader(source), {
5665
5666
  config: async (bytes) => {
@@ -5671,6 +5672,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5671
5672
  const raw = decode(payload);
5672
5673
  if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
5673
5674
  language = raw.language;
5675
+ analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
5674
5676
  },
5675
5677
  manifest: async (manifest, sourceName) => {
5676
5678
  const listed = manifest.partitions.map((partition) => partition.partitionId);
@@ -5697,7 +5699,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5697
5699
  file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
5698
5700
  });
5699
5701
  if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
5700
- return { language, manifest: received, store };
5702
+ return { language, analysisRevision, manifest: received, store };
5701
5703
  }
5702
5704
  function emptyPartition(manager, partitionId) {
5703
5705
  return {
@@ -5749,6 +5751,18 @@ async function replacePartition(manager, partitionId, archive) {
5749
5751
  vectorIndex.scheduleBuild();
5750
5752
  }
5751
5753
  }
5754
+ function recordReceivedAnalysis(core, indexName, receivedRevision) {
5755
+ const entry = core.requireIndex(indexName);
5756
+ const current = entry.language.revision;
5757
+ if (receivedRevision === current) return;
5758
+ entry.analysisRevision = entry.analysisRevision === current || entry.analysisRevision === receivedRevision ? receivedRevision : null;
5759
+ core.analysisRebuild.markStale({
5760
+ indexName,
5761
+ language: entry.language.name,
5762
+ storedRevision: entry.analysisRevision,
5763
+ currentRevision: current
5764
+ });
5765
+ }
5752
5766
  async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
5753
5767
  core.guardShutdown();
5754
5768
  requireRestoreTarget(indexName, partitionId, partitionCount);
@@ -5775,6 +5789,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
5775
5789
  const wasScaledOut = core.orchestrator.desyncIndex(indexName);
5776
5790
  try {
5777
5791
  await replacePartition(core.requireManager(indexName), partitionId, archive);
5792
+ recordReceivedAnalysis(core, indexName, archive.analysisRevision);
5778
5793
  } finally {
5779
5794
  await core.orchestrator.resyncIndex(indexName, wasScaledOut);
5780
5795
  }