@delali/narsil 0.3.1-next.84 → 0.3.1-next.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,15 +1,15 @@
1
- export { VERSION, createNarsil } from './chunk-XIQZKKTK.mjs';
1
+ export { VERSION, createNarsil } from './chunk-IDLP54FH.mjs';
2
2
  import './chunk-4D6OH4EF.mjs';
3
3
  import './chunk-GCM3JFVA.mjs';
4
4
  import './chunk-2453RJYK.mjs';
5
- export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-IA67SOJO.mjs';
5
+ export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-XPK3GB2P.mjs';
6
6
  export { registerPartialWordSearch, registerPatternSearch } from './chunk-XYDP7CCJ.mjs';
7
7
  export { getLanguage, registerLanguage } from './chunk-OLXM65YH.mjs';
8
8
  import './chunk-ZCKPRCYI.mjs';
9
9
  import './chunk-TGP5M5R7.mjs';
10
- export { isSimdAvailable } from './chunk-RZXNWQSM.mjs';
10
+ export { isSimdAvailable } from './chunk-B3F7LZXM.mjs';
11
11
  import './chunk-ZJ4CSAB2.mjs';
12
- import './chunk-B3UOQJD3.mjs';
12
+ import './chunk-NFF5TDRG.mjs';
13
13
  import './chunk-SREPGC6O.mjs';
14
14
  import './chunk-WQBCBKY3.mjs';
15
15
  export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-4VBG6PEH.mjs';
@@ -20,5 +20,5 @@ import './chunk-LS5IRM23.mjs';
20
20
  import './chunk-U2BKYEC7.mjs';
21
21
  import './chunk-R6A773WK.mjs';
22
22
  import './chunk-YTJCGKM4.mjs';
23
- import './chunk-POWOTUKJ.mjs';
23
+ import './chunk-YRX6XMOQ.mjs';
24
24
  export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-TPJU7NDJ.mjs';
@@ -1,15 +1,15 @@
1
- import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-XIQZKKTK.mjs';
1
+ import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-IDLP54FH.mjs';
2
2
  import '../../chunk-4D6OH4EF.mjs';
3
3
  import '../../chunk-GCM3JFVA.mjs';
4
4
  import '../../chunk-2453RJYK.mjs';
5
- import '../../chunk-IA67SOJO.mjs';
5
+ import '../../chunk-XPK3GB2P.mjs';
6
6
  import '../../chunk-XYDP7CCJ.mjs';
7
7
  import '../../chunk-OLXM65YH.mjs';
8
8
  import '../../chunk-ZCKPRCYI.mjs';
9
9
  import '../../chunk-TGP5M5R7.mjs';
10
- import '../../chunk-RZXNWQSM.mjs';
10
+ import '../../chunk-B3F7LZXM.mjs';
11
11
  import '../../chunk-ZJ4CSAB2.mjs';
12
- import '../../chunk-B3UOQJD3.mjs';
12
+ import '../../chunk-NFF5TDRG.mjs';
13
13
  import '../../chunk-SREPGC6O.mjs';
14
14
  import '../../chunk-WQBCBKY3.mjs';
15
15
  import '../../chunk-4VBG6PEH.mjs';
@@ -20,7 +20,7 @@ import '../../chunk-LS5IRM23.mjs';
20
20
  import '../../chunk-U2BKYEC7.mjs';
21
21
  import '../../chunk-R6A773WK.mjs';
22
22
  import '../../chunk-YTJCGKM4.mjs';
23
- import '../../chunk-POWOTUKJ.mjs';
23
+ import '../../chunk-YRX6XMOQ.mjs';
24
24
  import '../../chunk-TPJU7NDJ.mjs';
25
25
 
26
26
  // src/persistence/durability/rebuild.ts
package/dist/server.mjs CHANGED
@@ -1,11 +1,11 @@
1
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, badRequest, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-XZF4LZ3X.mjs';
2
- export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-XZF4LZ3X.mjs';
1
+ import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, badRequest, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-RM6FRYEC.mjs';
2
+ export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-RM6FRYEC.mjs';
3
3
  import { validateIndexName } from './chunk-GNYXCLJS.mjs';
4
4
  import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-2453RJYK.mjs';
5
5
  import { detectRuntime } from './chunk-OLXM65YH.mjs';
6
6
  import './chunk-ZCKPRCYI.mjs';
7
7
  import './chunk-TGP5M5R7.mjs';
8
- import './chunk-B3UOQJD3.mjs';
8
+ import './chunk-NFF5TDRG.mjs';
9
9
  import './chunk-WQBCBKY3.mjs';
10
10
  import './chunk-KJ7LB3TR.mjs';
11
11
  import './chunk-KEHK2O6P.mjs';
@@ -1038,7 +1038,7 @@ var RELAYED_TYPES = /* @__PURE__ */ new Set(["request", "authorize", "embed", "s
1038
1038
  function isRelayRequest(message) {
1039
1039
  if (typeof message !== "object" || message === null) return false;
1040
1040
  const { type, requestId } = message;
1041
- if (type === "touch") return true;
1041
+ if (type === "touch" || type === "leaving") return true;
1042
1042
  return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
1043
1043
  }
1044
1044
  function spooledBodyOf(relayed) {
@@ -1124,10 +1124,24 @@ async function startRequestThreads(options) {
1124
1124
  const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
1125
1125
  const threads = /* @__PURE__ */ new Map();
1126
1126
  let stopped = false;
1127
- function relayFrom(port) {
1127
+ function releaseDescriptor(thread) {
1128
+ if (thread.descriptor !== null) {
1129
+ try {
1130
+ acceptor.removeChildAppDescriptor(thread.descriptor);
1131
+ } catch {
1132
+ }
1133
+ }
1134
+ confirmDeparture(thread.departure);
1135
+ }
1136
+ function relayFrom(thread) {
1137
+ const { port } = thread;
1128
1138
  const streams = /* @__PURE__ */ new Map();
1129
1139
  port.on("message", (message) => {
1130
1140
  if (!isRelayRequest(message)) return;
1141
+ if (message.type === "leaving") {
1142
+ releaseDescriptor(thread);
1143
+ return;
1144
+ }
1131
1145
  if (message.type === "touch") {
1132
1146
  core.orchestrator.noteAccess(message.indexName);
1133
1147
  return;
@@ -1147,10 +1161,7 @@ async function startRequestThreads(options) {
1147
1161
  });
1148
1162
  }
1149
1163
  async function stopThread(workerId, thread) {
1150
- try {
1151
- acceptor.removeChildAppDescriptor(thread.descriptor);
1152
- } catch {
1153
- }
1164
+ releaseDescriptor(thread);
1154
1165
  thread.port.close();
1155
1166
  clearGateSlot(options.gate, gateSlotOfWorker(workerId));
1156
1167
  await thread.executor.execute({ type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(() => void 0);
@@ -1159,7 +1170,13 @@ async function startRequestThreads(options) {
1159
1170
  async onWorkerReady(workerId, executor) {
1160
1171
  if (stopped) return;
1161
1172
  const channel = new MessageChannel();
1162
- relayFrom(channel.port2);
1173
+ const thread = {
1174
+ descriptor: null,
1175
+ port: channel.port2,
1176
+ executor,
1177
+ departure: createDepartureWord()
1178
+ };
1179
+ relayFrom(thread);
1163
1180
  let result;
1164
1181
  try {
1165
1182
  result = await executor.execute(
@@ -1168,6 +1185,7 @@ async function startRequestThreads(options) {
1168
1185
  requestId: `serve-${workerId}`,
1169
1186
  settings: {
1170
1187
  port: channel.port1,
1188
+ departure: thread.departure.buffer,
1171
1189
  gate: options.gate,
1172
1190
  gateSlot: gateSlotOfWorker(workerId),
1173
1191
  maxConcurrentRequests: options.limits.maxConcurrentRequests,
@@ -1188,8 +1206,8 @@ async function startRequestThreads(options) {
1188
1206
  console.warn(`Worker ${workerId} could not start taking requests:`, err);
1189
1207
  return;
1190
1208
  }
1191
- const thread = { descriptor: result.descriptor, port: channel.port2, executor };
1192
- if (stopped) {
1209
+ thread.descriptor = result.descriptor;
1210
+ if (stopped || departureConfirmed(thread.departure)) {
1193
1211
  await stopThread(workerId, thread);
1194
1212
  return;
1195
1213
  }
@@ -1200,10 +1218,7 @@ async function startRequestThreads(options) {
1200
1218
  const thread = threads.get(workerId);
1201
1219
  if (thread === void 0) return;
1202
1220
  threads.delete(workerId);
1203
- try {
1204
- acceptor.removeChildAppDescriptor(thread.descriptor);
1205
- } catch {
1206
- }
1221
+ releaseDescriptor(thread);
1207
1222
  thread.port.close();
1208
1223
  clearGateSlot(options.gate, gateSlotOfWorker(workerId));
1209
1224
  }
@@ -1,9 +1,9 @@
1
- import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex } from './chunk-XZF4LZ3X.mjs';
1
+ import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-RM6FRYEC.mjs';
2
2
  import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-4D6OH4EF.mjs';
3
3
  import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-XYDP7CCJ.mjs';
4
4
  import './chunk-ZCKPRCYI.mjs';
5
5
  import './chunk-ZJ4CSAB2.mjs';
6
- import './chunk-B3UOQJD3.mjs';
6
+ import './chunk-NFF5TDRG.mjs';
7
7
  import './chunk-SREPGC6O.mjs';
8
8
  import './chunk-WQBCBKY3.mjs';
9
9
  import './chunk-4VBG6PEH.mjs';
@@ -13,7 +13,7 @@ import './chunk-GES7U2PM.mjs';
13
13
  import './chunk-LS5IRM23.mjs';
14
14
  import './chunk-R6A773WK.mjs';
15
15
  import './chunk-YTJCGKM4.mjs';
16
- import { ServerErrorCodes, NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
16
+ import { ServerErrorCodes, ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
17
17
 
18
18
  // src/server/request-threads/held-vectors.ts
19
19
  function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
@@ -58,23 +58,26 @@ function createThreadReadEngine(options) {
58
58
  (fieldPath, docId) => executor.heldVectorOf(indexName, fieldPath, docId)
59
59
  );
60
60
  }
61
- function requireContext(indexName) {
62
- const context = executor.queryContextOf(indexName);
63
- if (context === void 0) {
64
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `This thread holds no copy of index "${indexName}"`, {
65
- indexName
66
- });
61
+ function heldContext(indexName) {
62
+ return executor.queryContextOf(indexName) ?? null;
63
+ }
64
+ function wholeDocumentsContext(indexName) {
65
+ const context = heldContext(indexName);
66
+ return context !== null && holdsEveryVectorField(indexName, context) ? context : null;
67
+ }
68
+ function requireAnswerable(indexName, context) {
69
+ if (context === null) {
70
+ throw new NarsilError(
71
+ ErrorCodes.INDEX_NOT_FOUND,
72
+ `This thread holds no copy of index "${indexName}" that answers this request yet`,
73
+ { indexName }
74
+ );
67
75
  }
68
76
  relay.touch(indexName);
69
77
  return context;
70
78
  }
71
79
  function requireWholeDocuments(indexName) {
72
- const context = requireContext(indexName);
73
- if (!holdsEveryVectorField(indexName, context)) {
74
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `This thread holds no vectors of index "${indexName}" yet`, {
75
- indexName
76
- });
77
- }
80
+ const context = requireAnswerable(indexName, wholeDocumentsContext(indexName));
78
81
  return { context, manager: managerOf(indexName, context) };
79
82
  }
80
83
  async function resolvedParams(indexName, params) {
@@ -83,10 +86,6 @@ function createThreadReadEngine(options) {
83
86
  const { text: _text, ...vector } = params.vector;
84
87
  return { ...params, vector: { ...vector, value } };
85
88
  }
86
- function holdsDocumentsOf(indexName) {
87
- const context = executor.queryContextOf(indexName);
88
- return context !== void 0 && holdsEveryVectorField(indexName, context);
89
- }
90
89
  function searchableContext(indexName, params) {
91
90
  if (searchHooks) return null;
92
91
  const context = executor.queryContextOf(indexName);
@@ -95,24 +94,22 @@ function createThreadReadEngine(options) {
95
94
  if (field === void 0) return context;
96
95
  const vectorFields = extractVectorFieldsFromSchema(context.config.schema);
97
96
  if (!vectorFields.has(field)) return context;
98
- return context.vectorSearchers.has(field) ? context : null;
97
+ return context.vectorSearchers.has(field) && executor.holdsVectorField(indexName, field) ? context : null;
99
98
  }
100
- function canAnswer(indexName, params) {
99
+ function answerableContext(indexName, params) {
101
100
  const context = searchableContext(indexName, params);
102
- return context !== null && holdsEveryVectorFieldRead(indexName, context, params);
103
- }
104
- function canCount(indexName, params) {
105
- return searchableContext(indexName, params) !== null;
101
+ return context !== null && holdsEveryVectorFieldRead(indexName, context, params) ? context : null;
106
102
  }
107
103
  return {
108
- holdsCopyOf: (indexName) => executor.queryContextOf(indexName) !== void 0,
109
- holdsDocumentsOf,
110
- canAnswer,
111
- canCount,
104
+ holdsCopyOf: (indexName) => heldContext(indexName) !== null,
105
+ holdsDocumentsOf: (indexName) => wholeDocumentsContext(indexName) !== null,
106
+ canAnswer: (indexName, params) => answerableContext(indexName, params) !== null,
107
+ canCount: (indexName, params) => searchableContext(indexName, params) !== null,
112
108
  async query(indexName, params) {
113
- const { context, manager } = requireWholeDocuments(indexName);
114
- const result = await executeQuery(await resolvedParams(indexName, params), {
115
- manager,
109
+ const resolved = await resolvedParams(indexName, params);
110
+ const context = requireAnswerable(indexName, answerableContext(indexName, params));
111
+ const result = await executeQuery(resolved, {
112
+ manager: managerOf(indexName, context),
116
113
  language: context.language,
117
114
  config: context.config,
118
115
  indexName,
@@ -124,8 +121,9 @@ function createThreadReadEngine(options) {
124
121
  return result;
125
122
  },
126
123
  async preflight(indexName, params) {
127
- const context = requireContext(indexName);
128
- const result = await executePreflight(await resolvedParams(indexName, params), {
124
+ const resolved = await resolvedParams(indexName, params);
125
+ const context = requireAnswerable(indexName, searchableContext(indexName, params));
126
+ const result = await executePreflight(resolved, {
129
127
  manager: context.manager,
130
128
  language: context.language,
131
129
  config: context.config,
@@ -138,7 +136,7 @@ function createThreadReadEngine(options) {
138
136
  return result;
139
137
  },
140
138
  async suggest(indexName, params) {
141
- const context = requireContext(indexName);
139
+ const context = requireAnswerable(indexName, heldContext(indexName));
142
140
  const result = executeSuggest(context.manager, context.language, params);
143
141
  if (context.analysisStale) result.analysisStale = true;
144
142
  return result;
@@ -156,10 +154,10 @@ function createThreadReadEngine(options) {
156
154
  return found;
157
155
  },
158
156
  async has(indexName, docId) {
159
- return requireContext(indexName).manager.has(docId);
157
+ return requireAnswerable(indexName, heldContext(indexName)).manager.has(docId);
160
158
  },
161
159
  async countDocuments(indexName) {
162
- return requireContext(indexName).manager.countDocuments();
160
+ return requireAnswerable(indexName, heldContext(indexName)).manager.countDocuments();
163
161
  },
164
162
  async listDocuments(indexName, params) {
165
163
  const { context, manager } = requireWholeDocuments(indexName);
@@ -344,6 +342,10 @@ function createRelayClient(port, touchIntervalMs) {
344
342
  touchedAt.set(indexName, now);
345
343
  port.postMessage({ type: "touch", indexName });
346
344
  },
345
+ announceDeparture() {
346
+ if (closed) return;
347
+ port.postMessage({ type: "leaving" });
348
+ },
347
349
  close() {
348
350
  failAll();
349
351
  port.close();
@@ -351,8 +353,182 @@ function createRelayClient(port, touchIntervalMs) {
351
353
  };
352
354
  }
353
355
 
356
+ // src/server/request-threads/thread-departure.ts
357
+ var DEPARTURE_REFUSAL = {
358
+ error: {
359
+ code: ErrorCodes.WORKER_CRASHED,
360
+ message: "The request thread that held this connection failed, so retry the request on a new connection"
361
+ }
362
+ };
363
+ var ThreadDepartureSignal = class extends Error {
364
+ };
365
+ var exitThread = process.exit.bind(process);
366
+ var endThread = threadEndOf(process);
367
+ var active = null;
368
+ var installed = false;
369
+ function threadEndOf(target) {
370
+ const found = Reflect.get(target, "reallyExit");
371
+ if (typeof found !== "function") return null;
372
+ return (code) => {
373
+ Reflect.apply(found, target, [code]);
374
+ };
375
+ }
376
+ function refuse(request) {
377
+ if (request.abort.aborted) return;
378
+ const payload = encodeJson(DEPARTURE_REFUSAL);
379
+ try {
380
+ request.res.cork(() => {
381
+ request.res.writeStatus(statusLine(503)).writeHeader("Content-Type", "application/json").writeHeader("Connection", "close").end(payload, true);
382
+ });
383
+ } catch {
384
+ }
385
+ }
386
+ function departedError() {
387
+ return new NarsilError(ErrorCodes.WORKER_CRASHED, "This request thread is leaving after a failure");
388
+ }
389
+ function crashWith(error) {
390
+ process.off("uncaughtException", leaveOnUncaughtError);
391
+ setImmediate(() => {
392
+ throw error;
393
+ });
394
+ }
395
+ function leaveOnUncaughtError(error) {
396
+ if (error instanceof ThreadDepartureSignal) return;
397
+ if (process.listenerCount("uncaughtException") > 1) return;
398
+ const leaving = active;
399
+ if (leaving === null) {
400
+ crashWith(error);
401
+ return;
402
+ }
403
+ leaving.begin();
404
+ leaving.finish(() => crashWith(error));
405
+ }
406
+ function exitAfterDeparture(code) {
407
+ const leaving = active;
408
+ if (leaving === null) return exitThread(code);
409
+ leaving.begin();
410
+ leaving.finish(() => exitThread(code));
411
+ throw new ThreadDepartureSignal("This request thread is leaving through process.exit");
412
+ }
413
+ function endAfterDeparture(end) {
414
+ return (code) => {
415
+ const leaving = active;
416
+ if (leaving === null) {
417
+ end(code);
418
+ return;
419
+ }
420
+ leaving.begin();
421
+ leaving.finish(() => end(code));
422
+ throw new ThreadDepartureSignal("This request thread is leaving through an exit that it held before it served");
423
+ };
424
+ }
425
+ function leaveAtExit() {
426
+ active?.leaveAtExit();
427
+ }
428
+ function install() {
429
+ if (installed) return;
430
+ installed = true;
431
+ process.exit = exitAfterDeparture;
432
+ if (endThread !== null) Reflect.set(process, "reallyExit", endAfterDeparture(endThread));
433
+ process.on("uncaughtException", leaveOnUncaughtError);
434
+ process.on("exit", leaveAtExit);
435
+ }
436
+ function watchForDeparture(options) {
437
+ const { relay, departure, close } = options;
438
+ const running = /* @__PURE__ */ new Set();
439
+ let departing = false;
440
+ let finishing = false;
441
+ function closeOnce() {
442
+ if (active === state) active = null;
443
+ close();
444
+ }
445
+ const state = {
446
+ begin() {
447
+ if (departing) return;
448
+ departing = true;
449
+ for (const request of running) refuse(request);
450
+ if (!departureConfirmed(departure)) relay.announceDeparture();
451
+ },
452
+ finish(exit) {
453
+ if (finishing) return;
454
+ finishing = true;
455
+ departureConfirmation(departure).then(() => new Promise((resolve) => setTimeout(resolve, REQUEST_THREAD_DEPARTURE_DRAIN_MS))).catch(() => void 0).then(() => {
456
+ closeOnce();
457
+ exit();
458
+ });
459
+ },
460
+ leaveAtExit() {
461
+ state.begin();
462
+ if (endThread !== null) return;
463
+ awaitDepartureConfirmation(departure);
464
+ closeOnce();
465
+ }
466
+ };
467
+ active = state;
468
+ install();
469
+ function refusingOnceDeparting(call) {
470
+ return (...args) => {
471
+ if (departing) throw departedError();
472
+ return call(...args);
473
+ };
474
+ }
475
+ return {
476
+ track(res, abort) {
477
+ const request = { res, abort };
478
+ if (departing) {
479
+ refuse(request);
480
+ return () => void 0;
481
+ }
482
+ running.add(request);
483
+ return () => {
484
+ running.delete(request);
485
+ };
486
+ },
487
+ guardRoute(handler) {
488
+ return async (ctx) => {
489
+ if (departing) {
490
+ refuse(ctx);
491
+ return;
492
+ }
493
+ try {
494
+ await handler(ctx);
495
+ } catch (err) {
496
+ if (!departing) throw err;
497
+ }
498
+ };
499
+ },
500
+ guardExecutor(executor) {
501
+ return {
502
+ getManager: refusingOnceDeparting((indexName) => executor.getManager(indexName)),
503
+ queryContextOf: refusingOnceDeparting((indexName) => executor.queryContextOf(indexName)),
504
+ holdsVectorField: refusingOnceDeparting(
505
+ (indexName, fieldName) => executor.holdsVectorField(indexName, fieldName)
506
+ ),
507
+ heldVectorOf: refusingOnceDeparting(
508
+ (indexName, fieldName, docId) => executor.heldVectorOf(indexName, fieldName, docId)
509
+ ),
510
+ createIndex: refusingOnceDeparting(
511
+ (indexName, config, language, analysisStale) => executor.createIndex(indexName, config, language, analysisStale)
512
+ ),
513
+ dropIndex: refusingOnceDeparting((indexName) => executor.dropIndex(indexName)),
514
+ listIndexes: refusingOnceDeparting(() => executor.listIndexes())
515
+ };
516
+ }
517
+ };
518
+ }
519
+ function leaveWithoutDeparture() {
520
+ active = null;
521
+ }
522
+
354
523
  // src/server/request-threads/thread.ts
355
524
  var servingApp = null;
525
+ function guardEveryRoute(handlers, departure) {
526
+ const guarded = { ...handlers };
527
+ for (const route of Object.keys(guarded)) {
528
+ guarded[route] = departure.guardRoute(handlers[route]);
529
+ }
530
+ return guarded;
531
+ }
356
532
  function relayed(relay, route) {
357
533
  return (ctx) => relay.request(route, ctx);
358
534
  }
@@ -380,7 +556,8 @@ function searchHandler(options) {
380
556
  try {
381
557
  respondJson(ctx, await answer(ctx.params[0], params));
382
558
  } catch (err) {
383
- respondError(ctx, err);
559
+ if (staysOnThread(ctx.params[0], params)) respondError(ctx, err);
560
+ else await fallback(ctx);
384
561
  }
385
562
  };
386
563
  }
@@ -454,8 +631,13 @@ async function serveRequests(executor, settings) {
454
631
  const child = childAppOf(app);
455
632
  if (child === null) throw new Error("This uWebSockets.js build offers no child apps");
456
633
  const relay = createRelayClient(settings.port, settings.touchIntervalMs);
634
+ const departure = watchForDeparture({
635
+ relay,
636
+ departure: new Int32Array(settings.departure),
637
+ close: closeRequestThread
638
+ });
457
639
  const local = createThreadReadEngine({
458
- executor,
640
+ executor: departure.guardExecutor(executor),
459
641
  relay,
460
642
  searchHooks: settings.searchHooks,
461
643
  patternWorkCap: settings.patternWorkCap
@@ -463,13 +645,16 @@ async function serveRequests(executor, settings) {
463
645
  const run = createRouteRunner({
464
646
  authorize: settings.authorizes ? (context) => relay.authorize(context) : void 0,
465
647
  gate: createSharedRequestGate(settings.gate, settings.maxConcurrentRequests, settings.gateSlot),
466
- corsHeaders: settings.cors ? corsHeaderReader(settings.cors) : void 0
648
+ corsHeaders: settings.cors ? corsHeaderReader(settings.cors) : void 0,
649
+ track: departure.track
467
650
  });
468
- registerServerRoutes(app, run, threadHandlers(local, relay, settings), settings.limits, settings.cors);
651
+ const handlers = guardEveryRoute(threadHandlers(local, relay, settings), departure);
652
+ registerServerRoutes(app, run, handlers, settings.limits, settings.cors);
469
653
  servingApp = { app: child, relay };
470
654
  return { descriptor: child.getDescriptor() };
471
655
  }
472
656
  function closeRequestThread() {
657
+ leaveWithoutDeparture();
473
658
  if (servingApp === null) return;
474
659
  const { app, relay } = servingApp;
475
660
  servingApp = null;
@@ -156,6 +156,12 @@ interface SharedVectorStoreHandles {
156
156
  magnitudes: GrowableBuffer;
157
157
  /** This holds one byte per ordinal, which reads 1 where the ordinal holds a live vector. */
158
158
  present: GrowableBuffer;
159
+ /** This holds one byte per ordinal, which reads 1 where the graph has yet to place the ordinal's vector. */
160
+ unplaced: GrowableBuffer;
161
+ /** This holds one 32-bit count for each fixed run of ordinals, which counts the unplaced flags that the run holds, so that a reader can skip every run whose count reads 0. */
162
+ unplacedChunks: GrowableBuffer;
163
+ /** This holds one byte per ordinal, which reads 1 where the field has removed the ordinal's document while it keeps the vector. */
164
+ removed: GrowableBuffer;
159
165
  /** This holds each ordinal's partition, or a negative marker where the store knows none. */
160
166
  partitions: GrowableBuffer;
161
167
  /** This holds every document id, UTF-8 encoded end to end in ordinal order. */
@@ -217,6 +223,8 @@ interface SharedGraphHandles {
217
223
  locks: GrowableBuffer;
218
224
  /** This holds one byte per ordinal, which reads 1 once a caller removes the document. */
219
225
  tombstones: GrowableBuffer;
226
+ /** This holds one word per ordinal, the number of base-layer lists that hold that ordinal. */
227
+ incoming: GrowableBuffer;
220
228
  /** This records the node lock and the graph lock each thread slot holds right now. */
221
229
  heldLocks: Int32Array;
222
230
  }
@@ -1,6 +1,6 @@
1
- import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-RZXNWQSM.mjs';
1
+ import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-B3F7LZXM.mjs';
2
2
  import '../chunk-ZJ4CSAB2.mjs';
3
- import '../chunk-B3UOQJD3.mjs';
3
+ import '../chunk-NFF5TDRG.mjs';
4
4
  import '../chunk-KEHK2O6P.mjs';
5
5
  import '../chunk-R6A773WK.mjs';
6
6
  import '../chunk-YTJCGKM4.mjs';
@@ -1,5 +1,5 @@
1
1
  import { startIdleHeapCleanup } from '../chunk-5XNIRLFC.mjs';
2
- import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-POWOTUKJ.mjs';
2
+ import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-YRX6XMOQ.mjs';
3
3
  import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-TPJU7NDJ.mjs';
4
4
 
5
5
  // src/workers/action-handler.ts
@@ -18,7 +18,7 @@ async function importBootstrapModule(moduleUrl) {
18
18
  await import(moduleUrl);
19
19
  }
20
20
  async function stopServingRequests() {
21
- const { closeRequestThread } = await import('../thread-33M5AOEI.mjs');
21
+ const { closeRequestThread } = await import('../thread-EUUNJNKU.mjs');
22
22
  closeRequestThread();
23
23
  }
24
24
  function holdsCopies(executor) {
@@ -49,7 +49,7 @@ function createActionHandler(executor) {
49
49
  if (!holdsCopies(executor)) {
50
50
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
51
51
  }
52
- const starting = import('../thread-33M5AOEI.mjs').then(
52
+ const starting = import('../thread-EUUNJNKU.mjs').then(
53
53
  ({ serveRequests }) => serveRequests(executor, action.settings)
54
54
  );
55
55
  serving.start = starting.then(
@@ -102,7 +102,7 @@ async function setup() {
102
102
  } catch {
103
103
  parentPort = null;
104
104
  }
105
- const { createDirectExecutor } = await import('../direct-executor-6WWFY5EI.mjs');
105
+ const { createDirectExecutor } = await import('../direct-executor-SI2QV4YI.mjs');
106
106
  const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
107
107
  if (parentPort) {
108
108
  const port = parentPort;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@delali/narsil",
3
- "version": "0.3.1-next.84",
3
+ "version": "0.3.1-next.86",
4
4
  "type": "module",
5
5
  "description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
6
6
  "author": "Delali (https://sondelali.com)",
@@ -246,11 +246,11 @@
246
246
  }
247
247
  },
248
248
  "optionalDependencies": {
249
- "@delali/narsil-native-darwin-arm64": "0.3.1-next.84",
250
- "@delali/narsil-native-darwin-x64": "0.3.1-next.84",
251
- "@delali/narsil-native-linux-arm64": "0.3.1-next.84",
252
- "@delali/narsil-native-linux-x64": "0.3.1-next.84",
253
- "@delali/narsil-native-win32-arm64": "0.3.1-next.84",
254
- "@delali/narsil-native-win32-x64": "0.3.1-next.84"
249
+ "@delali/narsil-native-darwin-arm64": "0.3.1-next.86",
250
+ "@delali/narsil-native-darwin-x64": "0.3.1-next.86",
251
+ "@delali/narsil-native-linux-arm64": "0.3.1-next.86",
252
+ "@delali/narsil-native-linux-x64": "0.3.1-next.86",
253
+ "@delali/narsil-native-win32-arm64": "0.3.1-next.86",
254
+ "@delali/narsil-native-win32-x64": "0.3.1-next.86"
255
255
  }
256
256
  }