@delali/narsil 0.3.1-next.84 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XIQZKKTK.mjs → chunk-67MQF5CV.mjs} +39 -10
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-IA67SOJO.mjs → chunk-RCSXJCI5.mjs} +271 -93
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6WWFY5EI.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +5 -5
- package/dist/index.browser.mjs +763 -161
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
package/dist/index.mjs
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-67MQF5CV.mjs';
|
|
2
2
|
import './chunk-4D6OH4EF.mjs';
|
|
3
3
|
import './chunk-GCM3JFVA.mjs';
|
|
4
4
|
import './chunk-2453RJYK.mjs';
|
|
5
|
-
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-
|
|
5
|
+
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer } from './chunk-RCSXJCI5.mjs';
|
|
6
6
|
export { registerPartialWordSearch, registerPatternSearch } from './chunk-XYDP7CCJ.mjs';
|
|
7
7
|
export { getLanguage, registerLanguage } from './chunk-OLXM65YH.mjs';
|
|
8
8
|
import './chunk-ZCKPRCYI.mjs';
|
|
9
9
|
import './chunk-TGP5M5R7.mjs';
|
|
10
|
-
export { isSimdAvailable } from './chunk-
|
|
10
|
+
export { isSimdAvailable } from './chunk-NIWDQTZB.mjs';
|
|
11
11
|
import './chunk-ZJ4CSAB2.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-NFF5TDRG.mjs';
|
|
13
13
|
import './chunk-SREPGC6O.mjs';
|
|
14
14
|
import './chunk-WQBCBKY3.mjs';
|
|
15
15
|
export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-4VBG6PEH.mjs';
|
|
@@ -20,5 +20,5 @@ import './chunk-LS5IRM23.mjs';
|
|
|
20
20
|
import './chunk-U2BKYEC7.mjs';
|
|
21
21
|
import './chunk-R6A773WK.mjs';
|
|
22
22
|
import './chunk-YTJCGKM4.mjs';
|
|
23
|
-
import './chunk-
|
|
23
|
+
import './chunk-YRX6XMOQ.mjs';
|
|
24
24
|
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-
|
|
1
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-67MQF5CV.mjs';
|
|
2
2
|
import '../../chunk-4D6OH4EF.mjs';
|
|
3
3
|
import '../../chunk-GCM3JFVA.mjs';
|
|
4
4
|
import '../../chunk-2453RJYK.mjs';
|
|
5
|
-
import '../../chunk-
|
|
5
|
+
import '../../chunk-RCSXJCI5.mjs';
|
|
6
6
|
import '../../chunk-XYDP7CCJ.mjs';
|
|
7
7
|
import '../../chunk-OLXM65YH.mjs';
|
|
8
8
|
import '../../chunk-ZCKPRCYI.mjs';
|
|
9
9
|
import '../../chunk-TGP5M5R7.mjs';
|
|
10
|
-
import '../../chunk-
|
|
10
|
+
import '../../chunk-NIWDQTZB.mjs';
|
|
11
11
|
import '../../chunk-ZJ4CSAB2.mjs';
|
|
12
|
-
import '../../chunk-
|
|
12
|
+
import '../../chunk-NFF5TDRG.mjs';
|
|
13
13
|
import '../../chunk-SREPGC6O.mjs';
|
|
14
14
|
import '../../chunk-WQBCBKY3.mjs';
|
|
15
15
|
import '../../chunk-4VBG6PEH.mjs';
|
|
@@ -20,7 +20,7 @@ import '../../chunk-LS5IRM23.mjs';
|
|
|
20
20
|
import '../../chunk-U2BKYEC7.mjs';
|
|
21
21
|
import '../../chunk-R6A773WK.mjs';
|
|
22
22
|
import '../../chunk-YTJCGKM4.mjs';
|
|
23
|
-
import '../../chunk-
|
|
23
|
+
import '../../chunk-YRX6XMOQ.mjs';
|
|
24
24
|
import '../../chunk-TPJU7NDJ.mjs';
|
|
25
25
|
|
|
26
26
|
// src/persistence/durability/rebuild.ts
|
package/dist/server.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, badRequest, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-
|
|
2
|
-
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
1
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, parseTaskListQuery, parseJsonOptional, validateRebalance, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, badRequest, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-RM6FRYEC.mjs';
|
|
2
|
+
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-RM6FRYEC.mjs';
|
|
3
3
|
import { validateIndexName } from './chunk-GNYXCLJS.mjs';
|
|
4
4
|
import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-2453RJYK.mjs';
|
|
5
5
|
import { detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
6
6
|
import './chunk-ZCKPRCYI.mjs';
|
|
7
7
|
import './chunk-TGP5M5R7.mjs';
|
|
8
|
-
import './chunk-
|
|
8
|
+
import './chunk-NFF5TDRG.mjs';
|
|
9
9
|
import './chunk-WQBCBKY3.mjs';
|
|
10
10
|
import './chunk-KJ7LB3TR.mjs';
|
|
11
11
|
import './chunk-KEHK2O6P.mjs';
|
|
@@ -1038,7 +1038,7 @@ var RELAYED_TYPES = /* @__PURE__ */ new Set(["request", "authorize", "embed", "s
|
|
|
1038
1038
|
function isRelayRequest(message) {
|
|
1039
1039
|
if (typeof message !== "object" || message === null) return false;
|
|
1040
1040
|
const { type, requestId } = message;
|
|
1041
|
-
if (type === "touch") return true;
|
|
1041
|
+
if (type === "touch" || type === "leaving") return true;
|
|
1042
1042
|
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
1043
1043
|
}
|
|
1044
1044
|
function spooledBodyOf(relayed) {
|
|
@@ -1124,10 +1124,24 @@ async function startRequestThreads(options) {
|
|
|
1124
1124
|
const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
|
|
1125
1125
|
const threads = /* @__PURE__ */ new Map();
|
|
1126
1126
|
let stopped = false;
|
|
1127
|
-
function
|
|
1127
|
+
function releaseDescriptor(thread) {
|
|
1128
|
+
if (thread.descriptor !== null) {
|
|
1129
|
+
try {
|
|
1130
|
+
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
1131
|
+
} catch {
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
confirmDeparture(thread.departure);
|
|
1135
|
+
}
|
|
1136
|
+
function relayFrom(thread) {
|
|
1137
|
+
const { port } = thread;
|
|
1128
1138
|
const streams = /* @__PURE__ */ new Map();
|
|
1129
1139
|
port.on("message", (message) => {
|
|
1130
1140
|
if (!isRelayRequest(message)) return;
|
|
1141
|
+
if (message.type === "leaving") {
|
|
1142
|
+
releaseDescriptor(thread);
|
|
1143
|
+
return;
|
|
1144
|
+
}
|
|
1131
1145
|
if (message.type === "touch") {
|
|
1132
1146
|
core.orchestrator.noteAccess(message.indexName);
|
|
1133
1147
|
return;
|
|
@@ -1147,10 +1161,7 @@ async function startRequestThreads(options) {
|
|
|
1147
1161
|
});
|
|
1148
1162
|
}
|
|
1149
1163
|
async function stopThread(workerId, thread) {
|
|
1150
|
-
|
|
1151
|
-
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
1152
|
-
} catch {
|
|
1153
|
-
}
|
|
1164
|
+
releaseDescriptor(thread);
|
|
1154
1165
|
thread.port.close();
|
|
1155
1166
|
clearGateSlot(options.gate, gateSlotOfWorker(workerId));
|
|
1156
1167
|
await thread.executor.execute({ type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(() => void 0);
|
|
@@ -1159,7 +1170,13 @@ async function startRequestThreads(options) {
|
|
|
1159
1170
|
async onWorkerReady(workerId, executor) {
|
|
1160
1171
|
if (stopped) return;
|
|
1161
1172
|
const channel = new MessageChannel();
|
|
1162
|
-
|
|
1173
|
+
const thread = {
|
|
1174
|
+
descriptor: null,
|
|
1175
|
+
port: channel.port2,
|
|
1176
|
+
executor,
|
|
1177
|
+
departure: createDepartureWord()
|
|
1178
|
+
};
|
|
1179
|
+
relayFrom(thread);
|
|
1163
1180
|
let result;
|
|
1164
1181
|
try {
|
|
1165
1182
|
result = await executor.execute(
|
|
@@ -1168,6 +1185,7 @@ async function startRequestThreads(options) {
|
|
|
1168
1185
|
requestId: `serve-${workerId}`,
|
|
1169
1186
|
settings: {
|
|
1170
1187
|
port: channel.port1,
|
|
1188
|
+
departure: thread.departure.buffer,
|
|
1171
1189
|
gate: options.gate,
|
|
1172
1190
|
gateSlot: gateSlotOfWorker(workerId),
|
|
1173
1191
|
maxConcurrentRequests: options.limits.maxConcurrentRequests,
|
|
@@ -1188,8 +1206,8 @@ async function startRequestThreads(options) {
|
|
|
1188
1206
|
console.warn(`Worker ${workerId} could not start taking requests:`, err);
|
|
1189
1207
|
return;
|
|
1190
1208
|
}
|
|
1191
|
-
|
|
1192
|
-
if (stopped) {
|
|
1209
|
+
thread.descriptor = result.descriptor;
|
|
1210
|
+
if (stopped || departureConfirmed(thread.departure)) {
|
|
1193
1211
|
await stopThread(workerId, thread);
|
|
1194
1212
|
return;
|
|
1195
1213
|
}
|
|
@@ -1200,10 +1218,7 @@ async function startRequestThreads(options) {
|
|
|
1200
1218
|
const thread = threads.get(workerId);
|
|
1201
1219
|
if (thread === void 0) return;
|
|
1202
1220
|
threads.delete(workerId);
|
|
1203
|
-
|
|
1204
|
-
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
1205
|
-
} catch {
|
|
1206
|
-
}
|
|
1221
|
+
releaseDescriptor(thread);
|
|
1207
1222
|
thread.port.close();
|
|
1208
1223
|
clearGateSlot(options.gate, gateSlotOfWorker(workerId));
|
|
1209
1224
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex } from './chunk-
|
|
1
|
+
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-RM6FRYEC.mjs';
|
|
2
2
|
import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-4D6OH4EF.mjs';
|
|
3
3
|
import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
4
4
|
import './chunk-ZCKPRCYI.mjs';
|
|
5
5
|
import './chunk-ZJ4CSAB2.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-NFF5TDRG.mjs';
|
|
7
7
|
import './chunk-SREPGC6O.mjs';
|
|
8
8
|
import './chunk-WQBCBKY3.mjs';
|
|
9
9
|
import './chunk-4VBG6PEH.mjs';
|
|
@@ -13,7 +13,7 @@ import './chunk-GES7U2PM.mjs';
|
|
|
13
13
|
import './chunk-LS5IRM23.mjs';
|
|
14
14
|
import './chunk-R6A773WK.mjs';
|
|
15
15
|
import './chunk-YTJCGKM4.mjs';
|
|
16
|
-
import { ServerErrorCodes,
|
|
16
|
+
import { ServerErrorCodes, ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
17
17
|
|
|
18
18
|
// src/server/request-threads/held-vectors.ts
|
|
19
19
|
function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
|
|
@@ -58,23 +58,26 @@ function createThreadReadEngine(options) {
|
|
|
58
58
|
(fieldPath, docId) => executor.heldVectorOf(indexName, fieldPath, docId)
|
|
59
59
|
);
|
|
60
60
|
}
|
|
61
|
-
function
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
61
|
+
function heldContext(indexName) {
|
|
62
|
+
return executor.queryContextOf(indexName) ?? null;
|
|
63
|
+
}
|
|
64
|
+
function wholeDocumentsContext(indexName) {
|
|
65
|
+
const context = heldContext(indexName);
|
|
66
|
+
return context !== null && holdsEveryVectorField(indexName, context) ? context : null;
|
|
67
|
+
}
|
|
68
|
+
function requireAnswerable(indexName, context) {
|
|
69
|
+
if (context === null) {
|
|
70
|
+
throw new NarsilError(
|
|
71
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
72
|
+
`This thread holds no copy of index "${indexName}" that answers this request yet`,
|
|
73
|
+
{ indexName }
|
|
74
|
+
);
|
|
67
75
|
}
|
|
68
76
|
relay.touch(indexName);
|
|
69
77
|
return context;
|
|
70
78
|
}
|
|
71
79
|
function requireWholeDocuments(indexName) {
|
|
72
|
-
const context =
|
|
73
|
-
if (!holdsEveryVectorField(indexName, context)) {
|
|
74
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `This thread holds no vectors of index "${indexName}" yet`, {
|
|
75
|
-
indexName
|
|
76
|
-
});
|
|
77
|
-
}
|
|
80
|
+
const context = requireAnswerable(indexName, wholeDocumentsContext(indexName));
|
|
78
81
|
return { context, manager: managerOf(indexName, context) };
|
|
79
82
|
}
|
|
80
83
|
async function resolvedParams(indexName, params) {
|
|
@@ -83,10 +86,6 @@ function createThreadReadEngine(options) {
|
|
|
83
86
|
const { text: _text, ...vector } = params.vector;
|
|
84
87
|
return { ...params, vector: { ...vector, value } };
|
|
85
88
|
}
|
|
86
|
-
function holdsDocumentsOf(indexName) {
|
|
87
|
-
const context = executor.queryContextOf(indexName);
|
|
88
|
-
return context !== void 0 && holdsEveryVectorField(indexName, context);
|
|
89
|
-
}
|
|
90
89
|
function searchableContext(indexName, params) {
|
|
91
90
|
if (searchHooks) return null;
|
|
92
91
|
const context = executor.queryContextOf(indexName);
|
|
@@ -95,24 +94,22 @@ function createThreadReadEngine(options) {
|
|
|
95
94
|
if (field === void 0) return context;
|
|
96
95
|
const vectorFields = extractVectorFieldsFromSchema(context.config.schema);
|
|
97
96
|
if (!vectorFields.has(field)) return context;
|
|
98
|
-
return context.vectorSearchers.has(field) ? context : null;
|
|
97
|
+
return context.vectorSearchers.has(field) && executor.holdsVectorField(indexName, field) ? context : null;
|
|
99
98
|
}
|
|
100
|
-
function
|
|
99
|
+
function answerableContext(indexName, params) {
|
|
101
100
|
const context = searchableContext(indexName, params);
|
|
102
|
-
return context !== null && holdsEveryVectorFieldRead(indexName, context, params);
|
|
103
|
-
}
|
|
104
|
-
function canCount(indexName, params) {
|
|
105
|
-
return searchableContext(indexName, params) !== null;
|
|
101
|
+
return context !== null && holdsEveryVectorFieldRead(indexName, context, params) ? context : null;
|
|
106
102
|
}
|
|
107
103
|
return {
|
|
108
|
-
holdsCopyOf: (indexName) =>
|
|
109
|
-
holdsDocumentsOf,
|
|
110
|
-
canAnswer,
|
|
111
|
-
canCount,
|
|
104
|
+
holdsCopyOf: (indexName) => heldContext(indexName) !== null,
|
|
105
|
+
holdsDocumentsOf: (indexName) => wholeDocumentsContext(indexName) !== null,
|
|
106
|
+
canAnswer: (indexName, params) => answerableContext(indexName, params) !== null,
|
|
107
|
+
canCount: (indexName, params) => searchableContext(indexName, params) !== null,
|
|
112
108
|
async query(indexName, params) {
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
109
|
+
const resolved = await resolvedParams(indexName, params);
|
|
110
|
+
const context = requireAnswerable(indexName, answerableContext(indexName, params));
|
|
111
|
+
const result = await executeQuery(resolved, {
|
|
112
|
+
manager: managerOf(indexName, context),
|
|
116
113
|
language: context.language,
|
|
117
114
|
config: context.config,
|
|
118
115
|
indexName,
|
|
@@ -124,8 +121,9 @@ function createThreadReadEngine(options) {
|
|
|
124
121
|
return result;
|
|
125
122
|
},
|
|
126
123
|
async preflight(indexName, params) {
|
|
127
|
-
const
|
|
128
|
-
const
|
|
124
|
+
const resolved = await resolvedParams(indexName, params);
|
|
125
|
+
const context = requireAnswerable(indexName, searchableContext(indexName, params));
|
|
126
|
+
const result = await executePreflight(resolved, {
|
|
129
127
|
manager: context.manager,
|
|
130
128
|
language: context.language,
|
|
131
129
|
config: context.config,
|
|
@@ -138,7 +136,7 @@ function createThreadReadEngine(options) {
|
|
|
138
136
|
return result;
|
|
139
137
|
},
|
|
140
138
|
async suggest(indexName, params) {
|
|
141
|
-
const context =
|
|
139
|
+
const context = requireAnswerable(indexName, heldContext(indexName));
|
|
142
140
|
const result = executeSuggest(context.manager, context.language, params);
|
|
143
141
|
if (context.analysisStale) result.analysisStale = true;
|
|
144
142
|
return result;
|
|
@@ -156,10 +154,10 @@ function createThreadReadEngine(options) {
|
|
|
156
154
|
return found;
|
|
157
155
|
},
|
|
158
156
|
async has(indexName, docId) {
|
|
159
|
-
return
|
|
157
|
+
return requireAnswerable(indexName, heldContext(indexName)).manager.has(docId);
|
|
160
158
|
},
|
|
161
159
|
async countDocuments(indexName) {
|
|
162
|
-
return
|
|
160
|
+
return requireAnswerable(indexName, heldContext(indexName)).manager.countDocuments();
|
|
163
161
|
},
|
|
164
162
|
async listDocuments(indexName, params) {
|
|
165
163
|
const { context, manager } = requireWholeDocuments(indexName);
|
|
@@ -344,6 +342,10 @@ function createRelayClient(port, touchIntervalMs) {
|
|
|
344
342
|
touchedAt.set(indexName, now);
|
|
345
343
|
port.postMessage({ type: "touch", indexName });
|
|
346
344
|
},
|
|
345
|
+
announceDeparture() {
|
|
346
|
+
if (closed) return;
|
|
347
|
+
port.postMessage({ type: "leaving" });
|
|
348
|
+
},
|
|
347
349
|
close() {
|
|
348
350
|
failAll();
|
|
349
351
|
port.close();
|
|
@@ -351,8 +353,182 @@ function createRelayClient(port, touchIntervalMs) {
|
|
|
351
353
|
};
|
|
352
354
|
}
|
|
353
355
|
|
|
356
|
+
// src/server/request-threads/thread-departure.ts
|
|
357
|
+
var DEPARTURE_REFUSAL = {
|
|
358
|
+
error: {
|
|
359
|
+
code: ErrorCodes.WORKER_CRASHED,
|
|
360
|
+
message: "The request thread that held this connection failed, so retry the request on a new connection"
|
|
361
|
+
}
|
|
362
|
+
};
|
|
363
|
+
var ThreadDepartureSignal = class extends Error {
|
|
364
|
+
};
|
|
365
|
+
var exitThread = process.exit.bind(process);
|
|
366
|
+
var endThread = threadEndOf(process);
|
|
367
|
+
var active = null;
|
|
368
|
+
var installed = false;
|
|
369
|
+
function threadEndOf(target) {
|
|
370
|
+
const found = Reflect.get(target, "reallyExit");
|
|
371
|
+
if (typeof found !== "function") return null;
|
|
372
|
+
return (code) => {
|
|
373
|
+
Reflect.apply(found, target, [code]);
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
function refuse(request) {
|
|
377
|
+
if (request.abort.aborted) return;
|
|
378
|
+
const payload = encodeJson(DEPARTURE_REFUSAL);
|
|
379
|
+
try {
|
|
380
|
+
request.res.cork(() => {
|
|
381
|
+
request.res.writeStatus(statusLine(503)).writeHeader("Content-Type", "application/json").writeHeader("Connection", "close").end(payload, true);
|
|
382
|
+
});
|
|
383
|
+
} catch {
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
function departedError() {
|
|
387
|
+
return new NarsilError(ErrorCodes.WORKER_CRASHED, "This request thread is leaving after a failure");
|
|
388
|
+
}
|
|
389
|
+
function crashWith(error) {
|
|
390
|
+
process.off("uncaughtException", leaveOnUncaughtError);
|
|
391
|
+
setImmediate(() => {
|
|
392
|
+
throw error;
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
function leaveOnUncaughtError(error) {
|
|
396
|
+
if (error instanceof ThreadDepartureSignal) return;
|
|
397
|
+
if (process.listenerCount("uncaughtException") > 1) return;
|
|
398
|
+
const leaving = active;
|
|
399
|
+
if (leaving === null) {
|
|
400
|
+
crashWith(error);
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
leaving.begin();
|
|
404
|
+
leaving.finish(() => crashWith(error));
|
|
405
|
+
}
|
|
406
|
+
function exitAfterDeparture(code) {
|
|
407
|
+
const leaving = active;
|
|
408
|
+
if (leaving === null) return exitThread(code);
|
|
409
|
+
leaving.begin();
|
|
410
|
+
leaving.finish(() => exitThread(code));
|
|
411
|
+
throw new ThreadDepartureSignal("This request thread is leaving through process.exit");
|
|
412
|
+
}
|
|
413
|
+
function endAfterDeparture(end) {
|
|
414
|
+
return (code) => {
|
|
415
|
+
const leaving = active;
|
|
416
|
+
if (leaving === null) {
|
|
417
|
+
end(code);
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
leaving.begin();
|
|
421
|
+
leaving.finish(() => end(code));
|
|
422
|
+
throw new ThreadDepartureSignal("This request thread is leaving through an exit that it held before it served");
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
function leaveAtExit() {
|
|
426
|
+
active?.leaveAtExit();
|
|
427
|
+
}
|
|
428
|
+
function install() {
|
|
429
|
+
if (installed) return;
|
|
430
|
+
installed = true;
|
|
431
|
+
process.exit = exitAfterDeparture;
|
|
432
|
+
if (endThread !== null) Reflect.set(process, "reallyExit", endAfterDeparture(endThread));
|
|
433
|
+
process.on("uncaughtException", leaveOnUncaughtError);
|
|
434
|
+
process.on("exit", leaveAtExit);
|
|
435
|
+
}
|
|
436
|
+
function watchForDeparture(options) {
|
|
437
|
+
const { relay, departure, close } = options;
|
|
438
|
+
const running = /* @__PURE__ */ new Set();
|
|
439
|
+
let departing = false;
|
|
440
|
+
let finishing = false;
|
|
441
|
+
function closeOnce() {
|
|
442
|
+
if (active === state) active = null;
|
|
443
|
+
close();
|
|
444
|
+
}
|
|
445
|
+
const state = {
|
|
446
|
+
begin() {
|
|
447
|
+
if (departing) return;
|
|
448
|
+
departing = true;
|
|
449
|
+
for (const request of running) refuse(request);
|
|
450
|
+
if (!departureConfirmed(departure)) relay.announceDeparture();
|
|
451
|
+
},
|
|
452
|
+
finish(exit) {
|
|
453
|
+
if (finishing) return;
|
|
454
|
+
finishing = true;
|
|
455
|
+
departureConfirmation(departure).then(() => new Promise((resolve) => setTimeout(resolve, REQUEST_THREAD_DEPARTURE_DRAIN_MS))).catch(() => void 0).then(() => {
|
|
456
|
+
closeOnce();
|
|
457
|
+
exit();
|
|
458
|
+
});
|
|
459
|
+
},
|
|
460
|
+
leaveAtExit() {
|
|
461
|
+
state.begin();
|
|
462
|
+
if (endThread !== null) return;
|
|
463
|
+
awaitDepartureConfirmation(departure);
|
|
464
|
+
closeOnce();
|
|
465
|
+
}
|
|
466
|
+
};
|
|
467
|
+
active = state;
|
|
468
|
+
install();
|
|
469
|
+
function refusingOnceDeparting(call) {
|
|
470
|
+
return (...args) => {
|
|
471
|
+
if (departing) throw departedError();
|
|
472
|
+
return call(...args);
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
return {
|
|
476
|
+
track(res, abort) {
|
|
477
|
+
const request = { res, abort };
|
|
478
|
+
if (departing) {
|
|
479
|
+
refuse(request);
|
|
480
|
+
return () => void 0;
|
|
481
|
+
}
|
|
482
|
+
running.add(request);
|
|
483
|
+
return () => {
|
|
484
|
+
running.delete(request);
|
|
485
|
+
};
|
|
486
|
+
},
|
|
487
|
+
guardRoute(handler) {
|
|
488
|
+
return async (ctx) => {
|
|
489
|
+
if (departing) {
|
|
490
|
+
refuse(ctx);
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
try {
|
|
494
|
+
await handler(ctx);
|
|
495
|
+
} catch (err) {
|
|
496
|
+
if (!departing) throw err;
|
|
497
|
+
}
|
|
498
|
+
};
|
|
499
|
+
},
|
|
500
|
+
guardExecutor(executor) {
|
|
501
|
+
return {
|
|
502
|
+
getManager: refusingOnceDeparting((indexName) => executor.getManager(indexName)),
|
|
503
|
+
queryContextOf: refusingOnceDeparting((indexName) => executor.queryContextOf(indexName)),
|
|
504
|
+
holdsVectorField: refusingOnceDeparting(
|
|
505
|
+
(indexName, fieldName) => executor.holdsVectorField(indexName, fieldName)
|
|
506
|
+
),
|
|
507
|
+
heldVectorOf: refusingOnceDeparting(
|
|
508
|
+
(indexName, fieldName, docId) => executor.heldVectorOf(indexName, fieldName, docId)
|
|
509
|
+
),
|
|
510
|
+
createIndex: refusingOnceDeparting(
|
|
511
|
+
(indexName, config, language, analysisStale) => executor.createIndex(indexName, config, language, analysisStale)
|
|
512
|
+
),
|
|
513
|
+
dropIndex: refusingOnceDeparting((indexName) => executor.dropIndex(indexName)),
|
|
514
|
+
listIndexes: refusingOnceDeparting(() => executor.listIndexes())
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
function leaveWithoutDeparture() {
|
|
520
|
+
active = null;
|
|
521
|
+
}
|
|
522
|
+
|
|
354
523
|
// src/server/request-threads/thread.ts
|
|
355
524
|
var servingApp = null;
|
|
525
|
+
function guardEveryRoute(handlers, departure) {
|
|
526
|
+
const guarded = { ...handlers };
|
|
527
|
+
for (const route of Object.keys(guarded)) {
|
|
528
|
+
guarded[route] = departure.guardRoute(handlers[route]);
|
|
529
|
+
}
|
|
530
|
+
return guarded;
|
|
531
|
+
}
|
|
356
532
|
function relayed(relay, route) {
|
|
357
533
|
return (ctx) => relay.request(route, ctx);
|
|
358
534
|
}
|
|
@@ -380,7 +556,8 @@ function searchHandler(options) {
|
|
|
380
556
|
try {
|
|
381
557
|
respondJson(ctx, await answer(ctx.params[0], params));
|
|
382
558
|
} catch (err) {
|
|
383
|
-
respondError(ctx, err);
|
|
559
|
+
if (staysOnThread(ctx.params[0], params)) respondError(ctx, err);
|
|
560
|
+
else await fallback(ctx);
|
|
384
561
|
}
|
|
385
562
|
};
|
|
386
563
|
}
|
|
@@ -454,8 +631,13 @@ async function serveRequests(executor, settings) {
|
|
|
454
631
|
const child = childAppOf(app);
|
|
455
632
|
if (child === null) throw new Error("This uWebSockets.js build offers no child apps");
|
|
456
633
|
const relay = createRelayClient(settings.port, settings.touchIntervalMs);
|
|
634
|
+
const departure = watchForDeparture({
|
|
635
|
+
relay,
|
|
636
|
+
departure: new Int32Array(settings.departure),
|
|
637
|
+
close: closeRequestThread
|
|
638
|
+
});
|
|
457
639
|
const local = createThreadReadEngine({
|
|
458
|
-
executor,
|
|
640
|
+
executor: departure.guardExecutor(executor),
|
|
459
641
|
relay,
|
|
460
642
|
searchHooks: settings.searchHooks,
|
|
461
643
|
patternWorkCap: settings.patternWorkCap
|
|
@@ -463,13 +645,16 @@ async function serveRequests(executor, settings) {
|
|
|
463
645
|
const run = createRouteRunner({
|
|
464
646
|
authorize: settings.authorizes ? (context) => relay.authorize(context) : void 0,
|
|
465
647
|
gate: createSharedRequestGate(settings.gate, settings.maxConcurrentRequests, settings.gateSlot),
|
|
466
|
-
corsHeaders: settings.cors ? corsHeaderReader(settings.cors) : void 0
|
|
648
|
+
corsHeaders: settings.cors ? corsHeaderReader(settings.cors) : void 0,
|
|
649
|
+
track: departure.track
|
|
467
650
|
});
|
|
468
|
-
|
|
651
|
+
const handlers = guardEveryRoute(threadHandlers(local, relay, settings), departure);
|
|
652
|
+
registerServerRoutes(app, run, handlers, settings.limits, settings.cors);
|
|
469
653
|
servingApp = { app: child, relay };
|
|
470
654
|
return { descriptor: child.getDescriptor() };
|
|
471
655
|
}
|
|
472
656
|
function closeRequestThread() {
|
|
657
|
+
leaveWithoutDeparture();
|
|
473
658
|
if (servingApp === null) return;
|
|
474
659
|
const { app, relay } = servingApp;
|
|
475
660
|
servingApp = null;
|
|
@@ -156,6 +156,12 @@ interface SharedVectorStoreHandles {
|
|
|
156
156
|
magnitudes: GrowableBuffer;
|
|
157
157
|
/** This holds one byte per ordinal, which reads 1 where the ordinal holds a live vector. */
|
|
158
158
|
present: GrowableBuffer;
|
|
159
|
+
/** This holds one byte per ordinal, which reads 1 where the graph has yet to place the ordinal's vector. */
|
|
160
|
+
unplaced: GrowableBuffer;
|
|
161
|
+
/** This holds one 32-bit count for each fixed run of ordinals, which counts the unplaced flags that the run holds, so that a reader can skip every run whose count reads 0. */
|
|
162
|
+
unplacedChunks: GrowableBuffer;
|
|
163
|
+
/** This holds one byte per ordinal, which reads 1 where the field has removed the ordinal's document while it keeps the vector. */
|
|
164
|
+
removed: GrowableBuffer;
|
|
159
165
|
/** This holds each ordinal's partition, or a negative marker where the store knows none. */
|
|
160
166
|
partitions: GrowableBuffer;
|
|
161
167
|
/** This holds every document id, UTF-8 encoded end to end in ordinal order. */
|
|
@@ -217,6 +223,8 @@ interface SharedGraphHandles {
|
|
|
217
223
|
locks: GrowableBuffer;
|
|
218
224
|
/** This holds one byte per ordinal, which reads 1 once a caller removes the document. */
|
|
219
225
|
tombstones: GrowableBuffer;
|
|
226
|
+
/** This holds one word per ordinal, the number of base-layer lists that hold that ordinal. */
|
|
227
|
+
incoming: GrowableBuffer;
|
|
220
228
|
/** This records the node lock and the graph lock each thread slot holds right now. */
|
|
221
229
|
heldLocks: Int32Array;
|
|
222
230
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-
|
|
1
|
+
import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-NIWDQTZB.mjs';
|
|
2
2
|
import '../chunk-ZJ4CSAB2.mjs';
|
|
3
|
-
import '../chunk-
|
|
3
|
+
import '../chunk-NFF5TDRG.mjs';
|
|
4
4
|
import '../chunk-KEHK2O6P.mjs';
|
|
5
5
|
import '../chunk-R6A773WK.mjs';
|
|
6
6
|
import '../chunk-YTJCGKM4.mjs';
|
package/dist/workers/entry.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { startIdleHeapCleanup } from '../chunk-5XNIRLFC.mjs';
|
|
2
|
-
import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-
|
|
2
|
+
import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-YRX6XMOQ.mjs';
|
|
3
3
|
import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-TPJU7NDJ.mjs';
|
|
4
4
|
|
|
5
5
|
// src/workers/action-handler.ts
|
|
@@ -18,7 +18,7 @@ async function importBootstrapModule(moduleUrl) {
|
|
|
18
18
|
await import(moduleUrl);
|
|
19
19
|
}
|
|
20
20
|
async function stopServingRequests() {
|
|
21
|
-
const { closeRequestThread } = await import('../thread-
|
|
21
|
+
const { closeRequestThread } = await import('../thread-EUUNJNKU.mjs');
|
|
22
22
|
closeRequestThread();
|
|
23
23
|
}
|
|
24
24
|
function holdsCopies(executor) {
|
|
@@ -49,7 +49,7 @@ function createActionHandler(executor) {
|
|
|
49
49
|
if (!holdsCopies(executor)) {
|
|
50
50
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
|
|
51
51
|
}
|
|
52
|
-
const starting = import('../thread-
|
|
52
|
+
const starting = import('../thread-EUUNJNKU.mjs').then(
|
|
53
53
|
({ serveRequests }) => serveRequests(executor, action.settings)
|
|
54
54
|
);
|
|
55
55
|
serving.start = starting.then(
|
|
@@ -102,7 +102,7 @@ async function setup() {
|
|
|
102
102
|
} catch {
|
|
103
103
|
parentPort = null;
|
|
104
104
|
}
|
|
105
|
-
const { createDirectExecutor } = await import('../direct-executor-
|
|
105
|
+
const { createDirectExecutor } = await import('../direct-executor-NY7ZTZIH.mjs');
|
|
106
106
|
const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
|
|
107
107
|
if (parentPort) {
|
|
108
108
|
const port = parentPort;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@delali/narsil",
|
|
3
|
-
"version": "0.3.1-next.
|
|
3
|
+
"version": "0.3.1-next.85",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
|
|
6
6
|
"author": "Delali (https://sondelali.com)",
|
|
@@ -246,11 +246,11 @@
|
|
|
246
246
|
}
|
|
247
247
|
},
|
|
248
248
|
"optionalDependencies": {
|
|
249
|
-
"@delali/narsil-native-darwin-arm64": "0.3.1-next.
|
|
250
|
-
"@delali/narsil-native-darwin-x64": "0.3.1-next.
|
|
251
|
-
"@delali/narsil-native-linux-arm64": "0.3.1-next.
|
|
252
|
-
"@delali/narsil-native-linux-x64": "0.3.1-next.
|
|
253
|
-
"@delali/narsil-native-win32-arm64": "0.3.1-next.
|
|
254
|
-
"@delali/narsil-native-win32-x64": "0.3.1-next.
|
|
249
|
+
"@delali/narsil-native-darwin-arm64": "0.3.1-next.85",
|
|
250
|
+
"@delali/narsil-native-darwin-x64": "0.3.1-next.85",
|
|
251
|
+
"@delali/narsil-native-linux-arm64": "0.3.1-next.85",
|
|
252
|
+
"@delali/narsil-native-linux-x64": "0.3.1-next.85",
|
|
253
|
+
"@delali/narsil-native-win32-arm64": "0.3.1-next.85",
|
|
254
|
+
"@delali/narsil-native-win32-x64": "0.3.1-next.85"
|
|
255
255
|
}
|
|
256
256
|
}
|