@delali/narsil 0.3.1-next.83 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7JZZ3HNV.mjs → chunk-67MQF5CV.mjs} +215 -48
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-CUP2E4OH.mjs → chunk-RCSXJCI5.mjs} +373 -199
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6HASZ36Y.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +22 -7
- package/dist/index.browser.mjs +1040 -304
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
1
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-NFF5TDRG.mjs';
|
|
2
2
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
3
|
import { MAX_SORT_FIELDS } from './chunk-KJ7LB3TR.mjs';
|
|
4
4
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -251,6 +251,7 @@ var MIN_UNBOUNDED_SERVER_LIMIT = 0;
|
|
|
251
251
|
var RESPONSE_STREAM_THRESHOLD_BYTES = 64 * 1024;
|
|
252
252
|
var MAX_INDEX_TOUCH_INTERVAL_MS = 1e3;
|
|
253
253
|
var INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT = 4;
|
|
254
|
+
var REQUEST_THREAD_DEPARTURE_DRAIN_MS = 100;
|
|
254
255
|
var DEFAULT_MAX_RETAINED_TASKS = 1e3;
|
|
255
256
|
var DEFAULT_TASK_PAGE_SIZE = 20;
|
|
256
257
|
var TASK_PROGRESS_WRITE_INTERVAL_MS = 250;
|
|
@@ -913,8 +914,8 @@ function sinkWritingAfterStatus(res, headers) {
|
|
|
913
914
|
res.writeHeader(key, value);
|
|
914
915
|
return sink;
|
|
915
916
|
},
|
|
916
|
-
end(body) {
|
|
917
|
-
res.end(body);
|
|
917
|
+
end(body, closeConnection) {
|
|
918
|
+
res.end(body, closeConnection);
|
|
918
919
|
return sink;
|
|
919
920
|
},
|
|
920
921
|
write(chunk) {
|
|
@@ -981,7 +982,7 @@ function sendDenial(res, authorization) {
|
|
|
981
982
|
sendError(res, authorization.denial.status, authorization.denial.code, authorization.denial.message);
|
|
982
983
|
}
|
|
983
984
|
function createRouteRunner(deps) {
|
|
984
|
-
const { authorize, gate, corsHeaders } = deps;
|
|
985
|
+
const { authorize, gate, corsHeaders, track } = deps;
|
|
985
986
|
return (handler, opts) => {
|
|
986
987
|
const paramCount = opts.paramCount ?? 0;
|
|
987
988
|
const needsBody = opts.needsBody ?? false;
|
|
@@ -1026,6 +1027,7 @@ function createRouteRunner(deps) {
|
|
|
1026
1027
|
);
|
|
1027
1028
|
return;
|
|
1028
1029
|
}
|
|
1030
|
+
const release = track?.(res, abort);
|
|
1029
1031
|
const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
|
|
1030
1032
|
const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
|
|
1031
1033
|
const spoolPromise = opts.spoolsBody === true ? spoolBody(
|
|
@@ -1034,7 +1036,7 @@ function createRouteRunner(deps) {
|
|
|
1034
1036
|
abort,
|
|
1035
1037
|
hookPromise.then((authorization) => authorization.allowed)
|
|
1036
1038
|
).catch(() => null) : Promise.resolve(null);
|
|
1037
|
-
Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
1039
|
+
const running = Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
|
|
1038
1040
|
const discardSpool = () => {
|
|
1039
1041
|
void spooledBody?.remove().catch(() => void 0);
|
|
1040
1042
|
};
|
|
@@ -1064,6 +1066,7 @@ function createRouteRunner(deps) {
|
|
|
1064
1066
|
}
|
|
1065
1067
|
}).catch(() => {
|
|
1066
1068
|
});
|
|
1069
|
+
if (release !== void 0) void running.then(release);
|
|
1067
1070
|
};
|
|
1068
1071
|
};
|
|
1069
1072
|
}
|
|
@@ -1195,4 +1198,26 @@ function childAppOf(app) {
|
|
|
1195
1198
|
return candidate;
|
|
1196
1199
|
}
|
|
1197
1200
|
|
|
1198
|
-
|
|
1201
|
+
// src/server/request-threads/departure.ts
|
|
1202
|
+
var DEPARTURE_WORD = 0;
|
|
1203
|
+
var AWAITING_DEPARTURE = 0;
|
|
1204
|
+
var DEPARTURE_CONFIRMED = 1;
|
|
1205
|
+
function createDepartureWord() {
|
|
1206
|
+
return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
1207
|
+
}
|
|
1208
|
+
function confirmDeparture(word) {
|
|
1209
|
+
Atomics.store(word, DEPARTURE_WORD, DEPARTURE_CONFIRMED);
|
|
1210
|
+
Atomics.notify(word, DEPARTURE_WORD);
|
|
1211
|
+
}
|
|
1212
|
+
function departureConfirmed(word) {
|
|
1213
|
+
return Atomics.load(word, DEPARTURE_WORD) === DEPARTURE_CONFIRMED;
|
|
1214
|
+
}
|
|
1215
|
+
function awaitDepartureConfirmation(word) {
|
|
1216
|
+
while (!departureConfirmed(word)) Atomics.wait(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1217
|
+
}
|
|
1218
|
+
async function departureConfirmation(word) {
|
|
1219
|
+
const waiting = Atomics.waitAsync(word, DEPARTURE_WORD, AWAITING_DEPARTURE);
|
|
1220
|
+
if (waiting.async) await waiting.value;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-RCSXJCI5.mjs';
|
|
2
2
|
import './chunk-XYDP7CCJ.mjs';
|
|
3
3
|
import './chunk-OLXM65YH.mjs';
|
|
4
4
|
import './chunk-ZCKPRCYI.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-NIWDQTZB.mjs';
|
|
7
7
|
import './chunk-ZJ4CSAB2.mjs';
|
|
8
|
-
import './chunk-
|
|
8
|
+
import './chunk-NFF5TDRG.mjs';
|
|
9
9
|
import './chunk-SREPGC6O.mjs';
|
|
10
10
|
import './chunk-WQBCBKY3.mjs';
|
|
11
11
|
import './chunk-4VBG6PEH.mjs';
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
2
|
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf,
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-67MQF5CV.mjs';
|
|
4
4
|
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
|
|
5
5
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
6
6
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
|
|
7
|
-
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
7
|
+
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-RCSXJCI5.mjs';
|
|
8
8
|
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
|
|
9
9
|
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
10
10
|
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
11
11
|
import './chunk-TGP5M5R7.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-NIWDQTZB.mjs';
|
|
13
13
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
14
|
-
import './chunk-
|
|
14
|
+
import './chunk-NFF5TDRG.mjs';
|
|
15
15
|
import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
|
|
16
16
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
17
17
|
import './chunk-4VBG6PEH.mjs';
|
|
@@ -23,7 +23,7 @@ import './chunk-U2BKYEC7.mjs';
|
|
|
23
23
|
import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
24
24
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
25
25
|
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
26
|
-
import './chunk-
|
|
26
|
+
import './chunk-YRX6XMOQ.mjs';
|
|
27
27
|
import { ErrorCodes, NarsilError, describeError } from './chunk-TPJU7NDJ.mjs';
|
|
28
28
|
import { decode, encode } from '@msgpack/msgpack';
|
|
29
29
|
import { randomUUID } from 'crypto';
|
|
@@ -5623,6 +5623,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
5623
5623
|
const entry = core.requireIndex(indexName);
|
|
5624
5624
|
const manager = core.requireManager(indexName);
|
|
5625
5625
|
let captured = 0;
|
|
5626
|
+
const config = await encodeIndexConfiguration(entry);
|
|
5626
5627
|
const checkpoint = await buildArchiveFromMemory({
|
|
5627
5628
|
indexName,
|
|
5628
5629
|
metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
|
|
@@ -5634,7 +5635,6 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
5634
5635
|
},
|
|
5635
5636
|
partitionId
|
|
5636
5637
|
});
|
|
5637
|
-
const config = await encodeIndexConfiguration(entry);
|
|
5638
5638
|
return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
|
|
5639
5639
|
} finally {
|
|
5640
5640
|
release();
|
|
@@ -5660,6 +5660,7 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
|
|
|
5660
5660
|
async function receiveArchive(indexName, partitionId, source, schema) {
|
|
5661
5661
|
const store = memoryCheckpointStore();
|
|
5662
5662
|
let language = "";
|
|
5663
|
+
let analysisRevision = null;
|
|
5663
5664
|
let received = null;
|
|
5664
5665
|
await readSnapshotArchive(createByteReader(source), {
|
|
5665
5666
|
config: async (bytes) => {
|
|
@@ -5671,6 +5672,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
5671
5672
|
const raw = decode(payload);
|
|
5672
5673
|
if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
|
|
5673
5674
|
language = raw.language;
|
|
5675
|
+
analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
|
|
5674
5676
|
},
|
|
5675
5677
|
manifest: async (manifest, sourceName) => {
|
|
5676
5678
|
const listed = manifest.partitions.map((partition) => partition.partitionId);
|
|
@@ -5697,7 +5699,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
5697
5699
|
file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
|
|
5698
5700
|
});
|
|
5699
5701
|
if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
|
|
5700
|
-
return { language, manifest: received, store };
|
|
5702
|
+
return { language, analysisRevision, manifest: received, store };
|
|
5701
5703
|
}
|
|
5702
5704
|
function emptyPartition(manager, partitionId) {
|
|
5703
5705
|
return {
|
|
@@ -5749,6 +5751,18 @@ async function replacePartition(manager, partitionId, archive) {
|
|
|
5749
5751
|
vectorIndex.scheduleBuild();
|
|
5750
5752
|
}
|
|
5751
5753
|
}
|
|
5754
|
+
function recordReceivedAnalysis(core, indexName, receivedRevision) {
|
|
5755
|
+
const entry = core.requireIndex(indexName);
|
|
5756
|
+
const current = entry.language.revision;
|
|
5757
|
+
if (receivedRevision === current) return;
|
|
5758
|
+
entry.analysisRevision = entry.analysisRevision === current || entry.analysisRevision === receivedRevision ? receivedRevision : null;
|
|
5759
|
+
core.analysisRebuild.markStale({
|
|
5760
|
+
indexName,
|
|
5761
|
+
language: entry.language.name,
|
|
5762
|
+
storedRevision: entry.analysisRevision,
|
|
5763
|
+
currentRevision: current
|
|
5764
|
+
});
|
|
5765
|
+
}
|
|
5752
5766
|
async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
|
|
5753
5767
|
core.guardShutdown();
|
|
5754
5768
|
requireRestoreTarget(indexName, partitionId, partitionCount);
|
|
@@ -5775,6 +5789,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
5775
5789
|
const wasScaledOut = core.orchestrator.desyncIndex(indexName);
|
|
5776
5790
|
try {
|
|
5777
5791
|
await replacePartition(core.requireManager(indexName), partitionId, archive);
|
|
5792
|
+
recordReceivedAnalysis(core, indexName, archive.analysisRevision);
|
|
5778
5793
|
} finally {
|
|
5779
5794
|
await core.orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
5780
5795
|
}
|