@delali/narsil 0.3.1-next.112 → 0.3.1-next.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-UR5MNN53.mjs → chunk-2TMWIATH.mjs} +108 -67
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-73U5DWOZ.mjs} +39 -4
- package/dist/browser/{chunk-O5IKTA27.mjs → chunk-FODCC3MK.mjs} +1 -1
- package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-I45FAGOF.mjs} +2 -2
- package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-G2PPZV3B.mjs} +127 -33
- package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-N7N42PBS.mjs} +1 -1
- package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
- package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-EBH6RXQO.mjs} +6 -5
- package/dist/browser/{write-IABOLE4N.mjs → write-5ZKFINX3.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-4ZKBKPBV.mjs} +22 -2
- package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
- package/dist/{chunk-3COL24MT.mjs → chunk-7EWQZQDT.mjs} +1 -1
- package/dist/{chunk-QI6JW4PM.mjs → chunk-BCNUJVTT.mjs} +1 -1
- package/dist/{chunk-3RRATITJ.mjs → chunk-BCSWODXP.mjs} +75 -55
- package/dist/{thread-WGBCVJII.mjs → chunk-DEXID4ZT.mjs} +301 -200
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-YIDSHBQS.mjs → chunk-HC5NURS6.mjs} +33 -4
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/{chunk-63HFWQEA.mjs → chunk-IE43SRL3.mjs} +276 -102
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-2IFU7SXG.mjs → chunk-KGVC6KJ3.mjs} +4 -4
- package/dist/{chunk-7CI34JAP.mjs → chunk-KVLX65KD.mjs} +4 -4
- package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
- package/dist/chunk-N6J6EJW3.mjs +198 -0
- package/dist/{chunk-PT7VRWS7.mjs → chunk-NX7KU77V.mjs} +3 -2
- package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-V7ISFBFC.mjs → chunk-Q62NDKEC.mjs} +5 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-BVOUSX3U.mjs → chunk-UDZYIWDP.mjs} +3 -3
- package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
- package/dist/{chunk-IBTKJIV7.mjs → chunk-VOKEYJHU.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-IM76DLA7.mjs → chunk-YCFQ5754.mjs} +2 -2
- package/dist/{chunk-VLRT47GU.mjs → chunk-YCSRB3IN.mjs} +3 -3
- package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-OUY4RPTU.mjs} +7 -6
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +79 -42
- package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-5V6DVHFK.mjs} +17 -16
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +10 -9
- package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
- package/dist/{module-WP4VVGV2.mjs → module-3IMTXNPW.mjs} +92 -29
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -13
- package/dist/react.d.ts +6 -6
- package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
- package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
- package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +110 -258
- package/dist/{snapshot-EU6NNB2P.mjs → snapshot-57WF53MK.mjs} +18 -17
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/thread-FDEHU5G3.mjs +31 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +19 -7
- package/dist/{write-CKRWS3P5.mjs → write-YEI7A5OR.mjs} +11 -10
- package/package.json +7 -7
|
@@ -11359,7 +11359,7 @@ function capExceeded(cap) {
|
|
|
11359
11359
|
{ cap }
|
|
11360
11360
|
);
|
|
11361
11361
|
}
|
|
11362
|
-
function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
11362
|
+
function localMeter(cap, spent, matcherStateBytes, flushUnits, clock) {
|
|
11363
11363
|
let total = spent;
|
|
11364
11364
|
let sharedCount = null;
|
|
11365
11365
|
const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
|
|
@@ -11367,11 +11367,13 @@ function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
|
11367
11367
|
cap,
|
|
11368
11368
|
matcherStateBytes,
|
|
11369
11369
|
flushUnits,
|
|
11370
|
+
clock,
|
|
11370
11371
|
add(units) {
|
|
11371
11372
|
if (units <= 0) return;
|
|
11372
11373
|
if (sharedCount === null) total += units;
|
|
11373
11374
|
else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
|
|
11374
11375
|
if (total > cap) throw capExceeded(cap);
|
|
11376
|
+
if (clock !== void 0) clock.tick(units);
|
|
11375
11377
|
},
|
|
11376
11378
|
remaining() {
|
|
11377
11379
|
return cap - spentSoFar();
|
|
@@ -11410,8 +11412,8 @@ function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
|
11410
11412
|
};
|
|
11411
11413
|
return meter;
|
|
11412
11414
|
}
|
|
11413
|
-
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
|
|
11414
|
-
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
|
|
11415
|
+
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP, clock) {
|
|
11416
|
+
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS, clock);
|
|
11415
11417
|
}
|
|
11416
11418
|
function sharedMeter(work, counter) {
|
|
11417
11419
|
const view = new BigInt64Array(counter);
|
|
@@ -11425,11 +11427,12 @@ function sharedMeter(work, counter) {
|
|
|
11425
11427
|
},
|
|
11426
11428
|
remaining() {
|
|
11427
11429
|
return work.cap - Number(Atomics.load(view, 0));
|
|
11428
|
-
}
|
|
11430
|
+
},
|
|
11431
|
+
clock: void 0
|
|
11429
11432
|
};
|
|
11430
11433
|
}
|
|
11431
11434
|
function joinPatternWorkMeter(work) {
|
|
11432
|
-
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
|
|
11435
|
+
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits, void 0) : sharedMeter(work, work.counter);
|
|
11433
11436
|
let spent = 0;
|
|
11434
11437
|
return {
|
|
11435
11438
|
matcherStateBytes: meter.matcherStateBytes,
|
|
@@ -11440,6 +11443,7 @@ function joinPatternWorkMeter(work) {
|
|
|
11440
11443
|
meter.add(units);
|
|
11441
11444
|
},
|
|
11442
11445
|
remaining: () => meter.remaining(),
|
|
11446
|
+
clock: meter.clock,
|
|
11443
11447
|
spent: () => spent
|
|
11444
11448
|
};
|
|
11445
11449
|
}
|
|
@@ -11973,7 +11977,7 @@ function ordersAfterAnchor(result, anchor, sort) {
|
|
|
11973
11977
|
function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
|
|
11974
11978
|
const decoded = cursor ? decodePageCursor(cursor) : null;
|
|
11975
11979
|
if (decoded !== null && cursor !== void 0) {
|
|
11976
|
-
requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
|
|
11980
|
+
requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding());
|
|
11977
11981
|
}
|
|
11978
11982
|
if (limit <= 0) {
|
|
11979
11983
|
return { paginated: [] };
|
|
@@ -12001,14 +12005,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
12001
12005
|
sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
|
|
12002
12006
|
sortSignature: sort.signature,
|
|
12003
12007
|
depth,
|
|
12004
|
-
binding
|
|
12008
|
+
binding: binding()
|
|
12005
12009
|
}) : encodePageCursor({
|
|
12006
12010
|
anchor: lastResult.id,
|
|
12007
12011
|
score: lastResult.score ?? null,
|
|
12008
12012
|
sortKey: null,
|
|
12009
12013
|
sortSignature: null,
|
|
12010
12014
|
depth,
|
|
12011
|
-
binding
|
|
12015
|
+
binding: binding()
|
|
12012
12016
|
});
|
|
12013
12017
|
}
|
|
12014
12018
|
return { paginated: sliced, nextCursor };
|
|
@@ -12238,6 +12242,7 @@ async function fanOutQuery(manager, params, language, schema, config, searchOpti
|
|
|
12238
12242
|
const partitionFacets = [];
|
|
12239
12243
|
const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
|
|
12240
12244
|
for (const partition of partitions) {
|
|
12245
|
+
options.patternWork?.clock?.check();
|
|
12241
12246
|
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
12242
12247
|
if (everyValue) {
|
|
12243
12248
|
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
|
|
@@ -12248,6 +12253,7 @@ async function fanOutQuery(manager, params, language, schema, config, searchOpti
|
|
|
12248
12253
|
facets = mergeFacets(partitionFacets, params.facets);
|
|
12249
12254
|
}
|
|
12250
12255
|
}
|
|
12256
|
+
options.patternWork?.clock?.check();
|
|
12251
12257
|
const allScoredArrays = outcomes.map((o) => o.result.scored);
|
|
12252
12258
|
const merged = kWayMerge(allScoredArrays);
|
|
12253
12259
|
let totalMatched = 0;
|
|
@@ -12304,6 +12310,7 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
12304
12310
|
patternWork: options?.searchOptions?.patternWork ?? createPatternWorkMeter()
|
|
12305
12311
|
};
|
|
12306
12312
|
for (const partition of partitionsIn(manager, options?.partitionIds)) {
|
|
12313
|
+
searchOptions.patternWork.clock?.check();
|
|
12307
12314
|
const matches = fulltextMatches(partition, params, language, schema, searchOptions);
|
|
12308
12315
|
if (matches !== null) count += matches.count;
|
|
12309
12316
|
}
|
|
@@ -15087,7 +15094,7 @@ function requireLoadedVectorCode() {
|
|
|
15087
15094
|
function loadVectorCode() {
|
|
15088
15095
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
15089
15096
|
if (loading === null) {
|
|
15090
|
-
loading = import('./module-
|
|
15097
|
+
loading = import('./module-G2PPZV3B.mjs').then(
|
|
15091
15098
|
(code) => {
|
|
15092
15099
|
loaded = code;
|
|
15093
15100
|
return code;
|
|
@@ -15299,6 +15306,7 @@ function createDirectExecutor(options) {
|
|
|
15299
15306
|
);
|
|
15300
15307
|
}
|
|
15301
15308
|
case "serveRequests":
|
|
15309
|
+
case "serveLongQueries":
|
|
15302
15310
|
case "stopServing": {
|
|
15303
15311
|
throw new NarsilError(
|
|
15304
15312
|
ErrorCodes.CONFIG_INVALID,
|
|
@@ -17052,6 +17060,7 @@ var MAX_PENDING_REPLICATION_DOCUMENTS = 2e4;
|
|
|
17052
17060
|
var REPLICATION_WINDOW = 32;
|
|
17053
17061
|
var LOAD_BUFFER_YIELD_INTERVAL = 1e3;
|
|
17054
17062
|
var MAIN_COPY_LONE_QUERY_DOCUMENTS = 5e4;
|
|
17063
|
+
var LONG_QUERY_THREADS = 1;
|
|
17055
17064
|
var POOL_RESTART_DELAY_MS = 1e3;
|
|
17056
17065
|
var POOL_RESTART_DELAY_MAX_MS = 6e4;
|
|
17057
17066
|
var RETIRED_THREADS_EXIT_WAIT_MS = 1e4;
|
|
@@ -17911,15 +17920,41 @@ async function createWorkerFactory(entryPoint) {
|
|
|
17911
17920
|
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
|
|
17912
17921
|
}
|
|
17913
17922
|
|
|
17914
|
-
// src/workers/pool.ts
|
|
17923
|
+
// src/workers/pool-memory.ts
|
|
17915
17924
|
function toFinite(value) {
|
|
17916
17925
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
17917
17926
|
}
|
|
17918
17927
|
function toHeapLimit(value) {
|
|
17919
17928
|
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
17920
17929
|
}
|
|
17930
|
+
async function memoryStatsOf(slots) {
|
|
17931
|
+
const indices = [];
|
|
17932
|
+
const pending = [];
|
|
17933
|
+
for (const [slotIndex, slot] of slots) {
|
|
17934
|
+
indices.push(slotIndex);
|
|
17935
|
+
pending.push(
|
|
17936
|
+
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
17937
|
+
);
|
|
17938
|
+
}
|
|
17939
|
+
const reports = await Promise.all(pending);
|
|
17940
|
+
const stats = [];
|
|
17941
|
+
for (let n = 0; n < indices.length; n++) {
|
|
17942
|
+
const report = reports[n];
|
|
17943
|
+
stats.push({
|
|
17944
|
+
workerId: indices[n],
|
|
17945
|
+
heapUsed: toFinite(report?.heapUsed),
|
|
17946
|
+
heapTotal: toFinite(report?.heapTotal),
|
|
17947
|
+
heapLimit: toHeapLimit(report?.heapLimit),
|
|
17948
|
+
external: toFinite(report?.external)
|
|
17949
|
+
});
|
|
17950
|
+
}
|
|
17951
|
+
return stats;
|
|
17952
|
+
}
|
|
17953
|
+
|
|
17954
|
+
// src/workers/pool.ts
|
|
17921
17955
|
function createWorkerPool(config) {
|
|
17922
17956
|
const workerCount = resolveWorkerCount(config.count);
|
|
17957
|
+
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
17923
17958
|
const workers = /* @__PURE__ */ new Map();
|
|
17924
17959
|
const deadSlots = /* @__PURE__ */ new Set();
|
|
17925
17960
|
const goneSlots = /* @__PURE__ */ new Set();
|
|
@@ -18014,10 +18049,13 @@ function createWorkerPool(config) {
|
|
|
18014
18049
|
}
|
|
18015
18050
|
};
|
|
18016
18051
|
}
|
|
18052
|
+
function isLongQueryWorker(workerId) {
|
|
18053
|
+
return workerId >= queryWorkerCount;
|
|
18054
|
+
}
|
|
18017
18055
|
function assignSlot(indexName) {
|
|
18018
|
-
const preferred = fnv1a(indexName) %
|
|
18019
|
-
for (let probe = 0; probe <
|
|
18020
|
-
const candidate = (preferred + probe) %
|
|
18056
|
+
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
18057
|
+
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
18058
|
+
const candidate = (preferred + probe) % queryWorkerCount;
|
|
18021
18059
|
if (!deadSlots.has(candidate)) {
|
|
18022
18060
|
return candidate;
|
|
18023
18061
|
}
|
|
@@ -18062,29 +18100,6 @@ function createWorkerPool(config) {
|
|
|
18062
18100
|
}
|
|
18063
18101
|
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
18064
18102
|
}
|
|
18065
|
-
async function getMemoryStats() {
|
|
18066
|
-
const indices = [];
|
|
18067
|
-
const pending = [];
|
|
18068
|
-
for (const [slotIndex, slot] of workers) {
|
|
18069
|
-
indices.push(slotIndex);
|
|
18070
|
-
pending.push(
|
|
18071
|
-
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
18072
|
-
);
|
|
18073
|
-
}
|
|
18074
|
-
const reports = await Promise.all(pending);
|
|
18075
|
-
const stats = [];
|
|
18076
|
-
for (let n = 0; n < indices.length; n++) {
|
|
18077
|
-
const report = reports[n];
|
|
18078
|
-
stats.push({
|
|
18079
|
-
workerId: indices[n],
|
|
18080
|
-
heapUsed: toFinite(report?.heapUsed),
|
|
18081
|
-
heapTotal: toFinite(report?.heapTotal),
|
|
18082
|
-
heapLimit: toHeapLimit(report?.heapLimit),
|
|
18083
|
-
external: toFinite(report?.external)
|
|
18084
|
-
});
|
|
18085
|
-
}
|
|
18086
|
-
return stats;
|
|
18087
|
-
}
|
|
18088
18103
|
async function shutdown() {
|
|
18089
18104
|
if (isShutdown) {
|
|
18090
18105
|
return;
|
|
@@ -18132,7 +18147,7 @@ function createWorkerPool(config) {
|
|
|
18132
18147
|
let chosenId = -1;
|
|
18133
18148
|
let chosen = null;
|
|
18134
18149
|
for (const [workerId, slot] of workers) {
|
|
18135
|
-
if (!slot.serving) continue;
|
|
18150
|
+
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
18136
18151
|
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
18137
18152
|
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
18138
18153
|
chosenId = workerId;
|
|
@@ -18144,8 +18159,8 @@ function createWorkerPool(config) {
|
|
|
18144
18159
|
}
|
|
18145
18160
|
function queriesInFlight() {
|
|
18146
18161
|
let total = 0;
|
|
18147
|
-
for (const slot of workers
|
|
18148
|
-
if (slot.serving) total += slot.inFlight;
|
|
18162
|
+
for (const [workerId, slot] of workers) {
|
|
18163
|
+
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
18149
18164
|
}
|
|
18150
18165
|
return total;
|
|
18151
18166
|
}
|
|
@@ -18153,7 +18168,7 @@ function createWorkerPool(config) {
|
|
|
18153
18168
|
const leases = [];
|
|
18154
18169
|
for (const [workerId, slot] of workers) {
|
|
18155
18170
|
if (leases.length >= limit) break;
|
|
18156
|
-
if (slot.serving && slot.inFlight === 0) leases.push(lease(workerId, slot));
|
|
18171
|
+
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
18157
18172
|
}
|
|
18158
18173
|
return leases;
|
|
18159
18174
|
}
|
|
@@ -18179,6 +18194,7 @@ function createWorkerPool(config) {
|
|
|
18179
18194
|
executorsHolding,
|
|
18180
18195
|
leaseLeastBusy,
|
|
18181
18196
|
leaseIdle,
|
|
18197
|
+
isLongQueryWorker,
|
|
18182
18198
|
queriesInFlight,
|
|
18183
18199
|
spawnAll,
|
|
18184
18200
|
deadWorkerIds,
|
|
@@ -18189,7 +18205,7 @@ function createWorkerPool(config) {
|
|
|
18189
18205
|
addIndex,
|
|
18190
18206
|
addIndexToAll,
|
|
18191
18207
|
removeIndex,
|
|
18192
|
-
getMemoryStats,
|
|
18208
|
+
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
18193
18209
|
shutdown,
|
|
18194
18210
|
whenEveryThreadIsGone
|
|
18195
18211
|
};
|
|
@@ -18219,7 +18235,7 @@ async function insertOnLease(lease, indexName, fieldName, handle, ordinals) {
|
|
|
18219
18235
|
function sharedCopyHostOf(state) {
|
|
18220
18236
|
return {
|
|
18221
18237
|
get workerCount() {
|
|
18222
|
-
return state.workerPool?.workerCount ?? state.keywordWorkerCount;
|
|
18238
|
+
return state.workerPool?.workerCount ?? state.keywordWorkerCount + state.longQueryWorkerCount;
|
|
18223
18239
|
},
|
|
18224
18240
|
holdsIndex: (indexName) => holdsIndex(state, indexName),
|
|
18225
18241
|
resolvePartition: (indexName, docId) => state.executor.getManager(indexName)?.partitionIdOf(docId),
|
|
@@ -18281,8 +18297,12 @@ function refreshVectorCopies(state, indexName) {
|
|
|
18281
18297
|
async function announceRequestThreads(state, pool) {
|
|
18282
18298
|
const listener = state.requestThreads;
|
|
18283
18299
|
if (listener === null) return;
|
|
18284
|
-
|
|
18285
|
-
|
|
18300
|
+
const entries = pool.executorEntries();
|
|
18301
|
+
for (const { workerId, executor } of entries) {
|
|
18302
|
+
if (pool.isLongQueryWorker(workerId)) await listener.onLongQueryWorkerReady(workerId, executor);
|
|
18303
|
+
}
|
|
18304
|
+
for (const { workerId, executor } of entries) {
|
|
18305
|
+
if (!pool.isLongQueryWorker(workerId)) await listener.onWorkerReady(workerId, executor);
|
|
18286
18306
|
}
|
|
18287
18307
|
}
|
|
18288
18308
|
function scheduleRequestThreadPoolRestart(state) {
|
|
@@ -18303,13 +18323,14 @@ async function serveRequestsOnWorkers(state, listener) {
|
|
|
18303
18323
|
if (!state.workersEnabled) return 0;
|
|
18304
18324
|
state.requestThreads = listener;
|
|
18305
18325
|
state.keywordWorkerCount = resolveRequestThreadCount(state.config?.workers?.count);
|
|
18326
|
+
state.longQueryWorkerCount = LONG_QUERY_THREADS;
|
|
18306
18327
|
if (state.vectorCopyPolicy !== void 0) {
|
|
18307
18328
|
state.vectorCopyPolicy.host = sharedCopyHostOf(state);
|
|
18308
18329
|
state.vectorCopyPolicy.enabled = state.workersEnabled;
|
|
18309
18330
|
}
|
|
18310
18331
|
if (state.poolStart !== null) await state.poolStart.catch(() => void 0);
|
|
18311
18332
|
const running = state.workerPool;
|
|
18312
|
-
if (running !== null && running.workerCount !== state.keywordWorkerCount) {
|
|
18333
|
+
if (running !== null && running.workerCount !== resolveWorkerCount(state.keywordWorkerCount + LONG_QUERY_THREADS)) {
|
|
18313
18334
|
retirePool(state, running);
|
|
18314
18335
|
state.poolRetryAt = 0;
|
|
18315
18336
|
}
|
|
@@ -18421,7 +18442,9 @@ async function replaceWorker(state, pool, workerId) {
|
|
|
18421
18442
|
await loadOntoReplacement(state, replacement, indexName);
|
|
18422
18443
|
}
|
|
18423
18444
|
replacement.admit();
|
|
18424
|
-
|
|
18445
|
+
if (pool.isLongQueryWorker(workerId))
|
|
18446
|
+
await state.requestThreads?.onLongQueryWorkerReady(workerId, replacement.executor);
|
|
18447
|
+
else await state.requestThreads?.onWorkerReady(workerId, replacement.executor);
|
|
18425
18448
|
} catch (err) {
|
|
18426
18449
|
replacement.abandon();
|
|
18427
18450
|
throw err;
|
|
@@ -18480,7 +18503,8 @@ async function startPool(state) {
|
|
|
18480
18503
|
const factory = await createWorkerFactory();
|
|
18481
18504
|
let started = null;
|
|
18482
18505
|
const pool = createWorkerPool({
|
|
18483
|
-
count: state.keywordWorkerCount,
|
|
18506
|
+
count: state.keywordWorkerCount + state.longQueryWorkerCount,
|
|
18507
|
+
longQueryWorkers: state.longQueryWorkerCount,
|
|
18484
18508
|
workerFactory: factory,
|
|
18485
18509
|
onWorkerCrash(workerId, indexNames, error) {
|
|
18486
18510
|
if (started !== null) handleWorkerCrash(state, started, workerId, indexNames, error);
|
|
@@ -18905,6 +18929,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18905
18929
|
vectorCopyPolicy,
|
|
18906
18930
|
workersEnabled: config?.workers?.enabled ?? workersEnabledByDefault(),
|
|
18907
18931
|
keywordWorkerCount: splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).keyword,
|
|
18932
|
+
longQueryWorkerCount: 0,
|
|
18908
18933
|
requestThreads: null,
|
|
18909
18934
|
copyThreshold: config?.workers?.promotionThreshold ?? DEFAULT_COPY_THRESHOLD,
|
|
18910
18935
|
copyIdleTimeoutMs: config?.workers?.idleTimeoutMs ?? copyIdleTimeoutBeforeClose(config?.lifecycle?.idleTimeoutMs),
|
|
@@ -19164,7 +19189,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19164
19189
|
// src/engine/core.ts
|
|
19165
19190
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19166
19191
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19167
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19192
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-BX6XRFYR.mjs')).resolveRerankSettings(config);
|
|
19168
19193
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19169
19194
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19170
19195
|
const vectorCopyPolicy = {
|
|
@@ -19175,7 +19200,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19175
19200
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19176
19201
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19177
19202
|
}
|
|
19178
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19203
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-CSMTKEYX.mjs');
|
|
19179
19204
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19180
19205
|
const executor = createDirectExecutor({
|
|
19181
19206
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19472,7 +19497,7 @@ async function shutdownEngine(core) {
|
|
|
19472
19497
|
}
|
|
19473
19498
|
}
|
|
19474
19499
|
if (core.rerankAdapters.size > 0) {
|
|
19475
|
-
await (await import('./runtime-
|
|
19500
|
+
await (await import('./runtime-BX6XRFYR.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19476
19501
|
}
|
|
19477
19502
|
await orchestrator.shutdown();
|
|
19478
19503
|
await executor.shutdown();
|
|
@@ -21919,6 +21944,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
21919
21944
|
graphCount: status.graphCount,
|
|
21920
21945
|
bufferSize: status.bufferSize,
|
|
21921
21946
|
building: status.building,
|
|
21947
|
+
waitingForMemory: status.waitingForMemory,
|
|
21922
21948
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
21923
21949
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
21924
21950
|
});
|
|
@@ -22608,14 +22634,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22608
22634
|
return {
|
|
22609
22635
|
snapshot(indexName) {
|
|
22610
22636
|
return withOpenIndex(indexName, async () => {
|
|
22611
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22637
|
+
const { createSnapshot } = await import('./snapshot-EBH6RXQO.mjs');
|
|
22612
22638
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22613
22639
|
});
|
|
22614
22640
|
},
|
|
22615
22641
|
async restore(indexName, data) {
|
|
22616
22642
|
core.guardShutdown();
|
|
22617
22643
|
validateIndexName(indexName);
|
|
22618
|
-
const restored = import('./restore-queue-
|
|
22644
|
+
const restored = import('./restore-queue-N7N42PBS.mjs').then(
|
|
22619
22645
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22620
22646
|
);
|
|
22621
22647
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22624,8 +22650,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22624
22650
|
exportDocuments(indexName) {
|
|
22625
22651
|
return withOpenIndex(indexName, async () => {
|
|
22626
22652
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22627
|
-
import('./snapshot-
|
|
22628
|
-
import('./write-
|
|
22653
|
+
import('./snapshot-EBH6RXQO.mjs'),
|
|
22654
|
+
import('./write-5ZKFINX3.mjs')
|
|
22629
22655
|
]);
|
|
22630
22656
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22631
22657
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22690,6 +22716,7 @@ function writeEncodedString(stream, value) {
|
|
|
22690
22716
|
stream.length += encoded.length;
|
|
22691
22717
|
}
|
|
22692
22718
|
function writeString(stream, value) {
|
|
22719
|
+
if (stream.clock !== void 0) stream.clock.tick(value.length);
|
|
22693
22720
|
ensureCapacity(stream, LENGTH_BYTES + value.length);
|
|
22694
22721
|
const start = stream.length + LENGTH_BYTES;
|
|
22695
22722
|
const bytes = stream.bytes;
|
|
@@ -22710,6 +22737,7 @@ var NumberList = class {
|
|
|
22710
22737
|
}
|
|
22711
22738
|
};
|
|
22712
22739
|
function writeNumberList(stream, list) {
|
|
22740
|
+
if (stream.clock !== void 0) stream.clock.tick(list.members.length);
|
|
22713
22741
|
writeByte(stream, LIST_TAG);
|
|
22714
22742
|
writeUint32(stream, list.members.length);
|
|
22715
22743
|
flush2(stream);
|
|
@@ -22781,15 +22809,15 @@ function boundRerankOf(rerank) {
|
|
|
22781
22809
|
}
|
|
22782
22810
|
var FILTERS_SLOT = 2;
|
|
22783
22811
|
var BOUND_SLOT_COUNT = 16;
|
|
22784
|
-
function bindingOf(values) {
|
|
22785
|
-
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS };
|
|
22812
|
+
function bindingOf(values, clock) {
|
|
22813
|
+
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS, clock };
|
|
22786
22814
|
for (const value of values) {
|
|
22787
22815
|
writeValue(stream, value);
|
|
22788
22816
|
}
|
|
22789
22817
|
flush2(stream);
|
|
22790
22818
|
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
22791
22819
|
}
|
|
22792
|
-
function queryBindingOf(params) {
|
|
22820
|
+
function queryBindingOf(params, clock) {
|
|
22793
22821
|
const values = [
|
|
22794
22822
|
params.term,
|
|
22795
22823
|
params.fields,
|
|
@@ -22809,7 +22837,14 @@ function queryBindingOf(params) {
|
|
|
22809
22837
|
params.scoring ?? "local"
|
|
22810
22838
|
];
|
|
22811
22839
|
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
22812
|
-
return bindingOf(values);
|
|
22840
|
+
return bindingOf(values, clock);
|
|
22841
|
+
}
|
|
22842
|
+
function lazyQueryBinding(params, clock) {
|
|
22843
|
+
let binding;
|
|
22844
|
+
return () => {
|
|
22845
|
+
binding ??= queryBindingOf(params, clock);
|
|
22846
|
+
return binding;
|
|
22847
|
+
};
|
|
22813
22848
|
}
|
|
22814
22849
|
function listBindingOf(filters) {
|
|
22815
22850
|
const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
|
|
@@ -22889,7 +22924,7 @@ function requireValidTypoTolerance(params) {
|
|
|
22889
22924
|
|
|
22890
22925
|
// src/engine/query/shared.ts
|
|
22891
22926
|
function withPatternWork(context) {
|
|
22892
|
-
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
22927
|
+
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap, context.clock) };
|
|
22893
22928
|
}
|
|
22894
22929
|
function partitionsFor(manager, partitionIds) {
|
|
22895
22930
|
return partitionsIn(manager, partitionIds);
|
|
@@ -23655,6 +23690,7 @@ function filteredSearchOf(params, context) {
|
|
|
23655
23690
|
}
|
|
23656
23691
|
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow) {
|
|
23657
23692
|
const options = {
|
|
23693
|
+
clock: context.patternWork.clock,
|
|
23658
23694
|
metric: vectorConfig.metric ?? "cosine",
|
|
23659
23695
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
23660
23696
|
efSearch: vectorConfig.efSearch,
|
|
@@ -23802,7 +23838,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23802
23838
|
let anchorId = null;
|
|
23803
23839
|
if (params.searchAfter !== void 0) {
|
|
23804
23840
|
const decoded = decodePageCursor(params.searchAfter);
|
|
23805
|
-
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
23841
|
+
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding());
|
|
23806
23842
|
anchorKey = decoded.sortKey;
|
|
23807
23843
|
anchorId = decoded.anchor;
|
|
23808
23844
|
}
|
|
@@ -23819,8 +23855,11 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23819
23855
|
const partitionFacets = [];
|
|
23820
23856
|
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
23821
23857
|
let count = 0;
|
|
23858
|
+
const clock = context.patternWork.clock;
|
|
23822
23859
|
for (const partition of partitions) {
|
|
23860
|
+
clock?.check();
|
|
23823
23861
|
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
23862
|
+
clock?.check();
|
|
23824
23863
|
if (partitionFacetConfig !== void 0) {
|
|
23825
23864
|
partitionFacets.push(
|
|
23826
23865
|
partition.computeFacets(
|
|
@@ -23832,6 +23871,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23832
23871
|
}
|
|
23833
23872
|
if (matches === null || matches.count === 0) continue;
|
|
23834
23873
|
count += matches.count;
|
|
23874
|
+
clock?.check();
|
|
23835
23875
|
const page2 = partition.sortedPage({
|
|
23836
23876
|
fields,
|
|
23837
23877
|
directions,
|
|
@@ -23844,6 +23884,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23844
23884
|
});
|
|
23845
23885
|
for (const entry of page2) merged.push(entry);
|
|
23846
23886
|
}
|
|
23887
|
+
clock?.check();
|
|
23847
23888
|
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
23848
23889
|
const hasMore = merged.length > offset + limit;
|
|
23849
23890
|
const page = merged.slice(offset, offset + limit);
|
|
@@ -23855,7 +23896,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23855
23896
|
score: null,
|
|
23856
23897
|
sortKey: last.key,
|
|
23857
23898
|
sortSignature,
|
|
23858
|
-
binding: context.cursorBinding
|
|
23899
|
+
binding: context.cursorBinding()
|
|
23859
23900
|
});
|
|
23860
23901
|
}
|
|
23861
23902
|
return {
|
|
@@ -23870,18 +23911,18 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23870
23911
|
function requireCursorDepth(cursor, signature, binding) {
|
|
23871
23912
|
if (cursor === void 0) return 0;
|
|
23872
23913
|
const decoded = decodePageCursor(cursor);
|
|
23873
|
-
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
23914
|
+
requireMatchingCursor(decoded, cursor, signature, true, binding());
|
|
23874
23915
|
return decoded.depth ?? 0;
|
|
23875
23916
|
}
|
|
23876
23917
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
23877
23918
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
23878
|
-
return import('./runtime-
|
|
23919
|
+
return import('./runtime-BX6XRFYR.mjs').then(
|
|
23879
23920
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
23880
23921
|
);
|
|
23881
23922
|
}
|
|
23882
23923
|
function executeQuery(request, callerContext) {
|
|
23883
23924
|
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
23884
|
-
return import('./runtime-
|
|
23925
|
+
return import('./runtime-BX6XRFYR.mjs').then(
|
|
23885
23926
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
23886
23927
|
);
|
|
23887
23928
|
}
|
|
@@ -24262,7 +24303,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24262
24303
|
const manager = core.requireManager(indexName);
|
|
24263
24304
|
const scoped = scopedParams(params, options);
|
|
24264
24305
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24265
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24306
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-BX6XRFYR.mjs');
|
|
24266
24307
|
const resolvedParams = await resolveVectorText(
|
|
24267
24308
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24268
24309
|
entry.embeddingAdapter,
|
|
@@ -24279,7 +24320,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24279
24320
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24280
24321
|
partitionIds: options?.partitionIds,
|
|
24281
24322
|
rescoreWindow: options?.rescoreWindow,
|
|
24282
|
-
cursorBinding:
|
|
24323
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24283
24324
|
patternWorkCap: core.patternWorkCap,
|
|
24284
24325
|
rerank: rerank?.rerankRunnerFor(core)
|
|
24285
24326
|
};
|
|
@@ -24328,7 +24369,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24328
24369
|
indexName,
|
|
24329
24370
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24330
24371
|
partitionIds: options?.partitionIds,
|
|
24331
|
-
cursorBinding:
|
|
24372
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24332
24373
|
patternWorkCap: core.patternWorkCap
|
|
24333
24374
|
};
|
|
24334
24375
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
@@ -19,6 +19,15 @@ function createFixedBuffer(bytes) {
|
|
|
19
19
|
function nextGrowthTarget(have, needed, max) {
|
|
20
20
|
return Math.min(max, Math.max(needed, have * 2));
|
|
21
21
|
}
|
|
22
|
+
function grownByteLength(have, needed, max, step = 1) {
|
|
23
|
+
let size = have;
|
|
24
|
+
while (size < needed) {
|
|
25
|
+
const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
|
|
26
|
+
if (next <= size) return size;
|
|
27
|
+
size = next;
|
|
28
|
+
}
|
|
29
|
+
return size;
|
|
30
|
+
}
|
|
22
31
|
function growBufferTo(buffer, bytes) {
|
|
23
32
|
const held = buffer.byteLength;
|
|
24
33
|
if (held >= bytes) return;
|
|
@@ -1115,6 +1124,25 @@ function ensureBlockSlots(handle, slots) {
|
|
|
1115
1124
|
}
|
|
1116
1125
|
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1117
1126
|
}
|
|
1127
|
+
function blockGrowthBytes(handle, layout, slots) {
|
|
1128
|
+
const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
|
|
1129
|
+
if (handle !== null && handle.storage.kind === "bytes") {
|
|
1130
|
+
const buffer = handle.storage.buffer;
|
|
1131
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
1132
|
+
}
|
|
1133
|
+
const have = handle === null ? 0 : blockBuffer(handle).byteLength;
|
|
1134
|
+
const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
|
|
1135
|
+
return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
|
|
1136
|
+
}
|
|
1137
|
+
function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
|
|
1138
|
+
let bytes = 0;
|
|
1139
|
+
const lastBlock = Math.floor(lastOrdinal / layout.capacity);
|
|
1140
|
+
for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
|
|
1141
|
+
const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
|
|
1142
|
+
bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
|
|
1143
|
+
}
|
|
1144
|
+
return bytes;
|
|
1145
|
+
}
|
|
1118
1146
|
function blockBuffer(handle) {
|
|
1119
1147
|
const storage = handle.storage;
|
|
1120
1148
|
if (storage.kind === "bytes") return storage.buffer;
|
|
@@ -2075,7 +2103,7 @@ function readNeighborsLocked(state, ord, layer) {
|
|
|
2075
2103
|
if (nodeReadHeld(state.locks, ord, version)) return count;
|
|
2076
2104
|
}
|
|
2077
2105
|
}
|
|
2078
|
-
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results, accepted) {
|
|
2106
|
+
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results, accepted, clock) {
|
|
2079
2107
|
const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
|
|
2080
2108
|
const workspace = state.workspace;
|
|
2081
2109
|
const frontier = workspace.frontier;
|
|
@@ -2105,6 +2133,7 @@ function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, di
|
|
|
2105
2133
|
while (popHeap(frontier)) {
|
|
2106
2134
|
if (frontier.topDistance > furthestDist && found.size >= ef) break;
|
|
2107
2135
|
const count = readNeighborsLocked(state, frontier.topOrd, layer);
|
|
2136
|
+
if (clock !== void 0) clock.tick(count * state.dimension);
|
|
2108
2137
|
for (let i = 0; i < count; i++) {
|
|
2109
2138
|
const neighborOrd = scratch[i];
|
|
2110
2139
|
ensureVisited(state, neighborOrd + 1);
|
|
@@ -2638,12 +2667,12 @@ function traverse(state, query, k, searchMetric, options) {
|
|
|
2638
2667
|
if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
|
|
2639
2668
|
setSingleEntryPoint(workspace, entryPoint);
|
|
2640
2669
|
for (let layer = topLayerOf(state); layer >= 1; layer--) {
|
|
2641
|
-
searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
|
|
2670
|
+
searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates, void 0, options.clock);
|
|
2642
2671
|
if (candidates.size > 0) {
|
|
2643
2672
|
setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
2644
2673
|
}
|
|
2645
2674
|
}
|
|
2646
|
-
searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates, filter);
|
|
2675
|
+
searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates, filter, options.clock);
|
|
2647
2676
|
} finally {
|
|
2648
2677
|
unlockGraphShared(state.locks);
|
|
2649
2678
|
}
|
|
@@ -3458,6 +3487,7 @@ var BUILD_CHUNK_SIZE = 100;
|
|
|
3458
3487
|
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
3459
3488
|
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
3460
3489
|
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
3490
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
3461
3491
|
var REBUILD_REMOVED_RATIO = 0.2;
|
|
3462
3492
|
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
3463
3493
|
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
@@ -4480,4 +4510,9 @@ function recordCheckpoint(state, plan, written) {
|
|
|
4480
4510
|
state.savedSignature = plan.signature;
|
|
4481
4511
|
}
|
|
4482
4512
|
|
|
4483
|
-
|
|
4513
|
+
// src/runtime/idle-heap-cleanup.browser.ts
|
|
4514
|
+
function collectAndReturnPages() {
|
|
4515
|
+
return false;
|
|
4516
|
+
}
|
|
4517
|
+
|
|
4518
|
+
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-2TMWIATH.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
2
|
-
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-FODCC3MK.mjs';
|
|
2
|
+
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-2TMWIATH.mjs';
|
|
3
3
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|