@delali/narsil 0.3.1-next.111 → 0.3.1-next.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-6YUUEMQ4.mjs → chunk-3YPLBK4C.mjs} +2 -2
- package/dist/browser/{chunk-5BQPBG3G.mjs → chunk-4FVSBSXY.mjs} +1 -1
- package/dist/browser/{chunk-HQOYKRMB.mjs → chunk-DC2SZO4D.mjs} +183 -44
- package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
- package/dist/browser/{durability-wiring-CYAJBYCO.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
- package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
- package/dist/browser/{restore-queue-RTZGJT4D.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
- package/dist/browser/{snapshot-BU6I25CW.mjs → snapshot-BE3F2PNC.mjs} +6 -5
- package/dist/browser/{write-RUPYHPBZ.mjs → write-MV7K7OCU.mjs} +2 -2
- package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
- package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
- package/dist/{chunk-6S7YRUPQ.mjs → chunk-27ZEY73B.mjs} +4 -4
- package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
- package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
- package/dist/{chunk-R2MRMH6M.mjs → chunk-5QIRLJH4.mjs} +3 -3
- package/dist/{chunk-7JOONQ24.mjs → chunk-5WEBVYOA.mjs} +176 -38
- package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
- package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
- package/dist/{chunk-JBPYFYIU.mjs → chunk-DZXGNLK5.mjs} +3 -3
- package/dist/chunk-EBIHRL7B.mjs +21 -0
- package/dist/{chunk-UGKO3RG2.mjs → chunk-GLZI636G.mjs} +1 -1
- package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
- package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
- package/dist/chunk-IWVEG5CX.mjs +67 -0
- package/dist/{chunk-SXVDBHYJ.mjs → chunk-JUAFWJON.mjs} +3 -3
- package/dist/{chunk-PKE2PCTS.mjs → chunk-LVNBY2TS.mjs} +1 -1
- package/dist/{chunk-YLE75NQ5.mjs → chunk-OABTAUCW.mjs} +2 -2
- package/dist/{chunk-YSR5L3XJ.mjs → chunk-OM5G2J6X.mjs} +1 -1
- package/dist/{chunk-IW6UR3AF.mjs → chunk-PABYSTKO.mjs} +13 -10
- package/dist/chunk-PELHVTO7.mjs +24 -0
- package/dist/{chunk-CUVTRLA4.mjs → chunk-QGVMEMCG.mjs} +4 -4
- package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
- package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
- package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
- package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
- package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
- package/dist/{direct-executor-QFAUKTCQ.mjs → direct-executor-R4HOSVPE.mjs} +6 -5
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +74 -37
- package/dist/{durability-wiring-NUHTYXHA.mjs → durability-wiring-XZI6355I.mjs} +15 -14
- package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
- package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +8 -7
- package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +12 -11
- package/dist/react.d.ts +6 -6
- package/dist/{restore-queue-QSLGXZJB.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +20 -62
- package/dist/{snapshot-IV3S2TMI.mjs → snapshot-PDUGHQ25.mjs} +16 -15
- package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
- package/dist/{thread-THLCKMEB.mjs → thread-S4SUCV4Y.mjs} +10 -7
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +6 -6
- package/dist/{write-VGRTOS7N.mjs → write-KVJ7YJJ6.mjs} +9 -8
- package/package.json +7 -7
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
2
|
-
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-4FVSBSXY.mjs';
|
|
2
|
+
import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-DC2SZO4D.mjs';
|
|
3
3
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
|
|
4
4
|
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-DC2SZO4D.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -3245,6 +3245,8 @@ function createDocumentStore() {
|
|
|
3245
3245
|
// src/core/constants.ts
|
|
3246
3246
|
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
3247
3247
|
var MAX_TERM_FREQUENCY = 65535;
|
|
3248
|
+
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
3249
|
+
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
3248
3250
|
|
|
3249
3251
|
// src/core/fuzzy.ts
|
|
3250
3252
|
var FIRST_SURROGATE = 55296;
|
|
@@ -4944,6 +4946,7 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
|
4944
4946
|
}
|
|
4945
4947
|
|
|
4946
4948
|
// src/geo/geo-index.ts
|
|
4949
|
+
var REMOVED_DOC_ID = -1;
|
|
4947
4950
|
function firstAtOrAfter(entries, code) {
|
|
4948
4951
|
let lo = 0;
|
|
4949
4952
|
let hi = entries.length;
|
|
@@ -4954,25 +4957,58 @@ function firstAtOrAfter(entries, code) {
|
|
|
4954
4957
|
}
|
|
4955
4958
|
return lo;
|
|
4956
4959
|
}
|
|
4960
|
+
function dropRemoved(entries) {
|
|
4961
|
+
let write = 0;
|
|
4962
|
+
for (let read = 0; read < entries.length; read++) {
|
|
4963
|
+
if (entries[read].docId !== REMOVED_DOC_ID) entries[write++] = entries[read];
|
|
4964
|
+
}
|
|
4965
|
+
entries.length = write;
|
|
4966
|
+
}
|
|
4967
|
+
function mergeSortedPending(entries, pending) {
|
|
4968
|
+
pending.sort((left2, right2) => left2.code - right2.code || left2.docId - right2.docId);
|
|
4969
|
+
const mainLength = entries.length;
|
|
4970
|
+
for (const entry of pending) entries.push(entry);
|
|
4971
|
+
let left = mainLength - 1;
|
|
4972
|
+
let right = pending.length - 1;
|
|
4973
|
+
let write = entries.length - 1;
|
|
4974
|
+
while (right >= 0) {
|
|
4975
|
+
const later = left >= 0 ? entries[left] : void 0;
|
|
4976
|
+
const next = pending[right];
|
|
4977
|
+
if (later !== void 0 && (later.code > next.code || later.code === next.code && later.docId > next.docId)) {
|
|
4978
|
+
entries[write--] = later;
|
|
4979
|
+
left--;
|
|
4980
|
+
} else {
|
|
4981
|
+
entries[write--] = next;
|
|
4982
|
+
right--;
|
|
4983
|
+
}
|
|
4984
|
+
}
|
|
4985
|
+
}
|
|
4957
4986
|
function createGeoIndex() {
|
|
4958
|
-
|
|
4987
|
+
let entries = [];
|
|
4988
|
+
let pending = [];
|
|
4959
4989
|
const byDocument = /* @__PURE__ */ new Map();
|
|
4960
|
-
let
|
|
4990
|
+
let removedCount = 0;
|
|
4961
4991
|
function ensureSorted() {
|
|
4962
|
-
if (
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
if (
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
4972
|
-
|
|
4973
|
-
entries.splice(at, 1);
|
|
4974
|
-
return;
|
|
4992
|
+
if (removedCount > 0) {
|
|
4993
|
+
dropRemoved(entries);
|
|
4994
|
+
dropRemoved(pending);
|
|
4995
|
+
removedCount = 0;
|
|
4996
|
+
}
|
|
4997
|
+
if (pending.length === 0) return;
|
|
4998
|
+
if (entries.length === 0) {
|
|
4999
|
+
pending.sort((left, right) => left.code - right.code || left.docId - right.docId);
|
|
5000
|
+
entries = pending;
|
|
5001
|
+
} else {
|
|
5002
|
+
mergeSortedPending(entries, pending);
|
|
4975
5003
|
}
|
|
5004
|
+
pending = [];
|
|
5005
|
+
}
|
|
5006
|
+
function removeEntry(entry) {
|
|
5007
|
+
byDocument.delete(entry.docId);
|
|
5008
|
+
entry.docId = REMOVED_DOC_ID;
|
|
5009
|
+
removedCount++;
|
|
5010
|
+
const held = entries.length + pending.length;
|
|
5011
|
+
if (removedCount >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) ensureSorted();
|
|
4976
5012
|
}
|
|
4977
5013
|
function matching(boxes, passes, inside) {
|
|
4978
5014
|
ensureSorted();
|
|
@@ -4991,14 +5027,15 @@ function createGeoIndex() {
|
|
|
4991
5027
|
}
|
|
4992
5028
|
const index = {
|
|
4993
5029
|
insert(internalId, lat, lon) {
|
|
4994
|
-
|
|
5030
|
+
const existing = byDocument.get(internalId);
|
|
5031
|
+
if (existing !== void 0) removeEntry(existing);
|
|
4995
5032
|
const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
|
|
4996
|
-
|
|
5033
|
+
pending.push(entry);
|
|
4997
5034
|
byDocument.set(internalId, entry);
|
|
4998
|
-
sorted = false;
|
|
4999
5035
|
},
|
|
5000
5036
|
remove(internalId) {
|
|
5001
|
-
|
|
5037
|
+
const existing = byDocument.get(internalId);
|
|
5038
|
+
if (existing !== void 0) removeEntry(existing);
|
|
5002
5039
|
},
|
|
5003
5040
|
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
5004
5041
|
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
@@ -5014,13 +5051,14 @@ function createGeoIndex() {
|
|
|
5014
5051
|
},
|
|
5015
5052
|
documentsBitset(capacity) {
|
|
5016
5053
|
const bits = createBitSet(capacity);
|
|
5017
|
-
for (const
|
|
5054
|
+
for (const docId of byDocument.keys()) bitsetSet(bits, docId);
|
|
5018
5055
|
return bits;
|
|
5019
5056
|
},
|
|
5020
5057
|
clear() {
|
|
5021
|
-
entries
|
|
5058
|
+
entries = [];
|
|
5059
|
+
pending = [];
|
|
5022
5060
|
byDocument.clear();
|
|
5023
|
-
|
|
5061
|
+
removedCount = 0;
|
|
5024
5062
|
},
|
|
5025
5063
|
serialize() {
|
|
5026
5064
|
ensureSorted();
|
|
@@ -5348,29 +5386,126 @@ function collectDocIdsBitset(entries, from, to, capacity) {
|
|
|
5348
5386
|
}
|
|
5349
5387
|
return bs;
|
|
5350
5388
|
}
|
|
5389
|
+
function removalsFor(byDocument, docId, value) {
|
|
5390
|
+
const keys = byDocument.get(docId);
|
|
5391
|
+
if (keys === void 0) return void 0;
|
|
5392
|
+
for (const key of keys) {
|
|
5393
|
+
if (key.value === value) return key;
|
|
5394
|
+
}
|
|
5395
|
+
return void 0;
|
|
5396
|
+
}
|
|
5397
|
+
function groupRemovals(docIds, values, limits) {
|
|
5398
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
5399
|
+
for (let at = 0; at < docIds.length; at++) {
|
|
5400
|
+
const existing = removalsFor(byDocument, docIds[at], values[at]);
|
|
5401
|
+
if (existing !== void 0) {
|
|
5402
|
+
existing.limits.push(limits[at]);
|
|
5403
|
+
continue;
|
|
5404
|
+
}
|
|
5405
|
+
const key = { value: values[at], limits: [limits[at]], next: 0 };
|
|
5406
|
+
const keys = byDocument.get(docIds[at]);
|
|
5407
|
+
if (keys === void 0) byDocument.set(docIds[at], [key]);
|
|
5408
|
+
else keys.push(key);
|
|
5409
|
+
}
|
|
5410
|
+
return byDocument;
|
|
5411
|
+
}
|
|
5412
|
+
function markedDocuments(docIds) {
|
|
5413
|
+
let highest = 0;
|
|
5414
|
+
for (const docId of docIds) if (docId > highest) highest = docId;
|
|
5415
|
+
const marked = new Uint8Array(highest + 1);
|
|
5416
|
+
for (const docId of docIds) marked[docId] = 1;
|
|
5417
|
+
return marked;
|
|
5418
|
+
}
|
|
5419
|
+
function removeFromSorted(entries, from, marked, byDocument) {
|
|
5420
|
+
let write = from;
|
|
5421
|
+
for (let read = from; read < entries.length; read++) {
|
|
5422
|
+
const entry = entries[read];
|
|
5423
|
+
if (entry.docId < marked.length && marked[entry.docId] === 1) {
|
|
5424
|
+
const key = removalsFor(byDocument, entry.docId, entry.value);
|
|
5425
|
+
if (key !== void 0 && key.next < key.limits.length) {
|
|
5426
|
+
key.next++;
|
|
5427
|
+
continue;
|
|
5428
|
+
}
|
|
5429
|
+
}
|
|
5430
|
+
entries[write++] = entry;
|
|
5431
|
+
}
|
|
5432
|
+
entries.length = write;
|
|
5433
|
+
}
|
|
5434
|
+
function removeFromPending(pending, marked, byDocument) {
|
|
5435
|
+
let write = 0;
|
|
5436
|
+
for (let read = 0; read < pending.length; read++) {
|
|
5437
|
+
const entry = pending[read];
|
|
5438
|
+
if (entry.docId < marked.length && marked[entry.docId] === 1) {
|
|
5439
|
+
const key = removalsFor(byDocument, entry.docId, entry.value);
|
|
5440
|
+
if (key !== void 0) {
|
|
5441
|
+
while (key.next < key.limits.length && key.limits[key.next] <= read) key.next++;
|
|
5442
|
+
if (key.next < key.limits.length) {
|
|
5443
|
+
key.next++;
|
|
5444
|
+
continue;
|
|
5445
|
+
}
|
|
5446
|
+
}
|
|
5447
|
+
}
|
|
5448
|
+
pending[write++] = entry;
|
|
5449
|
+
}
|
|
5450
|
+
pending.length = write;
|
|
5451
|
+
}
|
|
5452
|
+
function mergeSortedPending2(entries, pending) {
|
|
5453
|
+
pending.sort((a, b) => a.value - b.value);
|
|
5454
|
+
const mainLength = entries.length;
|
|
5455
|
+
for (const entry of pending) entries.push(entry);
|
|
5456
|
+
let left = mainLength - 1;
|
|
5457
|
+
let right = pending.length - 1;
|
|
5458
|
+
let write = entries.length - 1;
|
|
5459
|
+
while (right >= 0) {
|
|
5460
|
+
if (left >= 0 && entries[left].value > pending[right].value) entries[write--] = entries[left--];
|
|
5461
|
+
else entries[write--] = pending[right--];
|
|
5462
|
+
}
|
|
5463
|
+
}
|
|
5351
5464
|
function createNumericIndex() {
|
|
5352
5465
|
let entries = [];
|
|
5353
|
-
let
|
|
5466
|
+
let pending = [];
|
|
5467
|
+
const removedDocIds = [];
|
|
5468
|
+
const removedValues = [];
|
|
5469
|
+
const removedLimits = [];
|
|
5470
|
+
function applyRemovals() {
|
|
5471
|
+
const byDocument = groupRemovals(removedDocIds, removedValues, removedLimits);
|
|
5472
|
+
const marked = markedDocuments(removedDocIds);
|
|
5473
|
+
let lowest = removedValues[0];
|
|
5474
|
+
for (const value of removedValues) if (value < lowest) lowest = value;
|
|
5475
|
+
removeFromSorted(entries, lowerBound(entries, lowest), marked, byDocument);
|
|
5476
|
+
if (pending.length > 0) removeFromPending(pending, marked, byDocument);
|
|
5477
|
+
removedDocIds.length = 0;
|
|
5478
|
+
removedValues.length = 0;
|
|
5479
|
+
removedLimits.length = 0;
|
|
5480
|
+
}
|
|
5354
5481
|
function ensureSorted() {
|
|
5355
|
-
if (
|
|
5356
|
-
|
|
5357
|
-
|
|
5482
|
+
if (removedDocIds.length > 0) applyRemovals();
|
|
5483
|
+
if (pending.length === 0) return;
|
|
5484
|
+
if (entries.length === 0) {
|
|
5485
|
+
pending.sort((a, b) => a.value - b.value);
|
|
5486
|
+
entries = pending;
|
|
5487
|
+
} else {
|
|
5488
|
+
mergeSortedPending2(entries, pending);
|
|
5358
5489
|
}
|
|
5490
|
+
pending = [];
|
|
5491
|
+
}
|
|
5492
|
+
function discardPending() {
|
|
5493
|
+
pending = [];
|
|
5494
|
+
removedDocIds.length = 0;
|
|
5495
|
+
removedValues.length = 0;
|
|
5496
|
+
removedLimits.length = 0;
|
|
5359
5497
|
}
|
|
5360
5498
|
return {
|
|
5361
5499
|
insert(internalId, value) {
|
|
5362
|
-
|
|
5363
|
-
sorted = false;
|
|
5500
|
+
pending.push({ value, docId: internalId });
|
|
5364
5501
|
},
|
|
5365
5502
|
remove(internalId, value) {
|
|
5366
|
-
|
|
5367
|
-
|
|
5368
|
-
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
return;
|
|
5373
|
-
}
|
|
5503
|
+
removedDocIds.push(internalId);
|
|
5504
|
+
removedValues.push(value);
|
|
5505
|
+
removedLimits.push(pending.length);
|
|
5506
|
+
const held = entries.length + pending.length;
|
|
5507
|
+
if (removedDocIds.length >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) {
|
|
5508
|
+
ensureSorted();
|
|
5374
5509
|
}
|
|
5375
5510
|
},
|
|
5376
5511
|
queryEq(value) {
|
|
@@ -5407,6 +5542,7 @@ function createNumericIndex() {
|
|
|
5407
5542
|
return collectDocIds(entries, lowerBound(entries, min), upperBound(entries, max));
|
|
5408
5543
|
},
|
|
5409
5544
|
getAllDocIds() {
|
|
5545
|
+
ensureSorted();
|
|
5410
5546
|
return collectDocIds(entries, 0, entries.length);
|
|
5411
5547
|
},
|
|
5412
5548
|
queryEqBitset(value, capacity) {
|
|
@@ -5434,9 +5570,11 @@ function createNumericIndex() {
|
|
|
5434
5570
|
return collectDocIdsBitset(entries, lowerBound(entries, min), upperBound(entries, max), capacity);
|
|
5435
5571
|
},
|
|
5436
5572
|
getAllDocIdsBitset(capacity) {
|
|
5573
|
+
ensureSorted();
|
|
5437
5574
|
return collectDocIdsBitset(entries, 0, entries.length, capacity);
|
|
5438
5575
|
},
|
|
5439
5576
|
count() {
|
|
5577
|
+
ensureSorted();
|
|
5440
5578
|
return entries.length;
|
|
5441
5579
|
},
|
|
5442
5580
|
facetValueCounts(matched) {
|
|
@@ -5474,7 +5612,7 @@ function createNumericIndex() {
|
|
|
5474
5612
|
},
|
|
5475
5613
|
clear() {
|
|
5476
5614
|
entries = [];
|
|
5477
|
-
|
|
5615
|
+
discardPending();
|
|
5478
5616
|
},
|
|
5479
5617
|
serialize() {
|
|
5480
5618
|
ensureSorted();
|
|
@@ -5482,7 +5620,7 @@ function createNumericIndex() {
|
|
|
5482
5620
|
},
|
|
5483
5621
|
deserialize(data) {
|
|
5484
5622
|
entries = data.slice();
|
|
5485
|
-
|
|
5623
|
+
discardPending();
|
|
5486
5624
|
}
|
|
5487
5625
|
};
|
|
5488
5626
|
}
|
|
@@ -14949,7 +15087,7 @@ function requireLoadedVectorCode() {
|
|
|
14949
15087
|
function loadVectorCode() {
|
|
14950
15088
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
14951
15089
|
if (loading === null) {
|
|
14952
|
-
loading = import('./module-
|
|
15090
|
+
loading = import('./module-5EWHBKHZ.mjs').then(
|
|
14953
15091
|
(code) => {
|
|
14954
15092
|
loaded = code;
|
|
14955
15093
|
return code;
|
|
@@ -19037,7 +19175,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19037
19175
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19038
19176
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19039
19177
|
}
|
|
19040
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19178
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-2RG22WQJ.mjs');
|
|
19041
19179
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19042
19180
|
const executor = createDirectExecutor({
|
|
19043
19181
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -21781,6 +21919,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
21781
21919
|
graphCount: status.graphCount,
|
|
21782
21920
|
bufferSize: status.bufferSize,
|
|
21783
21921
|
building: status.building,
|
|
21922
|
+
waitingForMemory: status.waitingForMemory,
|
|
21784
21923
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
21785
21924
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
21786
21925
|
});
|
|
@@ -22470,14 +22609,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22470
22609
|
return {
|
|
22471
22610
|
snapshot(indexName) {
|
|
22472
22611
|
return withOpenIndex(indexName, async () => {
|
|
22473
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22612
|
+
const { createSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
|
|
22474
22613
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22475
22614
|
});
|
|
22476
22615
|
},
|
|
22477
22616
|
async restore(indexName, data) {
|
|
22478
22617
|
core.guardShutdown();
|
|
22479
22618
|
validateIndexName(indexName);
|
|
22480
|
-
const restored = import('./restore-queue-
|
|
22619
|
+
const restored = import('./restore-queue-WCFUB4NS.mjs').then(
|
|
22481
22620
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22482
22621
|
);
|
|
22483
22622
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22486,8 +22625,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22486
22625
|
exportDocuments(indexName) {
|
|
22487
22626
|
return withOpenIndex(indexName, async () => {
|
|
22488
22627
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22489
|
-
import('./snapshot-
|
|
22490
|
-
import('./write-
|
|
22628
|
+
import('./snapshot-BE3F2PNC.mjs'),
|
|
22629
|
+
import('./write-MV7K7OCU.mjs')
|
|
22491
22630
|
]);
|
|
22492
22631
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22493
22632
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -19,6 +19,15 @@ function createFixedBuffer(bytes) {
|
|
|
19
19
|
function nextGrowthTarget(have, needed, max) {
|
|
20
20
|
return Math.min(max, Math.max(needed, have * 2));
|
|
21
21
|
}
|
|
22
|
+
function grownByteLength(have, needed, max, step = 1) {
|
|
23
|
+
let size = have;
|
|
24
|
+
while (size < needed) {
|
|
25
|
+
const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
|
|
26
|
+
if (next <= size) return size;
|
|
27
|
+
size = next;
|
|
28
|
+
}
|
|
29
|
+
return size;
|
|
30
|
+
}
|
|
22
31
|
function growBufferTo(buffer, bytes) {
|
|
23
32
|
const held = buffer.byteLength;
|
|
24
33
|
if (held >= bytes) return;
|
|
@@ -1115,6 +1124,25 @@ function ensureBlockSlots(handle, slots) {
|
|
|
1115
1124
|
}
|
|
1116
1125
|
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1117
1126
|
}
|
|
1127
|
+
function blockGrowthBytes(handle, layout, slots) {
|
|
1128
|
+
const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
|
|
1129
|
+
if (handle !== null && handle.storage.kind === "bytes") {
|
|
1130
|
+
const buffer = handle.storage.buffer;
|
|
1131
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
1132
|
+
}
|
|
1133
|
+
const have = handle === null ? 0 : blockBuffer(handle).byteLength;
|
|
1134
|
+
const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
|
|
1135
|
+
return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
|
|
1136
|
+
}
|
|
1137
|
+
function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
|
|
1138
|
+
let bytes = 0;
|
|
1139
|
+
const lastBlock = Math.floor(lastOrdinal / layout.capacity);
|
|
1140
|
+
for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
|
|
1141
|
+
const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
|
|
1142
|
+
bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
|
|
1143
|
+
}
|
|
1144
|
+
return bytes;
|
|
1145
|
+
}
|
|
1118
1146
|
function blockBuffer(handle) {
|
|
1119
1147
|
const storage = handle.storage;
|
|
1120
1148
|
if (storage.kind === "bytes") return storage.buffer;
|
|
@@ -3458,6 +3486,7 @@ var BUILD_CHUNK_SIZE = 100;
|
|
|
3458
3486
|
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
3459
3487
|
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
3460
3488
|
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
3489
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
|
|
3461
3490
|
var REBUILD_REMOVED_RATIO = 0.2;
|
|
3462
3491
|
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
3463
3492
|
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
@@ -4480,4 +4509,9 @@ function recordCheckpoint(state, plan, written) {
|
|
|
4480
4509
|
state.savedSignature = plan.signature;
|
|
4481
4510
|
}
|
|
4482
4511
|
|
|
4483
|
-
|
|
4512
|
+
// src/runtime/idle-heap-cleanup.browser.ts
|
|
4513
|
+
function collectAndReturnPages() {
|
|
4514
|
+
return false;
|
|
4515
|
+
}
|
|
4516
|
+
|
|
4517
|
+
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-3YPLBK4C.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-4FVSBSXY.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-DC2SZO4D.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
|
|
6
6
|
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
|
|
7
7
|
import './chunk-JPTQZWJQ.mjs';
|