@delali/narsil 0.3.1-next.87 → 0.3.1-next.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
- package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
- package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
- package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
- package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
- package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
- package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
- package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
- package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
- package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
- package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
- package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
- package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
- package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
- package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
- package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
- package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
- package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
- package/dist/distribution.mjs +14 -27
- package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -5
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
- package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
- package/dist/workers/entry.mjs +3 -3
- package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
- package/package.json +13 -8
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected,
|
|
1
|
+
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, copyVectorsInto, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, projectIncludedPaths, readProjectedPath, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-X4SEXL4U.mjs';
|
|
2
2
|
import { getLanguage } from './chunk-PR4O7CAP.mjs';
|
|
3
3
|
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-T6CNO72J.mjs';
|
|
4
4
|
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
5
|
-
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex,
|
|
5
|
+
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, visitDocumentSurfaceSpellings, visitDocumentSpellings, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-A3UB2MXL.mjs';
|
|
6
6
|
import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-CVQWQVFX.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
8
|
import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-BVOJIYRV.mjs';
|
|
9
|
-
import { toReducedSortValue,
|
|
9
|
+
import { toReducedSortValue, defaultSortMode, readSortField, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-R7OSDAFK.mjs';
|
|
12
|
+
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
13
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
13
14
|
import { totalmem } from 'os';
|
|
14
15
|
import { getHeapStatistics } from 'v8';
|
|
15
|
-
import { decode } from '@msgpack/msgpack';
|
|
16
|
+
import { Decoder, decode } from '@msgpack/msgpack';
|
|
16
17
|
|
|
17
18
|
var MEGABYTE_BYTES = 1048576;
|
|
18
19
|
var IN_MEGABYTES = /^--max-old-space-size=(\d+)$/;
|
|
@@ -145,7 +146,7 @@ function sharesABiggerBuffer(value) {
|
|
|
145
146
|
return false;
|
|
146
147
|
}
|
|
147
148
|
function withOwnBuffers(value) {
|
|
148
|
-
if (value === null || typeof value !== "object") return value;
|
|
149
|
+
if (value === null || typeof value !== "object" || value instanceof Date) return value;
|
|
149
150
|
if (ArrayBuffer.isView(value)) return detachedView(value);
|
|
150
151
|
if (Array.isArray(value)) {
|
|
151
152
|
const copy2 = new Array(value.length);
|
|
@@ -170,6 +171,9 @@ function documentWithOwnBuffers(document) {
|
|
|
170
171
|
if (!sharesABiggerBuffer(document)) return document;
|
|
171
172
|
return withOwnBuffers(document);
|
|
172
173
|
}
|
|
174
|
+
function valueWithOwnBuffers(value) {
|
|
175
|
+
return sharesABiggerBuffer(value) ? withOwnBuffers(value) : value;
|
|
176
|
+
}
|
|
173
177
|
var STRING_HEADER_BYTES = 16;
|
|
174
178
|
var TWO_BYTE_SAMPLE_LIMIT = 16;
|
|
175
179
|
var NUMBER_BYTES = 16;
|
|
@@ -367,337 +371,926 @@ function createDocumentStore() {
|
|
|
367
371
|
},
|
|
368
372
|
resolver() {
|
|
369
373
|
return idResolver;
|
|
374
|
+
},
|
|
375
|
+
fieldReader(fieldPath) {
|
|
376
|
+
return (internalId) => {
|
|
377
|
+
const docId = reverseMap[internalId];
|
|
378
|
+
const stored = docId === void 0 ? void 0 : docs.get(docId);
|
|
379
|
+
return stored === void 0 ? void 0 : getNestedValue(stored.fields, fieldPath);
|
|
380
|
+
};
|
|
381
|
+
},
|
|
382
|
+
projectedDocument(docId, projection) {
|
|
383
|
+
const stored = docs.get(docId);
|
|
384
|
+
return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
|
|
370
385
|
}
|
|
371
386
|
};
|
|
372
387
|
}
|
|
373
388
|
|
|
374
389
|
// src/core/constants.ts
|
|
375
|
-
var POSTING_LIST_INITIAL_CAPACITY = 8;
|
|
376
390
|
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
377
391
|
var MAX_TERM_FREQUENCY = 65535;
|
|
378
392
|
|
|
379
|
-
// src/core/
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
let
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
393
|
+
// src/core/inverted-index/constants.ts
|
|
394
|
+
var TERM_TABLE_START_CAPACITY = 1 << 10;
|
|
395
|
+
var POSTING_POOL_START_ENTRIES = 1 << 12;
|
|
396
|
+
var POSITION_POOL_START_VALUES = 1 << 12;
|
|
397
|
+
var RETIRED_DOCUMENTS_START_CAPACITY = 1 << 10;
|
|
398
|
+
var SLICE_GROWTH_FACTOR = 1.5;
|
|
399
|
+
var POOL_GROWTH_FACTOR = 1.5;
|
|
400
|
+
var POOL_VACUUM_FREE_FRACTION = 0.25;
|
|
401
|
+
var CACHED_VIEW_MIN_ENTRIES = 128;
|
|
402
|
+
var TERM_VACUUM_MIN_DEAD_TERMS = 1 << 10;
|
|
403
|
+
var POOL_VACUUM_MIN_FREE_ENTRIES = 1 << 14;
|
|
404
|
+
var POSITION_VACUUM_MIN_FREE_VALUES = 1 << 14;
|
|
405
|
+
var TERM_ENTRY_ESTIMATED_BYTES = 64;
|
|
406
|
+
var TOKEN_STRING_HEADER_BYTES = 16;
|
|
407
|
+
|
|
408
|
+
// src/core/inverted-index/pools.ts
|
|
409
|
+
var NO_SLICE = 255;
|
|
410
|
+
var EMPTY_POSITIONS = 0;
|
|
411
|
+
var MAX_POOL_ENTRIES = 2 ** 32 - 1;
|
|
412
|
+
var SLICE_CAPACITIES = (() => {
|
|
413
|
+
const capacities = [1];
|
|
414
|
+
while (capacities[capacities.length - 1] < MAX_POOL_ENTRIES) {
|
|
415
|
+
const previous = capacities[capacities.length - 1];
|
|
416
|
+
capacities.push(Math.min(MAX_POOL_ENTRIES, Math.max(previous + 1, Math.ceil(previous * SLICE_GROWTH_FACTOR))));
|
|
417
|
+
}
|
|
418
|
+
return Uint32Array.from(capacities);
|
|
419
|
+
})();
|
|
420
|
+
function sliceClassFor(entries) {
|
|
421
|
+
let low = 0;
|
|
422
|
+
let high = SLICE_CAPACITIES.length - 1;
|
|
423
|
+
while (low < high) {
|
|
424
|
+
const middle = low + high >>> 1;
|
|
425
|
+
if (SLICE_CAPACITIES[middle] < entries) low = middle + 1;
|
|
426
|
+
else high = middle;
|
|
427
|
+
}
|
|
428
|
+
return low;
|
|
429
|
+
}
|
|
430
|
+
function sliceCapacity(sliceClass) {
|
|
431
|
+
return sliceClass === NO_SLICE ? 0 : SLICE_CAPACITIES[sliceClass];
|
|
432
|
+
}
|
|
433
|
+
function grownCapacity(current, needed) {
|
|
434
|
+
return Math.min(MAX_POOL_ENTRIES, Math.max(needed, Math.ceil(current * POOL_GROWTH_FACTOR)));
|
|
435
|
+
}
|
|
436
|
+
var PostingPool = class {
|
|
437
|
+
docIds;
|
|
438
|
+
frequencies;
|
|
439
|
+
fieldIndices;
|
|
440
|
+
positionStarts = null;
|
|
441
|
+
end = 0;
|
|
442
|
+
epoch = 0;
|
|
443
|
+
freeEntries = 0;
|
|
444
|
+
freeLists = [];
|
|
445
|
+
constructor(capacity) {
|
|
446
|
+
this.docIds = new Uint32Array(capacity);
|
|
447
|
+
this.frequencies = new Uint16Array(capacity);
|
|
448
|
+
this.fieldIndices = new Uint16Array(capacity);
|
|
449
|
+
}
|
|
450
|
+
allocate(sliceClass) {
|
|
451
|
+
const size = sliceCapacity(sliceClass);
|
|
452
|
+
const free = this.freeLists[sliceClass];
|
|
453
|
+
if (free !== void 0) {
|
|
454
|
+
const reused = free.pop();
|
|
455
|
+
if (reused !== void 0) {
|
|
456
|
+
this.freeEntries -= size;
|
|
457
|
+
return reused;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
const start = this.end;
|
|
461
|
+
this.reserve(start + size);
|
|
462
|
+
this.end = start + size;
|
|
463
|
+
return start;
|
|
464
|
+
}
|
|
465
|
+
release(start, sliceClass) {
|
|
466
|
+
if (sliceClass === NO_SLICE) return;
|
|
467
|
+
let free = this.freeLists[sliceClass];
|
|
468
|
+
if (free === void 0) {
|
|
469
|
+
free = [];
|
|
470
|
+
this.freeLists[sliceClass] = free;
|
|
471
|
+
}
|
|
472
|
+
free.push(start);
|
|
473
|
+
this.freeEntries += sliceCapacity(sliceClass);
|
|
474
|
+
}
|
|
475
|
+
trackPositions() {
|
|
476
|
+
if (this.positionStarts === null) {
|
|
477
|
+
this.positionStarts = new Uint32Array(this.docIds.length);
|
|
478
|
+
this.epoch++;
|
|
479
|
+
}
|
|
480
|
+
return this.positionStarts;
|
|
481
|
+
}
|
|
482
|
+
move(from, to, count) {
|
|
483
|
+
this.docIds.copyWithin(to, from, from + count);
|
|
484
|
+
this.frequencies.copyWithin(to, from, from + count);
|
|
485
|
+
this.fieldIndices.copyWithin(to, from, from + count);
|
|
486
|
+
if (this.positionStarts !== null) this.positionStarts.copyWithin(to, from, from + count);
|
|
487
|
+
}
|
|
488
|
+
byteLength() {
|
|
489
|
+
const positionBytes = this.positionStarts === null ? 0 : this.positionStarts.byteLength;
|
|
490
|
+
return this.docIds.byteLength + this.frequencies.byteLength + this.fieldIndices.byteLength + positionBytes;
|
|
491
|
+
}
|
|
492
|
+
reserve(needed) {
|
|
493
|
+
if (needed <= this.docIds.length) return;
|
|
494
|
+
const capacity = grownCapacity(this.docIds.length, needed);
|
|
495
|
+
this.docIds = grownCopy(this.docIds, new Uint32Array(capacity), this.end);
|
|
496
|
+
this.frequencies = grownCopy(this.frequencies, new Uint16Array(capacity), this.end);
|
|
497
|
+
this.fieldIndices = grownCopy(this.fieldIndices, new Uint16Array(capacity), this.end);
|
|
498
|
+
if (this.positionStarts !== null) {
|
|
499
|
+
this.positionStarts = grownCopy(this.positionStarts, new Uint32Array(capacity), this.end);
|
|
500
|
+
}
|
|
501
|
+
this.epoch++;
|
|
445
502
|
}
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
}
|
|
503
|
+
};
|
|
504
|
+
function grownCopy(source, target, used) {
|
|
505
|
+
target.set(source.subarray(0, used));
|
|
506
|
+
return target;
|
|
507
|
+
}
|
|
508
|
+
var PositionPool = class {
|
|
509
|
+
values;
|
|
510
|
+
end = 1;
|
|
511
|
+
freeValues = 0;
|
|
512
|
+
constructor(capacity) {
|
|
513
|
+
this.values = new Uint32Array(Math.max(1, capacity));
|
|
514
|
+
}
|
|
515
|
+
append(positions) {
|
|
516
|
+
const count = positions.length;
|
|
517
|
+
if (count === 0) return EMPTY_POSITIONS;
|
|
518
|
+
const start = this.end;
|
|
519
|
+
this.reserve(start + 1 + count);
|
|
520
|
+
const values = this.values;
|
|
521
|
+
values[start] = count;
|
|
522
|
+
for (let i = 0; i < count; i++) values[start + 1 + i] = positions[i];
|
|
523
|
+
this.end = start + 1 + count;
|
|
524
|
+
return start;
|
|
525
|
+
}
|
|
526
|
+
appendFrom(source, start) {
|
|
527
|
+
if (start === EMPTY_POSITIONS) return EMPTY_POSITIONS;
|
|
528
|
+
const count = source.values[start];
|
|
529
|
+
const target = this.end;
|
|
530
|
+
this.reserve(target + 1 + count);
|
|
531
|
+
this.values.set(source.values.subarray(start, start + 1 + count), target);
|
|
532
|
+
this.end = target + 1 + count;
|
|
533
|
+
return target;
|
|
534
|
+
}
|
|
535
|
+
countAt(start) {
|
|
536
|
+
return this.values[start];
|
|
537
|
+
}
|
|
538
|
+
read(start) {
|
|
539
|
+
const count = this.values[start];
|
|
540
|
+
const out = new Array(count);
|
|
541
|
+
for (let i = 0; i < count; i++) out[i] = this.values[start + 1 + i];
|
|
542
|
+
return out;
|
|
543
|
+
}
|
|
544
|
+
copyInto(start, target, offset) {
|
|
545
|
+
const count = this.values[start];
|
|
546
|
+
if (count > 0) target.set(this.values.subarray(start + 1, start + 1 + count), offset);
|
|
547
|
+
return count;
|
|
548
|
+
}
|
|
549
|
+
release(start) {
|
|
550
|
+
if (start === EMPTY_POSITIONS) return;
|
|
551
|
+
this.freeValues += this.values[start] + 1;
|
|
552
|
+
}
|
|
553
|
+
reserve(needed) {
|
|
554
|
+
if (needed <= this.values.length) return;
|
|
555
|
+
const capacity = grownCapacity(this.values.length, needed);
|
|
556
|
+
const grown = new Uint32Array(capacity);
|
|
557
|
+
grown.set(this.values.subarray(0, this.end));
|
|
558
|
+
this.values = grown;
|
|
559
|
+
}
|
|
560
|
+
};
|
|
451
561
|
|
|
452
|
-
// src/core/inverted-index.ts
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
562
|
+
// src/core/inverted-index/terms.ts
|
|
563
|
+
var TERM_LIVE = 1;
|
|
564
|
+
var TERM_ORDERED = 2;
|
|
565
|
+
var TERM_POSITIONS = 4;
|
|
566
|
+
function grow(source, target) {
|
|
567
|
+
target.set(source);
|
|
568
|
+
return target;
|
|
569
|
+
}
|
|
570
|
+
var TermColumns = class _TermColumns {
|
|
571
|
+
constructor(startCapacity) {
|
|
572
|
+
this.startCapacity = startCapacity;
|
|
573
|
+
this.sliceStart = new Uint32Array(startCapacity);
|
|
574
|
+
this.sliceClass = new Uint8Array(startCapacity);
|
|
575
|
+
this.lengths = new Uint32Array(startCapacity);
|
|
576
|
+
this.docFrequency = new Uint32Array(startCapacity);
|
|
577
|
+
this.deadDocuments = new Uint32Array(startCapacity);
|
|
578
|
+
this.totalTermFrequency = new Float64Array(startCapacity);
|
|
579
|
+
this.revision = new Uint32Array(startCapacity);
|
|
580
|
+
this.flags = new Uint8Array(startCapacity);
|
|
581
|
+
}
|
|
582
|
+
ids = /* @__PURE__ */ new Map();
|
|
583
|
+
tokens = [];
|
|
584
|
+
sliceStart;
|
|
585
|
+
sliceClass;
|
|
586
|
+
lengths;
|
|
587
|
+
docFrequency;
|
|
588
|
+
deadDocuments;
|
|
589
|
+
totalTermFrequency;
|
|
590
|
+
revision;
|
|
591
|
+
flags;
|
|
592
|
+
count = 0;
|
|
593
|
+
liveCount = 0;
|
|
594
|
+
generation = 0;
|
|
595
|
+
tokenCodeUnits = 0;
|
|
596
|
+
buckets = /* @__PURE__ */ new Map();
|
|
597
|
+
get deadCount() {
|
|
598
|
+
return this.count - this.liveCount;
|
|
599
|
+
}
|
|
600
|
+
add(token) {
|
|
601
|
+
const ordinal = this.count;
|
|
602
|
+
if (ordinal === this.flags.length) this.growTo(ordinal * 2);
|
|
603
|
+
this.count = ordinal + 1;
|
|
604
|
+
this.liveCount++;
|
|
605
|
+
this.tokenCodeUnits += token.length;
|
|
606
|
+
this.ids.set(token, ordinal);
|
|
607
|
+
this.tokens.push(token);
|
|
608
|
+
this.sliceClass[ordinal] = NO_SLICE;
|
|
609
|
+
this.flags[ordinal] = TERM_LIVE | TERM_ORDERED;
|
|
610
|
+
this.track(token, ordinal);
|
|
611
|
+
return ordinal;
|
|
612
|
+
}
|
|
613
|
+
kill(ordinal) {
|
|
614
|
+
const token = this.tokens[ordinal];
|
|
615
|
+
this.ids.delete(token);
|
|
616
|
+
const bucket = token.length > 0 ? this.buckets.get(token[0]) : void 0;
|
|
617
|
+
if (bucket !== void 0) bucket.dead++;
|
|
618
|
+
this.tokens[ordinal] = "";
|
|
619
|
+
this.flags[ordinal] = 0;
|
|
620
|
+
this.lengths[ordinal] = 0;
|
|
621
|
+
this.docFrequency[ordinal] = 0;
|
|
622
|
+
this.deadDocuments[ordinal] = 0;
|
|
623
|
+
this.totalTermFrequency[ordinal] = 0;
|
|
624
|
+
this.sliceClass[ordinal] = NO_SLICE;
|
|
625
|
+
this.liveCount--;
|
|
626
|
+
this.tokenCodeUnits -= token.length;
|
|
627
|
+
}
|
|
628
|
+
isLive(ordinal) {
|
|
629
|
+
return (this.flags[ordinal] & TERM_LIVE) !== 0;
|
|
630
|
+
}
|
|
631
|
+
forEachLive(visit) {
|
|
632
|
+
const flags = this.flags;
|
|
633
|
+
for (let ordinal = 0; ordinal < this.count; ordinal++) {
|
|
634
|
+
if ((flags[ordinal] & TERM_LIVE) !== 0) visit(ordinal);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
forEachStartingWith(prefix, visit) {
|
|
638
|
+
if (prefix.length === 0) {
|
|
639
|
+
this.forEachLive(visit);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
const bucket = this.buckets.get(prefix[0]);
|
|
643
|
+
if (bucket === void 0) return;
|
|
644
|
+
this.compactBucket(prefix[0], bucket);
|
|
645
|
+
const { ordinals, length } = bucket;
|
|
646
|
+
const flags = this.flags;
|
|
647
|
+
const tokens = this.tokens;
|
|
648
|
+
const checkPrefix = prefix.length > 1;
|
|
649
|
+
for (let i = 0; i < length; i++) {
|
|
650
|
+
const ordinal = ordinals[i];
|
|
651
|
+
if ((flags[ordinal] & TERM_LIVE) === 0) continue;
|
|
652
|
+
if (checkPrefix && !tokens[ordinal].startsWith(prefix)) continue;
|
|
653
|
+
visit(ordinal);
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
renumber() {
|
|
657
|
+
const remap = new Int32Array(this.count).fill(-1);
|
|
658
|
+
const capacity = Math.max(this.startCapacity, this.liveCount);
|
|
659
|
+
const next = new _TermColumns(capacity);
|
|
660
|
+
for (let ordinal = 0; ordinal < this.count; ordinal++) {
|
|
661
|
+
if ((this.flags[ordinal] & TERM_LIVE) === 0) continue;
|
|
662
|
+
const target = next.add(this.tokens[ordinal]);
|
|
663
|
+
remap[ordinal] = target;
|
|
664
|
+
next.sliceStart[target] = this.sliceStart[ordinal];
|
|
665
|
+
next.sliceClass[target] = this.sliceClass[ordinal];
|
|
666
|
+
next.lengths[target] = this.lengths[ordinal];
|
|
667
|
+
next.docFrequency[target] = this.docFrequency[ordinal];
|
|
668
|
+
next.deadDocuments[target] = this.deadDocuments[ordinal];
|
|
669
|
+
next.totalTermFrequency[target] = this.totalTermFrequency[ordinal];
|
|
670
|
+
next.revision[target] = this.revision[ordinal];
|
|
671
|
+
next.flags[target] = this.flags[ordinal];
|
|
672
|
+
}
|
|
673
|
+
this.ids = next.ids;
|
|
674
|
+
this.tokens = next.tokens;
|
|
675
|
+
this.sliceStart = next.sliceStart;
|
|
676
|
+
this.sliceClass = next.sliceClass;
|
|
677
|
+
this.lengths = next.lengths;
|
|
678
|
+
this.docFrequency = next.docFrequency;
|
|
679
|
+
this.deadDocuments = next.deadDocuments;
|
|
680
|
+
this.totalTermFrequency = next.totalTermFrequency;
|
|
681
|
+
this.revision = next.revision;
|
|
682
|
+
this.flags = next.flags;
|
|
683
|
+
this.buckets = next.buckets;
|
|
684
|
+
this.count = next.count;
|
|
685
|
+
this.liveCount = next.liveCount;
|
|
686
|
+
this.generation++;
|
|
687
|
+
return remap;
|
|
688
|
+
}
|
|
689
|
+
clear() {
|
|
690
|
+
this.ids = /* @__PURE__ */ new Map();
|
|
691
|
+
this.tokens = [];
|
|
692
|
+
this.buckets = /* @__PURE__ */ new Map();
|
|
693
|
+
this.count = 0;
|
|
694
|
+
this.liveCount = 0;
|
|
695
|
+
this.tokenCodeUnits = 0;
|
|
696
|
+
this.generation++;
|
|
697
|
+
this.sliceStart = new Uint32Array(this.startCapacity);
|
|
698
|
+
this.sliceClass = new Uint8Array(this.startCapacity);
|
|
699
|
+
this.lengths = new Uint32Array(this.startCapacity);
|
|
700
|
+
this.docFrequency = new Uint32Array(this.startCapacity);
|
|
701
|
+
this.deadDocuments = new Uint32Array(this.startCapacity);
|
|
702
|
+
this.totalTermFrequency = new Float64Array(this.startCapacity);
|
|
703
|
+
this.revision = new Uint32Array(this.startCapacity);
|
|
704
|
+
this.flags = new Uint8Array(this.startCapacity);
|
|
705
|
+
}
|
|
706
|
+
byteLength() {
|
|
707
|
+
let bucketBytes = 0;
|
|
708
|
+
for (const bucket of this.buckets.values()) bucketBytes += bucket.ordinals.byteLength;
|
|
709
|
+
return this.sliceStart.byteLength + this.sliceClass.byteLength + this.lengths.byteLength + this.docFrequency.byteLength + this.deadDocuments.byteLength + this.totalTermFrequency.byteLength + this.revision.byteLength + this.flags.byteLength + bucketBytes;
|
|
710
|
+
}
|
|
711
|
+
track(token, ordinal) {
|
|
459
712
|
if (token.length === 0) return;
|
|
460
|
-
const
|
|
461
|
-
let bucket =
|
|
462
|
-
if (
|
|
463
|
-
bucket =
|
|
464
|
-
|
|
713
|
+
const first = token[0];
|
|
714
|
+
let bucket = this.buckets.get(first);
|
|
715
|
+
if (bucket === void 0) {
|
|
716
|
+
bucket = { ordinals: new Uint32Array(4), length: 0, dead: 0 };
|
|
717
|
+
this.buckets.set(first, bucket);
|
|
718
|
+
}
|
|
719
|
+
if (bucket.length === bucket.ordinals.length) {
|
|
720
|
+
bucket.ordinals = grow(bucket.ordinals, new Uint32Array(bucket.ordinals.length * 2));
|
|
721
|
+
}
|
|
722
|
+
bucket.ordinals[bucket.length++] = ordinal;
|
|
723
|
+
}
|
|
724
|
+
compactBucket(first, bucket) {
|
|
725
|
+
if (bucket.dead * 2 <= bucket.length) return;
|
|
726
|
+
const { ordinals } = bucket;
|
|
727
|
+
let kept = 0;
|
|
728
|
+
for (let i = 0; i < bucket.length; i++) {
|
|
729
|
+
const ordinal = ordinals[i];
|
|
730
|
+
if ((this.flags[ordinal] & TERM_LIVE) !== 0) ordinals[kept++] = ordinal;
|
|
731
|
+
}
|
|
732
|
+
bucket.length = kept;
|
|
733
|
+
bucket.dead = 0;
|
|
734
|
+
if (kept === 0) this.buckets.delete(first);
|
|
735
|
+
}
|
|
736
|
+
growTo(capacity) {
|
|
737
|
+
this.sliceStart = grow(this.sliceStart, new Uint32Array(capacity));
|
|
738
|
+
this.sliceClass = grow(this.sliceClass, new Uint8Array(capacity));
|
|
739
|
+
this.lengths = grow(this.lengths, new Uint32Array(capacity));
|
|
740
|
+
this.docFrequency = grow(this.docFrequency, new Uint32Array(capacity));
|
|
741
|
+
this.deadDocuments = grow(this.deadDocuments, new Uint32Array(capacity));
|
|
742
|
+
this.totalTermFrequency = grow(this.totalTermFrequency, new Float64Array(capacity));
|
|
743
|
+
this.revision = grow(this.revision, new Uint32Array(capacity));
|
|
744
|
+
this.flags = grow(this.flags, new Uint8Array(capacity));
|
|
745
|
+
}
|
|
746
|
+
};
|
|
747
|
+
|
|
748
|
+
// src/core/inverted-index/view.ts
|
|
749
|
+
var NO_TOMBSTONES = { size: 0, has: () => false };
|
|
750
|
+
var NO_DOCUMENT_IDS = new Uint32Array(0);
|
|
751
|
+
var NO_FREQUENCIES = new Uint16Array(0);
|
|
752
|
+
var DETACHED = -1;
|
|
753
|
+
var LivePostingView = class {
|
|
754
|
+
constructor(store, token, ordinal, keepsPositions) {
|
|
755
|
+
this.store = store;
|
|
756
|
+
this.token = token;
|
|
757
|
+
this.keepsPositions = keepsPositions;
|
|
758
|
+
this.ordinal = ordinal;
|
|
759
|
+
this.generation = store.terms.generation;
|
|
760
|
+
const view = this;
|
|
761
|
+
this.docIdSet = {
|
|
762
|
+
get size() {
|
|
763
|
+
const live = view.sync();
|
|
764
|
+
return live === DETACHED ? 0 : view.store.terms.docFrequency[live];
|
|
765
|
+
}
|
|
766
|
+
};
|
|
767
|
+
this.retiredRows = {
|
|
768
|
+
get size() {
|
|
769
|
+
const live = view.sync();
|
|
770
|
+
return live === DETACHED ? 0 : view.store.terms.deadDocuments[live];
|
|
771
|
+
},
|
|
772
|
+
has: (internalId) => store.isRetired(internalId)
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
docIdSet;
|
|
776
|
+
retiredRows;
|
|
777
|
+
ordinal;
|
|
778
|
+
generation;
|
|
779
|
+
syncedEpoch = -1;
|
|
780
|
+
syncedStart = -1;
|
|
781
|
+
syncedLength = 0;
|
|
782
|
+
syncedRevision = -1;
|
|
783
|
+
ids = NO_DOCUMENT_IDS;
|
|
784
|
+
frequencies = NO_FREQUENCIES;
|
|
785
|
+
fields = NO_FREQUENCIES;
|
|
786
|
+
materialized = null;
|
|
787
|
+
get length() {
|
|
788
|
+
this.sync();
|
|
789
|
+
return this.syncedLength;
|
|
790
|
+
}
|
|
791
|
+
get docIds() {
|
|
792
|
+
this.sync();
|
|
793
|
+
return this.ids;
|
|
794
|
+
}
|
|
795
|
+
get termFrequencies() {
|
|
796
|
+
this.sync();
|
|
797
|
+
return this.frequencies;
|
|
798
|
+
}
|
|
799
|
+
get fieldNameIndices() {
|
|
800
|
+
this.sync();
|
|
801
|
+
return this.fields;
|
|
802
|
+
}
|
|
803
|
+
get holdsPositions() {
|
|
804
|
+
const live = this.sync();
|
|
805
|
+
return live !== DETACHED && (this.store.terms.flags[live] & TERM_POSITIONS) !== 0;
|
|
806
|
+
}
|
|
807
|
+
get positions() {
|
|
808
|
+
if (!this.holdsPositions) return null;
|
|
809
|
+
if (this.materialized !== null) return this.materialized;
|
|
810
|
+
const starts = this.store.pool.positionStarts;
|
|
811
|
+
const rows = new Array(this.syncedLength);
|
|
812
|
+
for (let row = 0; row < this.syncedLength; row++) {
|
|
813
|
+
rows[row] = starts === null ? [] : this.store.positions.read(starts[this.syncedStart + row]);
|
|
814
|
+
}
|
|
815
|
+
if (this.keepsPositions) this.materialized = rows;
|
|
816
|
+
return rows;
|
|
817
|
+
}
|
|
818
|
+
positionCountAt(row) {
|
|
819
|
+
if (!this.holdsPositions) return 0;
|
|
820
|
+
const starts = this.store.pool.positionStarts;
|
|
821
|
+
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
822
|
+
}
|
|
823
|
+
copyPositionsAt(row, target, offset) {
|
|
824
|
+
if (!this.holdsPositions) return 0;
|
|
825
|
+
const starts = this.store.pool.positionStarts;
|
|
826
|
+
return starts === null ? 0 : this.store.positions.copyInto(starts[this.syncedStart + row], target, offset);
|
|
827
|
+
}
|
|
828
|
+
get deletedDocs() {
|
|
829
|
+
const live = this.sync();
|
|
830
|
+
if (live === DETACHED || this.store.terms.deadDocuments[live] === 0) return NO_TOMBSTONES;
|
|
831
|
+
return this.retiredRows;
|
|
832
|
+
}
|
|
833
|
+
get totalTermFrequency() {
|
|
834
|
+
const live = this.sync();
|
|
835
|
+
return live === DETACHED ? 0 : this.store.terms.totalTermFrequency[live];
|
|
836
|
+
}
|
|
837
|
+
get structureRevision() {
|
|
838
|
+
const live = this.sync();
|
|
839
|
+
return live === DETACHED ? this.syncedRevision : this.store.terms.revision[live];
|
|
840
|
+
}
|
|
841
|
+
get ordered() {
|
|
842
|
+
const live = this.sync();
|
|
843
|
+
return live === DETACHED || (this.store.terms.flags[live] & TERM_ORDERED) !== 0;
|
|
844
|
+
}
|
|
845
|
+
sync() {
|
|
846
|
+
const terms = this.store.terms;
|
|
847
|
+
if (this.generation !== terms.generation) {
|
|
848
|
+
this.ordinal = terms.ids.get(this.token) ?? DETACHED;
|
|
849
|
+
this.generation = terms.generation;
|
|
850
|
+
}
|
|
851
|
+
const ordinal = this.ordinal;
|
|
852
|
+
if (ordinal === DETACHED || !terms.isLive(ordinal)) {
|
|
853
|
+
this.detach();
|
|
854
|
+
return DETACHED;
|
|
855
|
+
}
|
|
856
|
+
const pool = this.store.pool;
|
|
857
|
+
const start = terms.sliceStart[ordinal];
|
|
858
|
+
const length = terms.lengths[ordinal];
|
|
859
|
+
const revision = terms.revision[ordinal];
|
|
860
|
+
if (pool.epoch !== this.syncedEpoch || start !== this.syncedStart || length !== this.syncedLength || revision !== this.syncedRevision) {
|
|
861
|
+
this.ids = pool.docIds.subarray(start, start + length);
|
|
862
|
+
this.frequencies = pool.frequencies.subarray(start, start + length);
|
|
863
|
+
this.fields = pool.fieldIndices.subarray(start, start + length);
|
|
864
|
+
this.materialized = null;
|
|
865
|
+
this.syncedEpoch = pool.epoch;
|
|
866
|
+
this.syncedStart = start;
|
|
867
|
+
this.syncedLength = length;
|
|
868
|
+
this.syncedRevision = revision;
|
|
465
869
|
}
|
|
466
|
-
|
|
870
|
+
return ordinal;
|
|
467
871
|
}
|
|
468
|
-
|
|
469
|
-
if (
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
872
|
+
detach() {
|
|
873
|
+
if (this.syncedStart === DETACHED && this.syncedLength === 0) return;
|
|
874
|
+
this.ids = NO_DOCUMENT_IDS;
|
|
875
|
+
this.frequencies = NO_FREQUENCIES;
|
|
876
|
+
this.fields = NO_FREQUENCIES;
|
|
877
|
+
this.materialized = null;
|
|
878
|
+
this.syncedStart = DETACHED;
|
|
879
|
+
this.syncedLength = 0;
|
|
880
|
+
this.syncedEpoch = -1;
|
|
881
|
+
}
|
|
882
|
+
};
|
|
883
|
+
|
|
884
|
+
// src/core/inverted-index/postings.ts
|
|
885
|
+
function holdsDocument(docIds, start, length, internalId, ordered) {
|
|
886
|
+
if (!ordered) {
|
|
887
|
+
for (let i = start; i < start + length; i++) {
|
|
888
|
+
if (docIds[i] === internalId) return true;
|
|
475
889
|
}
|
|
890
|
+
return false;
|
|
476
891
|
}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
if (
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
892
|
+
let low = start;
|
|
893
|
+
let high = start + length;
|
|
894
|
+
while (low < high) {
|
|
895
|
+
const middle = low + high >>> 1;
|
|
896
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
897
|
+
else high = middle;
|
|
898
|
+
}
|
|
899
|
+
return low < start + length && docIds[low] === internalId;
|
|
900
|
+
}
|
|
901
|
+
var LivePostings = class {
|
|
902
|
+
terms = new TermColumns(TERM_TABLE_START_CAPACITY);
|
|
903
|
+
pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
904
|
+
positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
905
|
+
retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
|
|
906
|
+
batchMode = false;
|
|
907
|
+
dirty = /* @__PURE__ */ new Set();
|
|
908
|
+
cachedViews = /* @__PURE__ */ new Map();
|
|
909
|
+
compactedSlack = 0;
|
|
910
|
+
view(ordinal, cached) {
|
|
911
|
+
if (!cached || this.terms.lengths[ordinal] < CACHED_VIEW_MIN_ENTRIES) {
|
|
912
|
+
return new LivePostingView(this, this.terms.tokens[ordinal], ordinal, cached);
|
|
913
|
+
}
|
|
914
|
+
let view = this.cachedViews.get(ordinal);
|
|
915
|
+
if (view === void 0) {
|
|
916
|
+
view = new LivePostingView(this, this.terms.tokens[ordinal], ordinal, true);
|
|
917
|
+
this.cachedViews.set(ordinal, view);
|
|
918
|
+
}
|
|
919
|
+
return view;
|
|
920
|
+
}
|
|
921
|
+
reserve(token, entries) {
|
|
922
|
+
const terms = this.terms;
|
|
923
|
+
const ordinal = terms.ids.get(token) ?? terms.add(token);
|
|
924
|
+
if (sliceCapacity(terms.sliceClass[ordinal]) < terms.lengths[ordinal] + entries) {
|
|
925
|
+
this.relocate(ordinal, terms.lengths[ordinal] + entries);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
insert(token, internalId, termFrequency, fieldIndex, positions) {
|
|
929
|
+
const terms = this.terms;
|
|
930
|
+
if (this.isRetired(internalId)) this.purgeRetired();
|
|
931
|
+
const ordinal = terms.ids.get(token) ?? terms.add(token);
|
|
932
|
+
const length = terms.lengths[ordinal];
|
|
933
|
+
if (length >= sliceCapacity(terms.sliceClass[ordinal])) this.relocate(ordinal, length + 1);
|
|
934
|
+
const pool = this.pool;
|
|
935
|
+
const start = terms.sliceStart[ordinal];
|
|
936
|
+
const at = start + length;
|
|
937
|
+
const flags = terms.flags[ordinal];
|
|
938
|
+
let newDocument = true;
|
|
939
|
+
if (length > 0) {
|
|
940
|
+
const last = pool.docIds[at - 1];
|
|
941
|
+
const ordered = (flags & TERM_ORDERED) !== 0;
|
|
942
|
+
if (internalId === last) {
|
|
943
|
+
newDocument = false;
|
|
944
|
+
} else if (!ordered || internalId < last) {
|
|
945
|
+
newDocument = !holdsDocument(pool.docIds, start, length, internalId, ordered);
|
|
946
|
+
if (internalId < last) terms.flags[ordinal] = flags & ~TERM_ORDERED;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
const stored = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
|
|
950
|
+
pool.docIds[at] = internalId;
|
|
951
|
+
pool.frequencies[at] = stored;
|
|
952
|
+
pool.fieldIndices[at] = fieldIndex;
|
|
953
|
+
terms.totalTermFrequency[ordinal] += stored;
|
|
954
|
+
if (positions !== null) {
|
|
955
|
+
const starts = pool.trackPositions();
|
|
956
|
+
if ((flags & TERM_POSITIONS) === 0) {
|
|
957
|
+
terms.flags[ordinal] |= TERM_POSITIONS;
|
|
958
|
+
starts.fill(EMPTY_POSITIONS, start, at);
|
|
959
|
+
}
|
|
960
|
+
starts[at] = this.positions.append(positions);
|
|
961
|
+
} else if ((flags & TERM_POSITIONS) !== 0) {
|
|
962
|
+
pool.trackPositions()[at] = EMPTY_POSITIONS;
|
|
963
|
+
}
|
|
964
|
+
if (newDocument) terms.docFrequency[ordinal]++;
|
|
965
|
+
terms.lengths[ordinal] = length + 1;
|
|
966
|
+
this.repackIfSparse();
|
|
967
|
+
}
|
|
968
|
+
remove(token, internalId, occurrences) {
|
|
969
|
+
const terms = this.terms;
|
|
970
|
+
const ordinal = terms.ids.get(token);
|
|
971
|
+
if (ordinal === void 0) return;
|
|
972
|
+
const length = terms.lengths[ordinal];
|
|
973
|
+
const ordered = (terms.flags[ordinal] & TERM_ORDERED) !== 0;
|
|
974
|
+
if (!holdsDocument(this.pool.docIds, terms.sliceStart[ordinal], length, internalId, ordered)) return;
|
|
975
|
+
terms.revision[ordinal]++;
|
|
976
|
+
terms.docFrequency[ordinal]--;
|
|
977
|
+
terms.deadDocuments[ordinal]++;
|
|
978
|
+
this.retire(internalId);
|
|
979
|
+
terms.totalTermFrequency[ordinal] = Math.max(0, terms.totalTermFrequency[ordinal] - occurrences);
|
|
980
|
+
if (terms.docFrequency[ordinal] === 0) {
|
|
981
|
+
this.dropTerm(ordinal);
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
if (this.batchMode) {
|
|
985
|
+
this.dirty.add(ordinal);
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
if (terms.deadDocuments[ordinal] / length > POSTING_LIST_COMPACTION_THRESHOLD) this.compact(ordinal);
|
|
989
|
+
this.repackIfSparse();
|
|
990
|
+
}
|
|
991
|
+
isRetired(internalId) {
|
|
992
|
+
return internalId < this.retired.length && this.retired[internalId] === 1;
|
|
993
|
+
}
|
|
994
|
+
beginBatch() {
|
|
995
|
+
this.batchMode = true;
|
|
996
|
+
this.dirty.clear();
|
|
997
|
+
}
|
|
998
|
+
endBatch() {
|
|
999
|
+
this.batchMode = false;
|
|
1000
|
+
const terms = this.terms;
|
|
1001
|
+
for (const ordinal of this.dirty) {
|
|
1002
|
+
if (!terms.isLive(ordinal)) continue;
|
|
1003
|
+
if (terms.deadDocuments[ordinal] / terms.lengths[ordinal] > POSTING_LIST_COMPACTION_THRESHOLD) {
|
|
1004
|
+
this.compact(ordinal);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
this.dirty.clear();
|
|
1008
|
+
this.repackIfSparse();
|
|
1009
|
+
}
|
|
1010
|
+
clear() {
|
|
1011
|
+
const epoch = this.pool.epoch + 1;
|
|
1012
|
+
this.terms.clear();
|
|
1013
|
+
this.pool = new PostingPool(POSTING_POOL_START_ENTRIES);
|
|
1014
|
+
this.pool.epoch = epoch;
|
|
1015
|
+
this.positions = new PositionPool(POSITION_POOL_START_VALUES);
|
|
1016
|
+
this.retired = new Uint8Array(RETIRED_DOCUMENTS_START_CAPACITY);
|
|
1017
|
+
this.dirty.clear();
|
|
1018
|
+
this.cachedViews.clear();
|
|
1019
|
+
this.compactedSlack = 0;
|
|
1020
|
+
}
|
|
1021
|
+
byteLength() {
|
|
1022
|
+
return this.terms.byteLength() + this.pool.byteLength() + this.positions.values.byteLength + this.retired.byteLength;
|
|
1023
|
+
}
|
|
1024
|
+
retire(internalId) {
|
|
1025
|
+
if (internalId >= this.retired.length) {
|
|
1026
|
+
const grown = new Uint8Array(Math.max(internalId + 1, this.retired.length * 2));
|
|
1027
|
+
grown.set(this.retired);
|
|
1028
|
+
this.retired = grown;
|
|
1029
|
+
}
|
|
1030
|
+
this.retired[internalId] = 1;
|
|
1031
|
+
}
|
|
1032
|
+
purgeRetired() {
|
|
1033
|
+
const terms = this.terms;
|
|
1034
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
1035
|
+
if (terms.isLive(ordinal) && terms.deadDocuments[ordinal] > 0) this.compact(ordinal);
|
|
1036
|
+
}
|
|
1037
|
+
this.retired.fill(0);
|
|
1038
|
+
}
|
|
1039
|
+
compact(ordinal) {
|
|
1040
|
+
if (this.terms.deadDocuments[ordinal] === 0) return;
|
|
1041
|
+
const retired = this.retired;
|
|
1042
|
+
this.filterEntries(ordinal, (id) => id < retired.length && retired[id] === 1);
|
|
1043
|
+
this.terms.deadDocuments[ordinal] = 0;
|
|
1044
|
+
}
|
|
1045
|
+
filterEntries(ordinal, drop) {
|
|
1046
|
+
const terms = this.terms;
|
|
1047
|
+
const pool = this.pool;
|
|
1048
|
+
const start = terms.sliceStart[ordinal];
|
|
1049
|
+
const length = terms.lengths[ordinal];
|
|
1050
|
+
const starts = (terms.flags[ordinal] & TERM_POSITIONS) !== 0 ? pool.positionStarts : null;
|
|
1051
|
+
const { docIds, frequencies, fieldIndices } = pool;
|
|
1052
|
+
terms.revision[ordinal]++;
|
|
1053
|
+
let kept = 0;
|
|
1054
|
+
let ascending = true;
|
|
1055
|
+
for (let i = start; i < start + length; i++) {
|
|
1056
|
+
const internalId = docIds[i];
|
|
1057
|
+
if (drop(internalId)) {
|
|
1058
|
+
if (starts !== null) this.positions.release(starts[i]);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
const target = start + kept;
|
|
1062
|
+
if (kept > 0 && internalId < docIds[target - 1]) ascending = false;
|
|
1063
|
+
if (target !== i) {
|
|
1064
|
+
docIds[target] = internalId;
|
|
1065
|
+
frequencies[target] = frequencies[i];
|
|
1066
|
+
fieldIndices[target] = fieldIndices[i];
|
|
1067
|
+
if (starts !== null) starts[target] = starts[i];
|
|
1068
|
+
}
|
|
1069
|
+
kept++;
|
|
1070
|
+
}
|
|
1071
|
+
terms.lengths[ordinal] = kept;
|
|
1072
|
+
terms.flags[ordinal] = ascending ? terms.flags[ordinal] | TERM_ORDERED : terms.flags[ordinal] & ~TERM_ORDERED;
|
|
1073
|
+
this.compactedSlack += length - kept;
|
|
1074
|
+
}
|
|
1075
|
+
relocate(ordinal, needed) {
|
|
1076
|
+
const terms = this.terms;
|
|
1077
|
+
const previousClass = terms.sliceClass[ordinal];
|
|
1078
|
+
const previousStart = terms.sliceStart[ordinal];
|
|
1079
|
+
const length = terms.lengths[ordinal];
|
|
1080
|
+
const grownClass = previousClass === NO_SLICE ? 0 : previousClass + 1;
|
|
1081
|
+
const nextClass = Math.max(grownClass, sliceClassFor(needed));
|
|
1082
|
+
const nextStart = this.pool.allocate(nextClass);
|
|
1083
|
+
if (length > 0) this.pool.move(previousStart, nextStart, length);
|
|
1084
|
+
this.pool.release(previousStart, previousClass);
|
|
1085
|
+
terms.sliceStart[ordinal] = nextStart;
|
|
1086
|
+
terms.sliceClass[ordinal] = nextClass;
|
|
1087
|
+
}
|
|
1088
|
+
dropTerm(ordinal) {
|
|
1089
|
+
const terms = this.terms;
|
|
1090
|
+
const pool = this.pool;
|
|
1091
|
+
if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && pool.positionStarts !== null) {
|
|
1092
|
+
const start = terms.sliceStart[ordinal];
|
|
1093
|
+
for (let i = start; i < start + terms.lengths[ordinal]; i++) this.positions.release(pool.positionStarts[i]);
|
|
1094
|
+
}
|
|
1095
|
+
pool.release(terms.sliceStart[ordinal], terms.sliceClass[ordinal]);
|
|
1096
|
+
this.dirty.delete(ordinal);
|
|
1097
|
+
this.cachedViews.delete(ordinal);
|
|
1098
|
+
terms.kill(ordinal);
|
|
1099
|
+
if (terms.deadCount >= TERM_VACUUM_MIN_DEAD_TERMS && terms.deadCount > terms.liveCount) this.renumberTerms();
|
|
1100
|
+
}
|
|
1101
|
+
renumberTerms() {
|
|
1102
|
+
const remap = this.terms.renumber();
|
|
1103
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
1104
|
+
for (const ordinal of this.dirty) {
|
|
1105
|
+
if (remap[ordinal] >= 0) dirty.add(remap[ordinal]);
|
|
1106
|
+
}
|
|
1107
|
+
this.dirty = dirty;
|
|
1108
|
+
this.cachedViews.clear();
|
|
1109
|
+
}
|
|
1110
|
+
repackIfSparse() {
|
|
1111
|
+
const pool = this.pool;
|
|
1112
|
+
const freeEntries = pool.freeEntries + this.compactedSlack;
|
|
1113
|
+
const sparsePostings = freeEntries >= POOL_VACUUM_MIN_FREE_ENTRIES && freeEntries > pool.end * POOL_VACUUM_FREE_FRACTION;
|
|
1114
|
+
const positions = this.positions;
|
|
1115
|
+
const sparsePositions = positions.freeValues >= POSITION_VACUUM_MIN_FREE_VALUES && positions.freeValues > positions.end * POOL_VACUUM_FREE_FRACTION;
|
|
1116
|
+
if (sparsePostings || sparsePositions) this.repack(sparsePositions);
|
|
1117
|
+
}
|
|
1118
|
+
repack(rebuildPositions) {
|
|
1119
|
+
const terms = this.terms;
|
|
1120
|
+
const pool = this.pool;
|
|
1121
|
+
const positions = this.positions;
|
|
1122
|
+
let capacity = 0;
|
|
1123
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
1124
|
+
if (terms.isLive(ordinal)) capacity += sliceCapacity(sliceClassFor(terms.lengths[ordinal]));
|
|
1125
|
+
}
|
|
1126
|
+
const nextPool = new PostingPool(Math.max(1, capacity));
|
|
1127
|
+
nextPool.epoch = pool.epoch + 1;
|
|
1128
|
+
const sourceStarts = pool.positionStarts;
|
|
1129
|
+
const targetStarts = sourceStarts === null ? null : nextPool.trackPositions();
|
|
1130
|
+
const nextPositions = rebuildPositions ? new PositionPool(positions.end - positions.freeValues) : positions;
|
|
1131
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
1132
|
+
if (!terms.isLive(ordinal)) continue;
|
|
1133
|
+
const start = terms.sliceStart[ordinal];
|
|
1134
|
+
const length = terms.lengths[ordinal];
|
|
1135
|
+
const sliceClass = sliceClassFor(length);
|
|
1136
|
+
const target = nextPool.allocate(sliceClass);
|
|
1137
|
+
nextPool.docIds.set(pool.docIds.subarray(start, start + length), target);
|
|
1138
|
+
nextPool.frequencies.set(pool.frequencies.subarray(start, start + length), target);
|
|
1139
|
+
nextPool.fieldIndices.set(pool.fieldIndices.subarray(start, start + length), target);
|
|
1140
|
+
if ((terms.flags[ordinal] & TERM_POSITIONS) !== 0 && sourceStarts !== null && targetStarts !== null) {
|
|
1141
|
+
if (rebuildPositions) {
|
|
1142
|
+
for (let i = 0; i < length; i++)
|
|
1143
|
+
targetStarts[target + i] = nextPositions.appendFrom(positions, sourceStarts[start + i]);
|
|
1144
|
+
} else {
|
|
1145
|
+
targetStarts.set(sourceStarts.subarray(start, start + length), target);
|
|
1146
|
+
}
|
|
486
1147
|
}
|
|
1148
|
+
terms.sliceStart[ordinal] = target;
|
|
1149
|
+
terms.sliceClass[ordinal] = sliceClass;
|
|
487
1150
|
}
|
|
488
|
-
|
|
1151
|
+
this.pool = nextPool;
|
|
1152
|
+
this.positions = nextPositions;
|
|
1153
|
+
this.compactedSlack = 0;
|
|
1154
|
+
}
|
|
1155
|
+
};
|
|
1156
|
+
|
|
1157
|
+
// src/core/inverted-index/index.ts
|
|
1158
|
+
function fieldIndexFor(fieldNameTable, fieldName) {
|
|
1159
|
+
let index = fieldNameTable.indexMap.get(fieldName);
|
|
1160
|
+
if (index === void 0) {
|
|
1161
|
+
index = fieldNameTable.names.length;
|
|
1162
|
+
fieldNameTable.names.push(fieldName);
|
|
1163
|
+
fieldNameTable.indexMap.set(fieldName, index);
|
|
489
1164
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
1165
|
+
return index;
|
|
1166
|
+
}
|
|
1167
|
+
function createInvertedIndex(fieldNameTable) {
|
|
1168
|
+
const postings = new LivePostings();
|
|
1169
|
+
function* liveTokens() {
|
|
1170
|
+
const terms = postings.terms;
|
|
1171
|
+
const generation = terms.generation;
|
|
1172
|
+
for (let ordinal = 0; ordinal < terms.count && terms.generation === generation; ordinal++) {
|
|
1173
|
+
if (terms.isLive(ordinal)) yield terms.tokens[ordinal];
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
function serializeTerm(ordinal, resolver) {
|
|
1177
|
+
const list = postings.view(ordinal, false);
|
|
1178
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
|
|
1179
|
+
const positions = list.positions;
|
|
1180
|
+
const entries = [];
|
|
1181
|
+
for (let i = 0; i < length; i++) {
|
|
1182
|
+
if (deletedDocs.size > 0 && deletedDocs.has(docIds[i])) continue;
|
|
1183
|
+
const externalId = resolver.toExternal(docIds[i]);
|
|
1184
|
+
if (externalId === void 0) continue;
|
|
1185
|
+
entries.push({
|
|
1186
|
+
docId: externalId,
|
|
1187
|
+
termFrequency: termFrequencies[i],
|
|
1188
|
+
fieldName: fieldNameTable.names[fieldNameIndices[i]],
|
|
1189
|
+
positions: positions === null ? [] : [...positions[i]]
|
|
1190
|
+
});
|
|
496
1191
|
}
|
|
497
|
-
return
|
|
1192
|
+
return entries;
|
|
498
1193
|
}
|
|
499
1194
|
return {
|
|
500
1195
|
insert(token, internalId, termFrequency, fieldNameIndex2, positions) {
|
|
501
|
-
|
|
502
|
-
if (list.deletedDocs.size > 0 && list.deletedDocs.has(internalId)) {
|
|
503
|
-
compactDocEntries(list, internalId);
|
|
504
|
-
list.deletedDocs.delete(internalId);
|
|
505
|
-
}
|
|
506
|
-
if (list.length >= list.termFrequencies.length) {
|
|
507
|
-
growTypedArrays(list);
|
|
508
|
-
}
|
|
509
|
-
const idx = list.length;
|
|
510
|
-
if (idx > 0 && internalId < list.docIds[idx - 1]) list.ordered = false;
|
|
511
|
-
list.docIds.push(internalId);
|
|
512
|
-
list.termFrequencies[idx] = termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : termFrequency;
|
|
513
|
-
list.fieldNameIndices[idx] = fieldNameIndex2;
|
|
514
|
-
list.totalTermFrequency += list.termFrequencies[idx];
|
|
515
|
-
if (positions !== null) {
|
|
516
|
-
if (!list.positions) {
|
|
517
|
-
list.positions = [];
|
|
518
|
-
for (let j = 0; j < idx; j++) list.positions.push([]);
|
|
519
|
-
}
|
|
520
|
-
list.positions.push(positions);
|
|
521
|
-
} else if (list.positions !== null) {
|
|
522
|
-
list.positions.push([]);
|
|
523
|
-
}
|
|
524
|
-
if (!list.docIdSet.has(internalId)) {
|
|
525
|
-
list.docIdSet.add(internalId);
|
|
526
|
-
}
|
|
527
|
-
list.length++;
|
|
1196
|
+
postings.insert(token, internalId, termFrequency, fieldNameIndex2, positions);
|
|
528
1197
|
},
|
|
529
1198
|
remove(token, internalId, occurrences) {
|
|
530
|
-
|
|
531
|
-
if (!list || !list.docIdSet.has(internalId)) return;
|
|
532
|
-
list.structureRevision++;
|
|
533
|
-
list.docIdSet.delete(internalId);
|
|
534
|
-
list.deletedDocs.add(internalId);
|
|
535
|
-
list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
|
|
536
|
-
if (list.docIdSet.size === 0) {
|
|
537
|
-
index.delete(token);
|
|
538
|
-
untrackToken(token);
|
|
539
|
-
if (batchMode) batchDirtyTokens.delete(token);
|
|
540
|
-
return;
|
|
541
|
-
}
|
|
542
|
-
if (batchMode) {
|
|
543
|
-
batchDirtyTokens.add(token);
|
|
544
|
-
return;
|
|
545
|
-
}
|
|
546
|
-
if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
|
|
547
|
-
compactList(list);
|
|
548
|
-
}
|
|
1199
|
+
postings.remove(token, internalId, occurrences);
|
|
549
1200
|
},
|
|
550
1201
|
beginBatch() {
|
|
551
|
-
|
|
552
|
-
batchDirtyTokens.clear();
|
|
1202
|
+
postings.beginBatch();
|
|
553
1203
|
},
|
|
554
1204
|
endBatch() {
|
|
555
|
-
|
|
556
|
-
for (const token of batchDirtyTokens) {
|
|
557
|
-
const list = index.get(token);
|
|
558
|
-
if (!list) continue;
|
|
559
|
-
if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
|
|
560
|
-
compactList(list);
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
batchDirtyTokens.clear();
|
|
1205
|
+
postings.endBatch();
|
|
564
1206
|
},
|
|
565
1207
|
lookup(token) {
|
|
566
|
-
|
|
1208
|
+
const ordinal = postings.terms.ids.get(token);
|
|
1209
|
+
return ordinal === void 0 ? void 0 : postings.view(ordinal, true);
|
|
1210
|
+
},
|
|
1211
|
+
lookupReadOnce(token) {
|
|
1212
|
+
const ordinal = postings.terms.ids.get(token);
|
|
1213
|
+
return ordinal === void 0 ? void 0 : postings.view(ordinal, false);
|
|
567
1214
|
},
|
|
568
1215
|
documentFrequency(token) {
|
|
569
|
-
|
|
1216
|
+
const ordinal = postings.terms.ids.get(token);
|
|
1217
|
+
return ordinal === void 0 ? 0 : postings.terms.docFrequency[ordinal];
|
|
570
1218
|
},
|
|
571
1219
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
1220
|
+
const terms = postings.terms;
|
|
572
1221
|
if (tolerance === 0) {
|
|
573
|
-
const exact =
|
|
574
|
-
return exact ? [{ token, postingList: exact }]
|
|
1222
|
+
const exact = terms.ids.get(token);
|
|
1223
|
+
return exact === void 0 ? [] : [{ token, postingList: postings.view(exact, true) }];
|
|
575
1224
|
}
|
|
576
1225
|
const results = [];
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
const postingList = index.get(candidate);
|
|
582
|
-
if (postingList) results.push({ token: candidate, postingList });
|
|
1226
|
+
terms.forEachStartingWith(fuzzyPrefixOf(token, prefixLength), (ordinal) => {
|
|
1227
|
+
const candidate = terms.tokens[ordinal];
|
|
1228
|
+
if (boundedLevenshtein(token, candidate, tolerance).withinTolerance) {
|
|
1229
|
+
results.push({ token: candidate, postingList: postings.view(ordinal, true) });
|
|
583
1230
|
}
|
|
584
|
-
}
|
|
1231
|
+
});
|
|
585
1232
|
return results;
|
|
586
1233
|
},
|
|
587
1234
|
prefixSearch(prefix, limit) {
|
|
588
1235
|
if (prefix.length === 0 || limit <= 0) return [];
|
|
589
|
-
const
|
|
1236
|
+
const terms = postings.terms;
|
|
590
1237
|
const results = [];
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
if (!list) continue;
|
|
595
|
-
results.push({ term, documentFrequency: list.docIdSet.size });
|
|
596
|
-
}
|
|
1238
|
+
terms.forEachStartingWith(prefix, (ordinal) => {
|
|
1239
|
+
results.push({ term: terms.tokens[ordinal], documentFrequency: terms.docFrequency[ordinal] });
|
|
1240
|
+
});
|
|
597
1241
|
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
|
|
598
1242
|
if (results.length > limit) results.length = limit;
|
|
599
1243
|
return results;
|
|
600
1244
|
},
|
|
601
1245
|
has(token) {
|
|
602
|
-
return
|
|
1246
|
+
return postings.terms.ids.has(token);
|
|
603
1247
|
},
|
|
604
1248
|
tokens() {
|
|
605
|
-
return
|
|
1249
|
+
return liveTokens();
|
|
606
1250
|
},
|
|
607
1251
|
size() {
|
|
608
|
-
return
|
|
1252
|
+
return postings.terms.liveCount;
|
|
609
1253
|
},
|
|
610
1254
|
clear() {
|
|
611
|
-
|
|
612
|
-
|
|
1255
|
+
postings.clear();
|
|
1256
|
+
},
|
|
1257
|
+
estimateBytes() {
|
|
1258
|
+
const terms = postings.terms;
|
|
1259
|
+
const perTerm = TERM_ENTRY_ESTIMATED_BYTES + TOKEN_STRING_HEADER_BYTES;
|
|
1260
|
+
return postings.byteLength() + terms.liveCount * perTerm + terms.tokenCodeUnits * 2;
|
|
613
1261
|
},
|
|
614
1262
|
serialize(resolver) {
|
|
615
1263
|
const result = /* @__PURE__ */ Object.create(null);
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
postings.push({
|
|
623
|
-
docId: externalId,
|
|
624
|
-
termFrequency: list.termFrequencies[i],
|
|
625
|
-
fieldName: fieldNameTable.names[list.fieldNameIndices[i]],
|
|
626
|
-
positions: list.positions ? list.positions[i] : []
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
if (postings.length > 0) {
|
|
630
|
-
result[token] = { docFrequency: list.docIdSet.size, postings };
|
|
1264
|
+
const terms = postings.terms;
|
|
1265
|
+
for (let ordinal = 0; ordinal < terms.count; ordinal++) {
|
|
1266
|
+
if (!terms.isLive(ordinal)) continue;
|
|
1267
|
+
const entries = serializeTerm(ordinal, resolver);
|
|
1268
|
+
if (entries.length > 0) {
|
|
1269
|
+
result[terms.tokens[ordinal]] = { docFrequency: terms.docFrequency[ordinal], postings: entries };
|
|
631
1270
|
}
|
|
632
1271
|
}
|
|
633
1272
|
return result;
|
|
634
1273
|
},
|
|
635
1274
|
deserialize(data, resolver) {
|
|
636
|
-
|
|
637
|
-
charBuckets.clear();
|
|
1275
|
+
postings.clear();
|
|
638
1276
|
for (const token of Object.keys(data)) {
|
|
639
|
-
const
|
|
640
|
-
const count = src.postings.length;
|
|
641
|
-
const docIdSet = /* @__PURE__ */ new Set();
|
|
642
|
-
const docIds = new Array(count);
|
|
643
|
-
const termFrequencies = new Uint16Array(count);
|
|
644
|
-
const fieldNameIndices = new Uint16Array(count);
|
|
1277
|
+
const source = data[token].postings;
|
|
645
1278
|
let hasPositions = false;
|
|
646
|
-
let
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
if (internalId === void 0) continue;
|
|
652
|
-
docIds[validCount] = internalId;
|
|
653
|
-
termFrequencies[validCount] = p.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : p.termFrequency;
|
|
654
|
-
totalTermFrequency += termFrequencies[validCount];
|
|
655
|
-
let fnIndex = fieldNameTable.indexMap.get(p.fieldName);
|
|
656
|
-
if (fnIndex === void 0) {
|
|
657
|
-
fnIndex = fieldNameTable.names.length;
|
|
658
|
-
fieldNameTable.names.push(p.fieldName);
|
|
659
|
-
fieldNameTable.indexMap.set(p.fieldName, fnIndex);
|
|
660
|
-
}
|
|
661
|
-
fieldNameIndices[validCount] = fnIndex;
|
|
662
|
-
docIdSet.add(internalId);
|
|
663
|
-
if (p.positions.length > 0) hasPositions = true;
|
|
664
|
-
validCount++;
|
|
665
|
-
}
|
|
666
|
-
if (validCount === 0) continue;
|
|
667
|
-
docIds.length = validCount;
|
|
668
|
-
const finalTF = validCount < count ? termFrequencies.slice(0, validCount) : termFrequencies;
|
|
669
|
-
const finalFNI = validCount < count ? fieldNameIndices.slice(0, validCount) : fieldNameIndices;
|
|
670
|
-
let positions = null;
|
|
671
|
-
if (hasPositions) {
|
|
672
|
-
positions = new Array(validCount);
|
|
673
|
-
let vi = 0;
|
|
674
|
-
for (let i = 0; i < count; i++) {
|
|
675
|
-
const internalId = resolver.toInternal(src.postings[i].docId);
|
|
676
|
-
if (internalId === void 0) continue;
|
|
677
|
-
positions[vi] = src.postings[i].positions;
|
|
678
|
-
vi++;
|
|
679
|
-
}
|
|
1279
|
+
let valid = 0;
|
|
1280
|
+
for (const posting of source) {
|
|
1281
|
+
if (resolver.toInternal(posting.docId) === void 0) continue;
|
|
1282
|
+
valid++;
|
|
1283
|
+
if (posting.positions.length > 0) hasPositions = true;
|
|
680
1284
|
}
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
1285
|
+
if (valid === 0) continue;
|
|
1286
|
+
postings.reserve(token, valid);
|
|
1287
|
+
for (const posting of source) {
|
|
1288
|
+
const internalId = resolver.toInternal(posting.docId);
|
|
1289
|
+
if (internalId === void 0) continue;
|
|
1290
|
+
const termFrequency = posting.termFrequency > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : posting.termFrequency;
|
|
1291
|
+
const fieldIndex = fieldIndexFor(fieldNameTable, posting.fieldName);
|
|
1292
|
+
postings.insert(token, internalId, termFrequency, fieldIndex, hasPositions ? posting.positions : null);
|
|
687
1293
|
}
|
|
688
|
-
index.set(token, {
|
|
689
|
-
length: validCount,
|
|
690
|
-
docIds,
|
|
691
|
-
termFrequencies: finalTF,
|
|
692
|
-
fieldNameIndices: finalFNI,
|
|
693
|
-
positions,
|
|
694
|
-
docIdSet,
|
|
695
|
-
deletedDocs: /* @__PURE__ */ new Set(),
|
|
696
|
-
totalTermFrequency,
|
|
697
|
-
structureRevision: 0,
|
|
698
|
-
ordered
|
|
699
|
-
});
|
|
700
|
-
trackToken(token);
|
|
701
1294
|
}
|
|
702
1295
|
}
|
|
703
1296
|
};
|
|
@@ -866,6 +1459,9 @@ function createSurfaceRegistry() {
|
|
|
866
1459
|
else totalsByToken.set(token, next);
|
|
867
1460
|
}
|
|
868
1461
|
return {
|
|
1462
|
+
holds(surface) {
|
|
1463
|
+
return entries.has(surface);
|
|
1464
|
+
},
|
|
869
1465
|
add(surface, token, occurrences) {
|
|
870
1466
|
if (surface.length === 0 || token.length === 0 || occurrences <= 0) return;
|
|
871
1467
|
if (surface === token) return;
|
|
@@ -2290,7 +2886,7 @@ var BYTES_PER_BOOLEAN_SLOT = 1;
|
|
|
2290
2886
|
var BYTES_PER_MIXED_SLOT = 8;
|
|
2291
2887
|
var BYTES_PER_STRING_CODE_UNIT = 2;
|
|
2292
2888
|
var STRING_HEADER_BYTES2 = 24;
|
|
2293
|
-
function
|
|
2889
|
+
function grow2(length, wanted) {
|
|
2294
2890
|
let next = length === 0 ? 64 : length;
|
|
2295
2891
|
while (next <= wanted) next *= 2;
|
|
2296
2892
|
return next;
|
|
@@ -2325,7 +2921,7 @@ function createValueStore(initialKind) {
|
|
|
2325
2921
|
function setNumber(internalId, value) {
|
|
2326
2922
|
if (numbers === null) return;
|
|
2327
2923
|
if (internalId >= numbers.length) {
|
|
2328
|
-
const grown = new Float64Array(
|
|
2924
|
+
const grown = new Float64Array(grow2(numbers.length, internalId));
|
|
2329
2925
|
grown.fill(Number.NaN, numbers.length);
|
|
2330
2926
|
grown.set(numbers);
|
|
2331
2927
|
numbers = grown;
|
|
@@ -2335,7 +2931,7 @@ function createValueStore(initialKind) {
|
|
|
2335
2931
|
function setBoolean(internalId, value) {
|
|
2336
2932
|
if (booleans === null) return;
|
|
2337
2933
|
if (internalId >= booleans.length) {
|
|
2338
|
-
const grown = new Uint8Array(
|
|
2934
|
+
const grown = new Uint8Array(grow2(booleans.length, internalId));
|
|
2339
2935
|
grown.fill(BOOLEAN_MISSING, booleans.length);
|
|
2340
2936
|
grown.set(booleans);
|
|
2341
2937
|
booleans = grown;
|
|
@@ -2446,10 +3042,9 @@ function createSortColumnSet(docStore) {
|
|
|
2446
3042
|
}
|
|
2447
3043
|
function backfill(key, field, fieldType, mode) {
|
|
2448
3044
|
const store = createValueStore(kindForFieldType(fieldType));
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
store.set(internalId, toReducedSortValue(readSortField(stored.fields, field), mode));
|
|
3045
|
+
const read = docStore.fieldReader(field);
|
|
3046
|
+
for (const internalId of docStore.allInternalIds()) {
|
|
3047
|
+
store.set(internalId, toReducedSortValue(read(internalId), mode));
|
|
2453
3048
|
}
|
|
2454
3049
|
const entry = {
|
|
2455
3050
|
field,
|
|
@@ -2781,15 +3376,18 @@ function sortedPageOf(state, request) {
|
|
|
2781
3376
|
function sortValuesOf(state, docId, fields, fieldTypes, modes) {
|
|
2782
3377
|
const set = state.sortColumns;
|
|
2783
3378
|
const internalId = state.docStore.getInternalId(docId);
|
|
2784
|
-
const stored = state.docStore.get(docId);
|
|
2785
3379
|
const key = new Array(fields.length);
|
|
2786
3380
|
for (let i = 0; i < fields.length; i++) {
|
|
2787
3381
|
const mode = modes[i] ?? "min";
|
|
2788
|
-
if (
|
|
3382
|
+
if (internalId === void 0) {
|
|
3383
|
+
key[i] = null;
|
|
3384
|
+
continue;
|
|
3385
|
+
}
|
|
3386
|
+
if (set?.holds(fields[i], fieldTypes[i], mode)) {
|
|
2789
3387
|
key[i] = set.column(fields[i], fieldTypes[i], mode).valueOf(internalId);
|
|
2790
3388
|
continue;
|
|
2791
3389
|
}
|
|
2792
|
-
key[i] =
|
|
3390
|
+
key[i] = toReducedSortValue(state.docStore.fieldReader(fields[i])(internalId), mode);
|
|
2793
3391
|
}
|
|
2794
3392
|
return key;
|
|
2795
3393
|
}
|
|
@@ -2832,7 +3430,15 @@ function buildFilterContext(state, schema, meter) {
|
|
|
2832
3430
|
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
2833
3431
|
if (pattern) patternFields.add(fieldPath);
|
|
2834
3432
|
}
|
|
2835
|
-
const
|
|
3433
|
+
const fieldReaders = /* @__PURE__ */ new Map();
|
|
3434
|
+
const readerOf = (fieldPath) => {
|
|
3435
|
+
let read = fieldReaders.get(fieldPath);
|
|
3436
|
+
if (read === void 0) {
|
|
3437
|
+
read = state.docStore.fieldReader(fieldPath);
|
|
3438
|
+
fieldReaders.set(fieldPath, read);
|
|
3439
|
+
}
|
|
3440
|
+
return read;
|
|
3441
|
+
};
|
|
2836
3442
|
let cachedAllDocIdsBitset = null;
|
|
2837
3443
|
const cachedPresentBitsets = /* @__PURE__ */ new Map();
|
|
2838
3444
|
return {
|
|
@@ -2856,7 +3462,7 @@ function buildFilterContext(state, schema, meter) {
|
|
|
2856
3462
|
matches: into,
|
|
2857
3463
|
meter,
|
|
2858
3464
|
host: PATTERN_HOST,
|
|
2859
|
-
valueOf: (
|
|
3465
|
+
valueOf: readerOf(fieldPath)
|
|
2860
3466
|
});
|
|
2861
3467
|
},
|
|
2862
3468
|
presentBitset: (fieldPath) => {
|
|
@@ -3215,38 +3821,18 @@ function searchFulltextMatches(state, params) {
|
|
|
3215
3821
|
// src/core/partition/memory.ts
|
|
3216
3822
|
var DOC_ENTRY_OVERHEAD_BYTES = 260;
|
|
3217
3823
|
var LENGTH_COLUMN_SLOT_BYTES = 6;
|
|
3218
|
-
var
|
|
3219
|
-
var POSITIONS_ARRAY_OVERHEAD_BYTES = 88;
|
|
3220
|
-
var POSITION_BYTES = 8;
|
|
3221
|
-
var PER_TERM_OVERHEAD_BYTES = 180;
|
|
3824
|
+
var DOC_FREQUENCY_ENTRY_BYTES = 44;
|
|
3222
3825
|
var SURFACE_ENTRY_OVERHEAD_BYTES = 140;
|
|
3223
3826
|
var FIELD_ENTRY_OVERHEAD_BYTES = 42;
|
|
3224
3827
|
function estimatePartitionBytes(state) {
|
|
3225
3828
|
const docCount = state.docStore.count();
|
|
3226
3829
|
if (docCount === 0) return 0;
|
|
3227
|
-
const
|
|
3228
|
-
let totalPostings = 0;
|
|
3229
|
-
let termCount = 0;
|
|
3230
|
-
for (const term in docFreqs) {
|
|
3231
|
-
totalPostings += docFreqs[term];
|
|
3232
|
-
termCount++;
|
|
3233
|
-
}
|
|
3234
|
-
const fieldLengthTotals = state.stats.totalFieldLengths;
|
|
3235
|
-
let totalTokenOccurrences = 0;
|
|
3236
|
-
let textFieldCount = 0;
|
|
3237
|
-
for (const field in fieldLengthTotals) {
|
|
3238
|
-
totalTokenOccurrences += fieldLengthTotals[field];
|
|
3239
|
-
textFieldCount++;
|
|
3240
|
-
}
|
|
3830
|
+
const textFieldCount = Object.keys(state.stats.totalFieldLengths).length;
|
|
3241
3831
|
let bytes = state.docStore.contentBytes();
|
|
3242
3832
|
bytes += docCount * DOC_ENTRY_OVERHEAD_BYTES;
|
|
3243
3833
|
bytes += textFieldCount * state.docStore.internalIdCapacity() * LENGTH_COLUMN_SLOT_BYTES;
|
|
3244
|
-
bytes +=
|
|
3245
|
-
|
|
3246
|
-
bytes += totalPostings * POSITIONS_ARRAY_OVERHEAD_BYTES;
|
|
3247
|
-
bytes += totalTokenOccurrences * POSITION_BYTES;
|
|
3248
|
-
}
|
|
3249
|
-
bytes += termCount * PER_TERM_OVERHEAD_BYTES;
|
|
3834
|
+
bytes += state.invertedIdx.estimateBytes();
|
|
3835
|
+
bytes += state.invertedIdx.size() * DOC_FREQUENCY_ENTRY_BYTES;
|
|
3250
3836
|
bytes += state.surfaceRegistry.size() * SURFACE_ENTRY_OVERHEAD_BYTES;
|
|
3251
3837
|
bytes += docCount * state.numericIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
3252
3838
|
bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
@@ -3282,17 +3868,18 @@ function mergeDocuments(target, source) {
|
|
|
3282
3868
|
}
|
|
3283
3869
|
function mergePostings(target, source, idMapping) {
|
|
3284
3870
|
for (const token of source.invertedIdx.tokens()) {
|
|
3285
|
-
const list = source.invertedIdx.lookup(token);
|
|
3871
|
+
const list = source.invertedIdx.lookupReadOnce?.(token) ?? source.invertedIdx.lookup(token);
|
|
3286
3872
|
if (list === void 0) continue;
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3873
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
|
|
3874
|
+
for (let i = 0; i < length; i++) {
|
|
3875
|
+
const sourceInternalId = docIds[i];
|
|
3876
|
+
if (deletedDocs.has(sourceInternalId)) continue;
|
|
3290
3877
|
const targetInternalId = idMapping.get(sourceInternalId);
|
|
3291
3878
|
if (targetInternalId === void 0) continue;
|
|
3292
|
-
const fieldName = source.fieldNameTable.names[
|
|
3879
|
+
const fieldName = source.fieldNameTable.names[fieldNameIndices[i]];
|
|
3293
3880
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(target.fieldNameTable, fieldName);
|
|
3294
|
-
const
|
|
3295
|
-
target.invertedIdx.insert(token, targetInternalId,
|
|
3881
|
+
const entryPositions = positions === null ? null : positions[i];
|
|
3882
|
+
target.invertedIdx.insert(token, targetInternalId, termFrequencies[i], fieldNameIndex2, entryPositions ?? null);
|
|
3296
3883
|
}
|
|
3297
3884
|
}
|
|
3298
3885
|
}
|
|
@@ -3404,8 +3991,9 @@ function entriesCovered(bounds) {
|
|
|
3404
3991
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
3405
3992
|
}
|
|
3406
3993
|
function scan(list, columns, previous, keepBlocks) {
|
|
3994
|
+
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
3407
3995
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
3408
|
-
const capacity = keepBlocks + Math.ceil((
|
|
3996
|
+
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
3409
3997
|
const entryEnd = new Int32Array(capacity);
|
|
3410
3998
|
const maxTermFrequency = new Int32Array(capacity);
|
|
3411
3999
|
const minFieldLength = new Float64Array(capacity);
|
|
@@ -3427,16 +4015,16 @@ function scan(list, columns, previous, keepBlocks) {
|
|
|
3427
4015
|
let blockMaxRun = 0;
|
|
3428
4016
|
let blockDocs = 0;
|
|
3429
4017
|
let entry = startEntry;
|
|
3430
|
-
while (entry <
|
|
3431
|
-
const docId =
|
|
4018
|
+
while (entry < length) {
|
|
4019
|
+
const docId = docIds[entry];
|
|
3432
4020
|
const tombstoned = hasDeleted && deleted.has(docId);
|
|
3433
4021
|
let run = 0;
|
|
3434
|
-
while (entry + run <
|
|
4022
|
+
while (entry + run < length && docIds[entry + run] === docId) {
|
|
3435
4023
|
if (!tombstoned) {
|
|
3436
4024
|
const position = entry + run;
|
|
3437
|
-
const termFrequency =
|
|
4025
|
+
const termFrequency = termFrequencies[position];
|
|
3438
4026
|
if (termFrequency > blockMaxTf) blockMaxTf = termFrequency;
|
|
3439
|
-
const fieldLength = fieldLengthBound(columns,
|
|
4027
|
+
const fieldLength = fieldLengthBound(columns, fieldNameIndices[position], docId);
|
|
3440
4028
|
if (fieldLength < blockMinLen) blockMinLen = fieldLength;
|
|
3441
4029
|
}
|
|
3442
4030
|
run++;
|
|
@@ -3446,7 +4034,7 @@ function scan(list, columns, previous, keepBlocks) {
|
|
|
3446
4034
|
blockDocs++;
|
|
3447
4035
|
}
|
|
3448
4036
|
entry += run;
|
|
3449
|
-
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry ===
|
|
4037
|
+
if (entry - blockStart >= TARGET_BLOCK_ENTRIES || entry === length) {
|
|
3450
4038
|
entryEnd[blockIndex] = entry;
|
|
3451
4039
|
maxTermFrequency[blockIndex] = blockMaxTf;
|
|
3452
4040
|
minFieldLength[blockIndex] = blockMinLen === Number.POSITIVE_INFINITY ? ABSENT_FIELD_LENGTH_BOUND : blockMinLen;
|
|
@@ -4203,10 +4791,11 @@ function countPostings(state, remap) {
|
|
|
4203
4791
|
const list = lookupReadOnce(state, token);
|
|
4204
4792
|
if (list === void 0) continue;
|
|
4205
4793
|
const listPositions = list.positionCountAt === void 0 ? list.positions : null;
|
|
4206
|
-
const tracksPositions = list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions;
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
|
|
4794
|
+
const tracksPositions = list.holdsPositions ?? (list.positionCountAt === void 0 ? listPositions !== null : state.trackPositions);
|
|
4795
|
+
const { docIds, deletedDocs, length } = list;
|
|
4796
|
+
for (let i = 0; i < length; i++) {
|
|
4797
|
+
const internalId = docIds[i];
|
|
4798
|
+
if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
|
|
4210
4799
|
postings++;
|
|
4211
4800
|
if (!tracksPositions) continue;
|
|
4212
4801
|
hasPositions = true;
|
|
@@ -4257,18 +4846,20 @@ function encodeSegmentState(state) {
|
|
|
4257
4846
|
for (const token of state.invertedIdx.tokens()) {
|
|
4258
4847
|
const list = lookupReadOnce(state, token);
|
|
4259
4848
|
if (list === void 0) continue;
|
|
4260
|
-
const
|
|
4849
|
+
const copyPositionsAt = list.copyPositionsAt?.bind(list);
|
|
4850
|
+
const listPositions = positionOffsets === null || copyPositionsAt !== void 0 ? null : list.positions;
|
|
4261
4851
|
const tokenStart = postingCursor;
|
|
4262
4852
|
const countViaSet = !list.ordered;
|
|
4263
4853
|
if (countViaSet) unorderedTokenDocs.clear();
|
|
4264
4854
|
let uniqueDocs = 0;
|
|
4265
4855
|
let lastCountedDoc = -1;
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4856
|
+
const { docIds: docIds2, termFrequencies, fieldNameIndices, deletedDocs, length } = list;
|
|
4857
|
+
for (let i = 0; i < length; i++) {
|
|
4858
|
+
const internalId = docIds2[i];
|
|
4859
|
+
if (deletedDocs.has(internalId) || remap[internalId] < 0) continue;
|
|
4269
4860
|
postingDocIds[postingCursor] = remap[internalId];
|
|
4270
|
-
postingFrequencies[postingCursor] =
|
|
4271
|
-
postingFieldIndices[postingCursor] =
|
|
4861
|
+
postingFrequencies[postingCursor] = termFrequencies[i];
|
|
4862
|
+
postingFieldIndices[postingCursor] = fieldNameIndices[i];
|
|
4272
4863
|
if (countViaSet) {
|
|
4273
4864
|
unorderedTokenDocs.add(internalId);
|
|
4274
4865
|
} else if (internalId !== lastCountedDoc) {
|
|
@@ -4278,7 +4869,9 @@ function encodeSegmentState(state) {
|
|
|
4278
4869
|
if (positionOffsets !== null && positionValues !== null) {
|
|
4279
4870
|
positionOffsets[postingCursor] = positionCursor;
|
|
4280
4871
|
const entryPositions = listPositions === null ? null : listPositions[i];
|
|
4281
|
-
if (
|
|
4872
|
+
if (copyPositionsAt !== void 0) {
|
|
4873
|
+
positionCursor += copyPositionsAt(i, positionValues, positionCursor);
|
|
4874
|
+
} else if (entryPositions !== null && entryPositions !== void 0) {
|
|
4282
4875
|
for (const position of entryPositions) positionValues[positionCursor++] = position;
|
|
4283
4876
|
}
|
|
4284
4877
|
}
|
|
@@ -4488,17 +5081,18 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
4488
5081
|
for (const token of state.invertedIdx.tokens()) {
|
|
4489
5082
|
const list = state.invertedIdx.lookup(token);
|
|
4490
5083
|
if (!list) continue;
|
|
4491
|
-
const
|
|
5084
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
|
|
5085
|
+
const hasDeleted = deletedDocs.size > 0;
|
|
4492
5086
|
const postings = [];
|
|
4493
|
-
for (let i = 0; i <
|
|
4494
|
-
if (hasDeleted &&
|
|
4495
|
-
const externalId = resolver.toExternal(
|
|
5087
|
+
for (let i = 0; i < length; i++) {
|
|
5088
|
+
if (hasDeleted && deletedDocs.has(docIds[i])) continue;
|
|
5089
|
+
const externalId = resolver.toExternal(docIds[i]);
|
|
4496
5090
|
if (externalId === void 0) continue;
|
|
4497
5091
|
postings.push({
|
|
4498
5092
|
docId: externalId,
|
|
4499
|
-
termFrequency:
|
|
4500
|
-
field: fieldNames[
|
|
4501
|
-
positions:
|
|
5093
|
+
termFrequency: termFrequencies[i],
|
|
5094
|
+
field: fieldNames[fieldNameIndices[i]],
|
|
5095
|
+
positions: positions === null ? [] : Array.from(positions[i] ?? [])
|
|
4502
5096
|
});
|
|
4503
5097
|
}
|
|
4504
5098
|
if (postings.length > 0) {
|
|
@@ -4821,6 +5415,11 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4821
5415
|
if (!stored) return void 0;
|
|
4822
5416
|
return stored.fields;
|
|
4823
5417
|
},
|
|
5418
|
+
fieldValues(docId, fieldPaths) {
|
|
5419
|
+
const stored = state.docStore.get(docId);
|
|
5420
|
+
if (!stored) return void 0;
|
|
5421
|
+
return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
|
|
5422
|
+
},
|
|
4824
5423
|
has(docId) {
|
|
4825
5424
|
return state.docStore.has(docId);
|
|
4826
5425
|
},
|
|
@@ -4910,7 +5509,7 @@ function frozenSegmentBytes(segment) {
|
|
|
4910
5509
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
4911
5510
|
}
|
|
4912
5511
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
4913
|
-
return bytes + segment.partialWords.bytes();
|
|
5512
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
4914
5513
|
}
|
|
4915
5514
|
|
|
4916
5515
|
// src/core/partition/composite/compaction.ts
|
|
@@ -4948,19 +5547,40 @@ function freezeLiveTailInto(live, liveState, frozen, freeze) {
|
|
|
4948
5547
|
frozen.push(segment);
|
|
4949
5548
|
return segment;
|
|
4950
5549
|
}
|
|
4951
|
-
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement) {
|
|
5550
|
+
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
|
|
4952
5551
|
const dropSet = new Set(dropSegmentIds);
|
|
4953
5552
|
const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
|
|
5553
|
+
const applied = /* @__PURE__ */ new Map();
|
|
4954
5554
|
for (const segment of dropped) {
|
|
4955
5555
|
for (const docId of segment.tombstonedDocIds()) {
|
|
4956
5556
|
if (!replacement.hasDocument(docId)) continue;
|
|
4957
5557
|
if (dropped.some((other) => other.hasDocument(docId))) continue;
|
|
4958
|
-
replacement.
|
|
5558
|
+
const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
|
|
5559
|
+
if (spellings !== void 0) applied.set(docId, spellings);
|
|
5560
|
+
replacement.tombstoneDocument(docId, spellings);
|
|
4959
5561
|
}
|
|
4960
5562
|
}
|
|
4961
5563
|
const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
|
|
4962
5564
|
frozen.length = 0;
|
|
4963
5565
|
frozen.push(...kept, replacement);
|
|
5566
|
+
return applied;
|
|
5567
|
+
}
|
|
5568
|
+
|
|
5569
|
+
// src/core/partition/composite/documents.ts
|
|
5570
|
+
function compositeDocument(frozen, live, docId, projection) {
|
|
5571
|
+
for (const segment of frozen) {
|
|
5572
|
+
const projected = segment.docStore.projectedDocument(docId, projection);
|
|
5573
|
+
if (projected !== void 0) return projected;
|
|
5574
|
+
}
|
|
5575
|
+
return live.get(docId, projection);
|
|
5576
|
+
}
|
|
5577
|
+
function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
5578
|
+
for (const segment of frozen) {
|
|
5579
|
+
const internalId = segment.docStore.getInternalId(docId);
|
|
5580
|
+
if (internalId === void 0) continue;
|
|
5581
|
+
return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
|
|
5582
|
+
}
|
|
5583
|
+
return live.fieldValues(docId, fieldPaths);
|
|
4964
5584
|
}
|
|
4965
5585
|
|
|
4966
5586
|
// src/core/partition/composite/filters.ts
|
|
@@ -5286,7 +5906,7 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
5286
5906
|
function isCompositePartition(partition) {
|
|
5287
5907
|
return "attachFrozenSegment" in partition;
|
|
5288
5908
|
}
|
|
5289
|
-
function createCompositePartition(partitionId, trackPositions = true, existingLive) {
|
|
5909
|
+
function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
|
|
5290
5910
|
const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
|
|
5291
5911
|
const liveState = partitionStateOf(live);
|
|
5292
5912
|
const frozen = [];
|
|
@@ -5294,6 +5914,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5294
5914
|
function invalidateDocFrequencies() {
|
|
5295
5915
|
docFrequenciesCache = null;
|
|
5296
5916
|
}
|
|
5917
|
+
function bindSegments() {
|
|
5918
|
+
invalidateDocFrequencies();
|
|
5919
|
+
if (textAnalysis === void 0) return;
|
|
5920
|
+
for (const segment of frozen) segment.removals.bind(textAnalysis);
|
|
5921
|
+
}
|
|
5297
5922
|
function cachedDocFrequencies() {
|
|
5298
5923
|
if (docFrequenciesCache === null) {
|
|
5299
5924
|
docFrequenciesCache = mergeDocFrequencies(subs());
|
|
@@ -5347,23 +5972,25 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5347
5972
|
frozenSegmentsById(segmentIds) {
|
|
5348
5973
|
return resolveFrozenSegments(frozen, segmentIds, partitionId);
|
|
5349
5974
|
},
|
|
5350
|
-
swapFrozenSegments(dropSegmentIds, replacement) {
|
|
5351
|
-
|
|
5352
|
-
|
|
5975
|
+
swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
|
|
5976
|
+
if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
|
|
5977
|
+
const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
|
|
5978
|
+
bindSegments();
|
|
5979
|
+
return applied;
|
|
5353
5980
|
},
|
|
5354
5981
|
freezeLiveTail(freeze) {
|
|
5355
5982
|
const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
|
|
5356
|
-
if (segment !== null)
|
|
5983
|
+
if (segment !== null) bindSegments();
|
|
5357
5984
|
return segment;
|
|
5358
5985
|
},
|
|
5359
5986
|
replaceLiveTail(segment) {
|
|
5360
5987
|
live.clear();
|
|
5361
5988
|
frozen.push(segment);
|
|
5362
|
-
|
|
5989
|
+
bindSegments();
|
|
5363
5990
|
},
|
|
5364
5991
|
attachFrozenSegment(segment) {
|
|
5365
5992
|
frozen.push(segment);
|
|
5366
|
-
|
|
5993
|
+
bindSegments();
|
|
5367
5994
|
},
|
|
5368
5995
|
mergeSegment(segment) {
|
|
5369
5996
|
invalidateDocFrequencies();
|
|
@@ -5402,7 +6029,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5402
6029
|
invalidateDocFrequencies();
|
|
5403
6030
|
live.insert(docId, document, schema, language, options);
|
|
5404
6031
|
},
|
|
5405
|
-
update(docId, document, schema, language, options) {
|
|
6032
|
+
update(docId, document, schema, language, options, removedSpellings) {
|
|
5406
6033
|
invalidateDocFrequencies();
|
|
5407
6034
|
const owner = frozenOwner(docId);
|
|
5408
6035
|
if (owner === void 0) {
|
|
@@ -5410,9 +6037,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5410
6037
|
return;
|
|
5411
6038
|
}
|
|
5412
6039
|
live.insert(docId, document, schema, language, options);
|
|
5413
|
-
owner.tombstoneDocument(docId);
|
|
6040
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
5414
6041
|
},
|
|
5415
|
-
remove(docId, schema, language, options) {
|
|
6042
|
+
remove(docId, schema, language, options, removedSpellings) {
|
|
5416
6043
|
if (live.has(docId)) {
|
|
5417
6044
|
invalidateDocFrequencies();
|
|
5418
6045
|
live.remove(docId, schema, language, options);
|
|
@@ -5425,7 +6052,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5425
6052
|
partitionId
|
|
5426
6053
|
});
|
|
5427
6054
|
}
|
|
5428
|
-
|
|
6055
|
+
invalidateDocFrequencies();
|
|
6056
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
6057
|
+
},
|
|
6058
|
+
removedSpellingsOf(docId) {
|
|
6059
|
+
return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
|
|
5429
6060
|
},
|
|
5430
6061
|
beginBatch() {
|
|
5431
6062
|
live.beginBatch();
|
|
@@ -5488,9 +6119,10 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5488
6119
|
expandPartialWords(subs(), analysis, []);
|
|
5489
6120
|
},
|
|
5490
6121
|
get(docId, projection) {
|
|
5491
|
-
|
|
5492
|
-
|
|
5493
|
-
|
|
6122
|
+
return compositeDocument(frozen, live, docId, projection);
|
|
6123
|
+
},
|
|
6124
|
+
fieldValues(docId, fieldPaths) {
|
|
6125
|
+
return compositeFieldValues(frozen, live, docId, fieldPaths);
|
|
5494
6126
|
},
|
|
5495
6127
|
getRef(docId) {
|
|
5496
6128
|
const stored = frozenStored(docId);
|
|
@@ -5509,8 +6141,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5509
6141
|
},
|
|
5510
6142
|
*docIds() {
|
|
5511
6143
|
for (const segment of frozen) {
|
|
5512
|
-
for (const
|
|
5513
|
-
|
|
6144
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
6145
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
6146
|
+
if (docId !== void 0) yield docId;
|
|
5514
6147
|
}
|
|
5515
6148
|
}
|
|
5516
6149
|
yield* live.docIds();
|
|
@@ -5601,6 +6234,18 @@ function createCapacityChecks(source) {
|
|
|
5601
6234
|
};
|
|
5602
6235
|
}
|
|
5603
6236
|
|
|
6237
|
+
// src/partitioning/manager/segments.ts
|
|
6238
|
+
function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
6239
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
6240
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
6241
|
+
if (docId === void 0 || !docPartitionMap.has(docId)) continue;
|
|
6242
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
|
|
6243
|
+
docId,
|
|
6244
|
+
partitionId: docPartitionMap.get(docId)
|
|
6245
|
+
});
|
|
6246
|
+
}
|
|
6247
|
+
}
|
|
6248
|
+
|
|
5604
6249
|
// src/partitioning/manager/stats.ts
|
|
5605
6250
|
function aggregateStatsOf(partitions) {
|
|
5606
6251
|
let totalDocuments = 0;
|
|
@@ -5683,6 +6328,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5683
6328
|
const count = initialPartitionCount ?? 1;
|
|
5684
6329
|
const analysis = resolveIndexAnalysis(config);
|
|
5685
6330
|
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
6331
|
+
const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
5686
6332
|
const trackPositions = config.trackPositions ?? true;
|
|
5687
6333
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
5688
6334
|
let partitions = [];
|
|
@@ -5711,10 +6357,8 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5711
6357
|
}
|
|
5712
6358
|
function asCompositePartition(partitionId) {
|
|
5713
6359
|
const existing = partitions[partitionId];
|
|
5714
|
-
if (
|
|
5715
|
-
|
|
5716
|
-
}
|
|
5717
|
-
const composite = createCompositePartition(partitionId, trackPositions, existing);
|
|
6360
|
+
if (isCompositePartition(existing)) return existing;
|
|
6361
|
+
const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
|
|
5718
6362
|
partitions[partitionId] = composite;
|
|
5719
6363
|
return composite;
|
|
5720
6364
|
}
|
|
@@ -5764,6 +6408,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5764
6408
|
get partialWordAnalysis() {
|
|
5765
6409
|
return partialWordAnalysis;
|
|
5766
6410
|
},
|
|
6411
|
+
get textAnalysis() {
|
|
6412
|
+
return textAnalysis;
|
|
6413
|
+
},
|
|
5767
6414
|
getPartition(partitionId) {
|
|
5768
6415
|
validatePartitionId(partitionId);
|
|
5769
6416
|
return partitions[partitionId];
|
|
@@ -5847,12 +6494,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5847
6494
|
docPartitionMap.set(docId, pid);
|
|
5848
6495
|
writeLog.documentWritten(docId);
|
|
5849
6496
|
},
|
|
5850
|
-
|
|
6497
|
+
removedSpellingsOf(docId) {
|
|
6498
|
+
const pid = locateDocument(docId);
|
|
6499
|
+
const partition = pid === void 0 ? void 0 : partitions[pid];
|
|
6500
|
+
return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
|
|
6501
|
+
},
|
|
6502
|
+
remove(docId, removedSpellings) {
|
|
5851
6503
|
const pid = locateDocument(docId);
|
|
5852
6504
|
if (pid === void 0) {
|
|
5853
6505
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
5854
6506
|
}
|
|
5855
|
-
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
|
|
6507
|
+
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
5856
6508
|
docPartitionMap.delete(docId);
|
|
5857
6509
|
writeLog.documentWritten(docId);
|
|
5858
6510
|
},
|
|
@@ -5866,13 +6518,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5866
6518
|
partitions[i].endBatch();
|
|
5867
6519
|
}
|
|
5868
6520
|
},
|
|
5869
|
-
update(docId, document, options) {
|
|
6521
|
+
update(docId, document, options, removedSpellings) {
|
|
5870
6522
|
const pid = locateDocument(docId);
|
|
5871
6523
|
if (pid === void 0) {
|
|
5872
6524
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
5873
6525
|
}
|
|
5874
6526
|
const updateOpts = resolveInsertOptions(options);
|
|
5875
|
-
partitions[pid].update(docId, document, config.schema, language, updateOpts);
|
|
6527
|
+
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
5876
6528
|
docPartitionMap.set(docId, pid);
|
|
5877
6529
|
writeLog.documentWritten(docId);
|
|
5878
6530
|
},
|
|
@@ -5880,14 +6532,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5880
6532
|
const pid = locateDocument(docId);
|
|
5881
6533
|
if (pid === void 0) return void 0;
|
|
5882
6534
|
const doc = partitions[pid].get(docId, projection);
|
|
5883
|
-
if (
|
|
5884
|
-
for (const [fieldPath, vecIndex] of vecIndexes) {
|
|
5885
|
-
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
5886
|
-
const vector = vecIndex.getVector(docId);
|
|
5887
|
-
if (vector) {
|
|
5888
|
-
setNestedValue(doc, fieldPath, vector);
|
|
5889
|
-
}
|
|
5890
|
-
}
|
|
6535
|
+
if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
|
|
5891
6536
|
return doc;
|
|
5892
6537
|
},
|
|
5893
6538
|
getRef(docId) {
|
|
@@ -5895,6 +6540,10 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5895
6540
|
if (pid === void 0) return void 0;
|
|
5896
6541
|
return partitions[pid].getRef(docId);
|
|
5897
6542
|
},
|
|
6543
|
+
fieldValues(docId, fieldPaths) {
|
|
6544
|
+
const pid = locateDocument(docId);
|
|
6545
|
+
return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
|
|
6546
|
+
},
|
|
5898
6547
|
sortValues(docId, fields, fieldTypes, modes) {
|
|
5899
6548
|
const pid = locateDocument(docId);
|
|
5900
6549
|
if (pid === void 0) return fields.map(() => null);
|
|
@@ -5930,22 +6579,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5930
6579
|
},
|
|
5931
6580
|
attachFrozenSegment(partitionId, segment) {
|
|
5932
6581
|
validatePartitionId(partitionId);
|
|
5933
|
-
|
|
5934
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
5935
|
-
if (docId === void 0 || !docPartitionMap.has(docId)) continue;
|
|
5936
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
|
|
5937
|
-
docId,
|
|
5938
|
-
partitionId: docPartitionMap.get(docId)
|
|
5939
|
-
});
|
|
5940
|
-
}
|
|
6582
|
+
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
5941
6583
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
5942
6584
|
registerSegmentDocuments(partitionId, segment);
|
|
5943
6585
|
writeLog.segmentWritten(segment);
|
|
5944
6586
|
},
|
|
5945
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
|
|
6587
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
5946
6588
|
validatePartitionId(partitionId);
|
|
5947
|
-
asCompositePartition(partitionId).swapFrozenSegments(
|
|
6589
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
6590
|
+
dropSegmentIds,
|
|
6591
|
+
replacement,
|
|
6592
|
+
removedSpellings
|
|
6593
|
+
);
|
|
5948
6594
|
registerSegmentDocuments(partitionId, replacement);
|
|
6595
|
+
return applied;
|
|
5949
6596
|
},
|
|
5950
6597
|
freezeLiveTail(partitionId, freeze) {
|
|
5951
6598
|
validatePartitionId(partitionId);
|
|
@@ -6033,21 +6680,15 @@ function generateId() {
|
|
|
6033
6680
|
// src/core/partition/frozen/doc-table.ts
|
|
6034
6681
|
function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
6035
6682
|
const capacity = idTable.count;
|
|
6036
|
-
const storedCache = new Array(capacity);
|
|
6037
6683
|
let sortedIds = null;
|
|
6038
6684
|
let sortedRevision = -1;
|
|
6039
6685
|
function storedAt(ordinal) {
|
|
6040
|
-
|
|
6041
|
-
|
|
6042
|
-
const
|
|
6043
|
-
|
|
6044
|
-
const length = column[ordinal];
|
|
6045
|
-
if (length > 0) fieldLengths[fieldName] = length;
|
|
6046
|
-
}
|
|
6047
|
-
stored = { fields: documents.docAt(ordinal), fieldLengths };
|
|
6048
|
-
storedCache[ordinal] = stored;
|
|
6686
|
+
const fieldLengths = {};
|
|
6687
|
+
for (const [fieldName, column] of lengthColumns) {
|
|
6688
|
+
const length = column[ordinal];
|
|
6689
|
+
if (length > 0) fieldLengths[fieldName] = length;
|
|
6049
6690
|
}
|
|
6050
|
-
return
|
|
6691
|
+
return { fields: documents.docAt(ordinal), fieldLengths };
|
|
6051
6692
|
}
|
|
6052
6693
|
function liveOrdinal(docId) {
|
|
6053
6694
|
const ordinal = idTable.ordinalOf(docId);
|
|
@@ -6116,43 +6757,275 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
|
6116
6757
|
},
|
|
6117
6758
|
resolver() {
|
|
6118
6759
|
return resolver;
|
|
6760
|
+
},
|
|
6761
|
+
fieldReader(fieldPath) {
|
|
6762
|
+
const read = documents.fieldReader(fieldPath);
|
|
6763
|
+
return (internalId) => resolver.isLive(internalId) ? read(internalId) : void 0;
|
|
6764
|
+
},
|
|
6765
|
+
projectedDocument(docId, projection) {
|
|
6766
|
+
const ordinal = liveOrdinal(docId);
|
|
6767
|
+
return ordinal === void 0 ? void 0 : documents.projectedAt(ordinal, projection);
|
|
6119
6768
|
}
|
|
6120
6769
|
};
|
|
6121
6770
|
}
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
6125
|
-
|
|
6126
|
-
|
|
6127
|
-
|
|
6128
|
-
|
|
6771
|
+
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
6772
|
+
var keyDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
6773
|
+
function headerAt(view, offset) {
|
|
6774
|
+
if (offset >= view.byteLength) return null;
|
|
6775
|
+
const tag = view.getUint8(offset);
|
|
6776
|
+
if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
|
|
6777
|
+
if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
|
|
6778
|
+
if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
|
|
6779
|
+
if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
|
|
6780
|
+
const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
|
|
6781
|
+
if (lengthBytes !== void 0) {
|
|
6782
|
+
if (offset + 1 + lengthBytes > view.byteLength) return null;
|
|
6783
|
+
const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
|
|
6784
|
+
const headerBytes = 1 + lengthBytes;
|
|
6785
|
+
if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
|
|
6786
|
+
if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
|
|
6787
|
+
return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
|
|
6129
6788
|
}
|
|
6130
|
-
|
|
6789
|
+
const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
|
|
6790
|
+
return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
|
|
6131
6791
|
}
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6792
|
+
var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
6793
|
+
[196, 1],
|
|
6794
|
+
[197, 2],
|
|
6795
|
+
[198, 4],
|
|
6796
|
+
[199, 1],
|
|
6797
|
+
[200, 2],
|
|
6798
|
+
[201, 4],
|
|
6799
|
+
[217, 1],
|
|
6800
|
+
[218, 2],
|
|
6801
|
+
[219, 4],
|
|
6802
|
+
[220, 2],
|
|
6803
|
+
[221, 4],
|
|
6804
|
+
[222, 2],
|
|
6805
|
+
[223, 4]
|
|
6806
|
+
]);
|
|
6807
|
+
var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
|
|
6808
|
+
var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
|
|
6809
|
+
var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
|
|
6810
|
+
var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
6811
|
+
[192, 0],
|
|
6812
|
+
[194, 0],
|
|
6813
|
+
[195, 0],
|
|
6814
|
+
[202, 4],
|
|
6815
|
+
[203, 8],
|
|
6816
|
+
[204, 1],
|
|
6817
|
+
[205, 2],
|
|
6818
|
+
[206, 4],
|
|
6819
|
+
[207, 8],
|
|
6820
|
+
[208, 1],
|
|
6821
|
+
[209, 2],
|
|
6822
|
+
[210, 4],
|
|
6823
|
+
[211, 8],
|
|
6824
|
+
[212, 2],
|
|
6825
|
+
[213, 3],
|
|
6826
|
+
[214, 5],
|
|
6827
|
+
[215, 9],
|
|
6828
|
+
[216, 17]
|
|
6829
|
+
]);
|
|
6830
|
+
function endOfValue(view, start) {
|
|
6831
|
+
let offset = start;
|
|
6832
|
+
let pending = 1;
|
|
6833
|
+
while (pending > 0) {
|
|
6834
|
+
const header = headerAt(view, offset);
|
|
6835
|
+
if (header === null) return null;
|
|
6836
|
+
offset += header.headerBytes + header.bodyBytes;
|
|
6837
|
+
if (offset > view.byteLength) return null;
|
|
6838
|
+
pending += header.children - 1;
|
|
6144
6839
|
}
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6840
|
+
return offset;
|
|
6841
|
+
}
|
|
6842
|
+
function stringKeyAt(bytes, view, offset) {
|
|
6843
|
+
const header = headerAt(view, offset);
|
|
6844
|
+
if (header === null || header.children !== 0) return null;
|
|
6845
|
+
const tag = view.getUint8(offset);
|
|
6846
|
+
const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
6847
|
+
const end = offset + header.headerBytes + header.bodyBytes;
|
|
6848
|
+
if (!isString || end > bytes.length) return null;
|
|
6849
|
+
try {
|
|
6850
|
+
return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
|
|
6851
|
+
} catch {
|
|
6852
|
+
return null;
|
|
6148
6853
|
}
|
|
6149
|
-
return document;
|
|
6150
6854
|
}
|
|
6151
|
-
function
|
|
6152
|
-
const
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6855
|
+
function topLevelMapKeys(bytes) {
|
|
6856
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6857
|
+
const top = headerAt(view, 0);
|
|
6858
|
+
if (top === null || top.bodyBytes !== 0) return null;
|
|
6859
|
+
const tag = view.getUint8(0);
|
|
6860
|
+
if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
|
|
6861
|
+
const keys = /* @__PURE__ */ new Set();
|
|
6862
|
+
let offset = top.headerBytes;
|
|
6863
|
+
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
6864
|
+
const key = stringKeyAt(bytes, view, offset);
|
|
6865
|
+
if (key === null) return null;
|
|
6866
|
+
const valueEnd = endOfValue(view, key.end);
|
|
6867
|
+
if (valueEnd === null) return null;
|
|
6868
|
+
keys.add(key.key);
|
|
6869
|
+
offset = valueEnd;
|
|
6870
|
+
}
|
|
6871
|
+
return keys;
|
|
6872
|
+
}
|
|
6873
|
+
function decodeMapWithoutFields(bytes, skippedFields) {
|
|
6874
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6875
|
+
const top = headerAt(view, 0);
|
|
6876
|
+
if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
|
|
6877
|
+
const tag = view.getUint8(0);
|
|
6878
|
+
const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
|
|
6879
|
+
if (!isMap) return null;
|
|
6880
|
+
const document = {};
|
|
6881
|
+
let offset = top.headerBytes;
|
|
6882
|
+
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
6883
|
+
const key = stringKeyAt(bytes, view, offset);
|
|
6884
|
+
if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
|
|
6885
|
+
const valueEnd = endOfValue(view, key.end);
|
|
6886
|
+
if (valueEnd === null) return null;
|
|
6887
|
+
if (!skippedFields.has(key.key)) {
|
|
6888
|
+
try {
|
|
6889
|
+
document[key.key] = decode(bytes.subarray(key.end, valueEnd));
|
|
6890
|
+
} catch {
|
|
6891
|
+
return null;
|
|
6892
|
+
}
|
|
6893
|
+
}
|
|
6894
|
+
offset = valueEnd;
|
|
6895
|
+
}
|
|
6896
|
+
return offset === bytes.length ? document : null;
|
|
6897
|
+
}
|
|
6898
|
+
|
|
6899
|
+
// src/serialization/msgpack-path.ts
|
|
6900
|
+
var encoder = new TextEncoder();
|
|
6901
|
+
var PROTO_KEY_BYTES = encoder.encode("__proto__");
|
|
6902
|
+
var MISSING = { found: false, value: void 0 };
|
|
6903
|
+
var pathDecoder = new Decoder();
|
|
6904
|
+
function encodePathSegments(segments) {
|
|
6905
|
+
if (segments.length === 0) return null;
|
|
6906
|
+
if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) return null;
|
|
6907
|
+
return segments.map((segment) => encoder.encode(segment));
|
|
6908
|
+
}
|
|
6909
|
+
function isStringTag(tag) {
|
|
6910
|
+
return tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
6911
|
+
}
|
|
6912
|
+
function isMapTag(tag) {
|
|
6913
|
+
return tag >= 128 && tag <= 143 || tag === 222 || tag === 223;
|
|
6914
|
+
}
|
|
6915
|
+
function isArrayTag(tag) {
|
|
6916
|
+
return tag >= 144 && tag <= 159 || tag === 220 || tag === 221;
|
|
6917
|
+
}
|
|
6918
|
+
function isScalarTag(tag) {
|
|
6919
|
+
if (tag <= 127 || tag >= 224 || isStringTag(tag)) return true;
|
|
6920
|
+
return tag === 192 || tag === 194 || tag === 195 || tag >= 202 && tag <= 211;
|
|
6921
|
+
}
|
|
6922
|
+
function sameBytes(bytes, start, end, target) {
|
|
6923
|
+
if (end - start !== target.length) return false;
|
|
6924
|
+
for (let at = 0; at < target.length; at++) {
|
|
6925
|
+
if (bytes[start + at] !== target[at]) return false;
|
|
6926
|
+
}
|
|
6927
|
+
return true;
|
|
6928
|
+
}
|
|
6929
|
+
function keyRangeAt(view, offset) {
|
|
6930
|
+
if (!isStringTag(view.getUint8(offset))) return null;
|
|
6931
|
+
const header = headerAt(view, offset);
|
|
6932
|
+
if (header === null) return null;
|
|
6933
|
+
const start = offset + header.headerBytes;
|
|
6934
|
+
return { start, end: start + header.bodyBytes };
|
|
6935
|
+
}
|
|
6936
|
+
function readEncodedPath(bytes, segments, arrays) {
|
|
6937
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6938
|
+
let start = 0;
|
|
6939
|
+
let end = bytes.length;
|
|
6940
|
+
for (let depth = 0; depth < segments.length; depth++) {
|
|
6941
|
+
if (start >= end) return null;
|
|
6942
|
+
const tag = view.getUint8(start);
|
|
6943
|
+
if (!isMapTag(tag)) {
|
|
6944
|
+
if (depth === 0) return null;
|
|
6945
|
+
if (isArrayTag(tag)) return arrays === "missing" ? MISSING : null;
|
|
6946
|
+
return isScalarTag(tag) ? MISSING : null;
|
|
6947
|
+
}
|
|
6948
|
+
const header = headerAt(view, start);
|
|
6949
|
+
if (header === null) return null;
|
|
6950
|
+
let offset = start + header.headerBytes;
|
|
6951
|
+
let valueStart = -1;
|
|
6952
|
+
let valueEnd = -1;
|
|
6953
|
+
for (let pair = 0; pair < header.children / 2; pair++) {
|
|
6954
|
+
if (offset >= end) return null;
|
|
6955
|
+
const key = keyRangeAt(view, offset);
|
|
6956
|
+
if (key === null || key.end > end) return null;
|
|
6957
|
+
if (sameBytes(bytes, key.start, key.end, PROTO_KEY_BYTES)) return null;
|
|
6958
|
+
const next = endOfValue(view, key.end);
|
|
6959
|
+
if (next === null || next > end) return null;
|
|
6960
|
+
if (sameBytes(bytes, key.start, key.end, segments[depth])) {
|
|
6961
|
+
valueStart = key.end;
|
|
6962
|
+
valueEnd = next;
|
|
6963
|
+
}
|
|
6964
|
+
offset = next;
|
|
6965
|
+
}
|
|
6966
|
+
if (offset !== end) return null;
|
|
6967
|
+
if (valueStart < 0) return MISSING;
|
|
6968
|
+
start = valueStart;
|
|
6969
|
+
end = valueEnd;
|
|
6970
|
+
}
|
|
6971
|
+
try {
|
|
6972
|
+
return { found: true, value: pathDecoder.decode(bytes.subarray(start, end)) };
|
|
6973
|
+
} catch {
|
|
6974
|
+
return null;
|
|
6975
|
+
}
|
|
6976
|
+
}
|
|
6977
|
+
|
|
6978
|
+
// src/core/partition/frozen/document-source.ts
|
|
6979
|
+
var encodedProjectionPaths = /* @__PURE__ */ new WeakMap();
|
|
6980
|
+
function encodedProjectionPathOf(path) {
|
|
6981
|
+
let segments = encodedProjectionPaths.get(path);
|
|
6982
|
+
if (segments === void 0) {
|
|
6983
|
+
segments = encodePathSegments(path);
|
|
6984
|
+
encodedProjectionPaths.set(path, segments);
|
|
6985
|
+
}
|
|
6986
|
+
return segments;
|
|
6987
|
+
}
|
|
6988
|
+
function sharesPrefix(left, right) {
|
|
6989
|
+
const shared = Math.min(left.length, right.length);
|
|
6990
|
+
for (let at = 0; at < shared; at++) {
|
|
6991
|
+
if (left[at] !== right[at]) return false;
|
|
6992
|
+
}
|
|
6993
|
+
return true;
|
|
6994
|
+
}
|
|
6995
|
+
function removeFieldAtPath(document, fieldPath) {
|
|
6996
|
+
const segments = fieldPath.split(".");
|
|
6997
|
+
let current = document;
|
|
6998
|
+
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
6999
|
+
const next = current[segments[i]];
|
|
7000
|
+
if (next === null || typeof next !== "object" || Array.isArray(next)) return;
|
|
7001
|
+
current = next;
|
|
7002
|
+
}
|
|
7003
|
+
delete current[segments[segments.length - 1]];
|
|
7004
|
+
}
|
|
7005
|
+
function decodeStoredDocument(bytes, discardedFields) {
|
|
7006
|
+
let decoded;
|
|
7007
|
+
try {
|
|
7008
|
+
decoded = decode(bytes);
|
|
7009
|
+
} catch (error) {
|
|
7010
|
+
throw new NarsilError(
|
|
7011
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
7012
|
+
`A stored document is not MessagePack: ${error instanceof Error ? error.message : String(error)}`
|
|
7013
|
+
);
|
|
7014
|
+
}
|
|
7015
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded) || decoded instanceof Uint8Array) {
|
|
7016
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "A stored document decodes to something other than a map");
|
|
7017
|
+
}
|
|
7018
|
+
const document = decoded;
|
|
7019
|
+
if (discardedFields !== void 0) {
|
|
7020
|
+
for (const fieldPath of discardedFields) removeFieldAtPath(document, fieldPath);
|
|
7021
|
+
}
|
|
7022
|
+
return document;
|
|
7023
|
+
}
|
|
7024
|
+
function* storedDocumentsOf(data) {
|
|
7025
|
+
const count = data.offsets.length - 1;
|
|
7026
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
7027
|
+
yield decodeStoredDocument(
|
|
7028
|
+
data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
|
|
6156
7029
|
data.discardedFields
|
|
6157
7030
|
);
|
|
6158
7031
|
}
|
|
@@ -6184,25 +7057,40 @@ function encodeDocumentTableData(documents) {
|
|
|
6184
7057
|
function wrapEncodedDocumentTable(data) {
|
|
6185
7058
|
const { blob, offsets } = data;
|
|
6186
7059
|
const count = offsets.length - 1;
|
|
6187
|
-
const
|
|
6188
|
-
const
|
|
6189
|
-
|
|
7060
|
+
const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
7061
|
+
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
7062
|
+
const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
|
|
7063
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
7064
|
+
function projectedPathAt(ordinal, path) {
|
|
7065
|
+
const segments = encodedProjectionPathOf(path);
|
|
7066
|
+
if (segments === null) return { found: false, value: void 0 };
|
|
7067
|
+
const read = touchesDiscarded(path) ? null : readEncodedPath(bytesAt(ordinal), segments, "missing");
|
|
7068
|
+
if (read === null) return readProjectedPath(docAt(ordinal), path);
|
|
7069
|
+
return read.found ? { found: true, value: valueWithOwnBuffers(read.value) } : read;
|
|
7070
|
+
}
|
|
6190
7071
|
return {
|
|
6191
7072
|
count,
|
|
6192
|
-
|
|
6193
|
-
|
|
6194
|
-
},
|
|
6195
|
-
docAt(ordinal) {
|
|
6196
|
-
let document = decoded[ordinal];
|
|
6197
|
-
if (document === void 0) {
|
|
6198
|
-
document = decodeStoredDocument(blob.slice(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
6199
|
-
decoded[ordinal] = document;
|
|
6200
|
-
decodedBytes += documentValueBytes(document);
|
|
6201
|
-
}
|
|
6202
|
-
return document;
|
|
6203
|
-
},
|
|
7073
|
+
byteLength: blob.byteLength + offsets.byteLength,
|
|
7074
|
+
docAt,
|
|
6204
7075
|
readAt(ordinal) {
|
|
6205
|
-
return
|
|
7076
|
+
return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
|
|
7077
|
+
},
|
|
7078
|
+
fieldReader(fieldPath) {
|
|
7079
|
+
const path = fieldPath.split(".");
|
|
7080
|
+
const segments = encodePathSegments(path);
|
|
7081
|
+
if (segments === null) return () => void 0;
|
|
7082
|
+
const decodesDocument = touchesDiscarded(path);
|
|
7083
|
+
return (ordinal) => {
|
|
7084
|
+
const read = decodesDocument ? null : readEncodedPath(bytesAt(ordinal), segments, "decode-document");
|
|
7085
|
+
if (read !== null) return valueWithOwnBuffers(read.value);
|
|
7086
|
+
return getNestedValue(docAt(ordinal), fieldPath);
|
|
7087
|
+
};
|
|
7088
|
+
},
|
|
7089
|
+
projectedAt(ordinal, projection) {
|
|
7090
|
+
if (projection === void 0 || projection.kind === "full") return docAt(ordinal);
|
|
7091
|
+
if (projection.kind === "none") return {};
|
|
7092
|
+
if (projection.include === null) return cloneProjected(docAt(ordinal), projection);
|
|
7093
|
+
return projectIncludedPaths(projection.include, projection.exclude, (path) => projectedPathAt(ordinal, path));
|
|
6206
7094
|
}
|
|
6207
7095
|
};
|
|
6208
7096
|
}
|
|
@@ -6219,6 +7107,12 @@ function wrapDocumentArray(documents) {
|
|
|
6219
7107
|
},
|
|
6220
7108
|
readAt(ordinal) {
|
|
6221
7109
|
return documents[ordinal];
|
|
7110
|
+
},
|
|
7111
|
+
fieldReader(fieldPath) {
|
|
7112
|
+
return (ordinal) => getNestedValue(documents[ordinal], fieldPath);
|
|
7113
|
+
},
|
|
7114
|
+
projectedAt(ordinal, projection) {
|
|
7115
|
+
return cloneProjected(documents[ordinal], projection);
|
|
6222
7116
|
}
|
|
6223
7117
|
};
|
|
6224
7118
|
}
|
|
@@ -6526,9 +7420,13 @@ function createFrozenEnumReader(entry) {
|
|
|
6526
7420
|
}
|
|
6527
7421
|
|
|
6528
7422
|
// src/core/partition/frozen/inverted-reader.ts
|
|
6529
|
-
function createFrozenInvertedReader(table, views) {
|
|
7423
|
+
function createFrozenInvertedReader(table, views, counts) {
|
|
6530
7424
|
function viewAtSorted(sortedIndex) {
|
|
6531
|
-
return views.viewAt(table.payloadSlot(sortedIndex),
|
|
7425
|
+
return views.viewAt(table.payloadSlot(sortedIndex), sortedIndex);
|
|
7426
|
+
}
|
|
7427
|
+
function liveAt(token) {
|
|
7428
|
+
const at = table.find(token);
|
|
7429
|
+
return at < 0 || counts.documentFrequency(at) === 0 ? -1 : at;
|
|
6532
7430
|
}
|
|
6533
7431
|
function candidateRange(prefix) {
|
|
6534
7432
|
if (prefix.length === 0) return { start: 0, end: table.size };
|
|
@@ -6536,22 +7434,22 @@ function createFrozenInvertedReader(table, views) {
|
|
|
6536
7434
|
}
|
|
6537
7435
|
return {
|
|
6538
7436
|
lookup(token) {
|
|
6539
|
-
const at =
|
|
7437
|
+
const at = liveAt(token);
|
|
6540
7438
|
if (at < 0) return void 0;
|
|
6541
7439
|
return viewAtSorted(at);
|
|
6542
7440
|
},
|
|
6543
7441
|
lookupReadOnce(token) {
|
|
6544
|
-
const at =
|
|
7442
|
+
const at = liveAt(token);
|
|
6545
7443
|
if (at < 0) return void 0;
|
|
6546
|
-
return views.viewReadOnce(table.payloadSlot(at),
|
|
7444
|
+
return views.viewReadOnce(table.payloadSlot(at), at);
|
|
6547
7445
|
},
|
|
6548
7446
|
documentFrequency(token) {
|
|
6549
7447
|
const at = table.find(token);
|
|
6550
|
-
return at < 0 ? 0 :
|
|
7448
|
+
return at < 0 ? 0 : counts.documentFrequency(at);
|
|
6551
7449
|
},
|
|
6552
7450
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
6553
7451
|
if (tolerance === 0) {
|
|
6554
|
-
const at =
|
|
7452
|
+
const at = liveAt(token);
|
|
6555
7453
|
return at < 0 ? [] : [{ token, postingList: viewAtSorted(at) }];
|
|
6556
7454
|
}
|
|
6557
7455
|
const prefix = fuzzyPrefixOf(token, prefixLength);
|
|
@@ -6562,7 +7460,7 @@ function createFrozenInvertedReader(table, views) {
|
|
|
6562
7460
|
const candidate = table.tokenAt(at);
|
|
6563
7461
|
if (!candidate.startsWith(requiredPrefix)) continue;
|
|
6564
7462
|
const { withinTolerance } = boundedLevenshtein(token, candidate, tolerance);
|
|
6565
|
-
if (withinTolerance) {
|
|
7463
|
+
if (withinTolerance && counts.documentFrequency(at) > 0) {
|
|
6566
7464
|
results.push({ token: candidate, postingList: viewAtSorted(at) });
|
|
6567
7465
|
}
|
|
6568
7466
|
}
|
|
@@ -6573,14 +7471,15 @@ function createFrozenInvertedReader(table, views) {
|
|
|
6573
7471
|
const range = table.prefixRange(prefix);
|
|
6574
7472
|
const results = [];
|
|
6575
7473
|
for (let at = range.start; at < range.end; at++) {
|
|
6576
|
-
|
|
7474
|
+
const documentFrequency = counts.documentFrequency(at);
|
|
7475
|
+
if (documentFrequency > 0) results.push({ term: table.tokenAt(at), documentFrequency });
|
|
6577
7476
|
}
|
|
6578
7477
|
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
|
|
6579
7478
|
if (results.length > limit) results.length = limit;
|
|
6580
7479
|
return results;
|
|
6581
7480
|
},
|
|
6582
7481
|
has(token) {
|
|
6583
|
-
return
|
|
7482
|
+
return liveAt(token) >= 0;
|
|
6584
7483
|
},
|
|
6585
7484
|
*tokens() {
|
|
6586
7485
|
for (let at = 0; at < table.size; at++) {
|
|
@@ -6592,6 +7491,171 @@ function createFrozenInvertedReader(table, views) {
|
|
|
6592
7491
|
}
|
|
6593
7492
|
};
|
|
6594
7493
|
}
|
|
7494
|
+
function processLimitBytes() {
|
|
7495
|
+
const hostBytes = totalmem();
|
|
7496
|
+
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
7497
|
+
return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
|
|
7498
|
+
}
|
|
7499
|
+
function readResidentMemory() {
|
|
7500
|
+
try {
|
|
7501
|
+
const residentBytes = process.memoryUsage.rss();
|
|
7502
|
+
const limitBytes = processLimitBytes();
|
|
7503
|
+
if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) return null;
|
|
7504
|
+
return { residentBytes, limitBytes };
|
|
7505
|
+
} catch {
|
|
7506
|
+
return null;
|
|
7507
|
+
}
|
|
7508
|
+
}
|
|
7509
|
+
|
|
7510
|
+
// src/core/partition/frozen/live-counts.ts
|
|
7511
|
+
var UINT32_BYTES = 4;
|
|
7512
|
+
var FLOAT64_BYTES = 8;
|
|
7513
|
+
function removalIndexBytes(columns, terms) {
|
|
7514
|
+
const documentWords = 2 * (columns.documentCount + 1);
|
|
7515
|
+
return (columns.postingDocIds.length + documentWords + 2 * terms) * UINT32_BYTES + terms * FLOAT64_BYTES;
|
|
7516
|
+
}
|
|
7517
|
+
function processHasRoomFor(bytes) {
|
|
7518
|
+
const memory = readResidentMemory();
|
|
7519
|
+
if (memory === null) return true;
|
|
7520
|
+
return memory.residentBytes + bytes <= FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION * memory.limitBytes;
|
|
7521
|
+
}
|
|
7522
|
+
function buildRemovalIndex(columns, table) {
|
|
7523
|
+
const { documentCount, postingDocIds } = columns;
|
|
7524
|
+
const starts = new Uint32Array(documentCount + 1);
|
|
7525
|
+
for (let row = 0; row < postingDocIds.length; row++) starts[postingDocIds[row] + 1]++;
|
|
7526
|
+
for (let ordinal = 0; ordinal < documentCount; ordinal++) starts[ordinal + 1] += starts[ordinal];
|
|
7527
|
+
const cursors = starts.slice(0, documentCount);
|
|
7528
|
+
const rows = new Uint32Array(postingDocIds.length);
|
|
7529
|
+
for (let row = 0; row < postingDocIds.length; row++) rows[cursors[postingDocIds[row]]++] = row;
|
|
7530
|
+
const termOfSlot = new Uint32Array(table.size);
|
|
7531
|
+
for (let sortedIndex = 0; sortedIndex < table.size; sortedIndex++) {
|
|
7532
|
+
termOfSlot[table.payloadSlot(sortedIndex)] = sortedIndex;
|
|
7533
|
+
}
|
|
7534
|
+
return {
|
|
7535
|
+
starts,
|
|
7536
|
+
rows,
|
|
7537
|
+
termOfSlot,
|
|
7538
|
+
documents: new Uint32Array(table.size),
|
|
7539
|
+
occurrences: new Float64Array(table.size)
|
|
7540
|
+
};
|
|
7541
|
+
}
|
|
7542
|
+
function tryBuildRemovalIndex(columns, table) {
|
|
7543
|
+
if (!processHasRoomFor(removalIndexBytes(columns, table.size))) return null;
|
|
7544
|
+
try {
|
|
7545
|
+
return buildRemovalIndex(columns, table);
|
|
7546
|
+
} catch (error) {
|
|
7547
|
+
if (error instanceof RangeError) return null;
|
|
7548
|
+
throw error;
|
|
7549
|
+
}
|
|
7550
|
+
}
|
|
7551
|
+
function slotOfRow(postingOffsets, slots, row) {
|
|
7552
|
+
let low = 0;
|
|
7553
|
+
let high = slots;
|
|
7554
|
+
while (low < high) {
|
|
7555
|
+
const middle = low + high >>> 1;
|
|
7556
|
+
if (postingOffsets[middle + 1] <= row) low = middle + 1;
|
|
7557
|
+
else high = middle;
|
|
7558
|
+
}
|
|
7559
|
+
return low;
|
|
7560
|
+
}
|
|
7561
|
+
function createFrozenLiveCounts(columns, table, tombstones, storedFrequencies) {
|
|
7562
|
+
const { postingOffsets, postingDocIds, postingFrequencies } = columns;
|
|
7563
|
+
let index = null;
|
|
7564
|
+
let lengths = null;
|
|
7565
|
+
let frequencies = null;
|
|
7566
|
+
let frequenciesRevision = -1;
|
|
7567
|
+
function countRemoval(built, ordinal) {
|
|
7568
|
+
const { starts, rows, termOfSlot, documents, occurrences } = built;
|
|
7569
|
+
let previousSlot = -1;
|
|
7570
|
+
for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
|
|
7571
|
+
const row = rows[at];
|
|
7572
|
+
const slot = slotOfRow(postingOffsets, table.size, row);
|
|
7573
|
+
const sortedIndex = termOfSlot[slot];
|
|
7574
|
+
if (slot !== previousSlot) documents[sortedIndex]++;
|
|
7575
|
+
occurrences[sortedIndex] += postingFrequencies[row];
|
|
7576
|
+
previousSlot = slot;
|
|
7577
|
+
}
|
|
7578
|
+
}
|
|
7579
|
+
function removedDocumentsOf(sortedIndex) {
|
|
7580
|
+
const slot = table.payloadSlot(sortedIndex);
|
|
7581
|
+
const end = postingOffsets[slot + 1];
|
|
7582
|
+
let removed = 0;
|
|
7583
|
+
let previous = -1;
|
|
7584
|
+
let ordered = true;
|
|
7585
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
7586
|
+
const ordinal = postingDocIds[row];
|
|
7587
|
+
if (ordinal < previous) ordered = false;
|
|
7588
|
+
if (ordinal !== previous && tombstones.has(ordinal)) removed++;
|
|
7589
|
+
previous = ordinal;
|
|
7590
|
+
}
|
|
7591
|
+
if (ordered) return removed;
|
|
7592
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7593
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
7594
|
+
const ordinal = postingDocIds[row];
|
|
7595
|
+
if (tombstones.has(ordinal)) seen.add(ordinal);
|
|
7596
|
+
}
|
|
7597
|
+
return seen.size;
|
|
7598
|
+
}
|
|
7599
|
+
function removedOccurrencesOf(sortedIndex) {
|
|
7600
|
+
const slot = table.payloadSlot(sortedIndex);
|
|
7601
|
+
const end = postingOffsets[slot + 1];
|
|
7602
|
+
let removed = 0;
|
|
7603
|
+
for (let row = postingOffsets[slot]; row < end; row++) {
|
|
7604
|
+
if (tombstones.has(postingDocIds[row])) removed += postingFrequencies[row];
|
|
7605
|
+
}
|
|
7606
|
+
return removed;
|
|
7607
|
+
}
|
|
7608
|
+
function documentFrequency(sortedIndex) {
|
|
7609
|
+
const stored = table.documentFrequencyAt(sortedIndex);
|
|
7610
|
+
if (tombstones.size === 0) return stored;
|
|
7611
|
+
if (index !== null) return stored - index.documents[sortedIndex];
|
|
7612
|
+
return stored - removedDocumentsOf(sortedIndex);
|
|
7613
|
+
}
|
|
7614
|
+
return {
|
|
7615
|
+
totalDocuments() {
|
|
7616
|
+
return columns.documentCount - tombstones.size;
|
|
7617
|
+
},
|
|
7618
|
+
totalFieldLengths() {
|
|
7619
|
+
return lengths ?? columns.totalFieldLengths;
|
|
7620
|
+
},
|
|
7621
|
+
documentFrequency,
|
|
7622
|
+
removedOccurrences(sortedIndex) {
|
|
7623
|
+
if (tombstones.size === 0) return 0;
|
|
7624
|
+
if (index !== null) return index.occurrences[sortedIndex];
|
|
7625
|
+
return removedOccurrencesOf(sortedIndex);
|
|
7626
|
+
},
|
|
7627
|
+
docFrequencies() {
|
|
7628
|
+
if (tombstones.size === 0) return storedFrequencies();
|
|
7629
|
+
if (frequencies !== null && frequenciesRevision === tombstones.revision) return frequencies;
|
|
7630
|
+
const live = /* @__PURE__ */ Object.create(null);
|
|
7631
|
+
for (let at = 0; at < table.size; at++) {
|
|
7632
|
+
const documents = documentFrequency(at);
|
|
7633
|
+
if (documents > 0) live[table.tokenAt(at)] = documents;
|
|
7634
|
+
}
|
|
7635
|
+
frequencies = live;
|
|
7636
|
+
frequenciesRevision = tombstones.revision;
|
|
7637
|
+
return frequencies;
|
|
7638
|
+
},
|
|
7639
|
+
subtractRemoved(ordinal) {
|
|
7640
|
+
lengths ??= { ...columns.totalFieldLengths };
|
|
7641
|
+
for (let field = 0; field < columns.fieldLengthNames.length; field++) {
|
|
7642
|
+
lengths[columns.fieldLengthNames[field]] -= columns.fieldLengthColumns[field][ordinal];
|
|
7643
|
+
}
|
|
7644
|
+
if (index !== null) {
|
|
7645
|
+
countRemoval(index, ordinal);
|
|
7646
|
+
return;
|
|
7647
|
+
}
|
|
7648
|
+
const built = tryBuildRemovalIndex(columns, table);
|
|
7649
|
+
if (built === null) return;
|
|
7650
|
+
for (const removed of tombstones.values()) countRemoval(built, removed);
|
|
7651
|
+
index = built;
|
|
7652
|
+
},
|
|
7653
|
+
bytes() {
|
|
7654
|
+
if (index === null) return 0;
|
|
7655
|
+
return index.starts.byteLength + index.rows.byteLength + index.termOfSlot.byteLength + index.documents.byteLength + index.occurrences.byteLength;
|
|
7656
|
+
}
|
|
7657
|
+
};
|
|
7658
|
+
}
|
|
6595
7659
|
|
|
6596
7660
|
// src/core/partition/frozen/posting-views.ts
|
|
6597
7661
|
function isOrdered(docIds) {
|
|
@@ -6605,16 +7669,16 @@ function sumFrequencies(frequencies) {
|
|
|
6605
7669
|
for (let i = 0; i < frequencies.length; i++) total += frequencies[i];
|
|
6606
7670
|
return total;
|
|
6607
7671
|
}
|
|
6608
|
-
function createFrozenPostingViews(payload, tombstones) {
|
|
7672
|
+
function createFrozenPostingViews(payload, tombstones, counts) {
|
|
6609
7673
|
const views = /* @__PURE__ */ new Map();
|
|
6610
|
-
function buildView(payloadSlot,
|
|
7674
|
+
function buildView(payloadSlot, sortedIndex, keepsPositions) {
|
|
6611
7675
|
const start = payload.postingOffsets[payloadSlot];
|
|
6612
7676
|
const end = payload.postingOffsets[payloadSlot + 1];
|
|
6613
7677
|
const docIds = payload.postingDocIds.subarray(start, end);
|
|
6614
7678
|
const termFrequencies = payload.postingFrequencies.subarray(start, end);
|
|
6615
7679
|
const fieldNameIndices = payload.postingFieldIndices.subarray(start, end);
|
|
6616
7680
|
const ordered = isOrdered(docIds);
|
|
6617
|
-
const
|
|
7681
|
+
const storedTermFrequency = sumFrequencies(termFrequencies);
|
|
6618
7682
|
const { positionOffsets, positionValues } = payload;
|
|
6619
7683
|
let materializedPositions = null;
|
|
6620
7684
|
return {
|
|
@@ -6636,9 +7700,15 @@ function createFrozenPostingViews(payload, tombstones) {
|
|
|
6636
7700
|
if (positionOffsets === null) return 0;
|
|
6637
7701
|
return positionOffsets[start + row + 1] - positionOffsets[start + row];
|
|
6638
7702
|
},
|
|
6639
|
-
docIdSet: {
|
|
7703
|
+
docIdSet: {
|
|
7704
|
+
get size() {
|
|
7705
|
+
return counts.documentFrequency(sortedIndex);
|
|
7706
|
+
}
|
|
7707
|
+
},
|
|
6640
7708
|
deletedDocs: tombstones,
|
|
6641
|
-
totalTermFrequency
|
|
7709
|
+
get totalTermFrequency() {
|
|
7710
|
+
return storedTermFrequency - counts.removedOccurrences(sortedIndex);
|
|
7711
|
+
},
|
|
6642
7712
|
get structureRevision() {
|
|
6643
7713
|
return tombstones.revision;
|
|
6644
7714
|
},
|
|
@@ -6646,16 +7716,138 @@ function createFrozenPostingViews(payload, tombstones) {
|
|
|
6646
7716
|
};
|
|
6647
7717
|
}
|
|
6648
7718
|
return {
|
|
6649
|
-
viewAt(payloadSlot,
|
|
7719
|
+
viewAt(payloadSlot, sortedIndex) {
|
|
6650
7720
|
let view = views.get(payloadSlot);
|
|
6651
7721
|
if (view === void 0) {
|
|
6652
|
-
view = buildView(payloadSlot,
|
|
7722
|
+
view = buildView(payloadSlot, sortedIndex, true);
|
|
6653
7723
|
views.set(payloadSlot, view);
|
|
6654
7724
|
}
|
|
6655
7725
|
return view;
|
|
6656
7726
|
},
|
|
6657
|
-
viewReadOnce(payloadSlot,
|
|
6658
|
-
return buildView(payloadSlot,
|
|
7727
|
+
viewReadOnce(payloadSlot, sortedIndex) {
|
|
7728
|
+
return buildView(payloadSlot, sortedIndex, false);
|
|
7729
|
+
}
|
|
7730
|
+
};
|
|
7731
|
+
}
|
|
7732
|
+
|
|
7733
|
+
// src/core/partition/frozen/removal-pass.ts
|
|
7734
|
+
var NO_REMOVED_SPELLINGS = [];
|
|
7735
|
+
function createFrozenRemovalPass(documents, surfaces) {
|
|
7736
|
+
let bound = null;
|
|
7737
|
+
let pending = [];
|
|
7738
|
+
function heldSpellingsOf(ordinal, analysis) {
|
|
7739
|
+
const spellings = [];
|
|
7740
|
+
if (!surfaces.holdsAny()) return spellings;
|
|
7741
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), analysis, (spelling) => {
|
|
7742
|
+
if (surfaces.holds(spelling)) spellings.push(spelling);
|
|
7743
|
+
});
|
|
7744
|
+
return spellings;
|
|
7745
|
+
}
|
|
7746
|
+
return {
|
|
7747
|
+
get settled() {
|
|
7748
|
+
return pending.length === 0;
|
|
7749
|
+
},
|
|
7750
|
+
bind(analysis) {
|
|
7751
|
+
if (bound !== null) return;
|
|
7752
|
+
const scoped = { ...analysis, fields: documents.textFields };
|
|
7753
|
+
bound = scoped;
|
|
7754
|
+
for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
|
|
7755
|
+
pending = [];
|
|
7756
|
+
},
|
|
7757
|
+
spellingsOf(ordinal) {
|
|
7758
|
+
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
7759
|
+
},
|
|
7760
|
+
afterRemoval(ordinal, spellings) {
|
|
7761
|
+
if (spellings !== void 0) {
|
|
7762
|
+
surfaces.subtractSpellings(ordinal, spellings);
|
|
7763
|
+
} else if (bound !== null) {
|
|
7764
|
+
surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
|
|
7765
|
+
} else {
|
|
7766
|
+
pending.push(ordinal);
|
|
7767
|
+
}
|
|
7768
|
+
}
|
|
7769
|
+
};
|
|
7770
|
+
}
|
|
7771
|
+
|
|
7772
|
+
// src/core/partition/frozen/surface-slot.ts
|
|
7773
|
+
function applyDocumentSurfaces(target, document, analysis, direction) {
|
|
7774
|
+
if (direction === -1) {
|
|
7775
|
+
visitDocumentSurfaceSpellings(document, analysis, (spelling) => target.subtract(spelling, 1));
|
|
7776
|
+
return;
|
|
7777
|
+
}
|
|
7778
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => {
|
|
7779
|
+
if (spelling !== token) target.add(spelling, token, 1);
|
|
7780
|
+
});
|
|
7781
|
+
}
|
|
7782
|
+
function addForms(target, forms) {
|
|
7783
|
+
for (const surface of Object.keys(forms)) {
|
|
7784
|
+
const value = forms[surface];
|
|
7785
|
+
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
7786
|
+
}
|
|
7787
|
+
}
|
|
7788
|
+
function createFrozenSurfaceSlot(load, documents) {
|
|
7789
|
+
let registry = null;
|
|
7790
|
+
let uncounted = null;
|
|
7791
|
+
function loaded() {
|
|
7792
|
+
if (registry === null) {
|
|
7793
|
+
const built = createSurfaceRegistry();
|
|
7794
|
+
const forms = load();
|
|
7795
|
+
if (forms !== null) built.deserialize(forms);
|
|
7796
|
+
registry = built;
|
|
7797
|
+
}
|
|
7798
|
+
return registry;
|
|
7799
|
+
}
|
|
7800
|
+
function isCounted(ordinal) {
|
|
7801
|
+
return uncounted === null || uncounted[ordinal] === 0;
|
|
7802
|
+
}
|
|
7803
|
+
return {
|
|
7804
|
+
reader: {
|
|
7805
|
+
candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
|
|
7806
|
+
stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
|
|
7807
|
+
size: () => loaded().size(),
|
|
7808
|
+
serialize: () => loaded().serialize()
|
|
7809
|
+
},
|
|
7810
|
+
holdsAny() {
|
|
7811
|
+
return loaded().size() > 0;
|
|
7812
|
+
},
|
|
7813
|
+
holds(spelling) {
|
|
7814
|
+
return loaded().holds(spelling);
|
|
7815
|
+
},
|
|
7816
|
+
subtractSpellings(ordinal, spellings) {
|
|
7817
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
7818
|
+
uncounted[ordinal] = 1;
|
|
7819
|
+
if (spellings.length === 0) return;
|
|
7820
|
+
const target = loaded();
|
|
7821
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
7822
|
+
},
|
|
7823
|
+
addKeptForms(target, mergedOrdinals, analysis) {
|
|
7824
|
+
const source = loaded();
|
|
7825
|
+
if (analysis === null) {
|
|
7826
|
+
addForms(target, source.serialize());
|
|
7827
|
+
return;
|
|
7828
|
+
}
|
|
7829
|
+
const scoped = { ...analysis, fields: documents.textFields };
|
|
7830
|
+
let kept = 0;
|
|
7831
|
+
let corrections = 0;
|
|
7832
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
7833
|
+
const keeps = mergedOrdinals[ordinal] >= 0;
|
|
7834
|
+
if (keeps) kept++;
|
|
7835
|
+
if (keeps !== isCounted(ordinal)) corrections++;
|
|
7836
|
+
}
|
|
7837
|
+
if (corrections > kept) {
|
|
7838
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
7839
|
+
if (mergedOrdinals[ordinal] >= 0) applyDocumentSurfaces(target, documents.documentAt(ordinal), scoped, 1);
|
|
7840
|
+
}
|
|
7841
|
+
return;
|
|
7842
|
+
}
|
|
7843
|
+
const corrected = createSurfaceRegistry();
|
|
7844
|
+
addForms(corrected, source.serialize());
|
|
7845
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
7846
|
+
const keeps = mergedOrdinals[ordinal] >= 0;
|
|
7847
|
+
if (keeps === isCounted(ordinal)) continue;
|
|
7848
|
+
applyDocumentSurfaces(corrected, documents.documentAt(ordinal), scoped, keeps ? 1 : -1);
|
|
7849
|
+
}
|
|
7850
|
+
addForms(target, corrected.serialize());
|
|
6659
7851
|
}
|
|
6660
7852
|
};
|
|
6661
7853
|
}
|
|
@@ -6683,27 +7875,9 @@ function decodeSurfaceTable(table) {
|
|
|
6683
7875
|
}
|
|
6684
7876
|
return forms;
|
|
6685
7877
|
}
|
|
6686
|
-
function createLazySurfaceReader(load) {
|
|
6687
|
-
let registry = null;
|
|
6688
|
-
function loaded() {
|
|
6689
|
-
if (registry === null) {
|
|
6690
|
-
const built = createSurfaceRegistry();
|
|
6691
|
-
const forms = load();
|
|
6692
|
-
if (forms !== null) built.deserialize(forms);
|
|
6693
|
-
registry = built;
|
|
6694
|
-
}
|
|
6695
|
-
return registry;
|
|
6696
|
-
}
|
|
6697
|
-
return {
|
|
6698
|
-
candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
|
|
6699
|
-
stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
|
|
6700
|
-
size: () => loaded().size(),
|
|
6701
|
-
serialize: () => loaded().serialize()
|
|
6702
|
-
};
|
|
6703
|
-
}
|
|
6704
7878
|
|
|
6705
7879
|
// src/core/partition/frozen/token-table.ts
|
|
6706
|
-
var
|
|
7880
|
+
var encoder2 = new TextEncoder();
|
|
6707
7881
|
var decoder3 = new TextDecoder();
|
|
6708
7882
|
function compareBytes2(blob, start, end, query) {
|
|
6709
7883
|
const length = end - start;
|
|
@@ -6790,7 +7964,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
6790
7964
|
return documentFrequencies[sortedIndex];
|
|
6791
7965
|
},
|
|
6792
7966
|
prefixRange(prefix) {
|
|
6793
|
-
const bytes =
|
|
7967
|
+
const bytes = encoder2.encode(prefix);
|
|
6794
7968
|
const start = lowerBound3(bytes);
|
|
6795
7969
|
return { start, end: prefixEnd(start, bytes) };
|
|
6796
7970
|
},
|
|
@@ -6808,8 +7982,8 @@ function wrapFrozenTokenTable(data) {
|
|
|
6808
7982
|
fromCodePoint = unit;
|
|
6809
7983
|
toCodePoint = nextCodePoint(unit);
|
|
6810
7984
|
}
|
|
6811
|
-
const from =
|
|
6812
|
-
const to =
|
|
7985
|
+
const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
|
|
7986
|
+
const to = encoder2.encode(String.fromCodePoint(toCodePoint));
|
|
6813
7987
|
return { start: lowerBound3(from), end: lowerBound3(to) };
|
|
6814
7988
|
}
|
|
6815
7989
|
};
|
|
@@ -6821,6 +7995,7 @@ function buildFrozenTokenTable(tokens, docFrequencies) {
|
|
|
6821
7995
|
// src/core/partition/frozen/tombstones.ts
|
|
6822
7996
|
function createFrozenTombstones() {
|
|
6823
7997
|
const removed = /* @__PURE__ */ new Set();
|
|
7998
|
+
const arrivals = [];
|
|
6824
7999
|
let revision = 0;
|
|
6825
8000
|
return {
|
|
6826
8001
|
get size() {
|
|
@@ -6835,9 +8010,13 @@ function createFrozenTombstones() {
|
|
|
6835
8010
|
add(internalId) {
|
|
6836
8011
|
if (removed.has(internalId)) return false;
|
|
6837
8012
|
removed.add(internalId);
|
|
8013
|
+
arrivals.push(internalId);
|
|
6838
8014
|
revision++;
|
|
6839
8015
|
return true;
|
|
6840
8016
|
},
|
|
8017
|
+
at(arrival) {
|
|
8018
|
+
return arrivals[arrival];
|
|
8019
|
+
},
|
|
6841
8020
|
values() {
|
|
6842
8021
|
return removed.values();
|
|
6843
8022
|
}
|
|
@@ -7008,26 +8187,36 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
|
|
|
7008
8187
|
}
|
|
7009
8188
|
|
|
7010
8189
|
// src/core/partition/frozen/index.ts
|
|
7011
|
-
function buildStatsView(
|
|
7012
|
-
const averageFieldLengths = {};
|
|
7013
|
-
if (source.documentCount > 0) {
|
|
7014
|
-
for (const [fieldName, total] of Object.entries(source.totalFieldLengths)) {
|
|
7015
|
-
averageFieldLengths[fieldName] = total / source.documentCount;
|
|
7016
|
-
}
|
|
7017
|
-
}
|
|
7018
|
-
let materializedFrequencies = null;
|
|
8190
|
+
function buildStatsView(counts) {
|
|
7019
8191
|
return {
|
|
7020
|
-
totalDocuments
|
|
7021
|
-
|
|
7022
|
-
|
|
7023
|
-
get
|
|
7024
|
-
|
|
7025
|
-
|
|
8192
|
+
get totalDocuments() {
|
|
8193
|
+
return counts.totalDocuments();
|
|
8194
|
+
},
|
|
8195
|
+
get totalFieldLengths() {
|
|
8196
|
+
return counts.totalFieldLengths();
|
|
8197
|
+
},
|
|
8198
|
+
get averageFieldLengths() {
|
|
8199
|
+
const documents = counts.totalDocuments();
|
|
8200
|
+
const averages = {};
|
|
8201
|
+
if (documents > 0) {
|
|
8202
|
+
for (const [fieldName, total] of Object.entries(counts.totalFieldLengths())) {
|
|
8203
|
+
averages[fieldName] = total / documents;
|
|
8204
|
+
}
|
|
7026
8205
|
}
|
|
7027
|
-
return
|
|
8206
|
+
return averages;
|
|
8207
|
+
},
|
|
8208
|
+
get docFrequencies() {
|
|
8209
|
+
return counts.docFrequencies();
|
|
7028
8210
|
}
|
|
7029
8211
|
};
|
|
7030
8212
|
}
|
|
8213
|
+
function storedFrequenciesOf(source) {
|
|
8214
|
+
let materialized = null;
|
|
8215
|
+
return () => {
|
|
8216
|
+
if (materialized === null) materialized = source.docFrequencies();
|
|
8217
|
+
return materialized;
|
|
8218
|
+
};
|
|
8219
|
+
}
|
|
7031
8220
|
function buildGeoReaders(entries) {
|
|
7032
8221
|
const readers = /* @__PURE__ */ new Map();
|
|
7033
8222
|
for (const entry of entries) {
|
|
@@ -7039,9 +8228,10 @@ function buildGeoReaders(entries) {
|
|
|
7039
8228
|
}
|
|
7040
8229
|
return readers;
|
|
7041
8230
|
}
|
|
7042
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
8231
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
7043
8232
|
const tombstones = createFrozenTombstones();
|
|
7044
|
-
const
|
|
8233
|
+
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
8234
|
+
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
7045
8235
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
7046
8236
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
7047
8237
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -7052,6 +8242,14 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
7052
8242
|
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
7053
8243
|
isTombstoned: (ordinal) => tombstones.has(ordinal)
|
|
7054
8244
|
});
|
|
8245
|
+
const textDocuments = {
|
|
8246
|
+
count: documentSource.count,
|
|
8247
|
+
textFields: source.fieldLengthNames,
|
|
8248
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
8249
|
+
};
|
|
8250
|
+
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
8251
|
+
const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
|
|
8252
|
+
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
7055
8253
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
7056
8254
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
7057
8255
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -7073,10 +8271,12 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
7073
8271
|
documentSource,
|
|
7074
8272
|
sharedSnapshot,
|
|
7075
8273
|
arrays: source,
|
|
7076
|
-
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews),
|
|
8274
|
+
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
7077
8275
|
docStore,
|
|
7078
|
-
stats: buildStatsView(
|
|
7079
|
-
surfaceRegistry:
|
|
8276
|
+
stats: buildStatsView(counts),
|
|
8277
|
+
surfaceRegistry: surfaces.reader,
|
|
8278
|
+
surfaces,
|
|
8279
|
+
removals,
|
|
7080
8280
|
numericIndexes,
|
|
7081
8281
|
booleanIndexes,
|
|
7082
8282
|
enumIndexes,
|
|
@@ -7099,15 +8299,21 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
7099
8299
|
isTombstoned(ordinal) {
|
|
7100
8300
|
return tombstones.has(ordinal);
|
|
7101
8301
|
},
|
|
7102
|
-
|
|
8302
|
+
removedSpellingsOf(docId) {
|
|
8303
|
+
const ordinal = docStore.getInternalId(docId);
|
|
8304
|
+
if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
|
|
8305
|
+
return removals.spellingsOf(ordinal);
|
|
8306
|
+
},
|
|
8307
|
+
tombstoneDocument(docId, removedSpellings) {
|
|
7103
8308
|
const ordinal = docStore.getInternalId(docId);
|
|
7104
8309
|
if (ordinal === void 0) return false;
|
|
7105
8310
|
const added = tombstones.add(ordinal);
|
|
7106
8311
|
if (added) {
|
|
8312
|
+
counts.subtractRemoved(ordinal);
|
|
7107
8313
|
partialWords.afterRemoval(ordinal);
|
|
8314
|
+
removals.afterRemoval(ordinal, removedSpellings);
|
|
7108
8315
|
docStore.releaseSortedDocIds();
|
|
7109
|
-
segment.sortColumns
|
|
7110
|
-
segment.scoreBuffer = null;
|
|
8316
|
+
segment.sortColumns?.forget(ordinal);
|
|
7111
8317
|
}
|
|
7112
8318
|
return added;
|
|
7113
8319
|
},
|
|
@@ -7115,6 +8321,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
7115
8321
|
const docIds = [];
|
|
7116
8322
|
for (const ordinal of tombstones.values()) docIds.push(source.idTable.idAt(ordinal));
|
|
7117
8323
|
return docIds;
|
|
8324
|
+
},
|
|
8325
|
+
removalBytes() {
|
|
8326
|
+
return counts.bytes();
|
|
7118
8327
|
}
|
|
7119
8328
|
};
|
|
7120
8329
|
return segment;
|
|
@@ -7136,7 +8345,7 @@ function createFrozenSegment(payload, documents, segmentId) {
|
|
|
7136
8345
|
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
7137
8346
|
);
|
|
7138
8347
|
}
|
|
7139
|
-
function createSharedFrozenSegment(snapshot) {
|
|
8348
|
+
function createSharedFrozenSegment(snapshot, survivingSurfaceForms) {
|
|
7140
8349
|
const tokenTable = wrapFrozenTokenTable(snapshot.tokenTable);
|
|
7141
8350
|
return assembleFrozenSegment(
|
|
7142
8351
|
snapshot.segmentId,
|
|
@@ -7156,136 +8365,10 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
7156
8365
|
},
|
|
7157
8366
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
7158
8367
|
snapshot,
|
|
7159
|
-
snapshot.partialWords
|
|
8368
|
+
snapshot.partialWords,
|
|
8369
|
+
survivingSurfaceForms
|
|
7160
8370
|
);
|
|
7161
8371
|
}
|
|
7162
|
-
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
7163
|
-
var keyDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
7164
|
-
function headerAt(view, offset) {
|
|
7165
|
-
if (offset >= view.byteLength) return null;
|
|
7166
|
-
const tag = view.getUint8(offset);
|
|
7167
|
-
if (tag <= 127 || tag >= 224) return { headerBytes: 1, bodyBytes: 0, children: 0 };
|
|
7168
|
-
if (tag <= 143) return { headerBytes: 1, bodyBytes: 0, children: (tag & 15) * 2 };
|
|
7169
|
-
if (tag <= 159) return { headerBytes: 1, bodyBytes: 0, children: tag & 15 };
|
|
7170
|
-
if (tag <= 191) return { headerBytes: 1, bodyBytes: tag & 31, children: 0 };
|
|
7171
|
-
const lengthBytes = LENGTH_BYTES_BY_TAG.get(tag);
|
|
7172
|
-
if (lengthBytes !== void 0) {
|
|
7173
|
-
if (offset + 1 + lengthBytes > view.byteLength) return null;
|
|
7174
|
-
const length = lengthBytes === 1 ? view.getUint8(offset + 1) : lengthBytes === 2 ? view.getUint16(offset + 1, false) : view.getUint32(offset + 1, false);
|
|
7175
|
-
const headerBytes = 1 + lengthBytes;
|
|
7176
|
-
if (ARRAY_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length };
|
|
7177
|
-
if (MAP_TAGS.has(tag)) return { headerBytes, bodyBytes: 0, children: length * 2 };
|
|
7178
|
-
return { headerBytes, bodyBytes: length + (EXT_TAGS.has(tag) ? 1 : 0), children: 0 };
|
|
7179
|
-
}
|
|
7180
|
-
const fixedBytes = FIXED_BODY_BYTES_BY_TAG.get(tag);
|
|
7181
|
-
return fixedBytes === void 0 ? null : { headerBytes: 1, bodyBytes: fixedBytes, children: 0 };
|
|
7182
|
-
}
|
|
7183
|
-
var LENGTH_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
7184
|
-
[196, 1],
|
|
7185
|
-
[197, 2],
|
|
7186
|
-
[198, 4],
|
|
7187
|
-
[199, 1],
|
|
7188
|
-
[200, 2],
|
|
7189
|
-
[201, 4],
|
|
7190
|
-
[217, 1],
|
|
7191
|
-
[218, 2],
|
|
7192
|
-
[219, 4],
|
|
7193
|
-
[220, 2],
|
|
7194
|
-
[221, 4],
|
|
7195
|
-
[222, 2],
|
|
7196
|
-
[223, 4]
|
|
7197
|
-
]);
|
|
7198
|
-
var ARRAY_TAGS = /* @__PURE__ */ new Set([220, 221]);
|
|
7199
|
-
var MAP_TAGS = /* @__PURE__ */ new Set([222, 223]);
|
|
7200
|
-
var EXT_TAGS = /* @__PURE__ */ new Set([199, 200, 201]);
|
|
7201
|
-
var FIXED_BODY_BYTES_BY_TAG = /* @__PURE__ */ new Map([
|
|
7202
|
-
[192, 0],
|
|
7203
|
-
[194, 0],
|
|
7204
|
-
[195, 0],
|
|
7205
|
-
[202, 4],
|
|
7206
|
-
[203, 8],
|
|
7207
|
-
[204, 1],
|
|
7208
|
-
[205, 2],
|
|
7209
|
-
[206, 4],
|
|
7210
|
-
[207, 8],
|
|
7211
|
-
[208, 1],
|
|
7212
|
-
[209, 2],
|
|
7213
|
-
[210, 4],
|
|
7214
|
-
[211, 8],
|
|
7215
|
-
[212, 2],
|
|
7216
|
-
[213, 3],
|
|
7217
|
-
[214, 5],
|
|
7218
|
-
[215, 9],
|
|
7219
|
-
[216, 17]
|
|
7220
|
-
]);
|
|
7221
|
-
function endOfValue(view, start) {
|
|
7222
|
-
let offset = start;
|
|
7223
|
-
let pending = 1;
|
|
7224
|
-
while (pending > 0) {
|
|
7225
|
-
const header = headerAt(view, offset);
|
|
7226
|
-
if (header === null) return null;
|
|
7227
|
-
offset += header.headerBytes + header.bodyBytes;
|
|
7228
|
-
if (offset > view.byteLength) return null;
|
|
7229
|
-
pending += header.children - 1;
|
|
7230
|
-
}
|
|
7231
|
-
return offset;
|
|
7232
|
-
}
|
|
7233
|
-
function stringKeyAt(bytes, view, offset) {
|
|
7234
|
-
const header = headerAt(view, offset);
|
|
7235
|
-
if (header === null || header.children !== 0) return null;
|
|
7236
|
-
const tag = view.getUint8(offset);
|
|
7237
|
-
const isString = tag >= 160 && tag <= 191 || tag === 217 || tag === 218 || tag === 219;
|
|
7238
|
-
const end = offset + header.headerBytes + header.bodyBytes;
|
|
7239
|
-
if (!isString || end > bytes.length) return null;
|
|
7240
|
-
try {
|
|
7241
|
-
return { key: keyDecoder.decode(bytes.subarray(offset + header.headerBytes, end)), end };
|
|
7242
|
-
} catch {
|
|
7243
|
-
return null;
|
|
7244
|
-
}
|
|
7245
|
-
}
|
|
7246
|
-
function topLevelMapKeys(bytes) {
|
|
7247
|
-
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
7248
|
-
const top = headerAt(view, 0);
|
|
7249
|
-
if (top === null || top.bodyBytes !== 0) return null;
|
|
7250
|
-
const tag = view.getUint8(0);
|
|
7251
|
-
if (!(tag >= 128 && tag <= 143 || MAP_TAGS.has(tag))) return null;
|
|
7252
|
-
const keys = /* @__PURE__ */ new Set();
|
|
7253
|
-
let offset = top.headerBytes;
|
|
7254
|
-
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
7255
|
-
const key = stringKeyAt(bytes, view, offset);
|
|
7256
|
-
if (key === null) return null;
|
|
7257
|
-
const valueEnd = endOfValue(view, key.end);
|
|
7258
|
-
if (valueEnd === null) return null;
|
|
7259
|
-
keys.add(key.key);
|
|
7260
|
-
offset = valueEnd;
|
|
7261
|
-
}
|
|
7262
|
-
return keys;
|
|
7263
|
-
}
|
|
7264
|
-
function decodeMapWithoutFields(bytes, skippedFields) {
|
|
7265
|
-
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
7266
|
-
const top = headerAt(view, 0);
|
|
7267
|
-
if (top === null || top.children === 0 || top.bodyBytes !== 0) return null;
|
|
7268
|
-
const tag = view.getUint8(0);
|
|
7269
|
-
const isMap = tag >= 128 && tag <= 143 || MAP_TAGS.has(tag);
|
|
7270
|
-
if (!isMap) return null;
|
|
7271
|
-
const document = {};
|
|
7272
|
-
let offset = top.headerBytes;
|
|
7273
|
-
for (let pair = 0; pair < top.children / 2; pair += 1) {
|
|
7274
|
-
const key = stringKeyAt(bytes, view, offset);
|
|
7275
|
-
if (key === null || FORBIDDEN_KEYS.has(key.key)) return null;
|
|
7276
|
-
const valueEnd = endOfValue(view, key.end);
|
|
7277
|
-
if (valueEnd === null) return null;
|
|
7278
|
-
if (!skippedFields.has(key.key)) {
|
|
7279
|
-
try {
|
|
7280
|
-
document[key.key] = decode(bytes.subarray(key.end, valueEnd));
|
|
7281
|
-
} catch {
|
|
7282
|
-
return null;
|
|
7283
|
-
}
|
|
7284
|
-
}
|
|
7285
|
-
offset = valueEnd;
|
|
7286
|
-
}
|
|
7287
|
-
return offset === bytes.length ? document : null;
|
|
7288
|
-
}
|
|
7289
8372
|
|
|
7290
8373
|
// src/core/partition/frozen/merge-fields.ts
|
|
7291
8374
|
function indexByFieldPath(entries) {
|
|
@@ -7677,16 +8760,9 @@ function mergeFieldLengths(inputs, documentCount) {
|
|
|
7677
8760
|
}
|
|
7678
8761
|
return { fieldLengthNames: names, fieldLengthColumns: columns, totalFieldLengths: totals };
|
|
7679
8762
|
}
|
|
7680
|
-
function mergeSurfaceForms2(inputs) {
|
|
8763
|
+
function mergeSurfaceForms2(inputs, analysis) {
|
|
7681
8764
|
const registry = createSurfaceRegistry();
|
|
7682
|
-
for (const input of inputs)
|
|
7683
|
-
const forms = input.segment.arrays.readSurfaceForms();
|
|
7684
|
-
if (forms === null) continue;
|
|
7685
|
-
for (const surface of Object.keys(forms)) {
|
|
7686
|
-
const value = forms[surface];
|
|
7687
|
-
if (Array.isArray(value)) registry.add(surface, value[1], value[0]);
|
|
7688
|
-
}
|
|
7689
|
-
}
|
|
8765
|
+
for (const input of inputs) input.segment.surfaces.addKeptForms(registry, input.remap, analysis);
|
|
7690
8766
|
return registry.size() === 0 ? null : registry.serialize();
|
|
7691
8767
|
}
|
|
7692
8768
|
function holdsADiscardedField(bytes, discardedFields) {
|
|
@@ -7756,15 +8832,16 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
|
7756
8832
|
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
7757
8833
|
return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
|
|
7758
8834
|
}
|
|
7759
|
-
function mergeFrozenSegments(segments, analysis) {
|
|
8835
|
+
function mergeFrozenSegments(segments, analysis, textAnalysis) {
|
|
7760
8836
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
7761
8837
|
const merged = mergeSegmentContents(
|
|
7762
8838
|
segments.map((segment) => ({ segment })),
|
|
7763
|
-
analysis
|
|
8839
|
+
analysis,
|
|
8840
|
+
textAnalysis
|
|
7764
8841
|
);
|
|
7765
8842
|
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
|
|
7766
8843
|
}
|
|
7767
|
-
function mergeSegmentContents(selections, analysis = null) {
|
|
8844
|
+
function mergeSegmentContents(selections, analysis = null, textAnalysis = null) {
|
|
7768
8845
|
const docIds = [];
|
|
7769
8846
|
const fieldNames = [];
|
|
7770
8847
|
const inputs = selections.map((selection) => ({
|
|
@@ -7791,7 +8868,7 @@ function mergeSegmentContents(selections, analysis = null) {
|
|
|
7791
8868
|
fieldLengthColumns: lengths.fieldLengthColumns,
|
|
7792
8869
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
7793
8870
|
docFrequencies: postings.docFrequencies,
|
|
7794
|
-
surfaceForms: mergeSurfaceForms2(inputs),
|
|
8871
|
+
surfaceForms: mergeSurfaceForms2(inputs, textAnalysis),
|
|
7795
8872
|
partialSpellings: partialWords.spellings,
|
|
7796
8873
|
...mergeFieldIndexes2(inputs)
|
|
7797
8874
|
};
|
|
@@ -8163,22 +9240,29 @@ function runSegmentAction(entry, action) {
|
|
|
8163
9240
|
case "attachSegments": {
|
|
8164
9241
|
for (const segment of action.segments) {
|
|
8165
9242
|
growPartitionsTo(entry.manager, segment.partitionId);
|
|
8166
|
-
const
|
|
8167
|
-
|
|
9243
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
9244
|
+
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
9245
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
9246
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
8168
9247
|
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
8169
9248
|
}
|
|
8170
9249
|
return void 0;
|
|
8171
9250
|
}
|
|
8172
9251
|
case "compactSegments": {
|
|
8173
9252
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
8174
|
-
return mergeFrozenSegments(
|
|
9253
|
+
return mergeFrozenSegments(
|
|
9254
|
+
partition.frozenSegmentsById(action.segmentIds),
|
|
9255
|
+
entry.manager.partialWordAnalysis,
|
|
9256
|
+
entry.manager.textAnalysis
|
|
9257
|
+
);
|
|
8175
9258
|
}
|
|
8176
9259
|
case "swapSegments": {
|
|
8177
9260
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
8178
9261
|
entry.manager.swapFrozenSegments(
|
|
8179
9262
|
action.partitionId,
|
|
8180
9263
|
action.dropSegmentIds,
|
|
8181
|
-
createSharedFrozenSegment(action.snapshot)
|
|
9264
|
+
createSharedFrozenSegment(action.snapshot),
|
|
9265
|
+
action.removedSpellings
|
|
8182
9266
|
);
|
|
8183
9267
|
return void 0;
|
|
8184
9268
|
}
|
|
@@ -8421,12 +9505,12 @@ function createDirectExecutor(options) {
|
|
|
8421
9505
|
}
|
|
8422
9506
|
case "remove": {
|
|
8423
9507
|
const entry = requireIndex(action.indexName);
|
|
8424
|
-
entry.manager.remove(action.docId);
|
|
9508
|
+
entry.manager.remove(action.docId, action.removedSpellings);
|
|
8425
9509
|
return void 0;
|
|
8426
9510
|
}
|
|
8427
9511
|
case "update": {
|
|
8428
9512
|
const entry = requireIndex(action.indexName);
|
|
8429
|
-
entry.manager.update(action.docId, action.document);
|
|
9513
|
+
entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
|
|
8430
9514
|
return void 0;
|
|
8431
9515
|
}
|
|
8432
9516
|
case "query": {
|