@delali/narsil 0.3.1-next.87 → 0.3.1-next.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
- package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
- package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
- package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
- package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
- package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
- package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
- package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
- package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
- package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
- package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
- package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
- package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
- package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
- package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
- package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
- package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
- package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
- package/dist/distribution.mjs +14 -27
- package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -5
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
- package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
- package/dist/workers/entry.mjs +3 -3
- package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
- package/package.json +13 -8
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-3GZQKGVQ.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-JACCR3BJ.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-RAHUQOJL.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-NQW2YXEX.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, compareCodePoints, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-FHFFWLEK.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-3GZQKGVQ.mjs';
|
|
3
|
+
import './chunk-JACCR3BJ.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-RAHUQOJL.mjs';
|
|
5
5
|
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-JACCR3BJ.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-RAHUQOJL.mjs';
|
|
3
3
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
|
|
4
4
|
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -348,97 +348,6 @@ function expandPartialWords(subs, analysis, requests) {
|
|
|
348
348
|
return requests.map((request) => readers.length === 0 ? [] : addedWordsFor(subs, readers, request));
|
|
349
349
|
}
|
|
350
350
|
|
|
351
|
-
// src/core/partition/partial-words/constants.ts
|
|
352
|
-
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
353
|
-
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
354
|
-
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
355
|
-
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
356
|
-
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
357
|
-
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
358
|
-
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
359
|
-
|
|
360
|
-
// src/core/partition/constants.ts
|
|
361
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
362
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
363
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
364
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
365
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
366
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
367
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
368
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
369
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
370
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
371
|
-
|
|
372
|
-
// src/core/partition/frozen/string-blob.ts
|
|
373
|
-
var SURROGATE = /[\ud800-\udfff]/;
|
|
374
|
-
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
375
|
-
var encoder2 = new TextEncoder();
|
|
376
|
-
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
377
|
-
function encodeLookupKey(text) {
|
|
378
|
-
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
|
|
379
|
-
const { written } = encoder2.encodeInto(text, lookupKeyBytes);
|
|
380
|
-
return lookupKeyBytes.subarray(0, written);
|
|
381
|
-
}
|
|
382
|
-
function encodeOneByOne(strings) {
|
|
383
|
-
const encoded = new Array(strings.length);
|
|
384
|
-
let blobLength = 0;
|
|
385
|
-
for (let i = 0; i < strings.length; i++) {
|
|
386
|
-
encoded[i] = encoder2.encode(strings[i]);
|
|
387
|
-
blobLength += encoded[i].length;
|
|
388
|
-
}
|
|
389
|
-
const blob = new Uint8Array(blobLength);
|
|
390
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
391
|
-
let cursor = 0;
|
|
392
|
-
for (let i = 0; i < strings.length; i++) {
|
|
393
|
-
blob.set(encoded[i], cursor);
|
|
394
|
-
cursor += encoded[i].length;
|
|
395
|
-
offsets[i + 1] = cursor;
|
|
396
|
-
}
|
|
397
|
-
return { blob, offsets };
|
|
398
|
-
}
|
|
399
|
-
function basicPlaneUtf8Bytes(text) {
|
|
400
|
-
let bytes = text.length;
|
|
401
|
-
for (let i = 0; i < text.length; i++) {
|
|
402
|
-
const unit = text.charCodeAt(i);
|
|
403
|
-
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
404
|
-
}
|
|
405
|
-
return bytes;
|
|
406
|
-
}
|
|
407
|
-
function encodeStringBlob(strings) {
|
|
408
|
-
const joined = strings.join("");
|
|
409
|
-
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
410
|
-
const blob = encoder2.encode(joined);
|
|
411
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
412
|
-
let cursor = 0;
|
|
413
|
-
for (let i = 0; i < strings.length; i++) {
|
|
414
|
-
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
415
|
-
offsets[i + 1] = cursor;
|
|
416
|
-
}
|
|
417
|
-
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
418
|
-
}
|
|
419
|
-
function orderByComparator(strings) {
|
|
420
|
-
const order = new Array(strings.length);
|
|
421
|
-
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
422
|
-
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
423
|
-
}
|
|
424
|
-
function alreadyInCodePointOrder(strings) {
|
|
425
|
-
for (let i = 1; i < strings.length; i++) {
|
|
426
|
-
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
427
|
-
}
|
|
428
|
-
return true;
|
|
429
|
-
}
|
|
430
|
-
function codePointOrder(strings) {
|
|
431
|
-
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
432
|
-
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
433
|
-
const positions = /* @__PURE__ */ new Map();
|
|
434
|
-
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
435
|
-
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
436
|
-
const order = new Array(strings.length);
|
|
437
|
-
const sorted = [...strings].sort();
|
|
438
|
-
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
439
|
-
return order;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
351
|
// src/core/tokenizer/ngrams.ts
|
|
443
352
|
var OTHER_SCRIPT = 0;
|
|
444
353
|
var HAN_SCRIPT = 1;
|
|
@@ -642,6 +551,17 @@ function visitTokens(text, language, options, visit) {
|
|
|
642
551
|
visit(processed, position, surface === processed ? void 0 : surface, part);
|
|
643
552
|
}
|
|
644
553
|
}
|
|
554
|
+
function visitSurfaceSpellings(text, language, visit) {
|
|
555
|
+
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
556
|
+
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
557
|
+
const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
|
|
558
|
+
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
559
|
+
for (const part of rawParts) {
|
|
560
|
+
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
561
|
+
if (candidate.length < minLength) continue;
|
|
562
|
+
visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
645
565
|
function tokenize(text, language, options) {
|
|
646
566
|
if (options?.customTokenizer) {
|
|
647
567
|
const customResult = options.customTokenizer.tokenize(text);
|
|
@@ -658,6 +578,98 @@ function tokenize(text, language, options) {
|
|
|
658
578
|
return { tokens, originalTokens, surfaces };
|
|
659
579
|
}
|
|
660
580
|
|
|
581
|
+
// src/core/partition/partial-words/constants.ts
|
|
582
|
+
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
583
|
+
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
584
|
+
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
585
|
+
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
586
|
+
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
587
|
+
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
588
|
+
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
589
|
+
|
|
590
|
+
// src/core/partition/constants.ts
|
|
591
|
+
var TARGET_BLOCK_ENTRIES = 128;
|
|
592
|
+
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
593
|
+
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
594
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
595
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
596
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
597
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
598
|
+
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
599
|
+
var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
|
|
600
|
+
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
601
|
+
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
602
|
+
|
|
603
|
+
// src/core/partition/frozen/string-blob.ts
|
|
604
|
+
var SURROGATE = /[\ud800-\udfff]/;
|
|
605
|
+
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
606
|
+
var encoder2 = new TextEncoder();
|
|
607
|
+
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
608
|
+
function encodeLookupKey(text) {
|
|
609
|
+
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
|
|
610
|
+
const { written } = encoder2.encodeInto(text, lookupKeyBytes);
|
|
611
|
+
return lookupKeyBytes.subarray(0, written);
|
|
612
|
+
}
|
|
613
|
+
function encodeOneByOne(strings) {
|
|
614
|
+
const encoded = new Array(strings.length);
|
|
615
|
+
let blobLength = 0;
|
|
616
|
+
for (let i = 0; i < strings.length; i++) {
|
|
617
|
+
encoded[i] = encoder2.encode(strings[i]);
|
|
618
|
+
blobLength += encoded[i].length;
|
|
619
|
+
}
|
|
620
|
+
const blob = new Uint8Array(blobLength);
|
|
621
|
+
const offsets = new Uint32Array(strings.length + 1);
|
|
622
|
+
let cursor = 0;
|
|
623
|
+
for (let i = 0; i < strings.length; i++) {
|
|
624
|
+
blob.set(encoded[i], cursor);
|
|
625
|
+
cursor += encoded[i].length;
|
|
626
|
+
offsets[i + 1] = cursor;
|
|
627
|
+
}
|
|
628
|
+
return { blob, offsets };
|
|
629
|
+
}
|
|
630
|
+
function basicPlaneUtf8Bytes(text) {
|
|
631
|
+
let bytes = text.length;
|
|
632
|
+
for (let i = 0; i < text.length; i++) {
|
|
633
|
+
const unit = text.charCodeAt(i);
|
|
634
|
+
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
635
|
+
}
|
|
636
|
+
return bytes;
|
|
637
|
+
}
|
|
638
|
+
function encodeStringBlob(strings) {
|
|
639
|
+
const joined = strings.join("");
|
|
640
|
+
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
641
|
+
const blob = encoder2.encode(joined);
|
|
642
|
+
const offsets = new Uint32Array(strings.length + 1);
|
|
643
|
+
let cursor = 0;
|
|
644
|
+
for (let i = 0; i < strings.length; i++) {
|
|
645
|
+
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
646
|
+
offsets[i + 1] = cursor;
|
|
647
|
+
}
|
|
648
|
+
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
649
|
+
}
|
|
650
|
+
function orderByComparator(strings) {
|
|
651
|
+
const order = new Array(strings.length);
|
|
652
|
+
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
653
|
+
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
654
|
+
}
|
|
655
|
+
function alreadyInCodePointOrder(strings) {
|
|
656
|
+
for (let i = 1; i < strings.length; i++) {
|
|
657
|
+
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
658
|
+
}
|
|
659
|
+
return true;
|
|
660
|
+
}
|
|
661
|
+
function codePointOrder(strings) {
|
|
662
|
+
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
663
|
+
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
664
|
+
const positions = /* @__PURE__ */ new Map();
|
|
665
|
+
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
666
|
+
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
667
|
+
const order = new Array(strings.length);
|
|
668
|
+
const sorted = [...strings].sort();
|
|
669
|
+
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
670
|
+
return order;
|
|
671
|
+
}
|
|
672
|
+
|
|
661
673
|
// src/core/partition/utils.ts
|
|
662
674
|
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
663
675
|
const existing = table.indexMap.get(fieldName);
|
|
@@ -719,16 +731,6 @@ function getFlatSchema(state, schema) {
|
|
|
719
731
|
state.lastSchemaRef = schema;
|
|
720
732
|
return state.flatSchemaCache;
|
|
721
733
|
}
|
|
722
|
-
function getFieldValueForDoc(docStore, docId, fieldPath) {
|
|
723
|
-
const stored = docStore.get(docId);
|
|
724
|
-
if (!stored) return void 0;
|
|
725
|
-
return getNestedValue(stored.fields, fieldPath);
|
|
726
|
-
}
|
|
727
|
-
function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
728
|
-
const externalId = docStore.getExternalId(internalId);
|
|
729
|
-
if (externalId === void 0) return void 0;
|
|
730
|
-
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
731
|
-
}
|
|
732
734
|
|
|
733
735
|
// src/core/partition/partial-words/spellings.ts
|
|
734
736
|
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
@@ -765,16 +767,24 @@ function textsOf(value) {
|
|
|
765
767
|
if (!Array.isArray(value)) return [];
|
|
766
768
|
return value.filter((element) => typeof element === "string");
|
|
767
769
|
}
|
|
768
|
-
function
|
|
770
|
+
function visitFieldTexts(document, fields, visit) {
|
|
771
|
+
for (const field of fields) {
|
|
772
|
+
for (const text of textsOf(getNestedValue(document, field))) visit(text);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function visitDocumentSpellings(document, analysis, visit) {
|
|
769
776
|
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
770
|
-
const
|
|
771
|
-
|
|
777
|
+
const visitOccurrence = (token, _position, surface) => {
|
|
778
|
+
visit(token, surface ?? token);
|
|
772
779
|
};
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
780
|
+
visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
|
|
781
|
+
}
|
|
782
|
+
function visitDocumentSurfaceSpellings(document, analysis, visit) {
|
|
783
|
+
if (analysis.options?.customTokenizer !== void 0) return;
|
|
784
|
+
visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
|
|
785
|
+
}
|
|
786
|
+
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
787
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => countSpelling(counts, token, spelling, direction));
|
|
778
788
|
}
|
|
779
789
|
function spellingCountsOf(documents, analysis) {
|
|
780
790
|
const counts = /* @__PURE__ */ new Map();
|
|
@@ -922,21 +932,21 @@ function readPartialWordTable(table, tokenTable, removedCounts) {
|
|
|
922
932
|
visit(tokenTable.tokenAt(spellingTokens[entry]), length);
|
|
923
933
|
}
|
|
924
934
|
},
|
|
925
|
-
entryOf(spelling
|
|
926
|
-
const tokenIndex = tokenTable.find(token);
|
|
927
|
-
if (tokenIndex < 0) return -1;
|
|
935
|
+
entryOf(spelling) {
|
|
928
936
|
const probe = partialWordProbe(spelling);
|
|
929
937
|
if (probe.codePoints.length <= PARTIAL_WORD_RUN_CODE_POINTS) return -1;
|
|
930
938
|
const run = narrowestRun(probe.codePoints);
|
|
931
939
|
if (run < 0) return -1;
|
|
940
|
+
let exhausted = -1;
|
|
932
941
|
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
933
942
|
const entry = runEntries[member];
|
|
934
|
-
if (spellingTokens[entry] !== tokenIndex) continue;
|
|
935
943
|
const start = spellingOffsets[entry];
|
|
936
944
|
const end = spellingOffsets[entry + 1];
|
|
937
|
-
if (end - start
|
|
945
|
+
if (end - start !== probe.utf8.length || !utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
946
|
+
if ((removedCounts.get(entry) ?? 0) < spellingCounts[entry]) return entry;
|
|
947
|
+
if (exhausted < 0) exhausted = entry;
|
|
938
948
|
}
|
|
939
|
-
return
|
|
949
|
+
return exhausted;
|
|
940
950
|
},
|
|
941
951
|
bytes() {
|
|
942
952
|
return partialWordTableBytes(table);
|
|
@@ -959,6 +969,38 @@ function addCounts(target, source) {
|
|
|
959
969
|
for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
|
|
960
970
|
}
|
|
961
971
|
}
|
|
972
|
+
function countRemovedSpellings(removed, document, analysis) {
|
|
973
|
+
const count = (spelling) => {
|
|
974
|
+
removed.set(spelling, (removed.get(spelling) ?? 0) + 1);
|
|
975
|
+
};
|
|
976
|
+
if (analysis.options?.customTokenizer !== void 0) {
|
|
977
|
+
visitDocumentSpellings(document, analysis, (_token, spelling) => count(spelling));
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
visitDocumentSurfaceSpellings(document, analysis, count);
|
|
981
|
+
}
|
|
982
|
+
function unstemmedSpellingsOf(analysis) {
|
|
983
|
+
const { language } = analysis;
|
|
984
|
+
if (language.normalizer === void 0 || language.tokenizer?.normalizeDiacritics === true) return () => null;
|
|
985
|
+
const stopWords = resolveStopWords(language, analysis.options?.stopWordOverride);
|
|
986
|
+
return (spelling) => {
|
|
987
|
+
if (stopWords.has(spelling)) return null;
|
|
988
|
+
const normalized = transformToken(spelling, language, false, false);
|
|
989
|
+
return normalized === spelling ? null : normalized;
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
function subtractRemovedSpellings(counts, removed, analysis) {
|
|
993
|
+
const tokenOf = /* @__PURE__ */ new Map();
|
|
994
|
+
for (const [token, bySpelling] of counts) {
|
|
995
|
+
for (const spelling of bySpelling.keys()) tokenOf.set(spelling, token);
|
|
996
|
+
}
|
|
997
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
998
|
+
for (const [spelling, count] of removed) {
|
|
999
|
+
const stored = tokenOf.has(spelling) ? spelling : unstemmedSpelling(spelling) ?? spelling;
|
|
1000
|
+
const token = tokenOf.get(stored);
|
|
1001
|
+
if (token !== void 0) countSpelling(counts, token, stored, -count);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
962
1004
|
function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
963
1005
|
let table = initial;
|
|
964
1006
|
let uncounted = null;
|
|
@@ -971,13 +1013,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
|
971
1013
|
function subtractPendingRemovals(current, analysis) {
|
|
972
1014
|
if (pendingRemovals.length === 0) return;
|
|
973
1015
|
const removed = /* @__PURE__ */ new Map();
|
|
974
|
-
for (const ordinal of pendingRemovals)
|
|
1016
|
+
for (const ordinal of pendingRemovals) countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
975
1017
|
pendingRemovals = [];
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
1018
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
1019
|
+
for (const [spelling, count] of removed) {
|
|
1020
|
+
let entry = current.entryOf(spelling);
|
|
1021
|
+
if (entry < 0) {
|
|
1022
|
+
const fallback = unstemmedSpelling(spelling);
|
|
1023
|
+
if (fallback !== null) entry = current.entryOf(fallback);
|
|
980
1024
|
}
|
|
1025
|
+
if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
|
|
981
1026
|
}
|
|
982
1027
|
}
|
|
983
1028
|
function rebuild(analysis) {
|
|
@@ -1031,11 +1076,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
|
1031
1076
|
}
|
|
1032
1077
|
const corrected = /* @__PURE__ */ new Map();
|
|
1033
1078
|
countTableSpellings(corrected, table, tokenTable);
|
|
1079
|
+
const removed = /* @__PURE__ */ new Map();
|
|
1080
|
+
const returned = [];
|
|
1034
1081
|
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1035
1082
|
const kept = mergedOrdinals[ordinal] >= 0;
|
|
1036
1083
|
if (kept === isCounted(ordinal)) continue;
|
|
1037
|
-
|
|
1084
|
+
if (kept) returned.push(ordinal);
|
|
1085
|
+
else countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
1038
1086
|
}
|
|
1087
|
+
subtractRemovedSpellings(corrected, removed, analysis);
|
|
1088
|
+
for (const ordinal of returned) countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, 1);
|
|
1039
1089
|
addCounts(counts, corrected);
|
|
1040
1090
|
},
|
|
1041
1091
|
spellings() {
|
|
@@ -1398,4 +1448,4 @@ var partialWordSearchModule = {
|
|
|
1398
1448
|
highlightWords: partialWordsToHighlight
|
|
1399
1449
|
};
|
|
1400
1450
|
|
|
1401
|
-
export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix,
|
|
1451
|
+
export { FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, resetNormalizationCache, sameFields, suggestDisplayTerms, textFieldsChanged, tokenize, tokenizeOptions, visitDocumentSpellings, visitDocumentSurfaceSpellings, visitTokens };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-YCJLVNWR.mjs';
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-
|
|
3
|
-
import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-QAMUC2BJ.mjs';
|
|
3
|
+
import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-R6BMMQXH.mjs';
|
|
4
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-EMXAYPTQ.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
6
6
|
import { flattenSchema, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor,
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-X4SEXL4U.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Q5DC6WAB.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
4
|
-
import { tokenize } from './chunk-
|
|
4
|
+
import { tokenize } from './chunk-A3UB2MXL.mjs';
|
|
5
5
|
import { partialFieldsOf, flattenSchema } from './chunk-CVQWQVFX.mjs';
|
|
6
6
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
|
|
7
7
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
8
8
|
import { isWordIndexedFieldType } from './chunk-BVOJIYRV.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType } from './chunk-L5ORED7V.mjs';
|
|
11
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
12
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
@@ -220,18 +220,27 @@ function requireFacetableFields(facets, flatSchema, strict) {
|
|
|
220
220
|
}
|
|
221
221
|
}
|
|
222
222
|
}
|
|
223
|
-
function
|
|
223
|
+
function requireGroupableFields(fields, flatSchema, strict) {
|
|
224
224
|
for (const field of fields) {
|
|
225
|
-
|
|
225
|
+
const fieldType = flatSchema[field];
|
|
226
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
227
|
+
if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
|
|
228
|
+
throw new NarsilError(
|
|
229
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
230
|
+
`The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
231
|
+
{ field, fieldType }
|
|
232
|
+
);
|
|
233
|
+
}
|
|
226
234
|
}
|
|
227
235
|
}
|
|
228
236
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
229
|
-
const
|
|
237
|
+
const groupFields = params.group?.fields;
|
|
238
|
+
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
230
239
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
231
240
|
const flatSchema = flattenSchema(schema);
|
|
232
241
|
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
233
242
|
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
234
|
-
if (checksGroup
|
|
243
|
+
if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
|
|
235
244
|
}
|
|
236
245
|
|
|
237
246
|
// src/search/typo-tolerance.ts
|
|
@@ -630,6 +639,7 @@ function executeListDocuments(params, context) {
|
|
|
630
639
|
const signature = sortSignatureOf(params.sort);
|
|
631
640
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
632
641
|
const filters = validQueryFilter(params.filters, schema);
|
|
642
|
+
const projection = resolveProjection(params.document);
|
|
633
643
|
const binding = listBindingOf(params.filters);
|
|
634
644
|
let cursor = null;
|
|
635
645
|
if (params.cursor !== void 0) {
|
|
@@ -640,7 +650,6 @@ function executeListDocuments(params, context) {
|
|
|
640
650
|
const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
|
|
641
651
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
642
652
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
643
|
-
const projection = resolveProjection(params.document);
|
|
644
653
|
const documents = [];
|
|
645
654
|
for (const docId of page.ids) {
|
|
646
655
|
if (projection.kind === "none") {
|
|
@@ -663,37 +672,23 @@ function executeListDocuments(params, context) {
|
|
|
663
672
|
}
|
|
664
673
|
|
|
665
674
|
// src/search/grouping.ts
|
|
666
|
-
function getNestedValue(obj, path) {
|
|
667
|
-
const segments = path.split(".");
|
|
668
|
-
let current = obj;
|
|
669
|
-
for (const segment of segments) {
|
|
670
|
-
if (current === null || current === void 0 || typeof current !== "object") {
|
|
671
|
-
return void 0;
|
|
672
|
-
}
|
|
673
|
-
if (!Object.hasOwn(current, segment)) {
|
|
674
|
-
return void 0;
|
|
675
|
-
}
|
|
676
|
-
current = current[segment];
|
|
677
|
-
}
|
|
678
|
-
return current;
|
|
679
|
-
}
|
|
680
675
|
function hitsKeptPerGroup(maxPerGroup) {
|
|
681
676
|
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
682
677
|
return Math.floor(maxPerGroup);
|
|
683
678
|
}
|
|
684
|
-
function applyGrouping(hits, group,
|
|
679
|
+
function applyGrouping(hits, group, documents) {
|
|
685
680
|
if (!group.fields || group.fields.length === 0) {
|
|
686
681
|
return [{ values: {}, hits }];
|
|
687
682
|
}
|
|
688
683
|
const groupMap = /* @__PURE__ */ new Map();
|
|
689
684
|
for (const hit of hits) {
|
|
690
|
-
const
|
|
685
|
+
const stored = documents.fieldValues(hit.id, group.fields);
|
|
691
686
|
const fieldValues = [];
|
|
692
687
|
const valuesRecord = {};
|
|
693
|
-
for (
|
|
694
|
-
const value =
|
|
688
|
+
for (let at = 0; at < group.fields.length; at++) {
|
|
689
|
+
const value = stored?.[at];
|
|
695
690
|
fieldValues.push(value);
|
|
696
|
-
valuesRecord[
|
|
691
|
+
valuesRecord[group.fields[at]] = value;
|
|
697
692
|
}
|
|
698
693
|
const compositeKey = JSON.stringify(fieldValues);
|
|
699
694
|
const existing = groupMap.get(compositeKey);
|
|
@@ -711,7 +706,7 @@ function applyGrouping(hits, group, getDocument) {
|
|
|
711
706
|
if (group.reduce) {
|
|
712
707
|
const folded = foldGroupReducer(
|
|
713
708
|
group.reduce,
|
|
714
|
-
everyHit.map((hit) => ({ document:
|
|
709
|
+
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
715
710
|
);
|
|
716
711
|
if (folded.reducerError !== void 0) {
|
|
717
712
|
result.reducerError = folded.reducerError;
|
|
@@ -1359,6 +1354,7 @@ async function executeQuery(request, callerContext) {
|
|
|
1359
1354
|
}
|
|
1360
1355
|
const params = validQueryParams(request, config);
|
|
1361
1356
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1357
|
+
const projection = resolveProjection(params.document);
|
|
1362
1358
|
let paginated;
|
|
1363
1359
|
let nextCursor;
|
|
1364
1360
|
let count;
|
|
@@ -1442,7 +1438,10 @@ async function executeQuery(request, callerContext) {
|
|
|
1442
1438
|
hits = applySorting(hits, params.sort, sortKeyOf);
|
|
1443
1439
|
}
|
|
1444
1440
|
if (params.group) {
|
|
1445
|
-
groups = applyGrouping(hits, params.group,
|
|
1441
|
+
groups = applyGrouping(hits, params.group, {
|
|
1442
|
+
fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
|
|
1443
|
+
document: (docId) => manager.getRef(docId)
|
|
1444
|
+
});
|
|
1446
1445
|
}
|
|
1447
1446
|
let pinsFromOutside = 0;
|
|
1448
1447
|
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
@@ -1482,7 +1481,6 @@ async function executeQuery(request, callerContext) {
|
|
|
1482
1481
|
for (const hit of hits) hit.score = void 0;
|
|
1483
1482
|
}
|
|
1484
1483
|
}
|
|
1485
|
-
const projection = resolveProjection(params.document);
|
|
1486
1484
|
if (projection.kind === "none") {
|
|
1487
1485
|
for (const hit of paginated) {
|
|
1488
1486
|
hit.document = {};
|