@delali/narsil 0.3.1-next.87 → 0.3.1-next.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
  2. package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
  3. package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
  4. package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
  5. package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
  6. package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
  7. package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
  8. package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
  9. package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
  10. package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
  11. package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
  12. package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
  13. package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
  14. package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
  15. package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
  16. package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
  17. package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
  18. package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
  19. package/dist/distribution/transport/grpc.mjs +45 -7
  20. package/dist/distribution.mjs +14 -27
  21. package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
  22. package/dist/index.browser.mjs +1 -1
  23. package/dist/index.mjs +5 -5
  24. package/dist/partial-words.mjs +1 -1
  25. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  26. package/dist/react.mjs +4 -2
  27. package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
  28. package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
  29. package/dist/workers/entry.mjs +3 -3
  30. package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
  31. package/package.json +13 -8
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-BBDAYUCO.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-3BXQPIXK.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-TBMNPGFO.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-3GZQKGVQ.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-JACCR3BJ.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-RAHUQOJL.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-NQW2YXEX.mjs';
6
6
  import { NarsilError, ErrorCodes, compareCodePoints, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-FHFFWLEK.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -1,7 +1,7 @@
1
1
  import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-BBDAYUCO.mjs';
3
- import './chunk-3BXQPIXK.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-TBMNPGFO.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-3GZQKGVQ.mjs';
3
+ import './chunk-JACCR3BJ.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-RAHUQOJL.mjs';
5
5
  import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
6
6
  import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-3BXQPIXK.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-TBMNPGFO.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-JACCR3BJ.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-RAHUQOJL.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
4
4
  import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
5
5
  import { decode } from '@msgpack/msgpack';
@@ -348,97 +348,6 @@ function expandPartialWords(subs, analysis, requests) {
348
348
  return requests.map((request) => readers.length === 0 ? [] : addedWordsFor(subs, readers, request));
349
349
  }
350
350
 
351
- // src/core/partition/partial-words/constants.ts
352
- var PARTIAL_WORD_RUN_CODE_POINTS = 3;
353
- var GROWING_PARTIAL_ENTRY_BYTES = 104;
354
- var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
355
- var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
356
- var GROWING_PARTIAL_RUN_BYTES = 120;
357
- var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
358
- var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
359
-
360
- // src/core/partition/constants.ts
361
- var TARGET_BLOCK_ENTRIES = 128;
362
- var PRUNING_REJECTION_SLACK = 1e-12;
363
- var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
364
- var SEGMENT_TOKEN_TABLE_START = 1 << 12;
365
- var SEGMENT_POSTING_LOG_START = 1 << 14;
366
- var SEGMENT_POSITION_POOL_START = 1 << 16;
367
- var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
368
- var LOOKUP_KEY_SCRATCH_BYTES = 4096;
369
- var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
370
- var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
371
-
372
- // src/core/partition/frozen/string-blob.ts
373
- var SURROGATE = /[\ud800-\udfff]/;
374
- var UTF8_BYTES_PER_UTF16_UNIT = 3;
375
- var encoder2 = new TextEncoder();
376
- var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
377
- function encodeLookupKey(text) {
378
- if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
379
- const { written } = encoder2.encodeInto(text, lookupKeyBytes);
380
- return lookupKeyBytes.subarray(0, written);
381
- }
382
- function encodeOneByOne(strings) {
383
- const encoded = new Array(strings.length);
384
- let blobLength = 0;
385
- for (let i = 0; i < strings.length; i++) {
386
- encoded[i] = encoder2.encode(strings[i]);
387
- blobLength += encoded[i].length;
388
- }
389
- const blob = new Uint8Array(blobLength);
390
- const offsets = new Uint32Array(strings.length + 1);
391
- let cursor = 0;
392
- for (let i = 0; i < strings.length; i++) {
393
- blob.set(encoded[i], cursor);
394
- cursor += encoded[i].length;
395
- offsets[i + 1] = cursor;
396
- }
397
- return { blob, offsets };
398
- }
399
- function basicPlaneUtf8Bytes(text) {
400
- let bytes = text.length;
401
- for (let i = 0; i < text.length; i++) {
402
- const unit = text.charCodeAt(i);
403
- if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
404
- }
405
- return bytes;
406
- }
407
- function encodeStringBlob(strings) {
408
- const joined = strings.join("");
409
- if (SURROGATE.test(joined)) return encodeOneByOne(strings);
410
- const blob = encoder2.encode(joined);
411
- const offsets = new Uint32Array(strings.length + 1);
412
- let cursor = 0;
413
- for (let i = 0; i < strings.length; i++) {
414
- cursor += basicPlaneUtf8Bytes(strings[i]);
415
- offsets[i + 1] = cursor;
416
- }
417
- return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
418
- }
419
- function orderByComparator(strings) {
420
- const order = new Array(strings.length);
421
- for (let i = 0; i < strings.length; i++) order[i] = i;
422
- return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
423
- }
424
- function alreadyInCodePointOrder(strings) {
425
- for (let i = 1; i < strings.length; i++) {
426
- if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
427
- }
428
- return true;
429
- }
430
- function codePointOrder(strings) {
431
- if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
432
- if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
433
- const positions = /* @__PURE__ */ new Map();
434
- for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
435
- if (positions.size !== strings.length) return orderByComparator(strings);
436
- const order = new Array(strings.length);
437
- const sorted = [...strings].sort();
438
- for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
439
- return order;
440
- }
441
-
442
351
  // src/core/tokenizer/ngrams.ts
443
352
  var OTHER_SCRIPT = 0;
444
353
  var HAN_SCRIPT = 1;
@@ -642,6 +551,17 @@ function visitTokens(text, language, options, visit) {
642
551
  visit(processed, position, surface === processed ? void 0 : surface, part);
643
552
  }
644
553
  }
554
+ function visitSurfaceSpellings(text, language, visit) {
555
+ const rawParts = splitText(normalizeForSplitting(text), language);
556
+ const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
557
+ const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
558
+ const stripPossessives = language.tokenizer?.stripPossessive ?? false;
559
+ for (const part of rawParts) {
560
+ const candidate = stripPossessives ? stripPossessive(part) : part;
561
+ if (candidate.length < minLength) continue;
562
+ visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
563
+ }
564
+ }
645
565
  function tokenize(text, language, options) {
646
566
  if (options?.customTokenizer) {
647
567
  const customResult = options.customTokenizer.tokenize(text);
@@ -658,6 +578,98 @@ function tokenize(text, language, options) {
658
578
  return { tokens, originalTokens, surfaces };
659
579
  }
660
580
 
581
+ // src/core/partition/partial-words/constants.ts
582
+ var PARTIAL_WORD_RUN_CODE_POINTS = 3;
583
+ var GROWING_PARTIAL_ENTRY_BYTES = 104;
584
+ var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
585
+ var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
586
+ var GROWING_PARTIAL_RUN_BYTES = 120;
587
+ var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
588
+ var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
589
+
590
+ // src/core/partition/constants.ts
591
+ var TARGET_BLOCK_ENTRIES = 128;
592
+ var PRUNING_REJECTION_SLACK = 1e-12;
593
+ var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
594
+ var SEGMENT_TOKEN_TABLE_START = 1 << 12;
595
+ var SEGMENT_POSTING_LOG_START = 1 << 14;
596
+ var SEGMENT_POSITION_POOL_START = 1 << 16;
597
+ var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
598
+ var LOOKUP_KEY_SCRATCH_BYTES = 4096;
599
+ var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
600
+ var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
601
+ var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
602
+
603
+ // src/core/partition/frozen/string-blob.ts
604
+ var SURROGATE = /[\ud800-\udfff]/;
605
+ var UTF8_BYTES_PER_UTF16_UNIT = 3;
606
+ var encoder2 = new TextEncoder();
607
+ var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
608
+ function encodeLookupKey(text) {
609
+ if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
610
+ const { written } = encoder2.encodeInto(text, lookupKeyBytes);
611
+ return lookupKeyBytes.subarray(0, written);
612
+ }
613
+ function encodeOneByOne(strings) {
614
+ const encoded = new Array(strings.length);
615
+ let blobLength = 0;
616
+ for (let i = 0; i < strings.length; i++) {
617
+ encoded[i] = encoder2.encode(strings[i]);
618
+ blobLength += encoded[i].length;
619
+ }
620
+ const blob = new Uint8Array(blobLength);
621
+ const offsets = new Uint32Array(strings.length + 1);
622
+ let cursor = 0;
623
+ for (let i = 0; i < strings.length; i++) {
624
+ blob.set(encoded[i], cursor);
625
+ cursor += encoded[i].length;
626
+ offsets[i + 1] = cursor;
627
+ }
628
+ return { blob, offsets };
629
+ }
630
+ function basicPlaneUtf8Bytes(text) {
631
+ let bytes = text.length;
632
+ for (let i = 0; i < text.length; i++) {
633
+ const unit = text.charCodeAt(i);
634
+ if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
635
+ }
636
+ return bytes;
637
+ }
638
+ function encodeStringBlob(strings) {
639
+ const joined = strings.join("");
640
+ if (SURROGATE.test(joined)) return encodeOneByOne(strings);
641
+ const blob = encoder2.encode(joined);
642
+ const offsets = new Uint32Array(strings.length + 1);
643
+ let cursor = 0;
644
+ for (let i = 0; i < strings.length; i++) {
645
+ cursor += basicPlaneUtf8Bytes(strings[i]);
646
+ offsets[i + 1] = cursor;
647
+ }
648
+ return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
649
+ }
650
+ function orderByComparator(strings) {
651
+ const order = new Array(strings.length);
652
+ for (let i = 0; i < strings.length; i++) order[i] = i;
653
+ return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
654
+ }
655
+ function alreadyInCodePointOrder(strings) {
656
+ for (let i = 1; i < strings.length; i++) {
657
+ if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
658
+ }
659
+ return true;
660
+ }
661
+ function codePointOrder(strings) {
662
+ if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
663
+ if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
664
+ const positions = /* @__PURE__ */ new Map();
665
+ for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
666
+ if (positions.size !== strings.length) return orderByComparator(strings);
667
+ const order = new Array(strings.length);
668
+ const sorted = [...strings].sort();
669
+ for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
670
+ return order;
671
+ }
672
+
661
673
  // src/core/partition/utils.ts
662
674
  function getOrCreateFieldNameIndex(table, fieldName) {
663
675
  const existing = table.indexMap.get(fieldName);
@@ -719,16 +731,6 @@ function getFlatSchema(state, schema) {
719
731
  state.lastSchemaRef = schema;
720
732
  return state.flatSchemaCache;
721
733
  }
722
- function getFieldValueForDoc(docStore, docId, fieldPath) {
723
- const stored = docStore.get(docId);
724
- if (!stored) return void 0;
725
- return getNestedValue(stored.fields, fieldPath);
726
- }
727
- function getFieldValueByInternalId(docStore, internalId, fieldPath) {
728
- const externalId = docStore.getExternalId(internalId);
729
- if (externalId === void 0) return void 0;
730
- return getFieldValueForDoc(docStore, externalId, fieldPath);
731
- }
732
734
 
733
735
  // src/core/partition/partial-words/spellings.ts
734
736
  var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
@@ -765,16 +767,24 @@ function textsOf(value) {
765
767
  if (!Array.isArray(value)) return [];
766
768
  return value.filter((element) => typeof element === "string");
767
769
  }
768
- function countDocumentSpellings(counts, document, analysis, direction) {
770
+ function visitFieldTexts(document, fields, visit) {
771
+ for (const field of fields) {
772
+ for (const text of textsOf(getNestedValue(document, field))) visit(text);
773
+ }
774
+ }
775
+ function visitDocumentSpellings(document, analysis, visit) {
769
776
  const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
770
- const countOccurrence = (token, _position, surface) => {
771
- countSpelling(counts, token, surface ?? token, direction);
777
+ const visitOccurrence = (token, _position, surface) => {
778
+ visit(token, surface ?? token);
772
779
  };
773
- for (const field of analysis.fields) {
774
- for (const text of textsOf(getNestedValue(document, field))) {
775
- visitTokens(text, analysis.language, tokenizing, countOccurrence);
776
- }
777
- }
780
+ visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
781
+ }
782
+ function visitDocumentSurfaceSpellings(document, analysis, visit) {
783
+ if (analysis.options?.customTokenizer !== void 0) return;
784
+ visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
785
+ }
786
+ function countDocumentSpellings(counts, document, analysis, direction) {
787
+ visitDocumentSpellings(document, analysis, (token, spelling) => countSpelling(counts, token, spelling, direction));
778
788
  }
779
789
  function spellingCountsOf(documents, analysis) {
780
790
  const counts = /* @__PURE__ */ new Map();
@@ -922,21 +932,21 @@ function readPartialWordTable(table, tokenTable, removedCounts) {
922
932
  visit(tokenTable.tokenAt(spellingTokens[entry]), length);
923
933
  }
924
934
  },
925
- entryOf(spelling, token) {
926
- const tokenIndex = tokenTable.find(token);
927
- if (tokenIndex < 0) return -1;
935
+ entryOf(spelling) {
928
936
  const probe = partialWordProbe(spelling);
929
937
  if (probe.codePoints.length <= PARTIAL_WORD_RUN_CODE_POINTS) return -1;
930
938
  const run = narrowestRun(probe.codePoints);
931
939
  if (run < 0) return -1;
940
+ let exhausted = -1;
932
941
  for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
933
942
  const entry = runEntries[member];
934
- if (spellingTokens[entry] !== tokenIndex) continue;
935
943
  const start = spellingOffsets[entry];
936
944
  const end = spellingOffsets[entry + 1];
937
- if (end - start === probe.utf8.length && utf8Contains(spellings, start, end, probe.utf8)) return entry;
945
+ if (end - start !== probe.utf8.length || !utf8Contains(spellings, start, end, probe.utf8)) continue;
946
+ if ((removedCounts.get(entry) ?? 0) < spellingCounts[entry]) return entry;
947
+ if (exhausted < 0) exhausted = entry;
938
948
  }
939
- return -1;
949
+ return exhausted;
940
950
  },
941
951
  bytes() {
942
952
  return partialWordTableBytes(table);
@@ -959,6 +969,38 @@ function addCounts(target, source) {
959
969
  for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
960
970
  }
961
971
  }
972
+ function countRemovedSpellings(removed, document, analysis) {
973
+ const count = (spelling) => {
974
+ removed.set(spelling, (removed.get(spelling) ?? 0) + 1);
975
+ };
976
+ if (analysis.options?.customTokenizer !== void 0) {
977
+ visitDocumentSpellings(document, analysis, (_token, spelling) => count(spelling));
978
+ return;
979
+ }
980
+ visitDocumentSurfaceSpellings(document, analysis, count);
981
+ }
982
+ function unstemmedSpellingsOf(analysis) {
983
+ const { language } = analysis;
984
+ if (language.normalizer === void 0 || language.tokenizer?.normalizeDiacritics === true) return () => null;
985
+ const stopWords = resolveStopWords(language, analysis.options?.stopWordOverride);
986
+ return (spelling) => {
987
+ if (stopWords.has(spelling)) return null;
988
+ const normalized = transformToken(spelling, language, false, false);
989
+ return normalized === spelling ? null : normalized;
990
+ };
991
+ }
992
+ function subtractRemovedSpellings(counts, removed, analysis) {
993
+ const tokenOf = /* @__PURE__ */ new Map();
994
+ for (const [token, bySpelling] of counts) {
995
+ for (const spelling of bySpelling.keys()) tokenOf.set(spelling, token);
996
+ }
997
+ const unstemmedSpelling = unstemmedSpellingsOf(analysis);
998
+ for (const [spelling, count] of removed) {
999
+ const stored = tokenOf.has(spelling) ? spelling : unstemmedSpelling(spelling) ?? spelling;
1000
+ const token = tokenOf.get(stored);
1001
+ if (token !== void 0) countSpelling(counts, token, stored, -count);
1002
+ }
1003
+ }
962
1004
  function createFrozenPartialWordSlot(initial, tokenTable, documents) {
963
1005
  let table = initial;
964
1006
  let uncounted = null;
@@ -971,13 +1013,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
971
1013
  function subtractPendingRemovals(current, analysis) {
972
1014
  if (pendingRemovals.length === 0) return;
973
1015
  const removed = /* @__PURE__ */ new Map();
974
- for (const ordinal of pendingRemovals) countDocumentSpellings(removed, documents.documentAt(ordinal), analysis, 1);
1016
+ for (const ordinal of pendingRemovals) countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
975
1017
  pendingRemovals = [];
976
- for (const [token, bySpelling] of removed) {
977
- for (const [spelling, count] of bySpelling) {
978
- const entry = current.entryOf(spelling, token);
979
- if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
1018
+ const unstemmedSpelling = unstemmedSpellingsOf(analysis);
1019
+ for (const [spelling, count] of removed) {
1020
+ let entry = current.entryOf(spelling);
1021
+ if (entry < 0) {
1022
+ const fallback = unstemmedSpelling(spelling);
1023
+ if (fallback !== null) entry = current.entryOf(fallback);
980
1024
  }
1025
+ if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
981
1026
  }
982
1027
  }
983
1028
  function rebuild(analysis) {
@@ -1031,11 +1076,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
1031
1076
  }
1032
1077
  const corrected = /* @__PURE__ */ new Map();
1033
1078
  countTableSpellings(corrected, table, tokenTable);
1079
+ const removed = /* @__PURE__ */ new Map();
1080
+ const returned = [];
1034
1081
  for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
1035
1082
  const kept = mergedOrdinals[ordinal] >= 0;
1036
1083
  if (kept === isCounted(ordinal)) continue;
1037
- countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, kept ? 1 : -1);
1084
+ if (kept) returned.push(ordinal);
1085
+ else countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
1038
1086
  }
1087
+ subtractRemovedSpellings(corrected, removed, analysis);
1088
+ for (const ordinal of returned) countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, 1);
1039
1089
  addCounts(counts, corrected);
1040
1090
  },
1041
1091
  spellings() {
@@ -1398,4 +1448,4 @@ var partialWordSearchModule = {
1398
1448
  highlightWords: partialWordsToHighlight
1399
1449
  };
1400
1450
 
1401
- export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, resetNormalizationCache, sameFields, suggestDisplayTerms, textFieldsChanged, tokenize, tokenizeOptions, visitTokens };
1451
+ export { FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, resetNormalizationCache, sameFields, suggestDisplayTerms, textFieldsChanged, tokenize, tokenizeOptions, visitDocumentSpellings, visitDocumentSurfaceSpellings, visitTokens };
@@ -1,7 +1,7 @@
1
1
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-YCJLVNWR.mjs';
2
- import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-T7DMUVTL.mjs';
3
- import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-AQU2QHSY.mjs';
4
- import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-XBXKPWMR.mjs';
2
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-QAMUC2BJ.mjs';
3
+ import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-R6BMMQXH.mjs';
4
+ import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-EMXAYPTQ.mjs';
5
5
  import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
6
6
  import { flattenSchema, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
7
7
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
@@ -1,14 +1,14 @@
1
- import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-SYRFSF6V.mjs';
1
+ import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-X4SEXL4U.mjs';
2
2
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Q5DC6WAB.mjs';
3
3
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
4
- import { tokenize } from './chunk-PSUJCOSE.mjs';
4
+ import { tokenize } from './chunk-A3UB2MXL.mjs';
5
5
  import { partialFieldsOf, flattenSchema } from './chunk-CVQWQVFX.mjs';
6
6
  import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
7
7
  import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
8
8
  import { isWordIndexedFieldType } from './chunk-BVOJIYRV.mjs';
9
9
  import { compareComparableKeys } from './chunk-R6A773WK.mjs';
10
10
  import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
11
- import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType } from './chunk-L5ORED7V.mjs';
11
+ import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
12
12
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
13
13
 
14
14
  // src/search/cursor-binding.ts
@@ -220,18 +220,27 @@ function requireFacetableFields(facets, flatSchema, strict) {
220
220
  }
221
221
  }
222
222
  }
223
- function requireDeclaredGroupFields(fields, flatSchema) {
223
+ function requireGroupableFields(fields, flatSchema, strict) {
224
224
  for (const field of fields) {
225
- if (flatSchema[field] === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
225
+ const fieldType = flatSchema[field];
226
+ if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
227
+ if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
228
+ throw new NarsilError(
229
+ ErrorCodes.SEARCH_INVALID_FIELD,
230
+ `The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
231
+ { field, fieldType }
232
+ );
233
+ }
226
234
  }
227
235
  }
228
236
  function requireUsableQueryFields(params, schema, strict = false) {
229
- const checksGroup = strict && Array.isArray(params.group?.fields);
237
+ const groupFields = params.group?.fields;
238
+ const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
230
239
  if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
231
240
  const flatSchema = flattenSchema(schema);
232
241
  if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
233
242
  if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
234
- if (checksGroup && params.group !== void 0) requireDeclaredGroupFields(params.group.fields, flatSchema);
243
+ if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
235
244
  }
236
245
 
237
246
  // src/search/typo-tolerance.ts
@@ -630,6 +639,7 @@ function executeListDocuments(params, context) {
630
639
  const signature = sortSignatureOf(params.sort);
631
640
  requireSortableFields(params.sort, schema, context.strict === true);
632
641
  const filters = validQueryFilter(params.filters, schema);
642
+ const projection = resolveProjection(params.document);
633
643
  const binding = listBindingOf(params.filters);
634
644
  let cursor = null;
635
645
  if (params.cursor !== void 0) {
@@ -640,7 +650,6 @@ function executeListDocuments(params, context) {
640
650
  const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
641
651
  const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
642
652
  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
643
- const projection = resolveProjection(params.document);
644
653
  const documents = [];
645
654
  for (const docId of page.ids) {
646
655
  if (projection.kind === "none") {
@@ -663,37 +672,23 @@ function executeListDocuments(params, context) {
663
672
  }
664
673
 
665
674
  // src/search/grouping.ts
666
- function getNestedValue(obj, path) {
667
- const segments = path.split(".");
668
- let current = obj;
669
- for (const segment of segments) {
670
- if (current === null || current === void 0 || typeof current !== "object") {
671
- return void 0;
672
- }
673
- if (!Object.hasOwn(current, segment)) {
674
- return void 0;
675
- }
676
- current = current[segment];
677
- }
678
- return current;
679
- }
680
675
  function hitsKeptPerGroup(maxPerGroup) {
681
676
  if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
682
677
  return Math.floor(maxPerGroup);
683
678
  }
684
- function applyGrouping(hits, group, getDocument) {
679
+ function applyGrouping(hits, group, documents) {
685
680
  if (!group.fields || group.fields.length === 0) {
686
681
  return [{ values: {}, hits }];
687
682
  }
688
683
  const groupMap = /* @__PURE__ */ new Map();
689
684
  for (const hit of hits) {
690
- const doc = getDocument(hit.id);
685
+ const stored = documents.fieldValues(hit.id, group.fields);
691
686
  const fieldValues = [];
692
687
  const valuesRecord = {};
693
- for (const field of group.fields) {
694
- const value = doc ? getNestedValue(doc, field) : void 0;
688
+ for (let at = 0; at < group.fields.length; at++) {
689
+ const value = stored?.[at];
695
690
  fieldValues.push(value);
696
- valuesRecord[field] = value;
691
+ valuesRecord[group.fields[at]] = value;
697
692
  }
698
693
  const compositeKey = JSON.stringify(fieldValues);
699
694
  const existing = groupMap.get(compositeKey);
@@ -711,7 +706,7 @@ function applyGrouping(hits, group, getDocument) {
711
706
  if (group.reduce) {
712
707
  const folded = foldGroupReducer(
713
708
  group.reduce,
714
- everyHit.map((hit) => ({ document: getDocument(hit.id), score: hit.score ?? 0 }))
709
+ everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
715
710
  );
716
711
  if (folded.reducerError !== void 0) {
717
712
  result.reducerError = folded.reducerError;
@@ -1359,6 +1354,7 @@ async function executeQuery(request, callerContext) {
1359
1354
  }
1360
1355
  const params = validQueryParams(request, config);
1361
1356
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
1357
+ const projection = resolveProjection(params.document);
1362
1358
  let paginated;
1363
1359
  let nextCursor;
1364
1360
  let count;
@@ -1442,7 +1438,10 @@ async function executeQuery(request, callerContext) {
1442
1438
  hits = applySorting(hits, params.sort, sortKeyOf);
1443
1439
  }
1444
1440
  if (params.group) {
1445
- groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
1441
+ groups = applyGrouping(hits, params.group, {
1442
+ fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
1443
+ document: (docId) => manager.getRef(docId)
1444
+ });
1446
1445
  }
1447
1446
  let pinsFromOutside = 0;
1448
1447
  if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
@@ -1482,7 +1481,6 @@ async function executeQuery(request, callerContext) {
1482
1481
  for (const hit of hits) hit.score = void 0;
1483
1482
  }
1484
1483
  }
1485
- const projection = resolveProjection(params.document);
1486
1484
  if (projection.kind === "none") {
1487
1485
  for (const hit of paginated) {
1488
1486
  hit.document = {};