@delali/narsil 0.3.1-next.109 → 0.3.1-next.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-RYWURYTW.mjs → chunk-2CBQ2PCC.mjs} +1 -1
  4. package/dist/browser/{chunk-EQ7BQ6VZ.mjs → chunk-LJS6PREA.mjs} +1 -0
  5. package/dist/browser/{chunk-3RXEGUGO.mjs → chunk-O5IKTA27.mjs} +41 -4
  6. package/dist/browser/{chunk-Z2K5AGZL.mjs → chunk-OVY4RPDS.mjs} +3 -3
  7. package/dist/browser/{chunk-S7KNCVPU.mjs → chunk-P7C2S5JX.mjs} +2 -2
  8. package/dist/browser/{chunk-SCBBEQWD.mjs → chunk-UR5MNN53.mjs} +189 -51
  9. package/dist/browser/{chunk-XHRHOH2P.mjs → chunk-YRYBTRWB.mjs} +139 -28
  10. package/dist/browser/{durability-wiring-JJGE7MED.mjs → durability-wiring-CPEMDDK7.mjs} +48 -20
  11. package/dist/browser/{module-PCKQADMQ.mjs → module-DYKR4INA.mjs} +5 -5
  12. package/dist/browser/{restore-queue-2STSWU2E.mjs → restore-queue-2H4QDQV4.mjs} +1 -1
  13. package/dist/browser/{runtime-ARKW5NZX.mjs → runtime-FL5K4JOH.mjs} +1 -1
  14. package/dist/browser/{snapshot-3AZDQKNA.mjs → snapshot-ISZMBDCN.mjs} +27 -14
  15. package/dist/browser/{write-XXLMPT3Y.mjs → write-IABOLE4N.mjs} +5 -5
  16. package/dist/{capabilities-D4MUmjfe.d.ts → capabilities-ByaX8Arf.d.ts} +2 -2
  17. package/dist/{chunk-CYXLPFBG.mjs → chunk-2IFU7SXG.mjs} +123 -35
  18. package/dist/{chunk-FN75BCXC.mjs → chunk-2RAQKITH.mjs} +1 -1
  19. package/dist/{chunk-VIXXQEX6.mjs → chunk-3COL24MT.mjs} +43 -6
  20. package/dist/{chunk-ICZW4J3I.mjs → chunk-3RRATITJ.mjs} +21 -21
  21. package/dist/{chunk-SAZB7ZHT.mjs → chunk-542TD4BL.mjs} +1 -1
  22. package/dist/{chunk-C57FPCTL.mjs → chunk-5WEBVYOA.mjs} +182 -44
  23. package/dist/{chunk-YP23IAV5.mjs → chunk-63HFWQEA.mjs} +4 -3
  24. package/dist/{chunk-FV5CP3E2.mjs → chunk-6ODIY6ZB.mjs} +1 -1
  25. package/dist/{chunk-RQ7LOBLD.mjs → chunk-6PN3U6XW.mjs} +1 -1
  26. package/dist/{chunk-SYHS56F7.mjs → chunk-7CI34JAP.mjs} +26 -13
  27. package/dist/{chunk-IBJWO5PF.mjs → chunk-7ZZQNOIY.mjs} +1 -1
  28. package/dist/{chunk-H5465KUZ.mjs → chunk-BVOUSX3U.mjs} +4 -3
  29. package/dist/{chunk-AWZVJCCW.mjs → chunk-DG62H7OD.mjs} +2 -2
  30. package/dist/{chunk-LWQTFKRS.mjs → chunk-FUUP45IS.mjs} +1 -1
  31. package/dist/{chunk-HL2GAS5N.mjs → chunk-GLZI636G.mjs} +7 -7
  32. package/dist/{chunk-C55OFKU6.mjs → chunk-HNAP2KTG.mjs} +2 -2
  33. package/dist/{chunk-A2HWRBQE.mjs → chunk-IBTKJIV7.mjs} +2 -2
  34. package/dist/{chunk-PA5WZWLC.mjs → chunk-IM76DLA7.mjs} +2 -2
  35. package/dist/{chunk-5LP7OU6B.mjs → chunk-IR4KQ4WB.mjs} +2 -2
  36. package/dist/{chunk-FABI2JA7.mjs → chunk-J6YVB2XM.mjs} +1 -1
  37. package/dist/{chunk-62GK7UUS.mjs → chunk-KCQG2PJT.mjs} +3 -3
  38. package/dist/{chunk-U6FT7LKG.mjs → chunk-LOBYK2FG.mjs} +3 -3
  39. package/dist/{chunk-XY6Z7T5W.mjs → chunk-N57XXGPG.mjs} +2 -2
  40. package/dist/{chunk-S2CKO6PP.mjs → chunk-OBWUWZRA.mjs} +1 -0
  41. package/dist/{chunk-CKRP7GCS.mjs → chunk-OCB7TMUQ.mjs} +1 -1
  42. package/dist/{chunk-NYWUBTMH.mjs → chunk-OFYKBKKV.mjs} +1 -1
  43. package/dist/{chunk-ZBZRBM2Z.mjs → chunk-PT7VRWS7.mjs} +3 -3
  44. package/dist/{chunk-DUWYU4D5.mjs → chunk-PWIWSUCX.mjs} +2 -2
  45. package/dist/{chunk-5W7ADHK7.mjs → chunk-QI6JW4PM.mjs} +1 -1
  46. package/dist/{chunk-SD2EYVU4.mjs → chunk-QIFOXLWF.mjs} +2 -2
  47. package/dist/{chunk-N75MAOM7.mjs → chunk-R2MRGYU5.mjs} +2 -2
  48. package/dist/{chunk-5DPTHLMX.mjs → chunk-V7G627TA.mjs} +1 -1
  49. package/dist/{chunk-4GD43FSW.mjs → chunk-V7ISFBFC.mjs} +10 -10
  50. package/dist/{chunk-MVWWAHTR.mjs → chunk-VK45J47R.mjs} +1 -1
  51. package/dist/{chunk-RSSLOFRP.mjs → chunk-VLRT47GU.mjs} +39 -8
  52. package/dist/{chunk-RPIE4QCL.mjs → chunk-YH5OIL7B.mjs} +2 -0
  53. package/dist/{chunk-JKOBHB4A.mjs → chunk-YIDSHBQS.mjs} +2 -2
  54. package/dist/{client-Ba9yP-Rr.d.ts → client-BdnzDFtO.d.ts} +2 -2
  55. package/dist/client.d.ts +6 -6
  56. package/dist/client.mjs +1 -0
  57. package/dist/{config-C-xXz8bC.d.ts → config-CSgvApt0.d.ts} +1 -1
  58. package/dist/{config-Dc_8-byz.d.ts → config-ujZESkJz.d.ts} +1 -1
  59. package/dist/{direct-executor-QH2M6AK2.mjs → direct-executor-ZSTKRA6Z.mjs} +11 -11
  60. package/dist/distribution/coordinator/etcd.mjs +94 -5
  61. package/dist/distribution/transport/grpc.mjs +24 -8
  62. package/dist/distribution/transport/in-memory.mjs +4 -4
  63. package/dist/distribution/transport/tcp.mjs +6 -6
  64. package/dist/distribution.d.ts +5 -5
  65. package/dist/distribution.mjs +1232 -377
  66. package/dist/{durability-wiring-4HWX7OVP.mjs → durability-wiring-EQTFL3B4.mjs} +65 -38
  67. package/dist/embeddings/openai.mjs +2 -2
  68. package/dist/{engine-CMZ5XSLy.d.ts → engine-C_HSy3iz.d.ts} +1 -1
  69. package/dist/{events-dnvgprIl.d.ts → events-CJsXF1F9.d.ts} +1 -0
  70. package/dist/index.browser.mjs +3 -3
  71. package/dist/index.d.ts +5 -5
  72. package/dist/index.mjs +16 -16
  73. package/dist/{module-IYILOTU3.mjs → module-WP4VVGV2.mjs} +9 -9
  74. package/dist/partial-words.mjs +5 -5
  75. package/dist/pattern.mjs +3 -3
  76. package/dist/persistence/durability/checkpoint-worker.d.ts +12 -0
  77. package/dist/persistence/durability/checkpoint-worker.mjs +25 -23
  78. package/dist/react.d.ts +6 -6
  79. package/dist/react.mjs +1 -0
  80. package/dist/{rerank-EHJVXEDP.mjs → rerank-3POMPQNX.mjs} +4 -4
  81. package/dist/rerankers/http.mjs +2 -2
  82. package/dist/{restore-queue-BONSQWCI.mjs → restore-queue-WZTMTJP2.mjs} +1 -1
  83. package/dist/{runtime-LEX2F7OW.mjs → runtime-C2435MPK.mjs} +5 -5
  84. package/dist/server.d.ts +8 -8
  85. package/dist/server.mjs +12 -12
  86. package/dist/{snapshot-DWDLN5YW.mjs → snapshot-EU6NNB2P.mjs} +25 -25
  87. package/dist/synonyms.mjs +3 -3
  88. package/dist/{tasks-B4B2cMK5.d.ts → tasks-CI3Fvkxw.d.ts} +1 -1
  89. package/dist/{thread-I32PUURR.mjs → thread-WGBCVJII.mjs} +11 -11
  90. package/dist/vector/search-worker.mjs +4 -4
  91. package/dist/workers/entry.mjs +5 -5
  92. package/dist/{write-AOFSHACR.mjs → write-CKRWS3P5.mjs} +18 -18
  93. package/package.json +7 -7
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-S2CKO6PP.mjs';
1
+ import { NarsilError, ErrorCodes } from '../chunk-OBWUWZRA.mjs';
2
2
 
3
3
  // src/persistence/filesystem.ts
4
4
  var cachedFs = null;
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-S2CKO6PP.mjs';
1
+ import { NarsilError, ErrorCodes } from '../chunk-OBWUWZRA.mjs';
2
2
 
3
3
  // src/persistence/constants.ts
4
4
  var DEFAULT_INDEXEDDB_NAME = "narsil";
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
1
+ import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
2
2
 
3
3
  // src/serialization/constants.ts
4
4
  var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
@@ -70,6 +70,7 @@ var ErrorCodes = {
70
70
  REPLICATION_INSYNC_REMOVAL_FAILED: "REPLICATION_INSYNC_REMOVAL_FAILED",
71
71
  REPLICATION_ROLLBACK_FAILED: "REPLICATION_ROLLBACK_FAILED",
72
72
  REPLICATION_OUTCOME_UNKNOWN: "REPLICATION_OUTCOME_UNKNOWN",
73
+ REPLICATION_WRITE_CONFLICT: "REPLICATION_WRITE_CONFLICT",
73
74
  REPLICATION_LOG_FULL: "REPLICATION_LOG_FULL",
74
75
  REPLICATION_ENTRY_CORRUPT: "REPLICATION_ENTRY_CORRUPT",
75
76
  REPLICATION_SNAPSHOT_CORRUPT: "REPLICATION_SNAPSHOT_CORRUPT",
@@ -1,5 +1,5 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-SCBBEQWD.mjs';
2
- import { NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-UR5MNN53.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
5
5
  function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
@@ -465,6 +465,42 @@ function decodePartialWords(raw, tokenCount, context) {
465
465
  return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
466
466
  }
467
467
 
468
+ // src/serialization/keyword-segment/write-records.ts
469
+ var UINT64_BYTES = 8;
470
+ function uint64Bytes(values) {
471
+ const bytes = new Uint8Array(values.length * UINT64_BYTES);
472
+ const view = new DataView(bytes.buffer);
473
+ for (let at = 0; at < values.length; at++) view.setBigUint64(at * UINT64_BYTES, BigInt(values[at]), true);
474
+ return bytes;
475
+ }
476
+ function uint64sOf(bytes, name) {
477
+ const values = new Float64Array(bytes.byteLength / UINT64_BYTES);
478
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
479
+ for (let at = 0; at < values.length; at++) {
480
+ const value = view.getBigUint64(at * UINT64_BYTES, true);
481
+ if (value > BigInt(Number.MAX_SAFE_INTEGER)) corrupt(`${name} holds a value above 2^53 - 1`, { at });
482
+ values[at] = Number(value);
483
+ }
484
+ return values;
485
+ }
486
+ function decodeWriteRecords(raw, documentCount) {
487
+ const rawSeqNos = requireBytesOrNil(raw.seqNos, "seqNos", UINT64_BYTES);
488
+ const rawTerms = requireBytesOrNil(raw.primaryTerms, "primaryTerms", UINT64_BYTES);
489
+ if (rawSeqNos === null !== (rawTerms === null))
490
+ corrupt("seqNos and primaryTerms must both be present or both absent");
491
+ if (rawSeqNos === null || rawTerms === null) return null;
492
+ const seqNos = uint64sOf(rawSeqNos, "seqNos");
493
+ const primaryTerms = uint64sOf(rawTerms, "primaryTerms");
494
+ requireCount(seqNos.length, documentCount, "seqNos");
495
+ requireCount(primaryTerms.length, documentCount, "primaryTerms");
496
+ for (let at = 0; at < documentCount; at++) {
497
+ if (seqNos[at] === 0 && primaryTerms[at] !== 0) {
498
+ corrupt(`primaryTerms gives document ${at} a term without a seqNo`, { at });
499
+ }
500
+ }
501
+ return { seqNos, primaryTerms };
502
+ }
503
+
468
504
  // src/serialization/keyword-segment/decode.ts
469
505
  function decodeSchema(raw) {
470
506
  const schema = requireRecord(raw, "schema");
@@ -596,7 +632,8 @@ function decodeKeywordSegment(payload, shared) {
596
632
  surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
597
633
  partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
598
634
  };
599
- return { schema, tombstones, arrays };
635
+ const writeRecords = decodeWriteRecords(raw, documentCount);
636
+ return writeRecords === null ? { schema, tombstones, arrays } : { schema, tombstones, arrays, writeRecords };
600
637
  }
601
638
 
602
- export { KEYWORD_SEGMENT_ENVELOPE_VERSION, KEYWORD_SEGMENT_PAYLOAD_VERSION, applyDeleteEntry, applyIndexEntry, decodeKeywordSegment, littleEndianBytes, restoreVectorFields, setNestedValue };
639
+ export { KEYWORD_SEGMENT_ENVELOPE_VERSION, KEYWORD_SEGMENT_PAYLOAD_VERSION, applyDeleteEntry, applyIndexEntry, decodeKeywordSegment, littleEndianBytes, restoreVectorFields, setNestedValue, uint64Bytes };
@@ -1,6 +1,6 @@
1
- import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-S7KNCVPU.mjs';
2
- import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-RYWURYTW.mjs';
3
- import { NarsilError, ErrorCodes, MAX_DOC_ID_LENGTH } from './chunk-EQ7BQ6VZ.mjs';
1
+ import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-P7C2S5JX.mjs';
2
+ import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, GRAPH_LOCK_WAIT_DEADLINE_MS, compareCodePoints, INITIAL_UPPER_SLOTS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, threadSlotGone, DEFAULT_EF_SEARCH, OSQ_LAMBDA, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-2CBQ2PCC.mjs';
3
+ import { NarsilError, ErrorCodes, MAX_DOC_ID_LENGTH } from './chunk-LJS6PREA.mjs';
4
4
 
5
5
  // src/vector/shared-buffers/growable.ts
6
6
  function sharedMemoryAvailable() {
@@ -1,5 +1,5 @@
1
- import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-RYWURYTW.mjs';
2
- import { NarsilError, ErrorCodes } from './chunk-EQ7BQ6VZ.mjs';
1
+ import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-2CBQ2PCC.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
3
3
 
4
4
  // src/vector/vector-index/dead-bits.ts
5
5
  var NO_LIVE_VECTOR_AT_POSITION = -1;
@@ -1,6 +1,6 @@
1
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-RYWURYTW.mjs';
1
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-2CBQ2PCC.mjs';
2
2
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
3
- import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-EQ7BQ6VZ.mjs';
3
+ import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-LJS6PREA.mjs';
4
4
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
5
5
 
6
6
  // src/analysis/registry.ts
@@ -3245,6 +3245,8 @@ function createDocumentStore() {
3245
3245
  // src/core/constants.ts
3246
3246
  var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
3247
3247
  var MAX_TERM_FREQUENCY = 65535;
3248
+ var PENDING_REMOVAL_MERGE_FLOOR = 64;
3249
+ var PENDING_REMOVAL_MERGE_DIVISOR = 256;
3248
3250
 
3249
3251
  // src/core/fuzzy.ts
3250
3252
  var FIRST_SURROGATE = 55296;
@@ -4944,6 +4946,7 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
4944
4946
  }
4945
4947
 
4946
4948
  // src/geo/geo-index.ts
4949
+ var REMOVED_DOC_ID = -1;
4947
4950
  function firstAtOrAfter(entries, code) {
4948
4951
  let lo = 0;
4949
4952
  let hi = entries.length;
@@ -4954,25 +4957,58 @@ function firstAtOrAfter(entries, code) {
4954
4957
  }
4955
4958
  return lo;
4956
4959
  }
4960
+ function dropRemoved(entries) {
4961
+ let write = 0;
4962
+ for (let read = 0; read < entries.length; read++) {
4963
+ if (entries[read].docId !== REMOVED_DOC_ID) entries[write++] = entries[read];
4964
+ }
4965
+ entries.length = write;
4966
+ }
4967
+ function mergeSortedPending(entries, pending) {
4968
+ pending.sort((left2, right2) => left2.code - right2.code || left2.docId - right2.docId);
4969
+ const mainLength = entries.length;
4970
+ for (const entry of pending) entries.push(entry);
4971
+ let left = mainLength - 1;
4972
+ let right = pending.length - 1;
4973
+ let write = entries.length - 1;
4974
+ while (right >= 0) {
4975
+ const later = left >= 0 ? entries[left] : void 0;
4976
+ const next = pending[right];
4977
+ if (later !== void 0 && (later.code > next.code || later.code === next.code && later.docId > next.docId)) {
4978
+ entries[write--] = later;
4979
+ left--;
4980
+ } else {
4981
+ entries[write--] = next;
4982
+ right--;
4983
+ }
4984
+ }
4985
+ }
4957
4986
  function createGeoIndex() {
4958
- const entries = [];
4987
+ let entries = [];
4988
+ let pending = [];
4959
4989
  const byDocument = /* @__PURE__ */ new Map();
4960
- let sorted = true;
4990
+ let removedCount = 0;
4961
4991
  function ensureSorted() {
4962
- if (sorted) return;
4963
- entries.sort((left, right) => left.code - right.code || left.docId - right.docId);
4964
- sorted = true;
4965
- }
4966
- function removeEntry(internalId) {
4967
- const entry = byDocument.get(internalId);
4968
- if (entry === void 0) return;
4969
- byDocument.delete(internalId);
4970
- ensureSorted();
4971
- for (let at = firstAtOrAfter(entries, entry.code); at < entries.length; at++) {
4972
- if (entries[at] !== entry) continue;
4973
- entries.splice(at, 1);
4974
- return;
4992
+ if (removedCount > 0) {
4993
+ dropRemoved(entries);
4994
+ dropRemoved(pending);
4995
+ removedCount = 0;
4996
+ }
4997
+ if (pending.length === 0) return;
4998
+ if (entries.length === 0) {
4999
+ pending.sort((left, right) => left.code - right.code || left.docId - right.docId);
5000
+ entries = pending;
5001
+ } else {
5002
+ mergeSortedPending(entries, pending);
4975
5003
  }
5004
+ pending = [];
5005
+ }
5006
+ function removeEntry(entry) {
5007
+ byDocument.delete(entry.docId);
5008
+ entry.docId = REMOVED_DOC_ID;
5009
+ removedCount++;
5010
+ const held = entries.length + pending.length;
5011
+ if (removedCount >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) ensureSorted();
4976
5012
  }
4977
5013
  function matching(boxes, passes, inside) {
4978
5014
  ensureSorted();
@@ -4991,14 +5027,15 @@ function createGeoIndex() {
4991
5027
  }
4992
5028
  const index = {
4993
5029
  insert(internalId, lat, lon) {
4994
- removeEntry(internalId);
5030
+ const existing = byDocument.get(internalId);
5031
+ if (existing !== void 0) removeEntry(existing);
4995
5032
  const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
4996
- entries.push(entry);
5033
+ pending.push(entry);
4997
5034
  byDocument.set(internalId, entry);
4998
- sorted = false;
4999
5035
  },
5000
5036
  remove(internalId) {
5001
- removeEntry(internalId);
5037
+ const existing = byDocument.get(internalId);
5038
+ if (existing !== void 0) removeEntry(existing);
5002
5039
  },
5003
5040
  radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
5004
5041
  const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
@@ -5014,13 +5051,14 @@ function createGeoIndex() {
5014
5051
  },
5015
5052
  documentsBitset(capacity) {
5016
5053
  const bits = createBitSet(capacity);
5017
- for (const entry of entries) bitsetSet(bits, entry.docId);
5054
+ for (const docId of byDocument.keys()) bitsetSet(bits, docId);
5018
5055
  return bits;
5019
5056
  },
5020
5057
  clear() {
5021
- entries.length = 0;
5058
+ entries = [];
5059
+ pending = [];
5022
5060
  byDocument.clear();
5023
- sorted = true;
5061
+ removedCount = 0;
5024
5062
  },
5025
5063
  serialize() {
5026
5064
  ensureSorted();
@@ -5348,29 +5386,126 @@ function collectDocIdsBitset(entries, from, to, capacity) {
5348
5386
  }
5349
5387
  return bs;
5350
5388
  }
5389
+ function removalsFor(byDocument, docId, value) {
5390
+ const keys = byDocument.get(docId);
5391
+ if (keys === void 0) return void 0;
5392
+ for (const key of keys) {
5393
+ if (key.value === value) return key;
5394
+ }
5395
+ return void 0;
5396
+ }
5397
+ function groupRemovals(docIds, values, limits) {
5398
+ const byDocument = /* @__PURE__ */ new Map();
5399
+ for (let at = 0; at < docIds.length; at++) {
5400
+ const existing = removalsFor(byDocument, docIds[at], values[at]);
5401
+ if (existing !== void 0) {
5402
+ existing.limits.push(limits[at]);
5403
+ continue;
5404
+ }
5405
+ const key = { value: values[at], limits: [limits[at]], next: 0 };
5406
+ const keys = byDocument.get(docIds[at]);
5407
+ if (keys === void 0) byDocument.set(docIds[at], [key]);
5408
+ else keys.push(key);
5409
+ }
5410
+ return byDocument;
5411
+ }
5412
+ function markedDocuments(docIds) {
5413
+ let highest = 0;
5414
+ for (const docId of docIds) if (docId > highest) highest = docId;
5415
+ const marked = new Uint8Array(highest + 1);
5416
+ for (const docId of docIds) marked[docId] = 1;
5417
+ return marked;
5418
+ }
5419
+ function removeFromSorted(entries, from, marked, byDocument) {
5420
+ let write = from;
5421
+ for (let read = from; read < entries.length; read++) {
5422
+ const entry = entries[read];
5423
+ if (entry.docId < marked.length && marked[entry.docId] === 1) {
5424
+ const key = removalsFor(byDocument, entry.docId, entry.value);
5425
+ if (key !== void 0 && key.next < key.limits.length) {
5426
+ key.next++;
5427
+ continue;
5428
+ }
5429
+ }
5430
+ entries[write++] = entry;
5431
+ }
5432
+ entries.length = write;
5433
+ }
5434
+ function removeFromPending(pending, marked, byDocument) {
5435
+ let write = 0;
5436
+ for (let read = 0; read < pending.length; read++) {
5437
+ const entry = pending[read];
5438
+ if (entry.docId < marked.length && marked[entry.docId] === 1) {
5439
+ const key = removalsFor(byDocument, entry.docId, entry.value);
5440
+ if (key !== void 0) {
5441
+ while (key.next < key.limits.length && key.limits[key.next] <= read) key.next++;
5442
+ if (key.next < key.limits.length) {
5443
+ key.next++;
5444
+ continue;
5445
+ }
5446
+ }
5447
+ }
5448
+ pending[write++] = entry;
5449
+ }
5450
+ pending.length = write;
5451
+ }
5452
+ function mergeSortedPending2(entries, pending) {
5453
+ pending.sort((a, b) => a.value - b.value);
5454
+ const mainLength = entries.length;
5455
+ for (const entry of pending) entries.push(entry);
5456
+ let left = mainLength - 1;
5457
+ let right = pending.length - 1;
5458
+ let write = entries.length - 1;
5459
+ while (right >= 0) {
5460
+ if (left >= 0 && entries[left].value > pending[right].value) entries[write--] = entries[left--];
5461
+ else entries[write--] = pending[right--];
5462
+ }
5463
+ }
5351
5464
  function createNumericIndex() {
5352
5465
  let entries = [];
5353
- let sorted = true;
5466
+ let pending = [];
5467
+ const removedDocIds = [];
5468
+ const removedValues = [];
5469
+ const removedLimits = [];
5470
+ function applyRemovals() {
5471
+ const byDocument = groupRemovals(removedDocIds, removedValues, removedLimits);
5472
+ const marked = markedDocuments(removedDocIds);
5473
+ let lowest = removedValues[0];
5474
+ for (const value of removedValues) if (value < lowest) lowest = value;
5475
+ removeFromSorted(entries, lowerBound(entries, lowest), marked, byDocument);
5476
+ if (pending.length > 0) removeFromPending(pending, marked, byDocument);
5477
+ removedDocIds.length = 0;
5478
+ removedValues.length = 0;
5479
+ removedLimits.length = 0;
5480
+ }
5354
5481
  function ensureSorted() {
5355
- if (!sorted) {
5356
- entries.sort((a, b) => a.value - b.value);
5357
- sorted = true;
5482
+ if (removedDocIds.length > 0) applyRemovals();
5483
+ if (pending.length === 0) return;
5484
+ if (entries.length === 0) {
5485
+ pending.sort((a, b) => a.value - b.value);
5486
+ entries = pending;
5487
+ } else {
5488
+ mergeSortedPending2(entries, pending);
5358
5489
  }
5490
+ pending = [];
5491
+ }
5492
+ function discardPending() {
5493
+ pending = [];
5494
+ removedDocIds.length = 0;
5495
+ removedValues.length = 0;
5496
+ removedLimits.length = 0;
5359
5497
  }
5360
5498
  return {
5361
5499
  insert(internalId, value) {
5362
- entries.push({ value, docId: internalId });
5363
- sorted = false;
5500
+ pending.push({ value, docId: internalId });
5364
5501
  },
5365
5502
  remove(internalId, value) {
5366
- ensureSorted();
5367
- const start = lowerBound(entries, value);
5368
- const end = upperBound(entries, value);
5369
- for (let i = start; i < end; i++) {
5370
- if (entries[i].docId === internalId) {
5371
- entries.splice(i, 1);
5372
- return;
5373
- }
5503
+ removedDocIds.push(internalId);
5504
+ removedValues.push(value);
5505
+ removedLimits.push(pending.length);
5506
+ const held = entries.length + pending.length;
5507
+ if (removedDocIds.length >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) {
5508
+ ensureSorted();
5374
5509
  }
5375
5510
  },
5376
5511
  queryEq(value) {
@@ -5407,6 +5542,7 @@ function createNumericIndex() {
5407
5542
  return collectDocIds(entries, lowerBound(entries, min), upperBound(entries, max));
5408
5543
  },
5409
5544
  getAllDocIds() {
5545
+ ensureSorted();
5410
5546
  return collectDocIds(entries, 0, entries.length);
5411
5547
  },
5412
5548
  queryEqBitset(value, capacity) {
@@ -5434,9 +5570,11 @@ function createNumericIndex() {
5434
5570
  return collectDocIdsBitset(entries, lowerBound(entries, min), upperBound(entries, max), capacity);
5435
5571
  },
5436
5572
  getAllDocIdsBitset(capacity) {
5573
+ ensureSorted();
5437
5574
  return collectDocIdsBitset(entries, 0, entries.length, capacity);
5438
5575
  },
5439
5576
  count() {
5577
+ ensureSorted();
5440
5578
  return entries.length;
5441
5579
  },
5442
5580
  facetValueCounts(matched) {
@@ -5474,7 +5612,7 @@ function createNumericIndex() {
5474
5612
  },
5475
5613
  clear() {
5476
5614
  entries = [];
5477
- sorted = true;
5615
+ discardPending();
5478
5616
  },
5479
5617
  serialize() {
5480
5618
  ensureSorted();
@@ -5482,7 +5620,7 @@ function createNumericIndex() {
5482
5620
  },
5483
5621
  deserialize(data) {
5484
5622
  entries = data.slice();
5485
- sorted = true;
5623
+ discardPending();
5486
5624
  }
5487
5625
  };
5488
5626
  }
@@ -14949,7 +15087,7 @@ function requireLoadedVectorCode() {
14949
15087
  function loadVectorCode() {
14950
15088
  if (loaded !== null) return Promise.resolve(loaded);
14951
15089
  if (loading === null) {
14952
- loading = import('./module-PCKQADMQ.mjs').then(
15090
+ loading = import('./module-DYKR4INA.mjs').then(
14953
15091
  (code) => {
14954
15092
  loaded = code;
14955
15093
  return code;
@@ -19026,7 +19164,7 @@ async function emitWriteEventAfter(source, indexName, write) {
19026
19164
  // src/engine/core.ts
19027
19165
  async function createEngineCore(config, hooks, capacityScope = "index") {
19028
19166
  validateWorkerConfig(config?.workers, config?.lifecycle);
19029
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-ARKW5NZX.mjs')).resolveRerankSettings(config);
19167
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-FL5K4JOH.mjs')).resolveRerankSettings(config);
19030
19168
  const eventHandlers = eventHandlersOf(config?.events);
19031
19169
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
19032
19170
  const vectorCopyPolicy = {
@@ -19037,7 +19175,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19037
19175
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19038
19176
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19039
19177
  }
19040
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-JJGE7MED.mjs');
19178
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-CPEMDDK7.mjs');
19041
19179
  const filesystemDurability = durabilityTier?.kind === "wal";
19042
19180
  const executor = createDirectExecutor({
19043
19181
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19334,7 +19472,7 @@ async function shutdownEngine(core) {
19334
19472
  }
19335
19473
  }
19336
19474
  if (core.rerankAdapters.size > 0) {
19337
- await (await import('./runtime-ARKW5NZX.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19475
+ await (await import('./runtime-FL5K4JOH.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19338
19476
  }
19339
19477
  await orchestrator.shutdown();
19340
19478
  await executor.shutdown();
@@ -22470,14 +22608,14 @@ function portabilityOperations(core, config, engine) {
22470
22608
  return {
22471
22609
  snapshot(indexName) {
22472
22610
  return withOpenIndex(indexName, async () => {
22473
- const { createSnapshot } = await import('./snapshot-3AZDQKNA.mjs');
22611
+ const { createSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
22474
22612
  return createSnapshot(snapshotDepsOf(indexName));
22475
22613
  });
22476
22614
  },
22477
22615
  async restore(indexName, data) {
22478
22616
  core.guardShutdown();
22479
22617
  validateIndexName(indexName);
22480
- const restored = import('./restore-queue-2STSWU2E.mjs').then(
22618
+ const restored = import('./restore-queue-2H4QDQV4.mjs').then(
22481
22619
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22482
22620
  );
22483
22621
  await emitWriteEventAfter(core, indexName, restored);
@@ -22486,8 +22624,8 @@ function portabilityOperations(core, config, engine) {
22486
22624
  exportDocuments(indexName) {
22487
22625
  return withOpenIndex(indexName, async () => {
22488
22626
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22489
- import('./snapshot-3AZDQKNA.mjs'),
22490
- import('./write-XXLMPT3Y.mjs')
22627
+ import('./snapshot-ISZMBDCN.mjs'),
22628
+ import('./write-IABOLE4N.mjs')
22491
22629
  ]);
22492
22630
  const header = indexConfigurationOf(core.requireIndex(indexName));
22493
22631
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -23737,13 +23875,13 @@ function requireCursorDepth(cursor, signature, binding) {
23737
23875
  }
23738
23876
  function hybridSearchOf(params, context, limit, offset, cursorDepth) {
23739
23877
  if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
23740
- return import('./runtime-ARKW5NZX.mjs').then(
23878
+ return import('./runtime-FL5K4JOH.mjs').then(
23741
23879
  (runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
23742
23880
  );
23743
23881
  }
23744
23882
  function executeQuery(request, callerContext) {
23745
23883
  if (request.rerank === void 0) return runQuery(request, callerContext);
23746
- return import('./runtime-ARKW5NZX.mjs').then(
23884
+ return import('./runtime-FL5K4JOH.mjs').then(
23747
23885
  (runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
23748
23886
  );
23749
23887
  }
@@ -24124,7 +24262,7 @@ async function runEngineQuery(core, indexName, params, options) {
24124
24262
  const manager = core.requireManager(indexName);
24125
24263
  const scoped = scopedParams(params, options);
24126
24264
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
24127
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-ARKW5NZX.mjs');
24265
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-FL5K4JOH.mjs');
24128
24266
  const resolvedParams = await resolveVectorText(
24129
24267
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
24130
24268
  entry.embeddingAdapter,