@delali/narsil 0.3.1-next.180 → 0.3.1-next.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
  4. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
  5. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  6. package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
  7. package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
  8. package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
  9. package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
  10. package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
  11. package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
  12. package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
  13. package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
  14. package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
  15. package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
  16. package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
  17. package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
  18. package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
  19. package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
  20. package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
  21. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  22. package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
  23. package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
  24. package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
  25. package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
  26. package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
  27. package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
  28. package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
  29. package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
  30. package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
  31. package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
  32. package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
  33. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  36. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  37. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
  38. package/dist/distribution.d.ts +4 -4
  39. package/dist/distribution.mjs +391 -138
  40. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  41. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  42. package/dist/index.browser.mjs +1 -1
  43. package/dist/index.d.ts +4 -4
  44. package/dist/index.mjs +1 -1
  45. package/dist/index.react-native.mjs +1 -1
  46. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
  47. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
  48. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
  49. package/dist/partial-words.browser.mjs +1 -1
  50. package/dist/partial-words.mjs +1 -1
  51. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  52. package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
  53. package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
  54. package/dist/query-suggestions/merge-worker.mjs +4 -4
  55. package/dist/query-suggestions.browser.mjs +1 -1
  56. package/dist/query-suggestions.d.ts +1 -1
  57. package/dist/query-suggestions.mjs +2 -2
  58. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
  59. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
  60. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
  61. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
  62. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
  64. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  65. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
  66. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
  67. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
  68. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
  69. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
  70. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
  71. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
  72. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
  73. package/dist/react-native/partial-words.mjs +1 -1
  74. package/dist/react-native/query-suggestions.mjs +1 -1
  75. package/dist/react-native/sync.mjs +1 -1
  76. package/dist/react-native/synonyms.mjs +1 -1
  77. package/dist/react.d.ts +7 -7
  78. package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
  79. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
  80. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
  81. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
  82. package/dist/server.d.ts +8 -8
  83. package/dist/server.mjs +16 -16
  84. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
  85. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
  86. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
  87. package/dist/sync/filesystem.d.ts +2 -2
  88. package/dist/sync/filesystem.mjs +6 -6
  89. package/dist/sync/indexeddb.d.ts +2 -2
  90. package/dist/sync.browser.mjs +1 -1
  91. package/dist/sync.d.ts +3 -3
  92. package/dist/sync.mjs +1 -1
  93. package/dist/synonyms.browser.mjs +1 -1
  94. package/dist/synonyms.mjs +1 -1
  95. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  96. package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
  97. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  98. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
  99. package/dist/workers/entry.mjs +7 -7
  100. package/openapi.json +32 -0
  101. package/package.json +7 -7
@@ -1,5 +1,5 @@
1
- import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-JCWC75NH.mjs';
2
- import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-2C3BXZGA.mjs';
1
+ import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-C77OBNPP.mjs';
2
+ import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-CWXR7OP4.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
4
4
 
5
5
  // src/query-suggestions/engine-hooks.ts
@@ -14,10 +14,12 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
14
14
  var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
15
15
  var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
16
16
  var LOG_RECORDS_PER_CHECKPOINT = 1e5;
17
+ var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
18
+ var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
17
19
  var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
18
20
  var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
19
21
  var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
20
22
  var CHECKPOINT_WORKER_IDLE_MS = 5e3;
21
23
  var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
22
24
 
23
- export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, SEGMENT_FILE_FRAMING_SLACK_BYTES, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES };
25
+ export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, SEGMENT_FILE_FRAMING_SLACK_BYTES, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES };
@@ -1,6 +1,6 @@
1
- import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-KHJPMZE2.mjs';
1
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-PK3BGGHO.mjs';
2
2
  import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-TNEU32EN.mjs';
3
- import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-JCWC75NH.mjs';
3
+ import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-C77OBNPP.mjs';
4
4
  import { decode } from './chunk-GXCDWLQI.mjs';
5
5
  import { requireValidFieldTypes } from './chunk-X5NHWOXT.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
@@ -1,4 +1,4 @@
1
- import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-2C3BXZGA.mjs';
1
+ import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-CWXR7OP4.mjs';
2
2
  import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-X5NHWOXT.mjs';
3
3
 
4
4
  // src/core/partition/pattern-snapshot.ts
@@ -1,4 +1,4 @@
1
- import { fnv1a } from './chunk-2C3BXZGA.mjs';
1
+ import { fnv1a } from './chunk-CWXR7OP4.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
3
3
 
4
4
  // src/persistence/durability/segment/layout.ts
@@ -1,6 +1,6 @@
1
- import { isValidInteger, isRecord } from './chunk-XQQKETAF.mjs';
1
+ import { isValidInteger, isRecord } from './chunk-F2UOTRY7.mjs';
2
2
  import { MAX_REPLICATION_FACTOR, MAX_PARTITION_COUNT } from './chunk-JDDXV6UF.mjs';
3
- import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-JCWC75NH.mjs';
3
+ import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-C77OBNPP.mjs';
4
4
  import { decode } from './chunk-GXCDWLQI.mjs';
5
5
  import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
6
6
  import { joinChunks } from './chunk-IE3Q62QD.mjs';
@@ -1,7 +1,7 @@
1
1
  import { encode } from './chunk-CV3Q2EKO.mjs';
2
2
  import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
3
- import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
4
- import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-2C3BXZGA.mjs';
3
+ import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
4
+ import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-CWXR7OP4.mjs';
5
5
  import { decode } from './chunk-GXCDWLQI.mjs';
6
6
  import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
7
7
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
@@ -1,5 +1,5 @@
1
1
  import { boundQuerySuggestionsOf } from './chunk-S2POFTLN.mjs';
2
- import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-2C3BXZGA.mjs';
2
+ import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-CWXR7OP4.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
4
4
 
5
5
  // src/query-suggestions/index.ts
@@ -1,7 +1,7 @@
1
- import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-KHJPMZE2.mjs';
1
+ import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-PK3BGGHO.mjs';
2
2
  import { encode } from './chunk-CV3Q2EKO.mjs';
3
3
  import { createFixedBuffer, createVectorFileReader } from './chunk-TNEU32EN.mjs';
4
- import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-JCWC75NH.mjs';
4
+ import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-C77OBNPP.mjs';
5
5
  import { decode } from './chunk-GXCDWLQI.mjs';
6
6
  import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-D6FYUTFN.mjs';
7
7
  import { ErrorCodes, NarsilError } from './chunk-Q3ZZH5Z7.mjs';
@@ -470,6 +470,12 @@ function createReadCache(capacity) {
470
470
  function evictDownTo(limit) {
471
471
  for (const [key, entry] of entries) {
472
472
  if (held <= limit) return;
473
+ if (entry.referenced) {
474
+ entry.referenced = false;
475
+ entries.delete(key);
476
+ entries.set(key, entry);
477
+ continue;
478
+ }
473
479
  drop(key, entry);
474
480
  }
475
481
  }
@@ -524,8 +530,7 @@ function createReadCache(capacity) {
524
530
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
525
531
  const found = entries.get(key);
526
532
  if (found === void 0) return void 0;
527
- entries.delete(key);
528
- entries.set(key, found);
533
+ found.referenced = true;
529
534
  return found.value;
530
535
  },
531
536
  put(fileNumber, kind, entry, value, bytes) {
@@ -533,7 +538,7 @@ function createReadCache(capacity) {
533
538
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
534
539
  const existing = entries.get(key);
535
540
  if (existing !== void 0) drop(key, existing);
536
- entries.set(key, { value, bytes, fileNumber });
541
+ entries.set(key, { value, bytes, fileNumber, referenced: true });
537
542
  held += bytes;
538
543
  perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
539
544
  evictDownTo(capacity - reserved);
@@ -1,6 +1,6 @@
1
- import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-XTSFRWAS.mjs';
1
+ import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-FIXUVFUC.mjs';
2
2
  import { SYNC_CLOSE_GOING_AWAY, encodeJson } from './chunk-FETZYCH7.mjs';
3
- import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-2C3BXZGA.mjs';
3
+ import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-CWXR7OP4.mjs';
4
4
  import { loadedMessagePack, loadMessagePack, extractVectorFieldsFromSchema } from './chunk-X5NHWOXT.mjs';
5
5
  import { answerFromOneState } from './chunk-DG6WIEB3.mjs';
6
6
  import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
@@ -1,8 +1,8 @@
1
- import { encode, Merger } from './chunk-2KWTQ2P7.mjs';
2
- import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-YXZDTKQS.mjs';
1
+ import { encode, Merger } from './chunk-JVR5KMO4.mjs';
2
+ import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-MCAJOLVK.mjs';
3
3
  import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
4
- import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-JCWC75NH.mjs';
5
- import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-2C3BXZGA.mjs';
4
+ import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-C77OBNPP.mjs';
5
+ import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-CWXR7OP4.mjs';
6
6
  import { decode } from './chunk-GXCDWLQI.mjs';
7
7
  import { spawnNodeWorker, resolveWorkerEntry } from './chunk-DG6WIEB3.mjs';
8
8
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
@@ -1,4 +1,4 @@
1
- import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-2C3BXZGA.mjs';
1
+ import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-CWXR7OP4.mjs';
2
2
 
3
3
  // src/partitioning/constants.ts
4
4
  var REBALANCE_CHUNK_SIZE = 1e3;
@@ -1,16 +1,16 @@
1
- import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-YPV2L7M5.mjs';
2
- import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-LOQHA6QK.mjs';
3
- import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-UISV46WX.mjs';
4
- import { openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, requirePagedPayloadChecksum, checkPagedContents, isPagedSegmentFile, bytesOfResidentSections, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, residentSectionsOf, partitionContentsOf, SEGMENT_HEADER_SIZE } from './chunk-F3FDHXCJ.mjs';
5
- import { createReadCache, MERGE_READ_CACHE_BYTES, loadFailed, decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-ZB5X5WAK.mjs';
6
- import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-KHJPMZE2.mjs';
7
- import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-QDZJM7NI.mjs';
1
+ import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-CKS4QQEC.mjs';
2
+ import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-SA7U67XF.mjs';
3
+ import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-NPXORGMK.mjs';
4
+ import { writeCheckpointSegments, commitCheckpointManifest, isPagedSegmentFile, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE } from './chunk-3OEXOSWI.mjs';
5
+ import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-TTTBE4TB.mjs';
6
+ import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-PK3BGGHO.mjs';
7
+ import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
8
8
  import { encode } from './chunk-CV3Q2EKO.mjs';
9
9
  import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
10
- import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-XFOL77WH.mjs';
10
+ import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-YWSIRKKQ.mjs';
11
11
  import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-TNEU32EN.mjs';
12
- import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-JCWC75NH.mjs';
13
- import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, decodeStoredDocument, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-2C3BXZGA.mjs';
12
+ import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-C77OBNPP.mjs';
13
+ import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-CWXR7OP4.mjs';
14
14
  import { decode } from './chunk-GXCDWLQI.mjs';
15
15
  import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
16
16
  import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-X5NHWOXT.mjs';
@@ -1543,61 +1543,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
1543
1543
  }
1544
1544
  }
1545
1545
 
1546
- // src/persistence/paged/segment-scan.ts
1547
- async function openPagedSegmentScan(directory, ref) {
1548
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
1549
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
1550
- const segment = createPagedFrozenSegment(snapshot, cache);
1551
- const { paged } = segment;
1552
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
1553
- const ids = segment.arrays.idTable;
1554
- const tombstones = new Set(snapshot.tombstones);
1555
- return {
1556
- count: snapshot.documentCount,
1557
- schema: snapshot.schema,
1558
- tombstones,
1559
- idAt: (ordinal) => ids.idAt(ordinal),
1560
- holds: (docId) => ids.ordinalOf(docId) >= 0,
1561
- storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
1562
- close: () => {
1563
- paged.close();
1564
- cache.clear();
1565
- }
1566
- };
1567
- }
1568
- async function checkPagedSegmentFully(directory, ref) {
1569
- const listing = await pagedListingOf(directory, ref);
1570
- await requirePagedPayloadChecksum(directory, listing);
1571
- const snapshot = openPagedSnapshot(listing);
1572
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
1573
- const segment = createPagedFrozenSegment(snapshot, cache);
1574
- const { paged } = segment;
1575
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
1576
- try {
1577
- await checkPagedContents(segment, cache);
1578
- return snapshot.schema;
1579
- } finally {
1580
- paged.close();
1581
- cache.clear();
1582
- }
1583
- }
1584
- async function bytesThatSegmentsHoldInMemory(directory, manifest) {
1585
- let bytes = 0;
1586
- for (const partition of manifest.partitions) {
1587
- for (const ref of partition.segments) {
1588
- const paged = await isPagedSegmentFile(directory, ref.key);
1589
- bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
1590
- }
1591
- }
1592
- return bytes;
1593
- }
1594
- function countsAsLatestHolder(scans, at, docId) {
1595
- for (let later = at + 1; later < scans.length; later++) {
1596
- if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
1597
- }
1598
- return true;
1599
- }
1600
-
1601
1546
  // src/persistence/archive/byte-reader.ts
1602
1547
  async function* chunksOf(source) {
1603
1548
  if (source instanceof Uint8Array) {
@@ -1691,7 +1636,7 @@ function vectorFieldsOf(schema, promotion) {
1691
1636
  }
1692
1637
  return Object.keys(fields).length === 0 ? null : fields;
1693
1638
  }
1694
- function metadataFromConfig(indexName, config, partitionCount, documentCount) {
1639
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
1695
1640
  const metadata = {
1696
1641
  indexName,
1697
1642
  schema: config.schema,
@@ -1706,6 +1651,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
1706
1651
  engineVersion: VERSION
1707
1652
  };
1708
1653
  metadata.documentCount = documentCount ?? config.documentCount;
1654
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
1655
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
1709
1656
  const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
1710
1657
  if (vectorFields !== null) {
1711
1658
  metadata.vectorFields = vectorFields;
@@ -1785,6 +1732,7 @@ function metadataConfigOf(entry) {
1785
1732
  ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
1786
1733
  ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
1787
1734
  documentCount: entry.documentCount,
1735
+ savedBytes: entry.savedBytes,
1788
1736
  partitionCount: entry.partitionCount,
1789
1737
  schema: flattenSchema(entry.config.schema),
1790
1738
  language: entry.language.name,
@@ -2773,6 +2721,9 @@ function metadataToWire(meta) {
2773
2721
  if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
2774
2722
  wire.document_count = meta.documentCount;
2775
2723
  }
2724
+ if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
2725
+ wire.saved_bytes = meta.savedBytes;
2726
+ }
2776
2727
  if (meta.embedding) {
2777
2728
  wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
2778
2729
  }
@@ -2888,6 +2839,9 @@ function wireToMetadata(raw) {
2888
2839
  if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
2889
2840
  meta.documentCount = raw.document_count;
2890
2841
  }
2842
+ if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
2843
+ meta.savedBytes = raw.saved_bytes;
2844
+ }
2891
2845
  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
2892
2846
  meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
2893
2847
  }
@@ -3211,6 +3165,20 @@ function requireAgreedSchema(agreed, schema, key) {
3211
3165
  }
3212
3166
  }
3213
3167
  }
3168
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
3169
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
3170
+ throw new NarsilError(
3171
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3172
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
3173
+ {
3174
+ key: ref.key,
3175
+ documentCount,
3176
+ tombstoneCount,
3177
+ listedDocCount: ref.docCount,
3178
+ listedTombstoneCount: ref.tombstoneCount
3179
+ }
3180
+ );
3181
+ }
3214
3182
  function removeWhatTheSegmentReplaces(manager, segment, contents) {
3215
3183
  manager.beginBatchRemove();
3216
3184
  try {
@@ -3245,6 +3213,7 @@ function removeFromPartition(manager, partitionId, docIds) {
3245
3213
  async function attachPagedSegment(directory, ref, partitionId, manager, options) {
3246
3214
  if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
3247
3215
  const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3216
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
3248
3217
  requireAgreedSchema(options.schema, snapshot.schema, ref.key);
3249
3218
  const segment = createPagedFrozenSegment(snapshot);
3250
3219
  removeFromPartition(manager, partitionId, snapshot.tombstones);
@@ -3290,6 +3259,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
3290
3259
  }
3291
3260
  }
3292
3261
  const contents = await readSegmentFile(store, ref);
3262
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
3293
3263
  requireAgreedSchema(options.schema, contents.schema, ref.key);
3294
3264
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
3295
3265
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
@@ -3615,4 +3585,4 @@ function createDetachedIndex(indexName, config, language, synonyms, executor, pa
3615
3585
  return { indexName, manager: entry.manager, entry, attach };
3616
3586
  }
3617
3587
 
3618
- export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, captureAppliedPrefix, changeIndexAnalysis, checkPagedSegmentFully, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, countsAsLatestHolder, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openPagedSegmentScan, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
3588
+ export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
@@ -1,8 +1,8 @@
1
1
  import { P as PartitionConfig, B as BM25Params, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-CzG9OUw5.js';
2
- import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-P-4xhUe4.js';
2
+ import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-De6JzTW8.js';
3
3
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
4
- import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-CXtQ3Zpp.js';
5
- import { B as BackupListing } from './config-CIpLXXxm.js';
4
+ import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-X2IJQc8J.js';
5
+ import { B as BackupListing } from './config-D9WR3MhO.js';
6
6
  import { d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, f as RecordQueryResult, c as QuerySuggestionsPreview, M as ManualQuerySuggestion } from './query-suggestions-CBYkrQ5T.js';
7
7
 
8
8
  /**
package/dist/client.d.ts CHANGED
@@ -1,10 +1,10 @@
1
- export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-CIpLXXxm.js';
2
- export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-P-4xhUe4.js';
3
- export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BQIdsaWy.js';
4
- export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-CXtQ3Zpp.js';
1
+ export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-D9WR3MhO.js';
2
+ export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-De6JzTW8.js';
3
+ export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-DAHXphRn.js';
4
+ export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-X2IJQc8J.js';
5
5
  export { M as ManualQuerySuggestion, Q as QuerySuggestionPreviewEntry, a as QuerySuggestionsBuild, b as QuerySuggestionsBuildTrigger, c as QuerySuggestionsPreview, d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, R as RecordQueryOutcome, f as RecordQueryResult } from './query-suggestions-CBYkrQ5T.js';
6
- export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-CeUxvIvd.js';
7
- import './config-DehU-VGL.js';
6
+ export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-DpqtT2XG.js';
7
+ import './config-C687-reW.js';
8
8
  import './adapters-pljAqmrC.js';
9
9
  import './schema-CzG9OUw5.js';
10
10
  import './synonyms-CpOAHPa-.js';
@@ -1,5 +1,5 @@
1
1
  import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
2
- import { Q as QueryParams, j as QueryResult, k as NarsilEventHandlers } from './engine-P-4xhUe4.js';
2
+ import { Q as QueryParams, j as QueryResult, k as NarsilEventHandlers } from './engine-De6JzTW8.js';
3
3
  import { I as IndexConfig, A as AnyDocument } from './schema-CzG9OUw5.js';
4
4
 
5
5
  /**
@@ -407,6 +407,16 @@ interface DurabilityConfig {
407
407
  * number of segments.
408
408
  */
409
409
  compactionThreshold?: number;
410
+ /**
411
+ * A checkpoint rewrites at most this many bytes of existing segment files to
412
+ * drop the copies of removed and replaced documents, 268,435,456 by default.
413
+ * Beyond that budget, each partition whose removed share exceeds 0.2 still
414
+ * rewrites one segment, so every such partition shrinks at every checkpoint.
415
+ * The value is a whole number of bytes of at least 1. The snapshot tier
416
+ * accepts it and writes each changed partition whole, so the budget applies
417
+ * to the write-ahead log tier.
418
+ */
419
+ rewriteBytesPerCheckpoint?: number;
410
420
  }
411
421
  /**
412
422
  * Which queries the main copy of an index answers once the index holds worker
@@ -1,4 +1,4 @@
1
- import { N as NarsilConfig } from './config-DehU-VGL.js';
1
+ import { N as NarsilConfig } from './config-C687-reW.js';
2
2
  import { P as PersistenceAdapter } from './adapters-pljAqmrC.js';
3
3
  import { A as AllocationConstraints, a as AllocationTable, N as NodeRole, b as NodeRegistration, C as ClusterCoordinator, c as NodeCapacity } from './index-CBUl4z3E.js';
4
4
  import { N as NodeTransport } from './types-DvncmvuQ.js';
@@ -1,4 +1,4 @@
1
- export { createDirectExecutor } from './chunk-2C3BXZGA.mjs';
1
+ export { createDirectExecutor } from './chunk-CWXR7OP4.mjs';
2
2
  import './chunk-KLVMK3R5.mjs';
3
3
  import './chunk-LPRRFCVU.mjs';
4
4
  import './chunk-GXCDWLQI.mjs';
@@ -1,13 +1,13 @@
1
- import { c as CreateIndexOptions, C as ClusterNamespace, d as ClusterNodeConfig } from './config-CIpLXXxm.js';
2
- export { B as BackupListing, e as BackupOperationOptions, f as BackupProgress, a as BackupSummary, g as ClusterBackupConfig, h as ClusterControllerConfig, i as ClusterHealth, j as ClusterHealthStatus, k as ClusterMemoryConfig, l as ClusterMetrics, b as ClusterNodeInfo, m as ClusterQueryConfig, n as CommitPointFigure, o as CoordinatorRequestFigure, p as CreateBackupOptions, D as DamagedBackup, I as IndexHealth, N as NodeReadiness, P as PartitionReplicationFigure, R as ReplicaLagFigure, q as ReplicationConfig, r as RestoreBackupOptions, V as VectorRebuildWaitFigure } from './config-CIpLXXxm.js';
1
+ import { c as CreateIndexOptions, C as ClusterNamespace, d as ClusterNodeConfig } from './config-D9WR3MhO.js';
2
+ export { B as BackupListing, e as BackupOperationOptions, f as BackupProgress, a as BackupSummary, g as ClusterBackupConfig, h as ClusterControllerConfig, i as ClusterHealth, j as ClusterHealthStatus, k as ClusterMemoryConfig, l as ClusterMetrics, b as ClusterNodeInfo, m as ClusterQueryConfig, n as CommitPointFigure, o as CoordinatorRequestFigure, p as CreateBackupOptions, D as DamagedBackup, I as IndexHealth, N as NodeReadiness, P as PartitionReplicationFigure, R as ReplicaLagFigure, q as ReplicationConfig, r as RestoreBackupOptions, V as VectorRebuildWaitFigure } from './config-D9WR3MhO.js';
3
3
  import { R as RerankAdapter } from './adapters-pljAqmrC.js';
4
- import { f as SyncSetting, g as SyncStatus, B as BatchResult, L as ListParams, l as ListResult, m as SuggestParams, n as SuggestResult, Q as QueryParams, o as PreflightResult, I as IndexInfo, A as AnalysisChange, p as IndexStats, P as PartitionStatsResult, M as MemoryStats, q as NarsilEventMap, j as QueryResult, h as Narsil } from './engine-P-4xhUe4.js';
4
+ import { f as SyncSetting, g as SyncStatus, B as BatchResult, L as ListParams, l as ListResult, m as SuggestParams, n as SuggestResult, Q as QueryParams, o as PreflightResult, I as IndexInfo, A as AnalysisChange, p as IndexStats, P as PartitionStatsResult, M as MemoryStats, q as NarsilEventMap, j as QueryResult, h as Narsil } from './engine-De6JzTW8.js';
5
5
  import { B as BM25Params, I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-CzG9OUw5.js';
6
6
  import { c as NodeCapacity, N as NodeRole } from './index-CBUl4z3E.js';
7
7
  export { A as AllocationConstraints, e as AllocationEvent, a as AllocationTable, C as ClusterCoordinator, I as IndexMetadataEvent, f as IndexMetadataWatch, g as IndexPartitionBytes, h as NodeEvent, d as NodeMemoryReport, b as NodeRegistration, P as PartitionAssignment, i as PartitionState, R as RegistrationLease, S as SchemaEvent, U as UnassignedReason } from './index-CBUl4z3E.js';
8
8
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
9
9
  export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-DvncmvuQ.js';
10
- import './config-DehU-VGL.js';
10
+ import './config-C687-reW.js';
11
11
  import './query-suggestions-CBYkrQ5T.js';
12
12
 
13
13
  /**