@delali/narsil 0.3.1-next.180 → 0.3.1-next.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
- package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
- package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
- package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
- package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
- package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
- package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
- package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
- package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
- package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
- package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
- package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
- package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
- package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +391 -138
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +1 -1
- package/dist/index.react-native.mjs +1 -1
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
- package/dist/query-suggestions/merge-worker.mjs +4 -4
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +2 -2
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
- package/dist/react-native/partial-words.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +1 -1
- package/dist/react-native/sync.mjs +1 -1
- package/dist/react-native/synonyms.mjs +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +16 -16
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +1 -1
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +1 -1
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/openapi.json +32 -0
- package/package.json +7 -7
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-
|
|
2
|
-
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-
|
|
1
|
+
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-C77OBNPP.mjs';
|
|
2
|
+
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-CWXR7OP4.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/engine-hooks.ts
|
|
@@ -14,10 +14,12 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
|
14
14
|
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
15
15
|
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
16
16
|
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
17
|
+
var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
|
|
18
|
+
var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
|
|
17
19
|
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
18
20
|
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
19
21
|
var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
|
|
20
22
|
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
21
23
|
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
22
24
|
|
|
23
|
-
export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, SEGMENT_FILE_FRAMING_SLACK_BYTES, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES };
|
|
25
|
+
export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, SEGMENT_FILE_FRAMING_SLACK_BYTES, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-
|
|
1
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-PK3BGGHO.mjs';
|
|
2
2
|
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-TNEU32EN.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-C77OBNPP.mjs';
|
|
4
4
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
5
5
|
import { requireValidFieldTypes } from './chunk-X5NHWOXT.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-
|
|
1
|
+
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-X5NHWOXT.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/pattern-snapshot.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { isValidInteger, isRecord } from './chunk-
|
|
1
|
+
import { isValidInteger, isRecord } from './chunk-F2UOTRY7.mjs';
|
|
2
2
|
import { MAX_REPLICATION_FACTOR, MAX_PARTITION_COUNT } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-C77OBNPP.mjs';
|
|
4
4
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
5
5
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
6
6
|
import { joinChunks } from './chunk-IE3Q62QD.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
2
2
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
|
|
4
|
+
import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-CWXR7OP4.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { boundQuerySuggestionsOf } from './chunk-S2POFTLN.mjs';
|
|
2
|
-
import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-
|
|
2
|
+
import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-CWXR7OP4.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/index.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-
|
|
1
|
+
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-PK3BGGHO.mjs';
|
|
2
2
|
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
3
3
|
import { createFixedBuffer, createVectorFileReader } from './chunk-TNEU32EN.mjs';
|
|
4
|
-
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
4
|
+
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-C77OBNPP.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-D6FYUTFN.mjs';
|
|
7
7
|
import { ErrorCodes, NarsilError } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -470,6 +470,12 @@ function createReadCache(capacity) {
|
|
|
470
470
|
function evictDownTo(limit) {
|
|
471
471
|
for (const [key, entry] of entries) {
|
|
472
472
|
if (held <= limit) return;
|
|
473
|
+
if (entry.referenced) {
|
|
474
|
+
entry.referenced = false;
|
|
475
|
+
entries.delete(key);
|
|
476
|
+
entries.set(key, entry);
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
473
479
|
drop(key, entry);
|
|
474
480
|
}
|
|
475
481
|
}
|
|
@@ -524,8 +530,7 @@ function createReadCache(capacity) {
|
|
|
524
530
|
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
525
531
|
const found = entries.get(key);
|
|
526
532
|
if (found === void 0) return void 0;
|
|
527
|
-
|
|
528
|
-
entries.set(key, found);
|
|
533
|
+
found.referenced = true;
|
|
529
534
|
return found.value;
|
|
530
535
|
},
|
|
531
536
|
put(fileNumber, kind, entry, value, bytes) {
|
|
@@ -533,7 +538,7 @@ function createReadCache(capacity) {
|
|
|
533
538
|
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
534
539
|
const existing = entries.get(key);
|
|
535
540
|
if (existing !== void 0) drop(key, existing);
|
|
536
|
-
entries.set(key, { value, bytes, fileNumber });
|
|
541
|
+
entries.set(key, { value, bytes, fileNumber, referenced: true });
|
|
537
542
|
held += bytes;
|
|
538
543
|
perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
|
|
539
544
|
evictDownTo(capacity - reserved);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-
|
|
1
|
+
import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-FIXUVFUC.mjs';
|
|
2
2
|
import { SYNC_CLOSE_GOING_AWAY, encodeJson } from './chunk-FETZYCH7.mjs';
|
|
3
|
-
import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-
|
|
3
|
+
import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-CWXR7OP4.mjs';
|
|
4
4
|
import { loadedMessagePack, loadMessagePack, extractVectorFieldsFromSchema } from './chunk-X5NHWOXT.mjs';
|
|
5
5
|
import { answerFromOneState } from './chunk-DG6WIEB3.mjs';
|
|
6
6
|
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { encode, Merger } from './chunk-
|
|
2
|
-
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-
|
|
1
|
+
import { encode, Merger } from './chunk-JVR5KMO4.mjs';
|
|
2
|
+
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-MCAJOLVK.mjs';
|
|
3
3
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
4
|
-
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-
|
|
5
|
-
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-
|
|
4
|
+
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-C77OBNPP.mjs';
|
|
5
|
+
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-CWXR7OP4.mjs';
|
|
6
6
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
7
7
|
import { spawnNodeWorker, resolveWorkerEntry } from './chunk-DG6WIEB3.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-
|
|
1
|
+
import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-CWXR7OP4.mjs';
|
|
2
2
|
|
|
3
3
|
// src/partitioning/constants.ts
|
|
4
4
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-
|
|
3
|
-
import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-
|
|
7
|
-
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-CKS4QQEC.mjs';
|
|
2
|
+
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-SA7U67XF.mjs';
|
|
3
|
+
import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-NPXORGMK.mjs';
|
|
4
|
+
import { writeCheckpointSegments, commitCheckpointManifest, isPagedSegmentFile, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE } from './chunk-3OEXOSWI.mjs';
|
|
5
|
+
import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-TTTBE4TB.mjs';
|
|
6
|
+
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-PK3BGGHO.mjs';
|
|
7
|
+
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
|
|
8
8
|
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
9
9
|
import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
|
|
10
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-
|
|
10
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-YWSIRKKQ.mjs';
|
|
11
11
|
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-TNEU32EN.mjs';
|
|
12
|
-
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-
|
|
13
|
-
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor,
|
|
12
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-C77OBNPP.mjs';
|
|
13
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-CWXR7OP4.mjs';
|
|
14
14
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
15
15
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
16
16
|
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-X5NHWOXT.mjs';
|
|
@@ -1543,61 +1543,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
|
|
|
1543
1543
|
}
|
|
1544
1544
|
}
|
|
1545
1545
|
|
|
1546
|
-
// src/persistence/paged/segment-scan.ts
|
|
1547
|
-
async function openPagedSegmentScan(directory, ref) {
|
|
1548
|
-
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1549
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1550
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1551
|
-
const { paged } = segment;
|
|
1552
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1553
|
-
const ids = segment.arrays.idTable;
|
|
1554
|
-
const tombstones = new Set(snapshot.tombstones);
|
|
1555
|
-
return {
|
|
1556
|
-
count: snapshot.documentCount,
|
|
1557
|
-
schema: snapshot.schema,
|
|
1558
|
-
tombstones,
|
|
1559
|
-
idAt: (ordinal) => ids.idAt(ordinal),
|
|
1560
|
-
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
1561
|
-
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
1562
|
-
close: () => {
|
|
1563
|
-
paged.close();
|
|
1564
|
-
cache.clear();
|
|
1565
|
-
}
|
|
1566
|
-
};
|
|
1567
|
-
}
|
|
1568
|
-
async function checkPagedSegmentFully(directory, ref) {
|
|
1569
|
-
const listing = await pagedListingOf(directory, ref);
|
|
1570
|
-
await requirePagedPayloadChecksum(directory, listing);
|
|
1571
|
-
const snapshot = openPagedSnapshot(listing);
|
|
1572
|
-
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1573
|
-
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1574
|
-
const { paged } = segment;
|
|
1575
|
-
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1576
|
-
try {
|
|
1577
|
-
await checkPagedContents(segment, cache);
|
|
1578
|
-
return snapshot.schema;
|
|
1579
|
-
} finally {
|
|
1580
|
-
paged.close();
|
|
1581
|
-
cache.clear();
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
1585
|
-
let bytes = 0;
|
|
1586
|
-
for (const partition of manifest.partitions) {
|
|
1587
|
-
for (const ref of partition.segments) {
|
|
1588
|
-
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
1589
|
-
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
1590
|
-
}
|
|
1591
|
-
}
|
|
1592
|
-
return bytes;
|
|
1593
|
-
}
|
|
1594
|
-
function countsAsLatestHolder(scans, at, docId) {
|
|
1595
|
-
for (let later = at + 1; later < scans.length; later++) {
|
|
1596
|
-
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
1597
|
-
}
|
|
1598
|
-
return true;
|
|
1599
|
-
}
|
|
1600
|
-
|
|
1601
1546
|
// src/persistence/archive/byte-reader.ts
|
|
1602
1547
|
async function* chunksOf(source) {
|
|
1603
1548
|
if (source instanceof Uint8Array) {
|
|
@@ -1691,7 +1636,7 @@ function vectorFieldsOf(schema, promotion) {
|
|
|
1691
1636
|
}
|
|
1692
1637
|
return Object.keys(fields).length === 0 ? null : fields;
|
|
1693
1638
|
}
|
|
1694
|
-
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
1639
|
+
function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
|
|
1695
1640
|
const metadata = {
|
|
1696
1641
|
indexName,
|
|
1697
1642
|
schema: config.schema,
|
|
@@ -1706,6 +1651,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
1706
1651
|
engineVersion: VERSION
|
|
1707
1652
|
};
|
|
1708
1653
|
metadata.documentCount = documentCount ?? config.documentCount;
|
|
1654
|
+
const recordedSavedBytes = savedBytes ?? config.savedBytes;
|
|
1655
|
+
if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
|
|
1709
1656
|
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
1710
1657
|
if (vectorFields !== null) {
|
|
1711
1658
|
metadata.vectorFields = vectorFields;
|
|
@@ -1785,6 +1732,7 @@ function metadataConfigOf(entry) {
|
|
|
1785
1732
|
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
1786
1733
|
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
1787
1734
|
documentCount: entry.documentCount,
|
|
1735
|
+
savedBytes: entry.savedBytes,
|
|
1788
1736
|
partitionCount: entry.partitionCount,
|
|
1789
1737
|
schema: flattenSchema(entry.config.schema),
|
|
1790
1738
|
language: entry.language.name,
|
|
@@ -2773,6 +2721,9 @@ function metadataToWire(meta) {
|
|
|
2773
2721
|
if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
|
|
2774
2722
|
wire.document_count = meta.documentCount;
|
|
2775
2723
|
}
|
|
2724
|
+
if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
|
|
2725
|
+
wire.saved_bytes = meta.savedBytes;
|
|
2726
|
+
}
|
|
2776
2727
|
if (meta.embedding) {
|
|
2777
2728
|
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
2778
2729
|
}
|
|
@@ -2888,6 +2839,9 @@ function wireToMetadata(raw) {
|
|
|
2888
2839
|
if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
|
|
2889
2840
|
meta.documentCount = raw.document_count;
|
|
2890
2841
|
}
|
|
2842
|
+
if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
|
|
2843
|
+
meta.savedBytes = raw.saved_bytes;
|
|
2844
|
+
}
|
|
2891
2845
|
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
2892
2846
|
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
2893
2847
|
}
|
|
@@ -3211,6 +3165,20 @@ function requireAgreedSchema(agreed, schema, key) {
|
|
|
3211
3165
|
}
|
|
3212
3166
|
}
|
|
3213
3167
|
}
|
|
3168
|
+
function requireListedCounts(ref, documentCount, tombstoneCount) {
|
|
3169
|
+
if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
|
|
3170
|
+
throw new NarsilError(
|
|
3171
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3172
|
+
`The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
|
|
3173
|
+
{
|
|
3174
|
+
key: ref.key,
|
|
3175
|
+
documentCount,
|
|
3176
|
+
tombstoneCount,
|
|
3177
|
+
listedDocCount: ref.docCount,
|
|
3178
|
+
listedTombstoneCount: ref.tombstoneCount
|
|
3179
|
+
}
|
|
3180
|
+
);
|
|
3181
|
+
}
|
|
3214
3182
|
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
3215
3183
|
manager.beginBatchRemove();
|
|
3216
3184
|
try {
|
|
@@ -3245,6 +3213,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
3245
3213
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
3246
3214
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
3247
3215
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3216
|
+
requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
|
|
3248
3217
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
3249
3218
|
const segment = createPagedFrozenSegment(snapshot);
|
|
3250
3219
|
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
@@ -3290,6 +3259,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
|
|
|
3290
3259
|
}
|
|
3291
3260
|
}
|
|
3292
3261
|
const contents = await readSegmentFile(store, ref);
|
|
3262
|
+
requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
|
|
3293
3263
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
3294
3264
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
3295
3265
|
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
@@ -3615,4 +3585,4 @@ function createDetachedIndex(indexName, config, language, synonyms, executor, pa
|
|
|
3615
3585
|
return { indexName, manager: entry.manager, entry, attach };
|
|
3616
3586
|
}
|
|
3617
3587
|
|
|
3618
|
-
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange,
|
|
3588
|
+
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { P as PartitionConfig, B as BM25Params, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-CzG9OUw5.js';
|
|
2
|
-
import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-
|
|
2
|
+
import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-De6JzTW8.js';
|
|
3
3
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
|
-
import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-
|
|
5
|
-
import { B as BackupListing } from './config-
|
|
4
|
+
import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-X2IJQc8J.js';
|
|
5
|
+
import { B as BackupListing } from './config-D9WR3MhO.js';
|
|
6
6
|
import { d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, f as RecordQueryResult, c as QuerySuggestionsPreview, M as ManualQuerySuggestion } from './query-suggestions-CBYkrQ5T.js';
|
|
7
7
|
|
|
8
8
|
/**
|
package/dist/client.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
4
|
-
export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-
|
|
1
|
+
export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-D9WR3MhO.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-De6JzTW8.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-DAHXphRn.js';
|
|
4
|
+
export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-X2IJQc8J.js';
|
|
5
5
|
export { M as ManualQuerySuggestion, Q as QuerySuggestionPreviewEntry, a as QuerySuggestionsBuild, b as QuerySuggestionsBuildTrigger, c as QuerySuggestionsPreview, d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, R as RecordQueryOutcome, f as RecordQueryResult } from './query-suggestions-CBYkrQ5T.js';
|
|
6
|
-
export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-
|
|
7
|
-
import './config-
|
|
6
|
+
export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-DpqtT2XG.js';
|
|
7
|
+
import './config-C687-reW.js';
|
|
8
8
|
import './adapters-pljAqmrC.js';
|
|
9
9
|
import './schema-CzG9OUw5.js';
|
|
10
10
|
import './synonyms-CpOAHPa-.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
2
|
-
import { Q as QueryParams, j as QueryResult, k as NarsilEventHandlers } from './engine-
|
|
2
|
+
import { Q as QueryParams, j as QueryResult, k as NarsilEventHandlers } from './engine-De6JzTW8.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument } from './schema-CzG9OUw5.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -407,6 +407,16 @@ interface DurabilityConfig {
|
|
|
407
407
|
* number of segments.
|
|
408
408
|
*/
|
|
409
409
|
compactionThreshold?: number;
|
|
410
|
+
/**
|
|
411
|
+
* A checkpoint rewrites at most this many bytes of existing segment files to
|
|
412
|
+
* drop the copies of removed and replaced documents, 268,435,456 by default.
|
|
413
|
+
* Beyond that budget, each partition whose removed share exceeds 0.2 still
|
|
414
|
+
* rewrites one segment, so every such partition shrinks at every checkpoint.
|
|
415
|
+
* The value is a whole number of bytes of at least 1. The snapshot tier
|
|
416
|
+
* accepts it and writes each changed partition whole, so the budget applies
|
|
417
|
+
* to the write-ahead log tier.
|
|
418
|
+
*/
|
|
419
|
+
rewriteBytesPerCheckpoint?: number;
|
|
410
420
|
}
|
|
411
421
|
/**
|
|
412
422
|
* Which queries the main copy of an index answers once the index holds worker
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
1
|
+
import { N as NarsilConfig } from './config-C687-reW.js';
|
|
2
2
|
import { P as PersistenceAdapter } from './adapters-pljAqmrC.js';
|
|
3
3
|
import { A as AllocationConstraints, a as AllocationTable, N as NodeRole, b as NodeRegistration, C as ClusterCoordinator, c as NodeCapacity } from './index-CBUl4z3E.js';
|
|
4
4
|
import { N as NodeTransport } from './types-DvncmvuQ.js';
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { c as CreateIndexOptions, C as ClusterNamespace, d as ClusterNodeConfig } from './config-
|
|
2
|
-
export { B as BackupListing, e as BackupOperationOptions, f as BackupProgress, a as BackupSummary, g as ClusterBackupConfig, h as ClusterControllerConfig, i as ClusterHealth, j as ClusterHealthStatus, k as ClusterMemoryConfig, l as ClusterMetrics, b as ClusterNodeInfo, m as ClusterQueryConfig, n as CommitPointFigure, o as CoordinatorRequestFigure, p as CreateBackupOptions, D as DamagedBackup, I as IndexHealth, N as NodeReadiness, P as PartitionReplicationFigure, R as ReplicaLagFigure, q as ReplicationConfig, r as RestoreBackupOptions, V as VectorRebuildWaitFigure } from './config-
|
|
1
|
+
import { c as CreateIndexOptions, C as ClusterNamespace, d as ClusterNodeConfig } from './config-D9WR3MhO.js';
|
|
2
|
+
export { B as BackupListing, e as BackupOperationOptions, f as BackupProgress, a as BackupSummary, g as ClusterBackupConfig, h as ClusterControllerConfig, i as ClusterHealth, j as ClusterHealthStatus, k as ClusterMemoryConfig, l as ClusterMetrics, b as ClusterNodeInfo, m as ClusterQueryConfig, n as CommitPointFigure, o as CoordinatorRequestFigure, p as CreateBackupOptions, D as DamagedBackup, I as IndexHealth, N as NodeReadiness, P as PartitionReplicationFigure, R as ReplicaLagFigure, q as ReplicationConfig, r as RestoreBackupOptions, V as VectorRebuildWaitFigure } from './config-D9WR3MhO.js';
|
|
3
3
|
import { R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
4
|
-
import { f as SyncSetting, g as SyncStatus, B as BatchResult, L as ListParams, l as ListResult, m as SuggestParams, n as SuggestResult, Q as QueryParams, o as PreflightResult, I as IndexInfo, A as AnalysisChange, p as IndexStats, P as PartitionStatsResult, M as MemoryStats, q as NarsilEventMap, j as QueryResult, h as Narsil } from './engine-
|
|
4
|
+
import { f as SyncSetting, g as SyncStatus, B as BatchResult, L as ListParams, l as ListResult, m as SuggestParams, n as SuggestResult, Q as QueryParams, o as PreflightResult, I as IndexInfo, A as AnalysisChange, p as IndexStats, P as PartitionStatsResult, M as MemoryStats, q as NarsilEventMap, j as QueryResult, h as Narsil } from './engine-De6JzTW8.js';
|
|
5
5
|
import { B as BM25Params, I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-CzG9OUw5.js';
|
|
6
6
|
import { c as NodeCapacity, N as NodeRole } from './index-CBUl4z3E.js';
|
|
7
7
|
export { A as AllocationConstraints, e as AllocationEvent, a as AllocationTable, C as ClusterCoordinator, I as IndexMetadataEvent, f as IndexMetadataWatch, g as IndexPartitionBytes, h as NodeEvent, d as NodeMemoryReport, b as NodeRegistration, P as PartitionAssignment, i as PartitionState, R as RegistrationLease, S as SchemaEvent, U as UnassignedReason } from './index-CBUl4z3E.js';
|
|
8
8
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
9
9
|
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-DvncmvuQ.js';
|
|
10
|
-
import './config-
|
|
10
|
+
import './config-C687-reW.js';
|
|
11
11
|
import './query-suggestions-CBYkrQ5T.js';
|
|
12
12
|
|
|
13
13
|
/**
|