@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,55 +1,59 @@
|
|
|
1
|
-
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, loadMetadata, snapshotCheckpointFor, replayWal } from './chunk-
|
|
3
|
-
import { decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope,
|
|
4
|
-
import {
|
|
1
|
+
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-NJJCPFXN.mjs';
|
|
2
|
+
import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, removeLeftoverRewriteFiles, loadMetadata, snapshotCheckpointFor, replayWal } from './chunk-RHKRTYGO.mjs';
|
|
3
|
+
import { decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, writeMetadataEnvelope, reclaimUnlistedKeys, loadSegmentedCheckpoint, isRecord, invalidManifest, unpackEnvelopeBytes, isId, isUint32, isSequenceNumber, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf, readCurrentManifest } from './chunk-YHUTF6EA.mjs';
|
|
4
|
+
import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
|
|
5
|
+
import './chunk-W2COJ5AH.mjs';
|
|
6
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-ICW45TTU.mjs';
|
|
5
7
|
import './chunk-37CD3OKS.mjs';
|
|
6
|
-
import { createDurableDirectory } from './chunk-
|
|
8
|
+
import { createDurableDirectory } from './chunk-AST3SHFA.mjs';
|
|
7
9
|
import { spawnNodeWorker } from './chunk-W55X2BYN.mjs';
|
|
8
10
|
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
9
|
-
import {
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
11
|
+
import { nextSegmentKey, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo } from './chunk-LA4WNDHL.mjs';
|
|
12
|
+
import './chunk-JDDXV6UF.mjs';
|
|
13
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, readSegmentFile, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-7AUASRE7.mjs';
|
|
14
|
+
import { directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-UUZCQQG2.mjs';
|
|
15
|
+
import './chunk-KEIYVZUX.mjs';
|
|
12
16
|
import './chunk-YH5OIL7B.mjs';
|
|
13
|
-
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
-
import
|
|
17
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-IO5F7ZBE.mjs';
|
|
18
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
19
|
+
import './chunk-5WYVUPGD.mjs';
|
|
20
|
+
import './chunk-HJSN7UYG.mjs';
|
|
21
|
+
import { resolveWorkerEntry } from './chunk-UIKN2OTQ.mjs';
|
|
17
22
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import './chunk-
|
|
23
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
|
|
24
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-WJGO5PGV.mjs';
|
|
25
|
+
import { fnv1a } from './chunk-ZPWS2SKJ.mjs';
|
|
26
|
+
import './chunk-N37VMC62.mjs';
|
|
21
27
|
import './chunk-ZATRDAYU.mjs';
|
|
22
28
|
import './chunk-EBIHRL7B.mjs';
|
|
23
|
-
import './chunk-HJSN7UYG.mjs';
|
|
24
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-RNSC4NFI.mjs';
|
|
25
|
-
import './chunk-TGP5M5R7.mjs';
|
|
26
|
-
import './chunk-U2BKYEC7.mjs';
|
|
27
29
|
import './chunk-KOKM4PQN.mjs';
|
|
28
30
|
import './chunk-QSK35UC2.mjs';
|
|
29
|
-
import './chunk-
|
|
30
|
-
import './chunk-
|
|
31
|
+
import './chunk-74XB2SXI.mjs';
|
|
32
|
+
import './chunk-RKAVHNRW.mjs';
|
|
31
33
|
import './chunk-WQBCBKY3.mjs';
|
|
32
34
|
import './chunk-JUKYYXSP.mjs';
|
|
33
|
-
import './chunk-
|
|
34
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-
|
|
35
|
+
import './chunk-DQNKVOJ2.mjs';
|
|
36
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-KGOXHLLZ.mjs';
|
|
35
37
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
36
38
|
import './chunk-FQBFZAWG.mjs';
|
|
37
39
|
import { MIN_M } from './chunk-VJS4JFBQ.mjs';
|
|
38
40
|
import './chunk-GES7U2PM.mjs';
|
|
39
|
-
import './chunk-
|
|
40
|
-
import './chunk-
|
|
41
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
42
|
+
import './chunk-O4LJQI4G.mjs';
|
|
41
43
|
import './chunk-X5CV2E2B.mjs';
|
|
42
|
-
import './chunk-
|
|
43
|
-
import './chunk-
|
|
44
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
45
|
+
import './chunk-TGP5M5R7.mjs';
|
|
46
|
+
import './chunk-U2BKYEC7.mjs';
|
|
47
|
+
import './chunk-NPNK57CA.mjs';
|
|
44
48
|
import './chunk-4UMZ7KQL.mjs';
|
|
45
49
|
import './chunk-R6A773WK.mjs';
|
|
46
50
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
47
|
-
import './chunk-
|
|
48
|
-
import './chunk-
|
|
51
|
+
import { createRequestId } from './chunk-QHPYS427.mjs';
|
|
52
|
+
import './chunk-GLZHNSGB.mjs';
|
|
49
53
|
import './chunk-22YPZVMG.mjs';
|
|
50
54
|
import './chunk-RFBLQLLZ.mjs';
|
|
51
55
|
import './chunk-L5ORED7V.mjs';
|
|
52
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
56
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
53
57
|
import { decode } from '@msgpack/msgpack';
|
|
54
58
|
|
|
55
59
|
// src/persistence/durability/sha256.ts
|
|
@@ -319,6 +323,7 @@ function storageFailed(code, message, key, error) {
|
|
|
319
323
|
}
|
|
320
324
|
function directoryRestoreStorage(directory) {
|
|
321
325
|
return {
|
|
326
|
+
directory,
|
|
322
327
|
read: (key) => directory.read(key),
|
|
323
328
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
324
329
|
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
@@ -363,6 +368,25 @@ function pathUnder(path, root) {
|
|
|
363
368
|
const next = path.charAt(root.length);
|
|
364
369
|
return next === "" || next === "/" || next === "\\";
|
|
365
370
|
}
|
|
371
|
+
function stagedDirectory(directory, staged, unstaged) {
|
|
372
|
+
const { readRange, byteLength } = directory;
|
|
373
|
+
return {
|
|
374
|
+
root: directory.root,
|
|
375
|
+
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
376
|
+
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
377
|
+
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
378
|
+
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
379
|
+
read: (key) => directory.read(staged(key)),
|
|
380
|
+
remove: (key) => directory.remove(staged(key)),
|
|
381
|
+
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
382
|
+
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
383
|
+
pathOf: (key) => directory.pathOf(staged(key)),
|
|
384
|
+
...readRange === void 0 ? {} : {
|
|
385
|
+
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
386
|
+
},
|
|
387
|
+
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
388
|
+
};
|
|
389
|
+
}
|
|
366
390
|
function createRestoreStaging(storage, indexName, options = {}) {
|
|
367
391
|
const stagingPrefix = newStagingPrefix();
|
|
368
392
|
const indexRoot = `${indexName}/`;
|
|
@@ -384,6 +408,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
384
408
|
};
|
|
385
409
|
const pathOf = storage.pathOf;
|
|
386
410
|
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
411
|
+
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
387
412
|
let retiredPrefix = null;
|
|
388
413
|
async function finishOnStorage() {
|
|
389
414
|
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
@@ -503,7 +528,8 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
503
528
|
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
504
529
|
return whole.slice(offset, offset + length);
|
|
505
530
|
},
|
|
506
|
-
release
|
|
531
|
+
release,
|
|
532
|
+
directory: deps.directory
|
|
507
533
|
};
|
|
508
534
|
} catch (error) {
|
|
509
535
|
release();
|
|
@@ -636,6 +662,22 @@ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
|
|
|
636
662
|
}
|
|
637
663
|
}
|
|
638
664
|
}
|
|
665
|
+
async function openNewLogSegments(indexState, failPartition2) {
|
|
666
|
+
const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
|
|
667
|
+
try {
|
|
668
|
+
if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
|
|
669
|
+
} catch (error) {
|
|
670
|
+
failPartition2(partitionId, partition, error);
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
|
|
674
|
+
failPartition2(partitionId, partition, error);
|
|
675
|
+
});
|
|
676
|
+
partition.appendChain = rolled;
|
|
677
|
+
await rolled;
|
|
678
|
+
});
|
|
679
|
+
await Promise.all(rolls);
|
|
680
|
+
}
|
|
639
681
|
function aTargetFailed(indexState, targets) {
|
|
640
682
|
return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
|
|
641
683
|
}
|
|
@@ -668,9 +710,10 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
668
710
|
return whole;
|
|
669
711
|
};
|
|
670
712
|
}
|
|
671
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf) {
|
|
713
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
672
714
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
673
715
|
return whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
716
|
+
onCaptured?.();
|
|
674
717
|
const documentCount = manager.countDocuments();
|
|
675
718
|
const targets = [];
|
|
676
719
|
let recordsLeftInLog = 0;
|
|
@@ -1016,37 +1059,83 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1016
1059
|
if (manager === void 0) {
|
|
1017
1060
|
return 0;
|
|
1018
1061
|
}
|
|
1019
|
-
const metadata = input.hooks.
|
|
1062
|
+
const metadata = input.hooks.buildTermsMetadata(input.indexName);
|
|
1020
1063
|
if (metadata === void 0) {
|
|
1021
1064
|
return 0;
|
|
1022
1065
|
}
|
|
1066
|
+
const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
|
|
1023
1067
|
const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
|
|
1024
1068
|
const store = directoryCheckpointStore(input.directory);
|
|
1025
1069
|
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1026
1070
|
const { indexName } = input;
|
|
1027
1071
|
const priorManifest = await readCurrentManifest(store, input.indexName);
|
|
1072
|
+
const textOnDisk = metadata.textStorage === "disk";
|
|
1073
|
+
const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
|
|
1074
|
+
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1075
|
+
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1076
|
+
everyPartition: fromMemory,
|
|
1077
|
+
aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
|
|
1078
|
+
...writeRecords === null ? {} : {
|
|
1079
|
+
captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
1080
|
+
}
|
|
1081
|
+
});
|
|
1082
|
+
let changesSinceCapture = null;
|
|
1028
1083
|
const capture2 = await captureCheckpoint(
|
|
1029
1084
|
input.indexState,
|
|
1030
1085
|
manager,
|
|
1031
1086
|
vectorIndexes,
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1035
|
-
everyPartition: input.fromMemory,
|
|
1036
|
-
aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable(),
|
|
1037
|
-
...writeRecords === null ? {} : {
|
|
1038
|
-
captureRecords: (partitionId) => (segments2) => writeRecords.captureSegments(indexName, partitionId, segments2)
|
|
1039
|
-
}
|
|
1040
|
-
}),
|
|
1041
|
-
input.fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1087
|
+
textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
|
|
1088
|
+
fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1042
1089
|
priorManifest?.vectors ?? [],
|
|
1043
|
-
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo)
|
|
1090
|
+
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
|
|
1091
|
+
textOnDisk ? () => {
|
|
1092
|
+
changesSinceCapture = manager.openCheckpointChangeLog();
|
|
1093
|
+
} : void 0
|
|
1044
1094
|
);
|
|
1095
|
+
try {
|
|
1096
|
+
await openNewLogSegments(input.indexState, input.failPartition);
|
|
1097
|
+
return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
|
|
1098
|
+
return changesSinceCapture;
|
|
1099
|
+
});
|
|
1100
|
+
} finally {
|
|
1101
|
+
changesSinceCapture?.close();
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1105
|
+
const { indexName, indexState, hooks } = input;
|
|
1106
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-IYPLWHGC.mjs');
|
|
1107
|
+
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1108
|
+
if (adoption === null) return [];
|
|
1109
|
+
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
1110
|
+
if (changes.partitionsReplaced) return false;
|
|
1111
|
+
const changed = changes.takeWrittenDocIds();
|
|
1112
|
+
swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
|
|
1113
|
+
hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
|
|
1114
|
+
});
|
|
1115
|
+
hooks.queueDiskAdoption?.({
|
|
1116
|
+
type: "adoptDiskSegments",
|
|
1117
|
+
indexName,
|
|
1118
|
+
partitions: actionPartitionsOf(adoption),
|
|
1119
|
+
changed: [...changed],
|
|
1120
|
+
requestId: createRequestId()
|
|
1121
|
+
});
|
|
1122
|
+
return true;
|
|
1123
|
+
});
|
|
1124
|
+
if (!adopted) {
|
|
1125
|
+
closeUnusedSegments(adoption);
|
|
1126
|
+
return [];
|
|
1127
|
+
}
|
|
1128
|
+
await hooks.awaitCopiesOfIndex?.(indexName);
|
|
1129
|
+
return adoption.droppedKeys;
|
|
1130
|
+
}
|
|
1131
|
+
async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
|
|
1132
|
+
const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
|
|
1133
|
+
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1045
1134
|
const { targets, documentCount } = capture2;
|
|
1046
1135
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
1047
1136
|
if (aTargetFailed(input.indexState, targets)) return 0;
|
|
1048
1137
|
const [liveVectors, segments] = await bothOrTheFirstFailure(
|
|
1049
|
-
writeCapturedVectors(store, input.indexName, capture2, priorManifest,
|
|
1138
|
+
writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
|
|
1050
1139
|
writeIndexCheckpointSegments({
|
|
1051
1140
|
directory: input.directory,
|
|
1052
1141
|
metadata,
|
|
@@ -1062,10 +1151,18 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1062
1151
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
1063
1152
|
if (aTargetFailed(input.indexState, targets)) return 0;
|
|
1064
1153
|
const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
|
|
1154
|
+
input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
|
|
1065
1155
|
recordWrittenVectors(vectorIndexes, liveVectors.written);
|
|
1066
1156
|
await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
|
|
1157
|
+
const changes = changesSinceCapture();
|
|
1158
|
+
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1067
1159
|
if (!input.holdsCheckpointFiles()) {
|
|
1068
|
-
|
|
1160
|
+
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1161
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-IYPLWHGC.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1162
|
+
await removeCheckpointGarbage(
|
|
1163
|
+
store,
|
|
1164
|
+
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
1165
|
+
);
|
|
1069
1166
|
}
|
|
1070
1167
|
const checkpointDocumentCount = documentCount;
|
|
1071
1168
|
await input.queueMetadataWrite(input.indexName, async () => {
|
|
@@ -1118,6 +1215,19 @@ function createIndexLoads(vectorIndexesOf2) {
|
|
|
1118
1215
|
startGraphRepairs
|
|
1119
1216
|
};
|
|
1120
1217
|
}
|
|
1218
|
+
async function switchInTurn(turns, turn) {
|
|
1219
|
+
let swapped = false;
|
|
1220
|
+
await checkpointInTurn(turns, {
|
|
1221
|
+
joinsInFlight: false,
|
|
1222
|
+
mayStart: turn.mayStart,
|
|
1223
|
+
perform: async () => {
|
|
1224
|
+
swapped = turn.swap();
|
|
1225
|
+
if (swapped) await turn.checkpoint();
|
|
1226
|
+
},
|
|
1227
|
+
afterSettling: turn.afterSettling
|
|
1228
|
+
});
|
|
1229
|
+
return swapped;
|
|
1230
|
+
}
|
|
1121
1231
|
async function checkpointInTurn(turns, turn) {
|
|
1122
1232
|
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
1123
1233
|
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
@@ -1421,6 +1531,19 @@ async function commitAppended(partition, address, deps) {
|
|
|
1421
1531
|
}
|
|
1422
1532
|
}
|
|
1423
1533
|
|
|
1534
|
+
// src/persistence/durability/mutation-entry.ts
|
|
1535
|
+
function buildMutationEntry(record, seqNo) {
|
|
1536
|
+
return buildEntry({
|
|
1537
|
+
seqNo,
|
|
1538
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
1539
|
+
operation: record.operation,
|
|
1540
|
+
partitionId: record.partitionId,
|
|
1541
|
+
indexName: record.indexName,
|
|
1542
|
+
documentId: record.documentId,
|
|
1543
|
+
document: record.document
|
|
1544
|
+
});
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1424
1547
|
// src/persistence/durability/partition-failure.ts
|
|
1425
1548
|
function createPartitionFailureRouting() {
|
|
1426
1549
|
let onFailed = null;
|
|
@@ -1534,6 +1657,8 @@ function parseSegmentStartSeqNo2(key) {
|
|
|
1534
1657
|
function createWalWriter(directory, config) {
|
|
1535
1658
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
1536
1659
|
const commitKey = `${config.indexName}/wal/${config.partitionId}/commit`;
|
|
1660
|
+
const preparedKey = nextSegmentKey(config.indexName, config.partitionId);
|
|
1661
|
+
let nextSegmentPrepared = false;
|
|
1537
1662
|
let handle = null;
|
|
1538
1663
|
let activeKey = null;
|
|
1539
1664
|
let activeStartSeqNo = 0;
|
|
@@ -1612,16 +1737,49 @@ function createWalWriter(directory, config) {
|
|
|
1612
1737
|
await withDurabilityLock(flushActiveSegment);
|
|
1613
1738
|
}
|
|
1614
1739
|
}
|
|
1615
|
-
async function
|
|
1616
|
-
|
|
1617
|
-
if (handle !== null) {
|
|
1618
|
-
await flushActiveSegment();
|
|
1619
|
-
await handle.close();
|
|
1620
|
-
handle = null;
|
|
1621
|
-
}
|
|
1622
|
-
await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
|
|
1740
|
+
async function rollUnderLock(startSeqNo) {
|
|
1741
|
+
if (handle !== null) {
|
|
1623
1742
|
await flushActiveSegment();
|
|
1743
|
+
await handle.close();
|
|
1744
|
+
handle = null;
|
|
1745
|
+
}
|
|
1746
|
+
await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
|
|
1747
|
+
await flushActiveSegment();
|
|
1748
|
+
}
|
|
1749
|
+
async function rollToNewSegment(startSeqNo) {
|
|
1750
|
+
await withDurabilityLock(() => rollUnderLock(startSeqNo));
|
|
1751
|
+
}
|
|
1752
|
+
function activeSegmentHoldsRecords() {
|
|
1753
|
+
return handle !== null && activeBytes > SEGMENT_HEADER_SIZE && highestAppendedSeqNo >= activeStartSeqNo;
|
|
1754
|
+
}
|
|
1755
|
+
async function prepareNextSegment() {
|
|
1756
|
+
const prepared = await directory.appendHandle(preparedKey);
|
|
1757
|
+
try {
|
|
1758
|
+
if (await prepared.size() > 0) await prepared.truncate(0);
|
|
1759
|
+
await prepared.append(writeSegmentHeader());
|
|
1760
|
+
await prepared.sync();
|
|
1761
|
+
nextSegmentPrepared = true;
|
|
1762
|
+
} finally {
|
|
1763
|
+
await prepared.close();
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
async function swapToPreparedSegment(sealed) {
|
|
1767
|
+
const writer = await getMarkerWriter();
|
|
1768
|
+
const startSeqNo = highestAppendedSeqNo + 1;
|
|
1769
|
+
const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
|
|
1770
|
+
const key = segmentKey(config.indexName, config.partitionId, startSeqNo);
|
|
1771
|
+
await Promise.all([sealed.sync(), directory.move(preparedKey, key)]);
|
|
1772
|
+
handle = await directory.appendHandle(key);
|
|
1773
|
+
activeKey = key;
|
|
1774
|
+
activeStartSeqNo = startSeqNo;
|
|
1775
|
+
activeBytes = SEGMENT_HEADER_SIZE;
|
|
1776
|
+
await writer.commit({
|
|
1777
|
+
activeSegmentSeqNo: startSeqNo,
|
|
1778
|
+
durableByteLength: SEGMENT_HEADER_SIZE,
|
|
1779
|
+
highestDurableSeqNo: durableSeqNo
|
|
1624
1780
|
});
|
|
1781
|
+
highestDurableSeqNo = durableSeqNo;
|
|
1782
|
+
await sealed.close();
|
|
1625
1783
|
}
|
|
1626
1784
|
async function maybeRoll(nextSeqNo, appendedBytes) {
|
|
1627
1785
|
if (handle === null || activeBytes <= SEGMENT_HEADER_SIZE) {
|
|
@@ -1707,6 +1865,19 @@ function createWalWriter(directory, config) {
|
|
|
1707
1865
|
await coordinator.commit();
|
|
1708
1866
|
},
|
|
1709
1867
|
rollToNewSegment,
|
|
1868
|
+
async prepareCheckpointRoll() {
|
|
1869
|
+
if (activeSegmentHoldsRecords()) await prepareNextSegment();
|
|
1870
|
+
},
|
|
1871
|
+
async rollForCheckpoint() {
|
|
1872
|
+
await withDurabilityLock(async () => {
|
|
1873
|
+
const sealed = handle;
|
|
1874
|
+
const prepared = nextSegmentPrepared;
|
|
1875
|
+
nextSegmentPrepared = false;
|
|
1876
|
+
if (sealed === null || !activeSegmentHoldsRecords()) return;
|
|
1877
|
+
if (prepared) await swapToPreparedSegment(sealed);
|
|
1878
|
+
else await rollUnderLock(highestAppendedSeqNo + 1);
|
|
1879
|
+
});
|
|
1880
|
+
},
|
|
1710
1881
|
async close() {
|
|
1711
1882
|
await withDurabilityLock(async () => {
|
|
1712
1883
|
if (handle !== null) {
|
|
@@ -2181,9 +2352,33 @@ async function readEarlierSegmentDocuments(store, key) {
|
|
|
2181
2352
|
|
|
2182
2353
|
// src/persistence/durability/checkpoint-count.ts
|
|
2183
2354
|
var decoder = new TextDecoder();
|
|
2355
|
+
async function countPagedPartition(store, partition) {
|
|
2356
|
+
const { directory } = store;
|
|
2357
|
+
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2358
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-IYPLWHGC.mjs');
|
|
2359
|
+
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2360
|
+
const scans = [];
|
|
2361
|
+
try {
|
|
2362
|
+
for (const ref of partition.segments) scans.push(await openPagedSegmentScan(directory, ref));
|
|
2363
|
+
let live = 0;
|
|
2364
|
+
for (let at = 0; at < scans.length; at++) {
|
|
2365
|
+
for (let ordinal = 0; ordinal < scans[at].count; ordinal++) {
|
|
2366
|
+
if (countsAsLatestHolder(scans, at, scans[at].idAt(ordinal))) live += 1;
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
return live;
|
|
2370
|
+
} finally {
|
|
2371
|
+
for (const scan of scans) scan.close();
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2184
2374
|
async function countCurrentLayout(store, manifest) {
|
|
2185
2375
|
let total = 0;
|
|
2186
2376
|
for (const partition of manifest.partitions) {
|
|
2377
|
+
const paged = await countPagedPartition(store, partition);
|
|
2378
|
+
if (paged !== null) {
|
|
2379
|
+
total += paged;
|
|
2380
|
+
continue;
|
|
2381
|
+
}
|
|
2187
2382
|
const live = /* @__PURE__ */ new Set();
|
|
2188
2383
|
for (const ref of partition.segments) {
|
|
2189
2384
|
const contents = await readSegmentFile(store, ref);
|
|
@@ -2458,6 +2653,45 @@ async function recoverCheckpoint(deps) {
|
|
|
2458
2653
|
return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
|
|
2459
2654
|
}
|
|
2460
2655
|
|
|
2656
|
+
// src/persistence/durability/stored-analysis.ts
|
|
2657
|
+
function generationOfStoredTerms(store, indexName) {
|
|
2658
|
+
return readSegmentManifest(store, indexName).then((decoded) => {
|
|
2659
|
+
if (decoded === null) return null;
|
|
2660
|
+
return decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
|
|
2661
|
+
});
|
|
2662
|
+
}
|
|
2663
|
+
function promotePendingAnalysis(metadata) {
|
|
2664
|
+
const pending = metadata.pendingAnalysis;
|
|
2665
|
+
if (pending === void 0) return;
|
|
2666
|
+
metadata.language = pending.language;
|
|
2667
|
+
metadata.analysisGeneration = pending.generation;
|
|
2668
|
+
metadata.analysisRevision = void 0;
|
|
2669
|
+
metadata.tokenizer = pending.tokenizer;
|
|
2670
|
+
metadata.stopWords = pending.stopWords;
|
|
2671
|
+
metadata.stopWordList = pending.stopWordList;
|
|
2672
|
+
metadata.pendingAnalysis = void 0;
|
|
2673
|
+
}
|
|
2674
|
+
async function settleStoredAnalysis(store, metadata, metadataOnly) {
|
|
2675
|
+
if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
|
|
2676
|
+
const stored = await generationOfStoredTerms(store, metadata.indexName);
|
|
2677
|
+
const current = metadata.analysisGeneration ?? 0;
|
|
2678
|
+
if (stored === null || stored === current) return false;
|
|
2679
|
+
if (stored === metadata.pendingAnalysis?.generation) {
|
|
2680
|
+
promotePendingAnalysis(metadata);
|
|
2681
|
+
return true;
|
|
2682
|
+
}
|
|
2683
|
+
throw new NarsilError(
|
|
2684
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2685
|
+
`The saved terms of index "${metadata.indexName}" come from analysis generation ${stored}, which matches no settings that its metadata records`,
|
|
2686
|
+
{
|
|
2687
|
+
indexName: metadata.indexName,
|
|
2688
|
+
storedGeneration: stored,
|
|
2689
|
+
currentGeneration: current,
|
|
2690
|
+
pendingGeneration: metadata.pendingAnalysis?.generation
|
|
2691
|
+
}
|
|
2692
|
+
);
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2461
2695
|
// src/persistence/durability/recover-index.ts
|
|
2462
2696
|
async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
2463
2697
|
await deps.queueMetadataWrite(indexName, async () => {
|
|
@@ -2483,9 +2717,12 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
2483
2717
|
derivedCount = void 0;
|
|
2484
2718
|
}
|
|
2485
2719
|
}
|
|
2720
|
+
const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
|
|
2486
2721
|
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
2487
2722
|
if (metadataOnly) {
|
|
2488
|
-
if (derivedCount !== void 0
|
|
2723
|
+
if (derivedCount !== void 0 || promoted) {
|
|
2724
|
+
await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
|
|
2725
|
+
}
|
|
2489
2726
|
return;
|
|
2490
2727
|
}
|
|
2491
2728
|
const manager = hooks.getManager(indexName);
|
|
@@ -2542,14 +2779,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
2542
2779
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
2543
2780
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
2544
2781
|
}
|
|
2545
|
-
if (recovered.upgradedDocumentCount !== null) {
|
|
2546
|
-
await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount);
|
|
2782
|
+
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
2783
|
+
await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
|
|
2547
2784
|
}
|
|
2548
2785
|
}
|
|
2549
2786
|
|
|
2550
2787
|
// src/persistence/durability/manager.ts
|
|
2551
2788
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
2552
|
-
const directory = createDurableDirectory(config.directory ?? "");
|
|
2789
|
+
const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
|
|
2553
2790
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
2554
2791
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
2555
2792
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -2632,17 +2869,6 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2632
2869
|
}
|
|
2633
2870
|
}
|
|
2634
2871
|
}
|
|
2635
|
-
function buildMutationEntry(record, seqNo) {
|
|
2636
|
-
return buildEntry({
|
|
2637
|
-
seqNo,
|
|
2638
|
-
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
2639
|
-
operation: record.operation,
|
|
2640
|
-
partitionId: record.partitionId,
|
|
2641
|
-
indexName: record.indexName,
|
|
2642
|
-
documentId: record.documentId,
|
|
2643
|
-
document: record.document
|
|
2644
|
-
});
|
|
2645
|
-
}
|
|
2646
2872
|
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
|
|
2647
2873
|
const indexState = indexes.get(indexName);
|
|
2648
2874
|
if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
|
|
@@ -2743,6 +2969,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2743
2969
|
await directoryWatch.claim();
|
|
2744
2970
|
await recoverDirectoryRestores(directory);
|
|
2745
2971
|
const names = await listPersistedIndexes(directory);
|
|
2972
|
+
await removeLeftoverRewriteFiles(directory, names);
|
|
2746
2973
|
for (const indexName of names) {
|
|
2747
2974
|
await recoverIndex(indexName, metadataOnly, false);
|
|
2748
2975
|
}
|
|
@@ -2766,6 +2993,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2766
2993
|
keepWriteRecordsWith(keeper) {
|
|
2767
2994
|
hooks.writeRecords = keeper;
|
|
2768
2995
|
},
|
|
2996
|
+
rewriteStorage: () => ({ directory, writeRecords: hooks.writeRecords ?? null }),
|
|
2769
2997
|
heldCheckpoints: () => heldCheckpoints.report(indexes),
|
|
2770
2998
|
async commitPartition(indexName, partitionId) {
|
|
2771
2999
|
const partition = indexes.get(indexName)?.partitions.get(partitionId);
|
|
@@ -2799,6 +3027,16 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2799
3027
|
checkpointFromMemory(indexName) {
|
|
2800
3028
|
return requestedCheckpoint(indexName, true);
|
|
2801
3029
|
},
|
|
3030
|
+
async checkpointAfterSwitch(indexName, swap) {
|
|
3031
|
+
const indexState = getOrCreateIndexState(indexName);
|
|
3032
|
+
heldCheckpoints.refuse(indexName, indexState);
|
|
3033
|
+
return switchInTurn(indexState, {
|
|
3034
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
3035
|
+
swap,
|
|
3036
|
+
checkpoint: () => performCheckpoint(indexName, indexState, true),
|
|
3037
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
3038
|
+
});
|
|
3039
|
+
},
|
|
2802
3040
|
async checkpointAfterInFlight(indexName) {
|
|
2803
3041
|
await loads.settled(indexName);
|
|
2804
3042
|
await checkpointIndex(indexName, false, true).catch((err) => markFatal(toError5(err)));
|
|
@@ -2911,6 +3149,17 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
2911
3149
|
return captured.documentCount;
|
|
2912
3150
|
}
|
|
2913
3151
|
|
|
3152
|
+
// src/persistence/durability/snapshot-only/index-state.ts
|
|
3153
|
+
function createSnapshotIndexState() {
|
|
3154
|
+
return {
|
|
3155
|
+
mutationsSinceCheckpoint: 0,
|
|
3156
|
+
checkpointInFlight: null,
|
|
3157
|
+
applyChain: Promise.resolve(),
|
|
3158
|
+
unloading: false,
|
|
3159
|
+
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
3160
|
+
};
|
|
3161
|
+
}
|
|
3162
|
+
|
|
2914
3163
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
2915
3164
|
function toError4(error) {
|
|
2916
3165
|
return error instanceof Error ? error : new Error(String(error));
|
|
@@ -2968,13 +3217,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
2968
3217
|
function getOrCreateIndexState(indexName) {
|
|
2969
3218
|
let state = indexes.get(indexName);
|
|
2970
3219
|
if (state === void 0) {
|
|
2971
|
-
state =
|
|
2972
|
-
mutationsSinceCheckpoint: 0,
|
|
2973
|
-
checkpointInFlight: null,
|
|
2974
|
-
applyChain: Promise.resolve(),
|
|
2975
|
-
unloading: false,
|
|
2976
|
-
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
2977
|
-
};
|
|
3220
|
+
state = createSnapshotIndexState();
|
|
2978
3221
|
indexes.set(indexName, state);
|
|
2979
3222
|
}
|
|
2980
3223
|
return state;
|
|
@@ -3002,7 +3245,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3002
3245
|
async function performCheckpoint(indexName, indexState, fromMemory) {
|
|
3003
3246
|
await settleRestore(indexName);
|
|
3004
3247
|
const manager = hooks.getManager(indexName);
|
|
3005
|
-
const metadata = hooks.
|
|
3248
|
+
const metadata = hooks.buildTermsMetadata(indexName);
|
|
3006
3249
|
if (manager === void 0 || metadata === void 0) {
|
|
3007
3250
|
return;
|
|
3008
3251
|
}
|
|
@@ -3013,8 +3256,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3013
3256
|
manager,
|
|
3014
3257
|
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
3015
3258
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
3016
|
-
rewritesEveryPartition: fromMemory
|
|
3259
|
+
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
|
|
3017
3260
|
});
|
|
3261
|
+
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
3018
3262
|
await queueMetadataWrite(indexName, async () => {
|
|
3019
3263
|
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
|
|
3020
3264
|
if (checkpointMetadata === void 0) return;
|
|
@@ -3050,6 +3294,11 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3050
3294
|
);
|
|
3051
3295
|
return queued;
|
|
3052
3296
|
}
|
|
3297
|
+
async function saveMetadata(indexName, documentCount) {
|
|
3298
|
+
const metadata = hooks.buildMetadata(indexName, documentCount);
|
|
3299
|
+
if (metadata === void 0) return;
|
|
3300
|
+
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
|
|
3301
|
+
}
|
|
3053
3302
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
3054
3303
|
return loads.track(indexName, loadIndex(indexName, metadataOnly), startsGraphRepairs);
|
|
3055
3304
|
}
|
|
@@ -3060,6 +3309,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3060
3309
|
return;
|
|
3061
3310
|
}
|
|
3062
3311
|
const { metadata } = await readMetadataEnvelope(metaBytes);
|
|
3312
|
+
const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
|
|
3063
3313
|
let derivedCount;
|
|
3064
3314
|
if (metadataOnly && metadata.documentCount === void 0) {
|
|
3065
3315
|
try {
|
|
@@ -3071,15 +3321,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3071
3321
|
}
|
|
3072
3322
|
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
3073
3323
|
if (metadataOnly) {
|
|
3074
|
-
if (derivedCount !== void 0) {
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
if (upgraded === void 0) {
|
|
3078
|
-
return;
|
|
3079
|
-
}
|
|
3080
|
-
const bytes = await writeMetadataEnvelope(upgraded, { checksum: true });
|
|
3081
|
-
await adapter.save(metadataKey(indexName), bytes);
|
|
3082
|
-
}).catch(() => void 0);
|
|
3324
|
+
if (derivedCount !== void 0 || promoted) {
|
|
3325
|
+
const counted = derivedCount ?? metadata.documentCount;
|
|
3326
|
+
await queueMetadataWrite(indexName, () => saveMetadata(indexName, counted)).catch(() => void 0);
|
|
3083
3327
|
}
|
|
3084
3328
|
return;
|
|
3085
3329
|
}
|
|
@@ -3107,13 +3351,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3107
3351
|
indexState.appliedSeqNoByPartition.set(partitionId, persistedSeqNo);
|
|
3108
3352
|
}
|
|
3109
3353
|
}
|
|
3110
|
-
if (recovered.upgradedDocumentCount !== null) {
|
|
3111
|
-
const upgradedCount = recovered.upgradedDocumentCount;
|
|
3112
|
-
await queueMetadataWrite(indexName,
|
|
3113
|
-
const upgraded = hooks.buildMetadata(indexName, upgradedCount);
|
|
3114
|
-
if (upgraded === void 0) return;
|
|
3115
|
-
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(upgraded, { checksum: true }));
|
|
3116
|
-
}).catch(() => void 0);
|
|
3354
|
+
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
3355
|
+
const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
|
|
3356
|
+
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
|
|
3117
3357
|
}
|
|
3118
3358
|
}
|
|
3119
3359
|
function countStoredBundleDocuments(indexName) {
|
|
@@ -3203,6 +3443,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3203
3443
|
checkpointFromMemory(indexName) {
|
|
3204
3444
|
return checkpointIndex(indexName, true);
|
|
3205
3445
|
},
|
|
3446
|
+
async checkpointAfterSwitch(indexName, swap) {
|
|
3447
|
+
const indexState = getOrCreateIndexState(indexName);
|
|
3448
|
+
return switchInTurn(indexState, {
|
|
3449
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
3450
|
+
swap,
|
|
3451
|
+
checkpoint: () => performCheckpoint(indexName, indexState, true),
|
|
3452
|
+
afterSettling: () => void 0
|
|
3453
|
+
});
|
|
3454
|
+
},
|
|
3206
3455
|
async checkpointAfterInFlight(indexName) {
|
|
3207
3456
|
await loads.settled(indexName);
|
|
3208
3457
|
await checkpointIndex(indexName, false, true).catch((err) => {
|
|
@@ -3277,6 +3526,14 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
3277
3526
|
getVectorFieldPaths: hooks.getVectorFieldPaths,
|
|
3278
3527
|
getVectorIndexes: hooks.getVectorIndexes,
|
|
3279
3528
|
buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
|
|
3529
|
+
buildTermsMetadata(indexName) {
|
|
3530
|
+
const config = hooks.getTermsConfig(indexName);
|
|
3531
|
+
return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
|
|
3532
|
+
},
|
|
3533
|
+
mustRewriteTerms: hooks.mustRewriteTerms,
|
|
3534
|
+
onManifestCommitted: hooks.onManifestCommitted,
|
|
3535
|
+
...hooks.queueDiskAdoption === void 0 ? {} : { queueDiskAdoption: hooks.queueDiskAdoption },
|
|
3536
|
+
...hooks.awaitCopiesOfIndex === void 0 ? {} : { awaitCopiesOfIndex: hooks.awaitCopiesOfIndex },
|
|
3280
3537
|
createIndexFromMetadata: hooks.createIndexFromMetadata,
|
|
3281
3538
|
recordCheckpoint: hooks.recordCheckpoint,
|
|
3282
3539
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
@@ -3347,9 +3604,20 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
3347
3604
|
getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
3348
3605
|
getVectorIndexes: (indexName) => (wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName)?.getVectorIndexes() : void 0) ?? /* @__PURE__ */ new Map(),
|
|
3349
3606
|
getIndexConfig: (indexName) => {
|
|
3607
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
3608
|
+
return entry === void 0 ? void 0 : durableMetadataConfigOf(entry);
|
|
3609
|
+
},
|
|
3610
|
+
getTermsConfig: (indexName) => {
|
|
3350
3611
|
const entry = wiring.indexRegistry.get(indexName);
|
|
3351
3612
|
return entry === void 0 ? void 0 : metadataConfigOf(entry);
|
|
3352
3613
|
},
|
|
3614
|
+
mustRewriteTerms: (indexName) => (wiring.indexRegistry.get(indexName)?.durableAnalysis ?? null) !== null,
|
|
3615
|
+
onManifestCommitted(indexName, analysisGeneration) {
|
|
3616
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
3617
|
+
if (entry !== void 0 && entry.analysisGeneration === analysisGeneration) entry.durableAnalysis = null;
|
|
3618
|
+
},
|
|
3619
|
+
queueDiskAdoption: wiring.queueDiskAdoption,
|
|
3620
|
+
awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
|
|
3353
3621
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
3354
3622
|
recordCheckpoint: wiring.recordCheckpoint,
|
|
3355
3623
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|