@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
1
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-MVM33ZUL.mjs';
|
|
2
|
+
import { restoreVectorFields } from './chunk-TCLEOZ4P.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-LVY2LI35.mjs';
|
|
4
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-5DB2VDGI.mjs';
|
|
5
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-D33A5VMU.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
6
7
|
import { decode, encode } from '@msgpack/msgpack';
|
|
7
8
|
|
|
8
9
|
// src/engine/durability-metadata.ts
|
|
@@ -42,6 +43,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
42
43
|
if (config.stopWordList !== void 0) {
|
|
43
44
|
metadata.stopWordList = config.stopWordList;
|
|
44
45
|
}
|
|
46
|
+
if (config.analysisGeneration !== void 0) {
|
|
47
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
48
|
+
}
|
|
49
|
+
if (config.pendingAnalysis !== void 0) {
|
|
50
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
51
|
+
}
|
|
45
52
|
if (config.partitionLimits !== void 0) {
|
|
46
53
|
metadata.partitionLimits = config.partitionLimits;
|
|
47
54
|
}
|
|
@@ -51,6 +58,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
51
58
|
if (config.trackPositions !== void 0) {
|
|
52
59
|
metadata.trackPositions = config.trackPositions;
|
|
53
60
|
}
|
|
61
|
+
if (config.textStorage !== void 0) {
|
|
62
|
+
metadata.textStorage = config.textStorage;
|
|
63
|
+
}
|
|
54
64
|
if (config.strict !== void 0) {
|
|
55
65
|
metadata.strict = config.strict;
|
|
56
66
|
}
|
|
@@ -96,9 +106,12 @@ function metadataConfigOf(entry) {
|
|
|
96
106
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
97
107
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
98
108
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
109
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
110
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
99
111
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
100
112
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
101
113
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
114
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
102
115
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
103
116
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
104
117
|
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
@@ -106,30 +119,42 @@ function metadataConfigOf(entry) {
|
|
|
106
119
|
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
|
|
107
120
|
};
|
|
108
121
|
}
|
|
122
|
+
function durableMetadataConfigOf(entry) {
|
|
123
|
+
const terms = metadataConfigOf(entry);
|
|
124
|
+
const durable = entry.durableAnalysis;
|
|
125
|
+
if (durable === null) return terms;
|
|
126
|
+
const {
|
|
127
|
+
language,
|
|
128
|
+
tokenizer,
|
|
129
|
+
stopWords,
|
|
130
|
+
stopWordList,
|
|
131
|
+
analysisRevision: _analysisRevision,
|
|
132
|
+
arrayPositionGap: _arrayPositionGap,
|
|
133
|
+
analysisGeneration: _analysisGeneration,
|
|
134
|
+
...rest
|
|
135
|
+
} = terms;
|
|
136
|
+
return {
|
|
137
|
+
...rest,
|
|
138
|
+
language: durable.language,
|
|
139
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
140
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
141
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
142
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
143
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
144
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
145
|
+
pendingAnalysis: {
|
|
146
|
+
generation: entry.analysisGeneration,
|
|
147
|
+
language,
|
|
148
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
149
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
150
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
}
|
|
109
154
|
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
110
155
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
111
156
|
}
|
|
112
157
|
|
|
113
|
-
// src/persistence/durability/constants.ts
|
|
114
|
-
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
115
|
-
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
116
|
-
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
117
|
-
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
118
|
-
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
119
|
-
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
120
|
-
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
121
|
-
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
122
|
-
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
123
|
-
var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
|
|
124
|
-
var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
125
|
-
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
126
|
-
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
127
|
-
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
128
|
-
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
129
|
-
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
130
|
-
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
131
|
-
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
132
|
-
|
|
133
158
|
// src/serialization/stored-vector-values.ts
|
|
134
159
|
function dropStoredVectorValues(documents, schema) {
|
|
135
160
|
const vectorFieldPaths = [];
|
|
@@ -337,6 +362,19 @@ function metadataToWire(meta) {
|
|
|
337
362
|
if (meta.stopWordList !== void 0) {
|
|
338
363
|
wire.stop_word_list = meta.stopWordList;
|
|
339
364
|
}
|
|
365
|
+
if (meta.analysisGeneration !== void 0) {
|
|
366
|
+
wire.analysis_generation = meta.analysisGeneration;
|
|
367
|
+
}
|
|
368
|
+
if (meta.pendingAnalysis !== void 0) {
|
|
369
|
+
const pending = meta.pendingAnalysis;
|
|
370
|
+
wire.pending_analysis = {
|
|
371
|
+
generation: pending.generation,
|
|
372
|
+
language: pending.language,
|
|
373
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
374
|
+
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
375
|
+
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
376
|
+
};
|
|
377
|
+
}
|
|
340
378
|
if (meta.partitionLimits !== void 0) {
|
|
341
379
|
wire.partition_limits = {
|
|
342
380
|
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
@@ -350,6 +388,9 @@ function metadataToWire(meta) {
|
|
|
350
388
|
if (meta.trackPositions !== void 0) {
|
|
351
389
|
wire.track_positions = meta.trackPositions;
|
|
352
390
|
}
|
|
391
|
+
if (meta.textStorage !== void 0) {
|
|
392
|
+
wire.text_storage = meta.textStorage;
|
|
393
|
+
}
|
|
353
394
|
if (meta.strict !== void 0) {
|
|
354
395
|
wire.strict = meta.strict;
|
|
355
396
|
}
|
|
@@ -432,6 +473,19 @@ function wireToMetadata(raw) {
|
|
|
432
473
|
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
433
474
|
meta.stopWordList = raw.stop_word_list;
|
|
434
475
|
}
|
|
476
|
+
if (isPositiveInteger(raw.analysis_generation)) {
|
|
477
|
+
meta.analysisGeneration = raw.analysis_generation;
|
|
478
|
+
}
|
|
479
|
+
const pending = raw.pending_analysis;
|
|
480
|
+
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
481
|
+
meta.pendingAnalysis = {
|
|
482
|
+
generation: pending.generation,
|
|
483
|
+
language: pending.language,
|
|
484
|
+
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
485
|
+
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
486
|
+
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
487
|
+
};
|
|
488
|
+
}
|
|
435
489
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
436
490
|
const limits = {
|
|
437
491
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
@@ -448,6 +502,14 @@ function wireToMetadata(raw) {
|
|
|
448
502
|
if (typeof raw.track_positions === "boolean") {
|
|
449
503
|
meta.trackPositions = raw.track_positions;
|
|
450
504
|
}
|
|
505
|
+
if (raw.text_storage !== void 0) {
|
|
506
|
+
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
507
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
508
|
+
textStorage: raw.text_storage
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
meta.textStorage = raw.text_storage;
|
|
512
|
+
}
|
|
451
513
|
if (typeof raw.strict === "boolean") {
|
|
452
514
|
meta.strict = raw.strict;
|
|
453
515
|
}
|
|
@@ -524,6 +586,9 @@ function wireToMetadata(raw) {
|
|
|
524
586
|
function isNonNegativeInteger(value) {
|
|
525
587
|
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
526
588
|
}
|
|
589
|
+
function isStringList(value) {
|
|
590
|
+
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
591
|
+
}
|
|
527
592
|
function isPositiveInteger(value) {
|
|
528
593
|
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
529
594
|
}
|
|
@@ -595,7 +660,7 @@ async function packEnvelope(payloadBytes, options) {
|
|
|
595
660
|
const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
|
|
596
661
|
const [major, minor, patch] = engineVersionParts();
|
|
597
662
|
const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
|
|
598
|
-
const
|
|
663
|
+
const header = {
|
|
599
664
|
magic: "NRSL",
|
|
600
665
|
envelopeFormatVersion,
|
|
601
666
|
engineVersionMajor: major,
|
|
@@ -606,64 +671,64 @@ async function packEnvelope(payloadBytes, options) {
|
|
|
606
671
|
checksum,
|
|
607
672
|
reserved: new Uint8Array(14)
|
|
608
673
|
};
|
|
609
|
-
const headerBytes = writeHeader(
|
|
674
|
+
const headerBytes = writeHeader(header);
|
|
610
675
|
const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
|
|
611
676
|
envelope.set(headerBytes, 0);
|
|
612
677
|
envelope.set(finalPayload, HEADER_SIZE);
|
|
613
678
|
return envelope;
|
|
614
679
|
}
|
|
615
680
|
async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
|
|
616
|
-
const
|
|
617
|
-
if (
|
|
681
|
+
const header = readHeader(data);
|
|
682
|
+
if (header.envelopeFormatVersion > maxAcceptedVersion) {
|
|
618
683
|
throw new NarsilError(
|
|
619
684
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
620
|
-
`This data holds Narsil envelope format v${
|
|
685
|
+
`This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
|
|
621
686
|
);
|
|
622
687
|
}
|
|
623
|
-
if (
|
|
688
|
+
if (header.flags.encryptionEnabled) {
|
|
624
689
|
throw new NarsilError(
|
|
625
690
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
626
691
|
"Encrypted envelopes are not supported in this version of Narsil"
|
|
627
692
|
);
|
|
628
693
|
}
|
|
629
|
-
const compAlgo =
|
|
630
|
-
if (
|
|
694
|
+
const compAlgo = header.flags.compressionAlgorithm;
|
|
695
|
+
if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
|
|
631
696
|
throw new NarsilError(
|
|
632
697
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
633
698
|
`Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
|
|
634
699
|
);
|
|
635
700
|
}
|
|
636
701
|
const availablePayload = data.length - HEADER_SIZE;
|
|
637
|
-
if (availablePayload <
|
|
702
|
+
if (availablePayload < header.payloadLength) {
|
|
638
703
|
throw new NarsilError(
|
|
639
704
|
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
640
|
-
`Envelope data is truncated: header declares ${
|
|
705
|
+
`Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
|
|
641
706
|
);
|
|
642
707
|
}
|
|
643
|
-
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE +
|
|
644
|
-
if (
|
|
708
|
+
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
|
|
709
|
+
if (header.flags.checksumPresent) {
|
|
645
710
|
const computed = crc32(payloadBytes);
|
|
646
|
-
if (computed !==
|
|
711
|
+
if (computed !== header.checksum) {
|
|
647
712
|
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
|
|
648
|
-
expected:
|
|
713
|
+
expected: header.checksum,
|
|
649
714
|
computed
|
|
650
715
|
});
|
|
651
716
|
}
|
|
652
717
|
}
|
|
653
|
-
if (
|
|
718
|
+
if (header.flags.compressionEnabled) {
|
|
654
719
|
payloadBytes = await decompressGzip(payloadBytes);
|
|
655
720
|
}
|
|
656
|
-
return { header
|
|
721
|
+
return { header, payloadBytes };
|
|
657
722
|
}
|
|
658
723
|
async function unpackEnvelopeBytes(data) {
|
|
659
724
|
return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
|
|
660
725
|
}
|
|
661
726
|
async function unpackIndexSnapshotEnvelope(data) {
|
|
662
|
-
const { header
|
|
663
|
-
if (
|
|
727
|
+
const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
|
|
728
|
+
if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
|
|
664
729
|
throw new NarsilError(
|
|
665
730
|
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
666
|
-
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${
|
|
731
|
+
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
|
|
667
732
|
);
|
|
668
733
|
}
|
|
669
734
|
return payloadBytes;
|
|
@@ -673,126 +738,9 @@ async function writeMetadataEnvelope(meta, options = {}) {
|
|
|
673
738
|
return packEnvelope(payloadBytes, options);
|
|
674
739
|
}
|
|
675
740
|
async function readMetadataEnvelope(data) {
|
|
676
|
-
const { header
|
|
741
|
+
const { header, payloadBytes } = await unpackEnvelope(data);
|
|
677
742
|
const metadata = deserializeMetadata(payloadBytes);
|
|
678
|
-
return { header
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
// src/persistence/durability/segment/layout.ts
|
|
682
|
-
var SEGMENT_ID_WIDTH = 16;
|
|
683
|
-
function manifestKey(indexName) {
|
|
684
|
-
return `${indexName}/manifest`;
|
|
685
|
-
}
|
|
686
|
-
function snapshotBundleKey(indexName) {
|
|
687
|
-
return `${indexName}/snapshot`;
|
|
688
|
-
}
|
|
689
|
-
function segmentPrefix(indexName, partitionId) {
|
|
690
|
-
return `${indexName}/segments/${partitionId}/`;
|
|
691
|
-
}
|
|
692
|
-
function segmentKey(indexName, partitionId, segmentId) {
|
|
693
|
-
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
694
|
-
}
|
|
695
|
-
function segmentsPrefix(indexName) {
|
|
696
|
-
return `${indexName}/segments/`;
|
|
697
|
-
}
|
|
698
|
-
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
699
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
700
|
-
}
|
|
701
|
-
function requireGeneration(generation) {
|
|
702
|
-
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
703
|
-
throw new NarsilError(
|
|
704
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
705
|
-
`Vector graph generation ${generation} must be a positive integer`,
|
|
706
|
-
{ generation }
|
|
707
|
-
);
|
|
708
|
-
}
|
|
709
|
-
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
710
|
-
requireGeneration(generation);
|
|
711
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
712
|
-
}
|
|
713
|
-
function formatSegmentId(segmentId) {
|
|
714
|
-
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
715
|
-
throw new NarsilError(
|
|
716
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
717
|
-
`Segment id ${segmentId} must be a non-negative integer`,
|
|
718
|
-
{ segmentId }
|
|
719
|
-
);
|
|
720
|
-
}
|
|
721
|
-
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
722
|
-
}
|
|
723
|
-
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
724
|
-
function encodeFieldPath(fieldPath) {
|
|
725
|
-
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
726
|
-
throw new NarsilError(
|
|
727
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
728
|
-
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
729
|
-
{ fieldPath }
|
|
730
|
-
);
|
|
731
|
-
}
|
|
732
|
-
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
733
|
-
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
734
|
-
}
|
|
735
|
-
var SEGMENT_MANIFEST_VERSION = 7;
|
|
736
|
-
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
737
|
-
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
738
|
-
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
739
|
-
var MAX_SEGMENT_ID = 281474976710655;
|
|
740
|
-
function fileRefOnTheWire(ref) {
|
|
741
|
-
return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
|
|
742
|
-
}
|
|
743
|
-
function encodeSegmentManifest(manifest) {
|
|
744
|
-
const payload = encode({
|
|
745
|
-
version: SEGMENT_MANIFEST_VERSION,
|
|
746
|
-
schema: manifest.schema,
|
|
747
|
-
language: manifest.language,
|
|
748
|
-
checkpoint: manifest.checkpoint.map((c) => ({
|
|
749
|
-
partitionId: c.partitionId,
|
|
750
|
-
lastSeqNo: c.lastSeqNo,
|
|
751
|
-
primaryTerm: c.primaryTerm,
|
|
752
|
-
...c.termHistory === void 0 ? {} : {
|
|
753
|
-
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
754
|
-
}
|
|
755
|
-
})),
|
|
756
|
-
partitions: manifest.partitions.map((p) => ({
|
|
757
|
-
partitionId: p.partitionId,
|
|
758
|
-
nextSegmentId: p.nextSegmentId,
|
|
759
|
-
segments: p.segments.map((s) => ({
|
|
760
|
-
id: s.id,
|
|
761
|
-
key: s.key,
|
|
762
|
-
docCount: s.docCount,
|
|
763
|
-
tombstoneCount: s.tombstoneCount,
|
|
764
|
-
bytes: s.bytes,
|
|
765
|
-
checksum: s.checksum
|
|
766
|
-
}))
|
|
767
|
-
})),
|
|
768
|
-
vectors: manifest.vectors.map((v) => ({
|
|
769
|
-
fieldPath: v.fieldPath,
|
|
770
|
-
nextFileId: v.nextFileId,
|
|
771
|
-
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
772
|
-
graphGeneration: v.graphGeneration,
|
|
773
|
-
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
774
|
-
}))
|
|
775
|
-
});
|
|
776
|
-
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
777
|
-
}
|
|
778
|
-
function manifestReferencedKeys(manifest) {
|
|
779
|
-
const keys = /* @__PURE__ */ new Set();
|
|
780
|
-
for (const partition of manifest.partitions) {
|
|
781
|
-
for (const segment of partition.segments) {
|
|
782
|
-
keys.add(segment.key);
|
|
783
|
-
}
|
|
784
|
-
}
|
|
785
|
-
for (const vector of manifest.vectors) {
|
|
786
|
-
for (const file of vector.files) keys.add(file.key);
|
|
787
|
-
for (const part of vector.graphParts) keys.add(part.key);
|
|
788
|
-
}
|
|
789
|
-
return keys;
|
|
790
|
-
}
|
|
791
|
-
function manifestFileRefs(manifest) {
|
|
792
|
-
const refs = [];
|
|
793
|
-
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
794
|
-
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
795
|
-
return refs;
|
|
743
|
+
return { header, metadata };
|
|
796
744
|
}
|
|
797
745
|
|
|
798
746
|
// src/persistence/durability/segment/manifest-fields.ts
|
|
@@ -823,7 +771,20 @@ function readFileRef(raw, expectedKey, what) {
|
|
|
823
771
|
invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
|
|
824
772
|
}
|
|
825
773
|
if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
|
|
826
|
-
return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
774
|
+
if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
775
|
+
const { headOffset, headLength, headChecksum } = raw;
|
|
776
|
+
if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
|
|
777
|
+
invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
|
|
778
|
+
}
|
|
779
|
+
if (headOffset + headLength > bytes - HEADER_SIZE) {
|
|
780
|
+
invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
|
|
781
|
+
key: expectedKey,
|
|
782
|
+
headOffset,
|
|
783
|
+
headLength,
|
|
784
|
+
bytes
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
|
|
827
788
|
}
|
|
828
789
|
|
|
829
790
|
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
@@ -920,12 +881,16 @@ function decodeManifestPayload(raw, indexName) {
|
|
|
920
881
|
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
921
882
|
requireValidFieldTypes(schema);
|
|
922
883
|
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
884
|
+
if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
|
|
885
|
+
invalidManifest("has an invalid analysis generation");
|
|
886
|
+
}
|
|
923
887
|
return {
|
|
924
888
|
kind: "current",
|
|
925
889
|
manifest: {
|
|
926
890
|
version: SEGMENT_MANIFEST_VERSION,
|
|
927
891
|
schema,
|
|
928
892
|
language: raw.language,
|
|
893
|
+
...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
|
|
929
894
|
checkpoint: normalizeCheckpoint(raw.checkpoint),
|
|
930
895
|
partitions: normalizePartitions(raw.partitions, indexName),
|
|
931
896
|
vectors: normalizeVectorFields(raw.vectors, indexName)
|
|
@@ -1077,83 +1042,292 @@ function contentsOfCaptured(captured) {
|
|
|
1077
1042
|
return contents;
|
|
1078
1043
|
}
|
|
1079
1044
|
|
|
1080
|
-
// src/
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1045
|
+
// src/serialization/crc32-combine.ts
|
|
1046
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
1047
|
+
var CRC_BITS = 32;
|
|
1048
|
+
var zeroByteOperators = [];
|
|
1049
|
+
function timesVector(operator, vector) {
|
|
1050
|
+
let sum = 0;
|
|
1051
|
+
let rest = vector >>> 0;
|
|
1052
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
1053
|
+
if (rest & 1) sum ^= operator[row];
|
|
1054
|
+
}
|
|
1055
|
+
return sum >>> 0;
|
|
1056
|
+
}
|
|
1057
|
+
function squared(operator) {
|
|
1058
|
+
const result = new Uint32Array(CRC_BITS);
|
|
1059
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1060
|
+
return result;
|
|
1061
|
+
}
|
|
1062
|
+
function firstOperator() {
|
|
1063
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1064
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1065
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1066
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1067
|
+
return oneZeroBit;
|
|
1068
|
+
}
|
|
1069
|
+
function operatorForZeroBytesPower(power) {
|
|
1070
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
1071
|
+
while (zeroByteOperators.length <= power) {
|
|
1072
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
1073
|
+
}
|
|
1074
|
+
return zeroByteOperators[power];
|
|
1075
|
+
}
|
|
1076
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
1077
|
+
let crc = first >>> 0;
|
|
1078
|
+
let rest = secondByteLength;
|
|
1079
|
+
for (let power = 0; rest > 0; power++) {
|
|
1080
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
1081
|
+
rest = Math.floor(rest / 2);
|
|
1082
|
+
}
|
|
1083
|
+
return (crc ^ second) >>> 0;
|
|
1084
|
+
}
|
|
1085
|
+
var BIN8 = 196;
|
|
1086
|
+
var BIN16 = 197;
|
|
1087
|
+
var BIN32 = 198;
|
|
1088
|
+
var ARRAY16 = 220;
|
|
1089
|
+
var ARRAY32 = 221;
|
|
1090
|
+
var FIXMAP = 128;
|
|
1091
|
+
var FIXARRAY = 144;
|
|
1092
|
+
var FIXED_LIST_MAX = 15;
|
|
1093
|
+
var FIXSTR = 160;
|
|
1094
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
1095
|
+
var ASCII_MAX = 127;
|
|
1096
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
1097
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
1098
|
+
var BIN8_MAX_BYTES = 255;
|
|
1099
|
+
var UINT16_MAX = 65535;
|
|
1100
|
+
function binHeader(byteLength) {
|
|
1101
|
+
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
1102
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1103
|
+
const header = new Uint8Array(5);
|
|
1104
|
+
header[0] = BIN32;
|
|
1105
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1106
|
+
return header;
|
|
1107
|
+
}
|
|
1108
|
+
function arrayHeader(count) {
|
|
1109
|
+
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
1110
|
+
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
1111
|
+
const header = new Uint8Array(5);
|
|
1112
|
+
header[0] = ARRAY32;
|
|
1113
|
+
new DataView(header.buffer).setUint32(1, count, false);
|
|
1114
|
+
return header;
|
|
1115
|
+
}
|
|
1116
|
+
function createPayloadBuilder() {
|
|
1117
|
+
const chunks = [];
|
|
1098
1118
|
let offset = 0;
|
|
1099
|
-
|
|
1100
|
-
|
|
1119
|
+
function push(chunk) {
|
|
1120
|
+
chunks.push(chunk);
|
|
1101
1121
|
offset += chunk.byteLength;
|
|
1102
1122
|
}
|
|
1123
|
+
return {
|
|
1124
|
+
chunks,
|
|
1125
|
+
get offset() {
|
|
1126
|
+
return offset;
|
|
1127
|
+
},
|
|
1128
|
+
push,
|
|
1129
|
+
entry(key, value) {
|
|
1130
|
+
push(encode(key));
|
|
1131
|
+
push(encode(value));
|
|
1132
|
+
},
|
|
1133
|
+
rawEntry(key, encoded) {
|
|
1134
|
+
push(encode(key));
|
|
1135
|
+
const at = offset;
|
|
1136
|
+
push(encoded);
|
|
1137
|
+
return at;
|
|
1138
|
+
},
|
|
1139
|
+
bytesEntry(key, bytes) {
|
|
1140
|
+
push(encode(key));
|
|
1141
|
+
push(binHeader(bytes.byteLength));
|
|
1142
|
+
const at = offset;
|
|
1143
|
+
push(bytes);
|
|
1144
|
+
return at;
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
function pageChecksums(bytes, pageBytes) {
|
|
1149
|
+
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
1150
|
+
const checksums = new Uint8Array(pages * 4);
|
|
1151
|
+
const view = new DataView(checksums.buffer);
|
|
1152
|
+
for (let page = 0; page < pages; page++) {
|
|
1153
|
+
const from = page * pageBytes;
|
|
1154
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
1155
|
+
}
|
|
1156
|
+
return checksums;
|
|
1157
|
+
}
|
|
1158
|
+
function sectionOf(name, offset, bytes, pageBytes) {
|
|
1159
|
+
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
1160
|
+
}
|
|
1161
|
+
function uint32Bytes(values) {
|
|
1162
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
1163
|
+
const view = new DataView(bytes.buffer);
|
|
1164
|
+
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
1103
1165
|
return bytes;
|
|
1104
1166
|
}
|
|
1105
|
-
function
|
|
1106
|
-
if (
|
|
1107
|
-
|
|
1108
|
-
|
|
1167
|
+
function writtenAsShortAscii(docId, list, at) {
|
|
1168
|
+
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
1169
|
+
for (let unit = 0; unit < docId.length; unit++) {
|
|
1170
|
+
const code = docId.charCodeAt(unit);
|
|
1171
|
+
if (code > ASCII_MAX) return false;
|
|
1172
|
+
list[at + 1 + unit] = code;
|
|
1173
|
+
}
|
|
1174
|
+
list[at] = FIXSTR | docId.length;
|
|
1175
|
+
return true;
|
|
1109
1176
|
}
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1177
|
+
function encodedIdList(docIds) {
|
|
1178
|
+
const header = arrayHeader(docIds.length);
|
|
1179
|
+
let capacity = header.byteLength;
|
|
1180
|
+
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
1181
|
+
const list = new Uint8Array(capacity);
|
|
1182
|
+
list.set(header);
|
|
1183
|
+
const offsets = new Uint32Array(docIds.length + 1);
|
|
1184
|
+
let cursor = header.byteLength;
|
|
1185
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
1186
|
+
offsets[i] = cursor;
|
|
1187
|
+
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
1188
|
+
cursor += docIds[i].length + 1;
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
const encoded = encode(docIds[i]);
|
|
1192
|
+
list.set(encoded, cursor);
|
|
1193
|
+
cursor += encoded.byteLength;
|
|
1115
1194
|
}
|
|
1195
|
+
offsets[docIds.length] = cursor;
|
|
1196
|
+
return { list: list.subarray(0, cursor), offsets };
|
|
1116
1197
|
}
|
|
1117
|
-
function
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
1122
|
-
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
1123
|
-
};
|
|
1198
|
+
function positionsInIdOrder(docIds) {
|
|
1199
|
+
const positions = Array.from(docIds.keys());
|
|
1200
|
+
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
1201
|
+
return Uint32Array.from(positions);
|
|
1124
1202
|
}
|
|
1125
|
-
|
|
1126
|
-
const
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
const bytes = await store.read(key);
|
|
1130
|
-
if (bytes === null) missingFile(expected.what, key);
|
|
1131
|
-
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
1203
|
+
function magnitudeBytes(magnitudes) {
|
|
1204
|
+
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
1205
|
+
const view = new DataView(bytes.buffer);
|
|
1206
|
+
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
1132
1207
|
return bytes;
|
|
1133
1208
|
}
|
|
1134
|
-
function
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1209
|
+
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
1210
|
+
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
1211
|
+
let whole = 0;
|
|
1212
|
+
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
1213
|
+
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
1214
|
+
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
1215
|
+
}
|
|
1216
|
+
return whole;
|
|
1217
|
+
}
|
|
1218
|
+
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
1219
|
+
let headLength = 0;
|
|
1220
|
+
let head = new Uint8Array(0);
|
|
1221
|
+
let vectorsSection = {
|
|
1222
|
+
name: "vectors",
|
|
1223
|
+
offset: 0,
|
|
1224
|
+
length: vectors.byteLength,
|
|
1225
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
1226
|
+
checksums
|
|
1227
|
+
};
|
|
1228
|
+
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
1229
|
+
vectorsSection = {
|
|
1230
|
+
...vectorsSection,
|
|
1231
|
+
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
1232
|
+
};
|
|
1233
|
+
head = encode({ sections: [...sections, vectorsSection] });
|
|
1234
|
+
if (head.byteLength === headLength) break;
|
|
1235
|
+
headLength = head.byteLength;
|
|
1236
|
+
}
|
|
1237
|
+
return { head, vectorsSection };
|
|
1238
|
+
}
|
|
1239
|
+
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
1240
|
+
const builder = createPayloadBuilder();
|
|
1241
|
+
const { codes } = payload;
|
|
1242
|
+
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
1243
|
+
builder.entry("v", payload.v);
|
|
1244
|
+
builder.entry("fieldName", payload.fieldName);
|
|
1245
|
+
builder.entry("dimension", payload.dimension);
|
|
1246
|
+
const { list, offsets } = encodedIdList(payload.docIds);
|
|
1247
|
+
const docIdsAt = builder.rawEntry("docIds", list);
|
|
1248
|
+
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
1249
|
+
builder.push(encode("codes"));
|
|
1250
|
+
if (codes === null) {
|
|
1251
|
+
builder.push(encode(null));
|
|
1252
|
+
} else {
|
|
1253
|
+
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
1254
|
+
builder.entry("bits", codes.bits);
|
|
1255
|
+
builder.entry("centroid", codes.centroid);
|
|
1256
|
+
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
1257
|
+
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
1258
|
+
}
|
|
1259
|
+
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
1260
|
+
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
1261
|
+
const idOffsets = uint32Bytes(offsets);
|
|
1262
|
+
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
1263
|
+
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
1264
|
+
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
1265
|
+
const headKey = encode("head");
|
|
1266
|
+
const vectorsKey = encode("vectors");
|
|
1267
|
+
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
1268
|
+
const beforeHead = builder.offset + headKey.byteLength;
|
|
1269
|
+
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
1270
|
+
const { head } = encodedHeadAndVectorsPlace(
|
|
1271
|
+
sections,
|
|
1272
|
+
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
1273
|
+
payload.vectors,
|
|
1274
|
+
vectorChecksums,
|
|
1275
|
+
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
1145
1276
|
);
|
|
1277
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1278
|
+
builder.push(vectorsKey);
|
|
1279
|
+
builder.push(vectorsHeader);
|
|
1280
|
+
const beforeVectors = [...builder.chunks];
|
|
1281
|
+
builder.push(payload.vectors);
|
|
1282
|
+
return {
|
|
1283
|
+
chunks: builder.chunks,
|
|
1284
|
+
headOffset,
|
|
1285
|
+
headLength: head.byteLength,
|
|
1286
|
+
headChecksum: crc32(head),
|
|
1287
|
+
beforeVectors,
|
|
1288
|
+
vectors: {
|
|
1289
|
+
byteLength: payload.vectors.byteLength,
|
|
1290
|
+
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
1291
|
+
}
|
|
1292
|
+
};
|
|
1146
1293
|
}
|
|
1147
|
-
function
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1294
|
+
function nodeOffsetsOf(graph) {
|
|
1295
|
+
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
1296
|
+
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
1297
|
+
let cursor = 0;
|
|
1298
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
1299
|
+
offsets[at] = cursor;
|
|
1300
|
+
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
1301
|
+
}
|
|
1302
|
+
offsets[graph.levels.length] = cursor;
|
|
1303
|
+
return uint32Bytes(offsets);
|
|
1304
|
+
}
|
|
1305
|
+
function pagedGraphPartChunks(part) {
|
|
1306
|
+
const builder = createPayloadBuilder();
|
|
1307
|
+
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
1308
|
+
builder.entry("v", part.v);
|
|
1309
|
+
builder.entry("fieldName", part.fieldName);
|
|
1310
|
+
builder.push(encode("graphs"));
|
|
1311
|
+
builder.push(arrayHeader(part.graphs.length));
|
|
1312
|
+
const layouts = [];
|
|
1313
|
+
for (const graph of part.graphs) {
|
|
1314
|
+
const { levels, neighbours, ...header } = graph;
|
|
1315
|
+
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
1316
|
+
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
1317
|
+
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
1318
|
+
const levelsAt = builder.bytesEntry("levels", levels);
|
|
1319
|
+
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
1320
|
+
layouts.push({
|
|
1321
|
+
...header,
|
|
1322
|
+
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
1323
|
+
nodeOffsets: nodeOffsetsOf(graph),
|
|
1324
|
+
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1327
|
+
const head = encode({ graphs: layouts });
|
|
1328
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1329
|
+
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
1154
1330
|
}
|
|
1155
|
-
|
|
1156
|
-
// src/persistence/durability/segment/vector-load.ts
|
|
1157
1331
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
1158
1332
|
var GRAPH_PART_ENVELOPE_VERSION = 2;
|
|
1159
1333
|
function liveVectorsOf(ref) {
|
|
@@ -1163,13 +1337,13 @@ function liveVectorsOf(ref) {
|
|
|
1163
1337
|
}
|
|
1164
1338
|
async function openListedFile(store, ref, what, allowsCompression) {
|
|
1165
1339
|
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
|
|
1166
|
-
const { header
|
|
1340
|
+
const { header, payload } = await openChecksummedEnvelope(bytes, {
|
|
1167
1341
|
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
1168
1342
|
what: `${what} "${ref.key}"`,
|
|
1169
1343
|
allowsCompression
|
|
1170
1344
|
});
|
|
1171
|
-
requireListedChecksum(what, ref.key,
|
|
1172
|
-
return { payload, payloadLength:
|
|
1345
|
+
requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
|
|
1346
|
+
return { payload, payloadLength: header.payloadLength };
|
|
1173
1347
|
}
|
|
1174
1348
|
async function readGraphParts(store, ref) {
|
|
1175
1349
|
if (ref.graphParts.length === 0) return null;
|
|
@@ -1207,7 +1381,46 @@ async function readVectorFile(store, file, fieldPath) {
|
|
|
1207
1381
|
}
|
|
1208
1382
|
return { payload, payloadLength };
|
|
1209
1383
|
}
|
|
1210
|
-
|
|
1384
|
+
function holdsEveryHead(ref) {
|
|
1385
|
+
if (ref.graphParts.length === 0 || ref.files.length === 0) return false;
|
|
1386
|
+
return [...ref.files, ...ref.graphParts].every((file) => file.headOffset !== void 0);
|
|
1387
|
+
}
|
|
1388
|
+
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
1389
|
+
const { pathOf } = store;
|
|
1390
|
+
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
1391
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-MJMKNUYJ.mjs');
|
|
1392
|
+
await Promise.all([
|
|
1393
|
+
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
1394
|
+
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
1395
|
+
]);
|
|
1396
|
+
const files = await Promise.all(
|
|
1397
|
+
ref.files.map(async (file) => ({ ref: file, path: await pathOf.call(store, file.key) }))
|
|
1398
|
+
);
|
|
1399
|
+
const graphParts = await Promise.all(
|
|
1400
|
+
ref.graphParts.map(async (part) => ({ ref: part, path: await pathOf.call(store, part.key) }))
|
|
1401
|
+
);
|
|
1402
|
+
const field = openColdVectorField({
|
|
1403
|
+
fieldPath: ref.fieldPath,
|
|
1404
|
+
dimension: vectorIndex.dimension,
|
|
1405
|
+
metric: vectorIndex.metric,
|
|
1406
|
+
codeBits: osqBitsOf(vectorIndex.quantization),
|
|
1407
|
+
files,
|
|
1408
|
+
graphParts,
|
|
1409
|
+
resolvePartition
|
|
1410
|
+
});
|
|
1411
|
+
if (field === null) return false;
|
|
1412
|
+
vectorIndex.openColdField(field, async (beginRestore, stillWanted, firstSearchAnswered) => {
|
|
1413
|
+
await waitBeforeColdLoad(firstSearchAnswered);
|
|
1414
|
+
if (!stillWanted()) return;
|
|
1415
|
+
const graph = await readGraphParts(store, ref);
|
|
1416
|
+
if (!stillWanted()) return;
|
|
1417
|
+
const restore = beginRestore({ liveVectors: liveVectorsOf(ref), holdsGraph: true, graph });
|
|
1418
|
+
await loadFilesByParts(files, restore, ref.fieldPath, vectorIndex.dimension, stillWanted);
|
|
1419
|
+
});
|
|
1420
|
+
return true;
|
|
1421
|
+
}
|
|
1422
|
+
async function loadVectorField(store, ref, vectorIndex, resolvePartition = null) {
|
|
1423
|
+
if (resolvePartition !== null && await loadFieldOnDemand(store, ref, vectorIndex, resolvePartition)) return;
|
|
1211
1424
|
const restore = vectorIndex.restoreCheckpoint({
|
|
1212
1425
|
liveVectors: liveVectorsOf(ref),
|
|
1213
1426
|
holdsGraph: ref.graphParts.length > 0,
|
|
@@ -1230,24 +1443,24 @@ async function loadVectorField(store, ref, vectorIndex) {
|
|
|
1230
1443
|
}
|
|
1231
1444
|
var EMPTY_BIN = new Uint8Array(0);
|
|
1232
1445
|
var EMPTY_BIN_BYTES = 2;
|
|
1233
|
-
var
|
|
1234
|
-
var
|
|
1235
|
-
var
|
|
1236
|
-
var
|
|
1446
|
+
var BIN82 = 196;
|
|
1447
|
+
var BIN162 = 197;
|
|
1448
|
+
var BIN322 = 198;
|
|
1449
|
+
var BIN8_MAX_BYTES2 = 255;
|
|
1237
1450
|
var BIN16_MAX_BYTES = 65535;
|
|
1238
|
-
function
|
|
1239
|
-
if (byteLength <=
|
|
1240
|
-
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(
|
|
1241
|
-
const
|
|
1242
|
-
|
|
1243
|
-
new DataView(
|
|
1244
|
-
return
|
|
1451
|
+
function binHeader2(byteLength) {
|
|
1452
|
+
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
1453
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1454
|
+
const header = new Uint8Array(5);
|
|
1455
|
+
header[0] = BIN322;
|
|
1456
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1457
|
+
return header;
|
|
1245
1458
|
}
|
|
1246
1459
|
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
1247
1460
|
const head = encode(withEmptyBytesInTheirPlace);
|
|
1248
|
-
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] ===
|
|
1461
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
1249
1462
|
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
1250
|
-
return [head.subarray(0, head.length - EMPTY_BIN_BYTES),
|
|
1463
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
1251
1464
|
}
|
|
1252
1465
|
function vectorPartChunks(part) {
|
|
1253
1466
|
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
@@ -1263,20 +1476,27 @@ function keptFileRef(input, prior, kept) {
|
|
|
1263
1476
|
{ key: kept.key }
|
|
1264
1477
|
);
|
|
1265
1478
|
}
|
|
1266
|
-
return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
|
|
1479
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
1480
|
+
}
|
|
1481
|
+
function headFieldsOf(ref) {
|
|
1482
|
+
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
1483
|
+
}
|
|
1484
|
+
function headRefOf(paged) {
|
|
1485
|
+
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
1267
1486
|
}
|
|
1268
1487
|
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
1269
1488
|
const parts = plan.graphParts ?? [];
|
|
1270
1489
|
const refs = [];
|
|
1271
1490
|
for (let part = 0; part < parts.length; part++) {
|
|
1272
1491
|
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
1492
|
+
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
1273
1493
|
const packed = await packChecksummedEnvelope(
|
|
1274
|
-
[encode(parts[part])],
|
|
1494
|
+
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
1275
1495
|
GRAPH_PART_ENVELOPE_VERSION,
|
|
1276
1496
|
`graph part "${key}"`
|
|
1277
1497
|
);
|
|
1278
1498
|
await input.store.write(key, packed.chunks);
|
|
1279
|
-
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
|
|
1499
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
1280
1500
|
}
|
|
1281
1501
|
return refs;
|
|
1282
1502
|
}
|
|
@@ -1290,17 +1510,28 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
1290
1510
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
1291
1511
|
const written = [];
|
|
1292
1512
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
1293
|
-
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
1513
|
+
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
1294
1514
|
const id = nextFileId;
|
|
1295
1515
|
nextFileId += 1;
|
|
1296
1516
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
1297
|
-
const
|
|
1298
|
-
|
|
1517
|
+
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
1518
|
+
const what = `vector file "${key}"`;
|
|
1519
|
+
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
1520
|
+
paged.chunks,
|
|
1299
1521
|
VECTOR_FILE_ENVELOPE_VERSION,
|
|
1300
|
-
|
|
1522
|
+
what,
|
|
1523
|
+
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
1301
1524
|
);
|
|
1302
1525
|
await input.store.write(key, packed.chunks);
|
|
1303
|
-
files.push({
|
|
1526
|
+
files.push({
|
|
1527
|
+
id,
|
|
1528
|
+
key,
|
|
1529
|
+
count: payload.docIds.length,
|
|
1530
|
+
dead,
|
|
1531
|
+
bytes: packed.byteLength,
|
|
1532
|
+
checksum: packed.checksum,
|
|
1533
|
+
...headRefOf(paged)
|
|
1534
|
+
});
|
|
1304
1535
|
written.push({ id, key, ordinals });
|
|
1305
1536
|
outcome.layouts.push({
|
|
1306
1537
|
fieldPath,
|
|
@@ -1503,10 +1734,10 @@ var RECORD_LENGTH_SIZE = 4;
|
|
|
1503
1734
|
var FRAME_CRC_SIZE = 4;
|
|
1504
1735
|
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
1505
1736
|
function writeSegmentHeader() {
|
|
1506
|
-
const
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
return
|
|
1737
|
+
const header = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
1738
|
+
header.set(WAL_MAGIC, 0);
|
|
1739
|
+
header[4] = WAL_FORMAT_VERSION & 255;
|
|
1740
|
+
return header;
|
|
1510
1741
|
}
|
|
1511
1742
|
function frameRecord(entry) {
|
|
1512
1743
|
const payload = encode(entry);
|
|
@@ -1547,9 +1778,9 @@ function corrupt(message, details) {
|
|
|
1547
1778
|
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
1548
1779
|
}
|
|
1549
1780
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
1550
|
-
const
|
|
1551
|
-
if (!
|
|
1552
|
-
corrupt(`WAL segment header invalid in durable region: ${
|
|
1781
|
+
const header = checkSegmentHeader(data);
|
|
1782
|
+
if (!header.ok) {
|
|
1783
|
+
corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
|
|
1553
1784
|
}
|
|
1554
1785
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
1555
1786
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -1648,6 +1879,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
|
|
|
1648
1879
|
function walPrefix(indexName, partitionId) {
|
|
1649
1880
|
return `${indexName}/wal/${partitionId}/`;
|
|
1650
1881
|
}
|
|
1882
|
+
function nextSegmentKey(indexName, partitionId) {
|
|
1883
|
+
return `${walPrefix(indexName, partitionId)}next`;
|
|
1884
|
+
}
|
|
1651
1885
|
function segmentStartSeqNo(key, prefix) {
|
|
1652
1886
|
if (!key.startsWith(prefix)) {
|
|
1653
1887
|
return null;
|
|
@@ -1692,14 +1926,14 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
1692
1926
|
continue;
|
|
1693
1927
|
}
|
|
1694
1928
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
1695
|
-
const
|
|
1696
|
-
if (!
|
|
1929
|
+
const header = checkSegmentHeader(bytes);
|
|
1930
|
+
if (!header.ok) {
|
|
1697
1931
|
throw new NarsilError(
|
|
1698
1932
|
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
1699
|
-
`Sealed WAL segment header invalid: ${
|
|
1933
|
+
`Sealed WAL segment header invalid: ${header.reason}`,
|
|
1700
1934
|
{
|
|
1701
1935
|
key,
|
|
1702
|
-
reason:
|
|
1936
|
+
reason: header.reason
|
|
1703
1937
|
}
|
|
1704
1938
|
);
|
|
1705
1939
|
}
|
|
@@ -1829,569 +2063,23 @@ async function buildSegmentFromEntries(input) {
|
|
|
1829
2063
|
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
|
|
1830
2064
|
return built;
|
|
1831
2065
|
}
|
|
1832
|
-
var FIXMAP = 128;
|
|
1833
|
-
var MAP16 = 222;
|
|
1834
|
-
var MAP32 = 223;
|
|
1835
|
-
var FIXARRAY = 144;
|
|
1836
|
-
var ARRAY16 = 220;
|
|
1837
|
-
var ARRAY32 = 221;
|
|
1838
|
-
var BIN82 = 196;
|
|
1839
|
-
var BIN162 = 197;
|
|
1840
|
-
var BIN322 = 198;
|
|
1841
|
-
var FIX_COUNT_LIMIT = 16;
|
|
1842
|
-
var UINT8_LIMIT = 256;
|
|
1843
|
-
var UINT16_LIMIT = 65536;
|
|
1844
|
-
function header(fixed, sixteen, thirtyTwo, count) {
|
|
1845
|
-
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
1846
|
-
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
1847
|
-
const bytes = new Uint8Array(5);
|
|
1848
|
-
bytes[0] = thirtyTwo;
|
|
1849
|
-
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
1850
|
-
return bytes;
|
|
1851
|
-
}
|
|
1852
|
-
function binHeader2(byteLength) {
|
|
1853
|
-
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN82, byteLength);
|
|
1854
|
-
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1855
|
-
const bytes = new Uint8Array(5);
|
|
1856
|
-
bytes[0] = BIN322;
|
|
1857
|
-
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
1858
|
-
return bytes;
|
|
1859
|
-
}
|
|
1860
|
-
var MessagePackChunks = class {
|
|
1861
|
-
chunks = [];
|
|
1862
|
-
byteLength = 0;
|
|
1863
|
-
push(chunk) {
|
|
1864
|
-
if (chunk.byteLength === 0) return;
|
|
1865
|
-
this.chunks.push(chunk);
|
|
1866
|
-
this.byteLength += chunk.byteLength;
|
|
1867
|
-
}
|
|
1868
|
-
map(entries) {
|
|
1869
|
-
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
1870
|
-
return this;
|
|
1871
|
-
}
|
|
1872
|
-
array(items) {
|
|
1873
|
-
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1874
|
-
return this;
|
|
1875
|
-
}
|
|
1876
|
-
value(value) {
|
|
1877
|
-
this.push(encode(value));
|
|
1878
|
-
return this;
|
|
1879
|
-
}
|
|
1880
|
-
key(name) {
|
|
1881
|
-
return this.value(name);
|
|
1882
|
-
}
|
|
1883
|
-
binary(bytes) {
|
|
1884
|
-
this.push(binHeader2(bytes.byteLength));
|
|
1885
|
-
this.push(bytes);
|
|
1886
|
-
return this;
|
|
1887
|
-
}
|
|
1888
|
-
binaryOrNil(bytes) {
|
|
1889
|
-
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1890
|
-
}
|
|
1891
|
-
};
|
|
1892
|
-
|
|
1893
|
-
// src/serialization/keyword-segment/token-order.ts
|
|
1894
|
-
function slotsAreInTokenOrder(payloadSlots) {
|
|
1895
|
-
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1896
|
-
if (payloadSlots[at] !== at) return false;
|
|
1897
|
-
}
|
|
1898
|
-
return true;
|
|
1899
|
-
}
|
|
1900
|
-
function reorderedPositions(arrays, postingOffsets) {
|
|
1901
|
-
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1902
|
-
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1903
|
-
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1904
|
-
const offsets = new Uint32Array(postings + 1);
|
|
1905
|
-
const values = new Uint32Array(positionValues.length);
|
|
1906
|
-
let cursor = 0;
|
|
1907
|
-
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1908
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1909
|
-
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1910
|
-
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1911
|
-
offsets[to] = cursor;
|
|
1912
|
-
const start = positionOffsets[from];
|
|
1913
|
-
const end = positionOffsets[from + 1];
|
|
1914
|
-
values.set(positionValues.subarray(start, end), cursor);
|
|
1915
|
-
cursor += end - start;
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
|
-
offsets[postings] = cursor;
|
|
1919
|
-
return { positionOffsets: offsets, positionValues: values };
|
|
1920
|
-
}
|
|
1921
|
-
function postingsInTokenOrder(arrays) {
|
|
1922
|
-
const { tokenTable } = arrays;
|
|
1923
|
-
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1924
|
-
const tokens = tokenTable.payloadSlots.length;
|
|
1925
|
-
const postingOffsets = new Uint32Array(tokens + 1);
|
|
1926
|
-
for (let token = 0; token < tokens; token++) {
|
|
1927
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1928
|
-
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
1929
|
-
}
|
|
1930
|
-
const postings = postingOffsets[tokens];
|
|
1931
|
-
const postingDocIds = new Uint32Array(postings);
|
|
1932
|
-
const postingFrequencies = new Uint16Array(postings);
|
|
1933
|
-
const postingFieldIndices = new Uint16Array(postings);
|
|
1934
|
-
for (let token = 0; token < tokens; token++) {
|
|
1935
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1936
|
-
const start = arrays.postingOffsets[slot];
|
|
1937
|
-
const end = arrays.postingOffsets[slot + 1];
|
|
1938
|
-
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
1939
|
-
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
1940
|
-
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
1941
|
-
}
|
|
1942
|
-
return {
|
|
1943
|
-
postingOffsets,
|
|
1944
|
-
postingDocIds,
|
|
1945
|
-
postingFrequencies,
|
|
1946
|
-
postingFieldIndices,
|
|
1947
|
-
...reorderedPositions(arrays, postingOffsets)
|
|
1948
|
-
};
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
// src/serialization/keyword-segment/encode.ts
|
|
1952
|
-
var TOP_LEVEL_ENTRIES = 28;
|
|
1953
|
-
function writeFieldLengths(out, arrays) {
|
|
1954
|
-
out.array(arrays.fieldLengthNames.length);
|
|
1955
|
-
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
1956
|
-
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
1957
|
-
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
1958
|
-
}
|
|
1959
|
-
}
|
|
1960
|
-
function writeValueColumns(out, columns) {
|
|
1961
|
-
out.array(columns.length);
|
|
1962
|
-
for (const column of columns) {
|
|
1963
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1964
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1965
|
-
out.key("values").binary(littleEndianBytes(column.values));
|
|
1966
|
-
}
|
|
1967
|
-
}
|
|
1968
|
-
function writeBooleanColumns(out, columns) {
|
|
1969
|
-
out.array(columns.length);
|
|
1970
|
-
for (const column of columns) {
|
|
1971
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1972
|
-
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
1973
|
-
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
1974
|
-
}
|
|
1975
|
-
}
|
|
1976
|
-
function writeEnumColumns(out, columns) {
|
|
1977
|
-
out.array(columns.length);
|
|
1978
|
-
for (const column of columns) {
|
|
1979
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1980
|
-
out.key("values").value(column.values);
|
|
1981
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1982
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1983
|
-
}
|
|
1984
|
-
}
|
|
1985
|
-
function writeGeopointColumns(out, columns) {
|
|
1986
|
-
out.array(columns.length);
|
|
1987
|
-
for (const column of columns) {
|
|
1988
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1989
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1990
|
-
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
1991
|
-
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
1992
|
-
}
|
|
1993
|
-
}
|
|
1994
|
-
function writePatternColumns(out, columns) {
|
|
1995
|
-
out.array(columns.length);
|
|
1996
|
-
for (const column of columns) {
|
|
1997
|
-
out.map(5).key("field").value(column.fieldPath);
|
|
1998
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1999
|
-
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
2000
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
2001
|
-
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
2002
|
-
}
|
|
2003
|
-
}
|
|
2004
|
-
function writeSurfaceForms(out, arrays) {
|
|
2005
|
-
const table = arrays.surfaceTable;
|
|
2006
|
-
if (table === null || table.counts.length === 0) {
|
|
2007
|
-
out.value(null);
|
|
2008
|
-
return;
|
|
2009
|
-
}
|
|
2010
|
-
out.map(3).key("text").binary(table.blob);
|
|
2011
|
-
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
2012
|
-
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
2013
|
-
}
|
|
2014
|
-
function writePartialWords(out, arrays) {
|
|
2015
|
-
const table = arrays.partialWords;
|
|
2016
|
-
if (table === null) {
|
|
2017
|
-
out.value(null);
|
|
2018
|
-
return;
|
|
2019
|
-
}
|
|
2020
|
-
out.map(8).key("fields").value(table.fields);
|
|
2021
|
-
out.key("text").binary(table.spellings);
|
|
2022
|
-
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
2023
|
-
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
2024
|
-
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
2025
|
-
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
2026
|
-
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
2027
|
-
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
2028
|
-
}
|
|
2029
|
-
function encodeKeywordSegment(contents) {
|
|
2030
|
-
const { arrays } = contents;
|
|
2031
|
-
const postings = postingsInTokenOrder(arrays);
|
|
2032
|
-
const out = new MessagePackChunks();
|
|
2033
|
-
const { writeRecords } = contents;
|
|
2034
|
-
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
2035
|
-
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
2036
|
-
out.key("schema").value(contents.schema);
|
|
2037
|
-
out.key("tombstones").value(contents.tombstones);
|
|
2038
|
-
out.key("documentCount").value(arrays.documentCount);
|
|
2039
|
-
out.key("ids").binary(arrays.idTable.blob);
|
|
2040
|
-
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
2041
|
-
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
2042
|
-
out.key("documents").binary(arrays.documentTable.blob);
|
|
2043
|
-
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
2044
|
-
out.key("fieldNames").value(arrays.fieldNames);
|
|
2045
|
-
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
2046
|
-
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
2047
|
-
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
2048
|
-
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
2049
|
-
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
2050
|
-
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
2051
|
-
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
2052
|
-
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
2053
|
-
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
2054
|
-
out.key("fieldLengths");
|
|
2055
|
-
writeFieldLengths(out, arrays);
|
|
2056
|
-
out.key("numeric");
|
|
2057
|
-
writeValueColumns(out, arrays.numeric);
|
|
2058
|
-
out.key("sizes");
|
|
2059
|
-
writeValueColumns(out, arrays.sizes);
|
|
2060
|
-
out.key("boolean");
|
|
2061
|
-
writeBooleanColumns(out, arrays.boolean);
|
|
2062
|
-
out.key("enum");
|
|
2063
|
-
writeEnumColumns(out, arrays.enums);
|
|
2064
|
-
out.key("geopoint");
|
|
2065
|
-
writeGeopointColumns(out, arrays.geo);
|
|
2066
|
-
out.key("pattern");
|
|
2067
|
-
writePatternColumns(out, arrays.pattern);
|
|
2068
|
-
out.key("surfaceForms");
|
|
2069
|
-
writeSurfaceForms(out, arrays);
|
|
2070
|
-
out.key("partialWords");
|
|
2071
|
-
writePartialWords(out, arrays);
|
|
2072
|
-
if (writeRecords !== void 0) {
|
|
2073
|
-
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
2074
|
-
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
2075
|
-
}
|
|
2076
|
-
return out;
|
|
2077
|
-
}
|
|
2078
|
-
|
|
2079
|
-
// src/persistence/durability/segment/segment-file.ts
|
|
2080
|
-
function sharesMemoryWithWorkers() {
|
|
2081
|
-
return typeof SharedArrayBuffer === "function";
|
|
2082
|
-
}
|
|
2083
|
-
async function readSegmentFile(store, ref) {
|
|
2084
|
-
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
2085
|
-
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
2086
|
-
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
2087
|
-
what: `segment "${ref.key}"`,
|
|
2088
|
-
allowsCompression: true
|
|
2089
|
-
});
|
|
2090
|
-
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
2091
|
-
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
2092
|
-
}
|
|
2093
|
-
var ORDINAL_BYTES = 4;
|
|
2094
|
-
var POSTING_BYTES = 8;
|
|
2095
|
-
var VALUE_ENTRY_BYTES = 12;
|
|
2096
|
-
var GEOPOINT_ENTRY_BYTES = 20;
|
|
2097
|
-
var TOKEN_ENTRY_BYTES = 12;
|
|
2098
|
-
var STRING_HEADER_BYTES = 5;
|
|
2099
|
-
var encoder = new TextEncoder();
|
|
2100
|
-
function documentBytes(segment, ordinal) {
|
|
2101
|
-
const table = segment.arrays.documentTable;
|
|
2102
|
-
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
2103
|
-
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
2104
|
-
}
|
|
2105
|
-
function addPerEntry(costs, docIds, bytes) {
|
|
2106
|
-
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
2107
|
-
}
|
|
2108
|
-
function chargePostings(segment, costs, chargesTokens) {
|
|
2109
|
-
const arrays = segment.arrays;
|
|
2110
|
-
const positions = arrays.positionOffsets;
|
|
2111
|
-
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
2112
|
-
let bytes = POSTING_BYTES;
|
|
2113
|
-
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
2114
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
2115
|
-
}
|
|
2116
|
-
if (!chargesTokens) return;
|
|
2117
|
-
const table = arrays.tokenTable;
|
|
2118
|
-
for (let at = 0; at < table.size; at++) {
|
|
2119
|
-
const slot = table.payloadSlot(at);
|
|
2120
|
-
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
2121
|
-
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
2122
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
2123
|
-
}
|
|
2124
|
-
}
|
|
2125
|
-
}
|
|
2126
|
-
function chargeColumns(segment, costs) {
|
|
2127
|
-
const arrays = segment.arrays;
|
|
2128
|
-
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
2129
|
-
for (const column of arrays.boolean) {
|
|
2130
|
-
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
2131
|
-
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
2132
|
-
}
|
|
2133
|
-
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
2134
|
-
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
2135
|
-
for (const column of arrays.pattern) {
|
|
2136
|
-
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
2137
|
-
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
2138
|
-
}
|
|
2139
|
-
}
|
|
2140
|
-
function stringListBytes(values) {
|
|
2141
|
-
let bytes = STRING_HEADER_BYTES;
|
|
2142
|
-
for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
|
|
2143
|
-
return bytes;
|
|
2144
|
-
}
|
|
2145
|
-
function fixedBytesOf(segment, chargesTokens) {
|
|
2146
|
-
const arrays = segment.arrays;
|
|
2147
|
-
let bytes = stringListBytes(arrays.fieldNames);
|
|
2148
|
-
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
2149
|
-
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
2150
|
-
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
2151
|
-
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
2152
|
-
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
2153
|
-
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
2154
|
-
return bytes + segment.partialWords.bytes();
|
|
2155
|
-
}
|
|
2156
|
-
function keptDocumentsOf(selection, chargesTokens) {
|
|
2157
|
-
const { segment } = selection;
|
|
2158
|
-
const count = segment.arrays.documentCount;
|
|
2159
|
-
const costs = new Float64Array(count);
|
|
2160
|
-
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
2161
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
2162
|
-
costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
2163
|
-
}
|
|
2164
|
-
chargePostings(segment, costs, chargesTokens);
|
|
2165
|
-
chargeColumns(segment, costs);
|
|
2166
|
-
const ordinals = [];
|
|
2167
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
2168
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2169
|
-
costs[ordinal] += documentBytes(segment, ordinal);
|
|
2170
|
-
ordinals.push(ordinal);
|
|
2171
|
-
}
|
|
2172
|
-
return { segment, ordinals, costs };
|
|
2173
|
-
}
|
|
2174
|
-
function selectionOf(segment, ordinals) {
|
|
2175
|
-
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
2176
|
-
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
2177
|
-
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
2178
|
-
}
|
|
2179
|
-
function tooLargeForOneFile(cost, budget) {
|
|
2180
|
-
throw new NarsilError(
|
|
2181
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2182
|
-
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
2183
|
-
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
2184
|
-
);
|
|
2185
|
-
}
|
|
2186
|
-
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
2187
|
-
let tokenBytes = 0;
|
|
2188
|
-
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
2189
|
-
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
2190
|
-
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
2191
|
-
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
2192
|
-
const budget = MAX_FILE_BYTES - fixed;
|
|
2193
|
-
const groups = [];
|
|
2194
|
-
let current = [];
|
|
2195
|
-
let currentBytes = 0;
|
|
2196
|
-
let currentCount = 0;
|
|
2197
|
-
const flush = () => {
|
|
2198
|
-
if (currentCount === 0) return;
|
|
2199
|
-
groups.push({
|
|
2200
|
-
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
2201
|
-
documentCount: currentCount
|
|
2202
|
-
});
|
|
2203
|
-
current = [];
|
|
2204
|
-
currentBytes = 0;
|
|
2205
|
-
currentCount = 0;
|
|
2206
|
-
};
|
|
2207
|
-
for (const selection of selections) {
|
|
2208
|
-
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
2209
|
-
let entry = null;
|
|
2210
|
-
for (const ordinal of kept.ordinals) {
|
|
2211
|
-
const cost = kept.costs[ordinal];
|
|
2212
|
-
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
2213
|
-
if (currentBytes + cost > budget) {
|
|
2214
|
-
flush();
|
|
2215
|
-
entry = null;
|
|
2216
|
-
}
|
|
2217
|
-
if (entry === null) {
|
|
2218
|
-
entry = { segment: kept.segment, ordinals: [] };
|
|
2219
|
-
current.push(entry);
|
|
2220
|
-
}
|
|
2221
|
-
entry.ordinals.push(ordinal);
|
|
2222
|
-
currentBytes += cost;
|
|
2223
|
-
currentCount += 1;
|
|
2224
|
-
}
|
|
2225
|
-
}
|
|
2226
|
-
flush();
|
|
2227
|
-
return groups;
|
|
2228
|
-
}
|
|
2229
|
-
function halvesOf(group) {
|
|
2230
|
-
const flat = [];
|
|
2231
|
-
for (const selection of group.selections) {
|
|
2232
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2233
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2234
|
-
flat.push({ segment: selection.segment, ordinal });
|
|
2235
|
-
}
|
|
2236
|
-
}
|
|
2237
|
-
const middle = Math.ceil(flat.length / 2);
|
|
2238
|
-
const groupOf = (entries) => {
|
|
2239
|
-
const bySegment = /* @__PURE__ */ new Map();
|
|
2240
|
-
for (const { segment, ordinal } of entries) {
|
|
2241
|
-
let ordinals = bySegment.get(segment);
|
|
2242
|
-
if (ordinals === void 0) {
|
|
2243
|
-
ordinals = [];
|
|
2244
|
-
bySegment.set(segment, ordinals);
|
|
2245
|
-
}
|
|
2246
|
-
ordinals.push(ordinal);
|
|
2247
|
-
}
|
|
2248
|
-
return {
|
|
2249
|
-
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
2250
|
-
documentCount: entries.length
|
|
2251
|
-
};
|
|
2252
|
-
};
|
|
2253
|
-
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
2254
|
-
}
|
|
2255
|
-
|
|
2256
|
-
// src/persistence/durability/segment/segment-write.ts
|
|
2257
|
-
function emptyPayload() {
|
|
2258
|
-
return {
|
|
2259
|
-
documentCount: 0,
|
|
2260
|
-
docIds: [],
|
|
2261
|
-
fieldNames: [],
|
|
2262
|
-
tokens: [],
|
|
2263
|
-
postingOffsets: new Uint32Array(1),
|
|
2264
|
-
postingDocIds: new Uint32Array(0),
|
|
2265
|
-
postingFrequencies: new Uint16Array(0),
|
|
2266
|
-
postingFieldIndices: new Uint16Array(0),
|
|
2267
|
-
positionOffsets: null,
|
|
2268
|
-
positionValues: null,
|
|
2269
|
-
fieldLengthNames: [],
|
|
2270
|
-
fieldLengthColumns: [],
|
|
2271
|
-
totalFieldLengths: {},
|
|
2272
|
-
docFrequencies: {},
|
|
2273
|
-
surfaceForms: null,
|
|
2274
|
-
partialSpellings: null,
|
|
2275
|
-
numeric: [],
|
|
2276
|
-
boolean: [],
|
|
2277
|
-
enums: [],
|
|
2278
|
-
geo: [],
|
|
2279
|
-
pattern: [],
|
|
2280
|
-
sizes: []
|
|
2281
|
-
};
|
|
2282
|
-
}
|
|
2283
|
-
function noSpellingsOf(analysis) {
|
|
2284
|
-
if (analysis === null || analysis.fields.length === 0) return null;
|
|
2285
|
-
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
2286
|
-
}
|
|
2287
|
-
function writeRecordsOfGroup(group, records, count) {
|
|
2288
|
-
const seqNos = new Float64Array(count);
|
|
2289
|
-
const primaryTerms = new Float64Array(count);
|
|
2290
|
-
let at = 0;
|
|
2291
|
-
for (const selection of group?.selections ?? []) {
|
|
2292
|
-
const source = records.get(selection.segment);
|
|
2293
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2294
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
2295
|
-
if (source !== void 0) {
|
|
2296
|
-
seqNos[at] = source.seqNos[ordinal];
|
|
2297
|
-
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
2298
|
-
}
|
|
2299
|
-
at += 1;
|
|
2300
|
-
}
|
|
2301
|
-
}
|
|
2302
|
-
return { seqNos, primaryTerms };
|
|
2303
|
-
}
|
|
2304
|
-
function withWriteRecords(contents, group, request) {
|
|
2305
|
-
if (request.writeRecords === void 0) return contents;
|
|
2306
|
-
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
2307
|
-
return { ...contents, writeRecords };
|
|
2308
|
-
}
|
|
2309
|
-
function contentsOfGroup(group, request, tombstones) {
|
|
2310
|
-
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
2311
|
-
if (group === null) {
|
|
2312
|
-
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
2313
|
-
return {
|
|
2314
|
-
schema,
|
|
2315
|
-
tombstones,
|
|
2316
|
-
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
2317
|
-
};
|
|
2318
|
-
}
|
|
2319
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
2320
|
-
return {
|
|
2321
|
-
schema,
|
|
2322
|
-
tombstones,
|
|
2323
|
-
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
2324
|
-
};
|
|
2325
|
-
}
|
|
2326
|
-
function requireRoomForAnotherSegment(state) {
|
|
2327
|
-
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
2328
|
-
throw new NarsilError(
|
|
2329
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2330
|
-
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
2331
|
-
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
2332
|
-
);
|
|
2333
|
-
}
|
|
2334
|
-
async function writeGroup(state, group) {
|
|
2335
|
-
const { request } = state;
|
|
2336
|
-
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
2337
|
-
const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
2338
|
-
const encoded = encodeKeywordSegment(contents);
|
|
2339
|
-
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
2340
|
-
const [first, second] = halvesOf(group);
|
|
2341
|
-
await writeGroup(state, first);
|
|
2342
|
-
await writeGroup(state, second);
|
|
2343
|
-
return;
|
|
2344
|
-
}
|
|
2345
|
-
requireRoomForAnotherSegment(state);
|
|
2346
|
-
const id = state.nextSegmentId;
|
|
2347
|
-
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
2348
|
-
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
2349
|
-
await request.store.write(key, packed.chunks);
|
|
2350
|
-
state.segments.push({
|
|
2351
|
-
id,
|
|
2352
|
-
key,
|
|
2353
|
-
bytes: packed.byteLength,
|
|
2354
|
-
checksum: packed.checksum,
|
|
2355
|
-
docCount: contents.arrays.documentCount,
|
|
2356
|
-
tombstoneCount: tombstones.length
|
|
2357
|
-
});
|
|
2358
|
-
state.nextSegmentId = id + 1;
|
|
2359
|
-
state.tombstonesWritten = true;
|
|
2360
|
-
}
|
|
2361
|
-
function keptDocumentCount(selections) {
|
|
2362
|
-
let count = 0;
|
|
2363
|
-
for (const selection of selections) {
|
|
2364
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
2365
|
-
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
2366
|
-
}
|
|
2367
|
-
}
|
|
2368
|
-
return count;
|
|
2369
|
-
}
|
|
2370
|
-
async function writeSegmentsOf(request) {
|
|
2371
|
-
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
2372
|
-
const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
2373
|
-
for (const group of groups) {
|
|
2374
|
-
if (group.documentCount > 0) await writeGroup(state, group);
|
|
2375
|
-
}
|
|
2376
|
-
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
2377
|
-
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
2378
|
-
}
|
|
2379
2066
|
|
|
2380
2067
|
// src/persistence/durability/segment/compaction.ts
|
|
2381
|
-
function windowFits(segments, start, windowLength) {
|
|
2068
|
+
function windowFits(segments, start, windowLength, paged) {
|
|
2382
2069
|
let documents = 0;
|
|
2383
2070
|
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
2384
2071
|
for (let at = start; at < start + windowLength; at++) {
|
|
2385
2072
|
documents += segments[at].docCount;
|
|
2386
2073
|
bytes += segments[at].bytes;
|
|
2387
2074
|
}
|
|
2075
|
+
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
2388
2076
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
2389
2077
|
}
|
|
2390
|
-
function smallestFittingWindow(segments, windowLength) {
|
|
2078
|
+
function smallestFittingWindow(segments, windowLength, paged) {
|
|
2391
2079
|
let best = null;
|
|
2392
2080
|
let bestBytes = Number.POSITIVE_INFINITY;
|
|
2393
2081
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
2394
|
-
if (!windowFits(segments, start, windowLength)) continue;
|
|
2082
|
+
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
2395
2083
|
let bytes = 0;
|
|
2396
2084
|
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
2397
2085
|
if (bytes < bestBytes) {
|
|
@@ -2401,9 +2089,9 @@ function smallestFittingWindow(segments, windowLength) {
|
|
|
2401
2089
|
}
|
|
2402
2090
|
return best;
|
|
2403
2091
|
}
|
|
2404
|
-
function chooseWindow(segments, wantedLength) {
|
|
2092
|
+
function chooseWindow(segments, wantedLength, paged) {
|
|
2405
2093
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
2406
|
-
const window = smallestFittingWindow(segments, windowLength);
|
|
2094
|
+
const window = smallestFittingWindow(segments, windowLength, paged);
|
|
2407
2095
|
if (window !== null) return window;
|
|
2408
2096
|
}
|
|
2409
2097
|
return null;
|
|
@@ -2444,11 +2132,39 @@ async function compactPartitionSegments(input) {
|
|
|
2444
2132
|
if (segments.length <= compactionThreshold) {
|
|
2445
2133
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
2446
2134
|
}
|
|
2447
|
-
const
|
|
2135
|
+
const paged = input.paged === true;
|
|
2136
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
2448
2137
|
if (window === null) {
|
|
2449
2138
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
2450
2139
|
}
|
|
2451
2140
|
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
2141
|
+
const replacing = (written2) => ({
|
|
2142
|
+
segments: [
|
|
2143
|
+
...segments.slice(0, window.start),
|
|
2144
|
+
...written2.segments,
|
|
2145
|
+
...segments.slice(window.start + window.windowLength)
|
|
2146
|
+
],
|
|
2147
|
+
nextSegmentId: written2.nextSegmentId
|
|
2148
|
+
});
|
|
2149
|
+
if (paged) {
|
|
2150
|
+
const { directory } = input.store;
|
|
2151
|
+
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
2152
|
+
const { mergePagedWindow } = await import('./module-BUFNZMEU.mjs');
|
|
2153
|
+
return replacing(
|
|
2154
|
+
await mergePagedWindow({
|
|
2155
|
+
store: input.store,
|
|
2156
|
+
directory,
|
|
2157
|
+
indexName: input.indexName,
|
|
2158
|
+
partitionId: input.partitionId,
|
|
2159
|
+
schema: input.schema,
|
|
2160
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
2161
|
+
textAnalysis: input.textAnalysis,
|
|
2162
|
+
nextSegmentId: input.nextSegmentId,
|
|
2163
|
+
refs: windowRefs,
|
|
2164
|
+
dropsEveryTombstone: window.start === 0
|
|
2165
|
+
})
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2452
2168
|
const contents = [];
|
|
2453
2169
|
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
2454
2170
|
const frozen = contents.map(frozenOf);
|
|
@@ -2467,14 +2183,7 @@ async function compactPartitionSegments(input) {
|
|
|
2467
2183
|
mergesIntoOneFile: true,
|
|
2468
2184
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
2469
2185
|
});
|
|
2470
|
-
return
|
|
2471
|
-
segments: [
|
|
2472
|
-
...segments.slice(0, window.start),
|
|
2473
|
-
...written.segments,
|
|
2474
|
-
...segments.slice(window.start + window.windowLength)
|
|
2475
|
-
],
|
|
2476
|
-
nextSegmentId: written.nextSegmentId
|
|
2477
|
-
};
|
|
2186
|
+
return replacing(written);
|
|
2478
2187
|
}
|
|
2479
2188
|
|
|
2480
2189
|
// src/persistence/durability/segment/write.ts
|
|
@@ -2521,6 +2230,21 @@ function wholeContentsOf(input, partitionId) {
|
|
|
2521
2230
|
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
2522
2231
|
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
2523
2232
|
}
|
|
2233
|
+
function pagedSettingsOf(context) {
|
|
2234
|
+
if (context.input.metadata.textStorage !== "disk") return {};
|
|
2235
|
+
return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
|
|
2236
|
+
}
|
|
2237
|
+
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
2238
|
+
const { directory } = context.input.store;
|
|
2239
|
+
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
2240
|
+
const { pagedTombstonesFor } = await import('./module-BUFNZMEU.mjs');
|
|
2241
|
+
return pagedTombstonesFor(
|
|
2242
|
+
directory,
|
|
2243
|
+
older,
|
|
2244
|
+
selections.map((selection) => selection.segment),
|
|
2245
|
+
deleted
|
|
2246
|
+
);
|
|
2247
|
+
}
|
|
2524
2248
|
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
2525
2249
|
const written = await writeSegmentsOf({
|
|
2526
2250
|
store: context.input.store,
|
|
@@ -2532,7 +2256,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
2532
2256
|
textAnalysis: context.textAnalysis,
|
|
2533
2257
|
selections: whole.selections,
|
|
2534
2258
|
tombstones: [],
|
|
2535
|
-
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
|
|
2259
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
|
|
2260
|
+
...pagedSettingsOf(context)
|
|
2536
2261
|
});
|
|
2537
2262
|
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
2538
2263
|
}
|
|
@@ -2548,6 +2273,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2548
2273
|
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
2549
2274
|
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
2550
2275
|
if (built !== null) {
|
|
2276
|
+
const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
|
|
2551
2277
|
const written = await writeSegmentsOf({
|
|
2552
2278
|
store: context.input.store,
|
|
2553
2279
|
indexName: context.indexName,
|
|
@@ -2557,8 +2283,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2557
2283
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2558
2284
|
textAnalysis: context.textAnalysis,
|
|
2559
2285
|
selections: built.selections,
|
|
2560
|
-
tombstones
|
|
2561
|
-
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
|
|
2286
|
+
tombstones,
|
|
2287
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
|
|
2288
|
+
...pagedSettingsOf(context)
|
|
2562
2289
|
});
|
|
2563
2290
|
segments.push(...written.segments);
|
|
2564
2291
|
nextSegmentId = written.nextSegmentId;
|
|
@@ -2579,7 +2306,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2579
2306
|
textAnalysis: context.textAnalysis,
|
|
2580
2307
|
segments,
|
|
2581
2308
|
nextSegmentId,
|
|
2582
|
-
compactionThreshold: context.compactionThreshold
|
|
2309
|
+
compactionThreshold: context.compactionThreshold,
|
|
2310
|
+
paged: context.input.metadata.textStorage === "disk"
|
|
2583
2311
|
});
|
|
2584
2312
|
segments = compacted.segments;
|
|
2585
2313
|
nextSegmentId = compacted.nextSegmentId;
|
|
@@ -2642,8 +2370,10 @@ async function writeCheckpointSegments(input) {
|
|
|
2642
2370
|
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
2643
2371
|
const indexName = metadata.indexName;
|
|
2644
2372
|
const manifest = {
|
|
2373
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
2645
2374
|
schema: metadata.schema,
|
|
2646
2375
|
language: metadata.language,
|
|
2376
|
+
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
2647
2377
|
checkpoint: written.checkpoint,
|
|
2648
2378
|
partitions: written.partitions,
|
|
2649
2379
|
vectors
|
|
@@ -2694,8 +2424,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
2694
2424
|
function yieldToEventLoop() {
|
|
2695
2425
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
2696
2426
|
}
|
|
2427
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
2428
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
2429
|
+
composite.beginBatch();
|
|
2430
|
+
try {
|
|
2431
|
+
for (const docId of docIds) {
|
|
2432
|
+
if (!composite.has(docId)) continue;
|
|
2433
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
2434
|
+
docPartitionMap.delete(docId);
|
|
2435
|
+
}
|
|
2436
|
+
} finally {
|
|
2437
|
+
composite.endBatch();
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
2441
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-BUFNZMEU.mjs');
|
|
2442
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
2443
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
2444
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
2445
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
2446
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
2447
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
2448
|
+
if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
|
|
2449
|
+
else segment.paged?.close();
|
|
2450
|
+
}
|
|
2697
2451
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
2452
|
+
const { directory } = store;
|
|
2698
2453
|
for (const ref of partition.segments) {
|
|
2454
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
2455
|
+
const { isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
|
|
2456
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
2457
|
+
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
2458
|
+
await yieldToEventLoop();
|
|
2459
|
+
continue;
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2699
2462
|
const contents = await readSegmentFile(store, ref);
|
|
2700
2463
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
2701
2464
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -2740,6 +2503,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2740
2503
|
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
2741
2504
|
}
|
|
2742
2505
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
2506
|
+
const { directory } = store;
|
|
2507
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
2508
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
2509
|
+
);
|
|
2510
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
2511
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-BUFNZMEU.mjs');
|
|
2512
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2513
|
+
}
|
|
2743
2514
|
for (const partition of manifest.partitions) {
|
|
2744
2515
|
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
2745
2516
|
throw new NarsilError(
|
|
@@ -2757,7 +2528,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2757
2528
|
for (const vector of manifest.vectors) {
|
|
2758
2529
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
2759
2530
|
if (vectorIndex === void 0) continue;
|
|
2760
|
-
await loadVectorField(
|
|
2531
|
+
await loadVectorField(
|
|
2532
|
+
store,
|
|
2533
|
+
vector,
|
|
2534
|
+
vectorIndex,
|
|
2535
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
2536
|
+
);
|
|
2761
2537
|
}
|
|
2762
2538
|
return manifest.checkpoint;
|
|
2763
2539
|
}
|
|
@@ -2866,4 +2642,4 @@ function deserializePayloadV2(data) {
|
|
|
2866
2642
|
};
|
|
2867
2643
|
}
|
|
2868
2644
|
|
|
2869
|
-
export {
|
|
2645
|
+
export { SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, nextSegmentKey, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
|