@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
|
|
2
|
+
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-W2COJ5AH.mjs';
|
|
3
|
+
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-ICW45TTU.mjs';
|
|
4
|
+
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-7AUASRE7.mjs';
|
|
5
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-UUZCQQG2.mjs';
|
|
6
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-IO5F7ZBE.mjs';
|
|
7
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
|
|
8
|
+
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-ZPWS2SKJ.mjs';
|
|
9
|
+
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-KGOXHLLZ.mjs';
|
|
7
10
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
8
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
12
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
10
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
13
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
11
14
|
import { decode, encode } from '@msgpack/msgpack';
|
|
12
15
|
|
|
13
16
|
// src/engine/durability-metadata.ts
|
|
@@ -47,6 +50,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
47
50
|
if (config.stopWordList !== void 0) {
|
|
48
51
|
metadata.stopWordList = config.stopWordList;
|
|
49
52
|
}
|
|
53
|
+
if (config.analysisGeneration !== void 0) {
|
|
54
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
55
|
+
}
|
|
56
|
+
if (config.pendingAnalysis !== void 0) {
|
|
57
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
58
|
+
}
|
|
50
59
|
if (config.partitionLimits !== void 0) {
|
|
51
60
|
metadata.partitionLimits = config.partitionLimits;
|
|
52
61
|
}
|
|
@@ -56,6 +65,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
56
65
|
if (config.trackPositions !== void 0) {
|
|
57
66
|
metadata.trackPositions = config.trackPositions;
|
|
58
67
|
}
|
|
68
|
+
if (config.textStorage !== void 0) {
|
|
69
|
+
metadata.textStorage = config.textStorage;
|
|
70
|
+
}
|
|
59
71
|
if (config.strict !== void 0) {
|
|
60
72
|
metadata.strict = config.strict;
|
|
61
73
|
}
|
|
@@ -101,9 +113,12 @@ function metadataConfigOf(entry) {
|
|
|
101
113
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
102
114
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
103
115
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
116
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
117
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
104
118
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
105
119
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
106
120
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
121
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
107
122
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
108
123
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
109
124
|
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
@@ -111,6 +126,38 @@ function metadataConfigOf(entry) {
|
|
|
111
126
|
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
|
|
112
127
|
};
|
|
113
128
|
}
|
|
129
|
+
function durableMetadataConfigOf(entry) {
|
|
130
|
+
const terms = metadataConfigOf(entry);
|
|
131
|
+
const durable = entry.durableAnalysis;
|
|
132
|
+
if (durable === null) return terms;
|
|
133
|
+
const {
|
|
134
|
+
language,
|
|
135
|
+
tokenizer,
|
|
136
|
+
stopWords,
|
|
137
|
+
stopWordList,
|
|
138
|
+
analysisRevision: _analysisRevision,
|
|
139
|
+
arrayPositionGap: _arrayPositionGap,
|
|
140
|
+
analysisGeneration: _analysisGeneration,
|
|
141
|
+
...rest
|
|
142
|
+
} = terms;
|
|
143
|
+
return {
|
|
144
|
+
...rest,
|
|
145
|
+
language: durable.language,
|
|
146
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
147
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
148
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
149
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
150
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
151
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
152
|
+
pendingAnalysis: {
|
|
153
|
+
generation: entry.analysisGeneration,
|
|
154
|
+
language,
|
|
155
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
156
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
157
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
}
|
|
114
161
|
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
115
162
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
116
163
|
}
|
|
@@ -322,6 +369,19 @@ function metadataToWire(meta) {
|
|
|
322
369
|
if (meta.stopWordList !== void 0) {
|
|
323
370
|
wire.stop_word_list = meta.stopWordList;
|
|
324
371
|
}
|
|
372
|
+
if (meta.analysisGeneration !== void 0) {
|
|
373
|
+
wire.analysis_generation = meta.analysisGeneration;
|
|
374
|
+
}
|
|
375
|
+
if (meta.pendingAnalysis !== void 0) {
|
|
376
|
+
const pending = meta.pendingAnalysis;
|
|
377
|
+
wire.pending_analysis = {
|
|
378
|
+
generation: pending.generation,
|
|
379
|
+
language: pending.language,
|
|
380
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
381
|
+
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
382
|
+
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
383
|
+
};
|
|
384
|
+
}
|
|
325
385
|
if (meta.partitionLimits !== void 0) {
|
|
326
386
|
wire.partition_limits = {
|
|
327
387
|
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
@@ -335,6 +395,9 @@ function metadataToWire(meta) {
|
|
|
335
395
|
if (meta.trackPositions !== void 0) {
|
|
336
396
|
wire.track_positions = meta.trackPositions;
|
|
337
397
|
}
|
|
398
|
+
if (meta.textStorage !== void 0) {
|
|
399
|
+
wire.text_storage = meta.textStorage;
|
|
400
|
+
}
|
|
338
401
|
if (meta.strict !== void 0) {
|
|
339
402
|
wire.strict = meta.strict;
|
|
340
403
|
}
|
|
@@ -417,6 +480,19 @@ function wireToMetadata(raw) {
|
|
|
417
480
|
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
418
481
|
meta.stopWordList = raw.stop_word_list;
|
|
419
482
|
}
|
|
483
|
+
if (isPositiveInteger(raw.analysis_generation)) {
|
|
484
|
+
meta.analysisGeneration = raw.analysis_generation;
|
|
485
|
+
}
|
|
486
|
+
const pending = raw.pending_analysis;
|
|
487
|
+
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
488
|
+
meta.pendingAnalysis = {
|
|
489
|
+
generation: pending.generation,
|
|
490
|
+
language: pending.language,
|
|
491
|
+
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
492
|
+
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
493
|
+
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
494
|
+
};
|
|
495
|
+
}
|
|
420
496
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
421
497
|
const limits = {
|
|
422
498
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
@@ -433,6 +509,14 @@ function wireToMetadata(raw) {
|
|
|
433
509
|
if (typeof raw.track_positions === "boolean") {
|
|
434
510
|
meta.trackPositions = raw.track_positions;
|
|
435
511
|
}
|
|
512
|
+
if (raw.text_storage !== void 0) {
|
|
513
|
+
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
514
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
515
|
+
textStorage: raw.text_storage
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
meta.textStorage = raw.text_storage;
|
|
519
|
+
}
|
|
436
520
|
if (typeof raw.strict === "boolean") {
|
|
437
521
|
meta.strict = raw.strict;
|
|
438
522
|
}
|
|
@@ -509,6 +593,9 @@ function wireToMetadata(raw) {
|
|
|
509
593
|
function isNonNegativeInteger(value) {
|
|
510
594
|
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
511
595
|
}
|
|
596
|
+
function isStringList(value) {
|
|
597
|
+
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
598
|
+
}
|
|
512
599
|
function isPositiveInteger(value) {
|
|
513
600
|
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
514
601
|
}
|
|
@@ -691,7 +778,20 @@ function readFileRef(raw, expectedKey, what) {
|
|
|
691
778
|
invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
|
|
692
779
|
}
|
|
693
780
|
if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
|
|
694
|
-
return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
781
|
+
if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
782
|
+
const { headOffset, headLength, headChecksum } = raw;
|
|
783
|
+
if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
|
|
784
|
+
invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
|
|
785
|
+
}
|
|
786
|
+
if (headOffset + headLength > bytes - HEADER_SIZE) {
|
|
787
|
+
invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
|
|
788
|
+
key: expectedKey,
|
|
789
|
+
headOffset,
|
|
790
|
+
headLength,
|
|
791
|
+
bytes
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
|
|
695
795
|
}
|
|
696
796
|
|
|
697
797
|
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
@@ -788,12 +888,16 @@ function decodeManifestPayload(raw, indexName) {
|
|
|
788
888
|
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
789
889
|
requireValidFieldTypes(schema);
|
|
790
890
|
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
891
|
+
if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
|
|
892
|
+
invalidManifest("has an invalid analysis generation");
|
|
893
|
+
}
|
|
791
894
|
return {
|
|
792
895
|
kind: "current",
|
|
793
896
|
manifest: {
|
|
794
897
|
version: SEGMENT_MANIFEST_VERSION,
|
|
795
898
|
schema,
|
|
796
899
|
language: raw.language,
|
|
900
|
+
...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
|
|
797
901
|
checkpoint: normalizeCheckpoint(raw.checkpoint),
|
|
798
902
|
partitions: normalizePartitions(raw.partitions, indexName),
|
|
799
903
|
vectors: normalizeVectorFields(raw.vectors, indexName)
|
|
@@ -878,100 +982,313 @@ function readNextSegmentId(value, segments, partitionId) {
|
|
|
878
982
|
}
|
|
879
983
|
return value;
|
|
880
984
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
async function readGraphParts(store, ref) {
|
|
899
|
-
if (ref.graphParts.length === 0) return null;
|
|
900
|
-
const parts = [];
|
|
901
|
-
for (const part of ref.graphParts) {
|
|
902
|
-
const { payload } = await openListedFile(store, part, "graph part", true);
|
|
903
|
-
const decoded = decodeVectorGraphPartPayload(decode(payload));
|
|
904
|
-
if (decoded.fieldName !== ref.fieldPath) {
|
|
905
|
-
throw new NarsilError(
|
|
906
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
907
|
-
`The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
|
|
908
|
-
{ key: part.key }
|
|
909
|
-
);
|
|
910
|
-
}
|
|
911
|
-
parts.push(decoded);
|
|
912
|
-
}
|
|
913
|
-
return assembleGraphParts(parts);
|
|
985
|
+
|
|
986
|
+
// src/serialization/crc32-combine.ts
|
|
987
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
988
|
+
var CRC_BITS = 32;
|
|
989
|
+
var zeroByteOperators = [];
|
|
990
|
+
function timesVector(operator, vector) {
|
|
991
|
+
let sum = 0;
|
|
992
|
+
let rest = vector >>> 0;
|
|
993
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
994
|
+
if (rest & 1) sum ^= operator[row];
|
|
995
|
+
}
|
|
996
|
+
return sum >>> 0;
|
|
997
|
+
}
|
|
998
|
+
function squared(operator) {
|
|
999
|
+
const result = new Uint32Array(CRC_BITS);
|
|
1000
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1001
|
+
return result;
|
|
914
1002
|
}
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
throw new NarsilError(
|
|
927
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
928
|
-
`The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
|
|
929
|
-
{ key: file.key }
|
|
930
|
-
);
|
|
1003
|
+
function firstOperator() {
|
|
1004
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1005
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1006
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1007
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1008
|
+
return oneZeroBit;
|
|
1009
|
+
}
|
|
1010
|
+
function operatorForZeroBytesPower(power) {
|
|
1011
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
1012
|
+
while (zeroByteOperators.length <= power) {
|
|
1013
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
931
1014
|
}
|
|
932
|
-
return
|
|
1015
|
+
return zeroByteOperators[power];
|
|
933
1016
|
}
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
for (const file of ref.files) {
|
|
941
|
-
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
942
|
-
restore.addFile({
|
|
943
|
-
id: file.id,
|
|
944
|
-
key: file.key,
|
|
945
|
-
payload,
|
|
946
|
-
dead: file.dead,
|
|
947
|
-
location: store.pathOf === void 0 ? null : {
|
|
948
|
-
path: await store.pathOf(file.key),
|
|
949
|
-
vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
|
|
950
|
-
}
|
|
951
|
-
});
|
|
1017
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
1018
|
+
let crc = first >>> 0;
|
|
1019
|
+
let rest = secondByteLength;
|
|
1020
|
+
for (let power = 0; rest > 0; power++) {
|
|
1021
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
1022
|
+
rest = Math.floor(rest / 2);
|
|
952
1023
|
}
|
|
953
|
-
|
|
1024
|
+
return (crc ^ second) >>> 0;
|
|
954
1025
|
}
|
|
955
|
-
var EMPTY_BIN = new Uint8Array(0);
|
|
956
|
-
var EMPTY_BIN_BYTES = 2;
|
|
957
1026
|
var BIN8 = 196;
|
|
958
1027
|
var BIN16 = 197;
|
|
959
1028
|
var BIN32 = 198;
|
|
1029
|
+
var ARRAY16 = 220;
|
|
1030
|
+
var ARRAY32 = 221;
|
|
1031
|
+
var FIXMAP = 128;
|
|
1032
|
+
var FIXARRAY = 144;
|
|
1033
|
+
var FIXED_LIST_MAX = 15;
|
|
1034
|
+
var FIXSTR = 160;
|
|
1035
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
1036
|
+
var ASCII_MAX = 127;
|
|
1037
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
1038
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
960
1039
|
var BIN8_MAX_BYTES = 255;
|
|
961
|
-
var
|
|
1040
|
+
var UINT16_MAX = 65535;
|
|
962
1041
|
function binHeader(byteLength) {
|
|
963
1042
|
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
964
|
-
if (byteLength <=
|
|
1043
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
965
1044
|
const header = new Uint8Array(5);
|
|
966
1045
|
header[0] = BIN32;
|
|
967
1046
|
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
968
1047
|
return header;
|
|
969
1048
|
}
|
|
1049
|
+
function arrayHeader(count) {
|
|
1050
|
+
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
1051
|
+
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
1052
|
+
const header = new Uint8Array(5);
|
|
1053
|
+
header[0] = ARRAY32;
|
|
1054
|
+
new DataView(header.buffer).setUint32(1, count, false);
|
|
1055
|
+
return header;
|
|
1056
|
+
}
|
|
1057
|
+
function createPayloadBuilder() {
|
|
1058
|
+
const chunks = [];
|
|
1059
|
+
let offset = 0;
|
|
1060
|
+
function push(chunk) {
|
|
1061
|
+
chunks.push(chunk);
|
|
1062
|
+
offset += chunk.byteLength;
|
|
1063
|
+
}
|
|
1064
|
+
return {
|
|
1065
|
+
chunks,
|
|
1066
|
+
get offset() {
|
|
1067
|
+
return offset;
|
|
1068
|
+
},
|
|
1069
|
+
push,
|
|
1070
|
+
entry(key, value) {
|
|
1071
|
+
push(encode(key));
|
|
1072
|
+
push(encode(value));
|
|
1073
|
+
},
|
|
1074
|
+
rawEntry(key, encoded) {
|
|
1075
|
+
push(encode(key));
|
|
1076
|
+
const at = offset;
|
|
1077
|
+
push(encoded);
|
|
1078
|
+
return at;
|
|
1079
|
+
},
|
|
1080
|
+
bytesEntry(key, bytes) {
|
|
1081
|
+
push(encode(key));
|
|
1082
|
+
push(binHeader(bytes.byteLength));
|
|
1083
|
+
const at = offset;
|
|
1084
|
+
push(bytes);
|
|
1085
|
+
return at;
|
|
1086
|
+
}
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
function pageChecksums(bytes, pageBytes) {
|
|
1090
|
+
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
1091
|
+
const checksums = new Uint8Array(pages * 4);
|
|
1092
|
+
const view = new DataView(checksums.buffer);
|
|
1093
|
+
for (let page = 0; page < pages; page++) {
|
|
1094
|
+
const from = page * pageBytes;
|
|
1095
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
1096
|
+
}
|
|
1097
|
+
return checksums;
|
|
1098
|
+
}
|
|
1099
|
+
function sectionOf(name, offset, bytes, pageBytes) {
|
|
1100
|
+
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
1101
|
+
}
|
|
1102
|
+
function uint32Bytes(values) {
|
|
1103
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
1104
|
+
const view = new DataView(bytes.buffer);
|
|
1105
|
+
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
1106
|
+
return bytes;
|
|
1107
|
+
}
|
|
1108
|
+
function writtenAsShortAscii(docId, list, at) {
|
|
1109
|
+
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
1110
|
+
for (let unit = 0; unit < docId.length; unit++) {
|
|
1111
|
+
const code = docId.charCodeAt(unit);
|
|
1112
|
+
if (code > ASCII_MAX) return false;
|
|
1113
|
+
list[at + 1 + unit] = code;
|
|
1114
|
+
}
|
|
1115
|
+
list[at] = FIXSTR | docId.length;
|
|
1116
|
+
return true;
|
|
1117
|
+
}
|
|
1118
|
+
function encodedIdList(docIds) {
|
|
1119
|
+
const header = arrayHeader(docIds.length);
|
|
1120
|
+
let capacity = header.byteLength;
|
|
1121
|
+
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
1122
|
+
const list = new Uint8Array(capacity);
|
|
1123
|
+
list.set(header);
|
|
1124
|
+
const offsets = new Uint32Array(docIds.length + 1);
|
|
1125
|
+
let cursor = header.byteLength;
|
|
1126
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
1127
|
+
offsets[i] = cursor;
|
|
1128
|
+
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
1129
|
+
cursor += docIds[i].length + 1;
|
|
1130
|
+
continue;
|
|
1131
|
+
}
|
|
1132
|
+
const encoded = encode(docIds[i]);
|
|
1133
|
+
list.set(encoded, cursor);
|
|
1134
|
+
cursor += encoded.byteLength;
|
|
1135
|
+
}
|
|
1136
|
+
offsets[docIds.length] = cursor;
|
|
1137
|
+
return { list: list.subarray(0, cursor), offsets };
|
|
1138
|
+
}
|
|
1139
|
+
function positionsInIdOrder(docIds) {
|
|
1140
|
+
const positions = Array.from(docIds.keys());
|
|
1141
|
+
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
1142
|
+
return Uint32Array.from(positions);
|
|
1143
|
+
}
|
|
1144
|
+
function magnitudeBytes(magnitudes) {
|
|
1145
|
+
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
1146
|
+
const view = new DataView(bytes.buffer);
|
|
1147
|
+
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
1148
|
+
return bytes;
|
|
1149
|
+
}
|
|
1150
|
+
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
1151
|
+
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
1152
|
+
let whole = 0;
|
|
1153
|
+
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
1154
|
+
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
1155
|
+
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
1156
|
+
}
|
|
1157
|
+
return whole;
|
|
1158
|
+
}
|
|
1159
|
+
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
1160
|
+
let headLength = 0;
|
|
1161
|
+
let head = new Uint8Array(0);
|
|
1162
|
+
let vectorsSection = {
|
|
1163
|
+
name: "vectors",
|
|
1164
|
+
offset: 0,
|
|
1165
|
+
length: vectors.byteLength,
|
|
1166
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
1167
|
+
checksums
|
|
1168
|
+
};
|
|
1169
|
+
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
1170
|
+
vectorsSection = {
|
|
1171
|
+
...vectorsSection,
|
|
1172
|
+
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
1173
|
+
};
|
|
1174
|
+
head = encode({ sections: [...sections, vectorsSection] });
|
|
1175
|
+
if (head.byteLength === headLength) break;
|
|
1176
|
+
headLength = head.byteLength;
|
|
1177
|
+
}
|
|
1178
|
+
return { head, vectorsSection };
|
|
1179
|
+
}
|
|
1180
|
+
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
1181
|
+
const builder = createPayloadBuilder();
|
|
1182
|
+
const { codes } = payload;
|
|
1183
|
+
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
1184
|
+
builder.entry("v", payload.v);
|
|
1185
|
+
builder.entry("fieldName", payload.fieldName);
|
|
1186
|
+
builder.entry("dimension", payload.dimension);
|
|
1187
|
+
const { list, offsets } = encodedIdList(payload.docIds);
|
|
1188
|
+
const docIdsAt = builder.rawEntry("docIds", list);
|
|
1189
|
+
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
1190
|
+
builder.push(encode("codes"));
|
|
1191
|
+
if (codes === null) {
|
|
1192
|
+
builder.push(encode(null));
|
|
1193
|
+
} else {
|
|
1194
|
+
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
1195
|
+
builder.entry("bits", codes.bits);
|
|
1196
|
+
builder.entry("centroid", codes.centroid);
|
|
1197
|
+
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
1198
|
+
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
1199
|
+
}
|
|
1200
|
+
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
1201
|
+
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
1202
|
+
const idOffsets = uint32Bytes(offsets);
|
|
1203
|
+
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
1204
|
+
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
1205
|
+
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
1206
|
+
const headKey = encode("head");
|
|
1207
|
+
const vectorsKey = encode("vectors");
|
|
1208
|
+
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
1209
|
+
const beforeHead = builder.offset + headKey.byteLength;
|
|
1210
|
+
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
1211
|
+
const { head } = encodedHeadAndVectorsPlace(
|
|
1212
|
+
sections,
|
|
1213
|
+
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
1214
|
+
payload.vectors,
|
|
1215
|
+
vectorChecksums,
|
|
1216
|
+
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
1217
|
+
);
|
|
1218
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1219
|
+
builder.push(vectorsKey);
|
|
1220
|
+
builder.push(vectorsHeader);
|
|
1221
|
+
const beforeVectors = [...builder.chunks];
|
|
1222
|
+
builder.push(payload.vectors);
|
|
1223
|
+
return {
|
|
1224
|
+
chunks: builder.chunks,
|
|
1225
|
+
headOffset,
|
|
1226
|
+
headLength: head.byteLength,
|
|
1227
|
+
headChecksum: crc32(head),
|
|
1228
|
+
beforeVectors,
|
|
1229
|
+
vectors: {
|
|
1230
|
+
byteLength: payload.vectors.byteLength,
|
|
1231
|
+
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
1232
|
+
}
|
|
1233
|
+
};
|
|
1234
|
+
}
|
|
1235
|
+
function nodeOffsetsOf(graph) {
|
|
1236
|
+
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
1237
|
+
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
1238
|
+
let cursor = 0;
|
|
1239
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
1240
|
+
offsets[at] = cursor;
|
|
1241
|
+
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
1242
|
+
}
|
|
1243
|
+
offsets[graph.levels.length] = cursor;
|
|
1244
|
+
return uint32Bytes(offsets);
|
|
1245
|
+
}
|
|
1246
|
+
function pagedGraphPartChunks(part) {
|
|
1247
|
+
const builder = createPayloadBuilder();
|
|
1248
|
+
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
1249
|
+
builder.entry("v", part.v);
|
|
1250
|
+
builder.entry("fieldName", part.fieldName);
|
|
1251
|
+
builder.push(encode("graphs"));
|
|
1252
|
+
builder.push(arrayHeader(part.graphs.length));
|
|
1253
|
+
const layouts = [];
|
|
1254
|
+
for (const graph of part.graphs) {
|
|
1255
|
+
const { levels, neighbours, ...header } = graph;
|
|
1256
|
+
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
1257
|
+
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
1258
|
+
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
1259
|
+
const levelsAt = builder.bytesEntry("levels", levels);
|
|
1260
|
+
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
1261
|
+
layouts.push({
|
|
1262
|
+
...header,
|
|
1263
|
+
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
1264
|
+
nodeOffsets: nodeOffsetsOf(graph),
|
|
1265
|
+
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
const head = encode({ graphs: layouts });
|
|
1269
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1270
|
+
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
1271
|
+
}
|
|
1272
|
+
var EMPTY_BIN = new Uint8Array(0);
|
|
1273
|
+
var EMPTY_BIN_BYTES = 2;
|
|
1274
|
+
var BIN82 = 196;
|
|
1275
|
+
var BIN162 = 197;
|
|
1276
|
+
var BIN322 = 198;
|
|
1277
|
+
var BIN8_MAX_BYTES2 = 255;
|
|
1278
|
+
var BIN16_MAX_BYTES = 65535;
|
|
1279
|
+
function binHeader2(byteLength) {
|
|
1280
|
+
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
1281
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1282
|
+
const header = new Uint8Array(5);
|
|
1283
|
+
header[0] = BIN322;
|
|
1284
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1285
|
+
return header;
|
|
1286
|
+
}
|
|
970
1287
|
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
971
1288
|
const head = encode(withEmptyBytesInTheirPlace);
|
|
972
|
-
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] ===
|
|
1289
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
973
1290
|
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
974
|
-
return [head.subarray(0, head.length - EMPTY_BIN_BYTES),
|
|
1291
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
975
1292
|
}
|
|
976
1293
|
function vectorPartChunks(part) {
|
|
977
1294
|
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
@@ -987,20 +1304,27 @@ function keptFileRef(input, prior, kept) {
|
|
|
987
1304
|
{ key: kept.key }
|
|
988
1305
|
);
|
|
989
1306
|
}
|
|
990
|
-
return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
|
|
1307
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
1308
|
+
}
|
|
1309
|
+
function headFieldsOf(ref) {
|
|
1310
|
+
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
1311
|
+
}
|
|
1312
|
+
function headRefOf(paged) {
|
|
1313
|
+
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
991
1314
|
}
|
|
992
1315
|
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
993
1316
|
const parts = plan.graphParts ?? [];
|
|
994
1317
|
const refs = [];
|
|
995
1318
|
for (let part = 0; part < parts.length; part++) {
|
|
996
1319
|
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
1320
|
+
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
997
1321
|
const packed = await packChecksummedEnvelope(
|
|
998
|
-
[encode(parts[part])],
|
|
1322
|
+
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
999
1323
|
GRAPH_PART_ENVELOPE_VERSION,
|
|
1000
1324
|
`graph part "${key}"`
|
|
1001
1325
|
);
|
|
1002
1326
|
await input.store.write(key, packed.chunks);
|
|
1003
|
-
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
|
|
1327
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
1004
1328
|
}
|
|
1005
1329
|
return refs;
|
|
1006
1330
|
}
|
|
@@ -1014,17 +1338,28 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
1014
1338
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
1015
1339
|
const written = [];
|
|
1016
1340
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
1017
|
-
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
1341
|
+
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
1018
1342
|
const id = nextFileId;
|
|
1019
1343
|
nextFileId += 1;
|
|
1020
1344
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
1021
|
-
const
|
|
1022
|
-
|
|
1345
|
+
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
1346
|
+
const what = `vector file "${key}"`;
|
|
1347
|
+
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
1348
|
+
paged.chunks,
|
|
1023
1349
|
VECTOR_FILE_ENVELOPE_VERSION,
|
|
1024
|
-
|
|
1350
|
+
what,
|
|
1351
|
+
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
1025
1352
|
);
|
|
1026
1353
|
await input.store.write(key, packed.chunks);
|
|
1027
|
-
files.push({
|
|
1354
|
+
files.push({
|
|
1355
|
+
id,
|
|
1356
|
+
key,
|
|
1357
|
+
count: payload.docIds.length,
|
|
1358
|
+
dead,
|
|
1359
|
+
bytes: packed.byteLength,
|
|
1360
|
+
checksum: packed.checksum,
|
|
1361
|
+
...headRefOf(paged)
|
|
1362
|
+
});
|
|
1028
1363
|
written.push({ id, key, ordinals });
|
|
1029
1364
|
outcome.layouts.push({
|
|
1030
1365
|
fieldPath,
|
|
@@ -1101,8 +1436,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
1101
1436
|
function yieldToEventLoop() {
|
|
1102
1437
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
1103
1438
|
}
|
|
1439
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
1440
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
1441
|
+
composite.beginBatch();
|
|
1442
|
+
try {
|
|
1443
|
+
for (const docId of docIds) {
|
|
1444
|
+
if (!composite.has(docId)) continue;
|
|
1445
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
1446
|
+
docPartitionMap.delete(docId);
|
|
1447
|
+
}
|
|
1448
|
+
} finally {
|
|
1449
|
+
composite.endBatch();
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
1453
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-IYPLWHGC.mjs');
|
|
1454
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
1455
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1456
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
1457
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
1458
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
1459
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
1460
|
+
if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
|
|
1461
|
+
else segment.paged?.close();
|
|
1462
|
+
}
|
|
1104
1463
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
1464
|
+
const { directory } = store;
|
|
1105
1465
|
for (const ref of partition.segments) {
|
|
1466
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
1467
|
+
const { isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
|
|
1468
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
1469
|
+
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
1470
|
+
await yieldToEventLoop();
|
|
1471
|
+
continue;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1106
1474
|
const contents = await readSegmentFile(store, ref);
|
|
1107
1475
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
1108
1476
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -1147,6 +1515,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1147
1515
|
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
1148
1516
|
}
|
|
1149
1517
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
1518
|
+
const { directory } = store;
|
|
1519
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
1520
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
1521
|
+
);
|
|
1522
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
1523
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-IYPLWHGC.mjs');
|
|
1524
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
1525
|
+
}
|
|
1150
1526
|
for (const partition of manifest.partitions) {
|
|
1151
1527
|
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
1152
1528
|
throw new NarsilError(
|
|
@@ -1164,7 +1540,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1164
1540
|
for (const vector of manifest.vectors) {
|
|
1165
1541
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
1166
1542
|
if (vectorIndex === void 0) continue;
|
|
1167
|
-
await loadVectorField(
|
|
1543
|
+
await loadVectorField(
|
|
1544
|
+
store,
|
|
1545
|
+
vector,
|
|
1546
|
+
vectorIndex,
|
|
1547
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
1548
|
+
);
|
|
1168
1549
|
}
|
|
1169
1550
|
return manifest.checkpoint;
|
|
1170
1551
|
}
|
|
@@ -1174,4 +1555,4 @@ async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
|
1174
1555
|
}
|
|
1175
1556
|
}
|
|
1176
1557
|
|
|
1177
|
-
export { SINGLE_NODE_PRIMARY_TERM,
|
|
1558
|
+
export { SINGLE_NODE_PRIMARY_TERM, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, durableMetadataConfigOf, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
|