@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,231 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
2
|
+
import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-7AUASRE7.mjs';
|
|
3
|
+
import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-UUZCQQG2.mjs';
|
|
4
|
+
import { restoreVectorFields } from './chunk-KEIYVZUX.mjs';
|
|
3
5
|
import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions,
|
|
6
|
-
import {
|
|
7
|
-
import { generateId
|
|
8
|
-
import { extractVectorFieldsFromSchema, partialFieldsOf
|
|
6
|
+
import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-IO5F7ZBE.mjs';
|
|
7
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
|
|
8
|
+
import { isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-ZPWS2SKJ.mjs';
|
|
9
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
10
|
+
import { extractVectorFieldsFromSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
|
|
9
11
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
10
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
11
|
-
import {
|
|
12
|
+
import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
|
|
13
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
12
15
|
import { encode, decode } from '@msgpack/msgpack';
|
|
13
16
|
|
|
14
|
-
// src/persistence/durability/constants.ts
|
|
15
|
-
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
16
|
-
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
17
|
-
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
18
|
-
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
19
|
-
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
20
|
-
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
21
|
-
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
22
|
-
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
23
|
-
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
24
|
-
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
25
|
-
var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
|
|
26
|
-
var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
27
|
-
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
28
|
-
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
29
|
-
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
30
|
-
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
31
|
-
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
32
|
-
var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
|
|
33
|
-
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
34
|
-
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
35
|
-
|
|
36
|
-
// src/persistence/durability/segment/layout.ts
|
|
37
|
-
var SEGMENT_ID_WIDTH = 16;
|
|
38
|
-
function manifestKey(indexName) {
|
|
39
|
-
return `${indexName}/manifest`;
|
|
40
|
-
}
|
|
41
|
-
function snapshotBundleKey(indexName) {
|
|
42
|
-
return `${indexName}/snapshot`;
|
|
43
|
-
}
|
|
44
|
-
function segmentPrefix(indexName, partitionId) {
|
|
45
|
-
return `${indexName}/segments/${partitionId}/`;
|
|
46
|
-
}
|
|
47
|
-
function segmentKey(indexName, partitionId, segmentId) {
|
|
48
|
-
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
49
|
-
}
|
|
50
|
-
function segmentsPrefix(indexName) {
|
|
51
|
-
return `${indexName}/segments/`;
|
|
52
|
-
}
|
|
53
|
-
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
54
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
55
|
-
}
|
|
56
|
-
function requireGeneration(generation) {
|
|
57
|
-
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
58
|
-
throw new NarsilError(
|
|
59
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
60
|
-
`Vector graph generation ${generation} must be a positive integer`,
|
|
61
|
-
{ generation }
|
|
62
|
-
);
|
|
63
|
-
}
|
|
64
|
-
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
65
|
-
requireGeneration(generation);
|
|
66
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
67
|
-
}
|
|
68
|
-
function formatSegmentId(segmentId) {
|
|
69
|
-
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
70
|
-
throw new NarsilError(
|
|
71
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
72
|
-
`Segment id ${segmentId} must be a non-negative integer`,
|
|
73
|
-
{ segmentId }
|
|
74
|
-
);
|
|
75
|
-
}
|
|
76
|
-
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
77
|
-
}
|
|
78
|
-
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
79
|
-
function encodeFieldPath(fieldPath) {
|
|
80
|
-
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
81
|
-
throw new NarsilError(
|
|
82
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
83
|
-
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
84
|
-
{ fieldPath }
|
|
85
|
-
);
|
|
86
|
-
}
|
|
87
|
-
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
88
|
-
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
89
|
-
}
|
|
90
|
-
var SEGMENT_MANIFEST_VERSION = 7;
|
|
91
|
-
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
92
|
-
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
93
|
-
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
94
|
-
var MAX_SEGMENT_ID = 281474976710655;
|
|
95
|
-
function fileRefOnTheWire(ref) {
|
|
96
|
-
return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
|
|
97
|
-
}
|
|
98
|
-
function encodeSegmentManifest(manifest) {
|
|
99
|
-
const payload = encode({
|
|
100
|
-
version: SEGMENT_MANIFEST_VERSION,
|
|
101
|
-
schema: manifest.schema,
|
|
102
|
-
language: manifest.language,
|
|
103
|
-
checkpoint: manifest.checkpoint.map((c) => ({
|
|
104
|
-
partitionId: c.partitionId,
|
|
105
|
-
lastSeqNo: c.lastSeqNo,
|
|
106
|
-
primaryTerm: c.primaryTerm,
|
|
107
|
-
...c.termHistory === void 0 ? {} : {
|
|
108
|
-
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
109
|
-
}
|
|
110
|
-
})),
|
|
111
|
-
partitions: manifest.partitions.map((p) => ({
|
|
112
|
-
partitionId: p.partitionId,
|
|
113
|
-
nextSegmentId: p.nextSegmentId,
|
|
114
|
-
segments: p.segments.map((s) => ({
|
|
115
|
-
id: s.id,
|
|
116
|
-
key: s.key,
|
|
117
|
-
docCount: s.docCount,
|
|
118
|
-
tombstoneCount: s.tombstoneCount,
|
|
119
|
-
bytes: s.bytes,
|
|
120
|
-
checksum: s.checksum
|
|
121
|
-
}))
|
|
122
|
-
})),
|
|
123
|
-
vectors: manifest.vectors.map((v) => ({
|
|
124
|
-
fieldPath: v.fieldPath,
|
|
125
|
-
nextFileId: v.nextFileId,
|
|
126
|
-
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
127
|
-
graphGeneration: v.graphGeneration,
|
|
128
|
-
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
129
|
-
}))
|
|
130
|
-
});
|
|
131
|
-
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
132
|
-
}
|
|
133
|
-
function manifestReferencedKeys(manifest) {
|
|
134
|
-
const keys = /* @__PURE__ */ new Set();
|
|
135
|
-
for (const partition of manifest.partitions) {
|
|
136
|
-
for (const segment of partition.segments) {
|
|
137
|
-
keys.add(segment.key);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
for (const vector of manifest.vectors) {
|
|
141
|
-
for (const file of vector.files) keys.add(file.key);
|
|
142
|
-
for (const part of vector.graphParts) keys.add(part.key);
|
|
143
|
-
}
|
|
144
|
-
return keys;
|
|
145
|
-
}
|
|
146
|
-
function manifestFileRefs(manifest) {
|
|
147
|
-
const refs = [];
|
|
148
|
-
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
149
|
-
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
150
|
-
return refs;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// src/persistence/durability/segment/store.ts
|
|
154
|
-
function directoryCheckpointStore(directory) {
|
|
155
|
-
const store = {
|
|
156
|
-
read: (key) => directory.read(key),
|
|
157
|
-
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
158
|
-
remove: (key) => directory.remove(key),
|
|
159
|
-
list: (prefix) => directory.list(prefix),
|
|
160
|
-
pathOf: (key) => directory.pathOf(key)
|
|
161
|
-
};
|
|
162
|
-
const byteLength = directory.byteLength;
|
|
163
|
-
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
164
|
-
return store;
|
|
165
|
-
}
|
|
166
|
-
function joined(chunks) {
|
|
167
|
-
if (chunks.length === 1) return chunks[0];
|
|
168
|
-
let total = 0;
|
|
169
|
-
for (const chunk of chunks) total += chunk.byteLength;
|
|
170
|
-
const bytes = new Uint8Array(total);
|
|
171
|
-
let offset = 0;
|
|
172
|
-
for (const chunk of chunks) {
|
|
173
|
-
bytes.set(chunk, offset);
|
|
174
|
-
offset += chunk.byteLength;
|
|
175
|
-
}
|
|
176
|
-
return bytes;
|
|
177
|
-
}
|
|
178
|
-
function adapterFailure(code, verb, key, error) {
|
|
179
|
-
if (error instanceof NarsilError) return error;
|
|
180
|
-
const cause = error instanceof Error ? error.message : String(error);
|
|
181
|
-
return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
|
|
182
|
-
}
|
|
183
|
-
async function guarded(run, code, verb, key) {
|
|
184
|
-
try {
|
|
185
|
-
return await run();
|
|
186
|
-
} catch (error) {
|
|
187
|
-
throw adapterFailure(code, verb, key, error);
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
function adapterCheckpointStore(adapter) {
|
|
191
|
-
return {
|
|
192
|
-
read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
|
|
193
|
-
write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
|
|
194
|
-
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
195
|
-
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
async function readListedFile(store, key, expected) {
|
|
199
|
-
const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
|
|
200
|
-
if (length === null) missingFile(expected.what, key);
|
|
201
|
-
if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
|
|
202
|
-
const bytes = await store.read(key);
|
|
203
|
-
if (bytes === null) missingFile(expected.what, key);
|
|
204
|
-
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
205
|
-
return bytes;
|
|
206
|
-
}
|
|
207
|
-
function missingFile(what, key) {
|
|
208
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
|
|
209
|
-
key
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
function requireListedLength(what, key, found, listed) {
|
|
213
|
-
if (found === listed) return;
|
|
214
|
-
throw new NarsilError(
|
|
215
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
216
|
-
`The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
|
|
217
|
-
{ key, bytes: found, listed }
|
|
218
|
-
);
|
|
219
|
-
}
|
|
220
|
-
function requireListedChecksum(what, key, found, listed) {
|
|
221
|
-
if (found === listed) return;
|
|
222
|
-
throw new NarsilError(
|
|
223
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
224
|
-
`The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
|
|
225
|
-
{ key, checksum: found, listed }
|
|
226
|
-
);
|
|
227
|
-
}
|
|
228
|
-
|
|
229
17
|
// src/persistence/durability/segment/captured-partition.ts
|
|
230
18
|
function keepMaskOf(segment) {
|
|
231
19
|
const keep = new Uint8Array(segment.arrays.documentCount);
|
|
@@ -676,10 +464,10 @@ var RECORD_LENGTH_SIZE = 4;
|
|
|
676
464
|
var FRAME_CRC_SIZE = 4;
|
|
677
465
|
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
678
466
|
function writeSegmentHeader() {
|
|
679
|
-
const
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
return
|
|
467
|
+
const header = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
468
|
+
header.set(WAL_MAGIC, 0);
|
|
469
|
+
header[4] = WAL_FORMAT_VERSION & 255;
|
|
470
|
+
return header;
|
|
683
471
|
}
|
|
684
472
|
function frameRecord(entry) {
|
|
685
473
|
const payload = encode(entry);
|
|
@@ -720,9 +508,9 @@ function corrupt(message, details) {
|
|
|
720
508
|
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
721
509
|
}
|
|
722
510
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
723
|
-
const
|
|
724
|
-
if (!
|
|
725
|
-
corrupt(`WAL segment header invalid in durable region: ${
|
|
511
|
+
const header = checkSegmentHeader(data);
|
|
512
|
+
if (!header.ok) {
|
|
513
|
+
corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
|
|
726
514
|
}
|
|
727
515
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
728
516
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -821,6 +609,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
|
|
|
821
609
|
function walPrefix(indexName, partitionId) {
|
|
822
610
|
return `${indexName}/wal/${partitionId}/`;
|
|
823
611
|
}
|
|
612
|
+
function nextSegmentKey(indexName, partitionId) {
|
|
613
|
+
return `${walPrefix(indexName, partitionId)}next`;
|
|
614
|
+
}
|
|
824
615
|
function segmentStartSeqNo(key, prefix) {
|
|
825
616
|
if (!key.startsWith(prefix)) {
|
|
826
617
|
return null;
|
|
@@ -865,14 +656,14 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
865
656
|
continue;
|
|
866
657
|
}
|
|
867
658
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
868
|
-
const
|
|
869
|
-
if (!
|
|
659
|
+
const header = checkSegmentHeader(bytes);
|
|
660
|
+
if (!header.ok) {
|
|
870
661
|
throw new NarsilError(
|
|
871
662
|
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
872
|
-
`Sealed WAL segment header invalid: ${
|
|
663
|
+
`Sealed WAL segment header invalid: ${header.reason}`,
|
|
873
664
|
{
|
|
874
665
|
key,
|
|
875
|
-
reason:
|
|
666
|
+
reason: header.reason
|
|
876
667
|
}
|
|
877
668
|
);
|
|
878
669
|
}
|
|
@@ -1002,569 +793,23 @@ async function buildSegmentFromEntries(input) {
|
|
|
1002
793
|
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
|
|
1003
794
|
return built;
|
|
1004
795
|
}
|
|
1005
|
-
var FIXMAP = 128;
|
|
1006
|
-
var MAP16 = 222;
|
|
1007
|
-
var MAP32 = 223;
|
|
1008
|
-
var FIXARRAY = 144;
|
|
1009
|
-
var ARRAY16 = 220;
|
|
1010
|
-
var ARRAY32 = 221;
|
|
1011
|
-
var BIN8 = 196;
|
|
1012
|
-
var BIN16 = 197;
|
|
1013
|
-
var BIN32 = 198;
|
|
1014
|
-
var FIX_COUNT_LIMIT = 16;
|
|
1015
|
-
var UINT8_LIMIT = 256;
|
|
1016
|
-
var UINT16_LIMIT = 65536;
|
|
1017
|
-
function header(fixed, sixteen, thirtyTwo, count) {
|
|
1018
|
-
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
1019
|
-
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
1020
|
-
const bytes = new Uint8Array(5);
|
|
1021
|
-
bytes[0] = thirtyTwo;
|
|
1022
|
-
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
1023
|
-
return bytes;
|
|
1024
|
-
}
|
|
1025
|
-
function binHeader(byteLength) {
|
|
1026
|
-
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
|
|
1027
|
-
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1028
|
-
const bytes = new Uint8Array(5);
|
|
1029
|
-
bytes[0] = BIN32;
|
|
1030
|
-
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
1031
|
-
return bytes;
|
|
1032
|
-
}
|
|
1033
|
-
var MessagePackChunks = class {
|
|
1034
|
-
chunks = [];
|
|
1035
|
-
byteLength = 0;
|
|
1036
|
-
push(chunk) {
|
|
1037
|
-
if (chunk.byteLength === 0) return;
|
|
1038
|
-
this.chunks.push(chunk);
|
|
1039
|
-
this.byteLength += chunk.byteLength;
|
|
1040
|
-
}
|
|
1041
|
-
map(entries) {
|
|
1042
|
-
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
1043
|
-
return this;
|
|
1044
|
-
}
|
|
1045
|
-
array(items) {
|
|
1046
|
-
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1047
|
-
return this;
|
|
1048
|
-
}
|
|
1049
|
-
value(value) {
|
|
1050
|
-
this.push(encode(value));
|
|
1051
|
-
return this;
|
|
1052
|
-
}
|
|
1053
|
-
key(name) {
|
|
1054
|
-
return this.value(name);
|
|
1055
|
-
}
|
|
1056
|
-
binary(bytes) {
|
|
1057
|
-
this.push(binHeader(bytes.byteLength));
|
|
1058
|
-
this.push(bytes);
|
|
1059
|
-
return this;
|
|
1060
|
-
}
|
|
1061
|
-
binaryOrNil(bytes) {
|
|
1062
|
-
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1063
|
-
}
|
|
1064
|
-
};
|
|
1065
|
-
|
|
1066
|
-
// src/serialization/keyword-segment/token-order.ts
|
|
1067
|
-
function slotsAreInTokenOrder(payloadSlots) {
|
|
1068
|
-
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1069
|
-
if (payloadSlots[at] !== at) return false;
|
|
1070
|
-
}
|
|
1071
|
-
return true;
|
|
1072
|
-
}
|
|
1073
|
-
function reorderedPositions(arrays, postingOffsets) {
|
|
1074
|
-
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1075
|
-
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1076
|
-
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1077
|
-
const offsets = new Uint32Array(postings + 1);
|
|
1078
|
-
const values = new Uint32Array(positionValues.length);
|
|
1079
|
-
let cursor = 0;
|
|
1080
|
-
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1081
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1082
|
-
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1083
|
-
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1084
|
-
offsets[to] = cursor;
|
|
1085
|
-
const start = positionOffsets[from];
|
|
1086
|
-
const end = positionOffsets[from + 1];
|
|
1087
|
-
values.set(positionValues.subarray(start, end), cursor);
|
|
1088
|
-
cursor += end - start;
|
|
1089
|
-
}
|
|
1090
|
-
}
|
|
1091
|
-
offsets[postings] = cursor;
|
|
1092
|
-
return { positionOffsets: offsets, positionValues: values };
|
|
1093
|
-
}
|
|
1094
|
-
function postingsInTokenOrder(arrays) {
|
|
1095
|
-
const { tokenTable } = arrays;
|
|
1096
|
-
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1097
|
-
const tokens = tokenTable.payloadSlots.length;
|
|
1098
|
-
const postingOffsets = new Uint32Array(tokens + 1);
|
|
1099
|
-
for (let token = 0; token < tokens; token++) {
|
|
1100
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1101
|
-
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
1102
|
-
}
|
|
1103
|
-
const postings = postingOffsets[tokens];
|
|
1104
|
-
const postingDocIds = new Uint32Array(postings);
|
|
1105
|
-
const postingFrequencies = new Uint16Array(postings);
|
|
1106
|
-
const postingFieldIndices = new Uint16Array(postings);
|
|
1107
|
-
for (let token = 0; token < tokens; token++) {
|
|
1108
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1109
|
-
const start = arrays.postingOffsets[slot];
|
|
1110
|
-
const end = arrays.postingOffsets[slot + 1];
|
|
1111
|
-
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
1112
|
-
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
1113
|
-
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
1114
|
-
}
|
|
1115
|
-
return {
|
|
1116
|
-
postingOffsets,
|
|
1117
|
-
postingDocIds,
|
|
1118
|
-
postingFrequencies,
|
|
1119
|
-
postingFieldIndices,
|
|
1120
|
-
...reorderedPositions(arrays, postingOffsets)
|
|
1121
|
-
};
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
// src/serialization/keyword-segment/encode.ts
|
|
1125
|
-
var TOP_LEVEL_ENTRIES = 28;
|
|
1126
|
-
function writeFieldLengths(out, arrays) {
|
|
1127
|
-
out.array(arrays.fieldLengthNames.length);
|
|
1128
|
-
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
1129
|
-
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
1130
|
-
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
function writeValueColumns(out, columns) {
|
|
1134
|
-
out.array(columns.length);
|
|
1135
|
-
for (const column of columns) {
|
|
1136
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1137
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1138
|
-
out.key("values").binary(littleEndianBytes(column.values));
|
|
1139
|
-
}
|
|
1140
|
-
}
|
|
1141
|
-
function writeBooleanColumns(out, columns) {
|
|
1142
|
-
out.array(columns.length);
|
|
1143
|
-
for (const column of columns) {
|
|
1144
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1145
|
-
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
1146
|
-
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
1147
|
-
}
|
|
1148
|
-
}
|
|
1149
|
-
function writeEnumColumns(out, columns) {
|
|
1150
|
-
out.array(columns.length);
|
|
1151
|
-
for (const column of columns) {
|
|
1152
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1153
|
-
out.key("values").value(column.values);
|
|
1154
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1155
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
function writeGeopointColumns(out, columns) {
|
|
1159
|
-
out.array(columns.length);
|
|
1160
|
-
for (const column of columns) {
|
|
1161
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1162
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1163
|
-
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
1164
|
-
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
1165
|
-
}
|
|
1166
|
-
}
|
|
1167
|
-
function writePatternColumns(out, columns) {
|
|
1168
|
-
out.array(columns.length);
|
|
1169
|
-
for (const column of columns) {
|
|
1170
|
-
out.map(5).key("field").value(column.fieldPath);
|
|
1171
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1172
|
-
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
1173
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1174
|
-
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
function writeSurfaceForms(out, arrays) {
|
|
1178
|
-
const table = arrays.surfaceTable;
|
|
1179
|
-
if (table === null || table.counts.length === 0) {
|
|
1180
|
-
out.value(null);
|
|
1181
|
-
return;
|
|
1182
|
-
}
|
|
1183
|
-
out.map(3).key("text").binary(table.blob);
|
|
1184
|
-
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
1185
|
-
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
1186
|
-
}
|
|
1187
|
-
function writePartialWords(out, arrays) {
|
|
1188
|
-
const table = arrays.partialWords;
|
|
1189
|
-
if (table === null) {
|
|
1190
|
-
out.value(null);
|
|
1191
|
-
return;
|
|
1192
|
-
}
|
|
1193
|
-
out.map(8).key("fields").value(table.fields);
|
|
1194
|
-
out.key("text").binary(table.spellings);
|
|
1195
|
-
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
1196
|
-
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
1197
|
-
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
1198
|
-
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
1199
|
-
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
1200
|
-
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
1201
|
-
}
|
|
1202
|
-
function encodeKeywordSegment(contents) {
|
|
1203
|
-
const { arrays } = contents;
|
|
1204
|
-
const postings = postingsInTokenOrder(arrays);
|
|
1205
|
-
const out = new MessagePackChunks();
|
|
1206
|
-
const { writeRecords } = contents;
|
|
1207
|
-
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
1208
|
-
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
1209
|
-
out.key("schema").value(contents.schema);
|
|
1210
|
-
out.key("tombstones").value(contents.tombstones);
|
|
1211
|
-
out.key("documentCount").value(arrays.documentCount);
|
|
1212
|
-
out.key("ids").binary(arrays.idTable.blob);
|
|
1213
|
-
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
1214
|
-
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
1215
|
-
out.key("documents").binary(arrays.documentTable.blob);
|
|
1216
|
-
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
1217
|
-
out.key("fieldNames").value(arrays.fieldNames);
|
|
1218
|
-
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
1219
|
-
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
1220
|
-
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
1221
|
-
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
1222
|
-
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
1223
|
-
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
1224
|
-
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
1225
|
-
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
1226
|
-
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
1227
|
-
out.key("fieldLengths");
|
|
1228
|
-
writeFieldLengths(out, arrays);
|
|
1229
|
-
out.key("numeric");
|
|
1230
|
-
writeValueColumns(out, arrays.numeric);
|
|
1231
|
-
out.key("sizes");
|
|
1232
|
-
writeValueColumns(out, arrays.sizes);
|
|
1233
|
-
out.key("boolean");
|
|
1234
|
-
writeBooleanColumns(out, arrays.boolean);
|
|
1235
|
-
out.key("enum");
|
|
1236
|
-
writeEnumColumns(out, arrays.enums);
|
|
1237
|
-
out.key("geopoint");
|
|
1238
|
-
writeGeopointColumns(out, arrays.geo);
|
|
1239
|
-
out.key("pattern");
|
|
1240
|
-
writePatternColumns(out, arrays.pattern);
|
|
1241
|
-
out.key("surfaceForms");
|
|
1242
|
-
writeSurfaceForms(out, arrays);
|
|
1243
|
-
out.key("partialWords");
|
|
1244
|
-
writePartialWords(out, arrays);
|
|
1245
|
-
if (writeRecords !== void 0) {
|
|
1246
|
-
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
1247
|
-
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
1248
|
-
}
|
|
1249
|
-
return out;
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
// src/persistence/durability/segment/segment-file.ts
|
|
1253
|
-
function sharesMemoryWithWorkers() {
|
|
1254
|
-
return typeof SharedArrayBuffer === "function";
|
|
1255
|
-
}
|
|
1256
|
-
async function readSegmentFile(store, ref) {
|
|
1257
|
-
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
1258
|
-
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
1259
|
-
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
1260
|
-
what: `segment "${ref.key}"`,
|
|
1261
|
-
allowsCompression: true
|
|
1262
|
-
});
|
|
1263
|
-
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
1264
|
-
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
1265
|
-
}
|
|
1266
|
-
var ORDINAL_BYTES = 4;
|
|
1267
|
-
var POSTING_BYTES = 8;
|
|
1268
|
-
var VALUE_ENTRY_BYTES = 12;
|
|
1269
|
-
var GEOPOINT_ENTRY_BYTES = 20;
|
|
1270
|
-
var TOKEN_ENTRY_BYTES = 12;
|
|
1271
|
-
var STRING_HEADER_BYTES = 5;
|
|
1272
|
-
var encoder = new TextEncoder();
|
|
1273
|
-
function documentBytes(segment, ordinal) {
|
|
1274
|
-
const table = segment.arrays.documentTable;
|
|
1275
|
-
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
1276
|
-
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
1277
|
-
}
|
|
1278
|
-
function addPerEntry(costs, docIds, bytes) {
|
|
1279
|
-
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
1280
|
-
}
|
|
1281
|
-
function chargePostings(segment, costs, chargesTokens) {
|
|
1282
|
-
const arrays = segment.arrays;
|
|
1283
|
-
const positions = arrays.positionOffsets;
|
|
1284
|
-
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
1285
|
-
let bytes = POSTING_BYTES;
|
|
1286
|
-
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
1287
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
1288
|
-
}
|
|
1289
|
-
if (!chargesTokens) return;
|
|
1290
|
-
const table = arrays.tokenTable;
|
|
1291
|
-
for (let at = 0; at < table.size; at++) {
|
|
1292
|
-
const slot = table.payloadSlot(at);
|
|
1293
|
-
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
1294
|
-
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
1295
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
1296
|
-
}
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
function chargeColumns(segment, costs) {
|
|
1300
|
-
const arrays = segment.arrays;
|
|
1301
|
-
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
1302
|
-
for (const column of arrays.boolean) {
|
|
1303
|
-
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
1304
|
-
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
1305
|
-
}
|
|
1306
|
-
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1307
|
-
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
1308
|
-
for (const column of arrays.pattern) {
|
|
1309
|
-
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1310
|
-
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
1311
|
-
}
|
|
1312
|
-
}
|
|
1313
|
-
function stringListBytes(values) {
|
|
1314
|
-
let bytes = STRING_HEADER_BYTES;
|
|
1315
|
-
for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
|
|
1316
|
-
return bytes;
|
|
1317
|
-
}
|
|
1318
|
-
function fixedBytesOf(segment, chargesTokens) {
|
|
1319
|
-
const arrays = segment.arrays;
|
|
1320
|
-
let bytes = stringListBytes(arrays.fieldNames);
|
|
1321
|
-
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
1322
|
-
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
1323
|
-
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
1324
|
-
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
1325
|
-
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
1326
|
-
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
1327
|
-
return bytes + segment.partialWords.bytes();
|
|
1328
|
-
}
|
|
1329
|
-
function keptDocumentsOf(selection, chargesTokens) {
|
|
1330
|
-
const { segment } = selection;
|
|
1331
|
-
const count = segment.arrays.documentCount;
|
|
1332
|
-
const costs = new Float64Array(count);
|
|
1333
|
-
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
1334
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1335
|
-
costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
1336
|
-
}
|
|
1337
|
-
chargePostings(segment, costs, chargesTokens);
|
|
1338
|
-
chargeColumns(segment, costs);
|
|
1339
|
-
const ordinals = [];
|
|
1340
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1341
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1342
|
-
costs[ordinal] += documentBytes(segment, ordinal);
|
|
1343
|
-
ordinals.push(ordinal);
|
|
1344
|
-
}
|
|
1345
|
-
return { segment, ordinals, costs };
|
|
1346
|
-
}
|
|
1347
|
-
function selectionOf(segment, ordinals) {
|
|
1348
|
-
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
1349
|
-
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
1350
|
-
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
1351
|
-
}
|
|
1352
|
-
function tooLargeForOneFile(cost, budget) {
|
|
1353
|
-
throw new NarsilError(
|
|
1354
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1355
|
-
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
1356
|
-
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
1357
|
-
);
|
|
1358
|
-
}
|
|
1359
|
-
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
1360
|
-
let tokenBytes = 0;
|
|
1361
|
-
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
1362
|
-
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
1363
|
-
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
1364
|
-
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
1365
|
-
const budget = MAX_FILE_BYTES - fixed;
|
|
1366
|
-
const groups = [];
|
|
1367
|
-
let current = [];
|
|
1368
|
-
let currentBytes = 0;
|
|
1369
|
-
let currentCount = 0;
|
|
1370
|
-
const flush = () => {
|
|
1371
|
-
if (currentCount === 0) return;
|
|
1372
|
-
groups.push({
|
|
1373
|
-
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
1374
|
-
documentCount: currentCount
|
|
1375
|
-
});
|
|
1376
|
-
current = [];
|
|
1377
|
-
currentBytes = 0;
|
|
1378
|
-
currentCount = 0;
|
|
1379
|
-
};
|
|
1380
|
-
for (const selection of selections) {
|
|
1381
|
-
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
1382
|
-
let entry = null;
|
|
1383
|
-
for (const ordinal of kept.ordinals) {
|
|
1384
|
-
const cost = kept.costs[ordinal];
|
|
1385
|
-
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
1386
|
-
if (currentBytes + cost > budget) {
|
|
1387
|
-
flush();
|
|
1388
|
-
entry = null;
|
|
1389
|
-
}
|
|
1390
|
-
if (entry === null) {
|
|
1391
|
-
entry = { segment: kept.segment, ordinals: [] };
|
|
1392
|
-
current.push(entry);
|
|
1393
|
-
}
|
|
1394
|
-
entry.ordinals.push(ordinal);
|
|
1395
|
-
currentBytes += cost;
|
|
1396
|
-
currentCount += 1;
|
|
1397
|
-
}
|
|
1398
|
-
}
|
|
1399
|
-
flush();
|
|
1400
|
-
return groups;
|
|
1401
|
-
}
|
|
1402
|
-
function halvesOf(group) {
|
|
1403
|
-
const flat = [];
|
|
1404
|
-
for (const selection of group.selections) {
|
|
1405
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1406
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1407
|
-
flat.push({ segment: selection.segment, ordinal });
|
|
1408
|
-
}
|
|
1409
|
-
}
|
|
1410
|
-
const middle = Math.ceil(flat.length / 2);
|
|
1411
|
-
const groupOf = (entries) => {
|
|
1412
|
-
const bySegment = /* @__PURE__ */ new Map();
|
|
1413
|
-
for (const { segment, ordinal } of entries) {
|
|
1414
|
-
let ordinals = bySegment.get(segment);
|
|
1415
|
-
if (ordinals === void 0) {
|
|
1416
|
-
ordinals = [];
|
|
1417
|
-
bySegment.set(segment, ordinals);
|
|
1418
|
-
}
|
|
1419
|
-
ordinals.push(ordinal);
|
|
1420
|
-
}
|
|
1421
|
-
return {
|
|
1422
|
-
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
1423
|
-
documentCount: entries.length
|
|
1424
|
-
};
|
|
1425
|
-
};
|
|
1426
|
-
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
// src/persistence/durability/segment/segment-write.ts
|
|
1430
|
-
function emptyPayload() {
|
|
1431
|
-
return {
|
|
1432
|
-
documentCount: 0,
|
|
1433
|
-
docIds: [],
|
|
1434
|
-
fieldNames: [],
|
|
1435
|
-
tokens: [],
|
|
1436
|
-
postingOffsets: new Uint32Array(1),
|
|
1437
|
-
postingDocIds: new Uint32Array(0),
|
|
1438
|
-
postingFrequencies: new Uint16Array(0),
|
|
1439
|
-
postingFieldIndices: new Uint16Array(0),
|
|
1440
|
-
positionOffsets: null,
|
|
1441
|
-
positionValues: null,
|
|
1442
|
-
fieldLengthNames: [],
|
|
1443
|
-
fieldLengthColumns: [],
|
|
1444
|
-
totalFieldLengths: {},
|
|
1445
|
-
docFrequencies: {},
|
|
1446
|
-
surfaceForms: null,
|
|
1447
|
-
partialSpellings: null,
|
|
1448
|
-
numeric: [],
|
|
1449
|
-
boolean: [],
|
|
1450
|
-
enums: [],
|
|
1451
|
-
geo: [],
|
|
1452
|
-
pattern: [],
|
|
1453
|
-
sizes: []
|
|
1454
|
-
};
|
|
1455
|
-
}
|
|
1456
|
-
function noSpellingsOf(analysis) {
|
|
1457
|
-
if (analysis === null || analysis.fields.length === 0) return null;
|
|
1458
|
-
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
1459
|
-
}
|
|
1460
|
-
function writeRecordsOfGroup(group, records, count) {
|
|
1461
|
-
const seqNos = new Float64Array(count);
|
|
1462
|
-
const primaryTerms = new Float64Array(count);
|
|
1463
|
-
let at = 0;
|
|
1464
|
-
for (const selection of group?.selections ?? []) {
|
|
1465
|
-
const source = records.get(selection.segment);
|
|
1466
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1467
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1468
|
-
if (source !== void 0) {
|
|
1469
|
-
seqNos[at] = source.seqNos[ordinal];
|
|
1470
|
-
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
1471
|
-
}
|
|
1472
|
-
at += 1;
|
|
1473
|
-
}
|
|
1474
|
-
}
|
|
1475
|
-
return { seqNos, primaryTerms };
|
|
1476
|
-
}
|
|
1477
|
-
function withWriteRecords(contents, group, request) {
|
|
1478
|
-
if (request.writeRecords === void 0) return contents;
|
|
1479
|
-
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
1480
|
-
return { ...contents, writeRecords };
|
|
1481
|
-
}
|
|
1482
|
-
function contentsOfGroup(group, request, tombstones) {
|
|
1483
|
-
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
1484
|
-
if (group === null) {
|
|
1485
|
-
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
1486
|
-
return {
|
|
1487
|
-
schema,
|
|
1488
|
-
tombstones,
|
|
1489
|
-
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
1490
|
-
};
|
|
1491
|
-
}
|
|
1492
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
1493
|
-
return {
|
|
1494
|
-
schema,
|
|
1495
|
-
tombstones,
|
|
1496
|
-
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
1497
|
-
};
|
|
1498
|
-
}
|
|
1499
|
-
function requireRoomForAnotherSegment(state) {
|
|
1500
|
-
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
1501
|
-
throw new NarsilError(
|
|
1502
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1503
|
-
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
1504
|
-
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
1505
|
-
);
|
|
1506
|
-
}
|
|
1507
|
-
async function writeGroup(state, group) {
|
|
1508
|
-
const { request } = state;
|
|
1509
|
-
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
1510
|
-
const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
1511
|
-
const encoded = encodeKeywordSegment(contents);
|
|
1512
|
-
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
1513
|
-
const [first, second] = halvesOf(group);
|
|
1514
|
-
await writeGroup(state, first);
|
|
1515
|
-
await writeGroup(state, second);
|
|
1516
|
-
return;
|
|
1517
|
-
}
|
|
1518
|
-
requireRoomForAnotherSegment(state);
|
|
1519
|
-
const id = state.nextSegmentId;
|
|
1520
|
-
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
1521
|
-
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
1522
|
-
await request.store.write(key, packed.chunks);
|
|
1523
|
-
state.segments.push({
|
|
1524
|
-
id,
|
|
1525
|
-
key,
|
|
1526
|
-
bytes: packed.byteLength,
|
|
1527
|
-
checksum: packed.checksum,
|
|
1528
|
-
docCount: contents.arrays.documentCount,
|
|
1529
|
-
tombstoneCount: tombstones.length
|
|
1530
|
-
});
|
|
1531
|
-
state.nextSegmentId = id + 1;
|
|
1532
|
-
state.tombstonesWritten = true;
|
|
1533
|
-
}
|
|
1534
|
-
function keptDocumentCount(selections) {
|
|
1535
|
-
let count = 0;
|
|
1536
|
-
for (const selection of selections) {
|
|
1537
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1538
|
-
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
1539
|
-
}
|
|
1540
|
-
}
|
|
1541
|
-
return count;
|
|
1542
|
-
}
|
|
1543
|
-
async function writeSegmentsOf(request) {
|
|
1544
|
-
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
1545
|
-
const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
1546
|
-
for (const group of groups) {
|
|
1547
|
-
if (group.documentCount > 0) await writeGroup(state, group);
|
|
1548
|
-
}
|
|
1549
|
-
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
1550
|
-
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
1551
|
-
}
|
|
1552
796
|
|
|
1553
797
|
// src/persistence/durability/segment/compaction.ts
|
|
1554
|
-
function windowFits(segments, start, windowLength) {
|
|
798
|
+
function windowFits(segments, start, windowLength, paged) {
|
|
1555
799
|
let documents = 0;
|
|
1556
800
|
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
1557
801
|
for (let at = start; at < start + windowLength; at++) {
|
|
1558
802
|
documents += segments[at].docCount;
|
|
1559
803
|
bytes += segments[at].bytes;
|
|
1560
804
|
}
|
|
805
|
+
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
1561
806
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
1562
807
|
}
|
|
1563
|
-
function smallestFittingWindow(segments, windowLength) {
|
|
808
|
+
function smallestFittingWindow(segments, windowLength, paged) {
|
|
1564
809
|
let best = null;
|
|
1565
810
|
let bestBytes = Number.POSITIVE_INFINITY;
|
|
1566
811
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
1567
|
-
if (!windowFits(segments, start, windowLength)) continue;
|
|
812
|
+
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
1568
813
|
let bytes = 0;
|
|
1569
814
|
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
1570
815
|
if (bytes < bestBytes) {
|
|
@@ -1574,9 +819,9 @@ function smallestFittingWindow(segments, windowLength) {
|
|
|
1574
819
|
}
|
|
1575
820
|
return best;
|
|
1576
821
|
}
|
|
1577
|
-
function chooseWindow(segments, wantedLength) {
|
|
822
|
+
function chooseWindow(segments, wantedLength, paged) {
|
|
1578
823
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
1579
|
-
const window = smallestFittingWindow(segments, windowLength);
|
|
824
|
+
const window = smallestFittingWindow(segments, windowLength, paged);
|
|
1580
825
|
if (window !== null) return window;
|
|
1581
826
|
}
|
|
1582
827
|
return null;
|
|
@@ -1617,11 +862,39 @@ async function compactPartitionSegments(input) {
|
|
|
1617
862
|
if (segments.length <= compactionThreshold) {
|
|
1618
863
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1619
864
|
}
|
|
1620
|
-
const
|
|
865
|
+
const paged = input.paged === true;
|
|
866
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
1621
867
|
if (window === null) {
|
|
1622
868
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1623
869
|
}
|
|
1624
870
|
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
871
|
+
const replacing = (written2) => ({
|
|
872
|
+
segments: [
|
|
873
|
+
...segments.slice(0, window.start),
|
|
874
|
+
...written2.segments,
|
|
875
|
+
...segments.slice(window.start + window.windowLength)
|
|
876
|
+
],
|
|
877
|
+
nextSegmentId: written2.nextSegmentId
|
|
878
|
+
});
|
|
879
|
+
if (paged) {
|
|
880
|
+
const { directory } = input.store;
|
|
881
|
+
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
882
|
+
const { mergePagedWindow } = await import('./module-IYPLWHGC.mjs');
|
|
883
|
+
return replacing(
|
|
884
|
+
await mergePagedWindow({
|
|
885
|
+
store: input.store,
|
|
886
|
+
directory,
|
|
887
|
+
indexName: input.indexName,
|
|
888
|
+
partitionId: input.partitionId,
|
|
889
|
+
schema: input.schema,
|
|
890
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
891
|
+
textAnalysis: input.textAnalysis,
|
|
892
|
+
nextSegmentId: input.nextSegmentId,
|
|
893
|
+
refs: windowRefs,
|
|
894
|
+
dropsEveryTombstone: window.start === 0
|
|
895
|
+
})
|
|
896
|
+
);
|
|
897
|
+
}
|
|
1625
898
|
const contents = [];
|
|
1626
899
|
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
1627
900
|
const frozen = contents.map(frozenOf);
|
|
@@ -1640,14 +913,7 @@ async function compactPartitionSegments(input) {
|
|
|
1640
913
|
mergesIntoOneFile: true,
|
|
1641
914
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
1642
915
|
});
|
|
1643
|
-
return
|
|
1644
|
-
segments: [
|
|
1645
|
-
...segments.slice(0, window.start),
|
|
1646
|
-
...written.segments,
|
|
1647
|
-
...segments.slice(window.start + window.windowLength)
|
|
1648
|
-
],
|
|
1649
|
-
nextSegmentId: written.nextSegmentId
|
|
1650
|
-
};
|
|
916
|
+
return replacing(written);
|
|
1651
917
|
}
|
|
1652
918
|
|
|
1653
919
|
// src/persistence/durability/segment/write.ts
|
|
@@ -1694,6 +960,21 @@ function wholeContentsOf(input, partitionId) {
|
|
|
1694
960
|
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
1695
961
|
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
1696
962
|
}
|
|
963
|
+
function pagedSettingsOf(context) {
|
|
964
|
+
if (context.input.metadata.textStorage !== "disk") return {};
|
|
965
|
+
return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
|
|
966
|
+
}
|
|
967
|
+
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
968
|
+
const { directory } = context.input.store;
|
|
969
|
+
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
970
|
+
const { pagedTombstonesFor } = await import('./module-IYPLWHGC.mjs');
|
|
971
|
+
return pagedTombstonesFor(
|
|
972
|
+
directory,
|
|
973
|
+
older,
|
|
974
|
+
selections.map((selection) => selection.segment),
|
|
975
|
+
deleted
|
|
976
|
+
);
|
|
977
|
+
}
|
|
1697
978
|
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
1698
979
|
const written = await writeSegmentsOf({
|
|
1699
980
|
store: context.input.store,
|
|
@@ -1705,7 +986,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
1705
986
|
textAnalysis: context.textAnalysis,
|
|
1706
987
|
selections: whole.selections,
|
|
1707
988
|
tombstones: [],
|
|
1708
|
-
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
|
|
989
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
|
|
990
|
+
...pagedSettingsOf(context)
|
|
1709
991
|
});
|
|
1710
992
|
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
1711
993
|
}
|
|
@@ -1721,6 +1003,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1721
1003
|
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
1722
1004
|
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
1723
1005
|
if (built !== null) {
|
|
1006
|
+
const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
|
|
1724
1007
|
const written = await writeSegmentsOf({
|
|
1725
1008
|
store: context.input.store,
|
|
1726
1009
|
indexName: context.indexName,
|
|
@@ -1730,8 +1013,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1730
1013
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
1731
1014
|
textAnalysis: context.textAnalysis,
|
|
1732
1015
|
selections: built.selections,
|
|
1733
|
-
tombstones
|
|
1734
|
-
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
|
|
1016
|
+
tombstones,
|
|
1017
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
|
|
1018
|
+
...pagedSettingsOf(context)
|
|
1735
1019
|
});
|
|
1736
1020
|
segments.push(...written.segments);
|
|
1737
1021
|
nextSegmentId = written.nextSegmentId;
|
|
@@ -1752,7 +1036,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1752
1036
|
textAnalysis: context.textAnalysis,
|
|
1753
1037
|
segments,
|
|
1754
1038
|
nextSegmentId,
|
|
1755
|
-
compactionThreshold: context.compactionThreshold
|
|
1039
|
+
compactionThreshold: context.compactionThreshold,
|
|
1040
|
+
paged: context.input.metadata.textStorage === "disk"
|
|
1756
1041
|
});
|
|
1757
1042
|
segments = compacted.segments;
|
|
1758
1043
|
nextSegmentId = compacted.nextSegmentId;
|
|
@@ -1815,8 +1100,10 @@ async function writeCheckpointSegments(input) {
|
|
|
1815
1100
|
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
1816
1101
|
const indexName = metadata.indexName;
|
|
1817
1102
|
const manifest = {
|
|
1103
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
1818
1104
|
schema: metadata.schema,
|
|
1819
1105
|
language: metadata.language,
|
|
1106
|
+
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
1820
1107
|
checkpoint: written.checkpoint,
|
|
1821
1108
|
partitions: written.partitions,
|
|
1822
1109
|
vectors
|
|
@@ -1836,4 +1123,4 @@ async function removeCheckpointGarbage(store, garbage) {
|
|
|
1836
1123
|
}
|
|
1837
1124
|
}
|
|
1838
1125
|
|
|
1839
|
-
export {
|
|
1126
|
+
export { SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, durableRecordMissing, encodeMarkerSlot, frameRecord, markerSlotOffset, nextMarkerSlot, nextSegmentKey, optionalIndexUuid, partitionContentsOf, readCommitMarker, readWalSegments, removeCheckpointGarbage, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writeSegmentHeader };
|