@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,233 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
2
|
+
import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-7AUASRE7.mjs';
|
|
3
|
+
import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-UUZCQQG2.mjs';
|
|
4
|
+
import { restoreVectorFields } from './chunk-KEIYVZUX.mjs';
|
|
3
5
|
import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions,
|
|
6
|
-
import {
|
|
6
|
+
import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-IO5F7ZBE.mjs';
|
|
7
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
|
|
8
|
+
import { isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-ZPWS2SKJ.mjs';
|
|
7
9
|
import { generateId } from './chunk-N37VMC62.mjs';
|
|
8
|
-
import { extractVectorFieldsFromSchema, partialFieldsOf
|
|
10
|
+
import { extractVectorFieldsFromSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
|
|
9
11
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
10
12
|
import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
|
|
11
13
|
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
12
14
|
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
13
15
|
import { encode, decode } from '@msgpack/msgpack';
|
|
14
16
|
|
|
15
|
-
// src/persistence/durability/constants.ts
|
|
16
|
-
var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
|
|
17
|
-
var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
|
|
18
|
-
var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
|
|
19
|
-
var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
|
|
20
|
-
var DIRECTORY_LOCK_ATTEMPTS = 3;
|
|
21
|
-
var DIRECTORY_LOCK_SETTLE_MS = 50;
|
|
22
|
-
var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
|
|
23
|
-
var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
|
|
24
|
-
var DEFAULT_COMPACTION_THRESHOLD = 12;
|
|
25
|
-
var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
|
|
26
|
-
var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
|
|
27
|
-
var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
|
|
28
|
-
var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
|
|
29
|
-
var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
|
|
30
|
-
var LOG_RECORDS_PER_CHECKPOINT = 1e5;
|
|
31
|
-
var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
|
|
32
|
-
var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
|
|
33
|
-
var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
|
|
34
|
-
var CHECKPOINT_WORKER_IDLE_MS = 5e3;
|
|
35
|
-
var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
|
|
36
|
-
|
|
37
|
-
// src/persistence/durability/segment/layout.ts
|
|
38
|
-
var SEGMENT_ID_WIDTH = 16;
|
|
39
|
-
function manifestKey(indexName) {
|
|
40
|
-
return `${indexName}/manifest`;
|
|
41
|
-
}
|
|
42
|
-
function snapshotBundleKey(indexName) {
|
|
43
|
-
return `${indexName}/snapshot`;
|
|
44
|
-
}
|
|
45
|
-
function segmentPrefix(indexName, partitionId) {
|
|
46
|
-
return `${indexName}/segments/${partitionId}/`;
|
|
47
|
-
}
|
|
48
|
-
function segmentKey(indexName, partitionId, segmentId) {
|
|
49
|
-
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
50
|
-
}
|
|
51
|
-
function segmentsPrefix(indexName) {
|
|
52
|
-
return `${indexName}/segments/`;
|
|
53
|
-
}
|
|
54
|
-
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
55
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
56
|
-
}
|
|
57
|
-
function requireGeneration(generation) {
|
|
58
|
-
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
59
|
-
throw new NarsilError(
|
|
60
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
61
|
-
`Vector graph generation ${generation} must be a positive integer`,
|
|
62
|
-
{ generation }
|
|
63
|
-
);
|
|
64
|
-
}
|
|
65
|
-
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
66
|
-
requireGeneration(generation);
|
|
67
|
-
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
68
|
-
}
|
|
69
|
-
function formatSegmentId(segmentId) {
|
|
70
|
-
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
71
|
-
throw new NarsilError(
|
|
72
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
73
|
-
`Segment id ${segmentId} must be a non-negative integer`,
|
|
74
|
-
{ segmentId }
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
78
|
-
}
|
|
79
|
-
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
80
|
-
function encodeFieldPath(fieldPath) {
|
|
81
|
-
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
82
|
-
throw new NarsilError(
|
|
83
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
84
|
-
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
85
|
-
{ fieldPath }
|
|
86
|
-
);
|
|
87
|
-
}
|
|
88
|
-
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
89
|
-
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
90
|
-
}
|
|
91
|
-
var SEGMENT_MANIFEST_VERSION = 7;
|
|
92
|
-
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
93
|
-
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
94
|
-
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
95
|
-
var MAX_SEGMENT_ID = 281474976710655;
|
|
96
|
-
function fileRefOnTheWire(ref) {
|
|
97
|
-
return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
|
|
98
|
-
}
|
|
99
|
-
function encodeSegmentManifest(manifest) {
|
|
100
|
-
const payload = encode({
|
|
101
|
-
version: SEGMENT_MANIFEST_VERSION,
|
|
102
|
-
schema: manifest.schema,
|
|
103
|
-
language: manifest.language,
|
|
104
|
-
...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
|
|
105
|
-
checkpoint: manifest.checkpoint.map((c) => ({
|
|
106
|
-
partitionId: c.partitionId,
|
|
107
|
-
lastSeqNo: c.lastSeqNo,
|
|
108
|
-
primaryTerm: c.primaryTerm,
|
|
109
|
-
...c.termHistory === void 0 ? {} : {
|
|
110
|
-
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
111
|
-
}
|
|
112
|
-
})),
|
|
113
|
-
partitions: manifest.partitions.map((p) => ({
|
|
114
|
-
partitionId: p.partitionId,
|
|
115
|
-
nextSegmentId: p.nextSegmentId,
|
|
116
|
-
segments: p.segments.map((s) => ({
|
|
117
|
-
id: s.id,
|
|
118
|
-
key: s.key,
|
|
119
|
-
docCount: s.docCount,
|
|
120
|
-
tombstoneCount: s.tombstoneCount,
|
|
121
|
-
bytes: s.bytes,
|
|
122
|
-
checksum: s.checksum
|
|
123
|
-
}))
|
|
124
|
-
})),
|
|
125
|
-
vectors: manifest.vectors.map((v) => ({
|
|
126
|
-
fieldPath: v.fieldPath,
|
|
127
|
-
nextFileId: v.nextFileId,
|
|
128
|
-
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
129
|
-
graphGeneration: v.graphGeneration,
|
|
130
|
-
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
131
|
-
}))
|
|
132
|
-
});
|
|
133
|
-
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
134
|
-
}
|
|
135
|
-
function manifestReferencedKeys(manifest) {
|
|
136
|
-
const keys = /* @__PURE__ */ new Set();
|
|
137
|
-
for (const partition of manifest.partitions) {
|
|
138
|
-
for (const segment of partition.segments) {
|
|
139
|
-
keys.add(segment.key);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
for (const vector of manifest.vectors) {
|
|
143
|
-
for (const file of vector.files) keys.add(file.key);
|
|
144
|
-
for (const part of vector.graphParts) keys.add(part.key);
|
|
145
|
-
}
|
|
146
|
-
return keys;
|
|
147
|
-
}
|
|
148
|
-
function manifestFileRefs(manifest) {
|
|
149
|
-
const refs = [];
|
|
150
|
-
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
151
|
-
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
152
|
-
return refs;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// src/persistence/durability/segment/store.ts
|
|
156
|
-
function directoryCheckpointStore(directory) {
|
|
157
|
-
const store = {
|
|
158
|
-
read: (key) => directory.read(key),
|
|
159
|
-
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
160
|
-
remove: (key) => directory.remove(key),
|
|
161
|
-
list: (prefix) => directory.list(prefix),
|
|
162
|
-
pathOf: (key) => directory.pathOf(key)
|
|
163
|
-
};
|
|
164
|
-
const byteLength = directory.byteLength;
|
|
165
|
-
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
166
|
-
return store;
|
|
167
|
-
}
|
|
168
|
-
function joined(chunks) {
|
|
169
|
-
if (chunks.length === 1) return chunks[0];
|
|
170
|
-
let total = 0;
|
|
171
|
-
for (const chunk of chunks) total += chunk.byteLength;
|
|
172
|
-
const bytes = new Uint8Array(total);
|
|
173
|
-
let offset = 0;
|
|
174
|
-
for (const chunk of chunks) {
|
|
175
|
-
bytes.set(chunk, offset);
|
|
176
|
-
offset += chunk.byteLength;
|
|
177
|
-
}
|
|
178
|
-
return bytes;
|
|
179
|
-
}
|
|
180
|
-
function adapterFailure(code, verb, key, error) {
|
|
181
|
-
if (error instanceof NarsilError) return error;
|
|
182
|
-
const cause = error instanceof Error ? error.message : String(error);
|
|
183
|
-
return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
|
|
184
|
-
}
|
|
185
|
-
async function guarded(run, code, verb, key) {
|
|
186
|
-
try {
|
|
187
|
-
return await run();
|
|
188
|
-
} catch (error) {
|
|
189
|
-
throw adapterFailure(code, verb, key, error);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
function adapterCheckpointStore(adapter) {
|
|
193
|
-
return {
|
|
194
|
-
read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
|
|
195
|
-
write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
|
|
196
|
-
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
197
|
-
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
async function readListedFile(store, key, expected) {
|
|
201
|
-
const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
|
|
202
|
-
if (length === null) missingFile(expected.what, key);
|
|
203
|
-
if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
|
|
204
|
-
const bytes = await store.read(key);
|
|
205
|
-
if (bytes === null) missingFile(expected.what, key);
|
|
206
|
-
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
207
|
-
return bytes;
|
|
208
|
-
}
|
|
209
|
-
function missingFile(what, key) {
|
|
210
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
|
|
211
|
-
key
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
|
-
function requireListedLength(what, key, found, listed) {
|
|
215
|
-
if (found === listed) return;
|
|
216
|
-
throw new NarsilError(
|
|
217
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
218
|
-
`The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
|
|
219
|
-
{ key, bytes: found, listed }
|
|
220
|
-
);
|
|
221
|
-
}
|
|
222
|
-
function requireListedChecksum(what, key, found, listed) {
|
|
223
|
-
if (found === listed) return;
|
|
224
|
-
throw new NarsilError(
|
|
225
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
226
|
-
`The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
|
|
227
|
-
{ key, checksum: found, listed }
|
|
228
|
-
);
|
|
229
|
-
}
|
|
230
|
-
|
|
231
17
|
// src/persistence/durability/segment/captured-partition.ts
|
|
232
18
|
function keepMaskOf(segment) {
|
|
233
19
|
const keep = new Uint8Array(segment.arrays.documentCount);
|
|
@@ -678,10 +464,10 @@ var RECORD_LENGTH_SIZE = 4;
|
|
|
678
464
|
var FRAME_CRC_SIZE = 4;
|
|
679
465
|
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
680
466
|
function writeSegmentHeader() {
|
|
681
|
-
const
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
return
|
|
467
|
+
const header = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
468
|
+
header.set(WAL_MAGIC, 0);
|
|
469
|
+
header[4] = WAL_FORMAT_VERSION & 255;
|
|
470
|
+
return header;
|
|
685
471
|
}
|
|
686
472
|
function frameRecord(entry) {
|
|
687
473
|
const payload = encode(entry);
|
|
@@ -722,9 +508,9 @@ function corrupt(message, details) {
|
|
|
722
508
|
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
723
509
|
}
|
|
724
510
|
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
725
|
-
const
|
|
726
|
-
if (!
|
|
727
|
-
corrupt(`WAL segment header invalid in durable region: ${
|
|
511
|
+
const header = checkSegmentHeader(data);
|
|
512
|
+
if (!header.ok) {
|
|
513
|
+
corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
|
|
728
514
|
}
|
|
729
515
|
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
730
516
|
corrupt("Commit marker durable byte length is outside the segment", {
|
|
@@ -823,6 +609,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
|
|
|
823
609
|
function walPrefix(indexName, partitionId) {
|
|
824
610
|
return `${indexName}/wal/${partitionId}/`;
|
|
825
611
|
}
|
|
612
|
+
function nextSegmentKey(indexName, partitionId) {
|
|
613
|
+
return `${walPrefix(indexName, partitionId)}next`;
|
|
614
|
+
}
|
|
826
615
|
function segmentStartSeqNo(key, prefix) {
|
|
827
616
|
if (!key.startsWith(prefix)) {
|
|
828
617
|
return null;
|
|
@@ -867,14 +656,14 @@ async function* readSegmentsInOrder(directory, wal) {
|
|
|
867
656
|
continue;
|
|
868
657
|
}
|
|
869
658
|
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
870
|
-
const
|
|
871
|
-
if (!
|
|
659
|
+
const header = checkSegmentHeader(bytes);
|
|
660
|
+
if (!header.ok) {
|
|
872
661
|
throw new NarsilError(
|
|
873
662
|
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
874
|
-
`Sealed WAL segment header invalid: ${
|
|
663
|
+
`Sealed WAL segment header invalid: ${header.reason}`,
|
|
875
664
|
{
|
|
876
665
|
key,
|
|
877
|
-
reason:
|
|
666
|
+
reason: header.reason
|
|
878
667
|
}
|
|
879
668
|
);
|
|
880
669
|
}
|
|
@@ -1004,569 +793,23 @@ async function buildSegmentFromEntries(input) {
|
|
|
1004
793
|
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
|
|
1005
794
|
return built;
|
|
1006
795
|
}
|
|
1007
|
-
var FIXMAP = 128;
|
|
1008
|
-
var MAP16 = 222;
|
|
1009
|
-
var MAP32 = 223;
|
|
1010
|
-
var FIXARRAY = 144;
|
|
1011
|
-
var ARRAY16 = 220;
|
|
1012
|
-
var ARRAY32 = 221;
|
|
1013
|
-
var BIN8 = 196;
|
|
1014
|
-
var BIN16 = 197;
|
|
1015
|
-
var BIN32 = 198;
|
|
1016
|
-
var FIX_COUNT_LIMIT = 16;
|
|
1017
|
-
var UINT8_LIMIT = 256;
|
|
1018
|
-
var UINT16_LIMIT = 65536;
|
|
1019
|
-
function header(fixed, sixteen, thirtyTwo, count) {
|
|
1020
|
-
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
1021
|
-
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
1022
|
-
const bytes = new Uint8Array(5);
|
|
1023
|
-
bytes[0] = thirtyTwo;
|
|
1024
|
-
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
1025
|
-
return bytes;
|
|
1026
|
-
}
|
|
1027
|
-
function binHeader(byteLength) {
|
|
1028
|
-
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
|
|
1029
|
-
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1030
|
-
const bytes = new Uint8Array(5);
|
|
1031
|
-
bytes[0] = BIN32;
|
|
1032
|
-
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
1033
|
-
return bytes;
|
|
1034
|
-
}
|
|
1035
|
-
var MessagePackChunks = class {
|
|
1036
|
-
chunks = [];
|
|
1037
|
-
byteLength = 0;
|
|
1038
|
-
push(chunk) {
|
|
1039
|
-
if (chunk.byteLength === 0) return;
|
|
1040
|
-
this.chunks.push(chunk);
|
|
1041
|
-
this.byteLength += chunk.byteLength;
|
|
1042
|
-
}
|
|
1043
|
-
map(entries) {
|
|
1044
|
-
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
1045
|
-
return this;
|
|
1046
|
-
}
|
|
1047
|
-
array(items) {
|
|
1048
|
-
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1049
|
-
return this;
|
|
1050
|
-
}
|
|
1051
|
-
value(value) {
|
|
1052
|
-
this.push(encode(value));
|
|
1053
|
-
return this;
|
|
1054
|
-
}
|
|
1055
|
-
key(name) {
|
|
1056
|
-
return this.value(name);
|
|
1057
|
-
}
|
|
1058
|
-
binary(bytes) {
|
|
1059
|
-
this.push(binHeader(bytes.byteLength));
|
|
1060
|
-
this.push(bytes);
|
|
1061
|
-
return this;
|
|
1062
|
-
}
|
|
1063
|
-
binaryOrNil(bytes) {
|
|
1064
|
-
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1065
|
-
}
|
|
1066
|
-
};
|
|
1067
|
-
|
|
1068
|
-
// src/serialization/keyword-segment/token-order.ts
|
|
1069
|
-
function slotsAreInTokenOrder(payloadSlots) {
|
|
1070
|
-
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1071
|
-
if (payloadSlots[at] !== at) return false;
|
|
1072
|
-
}
|
|
1073
|
-
return true;
|
|
1074
|
-
}
|
|
1075
|
-
function reorderedPositions(arrays, postingOffsets) {
|
|
1076
|
-
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1077
|
-
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1078
|
-
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1079
|
-
const offsets = new Uint32Array(postings + 1);
|
|
1080
|
-
const values = new Uint32Array(positionValues.length);
|
|
1081
|
-
let cursor = 0;
|
|
1082
|
-
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1083
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1084
|
-
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1085
|
-
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1086
|
-
offsets[to] = cursor;
|
|
1087
|
-
const start = positionOffsets[from];
|
|
1088
|
-
const end = positionOffsets[from + 1];
|
|
1089
|
-
values.set(positionValues.subarray(start, end), cursor);
|
|
1090
|
-
cursor += end - start;
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
offsets[postings] = cursor;
|
|
1094
|
-
return { positionOffsets: offsets, positionValues: values };
|
|
1095
|
-
}
|
|
1096
|
-
function postingsInTokenOrder(arrays) {
|
|
1097
|
-
const { tokenTable } = arrays;
|
|
1098
|
-
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1099
|
-
const tokens = tokenTable.payloadSlots.length;
|
|
1100
|
-
const postingOffsets = new Uint32Array(tokens + 1);
|
|
1101
|
-
for (let token = 0; token < tokens; token++) {
|
|
1102
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1103
|
-
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
1104
|
-
}
|
|
1105
|
-
const postings = postingOffsets[tokens];
|
|
1106
|
-
const postingDocIds = new Uint32Array(postings);
|
|
1107
|
-
const postingFrequencies = new Uint16Array(postings);
|
|
1108
|
-
const postingFieldIndices = new Uint16Array(postings);
|
|
1109
|
-
for (let token = 0; token < tokens; token++) {
|
|
1110
|
-
const slot = tokenTable.payloadSlots[token];
|
|
1111
|
-
const start = arrays.postingOffsets[slot];
|
|
1112
|
-
const end = arrays.postingOffsets[slot + 1];
|
|
1113
|
-
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
1114
|
-
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
1115
|
-
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
1116
|
-
}
|
|
1117
|
-
return {
|
|
1118
|
-
postingOffsets,
|
|
1119
|
-
postingDocIds,
|
|
1120
|
-
postingFrequencies,
|
|
1121
|
-
postingFieldIndices,
|
|
1122
|
-
...reorderedPositions(arrays, postingOffsets)
|
|
1123
|
-
};
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
// src/serialization/keyword-segment/encode.ts
|
|
1127
|
-
var TOP_LEVEL_ENTRIES = 28;
|
|
1128
|
-
function writeFieldLengths(out, arrays) {
|
|
1129
|
-
out.array(arrays.fieldLengthNames.length);
|
|
1130
|
-
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
1131
|
-
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
1132
|
-
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
function writeValueColumns(out, columns) {
|
|
1136
|
-
out.array(columns.length);
|
|
1137
|
-
for (const column of columns) {
|
|
1138
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1139
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1140
|
-
out.key("values").binary(littleEndianBytes(column.values));
|
|
1141
|
-
}
|
|
1142
|
-
}
|
|
1143
|
-
function writeBooleanColumns(out, columns) {
|
|
1144
|
-
out.array(columns.length);
|
|
1145
|
-
for (const column of columns) {
|
|
1146
|
-
out.map(3).key("field").value(column.fieldPath);
|
|
1147
|
-
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
1148
|
-
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
function writeEnumColumns(out, columns) {
|
|
1152
|
-
out.array(columns.length);
|
|
1153
|
-
for (const column of columns) {
|
|
1154
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1155
|
-
out.key("values").value(column.values);
|
|
1156
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1157
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
function writeGeopointColumns(out, columns) {
|
|
1161
|
-
out.array(columns.length);
|
|
1162
|
-
for (const column of columns) {
|
|
1163
|
-
out.map(4).key("field").value(column.fieldPath);
|
|
1164
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1165
|
-
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
1166
|
-
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1169
|
-
function writePatternColumns(out, columns) {
|
|
1170
|
-
out.array(columns.length);
|
|
1171
|
-
for (const column of columns) {
|
|
1172
|
-
out.map(5).key("field").value(column.fieldPath);
|
|
1173
|
-
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
1174
|
-
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
1175
|
-
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
1176
|
-
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
1177
|
-
}
|
|
1178
|
-
}
|
|
1179
|
-
function writeSurfaceForms(out, arrays) {
|
|
1180
|
-
const table = arrays.surfaceTable;
|
|
1181
|
-
if (table === null || table.counts.length === 0) {
|
|
1182
|
-
out.value(null);
|
|
1183
|
-
return;
|
|
1184
|
-
}
|
|
1185
|
-
out.map(3).key("text").binary(table.blob);
|
|
1186
|
-
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
1187
|
-
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
1188
|
-
}
|
|
1189
|
-
function writePartialWords(out, arrays) {
|
|
1190
|
-
const table = arrays.partialWords;
|
|
1191
|
-
if (table === null) {
|
|
1192
|
-
out.value(null);
|
|
1193
|
-
return;
|
|
1194
|
-
}
|
|
1195
|
-
out.map(8).key("fields").value(table.fields);
|
|
1196
|
-
out.key("text").binary(table.spellings);
|
|
1197
|
-
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
1198
|
-
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
1199
|
-
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
1200
|
-
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
1201
|
-
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
1202
|
-
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
1203
|
-
}
|
|
1204
|
-
function encodeKeywordSegment(contents) {
|
|
1205
|
-
const { arrays } = contents;
|
|
1206
|
-
const postings = postingsInTokenOrder(arrays);
|
|
1207
|
-
const out = new MessagePackChunks();
|
|
1208
|
-
const { writeRecords } = contents;
|
|
1209
|
-
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
1210
|
-
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
1211
|
-
out.key("schema").value(contents.schema);
|
|
1212
|
-
out.key("tombstones").value(contents.tombstones);
|
|
1213
|
-
out.key("documentCount").value(arrays.documentCount);
|
|
1214
|
-
out.key("ids").binary(arrays.idTable.blob);
|
|
1215
|
-
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
1216
|
-
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
1217
|
-
out.key("documents").binary(arrays.documentTable.blob);
|
|
1218
|
-
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
1219
|
-
out.key("fieldNames").value(arrays.fieldNames);
|
|
1220
|
-
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
1221
|
-
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
1222
|
-
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
1223
|
-
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
1224
|
-
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
1225
|
-
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
1226
|
-
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
1227
|
-
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
1228
|
-
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
1229
|
-
out.key("fieldLengths");
|
|
1230
|
-
writeFieldLengths(out, arrays);
|
|
1231
|
-
out.key("numeric");
|
|
1232
|
-
writeValueColumns(out, arrays.numeric);
|
|
1233
|
-
out.key("sizes");
|
|
1234
|
-
writeValueColumns(out, arrays.sizes);
|
|
1235
|
-
out.key("boolean");
|
|
1236
|
-
writeBooleanColumns(out, arrays.boolean);
|
|
1237
|
-
out.key("enum");
|
|
1238
|
-
writeEnumColumns(out, arrays.enums);
|
|
1239
|
-
out.key("geopoint");
|
|
1240
|
-
writeGeopointColumns(out, arrays.geo);
|
|
1241
|
-
out.key("pattern");
|
|
1242
|
-
writePatternColumns(out, arrays.pattern);
|
|
1243
|
-
out.key("surfaceForms");
|
|
1244
|
-
writeSurfaceForms(out, arrays);
|
|
1245
|
-
out.key("partialWords");
|
|
1246
|
-
writePartialWords(out, arrays);
|
|
1247
|
-
if (writeRecords !== void 0) {
|
|
1248
|
-
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
1249
|
-
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
1250
|
-
}
|
|
1251
|
-
return out;
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
// src/persistence/durability/segment/segment-file.ts
|
|
1255
|
-
function sharesMemoryWithWorkers() {
|
|
1256
|
-
return typeof SharedArrayBuffer === "function";
|
|
1257
|
-
}
|
|
1258
|
-
async function readSegmentFile(store, ref) {
|
|
1259
|
-
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
1260
|
-
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
1261
|
-
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
1262
|
-
what: `segment "${ref.key}"`,
|
|
1263
|
-
allowsCompression: true
|
|
1264
|
-
});
|
|
1265
|
-
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
1266
|
-
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
1267
|
-
}
|
|
1268
|
-
var ORDINAL_BYTES = 4;
|
|
1269
|
-
var POSTING_BYTES = 8;
|
|
1270
|
-
var VALUE_ENTRY_BYTES = 12;
|
|
1271
|
-
var GEOPOINT_ENTRY_BYTES = 20;
|
|
1272
|
-
var TOKEN_ENTRY_BYTES = 12;
|
|
1273
|
-
var STRING_HEADER_BYTES = 5;
|
|
1274
|
-
var encoder = new TextEncoder();
|
|
1275
|
-
function documentBytes(segment, ordinal) {
|
|
1276
|
-
const table = segment.arrays.documentTable;
|
|
1277
|
-
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
1278
|
-
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
1279
|
-
}
|
|
1280
|
-
function addPerEntry(costs, docIds, bytes) {
|
|
1281
|
-
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
1282
|
-
}
|
|
1283
|
-
function chargePostings(segment, costs, chargesTokens) {
|
|
1284
|
-
const arrays = segment.arrays;
|
|
1285
|
-
const positions = arrays.positionOffsets;
|
|
1286
|
-
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
1287
|
-
let bytes = POSTING_BYTES;
|
|
1288
|
-
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
1289
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
1290
|
-
}
|
|
1291
|
-
if (!chargesTokens) return;
|
|
1292
|
-
const table = arrays.tokenTable;
|
|
1293
|
-
for (let at = 0; at < table.size; at++) {
|
|
1294
|
-
const slot = table.payloadSlot(at);
|
|
1295
|
-
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
1296
|
-
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
1297
|
-
costs[arrays.postingDocIds[p]] += bytes;
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
function chargeColumns(segment, costs) {
|
|
1302
|
-
const arrays = segment.arrays;
|
|
1303
|
-
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
1304
|
-
for (const column of arrays.boolean) {
|
|
1305
|
-
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
1306
|
-
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
1307
|
-
}
|
|
1308
|
-
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1309
|
-
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
1310
|
-
for (const column of arrays.pattern) {
|
|
1311
|
-
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
1312
|
-
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
1313
|
-
}
|
|
1314
|
-
}
|
|
1315
|
-
function stringListBytes(values) {
|
|
1316
|
-
let bytes = STRING_HEADER_BYTES;
|
|
1317
|
-
for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
|
|
1318
|
-
return bytes;
|
|
1319
|
-
}
|
|
1320
|
-
function fixedBytesOf(segment, chargesTokens) {
|
|
1321
|
-
const arrays = segment.arrays;
|
|
1322
|
-
let bytes = stringListBytes(arrays.fieldNames);
|
|
1323
|
-
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
1324
|
-
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
1325
|
-
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
1326
|
-
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
1327
|
-
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
1328
|
-
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
1329
|
-
return bytes + segment.partialWords.bytes();
|
|
1330
|
-
}
|
|
1331
|
-
function keptDocumentsOf(selection, chargesTokens) {
|
|
1332
|
-
const { segment } = selection;
|
|
1333
|
-
const count = segment.arrays.documentCount;
|
|
1334
|
-
const costs = new Float64Array(count);
|
|
1335
|
-
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
1336
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1337
|
-
costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
1338
|
-
}
|
|
1339
|
-
chargePostings(segment, costs, chargesTokens);
|
|
1340
|
-
chargeColumns(segment, costs);
|
|
1341
|
-
const ordinals = [];
|
|
1342
|
-
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
1343
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1344
|
-
costs[ordinal] += documentBytes(segment, ordinal);
|
|
1345
|
-
ordinals.push(ordinal);
|
|
1346
|
-
}
|
|
1347
|
-
return { segment, ordinals, costs };
|
|
1348
|
-
}
|
|
1349
|
-
function selectionOf(segment, ordinals) {
|
|
1350
|
-
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
1351
|
-
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
1352
|
-
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
1353
|
-
}
|
|
1354
|
-
function tooLargeForOneFile(cost, budget) {
|
|
1355
|
-
throw new NarsilError(
|
|
1356
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1357
|
-
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
1358
|
-
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
1359
|
-
);
|
|
1360
|
-
}
|
|
1361
|
-
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
1362
|
-
let tokenBytes = 0;
|
|
1363
|
-
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
1364
|
-
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
1365
|
-
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
1366
|
-
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
1367
|
-
const budget = MAX_FILE_BYTES - fixed;
|
|
1368
|
-
const groups = [];
|
|
1369
|
-
let current = [];
|
|
1370
|
-
let currentBytes = 0;
|
|
1371
|
-
let currentCount = 0;
|
|
1372
|
-
const flush = () => {
|
|
1373
|
-
if (currentCount === 0) return;
|
|
1374
|
-
groups.push({
|
|
1375
|
-
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
1376
|
-
documentCount: currentCount
|
|
1377
|
-
});
|
|
1378
|
-
current = [];
|
|
1379
|
-
currentBytes = 0;
|
|
1380
|
-
currentCount = 0;
|
|
1381
|
-
};
|
|
1382
|
-
for (const selection of selections) {
|
|
1383
|
-
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
1384
|
-
let entry = null;
|
|
1385
|
-
for (const ordinal of kept.ordinals) {
|
|
1386
|
-
const cost = kept.costs[ordinal];
|
|
1387
|
-
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
1388
|
-
if (currentBytes + cost > budget) {
|
|
1389
|
-
flush();
|
|
1390
|
-
entry = null;
|
|
1391
|
-
}
|
|
1392
|
-
if (entry === null) {
|
|
1393
|
-
entry = { segment: kept.segment, ordinals: [] };
|
|
1394
|
-
current.push(entry);
|
|
1395
|
-
}
|
|
1396
|
-
entry.ordinals.push(ordinal);
|
|
1397
|
-
currentBytes += cost;
|
|
1398
|
-
currentCount += 1;
|
|
1399
|
-
}
|
|
1400
|
-
}
|
|
1401
|
-
flush();
|
|
1402
|
-
return groups;
|
|
1403
|
-
}
|
|
1404
|
-
function halvesOf(group) {
|
|
1405
|
-
const flat = [];
|
|
1406
|
-
for (const selection of group.selections) {
|
|
1407
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1408
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1409
|
-
flat.push({ segment: selection.segment, ordinal });
|
|
1410
|
-
}
|
|
1411
|
-
}
|
|
1412
|
-
const middle = Math.ceil(flat.length / 2);
|
|
1413
|
-
const groupOf = (entries) => {
|
|
1414
|
-
const bySegment = /* @__PURE__ */ new Map();
|
|
1415
|
-
for (const { segment, ordinal } of entries) {
|
|
1416
|
-
let ordinals = bySegment.get(segment);
|
|
1417
|
-
if (ordinals === void 0) {
|
|
1418
|
-
ordinals = [];
|
|
1419
|
-
bySegment.set(segment, ordinals);
|
|
1420
|
-
}
|
|
1421
|
-
ordinals.push(ordinal);
|
|
1422
|
-
}
|
|
1423
|
-
return {
|
|
1424
|
-
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
1425
|
-
documentCount: entries.length
|
|
1426
|
-
};
|
|
1427
|
-
};
|
|
1428
|
-
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
// src/persistence/durability/segment/segment-write.ts
|
|
1432
|
-
function emptyPayload() {
|
|
1433
|
-
return {
|
|
1434
|
-
documentCount: 0,
|
|
1435
|
-
docIds: [],
|
|
1436
|
-
fieldNames: [],
|
|
1437
|
-
tokens: [],
|
|
1438
|
-
postingOffsets: new Uint32Array(1),
|
|
1439
|
-
postingDocIds: new Uint32Array(0),
|
|
1440
|
-
postingFrequencies: new Uint16Array(0),
|
|
1441
|
-
postingFieldIndices: new Uint16Array(0),
|
|
1442
|
-
positionOffsets: null,
|
|
1443
|
-
positionValues: null,
|
|
1444
|
-
fieldLengthNames: [],
|
|
1445
|
-
fieldLengthColumns: [],
|
|
1446
|
-
totalFieldLengths: {},
|
|
1447
|
-
docFrequencies: {},
|
|
1448
|
-
surfaceForms: null,
|
|
1449
|
-
partialSpellings: null,
|
|
1450
|
-
numeric: [],
|
|
1451
|
-
boolean: [],
|
|
1452
|
-
enums: [],
|
|
1453
|
-
geo: [],
|
|
1454
|
-
pattern: [],
|
|
1455
|
-
sizes: []
|
|
1456
|
-
};
|
|
1457
|
-
}
|
|
1458
|
-
function noSpellingsOf(analysis) {
|
|
1459
|
-
if (analysis === null || analysis.fields.length === 0) return null;
|
|
1460
|
-
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
1461
|
-
}
|
|
1462
|
-
function writeRecordsOfGroup(group, records, count) {
|
|
1463
|
-
const seqNos = new Float64Array(count);
|
|
1464
|
-
const primaryTerms = new Float64Array(count);
|
|
1465
|
-
let at = 0;
|
|
1466
|
-
for (const selection of group?.selections ?? []) {
|
|
1467
|
-
const source = records.get(selection.segment);
|
|
1468
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1469
|
-
if (!selectionKeeps(selection, ordinal)) continue;
|
|
1470
|
-
if (source !== void 0) {
|
|
1471
|
-
seqNos[at] = source.seqNos[ordinal];
|
|
1472
|
-
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
1473
|
-
}
|
|
1474
|
-
at += 1;
|
|
1475
|
-
}
|
|
1476
|
-
}
|
|
1477
|
-
return { seqNos, primaryTerms };
|
|
1478
|
-
}
|
|
1479
|
-
function withWriteRecords(contents, group, request) {
|
|
1480
|
-
if (request.writeRecords === void 0) return contents;
|
|
1481
|
-
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
1482
|
-
return { ...contents, writeRecords };
|
|
1483
|
-
}
|
|
1484
|
-
function contentsOfGroup(group, request, tombstones) {
|
|
1485
|
-
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
1486
|
-
if (group === null) {
|
|
1487
|
-
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
1488
|
-
return {
|
|
1489
|
-
schema,
|
|
1490
|
-
tombstones,
|
|
1491
|
-
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
1492
|
-
};
|
|
1493
|
-
}
|
|
1494
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
1495
|
-
return {
|
|
1496
|
-
schema,
|
|
1497
|
-
tombstones,
|
|
1498
|
-
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
1499
|
-
};
|
|
1500
|
-
}
|
|
1501
|
-
function requireRoomForAnotherSegment(state) {
|
|
1502
|
-
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
1503
|
-
throw new NarsilError(
|
|
1504
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1505
|
-
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
1506
|
-
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
1507
|
-
);
|
|
1508
|
-
}
|
|
1509
|
-
async function writeGroup(state, group) {
|
|
1510
|
-
const { request } = state;
|
|
1511
|
-
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
1512
|
-
const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
1513
|
-
const encoded = encodeKeywordSegment(contents);
|
|
1514
|
-
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
1515
|
-
const [first, second] = halvesOf(group);
|
|
1516
|
-
await writeGroup(state, first);
|
|
1517
|
-
await writeGroup(state, second);
|
|
1518
|
-
return;
|
|
1519
|
-
}
|
|
1520
|
-
requireRoomForAnotherSegment(state);
|
|
1521
|
-
const id = state.nextSegmentId;
|
|
1522
|
-
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
1523
|
-
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
1524
|
-
await request.store.write(key, packed.chunks);
|
|
1525
|
-
state.segments.push({
|
|
1526
|
-
id,
|
|
1527
|
-
key,
|
|
1528
|
-
bytes: packed.byteLength,
|
|
1529
|
-
checksum: packed.checksum,
|
|
1530
|
-
docCount: contents.arrays.documentCount,
|
|
1531
|
-
tombstoneCount: tombstones.length
|
|
1532
|
-
});
|
|
1533
|
-
state.nextSegmentId = id + 1;
|
|
1534
|
-
state.tombstonesWritten = true;
|
|
1535
|
-
}
|
|
1536
|
-
function keptDocumentCount(selections) {
|
|
1537
|
-
let count = 0;
|
|
1538
|
-
for (const selection of selections) {
|
|
1539
|
-
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
1540
|
-
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
return count;
|
|
1544
|
-
}
|
|
1545
|
-
async function writeSegmentsOf(request) {
|
|
1546
|
-
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
1547
|
-
const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
1548
|
-
for (const group of groups) {
|
|
1549
|
-
if (group.documentCount > 0) await writeGroup(state, group);
|
|
1550
|
-
}
|
|
1551
|
-
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
1552
|
-
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
1553
|
-
}
|
|
1554
796
|
|
|
1555
797
|
// src/persistence/durability/segment/compaction.ts
|
|
1556
|
-
function windowFits(segments, start, windowLength) {
|
|
798
|
+
function windowFits(segments, start, windowLength, paged) {
|
|
1557
799
|
let documents = 0;
|
|
1558
800
|
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
1559
801
|
for (let at = start; at < start + windowLength; at++) {
|
|
1560
802
|
documents += segments[at].docCount;
|
|
1561
803
|
bytes += segments[at].bytes;
|
|
1562
804
|
}
|
|
805
|
+
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
1563
806
|
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
1564
807
|
}
|
|
1565
|
-
function smallestFittingWindow(segments, windowLength) {
|
|
808
|
+
function smallestFittingWindow(segments, windowLength, paged) {
|
|
1566
809
|
let best = null;
|
|
1567
810
|
let bestBytes = Number.POSITIVE_INFINITY;
|
|
1568
811
|
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
1569
|
-
if (!windowFits(segments, start, windowLength)) continue;
|
|
812
|
+
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
1570
813
|
let bytes = 0;
|
|
1571
814
|
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
1572
815
|
if (bytes < bestBytes) {
|
|
@@ -1576,9 +819,9 @@ function smallestFittingWindow(segments, windowLength) {
|
|
|
1576
819
|
}
|
|
1577
820
|
return best;
|
|
1578
821
|
}
|
|
1579
|
-
function chooseWindow(segments, wantedLength) {
|
|
822
|
+
function chooseWindow(segments, wantedLength, paged) {
|
|
1580
823
|
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
1581
|
-
const window = smallestFittingWindow(segments, windowLength);
|
|
824
|
+
const window = smallestFittingWindow(segments, windowLength, paged);
|
|
1582
825
|
if (window !== null) return window;
|
|
1583
826
|
}
|
|
1584
827
|
return null;
|
|
@@ -1619,11 +862,39 @@ async function compactPartitionSegments(input) {
|
|
|
1619
862
|
if (segments.length <= compactionThreshold) {
|
|
1620
863
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1621
864
|
}
|
|
1622
|
-
const
|
|
865
|
+
const paged = input.paged === true;
|
|
866
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
1623
867
|
if (window === null) {
|
|
1624
868
|
return { segments, nextSegmentId: input.nextSegmentId };
|
|
1625
869
|
}
|
|
1626
870
|
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
871
|
+
const replacing = (written2) => ({
|
|
872
|
+
segments: [
|
|
873
|
+
...segments.slice(0, window.start),
|
|
874
|
+
...written2.segments,
|
|
875
|
+
...segments.slice(window.start + window.windowLength)
|
|
876
|
+
],
|
|
877
|
+
nextSegmentId: written2.nextSegmentId
|
|
878
|
+
});
|
|
879
|
+
if (paged) {
|
|
880
|
+
const { directory } = input.store;
|
|
881
|
+
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
882
|
+
const { mergePagedWindow } = await import('./module-IYPLWHGC.mjs');
|
|
883
|
+
return replacing(
|
|
884
|
+
await mergePagedWindow({
|
|
885
|
+
store: input.store,
|
|
886
|
+
directory,
|
|
887
|
+
indexName: input.indexName,
|
|
888
|
+
partitionId: input.partitionId,
|
|
889
|
+
schema: input.schema,
|
|
890
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
891
|
+
textAnalysis: input.textAnalysis,
|
|
892
|
+
nextSegmentId: input.nextSegmentId,
|
|
893
|
+
refs: windowRefs,
|
|
894
|
+
dropsEveryTombstone: window.start === 0
|
|
895
|
+
})
|
|
896
|
+
);
|
|
897
|
+
}
|
|
1627
898
|
const contents = [];
|
|
1628
899
|
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
1629
900
|
const frozen = contents.map(frozenOf);
|
|
@@ -1642,14 +913,7 @@ async function compactPartitionSegments(input) {
|
|
|
1642
913
|
mergesIntoOneFile: true,
|
|
1643
914
|
...writeRecords === void 0 ? {} : { writeRecords }
|
|
1644
915
|
});
|
|
1645
|
-
return
|
|
1646
|
-
segments: [
|
|
1647
|
-
...segments.slice(0, window.start),
|
|
1648
|
-
...written.segments,
|
|
1649
|
-
...segments.slice(window.start + window.windowLength)
|
|
1650
|
-
],
|
|
1651
|
-
nextSegmentId: written.nextSegmentId
|
|
1652
|
-
};
|
|
916
|
+
return replacing(written);
|
|
1653
917
|
}
|
|
1654
918
|
|
|
1655
919
|
// src/persistence/durability/segment/write.ts
|
|
@@ -1696,6 +960,21 @@ function wholeContentsOf(input, partitionId) {
|
|
|
1696
960
|
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
1697
961
|
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
1698
962
|
}
|
|
963
|
+
function pagedSettingsOf(context) {
|
|
964
|
+
if (context.input.metadata.textStorage !== "disk") return {};
|
|
965
|
+
return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
|
|
966
|
+
}
|
|
967
|
+
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
968
|
+
const { directory } = context.input.store;
|
|
969
|
+
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
970
|
+
const { pagedTombstonesFor } = await import('./module-IYPLWHGC.mjs');
|
|
971
|
+
return pagedTombstonesFor(
|
|
972
|
+
directory,
|
|
973
|
+
older,
|
|
974
|
+
selections.map((selection) => selection.segment),
|
|
975
|
+
deleted
|
|
976
|
+
);
|
|
977
|
+
}
|
|
1699
978
|
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
1700
979
|
const written = await writeSegmentsOf({
|
|
1701
980
|
store: context.input.store,
|
|
@@ -1707,7 +986,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
1707
986
|
textAnalysis: context.textAnalysis,
|
|
1708
987
|
selections: whole.selections,
|
|
1709
988
|
tombstones: [],
|
|
1710
|
-
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
|
|
989
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
|
|
990
|
+
...pagedSettingsOf(context)
|
|
1711
991
|
});
|
|
1712
992
|
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
1713
993
|
}
|
|
@@ -1723,6 +1003,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1723
1003
|
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
1724
1004
|
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
1725
1005
|
if (built !== null) {
|
|
1006
|
+
const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
|
|
1726
1007
|
const written = await writeSegmentsOf({
|
|
1727
1008
|
store: context.input.store,
|
|
1728
1009
|
indexName: context.indexName,
|
|
@@ -1732,8 +1013,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1732
1013
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
1733
1014
|
textAnalysis: context.textAnalysis,
|
|
1734
1015
|
selections: built.selections,
|
|
1735
|
-
tombstones
|
|
1736
|
-
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
|
|
1016
|
+
tombstones,
|
|
1017
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
|
|
1018
|
+
...pagedSettingsOf(context)
|
|
1737
1019
|
});
|
|
1738
1020
|
segments.push(...written.segments);
|
|
1739
1021
|
nextSegmentId = written.nextSegmentId;
|
|
@@ -1754,7 +1036,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
1754
1036
|
textAnalysis: context.textAnalysis,
|
|
1755
1037
|
segments,
|
|
1756
1038
|
nextSegmentId,
|
|
1757
|
-
compactionThreshold: context.compactionThreshold
|
|
1039
|
+
compactionThreshold: context.compactionThreshold,
|
|
1040
|
+
paged: context.input.metadata.textStorage === "disk"
|
|
1758
1041
|
});
|
|
1759
1042
|
segments = compacted.segments;
|
|
1760
1043
|
nextSegmentId = compacted.nextSegmentId;
|
|
@@ -1817,6 +1100,7 @@ async function writeCheckpointSegments(input) {
|
|
|
1817
1100
|
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
1818
1101
|
const indexName = metadata.indexName;
|
|
1819
1102
|
const manifest = {
|
|
1103
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
1820
1104
|
schema: metadata.schema,
|
|
1821
1105
|
language: metadata.language,
|
|
1822
1106
|
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
@@ -1839,4 +1123,4 @@ async function removeCheckpointGarbage(store, garbage) {
|
|
|
1839
1123
|
}
|
|
1840
1124
|
}
|
|
1841
1125
|
|
|
1842
|
-
export {
|
|
1126
|
+
export { SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, durableRecordMissing, encodeMarkerSlot, frameRecord, markerSlotOffset, nextMarkerSlot, nextSegmentKey, optionalIndexUuid, partitionContentsOf, readCommitMarker, readWalSegments, removeCheckpointGarbage, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writeSegmentHeader };
|