@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes,
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, buildPartialWordTableFrom, createSurfaceRegistry, encodePartialSpellingsOf, recordValueSize, patternIndexOf, encodeFieldIndexes, countPartialSpelling, ARRAY_POSITION_GAP, documentWithOwnBuffers, readProjectedPath, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createGeoIndex, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes, createCompositePartition } from './chunk-ZPWS2SKJ.mjs';
|
|
2
2
|
import { generateId } from './chunk-N37VMC62.mjs';
|
|
3
3
|
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
4
4
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
|
|
5
|
-
import {
|
|
6
|
-
import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
5
|
+
import { getNestedValue, tokenizeOptions, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-DQNKVOJ2.mjs';
|
|
6
|
+
import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
8
|
import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
|
|
9
9
|
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-O4LJQI4G.mjs';
|
|
10
10
|
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
11
|
-
import {
|
|
11
|
+
import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder } from './chunk-NPNK57CA.mjs';
|
|
12
12
|
import { SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-4UMZ7KQL.mjs';
|
|
13
13
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
14
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
14
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
|
|
15
15
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
16
16
|
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
17
17
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
@@ -114,15 +114,19 @@ function createCapacityChecks(source) {
|
|
|
114
114
|
}
|
|
115
115
|
|
|
116
116
|
// src/partitioning/manager/doc-locations.ts
|
|
117
|
+
function documentIdsHeldInMemory(partition) {
|
|
118
|
+
return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
|
|
119
|
+
}
|
|
117
120
|
function rebuildDocLocations(locations, partitions) {
|
|
118
121
|
locations.clear();
|
|
119
122
|
for (let i = 0; i < partitions.length; i++) {
|
|
120
|
-
for (const docId of partitions[i]
|
|
123
|
+
for (const docId of documentIdsHeldInMemory(partitions[i])) {
|
|
121
124
|
locations.set(docId, i);
|
|
122
125
|
}
|
|
123
126
|
}
|
|
124
127
|
}
|
|
125
128
|
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
129
|
+
if (segment.paged !== null) return;
|
|
126
130
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
127
131
|
const docId = segment.docStore.getExternalId(internalId);
|
|
128
132
|
if (docId !== void 0) locations.set(docId, partitionId);
|
|
@@ -134,7 +138,7 @@ function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
|
134
138
|
locations.delete(docId);
|
|
135
139
|
}
|
|
136
140
|
}
|
|
137
|
-
for (const docId of partition
|
|
141
|
+
for (const docId of documentIdsHeldInMemory(partition)) {
|
|
138
142
|
locations.set(docId, partitionId);
|
|
139
143
|
}
|
|
140
144
|
}
|
|
@@ -151,6 +155,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
|
151
155
|
}
|
|
152
156
|
}
|
|
153
157
|
|
|
158
|
+
// src/partitioning/manager/write-log.ts
|
|
159
|
+
var REANALYSIS_WRITE_LOG = 0;
|
|
160
|
+
var CHECKPOINT_WRITE_LOG = 1;
|
|
161
|
+
function createWriteLogRecorder() {
|
|
162
|
+
const current = [null, null];
|
|
163
|
+
return {
|
|
164
|
+
open(owner) {
|
|
165
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
166
|
+
current[owner] = log;
|
|
167
|
+
return {
|
|
168
|
+
get partitionsReplaced() {
|
|
169
|
+
return log.replaced;
|
|
170
|
+
},
|
|
171
|
+
get pendingDocuments() {
|
|
172
|
+
return log.docIds.size;
|
|
173
|
+
},
|
|
174
|
+
takeWrittenDocIds() {
|
|
175
|
+
const taken = log.docIds;
|
|
176
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
177
|
+
return taken;
|
|
178
|
+
},
|
|
179
|
+
close() {
|
|
180
|
+
if (current[owner] === log) current[owner] = null;
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
},
|
|
184
|
+
documentWritten(docId) {
|
|
185
|
+
current[0]?.docIds.add(docId);
|
|
186
|
+
current[1]?.docIds.add(docId);
|
|
187
|
+
},
|
|
188
|
+
segmentWritten(segment) {
|
|
189
|
+
for (const log of current) {
|
|
190
|
+
if (log === null) continue;
|
|
191
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
192
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
193
|
+
if (docId !== void 0) log.docIds.add(docId);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
partitionsReplaced() {
|
|
198
|
+
for (const log of current) {
|
|
199
|
+
if (log !== null) log.replaced = true;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// src/partitioning/manager/segment-operations.ts
|
|
206
|
+
function createSegmentOperations(source) {
|
|
207
|
+
const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
|
|
208
|
+
return {
|
|
209
|
+
mergeSegment(partitionId, payload, documents) {
|
|
210
|
+
validatePartitionId(partitionId);
|
|
211
|
+
const partition = source.partitionAt(partitionId);
|
|
212
|
+
partition.beginBatch();
|
|
213
|
+
partition.mergeSegmentPayload(payload, documents);
|
|
214
|
+
partition.endBatch();
|
|
215
|
+
for (const docId of payload.docIds) {
|
|
216
|
+
docPartitionMap.set(docId, partitionId);
|
|
217
|
+
writeLogs.documentWritten(docId);
|
|
218
|
+
}
|
|
219
|
+
},
|
|
220
|
+
attachFrozenSegment(partitionId, segment) {
|
|
221
|
+
validatePartitionId(partitionId);
|
|
222
|
+
if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
|
|
223
|
+
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
224
|
+
if (segment.paged !== null) return;
|
|
225
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
226
|
+
writeLogs.segmentWritten(segment);
|
|
227
|
+
},
|
|
228
|
+
openCheckpointChangeLog() {
|
|
229
|
+
return writeLogs.open(CHECKPOINT_WRITE_LOG);
|
|
230
|
+
},
|
|
231
|
+
diskSegmentAccess(partitionId) {
|
|
232
|
+
validatePartitionId(partitionId);
|
|
233
|
+
return {
|
|
234
|
+
composite: asCompositePartition(partitionId),
|
|
235
|
+
docPartitionMap,
|
|
236
|
+
insertOptions: resolveInsertOptions()
|
|
237
|
+
};
|
|
238
|
+
},
|
|
239
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
240
|
+
validatePartitionId(partitionId);
|
|
241
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
242
|
+
dropSegmentIds,
|
|
243
|
+
replacement,
|
|
244
|
+
removedSpellings
|
|
245
|
+
);
|
|
246
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
247
|
+
return applied;
|
|
248
|
+
},
|
|
249
|
+
freezeLiveTail(partitionId, freeze) {
|
|
250
|
+
validatePartitionId(partitionId);
|
|
251
|
+
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
252
|
+
},
|
|
253
|
+
replaceLiveTail(partitionId, segment) {
|
|
254
|
+
validatePartitionId(partitionId);
|
|
255
|
+
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
256
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
154
261
|
// src/partitioning/manager/stats.ts
|
|
155
262
|
function aggregateStatsOf(partitions) {
|
|
156
263
|
let totalDocuments = 0;
|
|
@@ -186,47 +293,13 @@ function partitionStatsOf(partitions) {
|
|
|
186
293
|
}));
|
|
187
294
|
}
|
|
188
295
|
|
|
189
|
-
// src/partitioning/manager/write-log.ts
|
|
190
|
-
function createWriteLogRecorder() {
|
|
191
|
-
let current = null;
|
|
192
|
-
return {
|
|
193
|
-
open() {
|
|
194
|
-
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
195
|
-
current = log;
|
|
196
|
-
return {
|
|
197
|
-
get partitionsReplaced() {
|
|
198
|
-
return log.replaced;
|
|
199
|
-
},
|
|
200
|
-
get pendingDocuments() {
|
|
201
|
-
return log.docIds.size;
|
|
202
|
-
},
|
|
203
|
-
takeWrittenDocIds() {
|
|
204
|
-
const taken = log.docIds;
|
|
205
|
-
log.docIds = /* @__PURE__ */ new Set();
|
|
206
|
-
return taken;
|
|
207
|
-
},
|
|
208
|
-
close() {
|
|
209
|
-
if (current === log) current = null;
|
|
210
|
-
}
|
|
211
|
-
};
|
|
212
|
-
},
|
|
213
|
-
documentWritten(docId) {
|
|
214
|
-
if (current !== null) current.docIds.add(docId);
|
|
215
|
-
},
|
|
216
|
-
segmentWritten(segment) {
|
|
217
|
-
if (current === null) return;
|
|
218
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
219
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
220
|
-
if (docId !== void 0) current.docIds.add(docId);
|
|
221
|
-
}
|
|
222
|
-
},
|
|
223
|
-
partitionsReplaced() {
|
|
224
|
-
if (current !== null) current.replaced = true;
|
|
225
|
-
}
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
|
|
229
296
|
// src/partitioning/manager/index.ts
|
|
297
|
+
function closePagedSegmentsLeftBehind(earlier, kept) {
|
|
298
|
+
for (const partition of earlier) {
|
|
299
|
+
if (!isCompositePartition(partition) || kept.includes(partition)) continue;
|
|
300
|
+
for (const segment of partition.frozenSegments()) segment.paged?.close();
|
|
301
|
+
}
|
|
302
|
+
}
|
|
230
303
|
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
231
304
|
let config = initialConfig;
|
|
232
305
|
let language = initialLanguage;
|
|
@@ -240,7 +313,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
240
313
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
241
314
|
let partitions = [];
|
|
242
315
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
243
|
-
const
|
|
316
|
+
const writeLogs = createWriteLogRecorder();
|
|
244
317
|
for (let i = 0; i < count; i++) {
|
|
245
318
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
246
319
|
}
|
|
@@ -260,7 +333,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
260
333
|
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
261
334
|
}
|
|
262
335
|
function locateDocument(docId) {
|
|
263
|
-
|
|
336
|
+
const known = docPartitionMap.get(docId);
|
|
337
|
+
if (known !== void 0 || config.textStorage !== "disk") return known;
|
|
338
|
+
const routed = router.route(docId, partitions.length);
|
|
339
|
+
if (partitions[routed]?.has(docId)) return routed;
|
|
340
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
341
|
+
if (i !== routed && partitions[i].has(docId)) return i;
|
|
342
|
+
}
|
|
343
|
+
return void 0;
|
|
264
344
|
}
|
|
265
345
|
function asCompositePartition(partitionId) {
|
|
266
346
|
const existing = partitions[partitionId];
|
|
@@ -306,12 +386,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
306
386
|
return [...partitions];
|
|
307
387
|
},
|
|
308
388
|
setPartitions(newPartitions) {
|
|
309
|
-
|
|
389
|
+
writeLogs.partitionsReplaced();
|
|
390
|
+
const earlier = partitions;
|
|
310
391
|
partitions = [...newPartitions];
|
|
392
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
311
393
|
rebuildDocLocations(docPartitionMap, partitions);
|
|
312
394
|
},
|
|
313
395
|
openWriteLog() {
|
|
314
|
-
return
|
|
396
|
+
return writeLogs.open(REANALYSIS_WRITE_LOG);
|
|
315
397
|
},
|
|
316
398
|
adoptRebuiltPartitions(rebuilt) {
|
|
317
399
|
if (rebuilt.length !== partitions.length) {
|
|
@@ -321,7 +403,9 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
321
403
|
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
322
404
|
);
|
|
323
405
|
}
|
|
406
|
+
const earlier = partitions;
|
|
324
407
|
partitions = [...rebuilt];
|
|
408
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
325
409
|
},
|
|
326
410
|
adoptAnalysis(change) {
|
|
327
411
|
config = change.config;
|
|
@@ -341,13 +425,13 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
341
425
|
);
|
|
342
426
|
}
|
|
343
427
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
344
|
-
|
|
428
|
+
writeLogs.partitionsReplaced();
|
|
345
429
|
partitions.push(partition);
|
|
346
430
|
return partition;
|
|
347
431
|
},
|
|
348
432
|
removePartition(partitionId) {
|
|
349
433
|
validatePartitionId(partitionId);
|
|
350
|
-
|
|
434
|
+
writeLogs.partitionsReplaced();
|
|
351
435
|
partitions.splice(partitionId, 1);
|
|
352
436
|
rebuildDocLocations(docPartitionMap, partitions);
|
|
353
437
|
},
|
|
@@ -361,7 +445,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
361
445
|
if (newCount >= partitions.length) {
|
|
362
446
|
return;
|
|
363
447
|
}
|
|
364
|
-
|
|
448
|
+
writeLogs.partitionsReplaced();
|
|
365
449
|
for (const [docId, pid] of docPartitionMap) {
|
|
366
450
|
if (pid >= newCount) {
|
|
367
451
|
docPartitionMap.delete(docId);
|
|
@@ -384,7 +468,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
384
468
|
const insertOpts = resolveInsertOptions(options);
|
|
385
469
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
386
470
|
docPartitionMap.set(docId, pid);
|
|
387
|
-
|
|
471
|
+
writeLogs.documentWritten(docId);
|
|
388
472
|
},
|
|
389
473
|
removedSpellingsOf(docId) {
|
|
390
474
|
const pid = locateDocument(docId);
|
|
@@ -398,7 +482,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
398
482
|
}
|
|
399
483
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
400
484
|
docPartitionMap.delete(docId);
|
|
401
|
-
|
|
485
|
+
writeLogs.documentWritten(docId);
|
|
402
486
|
},
|
|
403
487
|
beginBatchRemove() {
|
|
404
488
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -418,7 +502,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
418
502
|
const updateOpts = resolveInsertOptions(options);
|
|
419
503
|
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
420
504
|
docPartitionMap.set(docId, pid);
|
|
421
|
-
|
|
505
|
+
writeLogs.documentWritten(docId);
|
|
422
506
|
},
|
|
423
507
|
get(docId, projection) {
|
|
424
508
|
const pid = locateDocument(docId);
|
|
@@ -458,46 +542,17 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
458
542
|
validatePartitionId(partitionId);
|
|
459
543
|
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
460
544
|
},
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
writeLog.documentWritten(docId);
|
|
470
|
-
}
|
|
471
|
-
},
|
|
472
|
-
attachFrozenSegment(partitionId, segment) {
|
|
473
|
-
validatePartitionId(partitionId);
|
|
474
|
-
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
475
|
-
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
476
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
477
|
-
writeLog.segmentWritten(segment);
|
|
478
|
-
},
|
|
479
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
480
|
-
validatePartitionId(partitionId);
|
|
481
|
-
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
482
|
-
dropSegmentIds,
|
|
483
|
-
replacement,
|
|
484
|
-
removedSpellings
|
|
485
|
-
);
|
|
486
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
487
|
-
return applied;
|
|
488
|
-
},
|
|
489
|
-
freezeLiveTail(partitionId, freeze) {
|
|
490
|
-
validatePartitionId(partitionId);
|
|
491
|
-
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
492
|
-
},
|
|
493
|
-
replaceLiveTail(partitionId, segment) {
|
|
494
|
-
validatePartitionId(partitionId);
|
|
495
|
-
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
496
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
497
|
-
},
|
|
545
|
+
...createSegmentOperations({
|
|
546
|
+
docPartitionMap,
|
|
547
|
+
writeLogs,
|
|
548
|
+
partitionAt: (partitionId) => partitions[partitionId],
|
|
549
|
+
validatePartitionId,
|
|
550
|
+
asCompositePartition,
|
|
551
|
+
resolveInsertOptions: () => resolveInsertOptions()
|
|
552
|
+
}),
|
|
498
553
|
deserializePartition(partitionId, data) {
|
|
499
554
|
validatePartitionId(partitionId);
|
|
500
|
-
|
|
555
|
+
writeLogs.partitionsReplaced();
|
|
501
556
|
partitions[partitionId].deserialize(data, config.schema);
|
|
502
557
|
preparePartialWords(partitionId);
|
|
503
558
|
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
@@ -974,11 +1029,17 @@ function encodeDocumentTableData(documents) {
|
|
|
974
1029
|
}
|
|
975
1030
|
function wrapEncodedDocumentTable(data) {
|
|
976
1031
|
const { blob, offsets } = data;
|
|
977
|
-
|
|
978
|
-
|
|
1032
|
+
return createEncodedDocumentSource(
|
|
1033
|
+
offsets.length - 1,
|
|
1034
|
+
blob.byteLength + offsets.byteLength,
|
|
1035
|
+
(ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
|
|
1036
|
+
data.discardedFields
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
1040
|
+
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
979
1041
|
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
980
|
-
const
|
|
981
|
-
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
1042
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
982
1043
|
function projectedPathAt(ordinal, path) {
|
|
983
1044
|
const segments = encodedProjectionPathOf(path);
|
|
984
1045
|
if (segments === null) return { found: false, value: void 0 };
|
|
@@ -988,10 +1049,10 @@ function wrapEncodedDocumentTable(data) {
|
|
|
988
1049
|
}
|
|
989
1050
|
return {
|
|
990
1051
|
count,
|
|
991
|
-
byteLength
|
|
1052
|
+
byteLength,
|
|
992
1053
|
docAt,
|
|
993
1054
|
readAt(ordinal) {
|
|
994
|
-
return decodeStoredDocument(bytesAt(ordinal),
|
|
1055
|
+
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
995
1056
|
},
|
|
996
1057
|
fieldReader(fieldPath) {
|
|
997
1058
|
const path = fieldPath.split(".");
|
|
@@ -1084,6 +1145,9 @@ function wrapExternalIdTable(data) {
|
|
|
1084
1145
|
const ordinal = sortedOrdinals[lo];
|
|
1085
1146
|
return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
|
|
1086
1147
|
},
|
|
1148
|
+
sortedOrdinalAt(position) {
|
|
1149
|
+
return sortedOrdinals[position];
|
|
1150
|
+
},
|
|
1087
1151
|
collectSortedIds(excluded) {
|
|
1088
1152
|
const ids = [];
|
|
1089
1153
|
for (let position = 0; position < count; position++) {
|
|
@@ -1103,6 +1167,7 @@ function buildExternalIdTable(docIds) {
|
|
|
1103
1167
|
characters += docIds[ordinal].length;
|
|
1104
1168
|
}
|
|
1105
1169
|
let sortedIds = null;
|
|
1170
|
+
let sortedOrdinals = null;
|
|
1106
1171
|
return {
|
|
1107
1172
|
count: docIds.length,
|
|
1108
1173
|
bytes: characters,
|
|
@@ -1112,6 +1177,10 @@ function buildExternalIdTable(docIds) {
|
|
|
1112
1177
|
ordinalOf(docId) {
|
|
1113
1178
|
return ordinals.get(docId) ?? -1;
|
|
1114
1179
|
},
|
|
1180
|
+
sortedOrdinalAt(position) {
|
|
1181
|
+
if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
|
|
1182
|
+
return sortedOrdinals[position];
|
|
1183
|
+
},
|
|
1115
1184
|
collectSortedIds(excluded) {
|
|
1116
1185
|
if (sortedIds === null) {
|
|
1117
1186
|
const ids = [...docIds];
|
|
@@ -1650,6 +1719,10 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
1650
1719
|
});
|
|
1651
1720
|
return spellings;
|
|
1652
1721
|
}
|
|
1722
|
+
function subtract(ordinal, analysis) {
|
|
1723
|
+
if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
|
|
1724
|
+
else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
|
|
1725
|
+
}
|
|
1653
1726
|
return {
|
|
1654
1727
|
get settled() {
|
|
1655
1728
|
return pending.length === 0;
|
|
@@ -1658,17 +1731,17 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
1658
1731
|
if (bound !== null) return;
|
|
1659
1732
|
const scoped = { ...analysis, fields: documents.textFields };
|
|
1660
1733
|
bound = scoped;
|
|
1661
|
-
for (const ordinal of pending)
|
|
1734
|
+
for (const ordinal of pending) subtract(ordinal, scoped);
|
|
1662
1735
|
pending = [];
|
|
1663
1736
|
},
|
|
1664
1737
|
spellingsOf(ordinal) {
|
|
1665
|
-
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
1738
|
+
return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
1666
1739
|
},
|
|
1667
1740
|
afterRemoval(ordinal, spellings) {
|
|
1668
1741
|
if (spellings !== void 0) {
|
|
1669
1742
|
surfaces.subtractSpellings(ordinal, spellings);
|
|
1670
1743
|
} else if (bound !== null) {
|
|
1671
|
-
|
|
1744
|
+
subtract(ordinal, bound);
|
|
1672
1745
|
} else {
|
|
1673
1746
|
pending.push(ordinal);
|
|
1674
1747
|
}
|
|
@@ -1692,15 +1765,32 @@ function addForms(target, forms) {
|
|
|
1692
1765
|
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
1693
1766
|
}
|
|
1694
1767
|
}
|
|
1695
|
-
function createFrozenSurfaceSlot(load, documents) {
|
|
1768
|
+
function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
1696
1769
|
let registry = null;
|
|
1697
1770
|
let uncounted = null;
|
|
1771
|
+
let table = tableOnDisk;
|
|
1772
|
+
let waitingRemovals = [];
|
|
1773
|
+
let waitingAnalysis = null;
|
|
1774
|
+
function subtractWaitingRemovals(target) {
|
|
1775
|
+
if (waitingAnalysis === null || target.size() === 0) return;
|
|
1776
|
+
for (const ordinal of waitingRemovals) {
|
|
1777
|
+
const spellings = [];
|
|
1778
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
|
|
1779
|
+
if (target.holds(spelling)) spellings.push(spelling);
|
|
1780
|
+
});
|
|
1781
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1698
1784
|
function loaded() {
|
|
1699
1785
|
if (registry === null) {
|
|
1700
1786
|
const built = createSurfaceRegistry();
|
|
1701
1787
|
const forms = load();
|
|
1702
1788
|
if (forms !== null) built.deserialize(forms);
|
|
1789
|
+
subtractWaitingRemovals(built);
|
|
1703
1790
|
registry = built;
|
|
1791
|
+
table = void 0;
|
|
1792
|
+
waitingRemovals = [];
|
|
1793
|
+
waitingAnalysis = null;
|
|
1704
1794
|
}
|
|
1705
1795
|
return registry;
|
|
1706
1796
|
}
|
|
@@ -1708,12 +1798,22 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
1708
1798
|
return uncounted === null || uncounted[ordinal] === 0;
|
|
1709
1799
|
}
|
|
1710
1800
|
return {
|
|
1801
|
+
heldBytes: () => table?.heldBytes() ?? 0,
|
|
1711
1802
|
reader: {
|
|
1712
|
-
candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
|
|
1803
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded().candidatesForPrefix(prefix),
|
|
1713
1804
|
stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
|
|
1714
1805
|
size: () => loaded().size(),
|
|
1715
1806
|
serialize: () => loaded().serialize()
|
|
1716
1807
|
},
|
|
1808
|
+
tableStaysOnDisk() {
|
|
1809
|
+
return registry === null && tableOnDisk !== void 0;
|
|
1810
|
+
},
|
|
1811
|
+
subtractOnceLoaded(ordinal, analysis) {
|
|
1812
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
1813
|
+
uncounted[ordinal] = 1;
|
|
1814
|
+
waitingAnalysis = analysis;
|
|
1815
|
+
waitingRemovals.push(ordinal);
|
|
1816
|
+
},
|
|
1717
1817
|
holdsAny() {
|
|
1718
1818
|
return loaded().size() > 0;
|
|
1719
1819
|
},
|
|
@@ -1861,6 +1961,9 @@ function wrapFrozenTokenTable(data) {
|
|
|
1861
1961
|
}
|
|
1862
1962
|
return token;
|
|
1863
1963
|
},
|
|
1964
|
+
tokenBytesAt(sortedIndex) {
|
|
1965
|
+
return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
|
|
1966
|
+
},
|
|
1864
1967
|
byteLengthAt(sortedIndex) {
|
|
1865
1968
|
return offsets[sortedIndex + 1] - offsets[sortedIndex];
|
|
1866
1969
|
},
|
|
@@ -2135,10 +2238,10 @@ function buildGeoReaders(entries) {
|
|
|
2135
2238
|
}
|
|
2136
2239
|
return readers;
|
|
2137
2240
|
}
|
|
2138
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
2241
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
|
|
2139
2242
|
const tombstones = createFrozenTombstones();
|
|
2140
|
-
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
2141
|
-
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
2243
|
+
const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
2244
|
+
const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
|
|
2142
2245
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
2143
2246
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
2144
2247
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -2155,7 +2258,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2155
2258
|
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
2156
2259
|
};
|
|
2157
2260
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
2158
|
-
const surfaces = createFrozenSurfaceSlot(
|
|
2261
|
+
const surfaces = createFrozenSurfaceSlot(
|
|
2262
|
+
loadSurfaceForms,
|
|
2263
|
+
textDocuments,
|
|
2264
|
+
survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
|
|
2265
|
+
);
|
|
2159
2266
|
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
2160
2267
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
2161
2268
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
@@ -2177,6 +2284,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2177
2284
|
segmentId,
|
|
2178
2285
|
documentSource,
|
|
2179
2286
|
sharedSnapshot,
|
|
2287
|
+
paged: pagedParts?.access ?? null,
|
|
2180
2288
|
arrays: source,
|
|
2181
2289
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
2182
2290
|
docStore,
|
|
@@ -2192,7 +2300,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2192
2300
|
sizeIndexes,
|
|
2193
2301
|
fieldNameTable,
|
|
2194
2302
|
partialWords,
|
|
2195
|
-
trackPositions: source.positionOffsets !== null,
|
|
2303
|
+
trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
|
|
2196
2304
|
flatSchemaCache: null,
|
|
2197
2305
|
lastSchemaRef: null,
|
|
2198
2306
|
sortColumns: null,
|
|
@@ -2236,6 +2344,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2236
2344
|
};
|
|
2237
2345
|
return segment;
|
|
2238
2346
|
}
|
|
2347
|
+
function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
|
|
2348
|
+
return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
|
|
2349
|
+
}
|
|
2239
2350
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
2240
2351
|
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
2241
2352
|
return assembleFrozenSegment(
|
|
@@ -3099,7 +3210,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
3099
3210
|
|
|
3100
3211
|
// src/workers/segment-actions.ts
|
|
3101
3212
|
function isSegmentAction(action) {
|
|
3102
|
-
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
|
|
3213
|
+
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
|
|
3214
|
+
}
|
|
3215
|
+
function isPagedSnapshot(snapshot) {
|
|
3216
|
+
return "path" in snapshot;
|
|
3217
|
+
}
|
|
3218
|
+
async function attachSegments(entry, action) {
|
|
3219
|
+
for (const segment of action.segments) {
|
|
3220
|
+
growPartitionsTo(entry.manager, segment.partitionId);
|
|
3221
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
3222
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
3223
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-IYPLWHGC.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
3224
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
3225
|
+
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
3226
|
+
}
|
|
3103
3227
|
}
|
|
3104
3228
|
function growPartitionsTo(manager, partitionId) {
|
|
3105
3229
|
while (manager.partitionCount <= partitionId) {
|
|
@@ -3148,17 +3272,10 @@ function runSegmentAction(entry, action) {
|
|
|
3148
3272
|
}
|
|
3149
3273
|
return void 0;
|
|
3150
3274
|
}
|
|
3151
|
-
case "attachSegments":
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
3156
|
-
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
3157
|
-
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
3158
|
-
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
3159
|
-
}
|
|
3160
|
-
return void 0;
|
|
3161
|
-
}
|
|
3275
|
+
case "attachSegments":
|
|
3276
|
+
return attachSegments(entry, action);
|
|
3277
|
+
case "adoptDiskSegments":
|
|
3278
|
+
return import('./module-IYPLWHGC.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
3162
3279
|
case "compactSegments": {
|
|
3163
3280
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
3164
3281
|
return mergeFrozenSegments(
|
|
@@ -3560,4 +3677,4 @@ function createDirectExecutor(options) {
|
|
|
3560
3677
|
};
|
|
3561
3678
|
}
|
|
3562
3679
|
|
|
3563
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
3680
|
+
export { assemblePagedFrozenSegment, availableSynonymModule, buildSegmentPayload, createDirectExecutor, createEncodedDocumentSource, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, encodeSurfaceTable, endOfValue, fieldIndexRemapOf, freezeSegmentShared, holdsADiscardedField, isEmptySynonymSet, mergeFieldIndexes, mergeFieldLengths, mergeFrozenSegments, mergePartialWords, mergeSegmentContents, mergeSurfaceForms, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule, wrapFrozenTokenTable };
|