@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes,
|
|
2
|
-
import { generateId
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, buildPartialWordTableFrom, createSurfaceRegistry, encodePartialSpellingsOf, recordValueSize, patternIndexOf, encodeFieldIndexes, countPartialSpelling, ARRAY_POSITION_GAP, documentWithOwnBuffers, readProjectedPath, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createGeoIndex, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes, createCompositePartition } from './chunk-ZPWS2SKJ.mjs';
|
|
2
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
3
3
|
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
4
4
|
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { getNestedValue, tokenizeOptions, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-DQNKVOJ2.mjs';
|
|
6
|
+
import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { isWordIndexedBase } from './chunk-
|
|
9
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
10
|
-
import {
|
|
8
|
+
import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
|
|
9
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-O4LJQI4G.mjs';
|
|
10
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
11
|
+
import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder } from './chunk-NPNK57CA.mjs';
|
|
11
12
|
import { SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-4UMZ7KQL.mjs';
|
|
12
13
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
13
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
14
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
|
|
14
15
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
15
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
16
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
16
17
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
17
18
|
|
|
18
19
|
// src/core/synonyms/platform.ts
|
|
@@ -112,6 +113,36 @@ function createCapacityChecks(source) {
|
|
|
112
113
|
};
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
// src/partitioning/manager/doc-locations.ts
|
|
117
|
+
function documentIdsHeldInMemory(partition) {
|
|
118
|
+
return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
|
|
119
|
+
}
|
|
120
|
+
function rebuildDocLocations(locations, partitions) {
|
|
121
|
+
locations.clear();
|
|
122
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
123
|
+
for (const docId of documentIdsHeldInMemory(partitions[i])) {
|
|
124
|
+
locations.set(docId, i);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
129
|
+
if (segment.paged !== null) return;
|
|
130
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
131
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
132
|
+
if (docId !== void 0) locations.set(docId, partitionId);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
136
|
+
for (const [docId, pid] of locations) {
|
|
137
|
+
if (pid === partitionId) {
|
|
138
|
+
locations.delete(docId);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
for (const docId of documentIdsHeldInMemory(partition)) {
|
|
142
|
+
locations.set(docId, partitionId);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
115
146
|
// src/partitioning/manager/segments.ts
|
|
116
147
|
function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
117
148
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
@@ -124,6 +155,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
|
124
155
|
}
|
|
125
156
|
}
|
|
126
157
|
|
|
158
|
+
// src/partitioning/manager/write-log.ts
|
|
159
|
+
var REANALYSIS_WRITE_LOG = 0;
|
|
160
|
+
var CHECKPOINT_WRITE_LOG = 1;
|
|
161
|
+
function createWriteLogRecorder() {
|
|
162
|
+
const current = [null, null];
|
|
163
|
+
return {
|
|
164
|
+
open(owner) {
|
|
165
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
166
|
+
current[owner] = log;
|
|
167
|
+
return {
|
|
168
|
+
get partitionsReplaced() {
|
|
169
|
+
return log.replaced;
|
|
170
|
+
},
|
|
171
|
+
get pendingDocuments() {
|
|
172
|
+
return log.docIds.size;
|
|
173
|
+
},
|
|
174
|
+
takeWrittenDocIds() {
|
|
175
|
+
const taken = log.docIds;
|
|
176
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
177
|
+
return taken;
|
|
178
|
+
},
|
|
179
|
+
close() {
|
|
180
|
+
if (current[owner] === log) current[owner] = null;
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
},
|
|
184
|
+
documentWritten(docId) {
|
|
185
|
+
current[0]?.docIds.add(docId);
|
|
186
|
+
current[1]?.docIds.add(docId);
|
|
187
|
+
},
|
|
188
|
+
segmentWritten(segment) {
|
|
189
|
+
for (const log of current) {
|
|
190
|
+
if (log === null) continue;
|
|
191
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
192
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
193
|
+
if (docId !== void 0) log.docIds.add(docId);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
partitionsReplaced() {
|
|
198
|
+
for (const log of current) {
|
|
199
|
+
if (log !== null) log.replaced = true;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// src/partitioning/manager/segment-operations.ts
|
|
206
|
+
function createSegmentOperations(source) {
|
|
207
|
+
const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
|
|
208
|
+
return {
|
|
209
|
+
mergeSegment(partitionId, payload, documents) {
|
|
210
|
+
validatePartitionId(partitionId);
|
|
211
|
+
const partition = source.partitionAt(partitionId);
|
|
212
|
+
partition.beginBatch();
|
|
213
|
+
partition.mergeSegmentPayload(payload, documents);
|
|
214
|
+
partition.endBatch();
|
|
215
|
+
for (const docId of payload.docIds) {
|
|
216
|
+
docPartitionMap.set(docId, partitionId);
|
|
217
|
+
writeLogs.documentWritten(docId);
|
|
218
|
+
}
|
|
219
|
+
},
|
|
220
|
+
attachFrozenSegment(partitionId, segment) {
|
|
221
|
+
validatePartitionId(partitionId);
|
|
222
|
+
if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
|
|
223
|
+
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
224
|
+
if (segment.paged !== null) return;
|
|
225
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
226
|
+
writeLogs.segmentWritten(segment);
|
|
227
|
+
},
|
|
228
|
+
openCheckpointChangeLog() {
|
|
229
|
+
return writeLogs.open(CHECKPOINT_WRITE_LOG);
|
|
230
|
+
},
|
|
231
|
+
diskSegmentAccess(partitionId) {
|
|
232
|
+
validatePartitionId(partitionId);
|
|
233
|
+
return {
|
|
234
|
+
composite: asCompositePartition(partitionId),
|
|
235
|
+
docPartitionMap,
|
|
236
|
+
insertOptions: resolveInsertOptions()
|
|
237
|
+
};
|
|
238
|
+
},
|
|
239
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
240
|
+
validatePartitionId(partitionId);
|
|
241
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
242
|
+
dropSegmentIds,
|
|
243
|
+
replacement,
|
|
244
|
+
removedSpellings
|
|
245
|
+
);
|
|
246
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
247
|
+
return applied;
|
|
248
|
+
},
|
|
249
|
+
freezeLiveTail(partitionId, freeze) {
|
|
250
|
+
validatePartitionId(partitionId);
|
|
251
|
+
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
252
|
+
},
|
|
253
|
+
replaceLiveTail(partitionId, segment) {
|
|
254
|
+
validatePartitionId(partitionId);
|
|
255
|
+
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
256
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
127
261
|
// src/partitioning/manager/stats.ts
|
|
128
262
|
function aggregateStatsOf(partitions) {
|
|
129
263
|
let totalDocuments = 0;
|
|
@@ -159,59 +293,27 @@ function partitionStatsOf(partitions) {
|
|
|
159
293
|
}));
|
|
160
294
|
}
|
|
161
295
|
|
|
162
|
-
// src/partitioning/manager/write-log.ts
|
|
163
|
-
function createWriteLogRecorder() {
|
|
164
|
-
let current = null;
|
|
165
|
-
return {
|
|
166
|
-
open() {
|
|
167
|
-
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
168
|
-
current = log;
|
|
169
|
-
return {
|
|
170
|
-
get partitionsReplaced() {
|
|
171
|
-
return log.replaced;
|
|
172
|
-
},
|
|
173
|
-
get pendingDocuments() {
|
|
174
|
-
return log.docIds.size;
|
|
175
|
-
},
|
|
176
|
-
takeWrittenDocIds() {
|
|
177
|
-
const taken = log.docIds;
|
|
178
|
-
log.docIds = /* @__PURE__ */ new Set();
|
|
179
|
-
return taken;
|
|
180
|
-
},
|
|
181
|
-
close() {
|
|
182
|
-
if (current === log) current = null;
|
|
183
|
-
}
|
|
184
|
-
};
|
|
185
|
-
},
|
|
186
|
-
documentWritten(docId) {
|
|
187
|
-
if (current !== null) current.docIds.add(docId);
|
|
188
|
-
},
|
|
189
|
-
segmentWritten(segment) {
|
|
190
|
-
if (current === null) return;
|
|
191
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
192
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
193
|
-
if (docId !== void 0) current.docIds.add(docId);
|
|
194
|
-
}
|
|
195
|
-
},
|
|
196
|
-
partitionsReplaced() {
|
|
197
|
-
if (current !== null) current.replaced = true;
|
|
198
|
-
}
|
|
199
|
-
};
|
|
200
|
-
}
|
|
201
|
-
|
|
202
296
|
// src/partitioning/manager/index.ts
|
|
203
|
-
function
|
|
297
|
+
function closePagedSegmentsLeftBehind(earlier, kept) {
|
|
298
|
+
for (const partition of earlier) {
|
|
299
|
+
if (!isCompositePartition(partition) || kept.includes(partition)) continue;
|
|
300
|
+
for (const segment of partition.frozenSegments()) segment.paged?.close();
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
304
|
+
let config = initialConfig;
|
|
305
|
+
let language = initialLanguage;
|
|
204
306
|
requirePatternSearchFor(config.schema);
|
|
205
307
|
requirePartialWordSearchFor(config.schema);
|
|
206
308
|
const count = initialPartitionCount ?? 1;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
309
|
+
let analysis = resolveIndexAnalysis(config);
|
|
310
|
+
let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
311
|
+
let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
210
312
|
const trackPositions = config.trackPositions ?? true;
|
|
211
313
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
212
314
|
let partitions = [];
|
|
213
315
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
214
|
-
const
|
|
316
|
+
const writeLogs = createWriteLogRecorder();
|
|
215
317
|
for (let i = 0; i < count; i++) {
|
|
216
318
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
217
319
|
}
|
|
@@ -231,7 +333,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
231
333
|
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
232
334
|
}
|
|
233
335
|
function locateDocument(docId) {
|
|
234
|
-
|
|
336
|
+
const known = docPartitionMap.get(docId);
|
|
337
|
+
if (known !== void 0 || config.textStorage !== "disk") return known;
|
|
338
|
+
const routed = router.route(docId, partitions.length);
|
|
339
|
+
if (partitions[routed]?.has(docId)) return routed;
|
|
340
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
341
|
+
if (i !== routed && partitions[i].has(docId)) return i;
|
|
342
|
+
}
|
|
343
|
+
return void 0;
|
|
235
344
|
}
|
|
236
345
|
function asCompositePartition(partitionId) {
|
|
237
346
|
const existing = partitions[partitionId];
|
|
@@ -240,30 +349,6 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
240
349
|
partitions[partitionId] = composite;
|
|
241
350
|
return composite;
|
|
242
351
|
}
|
|
243
|
-
function rebuildDocPartitionMap() {
|
|
244
|
-
docPartitionMap.clear();
|
|
245
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
246
|
-
for (const docId of partitions[i].docIds()) {
|
|
247
|
-
docPartitionMap.set(docId, i);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
function registerSegmentDocuments(partitionId, segment) {
|
|
252
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
253
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
254
|
-
if (docId !== void 0) docPartitionMap.set(docId, partitionId);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
function rebuildDocPartitionMapForPartition(partitionId) {
|
|
258
|
-
for (const [docId, pid] of docPartitionMap) {
|
|
259
|
-
if (pid === partitionId) {
|
|
260
|
-
docPartitionMap.delete(docId);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
for (const docId of partitions[partitionId].docIds()) {
|
|
264
|
-
docPartitionMap.set(docId, partitionId);
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
352
|
const manager = {
|
|
268
353
|
get partitionCount() {
|
|
269
354
|
return partitions.length;
|
|
@@ -301,12 +386,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
301
386
|
return [...partitions];
|
|
302
387
|
},
|
|
303
388
|
setPartitions(newPartitions) {
|
|
304
|
-
|
|
389
|
+
writeLogs.partitionsReplaced();
|
|
390
|
+
const earlier = partitions;
|
|
305
391
|
partitions = [...newPartitions];
|
|
306
|
-
|
|
392
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
393
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
307
394
|
},
|
|
308
395
|
openWriteLog() {
|
|
309
|
-
return
|
|
396
|
+
return writeLogs.open(REANALYSIS_WRITE_LOG);
|
|
310
397
|
},
|
|
311
398
|
adoptRebuiltPartitions(rebuilt) {
|
|
312
399
|
if (rebuilt.length !== partitions.length) {
|
|
@@ -316,7 +403,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
316
403
|
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
317
404
|
);
|
|
318
405
|
}
|
|
406
|
+
const earlier = partitions;
|
|
319
407
|
partitions = [...rebuilt];
|
|
408
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
409
|
+
},
|
|
410
|
+
adoptAnalysis(change) {
|
|
411
|
+
config = change.config;
|
|
412
|
+
language = change.language;
|
|
413
|
+
analysis = change.analysis;
|
|
414
|
+
partialWordAnalysis = change.partialWordAnalysis;
|
|
415
|
+
textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
416
|
+
manager.synonyms = change.synonyms;
|
|
320
417
|
},
|
|
321
418
|
addPartition() {
|
|
322
419
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
@@ -328,15 +425,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
328
425
|
);
|
|
329
426
|
}
|
|
330
427
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
331
|
-
|
|
428
|
+
writeLogs.partitionsReplaced();
|
|
332
429
|
partitions.push(partition);
|
|
333
430
|
return partition;
|
|
334
431
|
},
|
|
335
432
|
removePartition(partitionId) {
|
|
336
433
|
validatePartitionId(partitionId);
|
|
337
|
-
|
|
434
|
+
writeLogs.partitionsReplaced();
|
|
338
435
|
partitions.splice(partitionId, 1);
|
|
339
|
-
|
|
436
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
340
437
|
},
|
|
341
438
|
trimPartitions(newCount) {
|
|
342
439
|
if (newCount < 1) {
|
|
@@ -348,7 +445,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
348
445
|
if (newCount >= partitions.length) {
|
|
349
446
|
return;
|
|
350
447
|
}
|
|
351
|
-
|
|
448
|
+
writeLogs.partitionsReplaced();
|
|
352
449
|
for (const [docId, pid] of docPartitionMap) {
|
|
353
450
|
if (pid >= newCount) {
|
|
354
451
|
docPartitionMap.delete(docId);
|
|
@@ -358,7 +455,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
358
455
|
},
|
|
359
456
|
...createCapacityChecks({
|
|
360
457
|
indexName,
|
|
361
|
-
config,
|
|
458
|
+
config: initialConfig,
|
|
362
459
|
partitionCount: () => partitions.length,
|
|
363
460
|
documentCount: () => manager.countDocuments(),
|
|
364
461
|
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
@@ -371,7 +468,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
371
468
|
const insertOpts = resolveInsertOptions(options);
|
|
372
469
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
373
470
|
docPartitionMap.set(docId, pid);
|
|
374
|
-
|
|
471
|
+
writeLogs.documentWritten(docId);
|
|
375
472
|
},
|
|
376
473
|
removedSpellingsOf(docId) {
|
|
377
474
|
const pid = locateDocument(docId);
|
|
@@ -385,7 +482,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
385
482
|
}
|
|
386
483
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
387
484
|
docPartitionMap.delete(docId);
|
|
388
|
-
|
|
485
|
+
writeLogs.documentWritten(docId);
|
|
389
486
|
},
|
|
390
487
|
beginBatchRemove() {
|
|
391
488
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -405,7 +502,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
405
502
|
const updateOpts = resolveInsertOptions(options);
|
|
406
503
|
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
407
504
|
docPartitionMap.set(docId, pid);
|
|
408
|
-
|
|
505
|
+
writeLogs.documentWritten(docId);
|
|
409
506
|
},
|
|
410
507
|
get(docId, projection) {
|
|
411
508
|
const pid = locateDocument(docId);
|
|
@@ -445,49 +542,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
445
542
|
validatePartitionId(partitionId);
|
|
446
543
|
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
447
544
|
},
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
writeLog.documentWritten(docId);
|
|
457
|
-
}
|
|
458
|
-
},
|
|
459
|
-
attachFrozenSegment(partitionId, segment) {
|
|
460
|
-
validatePartitionId(partitionId);
|
|
461
|
-
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
462
|
-
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
463
|
-
registerSegmentDocuments(partitionId, segment);
|
|
464
|
-
writeLog.segmentWritten(segment);
|
|
465
|
-
},
|
|
466
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
467
|
-
validatePartitionId(partitionId);
|
|
468
|
-
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
469
|
-
dropSegmentIds,
|
|
470
|
-
replacement,
|
|
471
|
-
removedSpellings
|
|
472
|
-
);
|
|
473
|
-
registerSegmentDocuments(partitionId, replacement);
|
|
474
|
-
return applied;
|
|
475
|
-
},
|
|
476
|
-
freezeLiveTail(partitionId, freeze) {
|
|
477
|
-
validatePartitionId(partitionId);
|
|
478
|
-
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
479
|
-
},
|
|
480
|
-
replaceLiveTail(partitionId, segment) {
|
|
481
|
-
validatePartitionId(partitionId);
|
|
482
|
-
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
483
|
-
registerSegmentDocuments(partitionId, segment);
|
|
484
|
-
},
|
|
545
|
+
...createSegmentOperations({
|
|
546
|
+
docPartitionMap,
|
|
547
|
+
writeLogs,
|
|
548
|
+
partitionAt: (partitionId) => partitions[partitionId],
|
|
549
|
+
validatePartitionId,
|
|
550
|
+
asCompositePartition,
|
|
551
|
+
resolveInsertOptions: () => resolveInsertOptions()
|
|
552
|
+
}),
|
|
485
553
|
deserializePartition(partitionId, data) {
|
|
486
554
|
validatePartitionId(partitionId);
|
|
487
|
-
|
|
555
|
+
writeLogs.partitionsReplaced();
|
|
488
556
|
partitions[partitionId].deserialize(data, config.schema);
|
|
489
557
|
preparePartialWords(partitionId);
|
|
490
|
-
|
|
558
|
+
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
491
559
|
},
|
|
492
560
|
getAggregateStats() {
|
|
493
561
|
return aggregateStatsOf(partitions);
|
|
@@ -961,11 +1029,17 @@ function encodeDocumentTableData(documents) {
|
|
|
961
1029
|
}
|
|
962
1030
|
function wrapEncodedDocumentTable(data) {
|
|
963
1031
|
const { blob, offsets } = data;
|
|
964
|
-
|
|
965
|
-
|
|
1032
|
+
return createEncodedDocumentSource(
|
|
1033
|
+
offsets.length - 1,
|
|
1034
|
+
blob.byteLength + offsets.byteLength,
|
|
1035
|
+
(ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
|
|
1036
|
+
data.discardedFields
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
1040
|
+
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
966
1041
|
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
967
|
-
const
|
|
968
|
-
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
1042
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
969
1043
|
function projectedPathAt(ordinal, path) {
|
|
970
1044
|
const segments = encodedProjectionPathOf(path);
|
|
971
1045
|
if (segments === null) return { found: false, value: void 0 };
|
|
@@ -975,10 +1049,10 @@ function wrapEncodedDocumentTable(data) {
|
|
|
975
1049
|
}
|
|
976
1050
|
return {
|
|
977
1051
|
count,
|
|
978
|
-
byteLength
|
|
1052
|
+
byteLength,
|
|
979
1053
|
docAt,
|
|
980
1054
|
readAt(ordinal) {
|
|
981
|
-
return decodeStoredDocument(bytesAt(ordinal),
|
|
1055
|
+
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
982
1056
|
},
|
|
983
1057
|
fieldReader(fieldPath) {
|
|
984
1058
|
const path = fieldPath.split(".");
|
|
@@ -1071,6 +1145,9 @@ function wrapExternalIdTable(data) {
|
|
|
1071
1145
|
const ordinal = sortedOrdinals[lo];
|
|
1072
1146
|
return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
|
|
1073
1147
|
},
|
|
1148
|
+
sortedOrdinalAt(position) {
|
|
1149
|
+
return sortedOrdinals[position];
|
|
1150
|
+
},
|
|
1074
1151
|
collectSortedIds(excluded) {
|
|
1075
1152
|
const ids = [];
|
|
1076
1153
|
for (let position = 0; position < count; position++) {
|
|
@@ -1090,6 +1167,7 @@ function buildExternalIdTable(docIds) {
|
|
|
1090
1167
|
characters += docIds[ordinal].length;
|
|
1091
1168
|
}
|
|
1092
1169
|
let sortedIds = null;
|
|
1170
|
+
let sortedOrdinals = null;
|
|
1093
1171
|
return {
|
|
1094
1172
|
count: docIds.length,
|
|
1095
1173
|
bytes: characters,
|
|
@@ -1099,6 +1177,10 @@ function buildExternalIdTable(docIds) {
|
|
|
1099
1177
|
ordinalOf(docId) {
|
|
1100
1178
|
return ordinals.get(docId) ?? -1;
|
|
1101
1179
|
},
|
|
1180
|
+
sortedOrdinalAt(position) {
|
|
1181
|
+
if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
|
|
1182
|
+
return sortedOrdinals[position];
|
|
1183
|
+
},
|
|
1102
1184
|
collectSortedIds(excluded) {
|
|
1103
1185
|
if (sortedIds === null) {
|
|
1104
1186
|
const ids = [...docIds];
|
|
@@ -1637,6 +1719,10 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
1637
1719
|
});
|
|
1638
1720
|
return spellings;
|
|
1639
1721
|
}
|
|
1722
|
+
function subtract(ordinal, analysis) {
|
|
1723
|
+
if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
|
|
1724
|
+
else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
|
|
1725
|
+
}
|
|
1640
1726
|
return {
|
|
1641
1727
|
get settled() {
|
|
1642
1728
|
return pending.length === 0;
|
|
@@ -1645,17 +1731,17 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
1645
1731
|
if (bound !== null) return;
|
|
1646
1732
|
const scoped = { ...analysis, fields: documents.textFields };
|
|
1647
1733
|
bound = scoped;
|
|
1648
|
-
for (const ordinal of pending)
|
|
1734
|
+
for (const ordinal of pending) subtract(ordinal, scoped);
|
|
1649
1735
|
pending = [];
|
|
1650
1736
|
},
|
|
1651
1737
|
spellingsOf(ordinal) {
|
|
1652
|
-
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
1738
|
+
return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
1653
1739
|
},
|
|
1654
1740
|
afterRemoval(ordinal, spellings) {
|
|
1655
1741
|
if (spellings !== void 0) {
|
|
1656
1742
|
surfaces.subtractSpellings(ordinal, spellings);
|
|
1657
1743
|
} else if (bound !== null) {
|
|
1658
|
-
|
|
1744
|
+
subtract(ordinal, bound);
|
|
1659
1745
|
} else {
|
|
1660
1746
|
pending.push(ordinal);
|
|
1661
1747
|
}
|
|
@@ -1679,15 +1765,32 @@ function addForms(target, forms) {
|
|
|
1679
1765
|
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
1680
1766
|
}
|
|
1681
1767
|
}
|
|
1682
|
-
function createFrozenSurfaceSlot(load, documents) {
|
|
1768
|
+
function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
1683
1769
|
let registry = null;
|
|
1684
1770
|
let uncounted = null;
|
|
1771
|
+
let table = tableOnDisk;
|
|
1772
|
+
let waitingRemovals = [];
|
|
1773
|
+
let waitingAnalysis = null;
|
|
1774
|
+
function subtractWaitingRemovals(target) {
|
|
1775
|
+
if (waitingAnalysis === null || target.size() === 0) return;
|
|
1776
|
+
for (const ordinal of waitingRemovals) {
|
|
1777
|
+
const spellings = [];
|
|
1778
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
|
|
1779
|
+
if (target.holds(spelling)) spellings.push(spelling);
|
|
1780
|
+
});
|
|
1781
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1685
1784
|
function loaded() {
|
|
1686
1785
|
if (registry === null) {
|
|
1687
1786
|
const built = createSurfaceRegistry();
|
|
1688
1787
|
const forms = load();
|
|
1689
1788
|
if (forms !== null) built.deserialize(forms);
|
|
1789
|
+
subtractWaitingRemovals(built);
|
|
1690
1790
|
registry = built;
|
|
1791
|
+
table = void 0;
|
|
1792
|
+
waitingRemovals = [];
|
|
1793
|
+
waitingAnalysis = null;
|
|
1691
1794
|
}
|
|
1692
1795
|
return registry;
|
|
1693
1796
|
}
|
|
@@ -1695,12 +1798,22 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
1695
1798
|
return uncounted === null || uncounted[ordinal] === 0;
|
|
1696
1799
|
}
|
|
1697
1800
|
return {
|
|
1801
|
+
heldBytes: () => table?.heldBytes() ?? 0,
|
|
1698
1802
|
reader: {
|
|
1699
|
-
candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
|
|
1803
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded().candidatesForPrefix(prefix),
|
|
1700
1804
|
stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
|
|
1701
1805
|
size: () => loaded().size(),
|
|
1702
1806
|
serialize: () => loaded().serialize()
|
|
1703
1807
|
},
|
|
1808
|
+
tableStaysOnDisk() {
|
|
1809
|
+
return registry === null && tableOnDisk !== void 0;
|
|
1810
|
+
},
|
|
1811
|
+
subtractOnceLoaded(ordinal, analysis) {
|
|
1812
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
1813
|
+
uncounted[ordinal] = 1;
|
|
1814
|
+
waitingAnalysis = analysis;
|
|
1815
|
+
waitingRemovals.push(ordinal);
|
|
1816
|
+
},
|
|
1704
1817
|
holdsAny() {
|
|
1705
1818
|
return loaded().size() > 0;
|
|
1706
1819
|
},
|
|
@@ -1848,6 +1961,9 @@ function wrapFrozenTokenTable(data) {
|
|
|
1848
1961
|
}
|
|
1849
1962
|
return token;
|
|
1850
1963
|
},
|
|
1964
|
+
tokenBytesAt(sortedIndex) {
|
|
1965
|
+
return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
|
|
1966
|
+
},
|
|
1851
1967
|
byteLengthAt(sortedIndex) {
|
|
1852
1968
|
return offsets[sortedIndex + 1] - offsets[sortedIndex];
|
|
1853
1969
|
},
|
|
@@ -2122,10 +2238,10 @@ function buildGeoReaders(entries) {
|
|
|
2122
2238
|
}
|
|
2123
2239
|
return readers;
|
|
2124
2240
|
}
|
|
2125
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
2241
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
|
|
2126
2242
|
const tombstones = createFrozenTombstones();
|
|
2127
|
-
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
2128
|
-
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
2243
|
+
const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
2244
|
+
const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
|
|
2129
2245
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
2130
2246
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
2131
2247
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -2142,7 +2258,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2142
2258
|
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
2143
2259
|
};
|
|
2144
2260
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
2145
|
-
const surfaces = createFrozenSurfaceSlot(
|
|
2261
|
+
const surfaces = createFrozenSurfaceSlot(
|
|
2262
|
+
loadSurfaceForms,
|
|
2263
|
+
textDocuments,
|
|
2264
|
+
survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
|
|
2265
|
+
);
|
|
2146
2266
|
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
2147
2267
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
2148
2268
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
@@ -2164,6 +2284,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2164
2284
|
segmentId,
|
|
2165
2285
|
documentSource,
|
|
2166
2286
|
sharedSnapshot,
|
|
2287
|
+
paged: pagedParts?.access ?? null,
|
|
2167
2288
|
arrays: source,
|
|
2168
2289
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
2169
2290
|
docStore,
|
|
@@ -2179,7 +2300,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2179
2300
|
sizeIndexes,
|
|
2180
2301
|
fieldNameTable,
|
|
2181
2302
|
partialWords,
|
|
2182
|
-
trackPositions: source.positionOffsets !== null,
|
|
2303
|
+
trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
|
|
2183
2304
|
flatSchemaCache: null,
|
|
2184
2305
|
lastSchemaRef: null,
|
|
2185
2306
|
sortColumns: null,
|
|
@@ -2223,6 +2344,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2223
2344
|
};
|
|
2224
2345
|
return segment;
|
|
2225
2346
|
}
|
|
2347
|
+
function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
|
|
2348
|
+
return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
|
|
2349
|
+
}
|
|
2226
2350
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
2227
2351
|
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
2228
2352
|
return assembleFrozenSegment(
|
|
@@ -3086,7 +3210,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
3086
3210
|
|
|
3087
3211
|
// src/workers/segment-actions.ts
|
|
3088
3212
|
function isSegmentAction(action) {
|
|
3089
|
-
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
|
|
3213
|
+
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
|
|
3214
|
+
}
|
|
3215
|
+
function isPagedSnapshot(snapshot) {
|
|
3216
|
+
return "path" in snapshot;
|
|
3217
|
+
}
|
|
3218
|
+
async function attachSegments(entry, action) {
|
|
3219
|
+
for (const segment of action.segments) {
|
|
3220
|
+
growPartitionsTo(entry.manager, segment.partitionId);
|
|
3221
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
3222
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
3223
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-IYPLWHGC.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
3224
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
3225
|
+
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
3226
|
+
}
|
|
3090
3227
|
}
|
|
3091
3228
|
function growPartitionsTo(manager, partitionId) {
|
|
3092
3229
|
while (manager.partitionCount <= partitionId) {
|
|
@@ -3135,17 +3272,10 @@ function runSegmentAction(entry, action) {
|
|
|
3135
3272
|
}
|
|
3136
3273
|
return void 0;
|
|
3137
3274
|
}
|
|
3138
|
-
case "attachSegments":
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
3143
|
-
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
3144
|
-
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
3145
|
-
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
3146
|
-
}
|
|
3147
|
-
return void 0;
|
|
3148
|
-
}
|
|
3275
|
+
case "attachSegments":
|
|
3276
|
+
return attachSegments(entry, action);
|
|
3277
|
+
case "adoptDiskSegments":
|
|
3278
|
+
return import('./module-IYPLWHGC.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
3149
3279
|
case "compactSegments": {
|
|
3150
3280
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
3151
3281
|
return mergeFrozenSegments(
|
|
@@ -3247,6 +3377,34 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
3247
3377
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
3248
3378
|
}
|
|
3249
3379
|
|
|
3380
|
+
// src/workers/direct-executor/entry.ts
|
|
3381
|
+
function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
|
|
3382
|
+
return {
|
|
3383
|
+
manager,
|
|
3384
|
+
get config() {
|
|
3385
|
+
return manager.config;
|
|
3386
|
+
},
|
|
3387
|
+
get language() {
|
|
3388
|
+
return manager.language;
|
|
3389
|
+
},
|
|
3390
|
+
searchOptions: {
|
|
3391
|
+
get stopWords() {
|
|
3392
|
+
return manager.analysis.stopWords;
|
|
3393
|
+
},
|
|
3394
|
+
get customTokenizer() {
|
|
3395
|
+
return manager.analysis.customTokenizer;
|
|
3396
|
+
},
|
|
3397
|
+
get synonyms() {
|
|
3398
|
+
return manager.synonyms;
|
|
3399
|
+
}
|
|
3400
|
+
},
|
|
3401
|
+
vectorIndexes,
|
|
3402
|
+
vectorCopies: createHeldVectorCopies(),
|
|
3403
|
+
analysisStale,
|
|
3404
|
+
awaitedCopyLoad
|
|
3405
|
+
};
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3250
3408
|
// src/workers/direct-executor/index.ts
|
|
3251
3409
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
3252
3410
|
function createDirectExecutor(options) {
|
|
@@ -3298,22 +3456,7 @@ function createDirectExecutor(options) {
|
|
|
3298
3456
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
3299
3457
|
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
3300
3458
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
3301
|
-
return
|
|
3302
|
-
manager,
|
|
3303
|
-
config,
|
|
3304
|
-
language,
|
|
3305
|
-
searchOptions: {
|
|
3306
|
-
stopWords: manager.analysis.stopWords,
|
|
3307
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
3308
|
-
get synonyms() {
|
|
3309
|
-
return manager.synonyms;
|
|
3310
|
-
}
|
|
3311
|
-
},
|
|
3312
|
-
vectorIndexes,
|
|
3313
|
-
vectorCopies: createHeldVectorCopies(),
|
|
3314
|
-
analysisStale,
|
|
3315
|
-
awaitedCopyLoad
|
|
3316
|
-
};
|
|
3459
|
+
return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
|
|
3317
3460
|
}
|
|
3318
3461
|
function disposeEntry(entry) {
|
|
3319
3462
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
@@ -3534,4 +3677,4 @@ function createDirectExecutor(options) {
|
|
|
3534
3677
|
};
|
|
3535
3678
|
}
|
|
3536
3679
|
|
|
3537
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
3680
|
+
export { assemblePagedFrozenSegment, availableSynonymModule, buildSegmentPayload, createDirectExecutor, createEncodedDocumentSource, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, encodeSurfaceTable, endOfValue, fieldIndexRemapOf, freezeSegmentShared, holdsADiscardedField, isEmptySynonymSet, mergeFieldIndexes, mergeFieldLengths, mergeFrozenSegments, mergePartialWords, mergeSegmentContents, mergeSurfaceForms, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule, wrapFrozenTokenTable };
|