@delali/narsil 0.3.1-next.130 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +11 -1
- package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
- package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -9
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
- package/dist/server.d.ts +200 -11
- package/dist/server.mjs +66 -23
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/tasks-DjGjK70Y.d.ts +720 -0
- package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +5310 -0
- package/package.json +16 -8
- package/dist/config-hOWGBeC6.d.ts +0 -414
- package/dist/tasks-BfF-VRhm.d.ts +0 -302
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-
|
|
2
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-4MFH226Q.mjs';
|
|
3
3
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
4
|
-
import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN,
|
|
5
|
-
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-
|
|
4
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
|
|
5
|
+
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
|
|
6
6
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/analysis/registry.ts
|
|
@@ -7538,7 +7538,7 @@ function requireLoadedPhraseCode() {
|
|
|
7538
7538
|
}
|
|
7539
7539
|
function loadPhraseCode() {
|
|
7540
7540
|
if (loading === null) {
|
|
7541
|
-
loading = import('./phrase-module-
|
|
7541
|
+
loading = import('./phrase-module-HGVBO24W.mjs').then(
|
|
7542
7542
|
(code) => {
|
|
7543
7543
|
loaded = code;
|
|
7544
7544
|
return code;
|
|
@@ -11202,6 +11202,32 @@ function createCapacityChecks(source) {
|
|
|
11202
11202
|
};
|
|
11203
11203
|
}
|
|
11204
11204
|
|
|
11205
|
+
// src/partitioning/manager/doc-locations.ts
|
|
11206
|
+
function rebuildDocLocations(locations, partitions) {
|
|
11207
|
+
locations.clear();
|
|
11208
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11209
|
+
for (const docId of partitions[i].docIds()) {
|
|
11210
|
+
locations.set(docId, i);
|
|
11211
|
+
}
|
|
11212
|
+
}
|
|
11213
|
+
}
|
|
11214
|
+
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
11215
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11216
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11217
|
+
if (docId !== void 0) locations.set(docId, partitionId);
|
|
11218
|
+
}
|
|
11219
|
+
}
|
|
11220
|
+
function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
11221
|
+
for (const [docId, pid] of locations) {
|
|
11222
|
+
if (pid === partitionId) {
|
|
11223
|
+
locations.delete(docId);
|
|
11224
|
+
}
|
|
11225
|
+
}
|
|
11226
|
+
for (const docId of partition.docIds()) {
|
|
11227
|
+
locations.set(docId, partitionId);
|
|
11228
|
+
}
|
|
11229
|
+
}
|
|
11230
|
+
|
|
11205
11231
|
// src/partitioning/manager/nested-values.ts
|
|
11206
11232
|
function setNestedValue(obj, path, value) {
|
|
11207
11233
|
if (!path.includes(".")) {
|
|
@@ -11316,13 +11342,15 @@ function createWriteLogRecorder() {
|
|
|
11316
11342
|
}
|
|
11317
11343
|
|
|
11318
11344
|
// src/partitioning/manager/index.ts
|
|
11319
|
-
function createPartitionManager(indexName,
|
|
11345
|
+
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
11346
|
+
let config = initialConfig;
|
|
11347
|
+
let language = initialLanguage;
|
|
11320
11348
|
requirePatternSearchFor(config.schema);
|
|
11321
11349
|
requirePartialWordSearchFor(config.schema);
|
|
11322
11350
|
const count = initialPartitionCount ?? 1;
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11351
|
+
let analysis = resolveIndexAnalysis(config);
|
|
11352
|
+
let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11353
|
+
let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11326
11354
|
const trackPositions = config.trackPositions ?? true;
|
|
11327
11355
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11328
11356
|
let partitions = [];
|
|
@@ -11356,30 +11384,6 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11356
11384
|
partitions[partitionId] = composite;
|
|
11357
11385
|
return composite;
|
|
11358
11386
|
}
|
|
11359
|
-
function rebuildDocPartitionMap() {
|
|
11360
|
-
docPartitionMap.clear();
|
|
11361
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11362
|
-
for (const docId of partitions[i].docIds()) {
|
|
11363
|
-
docPartitionMap.set(docId, i);
|
|
11364
|
-
}
|
|
11365
|
-
}
|
|
11366
|
-
}
|
|
11367
|
-
function registerSegmentDocuments(partitionId, segment) {
|
|
11368
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11369
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
11370
|
-
if (docId !== void 0) docPartitionMap.set(docId, partitionId);
|
|
11371
|
-
}
|
|
11372
|
-
}
|
|
11373
|
-
function rebuildDocPartitionMapForPartition(partitionId) {
|
|
11374
|
-
for (const [docId, pid] of docPartitionMap) {
|
|
11375
|
-
if (pid === partitionId) {
|
|
11376
|
-
docPartitionMap.delete(docId);
|
|
11377
|
-
}
|
|
11378
|
-
}
|
|
11379
|
-
for (const docId of partitions[partitionId].docIds()) {
|
|
11380
|
-
docPartitionMap.set(docId, partitionId);
|
|
11381
|
-
}
|
|
11382
|
-
}
|
|
11383
11387
|
const manager = {
|
|
11384
11388
|
get partitionCount() {
|
|
11385
11389
|
return partitions.length;
|
|
@@ -11419,7 +11423,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11419
11423
|
setPartitions(newPartitions) {
|
|
11420
11424
|
writeLog.partitionsReplaced();
|
|
11421
11425
|
partitions = [...newPartitions];
|
|
11422
|
-
|
|
11426
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
11423
11427
|
},
|
|
11424
11428
|
openWriteLog() {
|
|
11425
11429
|
return writeLog.open();
|
|
@@ -11434,6 +11438,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11434
11438
|
}
|
|
11435
11439
|
partitions = [...rebuilt];
|
|
11436
11440
|
},
|
|
11441
|
+
adoptAnalysis(change) {
|
|
11442
|
+
config = change.config;
|
|
11443
|
+
language = change.language;
|
|
11444
|
+
analysis = change.analysis;
|
|
11445
|
+
partialWordAnalysis = change.partialWordAnalysis;
|
|
11446
|
+
textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11447
|
+
manager.synonyms = change.synonyms;
|
|
11448
|
+
},
|
|
11437
11449
|
addPartition() {
|
|
11438
11450
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
11439
11451
|
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
@@ -11452,7 +11464,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11452
11464
|
validatePartitionId(partitionId);
|
|
11453
11465
|
writeLog.partitionsReplaced();
|
|
11454
11466
|
partitions.splice(partitionId, 1);
|
|
11455
|
-
|
|
11467
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
11456
11468
|
},
|
|
11457
11469
|
trimPartitions(newCount) {
|
|
11458
11470
|
if (newCount < 1) {
|
|
@@ -11474,7 +11486,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11474
11486
|
},
|
|
11475
11487
|
...createCapacityChecks({
|
|
11476
11488
|
indexName,
|
|
11477
|
-
config,
|
|
11489
|
+
config: initialConfig,
|
|
11478
11490
|
partitionCount: () => partitions.length,
|
|
11479
11491
|
documentCount: () => manager.countDocuments(),
|
|
11480
11492
|
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
@@ -11576,7 +11588,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11576
11588
|
validatePartitionId(partitionId);
|
|
11577
11589
|
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11578
11590
|
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11579
|
-
|
|
11591
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11580
11592
|
writeLog.segmentWritten(segment);
|
|
11581
11593
|
},
|
|
11582
11594
|
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
@@ -11586,7 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11586
11598
|
replacement,
|
|
11587
11599
|
removedSpellings
|
|
11588
11600
|
);
|
|
11589
|
-
|
|
11601
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
11590
11602
|
return applied;
|
|
11591
11603
|
},
|
|
11592
11604
|
freezeLiveTail(partitionId, freeze) {
|
|
@@ -11596,14 +11608,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11596
11608
|
replaceLiveTail(partitionId, segment) {
|
|
11597
11609
|
validatePartitionId(partitionId);
|
|
11598
11610
|
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11599
|
-
|
|
11611
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11600
11612
|
},
|
|
11601
11613
|
deserializePartition(partitionId, data) {
|
|
11602
11614
|
validatePartitionId(partitionId);
|
|
11603
11615
|
writeLog.partitionsReplaced();
|
|
11604
11616
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11605
11617
|
preparePartialWords(partitionId);
|
|
11606
|
-
|
|
11618
|
+
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
11607
11619
|
},
|
|
11608
11620
|
getAggregateStats() {
|
|
11609
11621
|
return aggregateStatsOf(partitions);
|
|
@@ -13159,9 +13171,9 @@ function basicPlaneUtf8Bytes(text) {
|
|
|
13159
13171
|
return bytes;
|
|
13160
13172
|
}
|
|
13161
13173
|
function encodeStringBlob(strings) {
|
|
13162
|
-
const
|
|
13163
|
-
if (SURROGATE.test(
|
|
13164
|
-
const blob = encoder3.encode(
|
|
13174
|
+
const joined = strings.join("");
|
|
13175
|
+
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
13176
|
+
const blob = encoder3.encode(joined);
|
|
13165
13177
|
const offsets = new Uint32Array(strings.length + 1);
|
|
13166
13178
|
let cursor = 0;
|
|
13167
13179
|
for (let i = 0; i < strings.length; i++) {
|
|
@@ -15397,7 +15409,7 @@ function requireLoadedVectorCode() {
|
|
|
15397
15409
|
function loadVectorCode() {
|
|
15398
15410
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15399
15411
|
if (loading2 === null) {
|
|
15400
|
-
loading2 = import('./module-
|
|
15412
|
+
loading2 = import('./module-5J3PEUEB.mjs').then(
|
|
15401
15413
|
(code) => {
|
|
15402
15414
|
loaded2 = code;
|
|
15403
15415
|
return code;
|
|
@@ -15492,6 +15504,34 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
15492
15504
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
15493
15505
|
}
|
|
15494
15506
|
|
|
15507
|
+
// src/workers/direct-executor/entry.ts
|
|
15508
|
+
function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
|
|
15509
|
+
return {
|
|
15510
|
+
manager,
|
|
15511
|
+
get config() {
|
|
15512
|
+
return manager.config;
|
|
15513
|
+
},
|
|
15514
|
+
get language() {
|
|
15515
|
+
return manager.language;
|
|
15516
|
+
},
|
|
15517
|
+
searchOptions: {
|
|
15518
|
+
get stopWords() {
|
|
15519
|
+
return manager.analysis.stopWords;
|
|
15520
|
+
},
|
|
15521
|
+
get customTokenizer() {
|
|
15522
|
+
return manager.analysis.customTokenizer;
|
|
15523
|
+
},
|
|
15524
|
+
get synonyms() {
|
|
15525
|
+
return manager.synonyms;
|
|
15526
|
+
}
|
|
15527
|
+
},
|
|
15528
|
+
vectorIndexes,
|
|
15529
|
+
vectorCopies: createHeldVectorCopies(),
|
|
15530
|
+
analysisStale,
|
|
15531
|
+
awaitedCopyLoad
|
|
15532
|
+
};
|
|
15533
|
+
}
|
|
15534
|
+
|
|
15495
15535
|
// src/workers/direct-executor/index.ts
|
|
15496
15536
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
15497
15537
|
function createDirectExecutor(options) {
|
|
@@ -15543,22 +15583,7 @@ function createDirectExecutor(options) {
|
|
|
15543
15583
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
15544
15584
|
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
15545
15585
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
15546
|
-
return
|
|
15547
|
-
manager,
|
|
15548
|
-
config,
|
|
15549
|
-
language,
|
|
15550
|
-
searchOptions: {
|
|
15551
|
-
stopWords: manager.analysis.stopWords,
|
|
15552
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
15553
|
-
get synonyms() {
|
|
15554
|
-
return manager.synonyms;
|
|
15555
|
-
}
|
|
15556
|
-
},
|
|
15557
|
-
vectorIndexes,
|
|
15558
|
-
vectorCopies: createHeldVectorCopies(),
|
|
15559
|
-
analysisStale,
|
|
15560
|
-
awaitedCopyLoad
|
|
15561
|
-
};
|
|
15586
|
+
return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
|
|
15562
15587
|
}
|
|
15563
15588
|
function disposeEntry(entry) {
|
|
15564
15589
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
@@ -15780,11 +15805,14 @@ function createDirectExecutor(options) {
|
|
|
15780
15805
|
}
|
|
15781
15806
|
|
|
15782
15807
|
// src/partitioning/text-rebuild.ts
|
|
15808
|
+
function textRebuildTargetOf(config, language, analysis) {
|
|
15809
|
+
return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
|
|
15810
|
+
}
|
|
15783
15811
|
function interrupted(pass) {
|
|
15784
15812
|
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
15785
15813
|
}
|
|
15786
15814
|
async function buildPartition(pass, source, partitionId) {
|
|
15787
|
-
const { manager, options } = pass;
|
|
15815
|
+
const { manager, language, options } = pass;
|
|
15788
15816
|
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
15789
15817
|
const docIds = [...source.docIds()];
|
|
15790
15818
|
target.beginBatch();
|
|
@@ -15792,27 +15820,27 @@ async function buildPartition(pass, source, partitionId) {
|
|
|
15792
15820
|
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
15793
15821
|
for (let i = start; i < end; i++) {
|
|
15794
15822
|
const document = manager.getRef(docIds[i]);
|
|
15795
|
-
if (document !== void 0) target.insert(docIds[i], document, manager.schema,
|
|
15823
|
+
if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
|
|
15796
15824
|
}
|
|
15797
15825
|
await yieldToEventLoop();
|
|
15798
15826
|
if (interrupted(pass)) return null;
|
|
15799
15827
|
}
|
|
15800
15828
|
target.endBatch();
|
|
15801
|
-
if (
|
|
15829
|
+
if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
|
|
15802
15830
|
return target;
|
|
15803
15831
|
}
|
|
15804
15832
|
function applyWrittenDocuments(pass, built) {
|
|
15805
|
-
const { manager, options } = pass;
|
|
15833
|
+
const { manager, language, options } = pass;
|
|
15806
15834
|
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
15807
15835
|
for (const partition of built) {
|
|
15808
15836
|
if (!partition.has(docId)) continue;
|
|
15809
|
-
partition.remove(docId, manager.schema,
|
|
15837
|
+
partition.remove(docId, manager.schema, language, options);
|
|
15810
15838
|
break;
|
|
15811
15839
|
}
|
|
15812
15840
|
const partitionId = manager.partitionIdOf(docId);
|
|
15813
15841
|
const document = manager.getRef(docId);
|
|
15814
15842
|
if (partitionId === void 0 || document === void 0) continue;
|
|
15815
|
-
built[partitionId].insert(docId, document, manager.schema,
|
|
15843
|
+
built[partitionId].insert(docId, document, manager.schema, language, options);
|
|
15816
15844
|
}
|
|
15817
15845
|
}
|
|
15818
15846
|
async function rebuildOnce(pass) {
|
|
@@ -15829,17 +15857,36 @@ async function rebuildOnce(pass) {
|
|
|
15829
15857
|
await yieldToEventLoop();
|
|
15830
15858
|
if (interrupted(pass)) return false;
|
|
15831
15859
|
}
|
|
15832
|
-
|
|
15833
|
-
|
|
15834
|
-
|
|
15835
|
-
|
|
15860
|
+
const keepCurrent = () => {
|
|
15861
|
+
if (interrupted(pass)) return false;
|
|
15862
|
+
applyWrittenDocuments(pass, built);
|
|
15863
|
+
return true;
|
|
15864
|
+
};
|
|
15865
|
+
if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
|
|
15866
|
+
const swap = () => {
|
|
15867
|
+
if (interrupted(pass)) return false;
|
|
15868
|
+
applyWrittenDocuments(pass, built);
|
|
15869
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
15870
|
+
pass.hooks.onSwitch();
|
|
15871
|
+
return true;
|
|
15872
|
+
};
|
|
15873
|
+
return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
|
|
15836
15874
|
}
|
|
15837
|
-
async function rebuildTextIndexes(manager, hooks) {
|
|
15838
|
-
const options = resolvePartitionInsertOptions(
|
|
15875
|
+
async function rebuildTextIndexes(manager, hooks, target) {
|
|
15876
|
+
const options = resolvePartitionInsertOptions(
|
|
15877
|
+
target?.config ?? manager.config,
|
|
15878
|
+
target?.analysis ?? manager.analysis,
|
|
15879
|
+
{
|
|
15880
|
+
validate: false,
|
|
15881
|
+
skipClone: true
|
|
15882
|
+
}
|
|
15883
|
+
);
|
|
15884
|
+
const language = target?.language ?? manager.language;
|
|
15885
|
+
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
15839
15886
|
while (hooks.isCurrent()) {
|
|
15840
15887
|
const log = manager.openWriteLog();
|
|
15841
15888
|
try {
|
|
15842
|
-
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
15889
|
+
if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
|
|
15843
15890
|
} finally {
|
|
15844
15891
|
log.close();
|
|
15845
15892
|
}
|
|
@@ -15877,6 +15924,7 @@ function wireAnalysisRebuild(wiring) {
|
|
|
15877
15924
|
getManager: wiring.getManager,
|
|
15878
15925
|
desyncIndex: wiring.desyncIndex,
|
|
15879
15926
|
resyncIndex: wiring.resyncIndex,
|
|
15927
|
+
hasPendingChange: wiring.hasPendingChange,
|
|
15880
15928
|
async persistAnalysisRevision(indexName) {
|
|
15881
15929
|
const durability = wiring.durabilityManager;
|
|
15882
15930
|
if (durability === null) {
|
|
@@ -15894,6 +15942,7 @@ function wireAnalysisRebuild(wiring) {
|
|
|
15894
15942
|
}
|
|
15895
15943
|
function createAnalysisRebuildCoordinator(config, deps) {
|
|
15896
15944
|
const stale = /* @__PURE__ */ new Map();
|
|
15945
|
+
const exclusiveRuns = /* @__PURE__ */ new Map();
|
|
15897
15946
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
15898
15947
|
let queue = Promise.resolve();
|
|
15899
15948
|
let active = null;
|
|
@@ -15969,7 +16018,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15969
16018
|
await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
|
|
15970
16019
|
}
|
|
15971
16020
|
async function review(indexName, entry) {
|
|
15972
|
-
if (entry.run !== null || deps.getManager(indexName) === void 0) return;
|
|
16021
|
+
if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
|
|
15973
16022
|
try {
|
|
15974
16023
|
await announce(indexName, entry);
|
|
15975
16024
|
} catch (err) {
|
|
@@ -16001,8 +16050,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
16001
16050
|
return stale.has(indexName);
|
|
16002
16051
|
},
|
|
16003
16052
|
isRunning(indexName) {
|
|
16053
|
+
if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
|
|
16004
16054
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
16005
16055
|
},
|
|
16056
|
+
runExclusive(indexName, work, turnEnds) {
|
|
16057
|
+
exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
|
|
16058
|
+
const run = queue.then(work).finally(() => {
|
|
16059
|
+
const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
|
|
16060
|
+
if (remaining === 0) exclusiveRuns.delete(indexName);
|
|
16061
|
+
else exclusiveRuns.set(indexName, remaining);
|
|
16062
|
+
});
|
|
16063
|
+
queue = Promise.race([run, turnEnds ?? run]).then(
|
|
16064
|
+
() => void 0,
|
|
16065
|
+
() => void 0
|
|
16066
|
+
);
|
|
16067
|
+
return run;
|
|
16068
|
+
},
|
|
16006
16069
|
async abandon(indexName) {
|
|
16007
16070
|
const entry = stale.get(indexName);
|
|
16008
16071
|
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
@@ -16401,6 +16464,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16401
16464
|
if (loadingVectorCode !== null) await loadingVectorCode;
|
|
16402
16465
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
16403
16466
|
if (existing !== void 0) {
|
|
16467
|
+
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
16468
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-57MIRJWR.mjs');
|
|
16469
|
+
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
16470
|
+
}
|
|
16404
16471
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
16405
16472
|
deps.executor.createIndex(
|
|
16406
16473
|
metadata.indexName,
|
|
@@ -16449,7 +16516,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16449
16516
|
partitionCount: metadata.partitionCount,
|
|
16450
16517
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
16451
16518
|
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
16452
|
-
synonyms
|
|
16519
|
+
synonyms,
|
|
16520
|
+
analysisGeneration: metadata.analysisGeneration ?? 0,
|
|
16521
|
+
pendingAnalysis: metadata.pendingAnalysis ?? null,
|
|
16522
|
+
durableAnalysis: null
|
|
16453
16523
|
});
|
|
16454
16524
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
16455
16525
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -16849,6 +16919,7 @@ function wireIndexState(wiring) {
|
|
|
16849
16919
|
}
|
|
16850
16920
|
try {
|
|
16851
16921
|
await durability.manager.recoverIndex(indexName);
|
|
16922
|
+
await wiring.resumeChange(indexName);
|
|
16852
16923
|
await analysisRebuild.review(indexName);
|
|
16853
16924
|
await orchestrator.openIndex(indexName);
|
|
16854
16925
|
await wiring.onOpen?.(indexName);
|
|
@@ -17569,14 +17640,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
|
|
|
17569
17640
|
{ indexName }
|
|
17570
17641
|
);
|
|
17571
17642
|
}
|
|
17643
|
+
if (bootstrapModule !== void 0) return;
|
|
17572
17644
|
const language = config.language ?? "english";
|
|
17573
|
-
|
|
17645
|
+
const requireRegistered = (setting, name) => {
|
|
17574
17646
|
throw new NarsilError(
|
|
17575
17647
|
ErrorCodes.CONFIG_INVALID,
|
|
17576
|
-
`Index "${indexName}" uses
|
|
17577
|
-
{ indexName,
|
|
17648
|
+
`Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
|
|
17649
|
+
{ indexName, [setting]: name }
|
|
17578
17650
|
);
|
|
17579
|
-
}
|
|
17651
|
+
};
|
|
17652
|
+
if (language !== "english") requireRegistered("language", language);
|
|
17653
|
+
if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
|
|
17654
|
+
if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
|
|
17580
17655
|
}
|
|
17581
17656
|
function workerIneligibility(indexName, config, bootstrapModule) {
|
|
17582
17657
|
try {
|
|
@@ -19504,7 +19579,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19504
19579
|
// src/engine/core.ts
|
|
19505
19580
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19506
19581
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19507
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19582
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-VEUH674B.mjs')).resolveRerankSettings(config);
|
|
19508
19583
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19509
19584
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19510
19585
|
const vectorCopyPolicy = {
|
|
@@ -19515,7 +19590,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19515
19590
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19516
19591
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19517
19592
|
}
|
|
19518
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19593
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-MTIAN6VT.mjs');
|
|
19519
19594
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19520
19595
|
const executor = createDirectExecutor({
|
|
19521
19596
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19685,6 +19760,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19685
19760
|
entry.analysisRevision = entry.language.revision;
|
|
19686
19761
|
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
19687
19762
|
},
|
|
19763
|
+
hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
|
|
19688
19764
|
durabilityManager: durability?.manager ?? null
|
|
19689
19765
|
});
|
|
19690
19766
|
const indexState = wireIndexState({
|
|
@@ -19696,6 +19772,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19696
19772
|
indexRegistry,
|
|
19697
19773
|
rebalancingIndexes,
|
|
19698
19774
|
requireManager,
|
|
19775
|
+
resumeChange: async (indexName) => {
|
|
19776
|
+
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
19777
|
+
await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeChangeOf(engineCore, indexName);
|
|
19778
|
+
},
|
|
19699
19779
|
onClose: hooks?.onIndexClose,
|
|
19700
19780
|
onOpen: (indexName) => {
|
|
19701
19781
|
heapPressureNotifier.check(indexName);
|
|
@@ -19736,7 +19816,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19736
19816
|
checkWatermark: watermarkNotifier.check,
|
|
19737
19817
|
requireIndex
|
|
19738
19818
|
};
|
|
19739
|
-
|
|
19819
|
+
const engineCore = {
|
|
19740
19820
|
executor,
|
|
19741
19821
|
pluginRegistry,
|
|
19742
19822
|
durability,
|
|
@@ -19767,6 +19847,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19767
19847
|
rebalanceCtx,
|
|
19768
19848
|
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
19769
19849
|
};
|
|
19850
|
+
return engineCore;
|
|
19770
19851
|
}
|
|
19771
19852
|
|
|
19772
19853
|
// src/engine/lifecycle.ts
|
|
@@ -19777,6 +19858,7 @@ async function startEngineCore(core, config) {
|
|
|
19777
19858
|
checkHeapAfterRecovery(core);
|
|
19778
19859
|
}
|
|
19779
19860
|
if (core.invalidation) await core.invalidation.start();
|
|
19861
|
+
if (core.durability) await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeEveryChange(core);
|
|
19780
19862
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
19781
19863
|
} catch (err) {
|
|
19782
19864
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -19814,7 +19896,7 @@ async function shutdownEngine(core) {
|
|
|
19814
19896
|
}
|
|
19815
19897
|
}
|
|
19816
19898
|
if (core.rerankAdapters.size > 0) {
|
|
19817
|
-
await (await import('./runtime-
|
|
19899
|
+
await (await import('./runtime-VEUH674B.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19818
19900
|
}
|
|
19819
19901
|
await orchestrator.shutdown();
|
|
19820
19902
|
await executor.shutdown();
|
|
@@ -20244,7 +20326,8 @@ function indexConfigurationOf(entry) {
|
|
|
20244
20326
|
entry.embeddingAdapterName,
|
|
20245
20327
|
entry.analysisRevision,
|
|
20246
20328
|
entry.arrayPositionGap,
|
|
20247
|
-
synonyms
|
|
20329
|
+
synonyms,
|
|
20330
|
+
entry.analysisGeneration
|
|
20248
20331
|
);
|
|
20249
20332
|
}
|
|
20250
20333
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
@@ -20265,7 +20348,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
20265
20348
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
20266
20349
|
);
|
|
20267
20350
|
}
|
|
20268
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
|
|
20351
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
|
|
20269
20352
|
const bm25 = config.bm25;
|
|
20270
20353
|
return {
|
|
20271
20354
|
schema: config.schema,
|
|
@@ -20275,6 +20358,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
20275
20358
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
20276
20359
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
20277
20360
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
20361
|
+
...analysisGeneration > 0 ? { analysisGeneration } : {},
|
|
20278
20362
|
...bm25 !== void 0 ? {
|
|
20279
20363
|
bm25: {
|
|
20280
20364
|
...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
|
|
@@ -20449,7 +20533,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20449
20533
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
20450
20534
|
analysisRevision: language.revision,
|
|
20451
20535
|
arrayPositionGap: ARRAY_POSITION_GAP,
|
|
20452
|
-
synonyms: synonyms?.encoded ?? null
|
|
20536
|
+
synonyms: synonyms?.encoded ?? null,
|
|
20537
|
+
analysisGeneration: 0,
|
|
20538
|
+
pendingAnalysis: null,
|
|
20539
|
+
durableAnalysis: null
|
|
20453
20540
|
});
|
|
20454
20541
|
await core.indexState.registerOpen(name);
|
|
20455
20542
|
if (core.durability) {
|
|
@@ -20460,8 +20547,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20460
20547
|
async function dropEngineIndex(core, name) {
|
|
20461
20548
|
core.guardShutdown();
|
|
20462
20549
|
const entry = core.requireIndex(name);
|
|
20550
|
+
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
20551
|
+
const abandonChange = async () => {
|
|
20552
|
+
await (await import('./analysis-change-entry-ICGRR3IN.mjs')).abandonChangeOf(core, name);
|
|
20553
|
+
};
|
|
20554
|
+
if (changeRecorded()) await abandonChange();
|
|
20463
20555
|
await core.indexState.drop(name, async () => {
|
|
20464
20556
|
await core.analysisRebuild.abandon(name);
|
|
20557
|
+
if (changeRecorded()) await abandonChange();
|
|
20465
20558
|
await core.orchestrator.closeIndex(name);
|
|
20466
20559
|
if (core.durability) {
|
|
20467
20560
|
try {
|
|
@@ -22273,6 +22366,14 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
22273
22366
|
`Index "${indexName}" is already being rebalanced`
|
|
22274
22367
|
);
|
|
22275
22368
|
}
|
|
22369
|
+
const registered4 = ctx.requireIndex(indexName);
|
|
22370
|
+
if (registered4.pendingAnalysis !== null || registered4.durableAnalysis !== null) {
|
|
22371
|
+
throw new NarsilError(
|
|
22372
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22373
|
+
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
22374
|
+
{ indexName }
|
|
22375
|
+
);
|
|
22376
|
+
}
|
|
22276
22377
|
if (targetPartitionCount === manager.partitionCount) {
|
|
22277
22378
|
return;
|
|
22278
22379
|
}
|
|
@@ -22479,646 +22580,6 @@ function bindEngineCore(engine, core) {
|
|
|
22479
22580
|
engineCores.set(engine, core);
|
|
22480
22581
|
}
|
|
22481
22582
|
|
|
22482
|
-
// src/engine/snapshot/restore-config.ts
|
|
22483
|
-
function rejectSnapshot(message) {
|
|
22484
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
|
|
22485
|
-
}
|
|
22486
|
-
function isPlainObject3(value) {
|
|
22487
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
22488
|
-
}
|
|
22489
|
-
function restoredBm25(raw, reject2) {
|
|
22490
|
-
if (raw === void 0) {
|
|
22491
|
-
return void 0;
|
|
22492
|
-
}
|
|
22493
|
-
if (!isPlainObject3(raw)) {
|
|
22494
|
-
reject2("bm25 must be an object");
|
|
22495
|
-
}
|
|
22496
|
-
const fields = restoredBm25Fields(raw.fields, reject2);
|
|
22497
|
-
return {
|
|
22498
|
-
...restoredBm25Pair(raw, "bm25", reject2),
|
|
22499
|
-
...fields !== void 0 ? { fields } : {}
|
|
22500
|
-
};
|
|
22501
|
-
}
|
|
22502
|
-
function restoredBm25Pair(raw, label, reject2) {
|
|
22503
|
-
const { k1, b } = raw;
|
|
22504
|
-
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
22505
|
-
reject2(`${label} k1 must be a finite number`);
|
|
22506
|
-
}
|
|
22507
|
-
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
22508
|
-
reject2(`${label} b must be a finite number`);
|
|
22509
|
-
}
|
|
22510
|
-
return {
|
|
22511
|
-
...k1 !== void 0 ? { k1 } : {},
|
|
22512
|
-
...b !== void 0 ? { b } : {}
|
|
22513
|
-
};
|
|
22514
|
-
}
|
|
22515
|
-
function restoredBm25Fields(raw, reject2) {
|
|
22516
|
-
if (raw === void 0) return void 0;
|
|
22517
|
-
if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
|
|
22518
|
-
const source = raw;
|
|
22519
|
-
const fields = /* @__PURE__ */ Object.create(null);
|
|
22520
|
-
for (const name of Object.keys(source)) {
|
|
22521
|
-
const pair = source[name];
|
|
22522
|
-
if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
|
|
22523
|
-
fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
|
|
22524
|
-
}
|
|
22525
|
-
return fields;
|
|
22526
|
-
}
|
|
22527
|
-
function restoredStopWords(name, list, reject2) {
|
|
22528
|
-
if (name !== void 0 && typeof name !== "string") {
|
|
22529
|
-
reject2("stopWords must be a name");
|
|
22530
|
-
}
|
|
22531
|
-
if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
|
|
22532
|
-
reject2("stopWordList must be a list of words");
|
|
22533
|
-
}
|
|
22534
|
-
if (typeof name === "string" && Array.isArray(list)) {
|
|
22535
|
-
reject2("stopWords and stopWordList cannot both be present");
|
|
22536
|
-
}
|
|
22537
|
-
if (typeof name === "string") {
|
|
22538
|
-
return name;
|
|
22539
|
-
}
|
|
22540
|
-
if (Array.isArray(list)) {
|
|
22541
|
-
return new Set(list);
|
|
22542
|
-
}
|
|
22543
|
-
return void 0;
|
|
22544
|
-
}
|
|
22545
|
-
function restoredPartitionConfig(raw, reject2 = rejectSnapshot) {
|
|
22546
|
-
if (raw === void 0) {
|
|
22547
|
-
return void 0;
|
|
22548
|
-
}
|
|
22549
|
-
if (!isPlainObject3(raw)) {
|
|
22550
|
-
reject2("partitionConfig must be an object");
|
|
22551
|
-
}
|
|
22552
|
-
const { maxDocsPerPartition, maxPartitions, watermark } = raw;
|
|
22553
|
-
if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
22554
|
-
reject2("partitionConfig maxDocsPerPartition must be a positive integer");
|
|
22555
|
-
}
|
|
22556
|
-
if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
22557
|
-
reject2("partitionConfig maxPartitions must be a positive integer");
|
|
22558
|
-
}
|
|
22559
|
-
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
22560
|
-
reject2("partitionConfig watermark must be above 0 and at most 1");
|
|
22561
|
-
}
|
|
22562
|
-
return {
|
|
22563
|
-
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
22564
|
-
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
22565
|
-
...typeof watermark === "number" ? { watermark } : {}
|
|
22566
|
-
};
|
|
22567
|
-
}
|
|
22568
|
-
function restoredScoring(raw, reject2) {
|
|
22569
|
-
if (raw === void 0) {
|
|
22570
|
-
return void 0;
|
|
22571
|
-
}
|
|
22572
|
-
if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
|
|
22573
|
-
reject2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
|
|
22574
|
-
}
|
|
22575
|
-
return raw;
|
|
22576
|
-
}
|
|
22577
|
-
function restoredBoolean(raw, field, reject2) {
|
|
22578
|
-
if (raw === void 0) {
|
|
22579
|
-
return void 0;
|
|
22580
|
-
}
|
|
22581
|
-
if (typeof raw !== "boolean") {
|
|
22582
|
-
reject2(`${field} must be a boolean`);
|
|
22583
|
-
}
|
|
22584
|
-
return raw;
|
|
22585
|
-
}
|
|
22586
|
-
function restoredRequired(raw, reject2) {
|
|
22587
|
-
if (raw === void 0) {
|
|
22588
|
-
return void 0;
|
|
22589
|
-
}
|
|
22590
|
-
if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
|
|
22591
|
-
reject2("required must be a list of field paths");
|
|
22592
|
-
}
|
|
22593
|
-
return raw;
|
|
22594
|
-
}
|
|
22595
|
-
function restoredVectorPromotion(raw, reject2) {
|
|
22596
|
-
if (raw === void 0) {
|
|
22597
|
-
return void 0;
|
|
22598
|
-
}
|
|
22599
|
-
if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
|
|
22600
|
-
reject2("vectorPromotion must be an object");
|
|
22601
|
-
}
|
|
22602
|
-
return raw;
|
|
22603
|
-
}
|
|
22604
|
-
function restoredEmbedding(raw, reject2 = rejectSnapshot) {
|
|
22605
|
-
if (raw === void 0) {
|
|
22606
|
-
return void 0;
|
|
22607
|
-
}
|
|
22608
|
-
if (!isPlainObject3(raw)) {
|
|
22609
|
-
reject2("embedding must be an object");
|
|
22610
|
-
}
|
|
22611
|
-
const { fields, adapter } = raw;
|
|
22612
|
-
if (adapter !== void 0 && typeof adapter !== "string") {
|
|
22613
|
-
reject2("embedding adapter must be a name");
|
|
22614
|
-
}
|
|
22615
|
-
if (!isPlainObject3(fields)) {
|
|
22616
|
-
reject2("embedding fields must be an object");
|
|
22617
|
-
}
|
|
22618
|
-
for (const value of Object.values(fields)) {
|
|
22619
|
-
const isPath = typeof value === "string";
|
|
22620
|
-
const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
|
|
22621
|
-
if (!isPath && !isPathList) {
|
|
22622
|
-
reject2("embedding fields must map vector fields to source paths");
|
|
22623
|
-
}
|
|
22624
|
-
}
|
|
22625
|
-
return {
|
|
22626
|
-
fields,
|
|
22627
|
-
...adapter !== void 0 ? { adapter } : {}
|
|
22628
|
-
};
|
|
22629
|
-
}
|
|
22630
|
-
function readSharedConfigFields(source, reject2) {
|
|
22631
|
-
if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
|
|
22632
|
-
reject2("tokenizer must be a name");
|
|
22633
|
-
}
|
|
22634
|
-
const stopWords2 = restoredStopWords(source.stopWords, source.stopWordList, reject2);
|
|
22635
|
-
const bm25 = restoredBm25(source.bm25, reject2);
|
|
22636
|
-
const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject2);
|
|
22637
|
-
const defaultScoring = restoredScoring(source.defaultScoring, reject2);
|
|
22638
|
-
const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject2);
|
|
22639
|
-
const strict = restoredBoolean(source.strict, "strict", reject2);
|
|
22640
|
-
const required = restoredRequired(source.required, reject2);
|
|
22641
|
-
const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject2);
|
|
22642
|
-
if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
|
|
22643
|
-
reject2("patternValueLimit must be a number");
|
|
22644
|
-
}
|
|
22645
|
-
return {
|
|
22646
|
-
...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
|
|
22647
|
-
...stopWords2 !== void 0 ? { stopWords: stopWords2 } : {},
|
|
22648
|
-
...bm25 !== void 0 ? { bm25 } : {},
|
|
22649
|
-
...surfaceForms !== void 0 ? { surfaceForms } : {},
|
|
22650
|
-
...defaultScoring !== void 0 ? { defaultScoring } : {},
|
|
22651
|
-
...trackPositions !== void 0 ? { trackPositions } : {},
|
|
22652
|
-
...strict !== void 0 ? { strict } : {},
|
|
22653
|
-
...required !== void 0 ? { required } : {},
|
|
22654
|
-
...vectorPromotion !== void 0 ? { vectorPromotion } : {},
|
|
22655
|
-
...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
|
|
22656
|
-
};
|
|
22657
|
-
}
|
|
22658
|
-
function restoredConfigFields(envelope, reject2 = rejectSnapshot) {
|
|
22659
|
-
const shared2 = readSharedConfigFields(envelope, reject2);
|
|
22660
|
-
const partitions = restoredPartitionConfig(envelope.partitionConfig, reject2);
|
|
22661
|
-
return {
|
|
22662
|
-
...shared2,
|
|
22663
|
-
...partitions !== void 0 ? { partitions } : {}
|
|
22664
|
-
};
|
|
22665
|
-
}
|
|
22666
|
-
|
|
22667
|
-
// src/persistence/archive/byte-reader.ts
|
|
22668
|
-
async function* chunksOf(source) {
|
|
22669
|
-
if (source instanceof Uint8Array) {
|
|
22670
|
-
yield source;
|
|
22671
|
-
return;
|
|
22672
|
-
}
|
|
22673
|
-
if (typeof source.getReader === "function") {
|
|
22674
|
-
const reader = source.getReader();
|
|
22675
|
-
try {
|
|
22676
|
-
for (; ; ) {
|
|
22677
|
-
const step = await reader.read();
|
|
22678
|
-
if (step.done) return;
|
|
22679
|
-
yield step.value;
|
|
22680
|
-
}
|
|
22681
|
-
} finally {
|
|
22682
|
-
reader.releaseLock();
|
|
22683
|
-
}
|
|
22684
|
-
}
|
|
22685
|
-
yield* source;
|
|
22686
|
-
}
|
|
22687
|
-
function createByteReader(source) {
|
|
22688
|
-
const iterator = chunksOf(source);
|
|
22689
|
-
const pending = [];
|
|
22690
|
-
let pendingBytes = 0;
|
|
22691
|
-
let ended = false;
|
|
22692
|
-
async function fill(length) {
|
|
22693
|
-
while (pendingBytes < length && !ended) {
|
|
22694
|
-
const step = await iterator.next();
|
|
22695
|
-
if (step.done) {
|
|
22696
|
-
ended = true;
|
|
22697
|
-
break;
|
|
22698
|
-
}
|
|
22699
|
-
if (step.value.byteLength === 0) continue;
|
|
22700
|
-
pending.push(step.value);
|
|
22701
|
-
pendingBytes += step.value.byteLength;
|
|
22702
|
-
}
|
|
22703
|
-
}
|
|
22704
|
-
function take(length) {
|
|
22705
|
-
const out = new Uint8Array(length);
|
|
22706
|
-
let filled = 0;
|
|
22707
|
-
while (filled < length) {
|
|
22708
|
-
const head = pending[0];
|
|
22709
|
-
const needed = length - filled;
|
|
22710
|
-
if (head.byteLength <= needed) {
|
|
22711
|
-
out.set(head, filled);
|
|
22712
|
-
filled += head.byteLength;
|
|
22713
|
-
pending.shift();
|
|
22714
|
-
} else {
|
|
22715
|
-
out.set(head.subarray(0, needed), filled);
|
|
22716
|
-
pending[0] = head.subarray(needed);
|
|
22717
|
-
filled += needed;
|
|
22718
|
-
}
|
|
22719
|
-
}
|
|
22720
|
-
pendingBytes -= length;
|
|
22721
|
-
return out;
|
|
22722
|
-
}
|
|
22723
|
-
return {
|
|
22724
|
-
async readExactly(length) {
|
|
22725
|
-
await fill(length);
|
|
22726
|
-
if (pendingBytes === 0 && length > 0) return null;
|
|
22727
|
-
if (pendingBytes < length) return take(pendingBytes);
|
|
22728
|
-
return take(length);
|
|
22729
|
-
},
|
|
22730
|
-
async peek(length) {
|
|
22731
|
-
await fill(length);
|
|
22732
|
-
const available = Math.min(length, pendingBytes);
|
|
22733
|
-
const copy = take(available);
|
|
22734
|
-
pending.unshift(copy);
|
|
22735
|
-
pendingBytes += copy.byteLength;
|
|
22736
|
-
return copy;
|
|
22737
|
-
},
|
|
22738
|
-
async close() {
|
|
22739
|
-
await iterator.return(void 0);
|
|
22740
|
-
}
|
|
22741
|
-
};
|
|
22742
|
-
}
|
|
22743
|
-
|
|
22744
|
-
// src/engine/export/lines.ts
|
|
22745
|
-
var NEWLINE = 10;
|
|
22746
|
-
function joined(parts, total) {
|
|
22747
|
-
if (parts.length === 1) return parts[0];
|
|
22748
|
-
const bytes = new Uint8Array(total);
|
|
22749
|
-
let offset = 0;
|
|
22750
|
-
for (const part of parts) {
|
|
22751
|
-
bytes.set(part, offset);
|
|
22752
|
-
offset += part.byteLength;
|
|
22753
|
-
}
|
|
22754
|
-
return bytes;
|
|
22755
|
-
}
|
|
22756
|
-
async function* exportLines(source, maxLineBytes) {
|
|
22757
|
-
const decoder4 = new TextDecoder("utf-8", { fatal: true });
|
|
22758
|
-
let parts = [];
|
|
22759
|
-
let pending = 0;
|
|
22760
|
-
let overflowing = false;
|
|
22761
|
-
let line = 0;
|
|
22762
|
-
const finishLine = () => {
|
|
22763
|
-
line += 1;
|
|
22764
|
-
if (overflowing) {
|
|
22765
|
-
overflowing = false;
|
|
22766
|
-
parts = [];
|
|
22767
|
-
pending = 0;
|
|
22768
|
-
return { line, refused: "too-long" };
|
|
22769
|
-
}
|
|
22770
|
-
const bytes = joined(parts, pending);
|
|
22771
|
-
parts = [];
|
|
22772
|
-
pending = 0;
|
|
22773
|
-
try {
|
|
22774
|
-
return { line, text: decoder4.decode(bytes) };
|
|
22775
|
-
} catch {
|
|
22776
|
-
return { line, refused: "not-utf8" };
|
|
22777
|
-
}
|
|
22778
|
-
};
|
|
22779
|
-
for await (const chunk of chunksOf(source)) {
|
|
22780
|
-
let start = 0;
|
|
22781
|
-
for (; ; ) {
|
|
22782
|
-
const newline = chunk.indexOf(NEWLINE, start);
|
|
22783
|
-
const end = newline < 0 ? chunk.byteLength : newline;
|
|
22784
|
-
const piece = chunk.subarray(start, end);
|
|
22785
|
-
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
22786
|
-
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
22787
|
-
pending += piece.byteLength;
|
|
22788
|
-
} else {
|
|
22789
|
-
overflowing = true;
|
|
22790
|
-
parts = [];
|
|
22791
|
-
}
|
|
22792
|
-
if (newline < 0) break;
|
|
22793
|
-
yield finishLine();
|
|
22794
|
-
start = newline + 1;
|
|
22795
|
-
}
|
|
22796
|
-
}
|
|
22797
|
-
if (pending > 0 || overflowing) yield finishLine();
|
|
22798
|
-
}
|
|
22799
|
-
|
|
22800
|
-
// src/serialization/base64.ts
|
|
22801
|
-
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
22802
|
-
var PAD = 61;
|
|
22803
|
-
var BYTES_PER_GROUP = 3;
|
|
22804
|
-
var CHARACTERS_PER_GROUP = 4;
|
|
22805
|
-
var ASCII_LIMIT = 128;
|
|
22806
|
-
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
22807
|
-
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
22808
|
-
for (let at = 0; at < ALPHABET.length; at++) {
|
|
22809
|
-
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
22810
|
-
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
22811
|
-
}
|
|
22812
|
-
var asciiDecoder = new TextDecoder();
|
|
22813
|
-
function encodeBase64(bytes) {
|
|
22814
|
-
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
22815
|
-
let write = 0;
|
|
22816
|
-
let read = 0;
|
|
22817
|
-
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
22818
|
-
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
22819
|
-
out[write++] = ENCODE[group >>> 18];
|
|
22820
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
22821
|
-
out[write++] = ENCODE[group >>> 6 & 63];
|
|
22822
|
-
out[write++] = ENCODE[group & 63];
|
|
22823
|
-
}
|
|
22824
|
-
const left = bytes.byteLength - read;
|
|
22825
|
-
if (left > 0) {
|
|
22826
|
-
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
22827
|
-
out[write++] = ENCODE[group >>> 18];
|
|
22828
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
22829
|
-
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
22830
|
-
out[write++] = PAD;
|
|
22831
|
-
}
|
|
22832
|
-
return asciiDecoder.decode(out);
|
|
22833
|
-
}
|
|
22834
|
-
function sextetAt(text, at) {
|
|
22835
|
-
const code = text.charCodeAt(at);
|
|
22836
|
-
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
22837
|
-
}
|
|
22838
|
-
function decodeExactBase64(text) {
|
|
22839
|
-
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
22840
|
-
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
22841
|
-
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
22842
|
-
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
22843
|
-
let write = 0;
|
|
22844
|
-
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
22845
|
-
const a2 = sextetAt(text, at);
|
|
22846
|
-
const b2 = sextetAt(text, at + 1);
|
|
22847
|
-
const c2 = sextetAt(text, at + 2);
|
|
22848
|
-
const d = sextetAt(text, at + 3);
|
|
22849
|
-
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
22850
|
-
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
22851
|
-
out[write++] = group >>> 16;
|
|
22852
|
-
out[write++] = group >>> 8 & 255;
|
|
22853
|
-
out[write++] = group & 255;
|
|
22854
|
-
}
|
|
22855
|
-
if (padding === 0) return out;
|
|
22856
|
-
const a = sextetAt(text, fullEnd);
|
|
22857
|
-
const b = sextetAt(text, fullEnd + 1);
|
|
22858
|
-
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
22859
|
-
if ((a | b | c) < 0) return null;
|
|
22860
|
-
if (padding === 2 && (b & 15) !== 0) return null;
|
|
22861
|
-
if (padding === 1 && (c & 3) !== 0) return null;
|
|
22862
|
-
out[write++] = a << 2 | b >>> 4;
|
|
22863
|
-
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
22864
|
-
return out;
|
|
22865
|
-
}
|
|
22866
|
-
|
|
22867
|
-
// src/engine/export/tags.ts
|
|
22868
|
-
var TAG_PREFIX = "$";
|
|
22869
|
-
var DATE_TAG = "$date";
|
|
22870
|
-
var BYTES_TAG = "$bytes";
|
|
22871
|
-
var NUMBER_TAG = "$number";
|
|
22872
|
-
var OBJECT_TAG = "$object";
|
|
22873
|
-
var MILLISECONDS_PER_SECOND = 1e3;
|
|
22874
|
-
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
22875
|
-
var MAX_NANOSECONDS = 999999999;
|
|
22876
|
-
var MAX_DATE_MILLISECONDS = 864e13;
|
|
22877
|
-
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
22878
|
-
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
22879
|
-
["NaN", Number.NaN],
|
|
22880
|
-
["Infinity", Number.POSITIVE_INFINITY],
|
|
22881
|
-
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
22882
|
-
]);
|
|
22883
|
-
function refuse(path, reason) {
|
|
22884
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
22885
|
-
field: path
|
|
22886
|
-
});
|
|
22887
|
-
}
|
|
22888
|
-
function isRecord(value) {
|
|
22889
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
22890
|
-
}
|
|
22891
|
-
function startsWithTagPrefix(key) {
|
|
22892
|
-
return key.startsWith(TAG_PREFIX);
|
|
22893
|
-
}
|
|
22894
|
-
function soleTagKey(record) {
|
|
22895
|
-
let sole = null;
|
|
22896
|
-
for (const key of Object.keys(record)) {
|
|
22897
|
-
if (sole !== null) return null;
|
|
22898
|
-
sole = key;
|
|
22899
|
-
}
|
|
22900
|
-
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
22901
|
-
}
|
|
22902
|
-
function dateOf(value, path) {
|
|
22903
|
-
if (!Array.isArray(value) || value.length !== 2)
|
|
22904
|
-
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
22905
|
-
const seconds = value[0];
|
|
22906
|
-
const nanoseconds = value[1];
|
|
22907
|
-
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
22908
|
-
refuse(
|
|
22909
|
-
path,
|
|
22910
|
-
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
22911
|
-
);
|
|
22912
|
-
}
|
|
22913
|
-
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
22914
|
-
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
22915
|
-
}
|
|
22916
|
-
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
22917
|
-
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
22918
|
-
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
22919
|
-
return new Date(milliseconds);
|
|
22920
|
-
}
|
|
22921
|
-
function bytesOf(value, path) {
|
|
22922
|
-
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
22923
|
-
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
22924
|
-
return bytes;
|
|
22925
|
-
}
|
|
22926
|
-
function numberOf(value, path) {
|
|
22927
|
-
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
22928
|
-
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
22929
|
-
return number;
|
|
22930
|
-
}
|
|
22931
|
-
function requireDepth(path, depth) {
|
|
22932
|
-
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
22933
|
-
}
|
|
22934
|
-
function untagMap(record, path, depth) {
|
|
22935
|
-
requireDepth(path, depth);
|
|
22936
|
-
for (const key of Object.keys(record)) {
|
|
22937
|
-
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
22938
|
-
}
|
|
22939
|
-
return record;
|
|
22940
|
-
}
|
|
22941
|
-
function untagList(list, path, depth) {
|
|
22942
|
-
requireDepth(path, depth);
|
|
22943
|
-
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
22944
|
-
return list;
|
|
22945
|
-
}
|
|
22946
|
-
function untagTag(tag, value, path, depth) {
|
|
22947
|
-
switch (tag) {
|
|
22948
|
-
case DATE_TAG:
|
|
22949
|
-
return dateOf(value, path);
|
|
22950
|
-
case BYTES_TAG:
|
|
22951
|
-
return bytesOf(value, path);
|
|
22952
|
-
case NUMBER_TAG:
|
|
22953
|
-
return numberOf(value, path);
|
|
22954
|
-
case OBJECT_TAG:
|
|
22955
|
-
if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
22956
|
-
return untagMap(value, path, depth);
|
|
22957
|
-
default:
|
|
22958
|
-
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
22959
|
-
}
|
|
22960
|
-
}
|
|
22961
|
-
function untag(value, path, depth) {
|
|
22962
|
-
if (typeof value !== "object" || value === null) return value;
|
|
22963
|
-
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
22964
|
-
const record = value;
|
|
22965
|
-
const tag = soleTagKey(record);
|
|
22966
|
-
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
22967
|
-
}
|
|
22968
|
-
function untagDocumentLine(text, parsed) {
|
|
22969
|
-
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
22970
|
-
return untagMap(parsed, "", 0);
|
|
22971
|
-
}
|
|
22972
|
-
|
|
22973
|
-
// src/engine/export/import.ts
|
|
22974
|
-
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
22975
|
-
var DOCUMENT_EXPORT_VERSION = 1;
|
|
22976
|
-
function invalidHeader(message) {
|
|
22977
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
22978
|
-
}
|
|
22979
|
-
function isRecord2(value) {
|
|
22980
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
22981
|
-
}
|
|
22982
|
-
function configOfHeader(line) {
|
|
22983
|
-
if (line === void 0) invalidHeader("is missing");
|
|
22984
|
-
if (!("text" in line)) {
|
|
22985
|
-
invalidHeader(
|
|
22986
|
-
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
22987
|
-
);
|
|
22988
|
-
}
|
|
22989
|
-
let raw;
|
|
22990
|
-
try {
|
|
22991
|
-
raw = JSON.parse(line.text);
|
|
22992
|
-
} catch {
|
|
22993
|
-
invalidHeader("is not JSON");
|
|
22994
|
-
}
|
|
22995
|
-
if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
22996
|
-
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
22997
|
-
}
|
|
22998
|
-
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
22999
|
-
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
23000
|
-
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
23001
|
-
return {
|
|
23002
|
-
schema: configuration.schema,
|
|
23003
|
-
language: configuration.language,
|
|
23004
|
-
...fields,
|
|
23005
|
-
...embedding === void 0 ? {} : {
|
|
23006
|
-
embedding: {
|
|
23007
|
-
fields: embedding.fields,
|
|
23008
|
-
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
23009
|
-
}
|
|
23010
|
-
}
|
|
23011
|
-
};
|
|
23012
|
-
}
|
|
23013
|
-
async function prepareIndex(engine, indexName, config) {
|
|
23014
|
-
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
23015
|
-
await engine.createIndex(indexName, config);
|
|
23016
|
-
return;
|
|
23017
|
-
}
|
|
23018
|
-
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
23019
|
-
const exported = flattenSchema(config.schema);
|
|
23020
|
-
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
23021
|
-
for (const field of fields) {
|
|
23022
|
-
if (existing[field] === exported[field]) continue;
|
|
23023
|
-
throw new NarsilError(
|
|
23024
|
-
ErrorCodes.CONFIG_INVALID,
|
|
23025
|
-
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
23026
|
-
{ indexName, field }
|
|
23027
|
-
);
|
|
23028
|
-
}
|
|
23029
|
-
}
|
|
23030
|
-
function highestReportedAt(refused) {
|
|
23031
|
-
let highest = 0;
|
|
23032
|
-
for (let at = 1; at < refused.length; at++) {
|
|
23033
|
-
if (refused[at].line > refused[highest].line) highest = at;
|
|
23034
|
-
}
|
|
23035
|
-
return highest;
|
|
23036
|
-
}
|
|
23037
|
-
function refuse2(state, line, code, message) {
|
|
23038
|
-
state.result.refusedCount += 1;
|
|
23039
|
-
const refused = state.result.refused;
|
|
23040
|
-
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
23041
|
-
refused.push({ line, code, message });
|
|
23042
|
-
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
23043
|
-
return;
|
|
23044
|
-
}
|
|
23045
|
-
if (line >= refused[state.highestReported].line) return;
|
|
23046
|
-
refused[state.highestReported] = { line, code, message };
|
|
23047
|
-
state.highestReported = highestReportedAt(refused);
|
|
23048
|
-
}
|
|
23049
|
-
async function flush(state) {
|
|
23050
|
-
if (state.batch.length === 0) return;
|
|
23051
|
-
const batch = state.batch;
|
|
23052
|
-
state.batch = [];
|
|
23053
|
-
state.batchIds = /* @__PURE__ */ new Set();
|
|
23054
|
-
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
23055
|
-
const inserts = [];
|
|
23056
|
-
const updates = [];
|
|
23057
|
-
for (const entry of batch) {
|
|
23058
|
-
if (await state.engine.has(state.indexName, entry.docId))
|
|
23059
|
-
updates.push({ docId: entry.docId, document: entry.document });
|
|
23060
|
-
else inserts.push(entry.document);
|
|
23061
|
-
}
|
|
23062
|
-
for (const outcome of [
|
|
23063
|
-
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
23064
|
-
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
23065
|
-
]) {
|
|
23066
|
-
if (outcome === null) continue;
|
|
23067
|
-
state.result.imported += outcome.succeeded.length;
|
|
23068
|
-
for (const failure of outcome.failed) {
|
|
23069
|
-
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
23070
|
-
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
23071
|
-
}
|
|
23072
|
-
}
|
|
23073
|
-
}
|
|
23074
|
-
async function takeLine(state, entry) {
|
|
23075
|
-
if (!("text" in entry)) {
|
|
23076
|
-
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
23077
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
23078
|
-
return;
|
|
23079
|
-
}
|
|
23080
|
-
if (entry.text.length === 0) return;
|
|
23081
|
-
let document;
|
|
23082
|
-
try {
|
|
23083
|
-
document = JSON.parse(entry.text);
|
|
23084
|
-
} catch {
|
|
23085
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
23086
|
-
return;
|
|
23087
|
-
}
|
|
23088
|
-
if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
23089
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
23090
|
-
return;
|
|
23091
|
-
}
|
|
23092
|
-
try {
|
|
23093
|
-
untagDocumentLine(entry.text, document);
|
|
23094
|
-
} catch (error) {
|
|
23095
|
-
if (!(error instanceof NarsilError)) throw error;
|
|
23096
|
-
refuse2(state, entry.line, error.code, error.message);
|
|
23097
|
-
return;
|
|
23098
|
-
}
|
|
23099
|
-
if (state.batchIds.has(document.id)) await flush(state);
|
|
23100
|
-
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
23101
|
-
state.batchIds.add(document.id);
|
|
23102
|
-
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
23103
|
-
}
|
|
23104
|
-
async function readDocumentExport(engine, indexName, source) {
|
|
23105
|
-
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
23106
|
-
const header = await lines.next();
|
|
23107
|
-
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
23108
|
-
const state = {
|
|
23109
|
-
engine,
|
|
23110
|
-
indexName,
|
|
23111
|
-
batch: [],
|
|
23112
|
-
batchIds: /* @__PURE__ */ new Set(),
|
|
23113
|
-
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
23114
|
-
highestReported: 0
|
|
23115
|
-
};
|
|
23116
|
-
for await (const entry of lines) await takeLine(state, entry);
|
|
23117
|
-
await flush(state);
|
|
23118
|
-
state.result.refused.sort((a, b) => a.line - b.line);
|
|
23119
|
-
return state.result;
|
|
23120
|
-
}
|
|
23121
|
-
|
|
23122
22583
|
// src/narsil/portability.ts
|
|
23123
22584
|
function portabilityOperations(core, config, engine) {
|
|
23124
22585
|
async function withOpenIndex2(indexName, action) {
|
|
@@ -23150,14 +22611,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
23150
22611
|
return {
|
|
23151
22612
|
snapshot(indexName) {
|
|
23152
22613
|
return withOpenIndex2(indexName, async () => {
|
|
23153
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22614
|
+
const { createSnapshot } = await import('./snapshot-YBJ5MF2M.mjs');
|
|
23154
22615
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
23155
22616
|
});
|
|
23156
22617
|
},
|
|
23157
22618
|
async restore(indexName, data) {
|
|
23158
22619
|
core.guardShutdown();
|
|
23159
22620
|
validateIndexName(indexName);
|
|
23160
|
-
const restored = import('./restore-queue-
|
|
22621
|
+
const restored = import('./restore-queue-DISFECE4.mjs').then(
|
|
23161
22622
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
23162
22623
|
);
|
|
23163
22624
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -23166,8 +22627,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
23166
22627
|
exportDocuments(indexName) {
|
|
23167
22628
|
return withOpenIndex2(indexName, async () => {
|
|
23168
22629
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
23169
|
-
import('./snapshot-
|
|
23170
|
-
import('./write-
|
|
22630
|
+
import('./snapshot-YBJ5MF2M.mjs'),
|
|
22631
|
+
import('./write-RTWI5MXL.mjs')
|
|
23171
22632
|
]);
|
|
23172
22633
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
23173
22634
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -23175,7 +22636,10 @@ function portabilityOperations(core, config, engine) {
|
|
|
23175
22636
|
},
|
|
23176
22637
|
importDocumentExport(indexName, data) {
|
|
23177
22638
|
core.guardShutdown();
|
|
23178
|
-
|
|
22639
|
+
const imported = import('./import-Q7MXD6BS.mjs').then(
|
|
22640
|
+
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
22641
|
+
);
|
|
22642
|
+
return emitWriteEventAfter(core, indexName, imported);
|
|
23179
22643
|
}
|
|
23180
22644
|
};
|
|
23181
22645
|
}
|
|
@@ -23185,14 +22649,14 @@ var ABSENT_TAG = 0;
|
|
|
23185
22649
|
var NULL_TAG = 1;
|
|
23186
22650
|
var FALSE_TAG = 2;
|
|
23187
22651
|
var TRUE_TAG = 3;
|
|
23188
|
-
var
|
|
22652
|
+
var NUMBER_TAG = 4;
|
|
23189
22653
|
var STRING_TAG = 5;
|
|
23190
22654
|
var LIST_TAG = 6;
|
|
23191
22655
|
var MAP_TAG = 7;
|
|
23192
22656
|
var LENGTH_BYTES = 4;
|
|
23193
22657
|
var LAST_ASCII_UNIT = 127;
|
|
23194
22658
|
var encoder5 = new TextEncoder();
|
|
23195
|
-
function
|
|
22659
|
+
function flush(stream) {
|
|
23196
22660
|
stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
|
|
23197
22661
|
stream.length = 0;
|
|
23198
22662
|
}
|
|
@@ -23256,8 +22720,8 @@ function writeNumberList(stream, list) {
|
|
|
23256
22720
|
if (stream.clock !== void 0) stream.clock.tick(list.members.length);
|
|
23257
22721
|
writeByte(stream, LIST_TAG);
|
|
23258
22722
|
writeUint32(stream, list.members.length);
|
|
23259
|
-
|
|
23260
|
-
stream.hash = fnv1aContinueTaggedNumbers(stream.hash,
|
|
22723
|
+
flush(stream);
|
|
22724
|
+
stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
|
|
23261
22725
|
}
|
|
23262
22726
|
function writeValue(stream, value) {
|
|
23263
22727
|
if (value instanceof NumberList) {
|
|
@@ -23277,7 +22741,7 @@ function writeValue(stream, value) {
|
|
|
23277
22741
|
return;
|
|
23278
22742
|
}
|
|
23279
22743
|
if (typeof value === "number") {
|
|
23280
|
-
writeByte(stream,
|
|
22744
|
+
writeByte(stream, NUMBER_TAG);
|
|
23281
22745
|
writeNumber(stream, value);
|
|
23282
22746
|
return;
|
|
23283
22747
|
}
|
|
@@ -23351,7 +22815,7 @@ function bindingOf(values, clock) {
|
|
|
23351
22815
|
for (const value of values) {
|
|
23352
22816
|
writeValue(stream, value);
|
|
23353
22817
|
}
|
|
23354
|
-
|
|
22818
|
+
flush(stream);
|
|
23355
22819
|
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
23356
22820
|
}
|
|
23357
22821
|
function queryBindingOf(params, clock) {
|
|
@@ -24472,7 +23936,7 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
24472
23936
|
}
|
|
24473
23937
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
24474
23938
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
24475
|
-
return import('./runtime-
|
|
23939
|
+
return import('./runtime-VEUH674B.mjs').then(
|
|
24476
23940
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
24477
23941
|
);
|
|
24478
23942
|
}
|
|
@@ -24482,7 +23946,7 @@ function executeQuery(request, callerContext) {
|
|
|
24482
23946
|
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
24483
23947
|
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
24484
23948
|
}
|
|
24485
|
-
return import('./runtime-
|
|
23949
|
+
return import('./runtime-VEUH674B.mjs').then(
|
|
24486
23950
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
24487
23951
|
);
|
|
24488
23952
|
}
|
|
@@ -24841,13 +24305,19 @@ async function enterRead(core, indexName) {
|
|
|
24841
24305
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
24842
24306
|
}
|
|
24843
24307
|
async function readUnderOneAnalysis(core, indexName, read) {
|
|
24844
|
-
|
|
24845
|
-
|
|
24846
|
-
|
|
24847
|
-
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
24848
|
-
|
|
24308
|
+
const manager = core.requireManager(indexName);
|
|
24309
|
+
for (; ; ) {
|
|
24310
|
+
const analysis = manager.analysis;
|
|
24311
|
+
const analysisStale = core.analysisRebuild.isStale(indexName);
|
|
24312
|
+
try {
|
|
24313
|
+
const result = await read();
|
|
24314
|
+
if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
|
|
24315
|
+
return { result, analysisStale };
|
|
24316
|
+
}
|
|
24317
|
+
} catch (error) {
|
|
24318
|
+
if (manager.analysis === analysis) throw error;
|
|
24319
|
+
}
|
|
24849
24320
|
}
|
|
24850
|
-
return { result, analysisStale };
|
|
24851
24321
|
}
|
|
24852
24322
|
function phraseReadinessOf(core, indexName, config) {
|
|
24853
24323
|
return () => {
|
|
@@ -24875,7 +24345,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24875
24345
|
const manager = core.requireManager(indexName);
|
|
24876
24346
|
const scoped = scopedParams(params, options);
|
|
24877
24347
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24878
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24348
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-VEUH674B.mjs');
|
|
24879
24349
|
const resolvedParams = await resolveVectorText(
|
|
24880
24350
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24881
24351
|
entry.embeddingAdapter,
|
|
@@ -24883,25 +24353,24 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24883
24353
|
entry.embeddingAdapterName
|
|
24884
24354
|
);
|
|
24885
24355
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
24886
|
-
const context = {
|
|
24887
|
-
manager,
|
|
24888
|
-
language: entry.language,
|
|
24889
|
-
config: entry.config,
|
|
24890
|
-
workerSearch,
|
|
24891
|
-
indexName,
|
|
24892
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
24893
|
-
partitionIds: options?.partitionIds,
|
|
24894
|
-
rescoreWindow: options?.rescoreWindow,
|
|
24895
|
-
indexBm25: options?.indexBm25,
|
|
24896
|
-
cursorBinding: lazyQueryBinding(params),
|
|
24897
|
-
patternWorkCap: core.patternWorkCap,
|
|
24898
|
-
rerank: rerank?.rerankRunnerFor(core),
|
|
24899
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24900
|
-
};
|
|
24901
24356
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24902
24357
|
core,
|
|
24903
24358
|
indexName,
|
|
24904
|
-
() => executeQuery(resolvedParams,
|
|
24359
|
+
() => executeQuery(resolvedParams, {
|
|
24360
|
+
manager,
|
|
24361
|
+
language: manager.language,
|
|
24362
|
+
config: manager.config,
|
|
24363
|
+
workerSearch,
|
|
24364
|
+
indexName,
|
|
24365
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
24366
|
+
partitionIds: options?.partitionIds,
|
|
24367
|
+
rescoreWindow: options?.rescoreWindow,
|
|
24368
|
+
indexBm25: options?.indexBm25,
|
|
24369
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24370
|
+
patternWorkCap: core.patternWorkCap,
|
|
24371
|
+
rerank: rerank?.rerankRunnerFor(core),
|
|
24372
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24373
|
+
})
|
|
24905
24374
|
);
|
|
24906
24375
|
if (analysisStale) {
|
|
24907
24376
|
result.analysisStale = true;
|
|
@@ -24935,23 +24404,22 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24935
24404
|
entry.embeddingAdapterName
|
|
24936
24405
|
);
|
|
24937
24406
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
24938
|
-
const context = {
|
|
24939
|
-
manager,
|
|
24940
|
-
language: entry.language,
|
|
24941
|
-
config: entry.config,
|
|
24942
|
-
workerSearch,
|
|
24943
|
-
indexName,
|
|
24944
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
24945
|
-
partitionIds: options?.partitionIds,
|
|
24946
|
-
indexBm25: options?.indexBm25,
|
|
24947
|
-
cursorBinding: lazyQueryBinding(params),
|
|
24948
|
-
patternWorkCap: core.patternWorkCap,
|
|
24949
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24950
|
-
};
|
|
24951
24407
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24952
24408
|
core,
|
|
24953
24409
|
indexName,
|
|
24954
|
-
() => executePreflight(resolvedParams,
|
|
24410
|
+
() => executePreflight(resolvedParams, {
|
|
24411
|
+
manager,
|
|
24412
|
+
language: manager.language,
|
|
24413
|
+
config: manager.config,
|
|
24414
|
+
workerSearch,
|
|
24415
|
+
indexName,
|
|
24416
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
24417
|
+
partitionIds: options?.partitionIds,
|
|
24418
|
+
indexBm25: options?.indexBm25,
|
|
24419
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24420
|
+
patternWorkCap: core.patternWorkCap,
|
|
24421
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24422
|
+
})
|
|
24955
24423
|
);
|
|
24956
24424
|
if (analysisStale) {
|
|
24957
24425
|
result.analysisStale = true;
|
|
@@ -25164,9 +24632,20 @@ function createNarsilFromCore(core, config) {
|
|
|
25164
24632
|
return runEngineSuggest(core, indexName, params);
|
|
25165
24633
|
},
|
|
25166
24634
|
async rebuildAnalysis(indexName) {
|
|
24635
|
+
guardShutdown();
|
|
24636
|
+
const entry = requireIndex(indexName);
|
|
24637
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
24638
|
+
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
24639
|
+
return (await import('./analysis-change-57MIRJWR.mjs')).resumeAnalysisChange(core, indexName);
|
|
24640
|
+
});
|
|
24641
|
+
},
|
|
24642
|
+
async changeAnalysis(indexName, change) {
|
|
25167
24643
|
guardShutdown();
|
|
25168
24644
|
requireIndex(indexName);
|
|
25169
|
-
await withOpenIndexWrite(
|
|
24645
|
+
await withOpenIndexWrite(
|
|
24646
|
+
indexName,
|
|
24647
|
+
async () => (await import('./analysis-change-57MIRJWR.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
24648
|
+
);
|
|
25170
24649
|
},
|
|
25171
24650
|
...portabilityOperations(core, config, () => narsil),
|
|
25172
24651
|
...synonymOperations(core),
|
|
@@ -25312,4 +24791,4 @@ async function createNarsil(config) {
|
|
|
25312
24791
|
// src/index.ts
|
|
25313
24792
|
var VERSION = "0.1.0";
|
|
25314
24793
|
|
|
25315
|
-
export { ARRAY_POSITION_GAP,
|
|
24794
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, emitEngineEvent, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|