@delali/narsil 0.3.1-next.131 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-DXAmysqJ.d.ts} +10 -3
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-OGLV4I3L.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-DApBeLbk.d.ts} +17 -2
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +11 -1
- package/dist/{config-D1cDv3sE.d.ts → config-B0H-vp4U.d.ts} +4 -0
- package/dist/{config-CmYJO34X.d.ts → config-Brysl9NO.d.ts} +1 -1
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +21 -5
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CuLMbH1-.d.ts} +63 -6
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +11 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +51 -19
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-DjGjK70Y.d.ts} +22 -6
- package/dist/{thread-SKO7CN3R.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +101 -6
- package/package.json +7 -7
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig,
|
|
2
|
-
import { resolveVectorText, bindEngineCore,
|
|
3
|
-
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-XMQNOD5V.mjs';
|
|
2
|
+
import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-XXKWEG53.mjs';
|
|
3
|
+
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-UIKN2OTQ.mjs';
|
|
4
4
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
5
|
-
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared,
|
|
6
|
-
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode,
|
|
7
|
-
import { generateId
|
|
5
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-IBATGS3J.mjs';
|
|
6
|
+
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-U4X4AWKL.mjs';
|
|
7
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
8
8
|
import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
|
|
9
9
|
import { countEngineThread } from './chunk-EBIHRL7B.mjs';
|
|
10
10
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
11
|
-
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS,
|
|
12
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
13
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
14
|
-
import { getNestedValue } from './chunk-
|
|
15
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
11
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
12
|
+
import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
|
|
13
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
|
|
14
|
+
import { getNestedValue } from './chunk-J3EHATEZ.mjs';
|
|
15
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-UBXJQJII.mjs';
|
|
16
16
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
|
-
import { isSingleStringFieldType } from './chunk-
|
|
18
|
-
import { withOpenIndex, oneAtATime } from './chunk-
|
|
17
|
+
import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
|
|
18
|
+
import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
|
|
19
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
19
20
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
20
21
|
import { createRequestId, isValidWorkerResponse } from './chunk-UJWGQIXU.mjs';
|
|
21
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
22
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-FFVLPCLH.mjs';
|
|
22
23
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
23
|
-
import {
|
|
24
|
-
import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
|
|
24
|
+
import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
|
|
25
25
|
import { encode, decode } from '@msgpack/msgpack';
|
|
26
26
|
import * as zlib from 'zlib';
|
|
27
27
|
|
|
@@ -480,11 +480,14 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
480
480
|
}
|
|
481
481
|
|
|
482
482
|
// src/partitioning/text-rebuild.ts
|
|
483
|
+
function textRebuildTargetOf(config, language, analysis) {
|
|
484
|
+
return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
|
|
485
|
+
}
|
|
483
486
|
function interrupted(pass) {
|
|
484
487
|
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
485
488
|
}
|
|
486
489
|
async function buildPartition(pass, source, partitionId) {
|
|
487
|
-
const { manager, options } = pass;
|
|
490
|
+
const { manager, language, options } = pass;
|
|
488
491
|
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
489
492
|
const docIds = [...source.docIds()];
|
|
490
493
|
target.beginBatch();
|
|
@@ -492,27 +495,27 @@ async function buildPartition(pass, source, partitionId) {
|
|
|
492
495
|
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
493
496
|
for (let i = start; i < end; i++) {
|
|
494
497
|
const document = manager.getRef(docIds[i]);
|
|
495
|
-
if (document !== void 0) target.insert(docIds[i], document, manager.schema,
|
|
498
|
+
if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
|
|
496
499
|
}
|
|
497
500
|
await yieldToEventLoop();
|
|
498
501
|
if (interrupted(pass)) return null;
|
|
499
502
|
}
|
|
500
503
|
target.endBatch();
|
|
501
|
-
if (
|
|
504
|
+
if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
|
|
502
505
|
return target;
|
|
503
506
|
}
|
|
504
507
|
function applyWrittenDocuments(pass, built) {
|
|
505
|
-
const { manager, options } = pass;
|
|
508
|
+
const { manager, language, options } = pass;
|
|
506
509
|
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
507
510
|
for (const partition of built) {
|
|
508
511
|
if (!partition.has(docId)) continue;
|
|
509
|
-
partition.remove(docId, manager.schema,
|
|
512
|
+
partition.remove(docId, manager.schema, language, options);
|
|
510
513
|
break;
|
|
511
514
|
}
|
|
512
515
|
const partitionId = manager.partitionIdOf(docId);
|
|
513
516
|
const document = manager.getRef(docId);
|
|
514
517
|
if (partitionId === void 0 || document === void 0) continue;
|
|
515
|
-
built[partitionId].insert(docId, document, manager.schema,
|
|
518
|
+
built[partitionId].insert(docId, document, manager.schema, language, options);
|
|
516
519
|
}
|
|
517
520
|
}
|
|
518
521
|
async function rebuildOnce(pass) {
|
|
@@ -529,17 +532,36 @@ async function rebuildOnce(pass) {
|
|
|
529
532
|
await yieldToEventLoop();
|
|
530
533
|
if (interrupted(pass)) return false;
|
|
531
534
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
535
|
+
const keepCurrent = () => {
|
|
536
|
+
if (interrupted(pass)) return false;
|
|
537
|
+
applyWrittenDocuments(pass, built);
|
|
538
|
+
return true;
|
|
539
|
+
};
|
|
540
|
+
if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
|
|
541
|
+
const swap = () => {
|
|
542
|
+
if (interrupted(pass)) return false;
|
|
543
|
+
applyWrittenDocuments(pass, built);
|
|
544
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
545
|
+
pass.hooks.onSwitch();
|
|
546
|
+
return true;
|
|
547
|
+
};
|
|
548
|
+
return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
|
|
536
549
|
}
|
|
537
|
-
async function rebuildTextIndexes(manager, hooks) {
|
|
538
|
-
const options = resolvePartitionInsertOptions(
|
|
550
|
+
async function rebuildTextIndexes(manager, hooks, target) {
|
|
551
|
+
const options = resolvePartitionInsertOptions(
|
|
552
|
+
target?.config ?? manager.config,
|
|
553
|
+
target?.analysis ?? manager.analysis,
|
|
554
|
+
{
|
|
555
|
+
validate: false,
|
|
556
|
+
skipClone: true
|
|
557
|
+
}
|
|
558
|
+
);
|
|
559
|
+
const language = target?.language ?? manager.language;
|
|
560
|
+
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
539
561
|
while (hooks.isCurrent()) {
|
|
540
562
|
const log = manager.openWriteLog();
|
|
541
563
|
try {
|
|
542
|
-
if (await rebuildOnce({ manager, log, hooks, options })) return true;
|
|
564
|
+
if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
|
|
543
565
|
} finally {
|
|
544
566
|
log.close();
|
|
545
567
|
}
|
|
@@ -577,6 +599,7 @@ function wireAnalysisRebuild(wiring) {
|
|
|
577
599
|
getManager: wiring.getManager,
|
|
578
600
|
desyncIndex: wiring.desyncIndex,
|
|
579
601
|
resyncIndex: wiring.resyncIndex,
|
|
602
|
+
hasPendingChange: wiring.hasPendingChange,
|
|
580
603
|
async persistAnalysisRevision(indexName) {
|
|
581
604
|
const durability = wiring.durabilityManager;
|
|
582
605
|
if (durability === null) {
|
|
@@ -594,6 +617,7 @@ function wireAnalysisRebuild(wiring) {
|
|
|
594
617
|
}
|
|
595
618
|
function createAnalysisRebuildCoordinator(config, deps) {
|
|
596
619
|
const stale = /* @__PURE__ */ new Map();
|
|
620
|
+
const exclusiveRuns = /* @__PURE__ */ new Map();
|
|
597
621
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
598
622
|
let queue = Promise.resolve();
|
|
599
623
|
let active = null;
|
|
@@ -669,7 +693,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
669
693
|
await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
|
|
670
694
|
}
|
|
671
695
|
async function review(indexName, entry) {
|
|
672
|
-
if (entry.run !== null || deps.getManager(indexName) === void 0) return;
|
|
696
|
+
if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
|
|
673
697
|
try {
|
|
674
698
|
await announce(indexName, entry);
|
|
675
699
|
} catch (err) {
|
|
@@ -701,8 +725,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
701
725
|
return stale.has(indexName);
|
|
702
726
|
},
|
|
703
727
|
isRunning(indexName) {
|
|
728
|
+
if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
|
|
704
729
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
705
730
|
},
|
|
731
|
+
runExclusive(indexName, work, turnEnds) {
|
|
732
|
+
exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
|
|
733
|
+
const run = queue.then(work).finally(() => {
|
|
734
|
+
const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
|
|
735
|
+
if (remaining === 0) exclusiveRuns.delete(indexName);
|
|
736
|
+
else exclusiveRuns.set(indexName, remaining);
|
|
737
|
+
});
|
|
738
|
+
queue = Promise.race([run, turnEnds ?? run]).then(
|
|
739
|
+
() => void 0,
|
|
740
|
+
() => void 0
|
|
741
|
+
);
|
|
742
|
+
return run;
|
|
743
|
+
},
|
|
706
744
|
async abandon(indexName) {
|
|
707
745
|
const entry = stale.get(indexName);
|
|
708
746
|
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
@@ -958,6 +996,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
958
996
|
if (loadingVectorCode !== null) await loadingVectorCode;
|
|
959
997
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
960
998
|
if (existing !== void 0) {
|
|
999
|
+
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
1000
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-JN23AZQP.mjs');
|
|
1001
|
+
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1002
|
+
}
|
|
961
1003
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
962
1004
|
deps.executor.createIndex(
|
|
963
1005
|
metadata.indexName,
|
|
@@ -1006,7 +1048,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1006
1048
|
partitionCount: metadata.partitionCount,
|
|
1007
1049
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
1008
1050
|
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
1009
|
-
synonyms
|
|
1051
|
+
synonyms,
|
|
1052
|
+
analysisGeneration: metadata.analysisGeneration ?? 0,
|
|
1053
|
+
pendingAnalysis: metadata.pendingAnalysis ?? null,
|
|
1054
|
+
durableAnalysis: null
|
|
1010
1055
|
});
|
|
1011
1056
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
1012
1057
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -1406,6 +1451,7 @@ function wireIndexState(wiring) {
|
|
|
1406
1451
|
}
|
|
1407
1452
|
try {
|
|
1408
1453
|
await durability.manager.recoverIndex(indexName);
|
|
1454
|
+
await wiring.resumeChange(indexName);
|
|
1409
1455
|
await analysisRebuild.review(indexName);
|
|
1410
1456
|
await orchestrator.openIndex(indexName);
|
|
1411
1457
|
await wiring.onOpen?.(indexName);
|
|
@@ -2106,14 +2152,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
|
|
|
2106
2152
|
{ indexName }
|
|
2107
2153
|
);
|
|
2108
2154
|
}
|
|
2155
|
+
if (bootstrapModule !== void 0) return;
|
|
2109
2156
|
const language = config.language ?? "english";
|
|
2110
|
-
|
|
2157
|
+
const requireRegistered = (setting, name) => {
|
|
2111
2158
|
throw new NarsilError(
|
|
2112
2159
|
ErrorCodes.CONFIG_INVALID,
|
|
2113
|
-
`Index "${indexName}" uses
|
|
2114
|
-
{ indexName,
|
|
2160
|
+
`Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
|
|
2161
|
+
{ indexName, [setting]: name }
|
|
2115
2162
|
);
|
|
2116
|
-
}
|
|
2163
|
+
};
|
|
2164
|
+
if (language !== "english") requireRegistered("language", language);
|
|
2165
|
+
if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
|
|
2166
|
+
if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
|
|
2117
2167
|
}
|
|
2118
2168
|
function workerIneligibility(indexName, config, bootstrapModule) {
|
|
2119
2169
|
try {
|
|
@@ -4064,7 +4114,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
4064
4114
|
// src/engine/core.ts
|
|
4065
4115
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4066
4116
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4067
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
4117
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-DYV75DCB.mjs')).resolveRerankSettings(config);
|
|
4068
4118
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4069
4119
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4070
4120
|
const vectorCopyPolicy = {
|
|
@@ -4075,7 +4125,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4075
4125
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4076
4126
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4077
4127
|
}
|
|
4078
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4128
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-AYOTAPNQ.mjs');
|
|
4079
4129
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4080
4130
|
const executor = createDirectExecutor({
|
|
4081
4131
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4245,6 +4295,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4245
4295
|
entry.analysisRevision = entry.language.revision;
|
|
4246
4296
|
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
4247
4297
|
},
|
|
4298
|
+
hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
|
|
4248
4299
|
durabilityManager: durability?.manager ?? null
|
|
4249
4300
|
});
|
|
4250
4301
|
const indexState = wireIndexState({
|
|
@@ -4256,6 +4307,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4256
4307
|
indexRegistry,
|
|
4257
4308
|
rebalancingIndexes,
|
|
4258
4309
|
requireManager,
|
|
4310
|
+
resumeChange: async (indexName) => {
|
|
4311
|
+
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4312
|
+
await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4313
|
+
},
|
|
4259
4314
|
onClose: hooks?.onIndexClose,
|
|
4260
4315
|
onOpen: (indexName) => {
|
|
4261
4316
|
heapPressureNotifier.check(indexName);
|
|
@@ -4296,7 +4351,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4296
4351
|
checkWatermark: watermarkNotifier.check,
|
|
4297
4352
|
requireIndex
|
|
4298
4353
|
};
|
|
4299
|
-
|
|
4354
|
+
const engineCore = {
|
|
4300
4355
|
executor,
|
|
4301
4356
|
pluginRegistry,
|
|
4302
4357
|
durability,
|
|
@@ -4327,6 +4382,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4327
4382
|
rebalanceCtx,
|
|
4328
4383
|
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
4329
4384
|
};
|
|
4385
|
+
return engineCore;
|
|
4330
4386
|
}
|
|
4331
4387
|
|
|
4332
4388
|
// src/engine/lifecycle.ts
|
|
@@ -4337,6 +4393,7 @@ async function startEngineCore(core, config) {
|
|
|
4337
4393
|
checkHeapAfterRecovery(core);
|
|
4338
4394
|
}
|
|
4339
4395
|
if (core.invalidation) await core.invalidation.start();
|
|
4396
|
+
if (core.durability) await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeEveryChange(core);
|
|
4340
4397
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4341
4398
|
} catch (err) {
|
|
4342
4399
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4374,7 +4431,7 @@ async function shutdownEngine(core) {
|
|
|
4374
4431
|
}
|
|
4375
4432
|
}
|
|
4376
4433
|
if (core.rerankAdapters.size > 0) {
|
|
4377
|
-
await (await import('./runtime-
|
|
4434
|
+
await (await import('./runtime-DYV75DCB.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4378
4435
|
}
|
|
4379
4436
|
await orchestrator.shutdown();
|
|
4380
4437
|
await executor.shutdown();
|
|
@@ -4711,7 +4768,8 @@ function indexConfigurationOf(entry) {
|
|
|
4711
4768
|
entry.embeddingAdapterName,
|
|
4712
4769
|
entry.analysisRevision,
|
|
4713
4770
|
entry.arrayPositionGap,
|
|
4714
|
-
synonyms
|
|
4771
|
+
synonyms,
|
|
4772
|
+
entry.analysisGeneration
|
|
4715
4773
|
);
|
|
4716
4774
|
}
|
|
4717
4775
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
@@ -4732,7 +4790,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
4732
4790
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
4733
4791
|
);
|
|
4734
4792
|
}
|
|
4735
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
|
|
4793
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
|
|
4736
4794
|
const bm25 = config.bm25;
|
|
4737
4795
|
return {
|
|
4738
4796
|
schema: config.schema,
|
|
@@ -4742,6 +4800,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
4742
4800
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
4743
4801
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
4744
4802
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
4803
|
+
...analysisGeneration > 0 ? { analysisGeneration } : {},
|
|
4745
4804
|
...bm25 !== void 0 ? {
|
|
4746
4805
|
bm25: {
|
|
4747
4806
|
...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
|
|
@@ -4916,7 +4975,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4916
4975
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
4917
4976
|
analysisRevision: language.revision,
|
|
4918
4977
|
arrayPositionGap: ARRAY_POSITION_GAP,
|
|
4919
|
-
synonyms: synonyms?.encoded ?? null
|
|
4978
|
+
synonyms: synonyms?.encoded ?? null,
|
|
4979
|
+
analysisGeneration: 0,
|
|
4980
|
+
pendingAnalysis: null,
|
|
4981
|
+
durableAnalysis: null
|
|
4920
4982
|
});
|
|
4921
4983
|
await core.indexState.registerOpen(name);
|
|
4922
4984
|
if (core.durability) {
|
|
@@ -4927,8 +4989,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4927
4989
|
async function dropEngineIndex(core, name) {
|
|
4928
4990
|
core.guardShutdown();
|
|
4929
4991
|
const entry = core.requireIndex(name);
|
|
4992
|
+
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4993
|
+
const abandonChange = async () => {
|
|
4994
|
+
await (await import('./analysis-change-entry-5JSIMEKG.mjs')).abandonChangeOf(core, name);
|
|
4995
|
+
};
|
|
4996
|
+
if (changeRecorded()) await abandonChange();
|
|
4930
4997
|
await core.indexState.drop(name, async () => {
|
|
4931
4998
|
await core.analysisRebuild.abandon(name);
|
|
4999
|
+
if (changeRecorded()) await abandonChange();
|
|
4932
5000
|
await core.orchestrator.closeIndex(name);
|
|
4933
5001
|
if (core.durability) {
|
|
4934
5002
|
try {
|
|
@@ -6387,6 +6455,14 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
6387
6455
|
`Index "${indexName}" is already being rebalanced`
|
|
6388
6456
|
);
|
|
6389
6457
|
}
|
|
6458
|
+
const registered = ctx.requireIndex(indexName);
|
|
6459
|
+
if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
|
|
6460
|
+
throw new NarsilError(
|
|
6461
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6462
|
+
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
6463
|
+
{ indexName }
|
|
6464
|
+
);
|
|
6465
|
+
}
|
|
6390
6466
|
if (targetPartitionCount === manager.partitionCount) {
|
|
6391
6467
|
return;
|
|
6392
6468
|
}
|
|
@@ -6570,461 +6646,6 @@ function bm25Operations(core) {
|
|
|
6570
6646
|
};
|
|
6571
6647
|
}
|
|
6572
6648
|
|
|
6573
|
-
// src/persistence/archive/byte-reader.ts
|
|
6574
|
-
async function* chunksOf(source) {
|
|
6575
|
-
if (source instanceof Uint8Array) {
|
|
6576
|
-
yield source;
|
|
6577
|
-
return;
|
|
6578
|
-
}
|
|
6579
|
-
if (typeof source.getReader === "function") {
|
|
6580
|
-
const reader = source.getReader();
|
|
6581
|
-
try {
|
|
6582
|
-
for (; ; ) {
|
|
6583
|
-
const step = await reader.read();
|
|
6584
|
-
if (step.done) return;
|
|
6585
|
-
yield step.value;
|
|
6586
|
-
}
|
|
6587
|
-
} finally {
|
|
6588
|
-
reader.releaseLock();
|
|
6589
|
-
}
|
|
6590
|
-
}
|
|
6591
|
-
yield* source;
|
|
6592
|
-
}
|
|
6593
|
-
function createByteReader(source) {
|
|
6594
|
-
const iterator = chunksOf(source);
|
|
6595
|
-
const pending = [];
|
|
6596
|
-
let pendingBytes = 0;
|
|
6597
|
-
let ended = false;
|
|
6598
|
-
async function fill(length) {
|
|
6599
|
-
while (pendingBytes < length && !ended) {
|
|
6600
|
-
const step = await iterator.next();
|
|
6601
|
-
if (step.done) {
|
|
6602
|
-
ended = true;
|
|
6603
|
-
break;
|
|
6604
|
-
}
|
|
6605
|
-
if (step.value.byteLength === 0) continue;
|
|
6606
|
-
pending.push(step.value);
|
|
6607
|
-
pendingBytes += step.value.byteLength;
|
|
6608
|
-
}
|
|
6609
|
-
}
|
|
6610
|
-
function take(length) {
|
|
6611
|
-
const out = new Uint8Array(length);
|
|
6612
|
-
let filled = 0;
|
|
6613
|
-
while (filled < length) {
|
|
6614
|
-
const head = pending[0];
|
|
6615
|
-
const needed = length - filled;
|
|
6616
|
-
if (head.byteLength <= needed) {
|
|
6617
|
-
out.set(head, filled);
|
|
6618
|
-
filled += head.byteLength;
|
|
6619
|
-
pending.shift();
|
|
6620
|
-
} else {
|
|
6621
|
-
out.set(head.subarray(0, needed), filled);
|
|
6622
|
-
pending[0] = head.subarray(needed);
|
|
6623
|
-
filled += needed;
|
|
6624
|
-
}
|
|
6625
|
-
}
|
|
6626
|
-
pendingBytes -= length;
|
|
6627
|
-
return out;
|
|
6628
|
-
}
|
|
6629
|
-
return {
|
|
6630
|
-
async readExactly(length) {
|
|
6631
|
-
await fill(length);
|
|
6632
|
-
if (pendingBytes === 0 && length > 0) return null;
|
|
6633
|
-
if (pendingBytes < length) return take(pendingBytes);
|
|
6634
|
-
return take(length);
|
|
6635
|
-
},
|
|
6636
|
-
async peek(length) {
|
|
6637
|
-
await fill(length);
|
|
6638
|
-
const available = Math.min(length, pendingBytes);
|
|
6639
|
-
const copy = take(available);
|
|
6640
|
-
pending.unshift(copy);
|
|
6641
|
-
pendingBytes += copy.byteLength;
|
|
6642
|
-
return copy;
|
|
6643
|
-
},
|
|
6644
|
-
async close() {
|
|
6645
|
-
await iterator.return(void 0);
|
|
6646
|
-
}
|
|
6647
|
-
};
|
|
6648
|
-
}
|
|
6649
|
-
|
|
6650
|
-
// src/engine/export/lines.ts
|
|
6651
|
-
var NEWLINE = 10;
|
|
6652
|
-
function joined(parts, total) {
|
|
6653
|
-
if (parts.length === 1) return parts[0];
|
|
6654
|
-
const bytes = new Uint8Array(total);
|
|
6655
|
-
let offset = 0;
|
|
6656
|
-
for (const part of parts) {
|
|
6657
|
-
bytes.set(part, offset);
|
|
6658
|
-
offset += part.byteLength;
|
|
6659
|
-
}
|
|
6660
|
-
return bytes;
|
|
6661
|
-
}
|
|
6662
|
-
async function* exportLines(source, maxLineBytes) {
|
|
6663
|
-
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
6664
|
-
let parts = [];
|
|
6665
|
-
let pending = 0;
|
|
6666
|
-
let overflowing = false;
|
|
6667
|
-
let line = 0;
|
|
6668
|
-
const finishLine = () => {
|
|
6669
|
-
line += 1;
|
|
6670
|
-
if (overflowing) {
|
|
6671
|
-
overflowing = false;
|
|
6672
|
-
parts = [];
|
|
6673
|
-
pending = 0;
|
|
6674
|
-
return { line, refused: "too-long" };
|
|
6675
|
-
}
|
|
6676
|
-
const bytes = joined(parts, pending);
|
|
6677
|
-
parts = [];
|
|
6678
|
-
pending = 0;
|
|
6679
|
-
try {
|
|
6680
|
-
return { line, text: decoder.decode(bytes) };
|
|
6681
|
-
} catch {
|
|
6682
|
-
return { line, refused: "not-utf8" };
|
|
6683
|
-
}
|
|
6684
|
-
};
|
|
6685
|
-
for await (const chunk of chunksOf(source)) {
|
|
6686
|
-
let start = 0;
|
|
6687
|
-
for (; ; ) {
|
|
6688
|
-
const newline = chunk.indexOf(NEWLINE, start);
|
|
6689
|
-
const end = newline < 0 ? chunk.byteLength : newline;
|
|
6690
|
-
const piece = chunk.subarray(start, end);
|
|
6691
|
-
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
6692
|
-
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
6693
|
-
pending += piece.byteLength;
|
|
6694
|
-
} else {
|
|
6695
|
-
overflowing = true;
|
|
6696
|
-
parts = [];
|
|
6697
|
-
}
|
|
6698
|
-
if (newline < 0) break;
|
|
6699
|
-
yield finishLine();
|
|
6700
|
-
start = newline + 1;
|
|
6701
|
-
}
|
|
6702
|
-
}
|
|
6703
|
-
if (pending > 0 || overflowing) yield finishLine();
|
|
6704
|
-
}
|
|
6705
|
-
|
|
6706
|
-
// src/serialization/base64.ts
|
|
6707
|
-
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
6708
|
-
var PAD = 61;
|
|
6709
|
-
var BYTES_PER_GROUP = 3;
|
|
6710
|
-
var CHARACTERS_PER_GROUP = 4;
|
|
6711
|
-
var ASCII_LIMIT = 128;
|
|
6712
|
-
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
6713
|
-
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
6714
|
-
for (let at = 0; at < ALPHABET.length; at++) {
|
|
6715
|
-
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
6716
|
-
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
6717
|
-
}
|
|
6718
|
-
var asciiDecoder = new TextDecoder();
|
|
6719
|
-
function encodeBase64(bytes) {
|
|
6720
|
-
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
6721
|
-
let write = 0;
|
|
6722
|
-
let read = 0;
|
|
6723
|
-
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
6724
|
-
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
6725
|
-
out[write++] = ENCODE[group >>> 18];
|
|
6726
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
6727
|
-
out[write++] = ENCODE[group >>> 6 & 63];
|
|
6728
|
-
out[write++] = ENCODE[group & 63];
|
|
6729
|
-
}
|
|
6730
|
-
const left = bytes.byteLength - read;
|
|
6731
|
-
if (left > 0) {
|
|
6732
|
-
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
6733
|
-
out[write++] = ENCODE[group >>> 18];
|
|
6734
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
6735
|
-
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
6736
|
-
out[write++] = PAD;
|
|
6737
|
-
}
|
|
6738
|
-
return asciiDecoder.decode(out);
|
|
6739
|
-
}
|
|
6740
|
-
function sextetAt(text, at) {
|
|
6741
|
-
const code = text.charCodeAt(at);
|
|
6742
|
-
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
6743
|
-
}
|
|
6744
|
-
function decodeExactBase64(text) {
|
|
6745
|
-
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
6746
|
-
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
6747
|
-
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
6748
|
-
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
6749
|
-
let write = 0;
|
|
6750
|
-
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
6751
|
-
const a2 = sextetAt(text, at);
|
|
6752
|
-
const b2 = sextetAt(text, at + 1);
|
|
6753
|
-
const c2 = sextetAt(text, at + 2);
|
|
6754
|
-
const d = sextetAt(text, at + 3);
|
|
6755
|
-
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
6756
|
-
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
6757
|
-
out[write++] = group >>> 16;
|
|
6758
|
-
out[write++] = group >>> 8 & 255;
|
|
6759
|
-
out[write++] = group & 255;
|
|
6760
|
-
}
|
|
6761
|
-
if (padding === 0) return out;
|
|
6762
|
-
const a = sextetAt(text, fullEnd);
|
|
6763
|
-
const b = sextetAt(text, fullEnd + 1);
|
|
6764
|
-
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
6765
|
-
if ((a | b | c) < 0) return null;
|
|
6766
|
-
if (padding === 2 && (b & 15) !== 0) return null;
|
|
6767
|
-
if (padding === 1 && (c & 3) !== 0) return null;
|
|
6768
|
-
out[write++] = a << 2 | b >>> 4;
|
|
6769
|
-
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
6770
|
-
return out;
|
|
6771
|
-
}
|
|
6772
|
-
|
|
6773
|
-
// src/engine/export/tags.ts
|
|
6774
|
-
var TAG_PREFIX = "$";
|
|
6775
|
-
var DATE_TAG = "$date";
|
|
6776
|
-
var BYTES_TAG = "$bytes";
|
|
6777
|
-
var NUMBER_TAG = "$number";
|
|
6778
|
-
var OBJECT_TAG = "$object";
|
|
6779
|
-
var MILLISECONDS_PER_SECOND = 1e3;
|
|
6780
|
-
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
6781
|
-
var MAX_NANOSECONDS = 999999999;
|
|
6782
|
-
var MAX_DATE_MILLISECONDS = 864e13;
|
|
6783
|
-
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
6784
|
-
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
6785
|
-
["NaN", Number.NaN],
|
|
6786
|
-
["Infinity", Number.POSITIVE_INFINITY],
|
|
6787
|
-
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
6788
|
-
]);
|
|
6789
|
-
function refuse(path, reason) {
|
|
6790
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
6791
|
-
field: path
|
|
6792
|
-
});
|
|
6793
|
-
}
|
|
6794
|
-
function isRecord(value) {
|
|
6795
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6796
|
-
}
|
|
6797
|
-
function startsWithTagPrefix(key) {
|
|
6798
|
-
return key.startsWith(TAG_PREFIX);
|
|
6799
|
-
}
|
|
6800
|
-
function soleTagKey(record) {
|
|
6801
|
-
let sole = null;
|
|
6802
|
-
for (const key of Object.keys(record)) {
|
|
6803
|
-
if (sole !== null) return null;
|
|
6804
|
-
sole = key;
|
|
6805
|
-
}
|
|
6806
|
-
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
6807
|
-
}
|
|
6808
|
-
function dateOf(value, path) {
|
|
6809
|
-
if (!Array.isArray(value) || value.length !== 2)
|
|
6810
|
-
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
6811
|
-
const seconds = value[0];
|
|
6812
|
-
const nanoseconds = value[1];
|
|
6813
|
-
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
6814
|
-
refuse(
|
|
6815
|
-
path,
|
|
6816
|
-
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
6817
|
-
);
|
|
6818
|
-
}
|
|
6819
|
-
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
6820
|
-
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
6821
|
-
}
|
|
6822
|
-
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
6823
|
-
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
6824
|
-
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
6825
|
-
return new Date(milliseconds);
|
|
6826
|
-
}
|
|
6827
|
-
function bytesOf(value, path) {
|
|
6828
|
-
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
6829
|
-
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
6830
|
-
return bytes;
|
|
6831
|
-
}
|
|
6832
|
-
function numberOf(value, path) {
|
|
6833
|
-
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
6834
|
-
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
6835
|
-
return number;
|
|
6836
|
-
}
|
|
6837
|
-
function requireDepth(path, depth) {
|
|
6838
|
-
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
6839
|
-
}
|
|
6840
|
-
function untagMap(record, path, depth) {
|
|
6841
|
-
requireDepth(path, depth);
|
|
6842
|
-
for (const key of Object.keys(record)) {
|
|
6843
|
-
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
6844
|
-
}
|
|
6845
|
-
return record;
|
|
6846
|
-
}
|
|
6847
|
-
function untagList(list, path, depth) {
|
|
6848
|
-
requireDepth(path, depth);
|
|
6849
|
-
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
6850
|
-
return list;
|
|
6851
|
-
}
|
|
6852
|
-
function untagTag(tag, value, path, depth) {
|
|
6853
|
-
switch (tag) {
|
|
6854
|
-
case DATE_TAG:
|
|
6855
|
-
return dateOf(value, path);
|
|
6856
|
-
case BYTES_TAG:
|
|
6857
|
-
return bytesOf(value, path);
|
|
6858
|
-
case NUMBER_TAG:
|
|
6859
|
-
return numberOf(value, path);
|
|
6860
|
-
case OBJECT_TAG:
|
|
6861
|
-
if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
6862
|
-
return untagMap(value, path, depth);
|
|
6863
|
-
default:
|
|
6864
|
-
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
6865
|
-
}
|
|
6866
|
-
}
|
|
6867
|
-
function untag(value, path, depth) {
|
|
6868
|
-
if (typeof value !== "object" || value === null) return value;
|
|
6869
|
-
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
6870
|
-
const record = value;
|
|
6871
|
-
const tag = soleTagKey(record);
|
|
6872
|
-
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
6873
|
-
}
|
|
6874
|
-
function untagDocumentLine(text, parsed) {
|
|
6875
|
-
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
6876
|
-
return untagMap(parsed, "", 0);
|
|
6877
|
-
}
|
|
6878
|
-
|
|
6879
|
-
// src/engine/export/import.ts
|
|
6880
|
-
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
6881
|
-
var DOCUMENT_EXPORT_VERSION = 1;
|
|
6882
|
-
function invalidHeader(message) {
|
|
6883
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
6884
|
-
}
|
|
6885
|
-
function isRecord2(value) {
|
|
6886
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6887
|
-
}
|
|
6888
|
-
function configOfHeader(line) {
|
|
6889
|
-
if (line === void 0) invalidHeader("is missing");
|
|
6890
|
-
if (!("text" in line)) {
|
|
6891
|
-
invalidHeader(
|
|
6892
|
-
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
6893
|
-
);
|
|
6894
|
-
}
|
|
6895
|
-
let raw;
|
|
6896
|
-
try {
|
|
6897
|
-
raw = JSON.parse(line.text);
|
|
6898
|
-
} catch {
|
|
6899
|
-
invalidHeader("is not JSON");
|
|
6900
|
-
}
|
|
6901
|
-
if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
6902
|
-
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
6903
|
-
}
|
|
6904
|
-
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
6905
|
-
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
6906
|
-
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
6907
|
-
return {
|
|
6908
|
-
schema: configuration.schema,
|
|
6909
|
-
language: configuration.language,
|
|
6910
|
-
...fields,
|
|
6911
|
-
...embedding === void 0 ? {} : {
|
|
6912
|
-
embedding: {
|
|
6913
|
-
fields: embedding.fields,
|
|
6914
|
-
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
6915
|
-
}
|
|
6916
|
-
}
|
|
6917
|
-
};
|
|
6918
|
-
}
|
|
6919
|
-
async function prepareIndex(engine, indexName, config) {
|
|
6920
|
-
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
6921
|
-
await engine.createIndex(indexName, config);
|
|
6922
|
-
return;
|
|
6923
|
-
}
|
|
6924
|
-
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
6925
|
-
const exported = flattenSchema(config.schema);
|
|
6926
|
-
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
6927
|
-
for (const field of fields) {
|
|
6928
|
-
if (existing[field] === exported[field]) continue;
|
|
6929
|
-
throw new NarsilError(
|
|
6930
|
-
ErrorCodes.CONFIG_INVALID,
|
|
6931
|
-
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
6932
|
-
{ indexName, field }
|
|
6933
|
-
);
|
|
6934
|
-
}
|
|
6935
|
-
}
|
|
6936
|
-
function highestReportedAt(refused) {
|
|
6937
|
-
let highest = 0;
|
|
6938
|
-
for (let at = 1; at < refused.length; at++) {
|
|
6939
|
-
if (refused[at].line > refused[highest].line) highest = at;
|
|
6940
|
-
}
|
|
6941
|
-
return highest;
|
|
6942
|
-
}
|
|
6943
|
-
function refuse2(state, line, code, message) {
|
|
6944
|
-
state.result.refusedCount += 1;
|
|
6945
|
-
const refused = state.result.refused;
|
|
6946
|
-
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
6947
|
-
refused.push({ line, code, message });
|
|
6948
|
-
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
6949
|
-
return;
|
|
6950
|
-
}
|
|
6951
|
-
if (line >= refused[state.highestReported].line) return;
|
|
6952
|
-
refused[state.highestReported] = { line, code, message };
|
|
6953
|
-
state.highestReported = highestReportedAt(refused);
|
|
6954
|
-
}
|
|
6955
|
-
async function flush(state) {
|
|
6956
|
-
if (state.batch.length === 0) return;
|
|
6957
|
-
const batch = state.batch;
|
|
6958
|
-
state.batch = [];
|
|
6959
|
-
state.batchIds = /* @__PURE__ */ new Set();
|
|
6960
|
-
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
6961
|
-
const inserts = [];
|
|
6962
|
-
const updates = [];
|
|
6963
|
-
for (const entry of batch) {
|
|
6964
|
-
if (await state.engine.has(state.indexName, entry.docId))
|
|
6965
|
-
updates.push({ docId: entry.docId, document: entry.document });
|
|
6966
|
-
else inserts.push(entry.document);
|
|
6967
|
-
}
|
|
6968
|
-
for (const outcome of [
|
|
6969
|
-
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
6970
|
-
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
6971
|
-
]) {
|
|
6972
|
-
if (outcome === null) continue;
|
|
6973
|
-
state.result.imported += outcome.succeeded.length;
|
|
6974
|
-
for (const failure of outcome.failed) {
|
|
6975
|
-
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
6976
|
-
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
6977
|
-
}
|
|
6978
|
-
}
|
|
6979
|
-
}
|
|
6980
|
-
async function takeLine(state, entry) {
|
|
6981
|
-
if (!("text" in entry)) {
|
|
6982
|
-
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
6983
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
6984
|
-
return;
|
|
6985
|
-
}
|
|
6986
|
-
if (entry.text.length === 0) return;
|
|
6987
|
-
let document;
|
|
6988
|
-
try {
|
|
6989
|
-
document = JSON.parse(entry.text);
|
|
6990
|
-
} catch {
|
|
6991
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
6992
|
-
return;
|
|
6993
|
-
}
|
|
6994
|
-
if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
6995
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
6996
|
-
return;
|
|
6997
|
-
}
|
|
6998
|
-
try {
|
|
6999
|
-
untagDocumentLine(entry.text, document);
|
|
7000
|
-
} catch (error) {
|
|
7001
|
-
if (!(error instanceof NarsilError)) throw error;
|
|
7002
|
-
refuse2(state, entry.line, error.code, error.message);
|
|
7003
|
-
return;
|
|
7004
|
-
}
|
|
7005
|
-
if (state.batchIds.has(document.id)) await flush(state);
|
|
7006
|
-
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
7007
|
-
state.batchIds.add(document.id);
|
|
7008
|
-
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
7009
|
-
}
|
|
7010
|
-
async function readDocumentExport(engine, indexName, source) {
|
|
7011
|
-
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
7012
|
-
const header = await lines.next();
|
|
7013
|
-
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
7014
|
-
const state = {
|
|
7015
|
-
engine,
|
|
7016
|
-
indexName,
|
|
7017
|
-
batch: [],
|
|
7018
|
-
batchIds: /* @__PURE__ */ new Set(),
|
|
7019
|
-
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
7020
|
-
highestReported: 0
|
|
7021
|
-
};
|
|
7022
|
-
for await (const entry of lines) await takeLine(state, entry);
|
|
7023
|
-
await flush(state);
|
|
7024
|
-
state.result.refused.sort((a, b) => a.line - b.line);
|
|
7025
|
-
return state.result;
|
|
7026
|
-
}
|
|
7027
|
-
|
|
7028
6649
|
// src/narsil/portability.ts
|
|
7029
6650
|
function portabilityOperations(core, config, engine) {
|
|
7030
6651
|
async function withOpenIndex2(indexName, action) {
|
|
@@ -7056,14 +6677,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
7056
6677
|
return {
|
|
7057
6678
|
snapshot(indexName) {
|
|
7058
6679
|
return withOpenIndex2(indexName, async () => {
|
|
7059
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6680
|
+
const { createSnapshot } = await import('./snapshot-BTRZ3MZK.mjs');
|
|
7060
6681
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
7061
6682
|
});
|
|
7062
6683
|
},
|
|
7063
6684
|
async restore(indexName, data) {
|
|
7064
6685
|
core.guardShutdown();
|
|
7065
6686
|
validateIndexName(indexName);
|
|
7066
|
-
const restored = import('./restore-queue-
|
|
6687
|
+
const restored = import('./restore-queue-RYSRRFS4.mjs').then(
|
|
7067
6688
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
7068
6689
|
);
|
|
7069
6690
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -7072,8 +6693,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
7072
6693
|
exportDocuments(indexName) {
|
|
7073
6694
|
return withOpenIndex2(indexName, async () => {
|
|
7074
6695
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
7075
|
-
import('./snapshot-
|
|
7076
|
-
import('./write-
|
|
6696
|
+
import('./snapshot-BTRZ3MZK.mjs'),
|
|
6697
|
+
import('./write-GHTA56OW.mjs')
|
|
7077
6698
|
]);
|
|
7078
6699
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
7079
6700
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -7081,7 +6702,10 @@ function portabilityOperations(core, config, engine) {
|
|
|
7081
6702
|
},
|
|
7082
6703
|
importDocumentExport(indexName, data) {
|
|
7083
6704
|
core.guardShutdown();
|
|
7084
|
-
|
|
6705
|
+
const imported = import('./import-QRZAQ5RH.mjs').then(
|
|
6706
|
+
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6707
|
+
);
|
|
6708
|
+
return emitWriteEventAfter(core, indexName, imported);
|
|
7085
6709
|
}
|
|
7086
6710
|
};
|
|
7087
6711
|
}
|
|
@@ -7092,13 +6716,19 @@ async function enterRead(core, indexName) {
|
|
|
7092
6716
|
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
|
|
7093
6717
|
}
|
|
7094
6718
|
async function readUnderOneAnalysis(core, indexName, read) {
|
|
7095
|
-
|
|
7096
|
-
|
|
7097
|
-
|
|
7098
|
-
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
7099
|
-
|
|
6719
|
+
const manager = core.requireManager(indexName);
|
|
6720
|
+
for (; ; ) {
|
|
6721
|
+
const analysis = manager.analysis;
|
|
6722
|
+
const analysisStale = core.analysisRebuild.isStale(indexName);
|
|
6723
|
+
try {
|
|
6724
|
+
const result = await read();
|
|
6725
|
+
if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
|
|
6726
|
+
return { result, analysisStale };
|
|
6727
|
+
}
|
|
6728
|
+
} catch (error) {
|
|
6729
|
+
if (manager.analysis === analysis) throw error;
|
|
6730
|
+
}
|
|
7100
6731
|
}
|
|
7101
|
-
return { result, analysisStale };
|
|
7102
6732
|
}
|
|
7103
6733
|
function phraseReadinessOf(core, indexName, config) {
|
|
7104
6734
|
return () => {
|
|
@@ -7131,7 +6761,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
7131
6761
|
const manager = core.requireManager(indexName);
|
|
7132
6762
|
const scoped = scopedParams(params, options);
|
|
7133
6763
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
7134
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
6764
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-DYV75DCB.mjs');
|
|
7135
6765
|
const resolvedParams = await resolveVectorText(
|
|
7136
6766
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
7137
6767
|
entry.embeddingAdapter,
|
|
@@ -7139,25 +6769,24 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
7139
6769
|
entry.embeddingAdapterName
|
|
7140
6770
|
);
|
|
7141
6771
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
7142
|
-
const context = {
|
|
7143
|
-
manager,
|
|
7144
|
-
language: entry.language,
|
|
7145
|
-
config: entry.config,
|
|
7146
|
-
workerSearch,
|
|
7147
|
-
indexName,
|
|
7148
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
7149
|
-
partitionIds: options?.partitionIds,
|
|
7150
|
-
rescoreWindow: options?.rescoreWindow,
|
|
7151
|
-
indexBm25: options?.indexBm25,
|
|
7152
|
-
cursorBinding: lazyQueryBinding(params),
|
|
7153
|
-
patternWorkCap: core.patternWorkCap,
|
|
7154
|
-
rerank: rerank?.rerankRunnerFor(core),
|
|
7155
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
7156
|
-
};
|
|
7157
6772
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
7158
6773
|
core,
|
|
7159
6774
|
indexName,
|
|
7160
|
-
() => executeQuery(resolvedParams,
|
|
6775
|
+
() => executeQuery(resolvedParams, {
|
|
6776
|
+
manager,
|
|
6777
|
+
language: manager.language,
|
|
6778
|
+
config: manager.config,
|
|
6779
|
+
workerSearch,
|
|
6780
|
+
indexName,
|
|
6781
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
6782
|
+
partitionIds: options?.partitionIds,
|
|
6783
|
+
rescoreWindow: options?.rescoreWindow,
|
|
6784
|
+
indexBm25: options?.indexBm25,
|
|
6785
|
+
cursorBinding: lazyQueryBinding(params),
|
|
6786
|
+
patternWorkCap: core.patternWorkCap,
|
|
6787
|
+
rerank: rerank?.rerankRunnerFor(core),
|
|
6788
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
6789
|
+
})
|
|
7161
6790
|
);
|
|
7162
6791
|
if (analysisStale) {
|
|
7163
6792
|
result.analysisStale = true;
|
|
@@ -7191,23 +6820,22 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
7191
6820
|
entry.embeddingAdapterName
|
|
7192
6821
|
);
|
|
7193
6822
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
7194
|
-
const context = {
|
|
7195
|
-
manager,
|
|
7196
|
-
language: entry.language,
|
|
7197
|
-
config: entry.config,
|
|
7198
|
-
workerSearch,
|
|
7199
|
-
indexName,
|
|
7200
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
7201
|
-
partitionIds: options?.partitionIds,
|
|
7202
|
-
indexBm25: options?.indexBm25,
|
|
7203
|
-
cursorBinding: lazyQueryBinding(params),
|
|
7204
|
-
patternWorkCap: core.patternWorkCap,
|
|
7205
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
7206
|
-
};
|
|
7207
6823
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
7208
6824
|
core,
|
|
7209
6825
|
indexName,
|
|
7210
|
-
() => executePreflight(resolvedParams,
|
|
6826
|
+
() => executePreflight(resolvedParams, {
|
|
6827
|
+
manager,
|
|
6828
|
+
language: manager.language,
|
|
6829
|
+
config: manager.config,
|
|
6830
|
+
workerSearch,
|
|
6831
|
+
indexName,
|
|
6832
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
6833
|
+
partitionIds: options?.partitionIds,
|
|
6834
|
+
indexBm25: options?.indexBm25,
|
|
6835
|
+
cursorBinding: lazyQueryBinding(params),
|
|
6836
|
+
patternWorkCap: core.patternWorkCap,
|
|
6837
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
6838
|
+
})
|
|
7211
6839
|
);
|
|
7212
6840
|
if (analysisStale) {
|
|
7213
6841
|
result.analysisStale = true;
|
|
@@ -7448,9 +7076,20 @@ function createNarsilFromCore(core, config) {
|
|
|
7448
7076
|
return runEngineSuggest(core, indexName, params);
|
|
7449
7077
|
},
|
|
7450
7078
|
async rebuildAnalysis(indexName) {
|
|
7079
|
+
guardShutdown();
|
|
7080
|
+
const entry = requireIndex(indexName);
|
|
7081
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
7082
|
+
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
7083
|
+
return (await import('./analysis-change-JN23AZQP.mjs')).resumeAnalysisChange(core, indexName);
|
|
7084
|
+
});
|
|
7085
|
+
},
|
|
7086
|
+
async changeAnalysis(indexName, change) {
|
|
7451
7087
|
guardShutdown();
|
|
7452
7088
|
requireIndex(indexName);
|
|
7453
|
-
await withOpenIndexWrite(
|
|
7089
|
+
await withOpenIndexWrite(
|
|
7090
|
+
indexName,
|
|
7091
|
+
async () => (await import('./analysis-change-JN23AZQP.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
7092
|
+
);
|
|
7454
7093
|
},
|
|
7455
7094
|
...portabilityOperations(core, config, () => narsil),
|
|
7456
7095
|
...synonymOperations(core),
|
|
@@ -7596,4 +7235,4 @@ async function createNarsil(config) {
|
|
|
7596
7235
|
// src/index.ts
|
|
7597
7236
|
var VERSION = "0.1.0";
|
|
7598
7237
|
|
|
7599
|
-
export {
|
|
7238
|
+
export { HEADER_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };
|