@delali/narsil 0.3.1-next.144 → 0.3.1-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
- package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
- package/dist/browser/import-FTMLTPGQ.mjs +9 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
- package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
- package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
- package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
- package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
- package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
- package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
- package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
- package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
- package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
- package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
- package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
- package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
- package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
- package/dist/chunk-NWTEJ2T7.mjs +100 -0
- package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
- package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/chunk-PRID7CA6.mjs +104 -0
- package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
- package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
- package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
- package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
- package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
- package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
- package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
- package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
- package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
- package/dist/chunk-SVO6NV4D.mjs +248 -0
- package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
- package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
- package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
- package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
- package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
- package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
- package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +44 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
- package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
- package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16219 -14535
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
- package/dist/import-6KXFY6XI.mjs +35 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -30
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
- package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
- package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +9 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
- package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +153 -36
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
- package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +5 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
- package/openapi.json +349 -2
- package/package.json +27 -8
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-KGOXHLLZ.mjs +0 -693
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-QSK35UC2.mjs +0 -35
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
|
@@ -1,27 +1,21 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-
|
|
2
|
-
import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-5WYVUPGD.mjs';
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-HOC5JP2H.mjs';
|
|
3
2
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
4
|
-
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-
|
|
5
|
-
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
3
|
+
import { splitWorkerBudget, resolveWorkerCount, detectRuntime, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-G6Q5DEF5.mjs';
|
|
6
4
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
7
|
-
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-
|
|
8
|
-
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-
|
|
9
|
-
import { generateId } from './chunk-N37VMC62.mjs';
|
|
10
|
-
import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
|
|
5
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, generateId, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-OH7SX5MY.mjs';
|
|
6
|
+
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, readHeapStatistics, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-QYMAYU7M.mjs';
|
|
11
7
|
import { countEngineThread } from './chunk-EBIHRL7B.mjs';
|
|
12
|
-
import {
|
|
13
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
|
|
14
|
-
import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
|
|
15
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KGOXHLLZ.mjs';
|
|
16
|
-
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
|
-
import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
|
|
18
|
-
import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
|
|
19
|
-
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
20
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
|
+
import { getLanguage, withOpenIndex, oneAtATime } from './chunk-FRJ5RGX7.mjs';
|
|
21
9
|
import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
|
|
22
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
10
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-6Z22L5RN.mjs';
|
|
11
|
+
import { requireSearchBackendSettings } from './chunk-QXH2CS4V.mjs';
|
|
23
12
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
24
|
-
import {
|
|
13
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-LYKUJNAV.mjs';
|
|
14
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, getNestedValue, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-BTARFTEN.mjs';
|
|
15
|
+
import { isSingleStringFieldType } from './chunk-GVPYMSLO.mjs';
|
|
16
|
+
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
|
+
import { ErrorCodes, NarsilError } from './chunk-IQHZMCLX.mjs';
|
|
18
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
25
19
|
import { encode, decode } from '@msgpack/msgpack';
|
|
26
20
|
import * as zlib from 'zlib';
|
|
27
21
|
|
|
@@ -376,7 +370,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
376
370
|
// src/partitioning/rewrite-spill-of.ts
|
|
377
371
|
async function rewriteSpillOf(storage, manager, target) {
|
|
378
372
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
379
|
-
const { openRewriteSpill } = await import('./module-
|
|
373
|
+
const { openRewriteSpill } = await import('./module-TMFIWOD2.mjs');
|
|
380
374
|
return openRewriteSpill(storage, manager, target);
|
|
381
375
|
}
|
|
382
376
|
|
|
@@ -920,7 +914,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
920
914
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
921
915
|
if (existing !== void 0) {
|
|
922
916
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
923
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
917
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-D5APH4Z3.mjs');
|
|
924
918
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
925
919
|
}
|
|
926
920
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -4042,7 +4036,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
4042
4036
|
// src/engine/core.ts
|
|
4043
4037
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4044
4038
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4045
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
4039
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-RAOCKSYA.mjs')).resolveRerankSettings(config);
|
|
4046
4040
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4047
4041
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4048
4042
|
const vectorCopyPolicy = {
|
|
@@ -4053,7 +4047,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4053
4047
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4054
4048
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4055
4049
|
}
|
|
4056
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4050
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-K7RDXWL6.mjs');
|
|
4057
4051
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4058
4052
|
const executor = createDirectExecutor({
|
|
4059
4053
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4238,7 +4232,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4238
4232
|
requireManager,
|
|
4239
4233
|
resumeChange: async (indexName) => {
|
|
4240
4234
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4241
|
-
await (await import('./analysis-change-entry-
|
|
4235
|
+
await (await import('./analysis-change-entry-V4SQQSXQ.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4242
4236
|
},
|
|
4243
4237
|
onClose: hooks?.onIndexClose,
|
|
4244
4238
|
onOpen: (indexName) => {
|
|
@@ -4320,7 +4314,7 @@ async function startEngineCore(core, config) {
|
|
|
4320
4314
|
checkHeapAfterRecovery(core);
|
|
4321
4315
|
}
|
|
4322
4316
|
if (core.invalidation) await core.invalidation.start();
|
|
4323
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
4317
|
+
if (core.durability) await (await import('./analysis-change-entry-V4SQQSXQ.mjs')).resumeEveryChange(core);
|
|
4324
4318
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4325
4319
|
} catch (err) {
|
|
4326
4320
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4358,7 +4352,7 @@ async function shutdownEngine(core) {
|
|
|
4358
4352
|
}
|
|
4359
4353
|
}
|
|
4360
4354
|
if (core.rerankAdapters.size > 0) {
|
|
4361
|
-
await (await import('./runtime-
|
|
4355
|
+
await (await import('./runtime-RAOCKSYA.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4362
4356
|
}
|
|
4363
4357
|
await orchestrator.shutdown();
|
|
4364
4358
|
await executor.shutdown();
|
|
@@ -4924,7 +4918,7 @@ async function dropEngineIndex(core, name) {
|
|
|
4924
4918
|
const entry = core.requireIndex(name);
|
|
4925
4919
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4926
4920
|
const abandonChange = async () => {
|
|
4927
|
-
await (await import('./analysis-change-entry-
|
|
4921
|
+
await (await import('./analysis-change-entry-V4SQQSXQ.mjs')).abandonChangeOf(core, name);
|
|
4928
4922
|
};
|
|
4929
4923
|
if (changeRecorded()) await abandonChange();
|
|
4930
4924
|
await core.indexState.drop(name, async () => {
|
|
@@ -4975,6 +4969,224 @@ function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
|
4975
4969
|
}
|
|
4976
4970
|
}
|
|
4977
4971
|
|
|
4972
|
+
// src/engine/embed.ts
|
|
4973
|
+
function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
|
|
4974
|
+
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
4975
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4976
|
+
if (existing === void 0 || existing === null) {
|
|
4977
|
+
throw new NarsilError(
|
|
4978
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4979
|
+
adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
|
|
4980
|
+
adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
|
|
4981
|
+
);
|
|
4982
|
+
}
|
|
4983
|
+
}
|
|
4984
|
+
}
|
|
4985
|
+
function validateVector(vector, targetField, expectedDimensions) {
|
|
4986
|
+
if (!(vector instanceof Float32Array)) {
|
|
4987
|
+
throw new NarsilError(
|
|
4988
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4989
|
+
`Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
|
|
4990
|
+
{ field: targetField, returnedType: typeof vector }
|
|
4991
|
+
);
|
|
4992
|
+
}
|
|
4993
|
+
if (vector.length !== expectedDimensions) {
|
|
4994
|
+
throw new NarsilError(
|
|
4995
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
4996
|
+
`Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
|
|
4997
|
+
{ field: targetField, expected: expectedDimensions, actual: vector.length }
|
|
4998
|
+
);
|
|
4999
|
+
}
|
|
5000
|
+
for (let i = 0; i < vector.length; i++) {
|
|
5001
|
+
if (Number.isNaN(vector[i])) {
|
|
5002
|
+
throw new NarsilError(
|
|
5003
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
5004
|
+
`Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
|
|
5005
|
+
{ field: targetField, index: i }
|
|
5006
|
+
);
|
|
5007
|
+
}
|
|
5008
|
+
}
|
|
5009
|
+
return vector;
|
|
5010
|
+
}
|
|
5011
|
+
function resolveFieldValue(document, path) {
|
|
5012
|
+
if (!path.includes(".")) return document[path];
|
|
5013
|
+
const segments = path.split(".");
|
|
5014
|
+
let current = document;
|
|
5015
|
+
for (const segment of segments) {
|
|
5016
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
5017
|
+
current = current[segment];
|
|
5018
|
+
}
|
|
5019
|
+
return current;
|
|
5020
|
+
}
|
|
5021
|
+
function setFieldValue(document, path, value) {
|
|
5022
|
+
if (!path.includes(".")) {
|
|
5023
|
+
document[path] = value;
|
|
5024
|
+
return;
|
|
5025
|
+
}
|
|
5026
|
+
const segments = path.split(".");
|
|
5027
|
+
let current = document;
|
|
5028
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
5029
|
+
let next = current[segments[i]];
|
|
5030
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
5031
|
+
next = {};
|
|
5032
|
+
current[segments[i]] = next;
|
|
5033
|
+
}
|
|
5034
|
+
current = next;
|
|
5035
|
+
}
|
|
5036
|
+
current[segments[segments.length - 1]] = value;
|
|
5037
|
+
}
|
|
5038
|
+
function deleteFieldValue(document, path) {
|
|
5039
|
+
if (!path.includes(".")) {
|
|
5040
|
+
delete document[path];
|
|
5041
|
+
return;
|
|
5042
|
+
}
|
|
5043
|
+
const segments = path.split(".");
|
|
5044
|
+
let current = document;
|
|
5045
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
5046
|
+
const next = current[segments[i]];
|
|
5047
|
+
if (next === null || next === void 0 || typeof next !== "object") return;
|
|
5048
|
+
current = next;
|
|
5049
|
+
}
|
|
5050
|
+
delete current[segments[segments.length - 1]];
|
|
5051
|
+
}
|
|
5052
|
+
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
5053
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
5054
|
+
const parts = [];
|
|
5055
|
+
for (const source of sources) {
|
|
5056
|
+
const value = resolveFieldValue(document, source);
|
|
5057
|
+
if (value === void 0 || value === null || value === "") continue;
|
|
5058
|
+
if (typeof value !== "string") {
|
|
5059
|
+
throw new NarsilError(
|
|
5060
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5061
|
+
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
5062
|
+
{ field: source, targetField }
|
|
5063
|
+
);
|
|
5064
|
+
}
|
|
5065
|
+
parts.push(value);
|
|
5066
|
+
}
|
|
5067
|
+
if (parts.length === 0) {
|
|
5068
|
+
const details = { field: targetField };
|
|
5069
|
+
if (documentIndex !== void 0) {
|
|
5070
|
+
details.documentIndex = documentIndex;
|
|
5071
|
+
}
|
|
5072
|
+
throw new NarsilError(
|
|
5073
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
5074
|
+
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
5075
|
+
details
|
|
5076
|
+
);
|
|
5077
|
+
}
|
|
5078
|
+
return parts.join("\n");
|
|
5079
|
+
}
|
|
5080
|
+
function wrapAdapterError(err, targetField) {
|
|
5081
|
+
if (err instanceof NarsilError) return err;
|
|
5082
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
5083
|
+
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
5084
|
+
field: targetField,
|
|
5085
|
+
cause: message
|
|
5086
|
+
});
|
|
5087
|
+
}
|
|
5088
|
+
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
5089
|
+
const embeddedFields = /* @__PURE__ */ new Set();
|
|
5090
|
+
try {
|
|
5091
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
5092
|
+
const existing = resolveFieldValue(document, targetField);
|
|
5093
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
5094
|
+
const text = collectSourceText(document, sourceFields, targetField);
|
|
5095
|
+
let vector;
|
|
5096
|
+
try {
|
|
5097
|
+
const raw = await adapter.embed(text, "document", signal);
|
|
5098
|
+
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
5099
|
+
} catch (err) {
|
|
5100
|
+
throw wrapAdapterError(err, targetField);
|
|
5101
|
+
}
|
|
5102
|
+
setFieldValue(document, targetField, vector);
|
|
5103
|
+
embeddedFields.add(targetField);
|
|
5104
|
+
}
|
|
5105
|
+
} catch (err) {
|
|
5106
|
+
for (const field of embeddedFields) {
|
|
5107
|
+
deleteFieldValue(document, field);
|
|
5108
|
+
}
|
|
5109
|
+
throw err;
|
|
5110
|
+
}
|
|
5111
|
+
return embeddedFields;
|
|
5112
|
+
}
|
|
5113
|
+
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([
|
|
5114
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
5115
|
+
ErrorCodes.DOC_VALIDATION_FAILED
|
|
5116
|
+
]);
|
|
5117
|
+
function isDocumentSourceError(err) {
|
|
5118
|
+
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
5119
|
+
}
|
|
5120
|
+
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
5121
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
5122
|
+
const failed = /* @__PURE__ */ new Map();
|
|
5123
|
+
try {
|
|
5124
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
5125
|
+
const needsEmbedding = [];
|
|
5126
|
+
for (let i = 0; i < documents.length; i++) {
|
|
5127
|
+
if (failed.has(i)) continue;
|
|
5128
|
+
const doc = documents[i];
|
|
5129
|
+
const existing = resolveFieldValue(doc, targetField);
|
|
5130
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
5131
|
+
try {
|
|
5132
|
+
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
5133
|
+
needsEmbedding.push({ docIndex: i, text });
|
|
5134
|
+
} catch (err) {
|
|
5135
|
+
if (isDocumentSourceError(err)) {
|
|
5136
|
+
failed.set(i, err);
|
|
5137
|
+
} else {
|
|
5138
|
+
throw err;
|
|
5139
|
+
}
|
|
5140
|
+
}
|
|
5141
|
+
}
|
|
5142
|
+
if (needsEmbedding.length === 0) continue;
|
|
5143
|
+
const texts = needsEmbedding.map((entry) => entry.text);
|
|
5144
|
+
let vectors;
|
|
5145
|
+
try {
|
|
5146
|
+
if (adapter.embedBatch) {
|
|
5147
|
+
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
5148
|
+
} else {
|
|
5149
|
+
vectors = [];
|
|
5150
|
+
const concurrency = 8;
|
|
5151
|
+
for (let start = 0; start < texts.length; start += concurrency) {
|
|
5152
|
+
const chunk = texts.slice(start, start + concurrency);
|
|
5153
|
+
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
5154
|
+
vectors.push(...chunkResults);
|
|
5155
|
+
}
|
|
5156
|
+
}
|
|
5157
|
+
} catch (err) {
|
|
5158
|
+
throw wrapAdapterError(err, targetField);
|
|
5159
|
+
}
|
|
5160
|
+
if (vectors.length !== needsEmbedding.length) {
|
|
5161
|
+
throw new NarsilError(
|
|
5162
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
5163
|
+
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
5164
|
+
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
5165
|
+
);
|
|
5166
|
+
}
|
|
5167
|
+
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
5168
|
+
const { docIndex } = needsEmbedding[j];
|
|
5169
|
+
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
5170
|
+
setFieldValue(documents[docIndex], targetField, validated);
|
|
5171
|
+
let fieldSet = embedded.get(docIndex);
|
|
5172
|
+
if (!fieldSet) {
|
|
5173
|
+
fieldSet = /* @__PURE__ */ new Set();
|
|
5174
|
+
embedded.set(docIndex, fieldSet);
|
|
5175
|
+
}
|
|
5176
|
+
fieldSet.add(targetField);
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
} catch (err) {
|
|
5180
|
+
for (const [docIndex, fields] of embedded) {
|
|
5181
|
+
for (const field of fields) {
|
|
5182
|
+
deleteFieldValue(documents[docIndex], field);
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
throw err;
|
|
5186
|
+
}
|
|
5187
|
+
return { embedded, failed };
|
|
5188
|
+
}
|
|
5189
|
+
|
|
4978
5190
|
// src/engine/mutations/durable-rollback.ts
|
|
4979
5191
|
function rollbackFailed(docId, originalError, rollbackError) {
|
|
4980
5192
|
throw new NarsilError(
|
|
@@ -5019,6 +5231,141 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
5019
5231
|
}
|
|
5020
5232
|
}
|
|
5021
5233
|
|
|
5234
|
+
// src/engine/mutations/insert-admission.ts
|
|
5235
|
+
function providedDocId(document) {
|
|
5236
|
+
if (document === null || typeof document !== "object") return void 0;
|
|
5237
|
+
const id = document.id;
|
|
5238
|
+
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
5239
|
+
}
|
|
5240
|
+
function requireDocumentObject(document) {
|
|
5241
|
+
if (document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
5242
|
+
requireDocumentWithinSizeLimit(document);
|
|
5243
|
+
return;
|
|
5244
|
+
}
|
|
5245
|
+
const received = Array.isArray(document) ? "a list" : document === null ? "null" : `a ${typeof document}`;
|
|
5246
|
+
throw new NarsilError(
|
|
5247
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5248
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
5249
|
+
{
|
|
5250
|
+
received
|
|
5251
|
+
}
|
|
5252
|
+
);
|
|
5253
|
+
}
|
|
5254
|
+
function requireMatchingDocId(docId, document) {
|
|
5255
|
+
const embeddedDocId = providedDocId(document);
|
|
5256
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
5257
|
+
throw new NarsilError(
|
|
5258
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5259
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
5260
|
+
{ docId, documentId: embeddedDocId }
|
|
5261
|
+
);
|
|
5262
|
+
}
|
|
5263
|
+
function asBatchInsertError(err) {
|
|
5264
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
5265
|
+
}
|
|
5266
|
+
function createPendingAdmissions() {
|
|
5267
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
5268
|
+
}
|
|
5269
|
+
function capsEachPartition(ctx, manager) {
|
|
5270
|
+
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
5271
|
+
}
|
|
5272
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
5273
|
+
pending.total += 1;
|
|
5274
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
5275
|
+
const partitionId = manager.routePartition(docId);
|
|
5276
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
5277
|
+
}
|
|
5278
|
+
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
5279
|
+
const rebalancing = ctx.isRebalancing(indexName);
|
|
5280
|
+
if (rebalancing) {
|
|
5281
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
5282
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
5283
|
+
if (exists) {
|
|
5284
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
5285
|
+
}
|
|
5286
|
+
}
|
|
5287
|
+
assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing);
|
|
5288
|
+
}
|
|
5289
|
+
function admitBufferedUpsert(ctx, indexName, manager, docId) {
|
|
5290
|
+
assertRoomFor(ctx, indexName, manager, docId, void 0, true);
|
|
5291
|
+
}
|
|
5292
|
+
function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
5293
|
+
if (ctx.capacityScope === "partition") {
|
|
5294
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
5295
|
+
const partitionId = manager.routePartition(docId);
|
|
5296
|
+
manager.assertPartitionCapacity(partitionId, pending?.byPartition.get(partitionId) ?? 0);
|
|
5297
|
+
return;
|
|
5298
|
+
}
|
|
5299
|
+
const pendingAdmitted = pending?.total ?? 0;
|
|
5300
|
+
if (!rebalancing) {
|
|
5301
|
+
manager.assertCapacity(pendingAdmitted);
|
|
5302
|
+
return;
|
|
5303
|
+
}
|
|
5304
|
+
manager.assertCapacity(
|
|
5305
|
+
ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
|
|
5306
|
+
ctx.rebalanceTargetPartitionCount(indexName)
|
|
5307
|
+
);
|
|
5308
|
+
}
|
|
5309
|
+
function collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
5310
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5311
|
+
try {
|
|
5312
|
+
validateRequiredFields(documents[i], required);
|
|
5313
|
+
} catch (err) {
|
|
5314
|
+
chunkFailedIndexes.add(i);
|
|
5315
|
+
failed.push({ docId: providedDocId(documents[i]) ?? "", error: asBatchInsertError(err) });
|
|
5316
|
+
}
|
|
5317
|
+
}
|
|
5318
|
+
}
|
|
5319
|
+
async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
5320
|
+
if (entry.config.embedding === void 0) return;
|
|
5321
|
+
if (entry.embeddingAdapter === null) {
|
|
5322
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5323
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5324
|
+
try {
|
|
5325
|
+
assertDocumentCarriesMappedVectors(
|
|
5326
|
+
documents[i],
|
|
5327
|
+
entry.config.embedding,
|
|
5328
|
+
entry.embeddingAdapterName
|
|
5329
|
+
);
|
|
5330
|
+
} catch (err) {
|
|
5331
|
+
chunkFailedIndexes.add(i);
|
|
5332
|
+
failed.push({
|
|
5333
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
5334
|
+
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
5335
|
+
});
|
|
5336
|
+
}
|
|
5337
|
+
}
|
|
5338
|
+
return;
|
|
5339
|
+
}
|
|
5340
|
+
const embeddableSlice = [];
|
|
5341
|
+
const embeddableOriginalIndexes = [];
|
|
5342
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5343
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5344
|
+
embeddableSlice.push(documents[i]);
|
|
5345
|
+
embeddableOriginalIndexes.push(i);
|
|
5346
|
+
}
|
|
5347
|
+
if (embeddableSlice.length === 0) return;
|
|
5348
|
+
try {
|
|
5349
|
+
const embedResult = await embedBatchDocumentFields(
|
|
5350
|
+
embeddableSlice,
|
|
5351
|
+
entry.config.embedding,
|
|
5352
|
+
entry.embeddingAdapter,
|
|
5353
|
+
signal
|
|
5354
|
+
);
|
|
5355
|
+
for (const [sliceIndex, error] of embedResult.failed) {
|
|
5356
|
+
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
5357
|
+
chunkFailedIndexes.add(originalIdx);
|
|
5358
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
|
|
5359
|
+
}
|
|
5360
|
+
} catch (err) {
|
|
5361
|
+
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
5362
|
+
for (const originalIdx of embeddableOriginalIndexes) {
|
|
5363
|
+
chunkFailedIndexes.add(originalIdx);
|
|
5364
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
|
|
5365
|
+
}
|
|
5366
|
+
}
|
|
5367
|
+
}
|
|
5368
|
+
|
|
5022
5369
|
// src/engine/mutations/insert.ts
|
|
5023
5370
|
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
5024
5371
|
ctx.guardShutdown();
|
|
@@ -6302,6 +6649,24 @@ function bm25Operations(core) {
|
|
|
6302
6649
|
};
|
|
6303
6650
|
}
|
|
6304
6651
|
|
|
6652
|
+
// src/narsil/internals.ts
|
|
6653
|
+
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
6654
|
+
var rerankWindowSources = /* @__PURE__ */ new WeakMap();
|
|
6655
|
+
function bindEngineCore(engine, core) {
|
|
6656
|
+
engineCores.set(engine, core);
|
|
6657
|
+
}
|
|
6658
|
+
function engineCoreOf(engine) {
|
|
6659
|
+
return engineCores.get(engine);
|
|
6660
|
+
}
|
|
6661
|
+
function bindRerankWindowSource(owner, read) {
|
|
6662
|
+
rerankWindowSources.set(owner, read);
|
|
6663
|
+
}
|
|
6664
|
+
function rerankWindowCapOf(owner) {
|
|
6665
|
+
const core = engineCores.get(owner);
|
|
6666
|
+
if (core !== void 0) return core.rerankSettings.maxWindow;
|
|
6667
|
+
return rerankWindowSources.get(owner)?.();
|
|
6668
|
+
}
|
|
6669
|
+
|
|
6305
6670
|
// src/narsil/portability.ts
|
|
6306
6671
|
function portabilityOperations(core, config, engine) {
|
|
6307
6672
|
async function withOpenIndex2(indexName, action) {
|
|
@@ -6333,14 +6698,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6333
6698
|
return {
|
|
6334
6699
|
snapshot(indexName) {
|
|
6335
6700
|
return withOpenIndex2(indexName, async () => {
|
|
6336
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6701
|
+
const { createSnapshot } = await import('./snapshot-DB5ZVCDN.mjs');
|
|
6337
6702
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6338
6703
|
});
|
|
6339
6704
|
},
|
|
6340
6705
|
async restore(indexName, data) {
|
|
6341
6706
|
core.guardShutdown();
|
|
6342
6707
|
validateIndexName(indexName);
|
|
6343
|
-
const restored = import('./restore-queue-
|
|
6708
|
+
const restored = import('./restore-queue-UAVGBSJG.mjs').then(
|
|
6344
6709
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6345
6710
|
);
|
|
6346
6711
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6349,8 +6714,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6349
6714
|
exportDocuments(indexName) {
|
|
6350
6715
|
return withOpenIndex2(indexName, async () => {
|
|
6351
6716
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6352
|
-
import('./snapshot-
|
|
6353
|
-
import('./write-
|
|
6717
|
+
import('./snapshot-DB5ZVCDN.mjs'),
|
|
6718
|
+
import('./write-KZVTLGCI.mjs')
|
|
6354
6719
|
]);
|
|
6355
6720
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6356
6721
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6358,7 +6723,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6358
6723
|
},
|
|
6359
6724
|
importDocumentExport(indexName, data) {
|
|
6360
6725
|
core.guardShutdown();
|
|
6361
|
-
const imported = import('./import-
|
|
6726
|
+
const imported = import('./import-6KXFY6XI.mjs').then(
|
|
6362
6727
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6363
6728
|
);
|
|
6364
6729
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -6366,6 +6731,58 @@ function portabilityOperations(core, config, engine) {
|
|
|
6366
6731
|
};
|
|
6367
6732
|
}
|
|
6368
6733
|
|
|
6734
|
+
// src/engine/resolve-vector-text.ts
|
|
6735
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
6736
|
+
if (!params.vector) return params;
|
|
6737
|
+
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
6738
|
+
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
6739
|
+
if (params.vector.text !== void 0 && params.vector.value !== void 0) {
|
|
6740
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
|
|
6741
|
+
}
|
|
6742
|
+
if (!embeddingAdapter) {
|
|
6743
|
+
throw new NarsilError(
|
|
6744
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6745
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
6746
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
6747
|
+
);
|
|
6748
|
+
}
|
|
6749
|
+
if (typeof params.vector.text !== "string") {
|
|
6750
|
+
throw new NarsilError(
|
|
6751
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
6752
|
+
`Vector query 'text' must be a string, got ${typeof params.vector.text}`
|
|
6753
|
+
);
|
|
6754
|
+
}
|
|
6755
|
+
if (params.vector.text.length === 0) {
|
|
6756
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
|
|
6757
|
+
}
|
|
6758
|
+
let raw;
|
|
6759
|
+
try {
|
|
6760
|
+
raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
|
|
6761
|
+
} catch (err) {
|
|
6762
|
+
if (err instanceof NarsilError) throw err;
|
|
6763
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
6764
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `The embedding adapter cannot embed the query: ${message}`, {
|
|
6765
|
+
cause: message
|
|
6766
|
+
});
|
|
6767
|
+
}
|
|
6768
|
+
if (!(raw instanceof Float32Array)) {
|
|
6769
|
+
throw new NarsilError(
|
|
6770
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
6771
|
+
`Adapter returned ${typeof raw} for query embedding, expected Float32Array`
|
|
6772
|
+
);
|
|
6773
|
+
}
|
|
6774
|
+
if (raw.length !== embeddingAdapter.dimensions) {
|
|
6775
|
+
throw new NarsilError(
|
|
6776
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
6777
|
+
`Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
|
|
6778
|
+
{ expected: embeddingAdapter.dimensions, actual: raw.length }
|
|
6779
|
+
);
|
|
6780
|
+
}
|
|
6781
|
+
const resolved = { ...params.vector, value: Array.from(raw) };
|
|
6782
|
+
delete resolved.text;
|
|
6783
|
+
return { ...params, vector: resolved };
|
|
6784
|
+
}
|
|
6785
|
+
|
|
6369
6786
|
// src/narsil/reads.ts
|
|
6370
6787
|
async function enterRead(core, indexName) {
|
|
6371
6788
|
core.guardShutdown();
|
|
@@ -6417,7 +6834,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
6417
6834
|
const manager = core.requireManager(indexName);
|
|
6418
6835
|
const scoped = scopedParams(params, options);
|
|
6419
6836
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
6420
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
6837
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-RAOCKSYA.mjs');
|
|
6421
6838
|
const resolvedParams = await resolveVectorText(
|
|
6422
6839
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
6423
6840
|
entry.embeddingAdapter,
|
|
@@ -6739,7 +7156,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6739
7156
|
const entry = requireIndex(indexName);
|
|
6740
7157
|
await withOpenIndexWrite(indexName, async () => {
|
|
6741
7158
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
6742
|
-
return (await import('./analysis-change-
|
|
7159
|
+
return (await import('./analysis-change-D5APH4Z3.mjs')).resumeAnalysisChange(core, indexName);
|
|
6743
7160
|
});
|
|
6744
7161
|
},
|
|
6745
7162
|
async changeAnalysis(indexName, change) {
|
|
@@ -6747,7 +7164,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6747
7164
|
requireIndex(indexName);
|
|
6748
7165
|
const changed = withOpenIndex2(
|
|
6749
7166
|
indexName,
|
|
6750
|
-
async () => (await import('./analysis-change-
|
|
7167
|
+
async () => (await import('./analysis-change-D5APH4Z3.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
6751
7168
|
);
|
|
6752
7169
|
await emitWriteEventAfter(core, indexName, changed);
|
|
6753
7170
|
},
|
|
@@ -6785,7 +7202,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6785
7202
|
}
|
|
6786
7203
|
rebalancesStarting.set(indexName, started);
|
|
6787
7204
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
6788
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
7205
|
+
const { executeRebalance } = await import('./analysis-change-D5APH4Z3.mjs');
|
|
6789
7206
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
6790
7207
|
releaseWrites();
|
|
6791
7208
|
return running;
|
|
@@ -6909,4 +7326,4 @@ async function createNarsil(config) {
|
|
|
6909
7326
|
// src/index.ts
|
|
6910
7327
|
var VERSION = "0.1.0";
|
|
6911
7328
|
|
|
6912
|
-
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, resolveRunnableConfig, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|
|
7329
|
+
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, bindRerankWindowSource, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineCoreOf, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, providedDocId, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, rerankWindowCapOf, resolveRunnableConfig, resolveVectorText, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|