@delali/narsil 0.3.1-next.96 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-ZGMR46O3.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-KPMIHLOV.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-3MYKLHRO.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-R7P64REL.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-WMYC5ZK6.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-3A3WDNHO.mjs → snapshot-J57MZK6L.mjs} +8 -6
- package/dist/browser/{write-JODZHKIF.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-H75XXPMY.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-2BY6K52M.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-GMY7F2VA.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-VVFLEGZI.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-LJBJQPB5.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-O5WPRAVZ.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-TCYUAUHF.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-2VBNYYDY.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CbZmXUp6.d.ts → engine-BGCEm9E_.d.ts} +48 -18
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-UFIQJTSG.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-XII3ANGW.mjs → snapshot-HOGXNO5W.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-6M3VG4TU.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
2
|
-
import { generateId, getLanguage } from './chunk-
|
|
3
|
-
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-
|
|
4
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
5
|
-
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-K25LZA54.mjs';
|
|
2
|
+
import { generateId, getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
3
|
+
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-ASY5PHYI.mjs';
|
|
4
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-MHOSCQCQ.mjs';
|
|
5
|
+
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-LXPIXDUJ.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isWordIndexedBase } from './chunk-
|
|
8
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
9
|
-
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-
|
|
7
|
+
import { isWordIndexedBase } from './chunk-OOEXLUGR.mjs';
|
|
8
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-RONEOEON.mjs';
|
|
9
|
+
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-LU2CE5AE.mjs';
|
|
10
10
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
11
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-PPY2GSIB.mjs';
|
|
12
12
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
13
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
13
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
14
14
|
import { totalmem } from 'os';
|
|
15
15
|
import { getHeapStatistics } from 'v8';
|
|
16
16
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-3NONKBZX.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|
|
@@ -1,14 +1,23 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-LXPIXDUJ.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-KV3AZFT3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
6
|
+
var rerankWindowSources = /* @__PURE__ */ new WeakMap();
|
|
6
7
|
function bindEngineCore(engine, core) {
|
|
7
8
|
engineCores.set(engine, core);
|
|
8
9
|
}
|
|
9
10
|
function engineCoreOf(engine) {
|
|
10
11
|
return engineCores.get(engine);
|
|
11
12
|
}
|
|
13
|
+
function bindRerankWindowSource(owner, read) {
|
|
14
|
+
rerankWindowSources.set(owner, read);
|
|
15
|
+
}
|
|
16
|
+
function rerankWindowCapOf(owner) {
|
|
17
|
+
const core = engineCores.get(owner);
|
|
18
|
+
if (core !== void 0) return core.rerankSettings.maxWindow;
|
|
19
|
+
return rerankWindowSources.get(owner)?.();
|
|
20
|
+
}
|
|
12
21
|
|
|
13
22
|
// src/engine/snapshot/restore-config.ts
|
|
14
23
|
function rejectSnapshot(message) {
|
|
@@ -581,4 +590,4 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
|
|
|
581
590
|
return { ...params, vector: resolved };
|
|
582
591
|
}
|
|
583
592
|
|
|
584
|
-
export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, rejectSnapshot, requireDocumentObject, requireMatchingDocId, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
593
|
+
export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, bindRerankWindowSource, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, rejectSnapshot, requireDocumentObject, requireMatchingDocId, rerankWindowCapOf, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-LU2CE5AE.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
7
7
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-HXJBSHXF.mjs';
|
|
2
2
|
import { MAX_VECTOR_ORDINALS } from './chunk-QY7GC2MW.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
4
4
|
import { readSync, closeSync, openSync } from 'fs';
|
|
5
5
|
|
|
6
6
|
function createVectorFileReader() {
|
|
@@ -1,21 +1,22 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-CHOGQSHK.mjs';
|
|
5
|
-
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-WLOF6HSP.mjs';
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-NKRBEGJZ.mjs';
|
|
2
|
+
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-R7EEKAJE.mjs';
|
|
3
|
+
import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-AY3AQ3UH.mjs';
|
|
6
4
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
5
|
+
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, readHeapStatistics, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-OWNZYMRE.mjs';
|
|
6
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-K25LZA54.mjs';
|
|
7
|
+
import { generateId, getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
8
|
+
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
9
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-LXWDQLTW.mjs';
|
|
10
|
+
import { requireSearchBackendSettings } from './chunk-GGSH6KBQ.mjs';
|
|
11
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-ASY5PHYI.mjs';
|
|
12
|
+
import { getNestedValue } from './chunk-MHOSCQCQ.mjs';
|
|
13
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-LXPIXDUJ.mjs';
|
|
14
|
+
import { isSingleStringFieldType } from './chunk-OOEXLUGR.mjs';
|
|
14
15
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
16
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-R6PR4WYZ.mjs';
|
|
16
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
17
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-PPY2GSIB.mjs';
|
|
17
18
|
import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
18
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
19
|
+
import { ErrorCodes, NarsilError } from './chunk-KV3AZFT3.mjs';
|
|
19
20
|
import { encode, decode } from '@msgpack/msgpack';
|
|
20
21
|
import * as zlib from 'zlib';
|
|
21
22
|
|
|
@@ -28,6 +29,9 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
28
29
|
workers: true,
|
|
29
30
|
embedding: true,
|
|
30
31
|
embeddingAdapters: true,
|
|
32
|
+
rerankAdapters: true,
|
|
33
|
+
rerankTimeout: true,
|
|
34
|
+
maxRerankWindow: true,
|
|
31
35
|
durability: true,
|
|
32
36
|
analysis: true,
|
|
33
37
|
lifecycle: true,
|
|
@@ -334,6 +338,24 @@ function createPluginRegistry() {
|
|
|
334
338
|
};
|
|
335
339
|
}
|
|
336
340
|
|
|
341
|
+
// src/rerank/registry.ts
|
|
342
|
+
function validateRerankAdapter(name, adapter) {
|
|
343
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
344
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
|
|
345
|
+
}
|
|
346
|
+
const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
|
|
347
|
+
if (typeof rerank !== "function") {
|
|
348
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
|
|
349
|
+
adapter: name
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
function registerEngineRerankAdapter(core, name, adapter) {
|
|
354
|
+
core.guardShutdown();
|
|
355
|
+
validateRerankAdapter(name, adapter);
|
|
356
|
+
core.rerankAdapters.set(name, adapter);
|
|
357
|
+
}
|
|
358
|
+
|
|
337
359
|
// src/schema/embedding-validator.ts
|
|
338
360
|
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
339
361
|
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
@@ -3998,6 +4020,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
3998
4020
|
// src/engine/core.ts
|
|
3999
4021
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4000
4022
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4023
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-ZKYZFJJC.mjs')).resolveRerankSettings(config);
|
|
4001
4024
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4002
4025
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4003
4026
|
const vectorCopyPolicy = {
|
|
@@ -4008,7 +4031,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4008
4031
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4009
4032
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4010
4033
|
}
|
|
4011
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4034
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-QTHOGFMZ.mjs');
|
|
4012
4035
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4013
4036
|
const executor = createDirectExecutor({
|
|
4014
4037
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4027,6 +4050,11 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4027
4050
|
embeddingAdapters.set(name, adapter);
|
|
4028
4051
|
}
|
|
4029
4052
|
}
|
|
4053
|
+
const rerankAdapters = /* @__PURE__ */ new Map();
|
|
4054
|
+
for (const [name, adapter] of Object.entries(config?.rerankAdapters ?? {})) {
|
|
4055
|
+
validateRerankAdapter(name, adapter);
|
|
4056
|
+
rerankAdapters.set(name, adapter);
|
|
4057
|
+
}
|
|
4030
4058
|
const shutdownState = { isShutdown: false };
|
|
4031
4059
|
const abortController = new AbortController();
|
|
4032
4060
|
const rebalancingIndexes = /* @__PURE__ */ new Set();
|
|
@@ -4227,6 +4255,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4227
4255
|
idGenerator,
|
|
4228
4256
|
indexRegistry,
|
|
4229
4257
|
embeddingAdapters,
|
|
4258
|
+
rerankAdapters,
|
|
4259
|
+
rerankSettings,
|
|
4230
4260
|
eventHandlers,
|
|
4231
4261
|
shutdownState,
|
|
4232
4262
|
abortController,
|
|
@@ -4293,6 +4323,9 @@ async function shutdownEngine(core) {
|
|
|
4293
4323
|
} catch {
|
|
4294
4324
|
}
|
|
4295
4325
|
}
|
|
4326
|
+
if (core.rerankAdapters.size > 0) {
|
|
4327
|
+
await (await import('./runtime-ZKYZFJJC.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4328
|
+
}
|
|
4296
4329
|
await orchestrator.shutdown();
|
|
4297
4330
|
await executor.shutdown();
|
|
4298
4331
|
eventHandlers.clear();
|
|
@@ -6908,14 +6941,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6908
6941
|
return {
|
|
6909
6942
|
snapshot(indexName) {
|
|
6910
6943
|
return withOpenIndex(indexName, async () => {
|
|
6911
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6944
|
+
const { createSnapshot } = await import('./snapshot-HOGXNO5W.mjs');
|
|
6912
6945
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6913
6946
|
});
|
|
6914
6947
|
},
|
|
6915
6948
|
async restore(indexName, data) {
|
|
6916
6949
|
core.guardShutdown();
|
|
6917
6950
|
validateIndexName(indexName);
|
|
6918
|
-
const restored = import('./restore-queue-
|
|
6951
|
+
const restored = import('./restore-queue-JPRBUYII.mjs').then(
|
|
6919
6952
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6920
6953
|
);
|
|
6921
6954
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6924,8 +6957,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6924
6957
|
exportDocuments(indexName) {
|
|
6925
6958
|
return withOpenIndex(indexName, async () => {
|
|
6926
6959
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6927
|
-
import('./snapshot-
|
|
6928
|
-
import('./write-
|
|
6960
|
+
import('./snapshot-HOGXNO5W.mjs'),
|
|
6961
|
+
import('./write-XLIGTXBU.mjs')
|
|
6929
6962
|
]);
|
|
6930
6963
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6931
6964
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6977,8 +7010,9 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
6977
7010
|
const manager = core.requireManager(indexName);
|
|
6978
7011
|
const scoped = scopedParams(params, options);
|
|
6979
7012
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
7013
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-ZKYZFJJC.mjs');
|
|
6980
7014
|
const resolvedParams = await resolveVectorText(
|
|
6981
|
-
scoped,
|
|
7015
|
+
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
6982
7016
|
entry.embeddingAdapter,
|
|
6983
7017
|
core.abortController.signal,
|
|
6984
7018
|
entry.embeddingAdapterName
|
|
@@ -6992,8 +7026,10 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
6992
7026
|
indexName,
|
|
6993
7027
|
broadcastStats: broadcastStatsFor(core, options),
|
|
6994
7028
|
partitionIds: options?.partitionIds,
|
|
7029
|
+
rescoreWindow: options?.rescoreWindow,
|
|
6995
7030
|
cursorBinding: queryBindingOf(params),
|
|
6996
|
-
patternWorkCap: core.patternWorkCap
|
|
7031
|
+
patternWorkCap: core.patternWorkCap,
|
|
7032
|
+
rerank: rerank?.rerankRunnerFor(core)
|
|
6997
7033
|
};
|
|
6998
7034
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
6999
7035
|
core,
|
|
@@ -7173,6 +7209,9 @@ function createNarsilFromCore(core, config) {
|
|
|
7173
7209
|
registerEmbeddingAdapter(name, adapter) {
|
|
7174
7210
|
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
7175
7211
|
},
|
|
7212
|
+
registerRerankAdapter(name, adapter) {
|
|
7213
|
+
registerEngineRerankAdapter(core, name, adapter);
|
|
7214
|
+
},
|
|
7176
7215
|
dropIndex(name) {
|
|
7177
7216
|
return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
|
|
7178
7217
|
},
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
2
2
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/vector-index/dead-bits.ts
|
|
6
6
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
2
|
-
import {
|
|
2
|
+
import { DEFAULT_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
3
|
+
import { loadNativeCore } from './chunk-GGSH6KBQ.mjs';
|
|
3
4
|
import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
4
|
-
import { MAX_SORT_FIELDS } from './chunk-
|
|
5
|
-
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-
|
|
5
|
+
import { MAX_SORT_FIELDS } from './chunk-E74YLAIG.mjs';
|
|
6
|
+
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
6
7
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
7
8
|
import { tmpdir } from 'os';
|
|
8
9
|
import path from 'path';
|
|
@@ -128,6 +129,7 @@ var STATUS_BY_CODE = {
|
|
|
128
129
|
EMBEDDING_DIMENSION_MISMATCH: 400,
|
|
129
130
|
EMBEDDING_NO_SOURCE: 400,
|
|
130
131
|
EMBEDDING_CONFIG_INVALID: 400,
|
|
132
|
+
RERANK_CONFIG_INVALID: 400,
|
|
131
133
|
INDEX_NOT_FOUND: 404,
|
|
132
134
|
DOC_NOT_FOUND: 404,
|
|
133
135
|
INDEX_ALREADY_EXISTS: 409,
|
|
@@ -150,6 +152,7 @@ var STATUS_BY_CODE = {
|
|
|
150
152
|
PERSISTENCE_WAL_CORRUPT: 503,
|
|
151
153
|
PERSISTENCE_FSYNC_FAILED: 503,
|
|
152
154
|
EMBEDDING_FAILED: 503,
|
|
155
|
+
RERANK_FAILED: 503,
|
|
153
156
|
QUERY_ROUTING_FAILED: 503,
|
|
154
157
|
QUERY_PARTIAL_FAILURE: 503,
|
|
155
158
|
QUERY_NODE_TIMEOUT: 503,
|
|
@@ -246,6 +249,7 @@ var DEFAULT_IMPORT_BATCH_SIZE = 1e3;
|
|
|
246
249
|
var IMPORT_FLUSHES_IN_FLIGHT = 4;
|
|
247
250
|
var DEFAULT_MAX_CONCURRENT_REQUESTS = 0;
|
|
248
251
|
var DEFAULT_MAX_RESULT_WINDOW = 1e4;
|
|
252
|
+
var DEFAULT_MAX_RERANK_WINDOW = 100;
|
|
249
253
|
var DEFAULT_MAX_FETCH_DOCUMENTS = 1e4;
|
|
250
254
|
var DEFAULT_MAX_IMPORT_ERRORS = 100;
|
|
251
255
|
var DEFAULT_MAX_TASK_PAGE_SIZE = 1e3;
|
|
@@ -480,7 +484,28 @@ function overWindow(field, value, maxWindow) {
|
|
|
480
484
|
}
|
|
481
485
|
return null;
|
|
482
486
|
}
|
|
483
|
-
function
|
|
487
|
+
function overRerankWindow(rerank, maxRerankWindow) {
|
|
488
|
+
if (!isPlainObject(rerank)) return null;
|
|
489
|
+
const window = rerank.window;
|
|
490
|
+
if (typeof window === "number" && window > maxRerankWindow) {
|
|
491
|
+
return {
|
|
492
|
+
message: `Field "rerank.window" exceeds this server's maximum rerank window of ${maxRerankWindow}`,
|
|
493
|
+
details: { value: window, limit: maxRerankWindow }
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
function withServerRerankWindow(params, limits) {
|
|
499
|
+
const rerank = params.rerank;
|
|
500
|
+
if (rerank === void 0 || !isPlainObject(rerank) || rerank.window !== void 0) return params;
|
|
501
|
+
const depth = params.vector === void 0 ? void 0 : params.hybrid?.depth;
|
|
502
|
+
const fuses = params.term !== void 0 || params.mode === "hybrid";
|
|
503
|
+
const window = fuses && typeof depth === "number" && Number.isInteger(depth) ? Math.min(limits.maxRerankWindow, depth) : limits.maxRerankWindow;
|
|
504
|
+
if (window >= DEFAULT_RERANK_WINDOW) return params;
|
|
505
|
+
return { ...params, rerank: { ...rerank, window } };
|
|
506
|
+
}
|
|
507
|
+
function validateQuery(params, limits) {
|
|
508
|
+
const maxWindow = limits.maxResultWindow;
|
|
484
509
|
if (params.group && typeof params.group === "object" && "reduce" in params.group) {
|
|
485
510
|
return {
|
|
486
511
|
message: 'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
|
|
@@ -488,6 +513,8 @@ function validateQuery(params, maxWindow) {
|
|
|
488
513
|
}
|
|
489
514
|
const rowBound = overWindow("limit", params.limit, maxWindow) ?? overWindow("offset", params.offset, maxWindow);
|
|
490
515
|
if (rowBound) return rowBound;
|
|
516
|
+
const reranked = overRerankWindow(params.rerank, limits.maxRerankWindow);
|
|
517
|
+
if (reranked) return reranked;
|
|
491
518
|
if (params.group && typeof params.group === "object") {
|
|
492
519
|
const grouped = overWindow("group.maxPerGroup", params.group.maxPerGroup, maxWindow) ?? overWindow("group.limit", params.group.limit, maxWindow);
|
|
493
520
|
if (grouped) return grouped;
|
|
@@ -1226,4 +1253,4 @@ async function departureConfirmation(word) {
|
|
|
1226
1253
|
if (waiting.async) await waiting.value;
|
|
1227
1254
|
}
|
|
1228
1255
|
|
|
1229
|
-
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
|
|
1256
|
+
export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, REQUEST_THREAD_DEPARTURE_DRAIN_MS, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, awaitDepartureConfirmation, badRequest, childAppOf, clearGateSlot, confirmDeparture, corsHeaderReader, createDepartureWord, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, departureConfirmation, departureConfirmed, encodeJson, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, refuseRegex, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport, withServerRerankWindow };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-
|
|
3
|
-
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
5
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-
|
|
6
|
-
import { flattenSchema, requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-WCJKUGHB.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-LH4NZNJJ.mjs';
|
|
3
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-VWULPWRJ.mjs';
|
|
4
|
+
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
5
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-OWNZYMRE.mjs';
|
|
6
|
+
import { flattenSchema, requireValidFieldTypes } from './chunk-LXPIXDUJ.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
9
9
|
import { decode, encode } from '@msgpack/msgpack';
|
|
10
10
|
|
|
11
11
|
// src/engine/durability-metadata.ts
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
2
|
+
|
|
3
|
+
// src/rerank/call.ts
|
|
4
|
+
function unregistered(adapters, name) {
|
|
5
|
+
const registered = [...adapters.keys()].sort();
|
|
6
|
+
return new NarsilError(
|
|
7
|
+
ErrorCodes.RERANK_CONFIG_INVALID,
|
|
8
|
+
`The query gives rerank adapter "${name}", which this engine has not registered. Register it with registerRerankAdapter or the rerankAdapters setting`,
|
|
9
|
+
{ adapter: name, registered }
|
|
10
|
+
);
|
|
11
|
+
}
|
|
12
|
+
function scoresOf(raw, expected, name) {
|
|
13
|
+
const list = Array.isArray(raw) ? raw : ArrayBuffer.isView(raw) && !(raw instanceof DataView) ? Array.from(raw) : null;
|
|
14
|
+
if (list === null) {
|
|
15
|
+
throw new NarsilError(
|
|
16
|
+
ErrorCodes.RERANK_FAILED,
|
|
17
|
+
`Rerank adapter "${name}" returned ${typeof raw}, expected a list of scores`,
|
|
18
|
+
{
|
|
19
|
+
adapter: name
|
|
20
|
+
}
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
if (list.length !== expected) {
|
|
24
|
+
throw new NarsilError(
|
|
25
|
+
ErrorCodes.RERANK_FAILED,
|
|
26
|
+
`Rerank adapter "${name}" returned ${list.length} scores for ${expected} documents`,
|
|
27
|
+
{ adapter: name, expected, actual: list.length }
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
const scores = new Array(expected);
|
|
31
|
+
for (let i = 0; i < expected; i++) {
|
|
32
|
+
const score = list[i];
|
|
33
|
+
if (typeof score !== "number" || !Number.isFinite(score)) {
|
|
34
|
+
throw new NarsilError(
|
|
35
|
+
ErrorCodes.RERANK_FAILED,
|
|
36
|
+
`Rerank adapter "${name}" returned a score that is not a finite number at position ${i}`,
|
|
37
|
+
{ adapter: name, position: i }
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
scores[i] = score;
|
|
41
|
+
}
|
|
42
|
+
return scores;
|
|
43
|
+
}
|
|
44
|
+
function requireRerankAdapter(adapters, name) {
|
|
45
|
+
const adapter = adapters.get(name);
|
|
46
|
+
if (adapter === void 0) throw unregistered(adapters, name);
|
|
47
|
+
return adapter;
|
|
48
|
+
}
|
|
49
|
+
function timedOut(name, timeoutMs) {
|
|
50
|
+
return new NarsilError(
|
|
51
|
+
ErrorCodes.RERANK_FAILED,
|
|
52
|
+
`Rerank adapter "${name}" did not answer within ${timeoutMs} ms, so the engine cancelled the call`,
|
|
53
|
+
{ adapter: name, timeoutMs }
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
function cancelled(name) {
|
|
57
|
+
return new NarsilError(ErrorCodes.RERANK_FAILED, `The engine cancelled the call to rerank adapter "${name}"`, {
|
|
58
|
+
adapter: name
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
function wrapped(err, name) {
|
|
62
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.RERANK_FAILED) return err;
|
|
63
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
64
|
+
return new NarsilError(ErrorCodes.RERANK_FAILED, `Rerank adapter "${name}" failed: ${message}`, {
|
|
65
|
+
adapter: name,
|
|
66
|
+
cause: message,
|
|
67
|
+
...err instanceof NarsilError ? { causeCode: err.code } : {}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
async function callRerankAdapter(adapters, name, query, documents, options) {
|
|
71
|
+
const adapter = requireRerankAdapter(adapters, name);
|
|
72
|
+
if (documents.length === 0) return [];
|
|
73
|
+
const outer = options.signal;
|
|
74
|
+
if (outer?.aborted) throw cancelled(name);
|
|
75
|
+
const controller = new AbortController();
|
|
76
|
+
let timer;
|
|
77
|
+
let detach;
|
|
78
|
+
const stopped = new Promise((_resolve, reject) => {
|
|
79
|
+
timer = setTimeout(() => {
|
|
80
|
+
controller.abort();
|
|
81
|
+
reject(timedOut(name, options.timeoutMs));
|
|
82
|
+
}, options.timeoutMs);
|
|
83
|
+
if (outer !== void 0) {
|
|
84
|
+
const onAbort = () => {
|
|
85
|
+
controller.abort();
|
|
86
|
+
reject(cancelled(name));
|
|
87
|
+
};
|
|
88
|
+
outer.addEventListener("abort", onAbort, { once: true });
|
|
89
|
+
detach = () => outer.removeEventListener("abort", onAbort);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
let raw;
|
|
93
|
+
try {
|
|
94
|
+
raw = await Promise.race([
|
|
95
|
+
Promise.resolve().then(() => adapter.rerank(query, documents, controller.signal)),
|
|
96
|
+
stopped
|
|
97
|
+
]);
|
|
98
|
+
} catch (err) {
|
|
99
|
+
throw wrapped(err, name);
|
|
100
|
+
} finally {
|
|
101
|
+
clearTimeout(timer);
|
|
102
|
+
detach?.();
|
|
103
|
+
}
|
|
104
|
+
return scoresOf(raw, documents.length, name);
|
|
105
|
+
}
|
|
106
|
+
function rerankOnHost(host, name, query, documents) {
|
|
107
|
+
return callRerankAdapter(host.rerankAdapters, name, query, documents, {
|
|
108
|
+
timeoutMs: host.rerankSettings.timeoutMs,
|
|
109
|
+
signal: host.abortController.signal
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export { rerankOnHost };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-
|
|
2
|
-
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams,
|
|
1
|
+
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-4W_7tk1f.js';
|
|
2
|
+
import { V as VectorMaintenanceResult, B as BatchResult, L as ListParams, g as ListResult, m as IndexInfo, I as IndexStats, k as PartitionStatsResult, Q as QueryParams, e as QueryResult, P as PreflightResult, h as SuggestParams, j as SuggestResult } from './memory-CUEDwr-F.js';
|
|
3
3
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
|
-
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
4
|
+
import { e as TaskRecord, M as MemoryStatsResponse, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-HG3rlxx6.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* This is the `fetch` implementation the client sends every request through.
|
package/dist/client.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-
|
|
2
|
-
export { A as ASYNC_IMPORT_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
3
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
5
|
-
import './schema-
|
|
6
|
-
import './adapters-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-CUEDwr-F.js';
|
|
2
|
+
export { A as ASYNC_IMPORT_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BTTIs96E.js';
|
|
3
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-HG3rlxx6.js';
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-i9eLJrw5.js';
|
|
5
|
+
import './schema-4W_7tk1f.js';
|
|
6
|
+
import './adapters-BQFQ4PRz.js';
|
|
7
7
|
import './synonyms-CpOAHPa-.js';
|
|
8
|
-
import './config-
|
|
9
|
-
import './config-
|
|
10
|
-
import './index-
|
|
8
|
+
import './config-BGB_fDtq.js';
|
|
9
|
+
import './config-P__Diiy4.js';
|
|
10
|
+
import './index-lcdGLyJH.js';
|
|
11
11
|
import './types-Cb9kiY5P.js';
|
package/dist/client.mjs
CHANGED
|
@@ -41,6 +41,8 @@ var ErrorCodes = {
|
|
|
41
41
|
EMBEDDING_DIMENSION_MISMATCH: "EMBEDDING_DIMENSION_MISMATCH",
|
|
42
42
|
EMBEDDING_NO_SOURCE: "EMBEDDING_NO_SOURCE",
|
|
43
43
|
EMBEDDING_CONFIG_INVALID: "EMBEDDING_CONFIG_INVALID",
|
|
44
|
+
RERANK_FAILED: "RERANK_FAILED",
|
|
45
|
+
RERANK_CONFIG_INVALID: "RERANK_CONFIG_INVALID",
|
|
44
46
|
DOC_MISSING_REQUIRED_FIELD: "DOC_MISSING_REQUIRED_FIELD",
|
|
45
47
|
CONFIG_INVALID: "CONFIG_INVALID",
|
|
46
48
|
QUERY_ROUTING_FAILED: "QUERY_ROUTING_FAILED",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
2
|
-
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './index-
|
|
1
|
+
import { N as NarsilConfig } from './config-P__Diiy4.js';
|
|
2
|
+
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './index-lcdGLyJH.js';
|
|
3
3
|
import { N as NodeTransport } from './types-Cb9kiY5P.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-
|
|
2
|
-
import { Q as QueryParams, e as QueryResult, f as NarsilEventHandlers } from './memory-
|
|
3
|
-
import { I as IndexConfig, A as AnyDocument } from './schema-
|
|
1
|
+
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { Q as QueryParams, e as QueryResult, f as NarsilEventHandlers } from './memory-CUEDwr-F.js';
|
|
3
|
+
import { I as IndexConfig, A as AnyDocument } from './schema-4W_7tk1f.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Hooks that the engine calls alongside its own work, so that an application
|
|
@@ -191,6 +191,28 @@ interface NarsilConfig {
|
|
|
191
191
|
embedding?: EmbeddingAdapter;
|
|
192
192
|
/** Named adapters; names persist in index metadata so recovery can rebind. */
|
|
193
193
|
embeddingAdapters?: Record<string, EmbeddingAdapter>;
|
|
194
|
+
/**
|
|
195
|
+
* These rerank adapters are registered under their keys, so that a query
|
|
196
|
+
* can give a key in `rerank.adapter`. The engine throws
|
|
197
|
+
* `RERANK_CONFIG_INVALID` for an empty key and for an entry without a
|
|
198
|
+
* `rerank` function.
|
|
199
|
+
*/
|
|
200
|
+
rerankAdapters?: Record<string, RerankAdapter>;
|
|
201
|
+
/**
|
|
202
|
+
* The engine waits this many milliseconds for a rerank adapter to score one
|
|
203
|
+
* query, and 30,000 by default. It then cancels the call through the
|
|
204
|
+
* adapter's abort signal and throws `RERANK_FAILED`. Set a whole number of
|
|
205
|
+
* at least 1; {@link createNarsil} throws `CONFIG_INVALID` for any other
|
|
206
|
+
* value.
|
|
207
|
+
*/
|
|
208
|
+
rerankTimeout?: number;
|
|
209
|
+
/**
|
|
210
|
+
* A query may rerank at most this many of its top hits, which caps the
|
|
211
|
+
* documents that one query sends to a model. Set a whole number from 1 to
|
|
212
|
+
* 1,000, or leave it out for 100; {@link createNarsil} throws
|
|
213
|
+
* `CONFIG_INVALID` for any other value.
|
|
214
|
+
*/
|
|
215
|
+
maxRerankWindow?: number;
|
|
194
216
|
/** These settings recover the writes made since the last snapshot. */
|
|
195
217
|
durability?: DurabilityConfig;
|
|
196
218
|
/** These settings control what the engine does when an index's stored analysis differs from its language module. */
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export { createDirectExecutor } from './chunk-OWNZYMRE.mjs';
|
|
2
|
+
import './chunk-K25LZA54.mjs';
|
|
3
|
+
import './chunk-JFRNNBOM.mjs';
|
|
4
|
+
import './chunk-TGP5M5R7.mjs';
|
|
5
|
+
import './chunk-U2BKYEC7.mjs';
|
|
6
|
+
import './chunk-ASY5PHYI.mjs';
|
|
7
|
+
import './chunk-WQBCBKY3.mjs';
|
|
8
|
+
import './chunk-MHOSCQCQ.mjs';
|
|
9
|
+
import './chunk-LXPIXDUJ.mjs';
|
|
10
|
+
import './chunk-E74YLAIG.mjs';
|
|
11
|
+
import './chunk-KEHK2O6P.mjs';
|
|
12
|
+
import './chunk-GES7U2PM.mjs';
|
|
13
|
+
import './chunk-OOEXLUGR.mjs';
|
|
14
|
+
import './chunk-RONEOEON.mjs';
|
|
15
|
+
import './chunk-X5CV2E2B.mjs';
|
|
16
|
+
import './chunk-7X5JST5L.mjs';
|
|
17
|
+
import './chunk-LU2CE5AE.mjs';
|
|
18
|
+
import './chunk-R6A773WK.mjs';
|
|
19
|
+
import './chunk-YTJCGKM4.mjs';
|
|
20
|
+
import './chunk-PPY2GSIB.mjs';
|
|
21
|
+
import './chunk-L5ORED7V.mjs';
|
|
22
|
+
import './chunk-KV3AZFT3.mjs';
|