@delali/narsil 0.3.1-next.96 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-ZGMR46O3.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-KPMIHLOV.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-3MYKLHRO.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-R7P64REL.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-WMYC5ZK6.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-3A3WDNHO.mjs → snapshot-J57MZK6L.mjs} +8 -6
- package/dist/browser/{write-JODZHKIF.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-H75XXPMY.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-2BY6K52M.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-GMY7F2VA.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-VVFLEGZI.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-LJBJQPB5.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-O5WPRAVZ.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-TCYUAUHF.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-2VBNYYDY.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CbZmXUp6.d.ts → engine-BGCEm9E_.d.ts} +48 -18
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-UFIQJTSG.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-XII3ANGW.mjs → snapshot-HOGXNO5W.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-6M3VG4TU.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { C as ClusterCoordinator } from '../../index-
|
|
2
|
-
import '../../schema-
|
|
3
|
-
import '../../adapters-
|
|
1
|
+
import { C as ClusterCoordinator } from '../../index-lcdGLyJH.js';
|
|
2
|
+
import '../../schema-4W_7tk1f.js';
|
|
3
|
+
import '../../adapters-BQFQ4PRz.js';
|
|
4
4
|
import '../../synonyms-CpOAHPa-.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { MAX_NODE_ID_LENGTH, MAX_WATCHERS } from '../../chunk-ORBLD3EL.mjs';
|
|
2
2
|
import '../../chunk-R6A773WK.mjs';
|
|
3
3
|
import { compareCodePoints } from '../../chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../../chunk-KV3AZFT3.mjs';
|
|
5
5
|
import { decode, encode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
7
7
|
// src/distribution/coordinator/etcd/constants.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { C as ClusterCoordinator } from '../../index-
|
|
2
|
-
import '../../schema-
|
|
3
|
-
import '../../adapters-
|
|
1
|
+
import { C as ClusterCoordinator } from '../../index-lcdGLyJH.js';
|
|
2
|
+
import '../../schema-4W_7tk1f.js';
|
|
3
|
+
import '../../adapters-BQFQ4PRz.js';
|
|
4
4
|
import '../../synonyms-CpOAHPa-.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-RSRVGAIQ.mjs';
|
|
2
2
|
import { MAX_MESSAGE_SIZE_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, GRPC_ENVELOPE_OVERHEAD_BYTES } from '../../chunk-UNEEFTEU.mjs';
|
|
3
3
|
import { TransportError, TransportErrorCodes } from '../../chunk-OHP7QR33.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../../chunk-KV3AZFT3.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/transport/grpc/codec.ts
|
|
7
7
|
var FIELD_ONE_LENGTH_DELIMITED_KEY = 10;
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-
|
|
2
|
-
export { c as ClusterControllerConfig, d as ClusterMemoryConfig, e as ClusterNodeInfo, f as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-
|
|
3
|
-
import {
|
|
4
|
-
import { I as
|
|
1
|
+
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-BGB_fDtq.js';
|
|
2
|
+
export { c as ClusterControllerConfig, d as ClusterMemoryConfig, e as ClusterNodeInfo, f as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-BGB_fDtq.js';
|
|
3
|
+
import { R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
4
|
+
import { B as BatchResult, L as ListParams, g as ListResult, h as SuggestParams, j as SuggestResult, Q as QueryParams, P as PreflightResult, I as IndexStats, k as PartitionStatsResult, M as MemoryStats, l as NarsilEventMap, e as QueryResult } from './memory-CUEDwr-F.js';
|
|
5
|
+
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-4W_7tk1f.js';
|
|
5
6
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
6
|
-
import { b as NodeCapacity, N as NodeRole } from './index-
|
|
7
|
-
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, I as IndexPartitionBytes, e as NodeEvent, f as NodeMemoryReport, a as NodeRegistration, P as PartitionAssignment, g as PartitionState, S as SchemaEvent, U as UnassignedReason } from './index-
|
|
8
|
-
import { N as Narsil } from './engine-
|
|
7
|
+
import { b as NodeCapacity, N as NodeRole } from './index-lcdGLyJH.js';
|
|
8
|
+
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, I as IndexPartitionBytes, e as NodeEvent, f as NodeMemoryReport, a as NodeRegistration, P as PartitionAssignment, g as PartitionState, S as SchemaEvent, U as UnassignedReason } from './index-lcdGLyJH.js';
|
|
9
|
+
import { N as Narsil } from './engine-BGCEm9E_.js';
|
|
9
10
|
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-Cb9kiY5P.js';
|
|
10
|
-
import './config-
|
|
11
|
-
import './adapters-79OIa4xm.js';
|
|
11
|
+
import './config-P__Diiy4.js';
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* The cluster splits an index into this many partitions when
|
|
@@ -268,6 +268,16 @@ interface ClusterNode {
|
|
|
268
268
|
* @returns Process heap figures and this node's engine estimate.
|
|
269
269
|
*/
|
|
270
270
|
getMemoryStats(): Promise<MemoryStats>;
|
|
271
|
+
/**
|
|
272
|
+
* Registers a rerank adapter on this node under a name that a query gives
|
|
273
|
+
* in `rerank.adapter`. The node that receives a reranked query scores it
|
|
274
|
+
* with its own adapter, so register the adapter on every node that takes
|
|
275
|
+
* queries from clients. A data node needs none.
|
|
276
|
+
*
|
|
277
|
+
* @param name - The name that a query gives in `rerank.adapter`.
|
|
278
|
+
* @param adapter - The adapter that scores the top hits of a query.
|
|
279
|
+
*/
|
|
280
|
+
registerRerankAdapter(name: string, adapter: RerankAdapter): void;
|
|
271
281
|
/**
|
|
272
282
|
* Starts delivering one kind of engine event from this node's local engine
|
|
273
283
|
* to your listener.
|
|
@@ -372,9 +382,11 @@ interface ClusterEngineOptions {
|
|
|
372
382
|
* serves a single engine.
|
|
373
383
|
*
|
|
374
384
|
* Writes, updates, searches, suggestions, listings, counts, index creation,
|
|
375
|
-
* index removal, synonym sets, and document reads route through the cluster.
|
|
376
|
-
* memory statistics, index opens, index closes,
|
|
377
|
-
* node's local engine, because
|
|
385
|
+
* index removal, synonym sets, and document reads route through the cluster.
|
|
386
|
+
* Checkpointing, memory statistics, index opens, index closes, engine events,
|
|
387
|
+
* and rerank adapter registration reach this node's local engine, because
|
|
388
|
+
* each covers a per-node fact. This node reranks the searches that it
|
|
389
|
+
* receives with the adapters registered on it. Every other operation fails with
|
|
378
390
|
* `CLUSTER_OPERATION_UNSUPPORTED`, which the HTTP server answers with status
|
|
379
391
|
* 501, so a caller learns the operation is missing rather than reading a
|
|
380
392
|
* wrong answer. That covers `getStats`, `getPartitionStats`, and
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,49 +1,51 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import './chunk-
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
1
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexConfigKey, indexUuidOfStoredMetadata, getIndexMetadata, withPartitionCount, indexSettingsOf, putIndexMetadata, indexConfigFromSettings, metadataWithSynonymSet, isRecord, storedSynonymSetOf, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, isValidInteger, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, getClusterIndexConfig } from './chunk-2WYMG3OZ.mjs';
|
|
2
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-FMWRCP2C.mjs';
|
|
3
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-NTRMFARG.mjs';
|
|
4
|
+
import './chunk-COVCQ73F.mjs';
|
|
5
|
+
import { indexMetadataOf, loadPartitionSegments } from './chunk-ZG477IPV.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-WCJKUGHB.mjs';
|
|
5
7
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
6
|
-
import
|
|
7
|
-
import './chunk-ICL2AELD.mjs';
|
|
8
|
-
import './chunk-37CD3OKS.mjs';
|
|
8
|
+
import './chunk-HXJBSHXF.mjs';
|
|
9
9
|
import './chunk-DJHHF2L5.mjs';
|
|
10
|
-
import
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
10
|
+
import './chunk-37CD3OKS.mjs';
|
|
11
|
+
import { validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-LH4NZNJJ.mjs';
|
|
12
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-L3L5YFXC.mjs';
|
|
13
13
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import './chunk-
|
|
17
|
-
import {
|
|
18
|
-
import
|
|
19
|
-
import { providedDocId } from './chunk-WLOF6HSP.mjs';
|
|
14
|
+
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-OHP7QR33.mjs';
|
|
15
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update } from './chunk-VWULPWRJ.mjs';
|
|
16
|
+
import { runtimeHost, queryBindingOf, now, listBindingOf, requireSurfaceFormsForPartialFields, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-NKRBEGJZ.mjs';
|
|
17
|
+
import { bindRerankWindowSource, rerankWindowCapOf, engineCoreOf, providedDocId } from './chunk-R7EEKAJE.mjs';
|
|
18
|
+
import './chunk-AY3AQ3UH.mjs';
|
|
20
19
|
import './chunk-AV4IJKD2.mjs';
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
20
|
+
import { synonymModule, readHeapStatistics } from './chunk-OWNZYMRE.mjs';
|
|
21
|
+
import { oversampledShardSize, applyProjection, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-K25LZA54.mjs';
|
|
22
|
+
import { generateId, getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
23
|
+
import './chunk-HJSN7UYG.mjs';
|
|
24
|
+
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-LXWDQLTW.mjs';
|
|
23
25
|
import './chunk-TGP5M5R7.mjs';
|
|
24
26
|
import './chunk-U2BKYEC7.mjs';
|
|
25
27
|
import './chunk-KOKM4PQN.mjs';
|
|
26
28
|
import { MIN_OVERSAMPLE } from './chunk-QY7GC2MW.mjs';
|
|
27
|
-
import './chunk-
|
|
28
|
-
import { isPatternTolerance } from './chunk-
|
|
29
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
30
|
+
import { isPatternTolerance } from './chunk-ASY5PHYI.mjs';
|
|
29
31
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
-
import { validateSchema, requireCreatableSchema, encodeDocument, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
32
|
-
import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW } from './chunk-
|
|
32
|
+
import { getNestedValue } from './chunk-MHOSCQCQ.mjs';
|
|
33
|
+
import { validateSchema, requireCreatableSchema, encodeDocument, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-LXPIXDUJ.mjs';
|
|
34
|
+
import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, HYBRID_RESCORE_WINDOW, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-E74YLAIG.mjs';
|
|
33
35
|
import './chunk-KEHK2O6P.mjs';
|
|
34
36
|
import './chunk-GES7U2PM.mjs';
|
|
35
|
-
import './chunk-
|
|
36
|
-
import './chunk-
|
|
37
|
+
import './chunk-OOEXLUGR.mjs';
|
|
38
|
+
import './chunk-RONEOEON.mjs';
|
|
37
39
|
import './chunk-X5CV2E2B.mjs';
|
|
38
40
|
import './chunk-7X5JST5L.mjs';
|
|
39
|
-
import './chunk-
|
|
41
|
+
import './chunk-LU2CE5AE.mjs';
|
|
40
42
|
import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
41
43
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
42
44
|
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
43
45
|
import './chunk-R6PR4WYZ.mjs';
|
|
44
|
-
import './chunk-
|
|
46
|
+
import './chunk-PPY2GSIB.mjs';
|
|
45
47
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
46
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
48
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-KV3AZFT3.mjs';
|
|
47
49
|
import { decode, encode } from '@msgpack/msgpack';
|
|
48
50
|
import { randomUUID } from 'crypto';
|
|
49
51
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -5817,7 +5819,7 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
5817
5819
|
applyReplicationEntry: (entry) => applyLocalReplicationEntry(core, entry),
|
|
5818
5820
|
archiveReplicationPartition: (indexName, partitionId, lastSeqNo) => archiveReplicationPartition(core, indexName, partitionId, lastSeqNo),
|
|
5819
5821
|
restoreReplicationPartition: (indexName, partitionId, source, indexConfig, partitionCount) => restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount),
|
|
5820
|
-
queryPartitions: (indexName, params, partitionIds, globalStats) => runEngineQuery(core, indexName, params, { partitionIds, globalStats }),
|
|
5822
|
+
queryPartitions: (indexName, params, partitionIds, globalStats, rescoreWindow) => runEngineQuery(core, indexName, params, { partitionIds, globalStats, rescoreWindow }),
|
|
5821
5823
|
preflightPartitions: (indexName, params, partitionIds) => runEnginePreflight(core, indexName, params, { partitionIds }),
|
|
5822
5824
|
suggestPartitions: (indexName, params, partitionIds) => runEngineSuggest(core, indexName, params, partitionIds),
|
|
5823
5825
|
listPartitions: (indexName, params, partitionIds) => runEngineListDocuments(core, indexName, params, partitionIds),
|
|
@@ -6432,6 +6434,7 @@ function validateHybridParams(value) {
|
|
|
6432
6434
|
if (!isRecord2(value)) {
|
|
6433
6435
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
6434
6436
|
}
|
|
6437
|
+
if (!Object.hasOwn(value, "depth")) value.depth = null;
|
|
6435
6438
|
if (value.strategy !== null && !ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
6436
6439
|
throwInvalid(
|
|
6437
6440
|
SEARCH_INVALID_MODE,
|
|
@@ -6450,6 +6453,12 @@ function validateHybridParams(value) {
|
|
|
6450
6453
|
'Invalid SearchPayload: "params.hybrid.alpha" must be a finite number in [0, 1], or null'
|
|
6451
6454
|
);
|
|
6452
6455
|
}
|
|
6456
|
+
if (value.depth !== null && (!isInteger(value.depth) || value.depth < MIN_HYBRID_DEPTH || value.depth > MAX_HYBRID_DEPTH)) {
|
|
6457
|
+
throwInvalid(
|
|
6458
|
+
CONFIG_INVALID,
|
|
6459
|
+
`Invalid SearchPayload: "params.hybrid.depth" must be an integer between ${MIN_HYBRID_DEPTH} and ${MAX_HYBRID_DEPTH}, or null`
|
|
6460
|
+
);
|
|
6461
|
+
}
|
|
6453
6462
|
}
|
|
6454
6463
|
function validateGroupParams(value) {
|
|
6455
6464
|
if (!isRecord2(value)) {
|
|
@@ -6608,6 +6617,14 @@ function validateEfSearchParam(value) {
|
|
|
6608
6617
|
);
|
|
6609
6618
|
}
|
|
6610
6619
|
}
|
|
6620
|
+
function validateRescoreWindowParam(value) {
|
|
6621
|
+
if (!isInteger(value) || value < MIN_HYBRID_DEPTH || value > MAX_HYBRID_DEPTH) {
|
|
6622
|
+
throwInvalid(
|
|
6623
|
+
CONFIG_INVALID,
|
|
6624
|
+
`Invalid SearchPayload: "params.vector.rescoreWindow" must be an integer between ${MIN_HYBRID_DEPTH} and ${MAX_HYBRID_DEPTH}, or null`
|
|
6625
|
+
);
|
|
6626
|
+
}
|
|
6627
|
+
}
|
|
6611
6628
|
function validateOversampleParam(value) {
|
|
6612
6629
|
if (typeof value !== "number" || !Number.isFinite(value) || value < MIN_OVERSAMPLE || value > MAX_OVERSAMPLE) {
|
|
6613
6630
|
throwInvalid(
|
|
@@ -6632,6 +6649,7 @@ function validateVectorParams(value) {
|
|
|
6632
6649
|
if (!isRecord2(value)) {
|
|
6633
6650
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
|
|
6634
6651
|
}
|
|
6652
|
+
if (!Object.hasOwn(value, "rescoreWindow")) value.rescoreWindow = null;
|
|
6635
6653
|
validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
|
|
6636
6654
|
if (value.value !== null) {
|
|
6637
6655
|
if (!Array.isArray(value.value)) {
|
|
@@ -6681,6 +6699,9 @@ function validateVectorParams(value) {
|
|
|
6681
6699
|
if (value.oversample !== null) {
|
|
6682
6700
|
validateOversampleParam(value.oversample);
|
|
6683
6701
|
}
|
|
6702
|
+
if (value.rescoreWindow !== null) {
|
|
6703
|
+
validateRescoreWindowParam(value.rescoreWindow);
|
|
6704
|
+
}
|
|
6684
6705
|
}
|
|
6685
6706
|
function validateSortParams(value) {
|
|
6686
6707
|
if (!Array.isArray(value)) {
|
|
@@ -7572,6 +7593,26 @@ async function sendThroughTargets(targets, send, failure) {
|
|
|
7572
7593
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
7573
7594
|
}
|
|
7574
7595
|
|
|
7596
|
+
// src/distribution/cluster-node/vector-bytes.ts
|
|
7597
|
+
var FLOAT32_BYTES = 4;
|
|
7598
|
+
function float32Of(bytes) {
|
|
7599
|
+
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
|
|
7600
|
+
}
|
|
7601
|
+
function restoreVectorValues(value) {
|
|
7602
|
+
if (value instanceof Uint8Array) {
|
|
7603
|
+
return value.byteLength % FLOAT32_BYTES === 0 ? float32Of(value) : value;
|
|
7604
|
+
}
|
|
7605
|
+
if (Array.isArray(value)) {
|
|
7606
|
+
for (let i = 0; i < value.length; i++) value[i] = restoreVectorValues(value[i]);
|
|
7607
|
+
return value;
|
|
7608
|
+
}
|
|
7609
|
+
if (typeof value === "object" && value !== null && !ArrayBuffer.isView(value)) {
|
|
7610
|
+
const record = value;
|
|
7611
|
+
for (const key of Object.keys(record)) record[key] = restoreVectorValues(record[key]);
|
|
7612
|
+
}
|
|
7613
|
+
return value;
|
|
7614
|
+
}
|
|
7615
|
+
|
|
7575
7616
|
// src/distribution/cluster-node/allocation-wait.ts
|
|
7576
7617
|
function everyPartitionActive(table, partitionCount) {
|
|
7577
7618
|
if (table === null || table.assignments.size < partitionCount) {
|
|
@@ -9151,7 +9192,8 @@ async function fetchDistributedDocuments(config, nodeId, engine, indexName, resu
|
|
|
9151
9192
|
const decoded = decode(response.payload);
|
|
9152
9193
|
const payload = validateFetchResultPayload(decoded);
|
|
9153
9194
|
for (const fetched of payload.documents) {
|
|
9154
|
-
|
|
9195
|
+
const document = restoreVectorValues(fetched.document);
|
|
9196
|
+
documents.set(fetched.docId, applyProjection(document, projection));
|
|
9155
9197
|
}
|
|
9156
9198
|
}
|
|
9157
9199
|
return documents;
|
|
@@ -9203,7 +9245,7 @@ async function readDistributedDocuments(config, nodeId, engine, indexName, docId
|
|
|
9203
9245
|
});
|
|
9204
9246
|
const payload = validateFetchResultPayload(decode(response.payload));
|
|
9205
9247
|
for (const fetched of payload.documents) {
|
|
9206
|
-
documents.set(fetched.docId, fetched.document);
|
|
9248
|
+
documents.set(fetched.docId, restoreVectorValues(fetched.document));
|
|
9207
9249
|
}
|
|
9208
9250
|
}
|
|
9209
9251
|
}
|
|
@@ -10267,6 +10309,7 @@ function convertWireHybridToLocal(hybrid) {
|
|
|
10267
10309
|
if (hybrid.strategy !== null) local.strategy = hybrid.strategy;
|
|
10268
10310
|
if (hybrid.k !== null) local.k = hybrid.k;
|
|
10269
10311
|
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
10312
|
+
if (hybrid.depth !== null) local.depth = hybrid.depth;
|
|
10270
10313
|
return local;
|
|
10271
10314
|
}
|
|
10272
10315
|
function convertWireSortToLocal(wireSort) {
|
|
@@ -10393,7 +10436,8 @@ function convertLocalVectorToWire(vector) {
|
|
|
10393
10436
|
similarity: vector.similarity ?? null,
|
|
10394
10437
|
metric: vector.metric ?? null,
|
|
10395
10438
|
efSearch: vector.efSearch ?? null,
|
|
10396
|
-
oversample: vector.oversample ?? null
|
|
10439
|
+
oversample: vector.oversample ?? null,
|
|
10440
|
+
rescoreWindow: null
|
|
10397
10441
|
};
|
|
10398
10442
|
}
|
|
10399
10443
|
function convertLocalHybridToWire(hybrid) {
|
|
@@ -10403,7 +10447,8 @@ function convertLocalHybridToWire(hybrid) {
|
|
|
10403
10447
|
return {
|
|
10404
10448
|
strategy: hybrid.strategy ?? null,
|
|
10405
10449
|
k: hybrid.k ?? null,
|
|
10406
|
-
alpha: hybrid.alpha ?? null
|
|
10450
|
+
alpha: hybrid.alpha ?? null,
|
|
10451
|
+
depth: hybrid.depth ?? null
|
|
10407
10452
|
};
|
|
10408
10453
|
}
|
|
10409
10454
|
function facetLimitOf(options) {
|
|
@@ -10444,11 +10489,16 @@ async function handleSearch(message, respond, deps) {
|
|
|
10444
10489
|
if (queryParams.group?.limit !== void 0) {
|
|
10445
10490
|
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
10446
10491
|
}
|
|
10492
|
+
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
10493
|
+
if (rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
10494
|
+
queryParams.document = false;
|
|
10495
|
+
}
|
|
10447
10496
|
const queryResult = await deps.engine.queryPartitions(
|
|
10448
10497
|
payload.indexName,
|
|
10449
10498
|
queryParams,
|
|
10450
10499
|
payload.partitionIds,
|
|
10451
|
-
payload.globalStats ?? void 0
|
|
10500
|
+
payload.globalStats ?? void 0,
|
|
10501
|
+
rescoreWindow
|
|
10452
10502
|
);
|
|
10453
10503
|
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
10454
10504
|
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
@@ -11517,21 +11567,66 @@ function placePinnedEntries(merged, pinned, depth, allMatchesPresent) {
|
|
|
11517
11567
|
}
|
|
11518
11568
|
|
|
11519
11569
|
// src/distribution/query/hybrid.ts
|
|
11520
|
-
|
|
11521
|
-
|
|
11522
|
-
|
|
11523
|
-
|
|
11524
|
-
|
|
11570
|
+
function positionedOf(entry) {
|
|
11571
|
+
return entry.score === null ? { id: entry.docId, entry } : { id: entry.docId, score: entry.score, entry };
|
|
11572
|
+
}
|
|
11573
|
+
function pageLegsOf(params, depth) {
|
|
11574
|
+
return {
|
|
11575
|
+
text: { ...params, vector: null, hybrid: null, mode: null, limit: depth, offset: 0 },
|
|
11576
|
+
vector: {
|
|
11577
|
+
...params,
|
|
11578
|
+
term: null,
|
|
11579
|
+
hybrid: null,
|
|
11580
|
+
mode: null,
|
|
11581
|
+
group: null,
|
|
11582
|
+
facets: null,
|
|
11583
|
+
facetSize: null,
|
|
11584
|
+
limit: depth,
|
|
11585
|
+
offset: 0
|
|
11586
|
+
}
|
|
11587
|
+
};
|
|
11588
|
+
}
|
|
11589
|
+
function fixedLegsOf(params, depth) {
|
|
11590
|
+
const legParams = {
|
|
11525
11591
|
...params,
|
|
11526
|
-
term: null,
|
|
11527
11592
|
hybrid: null,
|
|
11528
11593
|
mode: null,
|
|
11529
|
-
|
|
11530
|
-
|
|
11531
|
-
|
|
11594
|
+
sort: null,
|
|
11595
|
+
searchAfter: null,
|
|
11596
|
+
pinned: null,
|
|
11532
11597
|
limit: depth,
|
|
11533
11598
|
offset: 0
|
|
11534
11599
|
};
|
|
11600
|
+
return {
|
|
11601
|
+
text: { ...legParams, vector: null },
|
|
11602
|
+
vector: {
|
|
11603
|
+
...legParams,
|
|
11604
|
+
vector: legParams.vector === null ? null : { ...legParams.vector, rescoreWindow: HYBRID_RESCORE_WINDOW },
|
|
11605
|
+
term: null,
|
|
11606
|
+
group: null,
|
|
11607
|
+
facets: null,
|
|
11608
|
+
facetSize: null
|
|
11609
|
+
}
|
|
11610
|
+
};
|
|
11611
|
+
}
|
|
11612
|
+
function pageFixedDepth(placed, fused, pins, allMatchesPresent, start, page, depth) {
|
|
11613
|
+
const placedIds = pins === null ? void 0 : new Set(pins.filter((entry) => allMatchesPresent || entry.position < depth).map((entry) => entry.docId));
|
|
11614
|
+
const paged = pageByPosition(placed.map(positionedOf), fused.map(positionedOf), {
|
|
11615
|
+
start,
|
|
11616
|
+
limit: page.limit,
|
|
11617
|
+
pinnedIds: placedIds,
|
|
11618
|
+
binding: page.binding,
|
|
11619
|
+
encodeCursor: encodePageCursor
|
|
11620
|
+
});
|
|
11621
|
+
return { page: paged.page.map((hit) => hit.entry), cursor: paged.cursor ?? null };
|
|
11622
|
+
}
|
|
11623
|
+
async function executeHybridQuery(indexName, params, page, facetShardSize, facetSize, totalPartitions, routing, deps, config) {
|
|
11624
|
+
const fusion = resolveHybridFusion(params.hybrid);
|
|
11625
|
+
const fixedDepth = fixedHybridDepthOf(params.hybrid);
|
|
11626
|
+
const cursorDepth = fixedDepth === void 0 || params.searchAfter === null ? 0 : decodePageCursor(params.searchAfter).depth ?? 0;
|
|
11627
|
+
if (fixedDepth !== void 0) requireWithinHybridDepth(cursorDepth, page.offset, fixedDepth);
|
|
11628
|
+
const depth = fixedDepth ?? page.limit + page.offset;
|
|
11629
|
+
const { text: textParams, vector: vectorParams } = fixedDepth === void 0 ? pageLegsOf(params, depth) : fixedLegsOf(params, depth);
|
|
11535
11630
|
let textGlobalStats = null;
|
|
11536
11631
|
if (params.scoring === "dfs") {
|
|
11537
11632
|
textGlobalStats = await collectDistributedStats(indexName, textParams, routing.nodeToPartitions, deps, config);
|
|
@@ -11555,6 +11650,7 @@ async function executeHybridQuery(indexName, params, limit, offset, facetShardSi
|
|
|
11555
11650
|
const allFacetBounds = [];
|
|
11556
11651
|
const allGroups = [];
|
|
11557
11652
|
let totalHits = 0;
|
|
11653
|
+
let vectorHits = 0;
|
|
11558
11654
|
for (const outcome of textOutcomes) {
|
|
11559
11655
|
if (outcome.status !== "success" || outcome.results === null) continue;
|
|
11560
11656
|
for (const partitionResult of outcome.results.results) {
|
|
@@ -11577,6 +11673,7 @@ async function executeHybridQuery(indexName, params, limit, offset, facetShardSi
|
|
|
11577
11673
|
if (partitionResult.scored.length > 0) {
|
|
11578
11674
|
vectorScored.push(partitionResult.scored);
|
|
11579
11675
|
}
|
|
11676
|
+
vectorHits += partitionResult.totalHits;
|
|
11580
11677
|
}
|
|
11581
11678
|
}
|
|
11582
11679
|
const mergedText = mergeAndTruncateScoredEntries(textScored, depth);
|
|
@@ -11587,18 +11684,20 @@ async function executeHybridQuery(indexName, params, limit, offset, facetShardSi
|
|
|
11587
11684
|
} else {
|
|
11588
11685
|
fused = distributedLinearCombination(mergedText, mergedVector, { alpha: fusion.alpha });
|
|
11589
11686
|
}
|
|
11590
|
-
|
|
11591
|
-
const
|
|
11592
|
-
const
|
|
11687
|
+
if (fixedDepth !== void 0 && fused.length > depth) fused = fused.slice(0, depth);
|
|
11688
|
+
const allMatchesPresent = coverage.queriedPartitions === coverage.totalPartitions && totalHits <= mergedText.length && (fixedDepth === void 0 ? mergedVector.length < depth : params.vector?.similarity === null && vectorHits <= mergedVector.length);
|
|
11689
|
+
const pins = fixedDepth === void 0 || params.searchAfter === null ? params.pinned : null;
|
|
11690
|
+
const placement = pins !== null ? placePinnedEntries(fused, pins, depth, allMatchesPresent) : { entries: fused, placedFromOutside: [] };
|
|
11691
|
+
const paged = fixedDepth === void 0 ? { page: placement.entries.slice(page.offset, depth), cursor: null } : pageFixedDepth(placement.entries, fused, pins, allMatchesPresent, cursorDepth + page.offset, page, depth);
|
|
11593
11692
|
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize, ascendingFacetFields(params.facets)) : null;
|
|
11594
11693
|
return {
|
|
11595
|
-
scored:
|
|
11694
|
+
scored: paged.page,
|
|
11596
11695
|
totalHits,
|
|
11597
11696
|
facets: mergedFacets?.facets ?? null,
|
|
11598
11697
|
facetErrorBounds: mergedFacets?.errorBounds ?? null,
|
|
11599
11698
|
facetUndercounts: mergedFacets?.undercounts ?? null,
|
|
11600
11699
|
groups: mergeGroupsFor(params, allGroups, null),
|
|
11601
|
-
cursor:
|
|
11700
|
+
cursor: paged.cursor,
|
|
11602
11701
|
coverage,
|
|
11603
11702
|
pinnedFromOutside: placement.placedFromOutside,
|
|
11604
11703
|
mergeHeldEveryMatch: allMatchesPresent
|
|
@@ -11657,10 +11756,10 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
11657
11756
|
);
|
|
11658
11757
|
}
|
|
11659
11758
|
const isHybrid = params.vector !== null && (params.term !== null || params.mode === "hybrid");
|
|
11660
|
-
if (isHybrid && params.searchAfter !== null) {
|
|
11759
|
+
if (isHybrid && params.searchAfter !== null && (params.hybrid?.depth ?? null) === null) {
|
|
11661
11760
|
throw new NarsilError(
|
|
11662
11761
|
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
11663
|
-
"Cursor pagination is not supported for hybrid queries, because fused ranks do not seek",
|
|
11762
|
+
"Cursor pagination is not supported for hybrid queries without hybrid.depth, because fused ranks do not seek. Set hybrid.depth to page by cursor",
|
|
11664
11763
|
{ indexName }
|
|
11665
11764
|
);
|
|
11666
11765
|
}
|
|
@@ -11675,8 +11774,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
11675
11774
|
return executeHybridQuery(
|
|
11676
11775
|
indexName,
|
|
11677
11776
|
params,
|
|
11678
|
-
limit,
|
|
11679
|
-
offset,
|
|
11777
|
+
{ limit, offset, binding },
|
|
11680
11778
|
facetShardSize,
|
|
11681
11779
|
facetSize,
|
|
11682
11780
|
totalPartitions,
|
|
@@ -12146,7 +12244,7 @@ async function listCluster(deps, indexName, params) {
|
|
|
12146
12244
|
}
|
|
12147
12245
|
const documents = page.map((item) => ({
|
|
12148
12246
|
id: item.entry.docId,
|
|
12149
|
-
document: projection.kind === "none" ? {} : applyProjection(item.entry.document, projection)
|
|
12247
|
+
document: projection.kind === "none" ? {} : applyProjection(restoreVectorValues(item.entry.document), projection)
|
|
12150
12248
|
}));
|
|
12151
12249
|
return { documents, cursor, total, elapsed: now() - startTime };
|
|
12152
12250
|
}
|
|
@@ -12239,6 +12337,13 @@ async function preflightCluster(deps, indexName, params) {
|
|
|
12239
12337
|
}
|
|
12240
12338
|
|
|
12241
12339
|
// src/distribution/cluster-node/reads/index.ts
|
|
12340
|
+
var clusterRerankHost = {
|
|
12341
|
+
...runtimeHost,
|
|
12342
|
+
engineCoreOf,
|
|
12343
|
+
queryBindingOf,
|
|
12344
|
+
getNestedValue,
|
|
12345
|
+
applyProjection
|
|
12346
|
+
};
|
|
12242
12347
|
async function readClusterDocuments(deps, indexName, docIds) {
|
|
12243
12348
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
12244
12349
|
if (allocation === null) {
|
|
@@ -12264,6 +12369,17 @@ async function queryCluster(deps, indexName, params) {
|
|
|
12264
12369
|
if (allocation === null) {
|
|
12265
12370
|
return deps.engine.query(indexName, params);
|
|
12266
12371
|
}
|
|
12372
|
+
if (params.rerank !== void 0) {
|
|
12373
|
+
const { rerankClusterQuery } = await import('./rerank-NWTKK636.mjs');
|
|
12374
|
+
return rerankClusterQuery(
|
|
12375
|
+
deps,
|
|
12376
|
+
indexName,
|
|
12377
|
+
params,
|
|
12378
|
+
(firstStage) => queryCluster(deps, indexName, firstStage),
|
|
12379
|
+
(docIds) => readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, docIds, allocation),
|
|
12380
|
+
clusterRerankHost
|
|
12381
|
+
);
|
|
12382
|
+
}
|
|
12267
12383
|
const wireParams = localParamsToWire(params);
|
|
12268
12384
|
const queryDeps = {
|
|
12269
12385
|
transport: deps.config.transport,
|
|
@@ -13004,6 +13120,7 @@ async function createClusterNode(config) {
|
|
|
13004
13120
|
},
|
|
13005
13121
|
open: (indexName) => trackOp(indexName, () => engine.open(indexName)),
|
|
13006
13122
|
close: (indexName) => transitionIndex(indexName, () => engine.close(indexName)),
|
|
13123
|
+
registerRerankAdapter: (name, adapter) => engine.registerRerankAdapter(name, adapter),
|
|
13007
13124
|
...createClusterNodeOperations({
|
|
13008
13125
|
trackOp,
|
|
13009
13126
|
transitionIndex,
|
|
@@ -13084,6 +13201,7 @@ async function createClusterNode(config) {
|
|
|
13084
13201
|
await snapshotSpool.dispose();
|
|
13085
13202
|
}
|
|
13086
13203
|
};
|
|
13204
|
+
bindRerankWindowSource(node, () => rerankWindowCapOf(engine));
|
|
13087
13205
|
return node;
|
|
13088
13206
|
function resolveSnapshotHeaderMetadata(indexName, partitionId) {
|
|
13089
13207
|
return nodeSnapshotHeaderMetadata(
|
|
@@ -13106,7 +13224,7 @@ function rejectWait(options) {
|
|
|
13106
13224
|
if (options?.wait === true) unsupported("wait");
|
|
13107
13225
|
}
|
|
13108
13226
|
function clusterNodeEngine(node, options) {
|
|
13109
|
-
|
|
13227
|
+
const engine = {
|
|
13110
13228
|
open(indexName) {
|
|
13111
13229
|
return node.open(indexName);
|
|
13112
13230
|
},
|
|
@@ -13185,6 +13303,9 @@ function clusterNodeEngine(node, options) {
|
|
|
13185
13303
|
on(event, handler) {
|
|
13186
13304
|
node.on(event, handler);
|
|
13187
13305
|
},
|
|
13306
|
+
registerRerankAdapter(name, adapter) {
|
|
13307
|
+
node.registerRerankAdapter(name, adapter);
|
|
13308
|
+
},
|
|
13188
13309
|
off(event, handler) {
|
|
13189
13310
|
node.off(event, handler);
|
|
13190
13311
|
},
|
|
@@ -13207,6 +13328,8 @@ function clusterNodeEngine(node, options) {
|
|
|
13207
13328
|
optimizeVectors: () => unsupported("optimizeVectors"),
|
|
13208
13329
|
vectorMaintenanceStatus: () => unsupported("vectorMaintenanceStatus")
|
|
13209
13330
|
};
|
|
13331
|
+
bindRerankWindowSource(engine, () => rerankWindowCapOf(node));
|
|
13332
|
+
return engine;
|
|
13210
13333
|
}
|
|
13211
13334
|
|
|
13212
13335
|
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, clusterNodeEngine, createClusterNode };
|
|
@@ -1,45 +1,46 @@
|
|
|
1
|
-
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-
|
|
3
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-BKB3XTMI.mjs';
|
|
2
|
+
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-ZG477IPV.mjs';
|
|
3
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WCJKUGHB.mjs';
|
|
4
4
|
import './chunk-37CD3OKS.mjs';
|
|
5
|
-
import { createDurableDirectory, getFs } from './chunk-
|
|
5
|
+
import { createDurableDirectory, getFs } from './chunk-45AVKHMJ.mjs';
|
|
6
6
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
7
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-
|
|
8
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
9
|
-
import './chunk-OHP7QR33.mjs';
|
|
7
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-LH4NZNJJ.mjs';
|
|
8
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-L3L5YFXC.mjs';
|
|
10
9
|
import './chunk-P36MUFEJ.mjs';
|
|
11
|
-
import
|
|
12
|
-
import './chunk-
|
|
13
|
-
import
|
|
14
|
-
import
|
|
15
|
-
import {
|
|
16
|
-
import './chunk-WLOF6HSP.mjs';
|
|
10
|
+
import './chunk-OHP7QR33.mjs';
|
|
11
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-VWULPWRJ.mjs';
|
|
12
|
+
import './chunk-NKRBEGJZ.mjs';
|
|
13
|
+
import './chunk-R7EEKAJE.mjs';
|
|
14
|
+
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
17
15
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
18
|
-
import {
|
|
19
|
-
import './chunk-
|
|
16
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-OWNZYMRE.mjs';
|
|
17
|
+
import { fnv1a } from './chunk-K25LZA54.mjs';
|
|
18
|
+
import './chunk-JFRNNBOM.mjs';
|
|
19
|
+
import './chunk-HJSN7UYG.mjs';
|
|
20
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LXWDQLTW.mjs';
|
|
20
21
|
import './chunk-TGP5M5R7.mjs';
|
|
21
22
|
import './chunk-U2BKYEC7.mjs';
|
|
22
23
|
import './chunk-KOKM4PQN.mjs';
|
|
23
24
|
import './chunk-QY7GC2MW.mjs';
|
|
24
|
-
import './chunk-
|
|
25
|
-
import './chunk-
|
|
25
|
+
import './chunk-GGSH6KBQ.mjs';
|
|
26
|
+
import './chunk-ASY5PHYI.mjs';
|
|
26
27
|
import './chunk-WQBCBKY3.mjs';
|
|
27
|
-
import './chunk-
|
|
28
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-
|
|
29
|
-
import './chunk-
|
|
28
|
+
import './chunk-MHOSCQCQ.mjs';
|
|
29
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-LXPIXDUJ.mjs';
|
|
30
|
+
import './chunk-E74YLAIG.mjs';
|
|
30
31
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
31
32
|
import './chunk-GES7U2PM.mjs';
|
|
32
|
-
import './chunk-
|
|
33
|
-
import './chunk-
|
|
33
|
+
import './chunk-OOEXLUGR.mjs';
|
|
34
|
+
import './chunk-RONEOEON.mjs';
|
|
34
35
|
import './chunk-X5CV2E2B.mjs';
|
|
35
36
|
import './chunk-7X5JST5L.mjs';
|
|
36
|
-
import './chunk-
|
|
37
|
+
import './chunk-LU2CE5AE.mjs';
|
|
37
38
|
import './chunk-R6A773WK.mjs';
|
|
38
39
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
39
40
|
import './chunk-R6PR4WYZ.mjs';
|
|
40
|
-
import './chunk-
|
|
41
|
+
import './chunk-PPY2GSIB.mjs';
|
|
41
42
|
import './chunk-L5ORED7V.mjs';
|
|
42
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
43
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
43
44
|
import { decode } from '@msgpack/msgpack';
|
|
44
45
|
|
|
45
46
|
// src/persistence/durability/sha256.ts
|