@delali/narsil 0.3.1-next.95 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-3U636ADD.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-LULHQXCK.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-J57MZK6L.mjs} +115 -61
- package/dist/browser/{write-ZUBDNLCH.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-7QONURHW.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-54HYC5FT.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-J7SDAOTP.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-Q4UTU5ZE.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-L26LEDHO.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-JFUE6F2Y.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-MASE2LTJ.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-BGt9lI1V.d.ts → engine-BGCEm9E_.d.ts} +50 -20
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-YC6TS4OP.mjs → snapshot-HOGXNO5W.mjs} +138 -83
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-M2OALSA4.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-K25LZA54.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QY7GC2MW.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT,
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-LXPIXDUJ.mjs';
|
|
5
|
+
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-E74YLAIG.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import { tokenize } from './chunk-
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-OOEXLUGR.mjs';
|
|
8
|
+
import { tokenize } from './chunk-LU2CE5AE.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
|
-
import {
|
|
10
|
+
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -23,6 +23,10 @@ var MAP_TAG = 7;
|
|
|
23
23
|
var LENGTH_BYTES = 4;
|
|
24
24
|
var LAST_ASCII_UNIT = 127;
|
|
25
25
|
var encoder = new TextEncoder();
|
|
26
|
+
function flush(stream) {
|
|
27
|
+
stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
|
|
28
|
+
stream.length = 0;
|
|
29
|
+
}
|
|
26
30
|
function ensureCapacity(stream, extra) {
|
|
27
31
|
if (stream.length + extra <= stream.bytes.length) return;
|
|
28
32
|
const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
@@ -78,23 +82,11 @@ var NumberList = class {
|
|
|
78
82
|
this.members = members;
|
|
79
83
|
}
|
|
80
84
|
};
|
|
81
|
-
var NUMBER_MEMBER_BYTES = 9;
|
|
82
|
-
var LIST_HEADER_BYTES = 5;
|
|
83
85
|
function writeNumberList(stream, list) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
let at = stream.length;
|
|
89
|
-
bytes[at] = LIST_TAG;
|
|
90
|
-
view.setUint32(at + 1, count, false);
|
|
91
|
-
at += LIST_HEADER_BYTES;
|
|
92
|
-
for (let i = 0; i < count; i++) {
|
|
93
|
-
bytes[at] = NUMBER_TAG;
|
|
94
|
-
view.setFloat64(at + 1, Math.fround(list.members[i]), false);
|
|
95
|
-
at += NUMBER_MEMBER_BYTES;
|
|
96
|
-
}
|
|
97
|
-
stream.length = at;
|
|
86
|
+
writeByte(stream, LIST_TAG);
|
|
87
|
+
writeUint32(stream, list.members.length);
|
|
88
|
+
flush(stream);
|
|
89
|
+
stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
|
|
98
90
|
}
|
|
99
91
|
function writeValue(stream, value) {
|
|
100
92
|
if (value instanceof NumberList) {
|
|
@@ -152,17 +144,26 @@ function boundVectorOf(vector) {
|
|
|
152
144
|
}
|
|
153
145
|
return bound;
|
|
154
146
|
}
|
|
147
|
+
function boundRerankOf(rerank) {
|
|
148
|
+
if (typeof rerank !== "object" || rerank === null || Array.isArray(rerank)) return rerank;
|
|
149
|
+
const present = {};
|
|
150
|
+
for (const [key, value] of Object.entries(rerank)) {
|
|
151
|
+
if (value !== void 0) present[key] = value;
|
|
152
|
+
}
|
|
153
|
+
return present;
|
|
154
|
+
}
|
|
155
155
|
var FILTERS_SLOT = 2;
|
|
156
156
|
var BOUND_SLOT_COUNT = 16;
|
|
157
157
|
function bindingOf(values) {
|
|
158
|
-
const stream = { bytes: new Uint8Array(256), length: 0 };
|
|
158
|
+
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS };
|
|
159
159
|
for (const value of values) {
|
|
160
160
|
writeValue(stream, value);
|
|
161
161
|
}
|
|
162
|
-
|
|
162
|
+
flush(stream);
|
|
163
|
+
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
163
164
|
}
|
|
164
165
|
function queryBindingOf(params) {
|
|
165
|
-
|
|
166
|
+
const values = [
|
|
166
167
|
params.term,
|
|
167
168
|
params.fields,
|
|
168
169
|
params.filters,
|
|
@@ -179,7 +180,9 @@ function queryBindingOf(params) {
|
|
|
179
180
|
params.hybrid,
|
|
180
181
|
boundVectorOf(params.vector),
|
|
181
182
|
params.scoring ?? "local"
|
|
182
|
-
]
|
|
183
|
+
];
|
|
184
|
+
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
185
|
+
return bindingOf(values);
|
|
183
186
|
}
|
|
184
187
|
function listBindingOf(filters) {
|
|
185
188
|
const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
|
|
@@ -997,81 +1000,6 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
997
1000
|
}
|
|
998
1001
|
}
|
|
999
1002
|
|
|
1000
|
-
// src/engine/query/sorted.ts
|
|
1001
|
-
function sortsWithoutScores(params) {
|
|
1002
|
-
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
1003
|
-
}
|
|
1004
|
-
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1005
|
-
const { manager, language, config } = context;
|
|
1006
|
-
const schema = config.schema;
|
|
1007
|
-
let anchorKey = null;
|
|
1008
|
-
let anchorId = null;
|
|
1009
|
-
if (params.searchAfter !== void 0) {
|
|
1010
|
-
const decoded = decodePageCursor(params.searchAfter);
|
|
1011
|
-
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
1012
|
-
anchorKey = decoded.sortKey;
|
|
1013
|
-
anchorId = decoded.anchor;
|
|
1014
|
-
}
|
|
1015
|
-
const normalized = normalizeSort(params.sort);
|
|
1016
|
-
const fields = normalized.map((entry) => entry.field);
|
|
1017
|
-
const directions = normalized.map((entry) => entry.direction);
|
|
1018
|
-
const flatSchema = flattenSchema(schema);
|
|
1019
|
-
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
1020
|
-
const modes = sortModesOf(normalized);
|
|
1021
|
-
const options = searchOptionsFor(manager, context.patternWork);
|
|
1022
|
-
const partitions = partitionsFor(manager, context.partitionIds);
|
|
1023
|
-
const partitionLimit = offset + limit + 1;
|
|
1024
|
-
const merged = [];
|
|
1025
|
-
const partitionFacets = [];
|
|
1026
|
-
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
1027
|
-
let count = 0;
|
|
1028
|
-
for (const partition of partitions) {
|
|
1029
|
-
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1030
|
-
if (partitionFacetConfig !== void 0) {
|
|
1031
|
-
partitionFacets.push(
|
|
1032
|
-
partition.computeFacets(
|
|
1033
|
-
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
1034
|
-
partitionFacetConfig,
|
|
1035
|
-
schema
|
|
1036
|
-
)
|
|
1037
|
-
);
|
|
1038
|
-
}
|
|
1039
|
-
if (matches === null || matches.count === 0) continue;
|
|
1040
|
-
count += matches.count;
|
|
1041
|
-
const page2 = partition.sortedPage({
|
|
1042
|
-
fields,
|
|
1043
|
-
directions,
|
|
1044
|
-
fieldTypes,
|
|
1045
|
-
modes,
|
|
1046
|
-
limit: partitionLimit,
|
|
1047
|
-
anchorKey,
|
|
1048
|
-
anchorId,
|
|
1049
|
-
matches
|
|
1050
|
-
});
|
|
1051
|
-
for (const entry of page2) merged.push(entry);
|
|
1052
|
-
}
|
|
1053
|
-
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
1054
|
-
const hasMore = merged.length > offset + limit;
|
|
1055
|
-
const page = merged.slice(offset, offset + limit);
|
|
1056
|
-
const last = page[page.length - 1];
|
|
1057
|
-
let cursor;
|
|
1058
|
-
if (hasMore && last !== void 0) {
|
|
1059
|
-
cursor = encodePageCursor({
|
|
1060
|
-
anchor: last.id,
|
|
1061
|
-
score: null,
|
|
1062
|
-
sortKey: last.key,
|
|
1063
|
-
sortSignature,
|
|
1064
|
-
binding: context.cursorBinding
|
|
1065
|
-
});
|
|
1066
|
-
}
|
|
1067
|
-
return {
|
|
1068
|
-
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
1069
|
-
count,
|
|
1070
|
-
cursor,
|
|
1071
|
-
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
1072
|
-
};
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
1003
|
// src/search/fusion.ts
|
|
1076
1004
|
function rankConstantOf(k) {
|
|
1077
1005
|
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
@@ -1104,30 +1032,27 @@ function resolveHybridFusion(hybrid) {
|
|
|
1104
1032
|
}
|
|
1105
1033
|
function reciprocalRankFusion(lists, options) {
|
|
1106
1034
|
const k = options.k;
|
|
1107
|
-
const
|
|
1108
|
-
const
|
|
1035
|
+
const positions = /* @__PURE__ */ new Map();
|
|
1036
|
+
const result = [];
|
|
1109
1037
|
for (const list of lists) {
|
|
1110
1038
|
for (let rank = 0; rank < list.length; rank++) {
|
|
1111
1039
|
const doc = list[rank];
|
|
1112
1040
|
const rrfContribution = 1 / (k + rank + 1);
|
|
1113
|
-
|
|
1114
|
-
if (
|
|
1115
|
-
|
|
1041
|
+
const position = positions.get(doc.docId);
|
|
1042
|
+
if (position === void 0) {
|
|
1043
|
+
positions.set(doc.docId, result.length);
|
|
1044
|
+
result.push({
|
|
1045
|
+
docId: doc.docId,
|
|
1046
|
+
score: rrfContribution,
|
|
1047
|
+
termFrequencies: doc.termFrequencies,
|
|
1048
|
+
fieldLengths: doc.fieldLengths,
|
|
1049
|
+
idf: doc.idf
|
|
1050
|
+
});
|
|
1051
|
+
} else {
|
|
1052
|
+
result[position].score += rrfContribution;
|
|
1116
1053
|
}
|
|
1117
1054
|
}
|
|
1118
1055
|
}
|
|
1119
|
-
const result = [];
|
|
1120
|
-
for (const [docId, score] of scores) {
|
|
1121
|
-
const original = docData.get(docId);
|
|
1122
|
-
if (!original) continue;
|
|
1123
|
-
result.push({
|
|
1124
|
-
docId,
|
|
1125
|
-
score,
|
|
1126
|
-
termFrequencies: original.termFrequencies,
|
|
1127
|
-
fieldLengths: original.fieldLengths,
|
|
1128
|
-
idf: original.idf
|
|
1129
|
-
});
|
|
1130
|
-
}
|
|
1131
1056
|
result.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId));
|
|
1132
1057
|
return result;
|
|
1133
1058
|
}
|
|
@@ -1254,12 +1179,13 @@ function filteredSearchOf(params, context) {
|
|
|
1254
1179
|
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
1255
1180
|
return { first: collectAgain(), collectAgain };
|
|
1256
1181
|
}
|
|
1257
|
-
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered) {
|
|
1182
|
+
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow) {
|
|
1258
1183
|
const options = {
|
|
1259
1184
|
metric: vectorConfig.metric ?? "cosine",
|
|
1260
1185
|
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1261
1186
|
efSearch: vectorConfig.efSearch,
|
|
1262
|
-
oversample: oversampleOf(vectorConfig)
|
|
1187
|
+
oversample: oversampleOf(vectorConfig),
|
|
1188
|
+
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
1263
1189
|
};
|
|
1264
1190
|
if (filtered !== void 0) {
|
|
1265
1191
|
return searchCurrentSubsets(
|
|
@@ -1291,7 +1217,7 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
1291
1217
|
}
|
|
1292
1218
|
const queryVec = new Float32Array(vectorConfig.value);
|
|
1293
1219
|
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1294
|
-
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered);
|
|
1220
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered, context.rescoreWindow);
|
|
1295
1221
|
const scored = vectorResultsToScored(outcome.results);
|
|
1296
1222
|
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1297
1223
|
}
|
|
@@ -1317,10 +1243,10 @@ async function textLeg(textOnlyParams, context) {
|
|
|
1317
1243
|
searchOptionsFor(manager, context.patternWork)
|
|
1318
1244
|
);
|
|
1319
1245
|
}
|
|
1320
|
-
async function vectorLeg(vectorConfig, queryVector, context, filtered, k) {
|
|
1246
|
+
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
1321
1247
|
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1322
1248
|
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1323
|
-
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered);
|
|
1249
|
+
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow);
|
|
1324
1250
|
return {
|
|
1325
1251
|
scored: vectorResultsToScored(outcome.results),
|
|
1326
1252
|
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
@@ -1365,6 +1291,107 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
1365
1291
|
};
|
|
1366
1292
|
}
|
|
1367
1293
|
|
|
1294
|
+
// src/engine/query/runtime-host.ts
|
|
1295
|
+
var runtimeHost = {
|
|
1296
|
+
now,
|
|
1297
|
+
compareCodePoints,
|
|
1298
|
+
clampRowCount,
|
|
1299
|
+
resolveProjection,
|
|
1300
|
+
projectionKeepsField,
|
|
1301
|
+
decodePageCursor,
|
|
1302
|
+
encodePageCursor,
|
|
1303
|
+
requireMatchingCursor,
|
|
1304
|
+
dedupePinnedEntries,
|
|
1305
|
+
placePinned,
|
|
1306
|
+
normalizeSort,
|
|
1307
|
+
flattenSchema,
|
|
1308
|
+
parsedTypeOf,
|
|
1309
|
+
isStringValuedBase,
|
|
1310
|
+
applyHighlights,
|
|
1311
|
+
textLeg,
|
|
1312
|
+
vectorLeg,
|
|
1313
|
+
filteredSearchOf,
|
|
1314
|
+
filterSubsetsMatchNothing,
|
|
1315
|
+
reciprocalRankFusion,
|
|
1316
|
+
linearCombination,
|
|
1317
|
+
resolveHybridFusion
|
|
1318
|
+
};
|
|
1319
|
+
|
|
1320
|
+
// src/engine/query/sorted.ts
|
|
1321
|
+
function sortsWithoutScores(params) {
|
|
1322
|
+
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
1323
|
+
}
|
|
1324
|
+
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1325
|
+
const { manager, language, config } = context;
|
|
1326
|
+
const schema = config.schema;
|
|
1327
|
+
let anchorKey = null;
|
|
1328
|
+
let anchorId = null;
|
|
1329
|
+
if (params.searchAfter !== void 0) {
|
|
1330
|
+
const decoded = decodePageCursor(params.searchAfter);
|
|
1331
|
+
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
1332
|
+
anchorKey = decoded.sortKey;
|
|
1333
|
+
anchorId = decoded.anchor;
|
|
1334
|
+
}
|
|
1335
|
+
const normalized = normalizeSort(params.sort);
|
|
1336
|
+
const fields = normalized.map((entry) => entry.field);
|
|
1337
|
+
const directions = normalized.map((entry) => entry.direction);
|
|
1338
|
+
const flatSchema = flattenSchema(schema);
|
|
1339
|
+
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
1340
|
+
const modes = sortModesOf(normalized);
|
|
1341
|
+
const options = searchOptionsFor(manager, context.patternWork);
|
|
1342
|
+
const partitions = partitionsFor(manager, context.partitionIds);
|
|
1343
|
+
const partitionLimit = offset + limit + 1;
|
|
1344
|
+
const merged = [];
|
|
1345
|
+
const partitionFacets = [];
|
|
1346
|
+
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
1347
|
+
let count = 0;
|
|
1348
|
+
for (const partition of partitions) {
|
|
1349
|
+
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1350
|
+
if (partitionFacetConfig !== void 0) {
|
|
1351
|
+
partitionFacets.push(
|
|
1352
|
+
partition.computeFacets(
|
|
1353
|
+
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
1354
|
+
partitionFacetConfig,
|
|
1355
|
+
schema
|
|
1356
|
+
)
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
if (matches === null || matches.count === 0) continue;
|
|
1360
|
+
count += matches.count;
|
|
1361
|
+
const page2 = partition.sortedPage({
|
|
1362
|
+
fields,
|
|
1363
|
+
directions,
|
|
1364
|
+
fieldTypes,
|
|
1365
|
+
modes,
|
|
1366
|
+
limit: partitionLimit,
|
|
1367
|
+
anchorKey,
|
|
1368
|
+
anchorId,
|
|
1369
|
+
matches
|
|
1370
|
+
});
|
|
1371
|
+
for (const entry of page2) merged.push(entry);
|
|
1372
|
+
}
|
|
1373
|
+
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
1374
|
+
const hasMore = merged.length > offset + limit;
|
|
1375
|
+
const page = merged.slice(offset, offset + limit);
|
|
1376
|
+
const last = page[page.length - 1];
|
|
1377
|
+
let cursor;
|
|
1378
|
+
if (hasMore && last !== void 0) {
|
|
1379
|
+
cursor = encodePageCursor({
|
|
1380
|
+
anchor: last.id,
|
|
1381
|
+
score: null,
|
|
1382
|
+
sortKey: last.key,
|
|
1383
|
+
sortSignature,
|
|
1384
|
+
binding: context.cursorBinding
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
return {
|
|
1388
|
+
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
1389
|
+
count,
|
|
1390
|
+
cursor,
|
|
1391
|
+
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
1392
|
+
};
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1368
1395
|
// src/engine/query/index.ts
|
|
1369
1396
|
function requireCursorDepth(cursor, signature, binding) {
|
|
1370
1397
|
if (cursor === void 0) return 0;
|
|
@@ -1372,7 +1399,19 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1372
1399
|
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
1373
1400
|
return decoded.depth ?? 0;
|
|
1374
1401
|
}
|
|
1375
|
-
|
|
1402
|
+
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1403
|
+
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1404
|
+
return import('./runtime-ZKYZFJJC.mjs').then(
|
|
1405
|
+
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
function executeQuery(request, callerContext) {
|
|
1409
|
+
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
1410
|
+
return import('./runtime-ZKYZFJJC.mjs').then(
|
|
1411
|
+
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1412
|
+
);
|
|
1413
|
+
}
|
|
1414
|
+
async function runQuery(request, callerContext) {
|
|
1376
1415
|
const context = withPatternWork(callerContext);
|
|
1377
1416
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
1378
1417
|
const startTime = now();
|
|
@@ -1417,7 +1456,7 @@ async function executeQuery(request, callerContext) {
|
|
|
1417
1456
|
fanOutResult = await executeVectorSearch(params, context, limit, offset, depth);
|
|
1418
1457
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1419
1458
|
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1420
|
-
fanOutResult = await
|
|
1459
|
+
fanOutResult = await hybridSearchOf(params, context, limit, offset, depth);
|
|
1421
1460
|
} else {
|
|
1422
1461
|
const scoring = scoringConfigFor(params, context);
|
|
1423
1462
|
const workerResult = workerSearch ? await workerSearch(
|
|
@@ -1503,7 +1542,7 @@ async function executeQuery(request, callerContext) {
|
|
|
1503
1542
|
};
|
|
1504
1543
|
}
|
|
1505
1544
|
const pinnedIds = params.pinned !== void 0 && params.searchAfter === void 0 ? new Set(params.pinned.map((entry) => entry.docId)) : void 0;
|
|
1506
|
-
const paged = applyPagination(
|
|
1545
|
+
const paged = (fanOutResult.paginate ?? applyPagination)(
|
|
1507
1546
|
hits,
|
|
1508
1547
|
limit,
|
|
1509
1548
|
offset,
|
|
@@ -1581,7 +1620,7 @@ async function executePreflight(request, callerContext) {
|
|
|
1581
1620
|
totalMatched = result.totalMatched;
|
|
1582
1621
|
countExact = result.matchedExact !== false;
|
|
1583
1622
|
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1584
|
-
const result = await
|
|
1623
|
+
const result = await hybridSearchOf(params, context, preflightLimit, preflightOffset);
|
|
1585
1624
|
totalMatched = result.totalMatched;
|
|
1586
1625
|
countExact = result.matchedExact !== false;
|
|
1587
1626
|
} else if (countsWithoutScores(params)) {
|
|
@@ -1663,4 +1702,4 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
1663
1702
|
return best;
|
|
1664
1703
|
}
|
|
1665
1704
|
|
|
1666
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
1705
|
+
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
3
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
6
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-COVCQ73F.mjs';
|
|
2
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-ZG477IPV.mjs';
|
|
3
|
+
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-WCJKUGHB.mjs';
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-LH4NZNJJ.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-VWULPWRJ.mjs';
|
|
6
|
+
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
8
8
|
|
|
9
9
|
// src/persistence/archive/memory-store.ts
|
|
10
10
|
function joined(chunks) {
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { rerankQueryTextOf } from './chunk-KZ3NX2P7.mjs';
|
|
2
|
+
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-FMWRCP2C.mjs';
|
|
3
|
+
import { DEFAULT_RERANK_WINDOW, MIN_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
5
|
+
import { RESULT_WINDOW } from './chunk-E74YLAIG.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
7
|
+
|
|
8
|
+
// src/rerank/params.ts
|
|
9
|
+
function invalid(message, details = {}) {
|
|
10
|
+
return new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, message, details);
|
|
11
|
+
}
|
|
12
|
+
function requireTextFields(fields, schema, host) {
|
|
13
|
+
if (!Array.isArray(fields) || fields.length === 0) {
|
|
14
|
+
throw invalid("rerank.fields must list at least one text field");
|
|
15
|
+
}
|
|
16
|
+
const flat = host.flattenSchema(schema);
|
|
17
|
+
const checked = [];
|
|
18
|
+
const seen = /* @__PURE__ */ new Set();
|
|
19
|
+
for (const field of fields) {
|
|
20
|
+
if (typeof field !== "string") {
|
|
21
|
+
throw invalid("rerank.fields accepts field paths written as strings", { field: String(field) });
|
|
22
|
+
}
|
|
23
|
+
const parsed = host.parsedTypeOf(Object.hasOwn(flat, field) ? flat[field] : void 0);
|
|
24
|
+
if (parsed === void 0 || !host.isStringValuedBase(parsed.base)) {
|
|
25
|
+
throw invalid(`rerank.fields gives "${field}", which the schema does not declare as a text field`, { field });
|
|
26
|
+
}
|
|
27
|
+
if (seen.has(field)) {
|
|
28
|
+
throw invalid(`rerank.fields gives "${field}" more than once`, { field });
|
|
29
|
+
}
|
|
30
|
+
seen.add(field);
|
|
31
|
+
checked.push(field);
|
|
32
|
+
}
|
|
33
|
+
return checked;
|
|
34
|
+
}
|
|
35
|
+
function requireWindow(window, maxWindow, hybridDepth) {
|
|
36
|
+
if (window === void 0) return Math.min(DEFAULT_RERANK_WINDOW, maxWindow, hybridDepth ?? maxWindow);
|
|
37
|
+
if (typeof window !== "number" || !Number.isInteger(window) || window < MIN_RERANK_WINDOW || window > maxWindow) {
|
|
38
|
+
throw invalid(`rerank.window must be a whole number from ${MIN_RERANK_WINDOW} to ${maxWindow}`, {
|
|
39
|
+
window: typeof window === "number" ? window : String(window),
|
|
40
|
+
maxRerankWindow: maxWindow
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
if (hybridDepth !== void 0 && window > hybridDepth) {
|
|
44
|
+
throw invalid(
|
|
45
|
+
`rerank.window is ${window}, above the hybrid.depth of ${hybridDepth} that a hybrid query reranks from`,
|
|
46
|
+
{
|
|
47
|
+
window,
|
|
48
|
+
depth: hybridDepth
|
|
49
|
+
}
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
return window;
|
|
53
|
+
}
|
|
54
|
+
function resolveRerankParams(params, schema, maxWindow, hybridDepth, host) {
|
|
55
|
+
const rerank = params.rerank;
|
|
56
|
+
if (typeof rerank !== "object" || rerank === null || Array.isArray(rerank)) {
|
|
57
|
+
throw invalid("rerank must be an object that gives an adapter and the fields to rerank by");
|
|
58
|
+
}
|
|
59
|
+
if (host.normalizeSort(params.sort).length > 0) {
|
|
60
|
+
throw new NarsilError(
|
|
61
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
62
|
+
"A query cannot carry both rerank and a sort, because each of them sets the order of the hits"
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
const { adapter, fields, window } = rerank;
|
|
66
|
+
if (typeof adapter !== "string" || adapter.length === 0) {
|
|
67
|
+
throw invalid("rerank.adapter must give the name of a registered rerank adapter");
|
|
68
|
+
}
|
|
69
|
+
const checkedFields = requireTextFields(fields, schema, host);
|
|
70
|
+
const checkedWindow = requireWindow(window, maxWindow, hybridDepth);
|
|
71
|
+
const query = rerankQueryTextOf(params);
|
|
72
|
+
if (query === void 0) {
|
|
73
|
+
throw invalid("A reranked query needs text to compare each hit with: set rerank.query, term, or vector.text");
|
|
74
|
+
}
|
|
75
|
+
return { adapter, fields: checkedFields, query, window: checkedWindow };
|
|
76
|
+
}
|
|
77
|
+
function rerankTextOf(values) {
|
|
78
|
+
if (values === void 0) return "";
|
|
79
|
+
const lines = [];
|
|
80
|
+
for (const value of values) {
|
|
81
|
+
if (typeof value === "string") {
|
|
82
|
+
lines.push(value);
|
|
83
|
+
} else if (Array.isArray(value)) {
|
|
84
|
+
for (const member of value) if (typeof member === "string") lines.push(member);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return lines.join("\n");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/rerank/pipeline.ts
|
|
91
|
+
function rerankCursorDepth(cursor, binding, host) {
|
|
92
|
+
const decoded = host.decodePageCursor(cursor);
|
|
93
|
+
host.requireMatchingCursor(decoded, cursor, null, true, binding);
|
|
94
|
+
return decoded.depth ?? 0;
|
|
95
|
+
}
|
|
96
|
+
function requireWithinReach(cursorDepth, offset, limit) {
|
|
97
|
+
const reach = cursorDepth + offset + limit;
|
|
98
|
+
if (reach <= RESULT_WINDOW) return;
|
|
99
|
+
throw new NarsilError(
|
|
100
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
101
|
+
`The engine pages a reranked query through the first ${RESULT_WINDOW} results, and this page ends at ${reach}`,
|
|
102
|
+
{ limit, offset, cursorDepth, window: RESULT_WINDOW }
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
function reorderByRerank(ranked, windowSize, scores, compareCodePoints) {
|
|
106
|
+
let initialLow = Number.POSITIVE_INFINITY;
|
|
107
|
+
let initialHigh = Number.NEGATIVE_INFINITY;
|
|
108
|
+
let finalLow = Number.POSITIVE_INFINITY;
|
|
109
|
+
let finalHigh = Number.NEGATIVE_INFINITY;
|
|
110
|
+
const ordered = new Array(windowSize);
|
|
111
|
+
for (let i = 0; i < windowSize; i++) {
|
|
112
|
+
const first = ranked[i].score ?? 0;
|
|
113
|
+
const score = scores[i];
|
|
114
|
+
if (first < initialLow) initialLow = first;
|
|
115
|
+
if (first > initialHigh) initialHigh = first;
|
|
116
|
+
if (score < finalLow) finalLow = score;
|
|
117
|
+
if (score > finalHigh) finalHigh = score;
|
|
118
|
+
ordered[i] = { ...ranked[i], score };
|
|
119
|
+
}
|
|
120
|
+
ordered.sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || compareCodePoints(a.id, b.id));
|
|
121
|
+
const rescale = initialLow > finalLow;
|
|
122
|
+
const scale = rescale ? Math.max(finalHigh - finalLow, 1) / Math.max(initialHigh - initialLow, 1) : 1;
|
|
123
|
+
const bias = rescale ? finalLow - initialLow * scale : 0;
|
|
124
|
+
for (let i = windowSize; i < ranked.length; i++) {
|
|
125
|
+
const hit = ranked[i];
|
|
126
|
+
ordered.push(rescale ? { ...hit, score: (hit.score ?? 0) * scale + bias } : hit);
|
|
127
|
+
}
|
|
128
|
+
return ordered;
|
|
129
|
+
}
|
|
130
|
+
function firstStageDepth(hybridDepth, window, cursorDepth, offset, limit) {
|
|
131
|
+
if (hybridDepth === void 0) {
|
|
132
|
+
requireWithinReach(cursorDepth, offset, limit);
|
|
133
|
+
return Math.max(window + 1, Math.min(cursorDepth + offset + limit + 1, RESULT_WINDOW));
|
|
134
|
+
}
|
|
135
|
+
requireWithinHybridDepth(cursorDepth, offset, hybridDepth);
|
|
136
|
+
return hybridDepth;
|
|
137
|
+
}
|
|
138
|
+
async function runRerankedQuery(request, io) {
|
|
139
|
+
const fuses = io.fusesRankings(request);
|
|
140
|
+
const hybridDepth = fuses ? fixedHybridDepthOf(request.hybrid) : void 0;
|
|
141
|
+
const { host } = io;
|
|
142
|
+
const plan = resolveRerankParams(request, io.schema, io.maxWindow, hybridDepth, host);
|
|
143
|
+
const limit = host.clampRowCount(request.limit, DEFAULT_LIMIT);
|
|
144
|
+
const offset = host.clampRowCount(request.offset, DEFAULT_OFFSET);
|
|
145
|
+
const cursorDepth = request.searchAfter === void 0 ? 0 : rerankCursorDepth(request.searchAfter, io.binding, host);
|
|
146
|
+
const depth = firstStageDepth(hybridDepth, plan.window, cursorDepth, offset, limit);
|
|
147
|
+
const projection = host.resolveProjection(request.document);
|
|
148
|
+
const { rerank: _rerank, pinned: _pinned, searchAfter: _searchAfter, highlight: _highlight, ...firstParams } = request;
|
|
149
|
+
const placesPins = request.searchAfter === void 0 && request.pinned !== void 0;
|
|
150
|
+
const countsApart = placesPins || request.vector !== void 0 && !fuses;
|
|
151
|
+
const [first, plain] = await Promise.all([
|
|
152
|
+
io.firstStage({ ...firstParams, offset: 0, limit: depth, document: false }),
|
|
153
|
+
countsApart ? io.firstStage({
|
|
154
|
+
...firstParams,
|
|
155
|
+
...placesPins ? { pinned: request.pinned } : {},
|
|
156
|
+
facets: void 0,
|
|
157
|
+
group: void 0,
|
|
158
|
+
offset: cursorDepth + offset,
|
|
159
|
+
limit,
|
|
160
|
+
document: false
|
|
161
|
+
}) : void 0
|
|
162
|
+
]);
|
|
163
|
+
const ranked = first.hits;
|
|
164
|
+
let pins = placesPins ? request.pinned : void 0;
|
|
165
|
+
if (pins !== void 0) {
|
|
166
|
+
const stored = await io.storedPins(host.dedupePinnedEntries(pins).map((entry) => entry.docId));
|
|
167
|
+
const clamps = io.clampsPins || first.countExact && ranked.length >= first.count;
|
|
168
|
+
pins = pins.filter((entry) => stored.has(entry.docId) && (clamps || entry.position < depth));
|
|
169
|
+
}
|
|
170
|
+
const placedPins = pins;
|
|
171
|
+
const place = (hits) => placedPins === void 0 ? hits : host.placePinned(hits, placedPins, (docId) => ({ id: docId, score: 0, document: void 0 })).hits;
|
|
172
|
+
const start = cursorDepth + offset;
|
|
173
|
+
const windowSize = Math.min(plan.window, ranked.length);
|
|
174
|
+
const pageHoldsHits = limit > 0 && start < place(ranked).length;
|
|
175
|
+
let ordered = ranked;
|
|
176
|
+
if (windowSize > 0 && pageHoldsHits) {
|
|
177
|
+
const texts = await io.texts(
|
|
178
|
+
ranked.slice(0, windowSize).map((hit) => hit.id),
|
|
179
|
+
plan.fields
|
|
180
|
+
);
|
|
181
|
+
ordered = reorderByRerank(
|
|
182
|
+
ranked,
|
|
183
|
+
windowSize,
|
|
184
|
+
await io.score(plan.adapter, plan.query, texts),
|
|
185
|
+
host.compareCodePoints
|
|
186
|
+
);
|
|
187
|
+
} else {
|
|
188
|
+
await io.score(plan.adapter, plan.query, []);
|
|
189
|
+
}
|
|
190
|
+
const pinnedIds = placedPins === void 0 ? void 0 : new Set(placedPins.map((entry) => entry.docId));
|
|
191
|
+
const { page, cursor } = pageByPosition(place(ordered), ordered, {
|
|
192
|
+
start,
|
|
193
|
+
limit,
|
|
194
|
+
pinnedIds,
|
|
195
|
+
binding: io.binding,
|
|
196
|
+
encodeCursor: host.encodePageCursor
|
|
197
|
+
});
|
|
198
|
+
await io.fillDocuments(page, first.groups, projection);
|
|
199
|
+
if (request.highlight) io.highlight?.(page, projection);
|
|
200
|
+
return {
|
|
201
|
+
hits: page,
|
|
202
|
+
count: (plain ?? first).count,
|
|
203
|
+
countExact: (plain ?? first).countExact,
|
|
204
|
+
elapsed: 0,
|
|
205
|
+
cursor,
|
|
206
|
+
facets: first.facets,
|
|
207
|
+
groups: first.groups,
|
|
208
|
+
coverage: first.coverage
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export { rerankTextOf, runRerankedQuery };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/schema/validator/field-type.ts
|
|
5
5
|
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|