@delali/narsil 0.2.3 → 0.2.4-next.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{chunk-MMSWEEZT.mjs → chunk-KEDL6FFZ.mjs} +315 -51
- package/dist/{chunk-WU2UZEVD.mjs → chunk-O4Q5JYNK.mjs} +1 -1
- package/dist/{chunk-GB2IIJJG.mjs → chunk-TLRUUMVE.mjs} +1 -4
- package/dist/{chunk-GRWFY2HI.mjs → chunk-YGQMKTKG.mjs} +120 -36
- package/dist/{client-BobVnoIK.d.ts → client-B5uO5jL1.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -4
- package/dist/{config-Byyl1QqH.d.ts → config-B09-cud9.d.ts} +2 -2
- package/dist/{config-fLC4lbmg.d.ts → config-NgUCkc6a.d.ts} +2 -2
- package/dist/{direct-executor-MSJEVPZU.mjs → direct-executor-7O23GEHE.mjs} +2 -2
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +705 -272
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-BE45D5ZG.d.ts → engine-CjLhdDIm.d.ts} +21 -20
- package/dist/index.browser.mjs +436 -91
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/{options-EXYFcem-.d.ts → options-BakECW-r.d.ts} +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +4 -4
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -4
- package/dist/{schema-ZRbk8BgH.d.ts → schema-DXOd96kx.d.ts} +16 -5
- package/dist/{search-DHuXwJYk.d.ts → search-BGouJxH4.d.ts} +22 -9
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +6 -9
- package/dist/{tasks-CnMLpfk-.d.ts → tasks-Bn4R1NPN.d.ts} +1 -1
- package/dist/{types-B5Tsaalz.d.ts → types-BTE33VZS.d.ts} +1 -1
- package/dist/workers/entry.mjs +2 -2
- package/package.json +2 -2
package/dist/distribution.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
1
|
+
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, listBindingOf, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, queryBindingOf, foldGroupReducer, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, dedupePinnedEntries, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-KEDL6FFZ.mjs';
|
|
2
2
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-JQUL2T6I.mjs';
|
|
3
|
-
import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH } from './chunk-
|
|
4
|
-
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-
|
|
3
|
+
import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH } from './chunk-O4Q5JYNK.mjs';
|
|
4
|
+
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, RESULT_WINDOW, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-YGQMKTKG.mjs';
|
|
5
5
|
import './chunk-TGP5M5R7.mjs';
|
|
6
6
|
import './chunk-OBXS7OZM.mjs';
|
|
7
7
|
import './chunk-WDL632CN.mjs';
|
|
@@ -9,7 +9,7 @@ import { crc32 } from './chunk-KMMJMDUA.mjs';
|
|
|
9
9
|
import './chunk-U2BKYEC7.mjs';
|
|
10
10
|
import { compareCodePoints, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-BFIH7OV5.mjs';
|
|
11
11
|
import { generateId } from './chunk-WTONLDU3.mjs';
|
|
12
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
12
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-TLRUUMVE.mjs';
|
|
13
13
|
import { encode, decode } from '@msgpack/msgpack';
|
|
14
14
|
|
|
15
15
|
// src/distribution/cluster/controller/types.ts
|
|
@@ -5748,6 +5748,11 @@ var MAX_LIMIT = 1e4;
|
|
|
5748
5748
|
var MAX_OFFSET = 1e4;
|
|
5749
5749
|
var MAX_HYBRID_K = 1e4;
|
|
5750
5750
|
var MIN_HYBRID_K = 1;
|
|
5751
|
+
var MAX_PREFIX_LENGTH = 1024;
|
|
5752
|
+
var MAX_PINNED_ENTRIES = 1e3;
|
|
5753
|
+
var MAX_PINNED_POSITION = RESULT_WINDOW;
|
|
5754
|
+
var MAX_EF_SEARCH = 1e4;
|
|
5755
|
+
var MAX_GROUP_FIELDS = 64;
|
|
5751
5756
|
var NULL_BYTE = String.fromCharCode(0);
|
|
5752
5757
|
function throwInvalid(code, message, details) {
|
|
5753
5758
|
throw new NarsilError(code, message, details);
|
|
@@ -6224,13 +6229,153 @@ function validateFilterExpression(value, fieldLabel, errorCode = SEARCH_INVALID_
|
|
|
6224
6229
|
}
|
|
6225
6230
|
}
|
|
6226
6231
|
|
|
6232
|
+
// src/distribution/query/oversample.ts
|
|
6233
|
+
function oversampledShardSize(size) {
|
|
6234
|
+
return Math.ceil(size * 1.5) + 10;
|
|
6235
|
+
}
|
|
6236
|
+
|
|
6237
|
+
// src/distribution/query/validators/search-options.ts
|
|
6238
|
+
var ALLOWED_MODES = ["fulltext", "vector", "hybrid"];
|
|
6239
|
+
var ALLOWED_METRICS = ["cosine", "dotProduct", "euclidean"];
|
|
6240
|
+
var ALLOWED_HYBRID_STRATEGIES = ["rrf", "linear"];
|
|
6241
|
+
function validateHybridParams(value) {
|
|
6242
|
+
if (!isRecord2(value)) {
|
|
6243
|
+
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
6244
|
+
}
|
|
6245
|
+
if (value.strategy !== null && !ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
6246
|
+
throwInvalid(
|
|
6247
|
+
SEARCH_INVALID_MODE,
|
|
6248
|
+
`Invalid SearchPayload: "params.hybrid.strategy" must be one of: ${ALLOWED_HYBRID_STRATEGIES.join(", ")}, or null`
|
|
6249
|
+
);
|
|
6250
|
+
}
|
|
6251
|
+
if (value.k !== null && (!isInteger(value.k) || value.k < MIN_HYBRID_K || value.k > MAX_HYBRID_K)) {
|
|
6252
|
+
throwInvalid(
|
|
6253
|
+
CONFIG_INVALID,
|
|
6254
|
+
`Invalid SearchPayload: "params.hybrid.k" must be an integer between ${MIN_HYBRID_K} and ${MAX_HYBRID_K}, or null`
|
|
6255
|
+
);
|
|
6256
|
+
}
|
|
6257
|
+
if (value.alpha !== null && (!isFiniteNumber(value.alpha) || value.alpha < 0 || value.alpha > 1)) {
|
|
6258
|
+
throwInvalid(
|
|
6259
|
+
CONFIG_INVALID,
|
|
6260
|
+
'Invalid SearchPayload: "params.hybrid.alpha" must be a finite number in [0, 1], or null'
|
|
6261
|
+
);
|
|
6262
|
+
}
|
|
6263
|
+
}
|
|
6264
|
+
function validateGroupParams(value) {
|
|
6265
|
+
if (!isRecord2(value)) {
|
|
6266
|
+
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
|
|
6267
|
+
}
|
|
6268
|
+
const fields = validateStringArray(
|
|
6269
|
+
value.fields,
|
|
6270
|
+
"params.group.fields",
|
|
6271
|
+
MAX_GROUP_FIELDS,
|
|
6272
|
+
MAX_FIELD_NAME_LENGTH,
|
|
6273
|
+
SEARCH_INVALID_FIELD
|
|
6274
|
+
);
|
|
6275
|
+
if (fields.length === 0) {
|
|
6276
|
+
throwInvalid(SEARCH_INVALID_FIELD, 'Invalid SearchPayload: "params.group.fields" must name at least one field');
|
|
6277
|
+
}
|
|
6278
|
+
if (!isInteger(value.maxPerGroup) || value.maxPerGroup <= 0 || value.maxPerGroup > MAX_LIMIT) {
|
|
6279
|
+
throwInvalid(
|
|
6280
|
+
CONFIG_INVALID,
|
|
6281
|
+
`Invalid SearchPayload: "params.group.maxPerGroup" must be a positive integer at most ${MAX_LIMIT}`
|
|
6282
|
+
);
|
|
6283
|
+
}
|
|
6284
|
+
if (value.limit !== null && (!isInteger(value.limit) || value.limit < 0 || value.limit > MAX_LIMIT)) {
|
|
6285
|
+
throwInvalid(
|
|
6286
|
+
CONFIG_INVALID,
|
|
6287
|
+
`Invalid SearchPayload: "params.group.limit" must be a non-negative integer at most ${MAX_LIMIT}, or null`
|
|
6288
|
+
);
|
|
6289
|
+
}
|
|
6290
|
+
}
|
|
6291
|
+
function validateTermMatchParam(value) {
|
|
6292
|
+
if (value === "all" || value === "any") {
|
|
6293
|
+
return;
|
|
6294
|
+
}
|
|
6295
|
+
if (typeof value === "string") {
|
|
6296
|
+
throwInvalid(SEARCH_INVALID_MODE, 'Invalid SearchPayload: "params.termMatch" must be "all", "any", or a count');
|
|
6297
|
+
}
|
|
6298
|
+
if (!isInteger(value) || value < 1 || value > MAX_TERMS_COUNT) {
|
|
6299
|
+
throwInvalid(
|
|
6300
|
+
CONFIG_INVALID,
|
|
6301
|
+
`Invalid SearchPayload: a "params.termMatch" count must be an integer between 1 and ${MAX_TERMS_COUNT}`
|
|
6302
|
+
);
|
|
6303
|
+
}
|
|
6304
|
+
}
|
|
6305
|
+
function validatePrefixLengthParam(value) {
|
|
6306
|
+
if (!isInteger(value) || value < 0 || value > MAX_PREFIX_LENGTH) {
|
|
6307
|
+
throwInvalid(
|
|
6308
|
+
CONFIG_INVALID,
|
|
6309
|
+
`Invalid SearchPayload: "params.prefixLength" must be a non-negative integer at most ${MAX_PREFIX_LENGTH}`
|
|
6310
|
+
);
|
|
6311
|
+
}
|
|
6312
|
+
}
|
|
6313
|
+
function validateBooleanParam(value, fieldLabel) {
|
|
6314
|
+
if (typeof value !== "boolean") {
|
|
6315
|
+
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.${fieldLabel}" must be a boolean or null`);
|
|
6316
|
+
}
|
|
6317
|
+
}
|
|
6318
|
+
function validatePinnedParam(value) {
|
|
6319
|
+
if (!Array.isArray(value)) {
|
|
6320
|
+
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.pinned" must be an array or null');
|
|
6321
|
+
}
|
|
6322
|
+
if (value.length > MAX_PINNED_ENTRIES) {
|
|
6323
|
+
throwInvalid(
|
|
6324
|
+
CONFIG_INVALID,
|
|
6325
|
+
`Invalid SearchPayload: "params.pinned" exceeds maximum length of ${MAX_PINNED_ENTRIES}`,
|
|
6326
|
+
{ length: value.length, limit: MAX_PINNED_ENTRIES }
|
|
6327
|
+
);
|
|
6328
|
+
}
|
|
6329
|
+
for (let i = 0; i < value.length; i++) {
|
|
6330
|
+
const entry = value[i];
|
|
6331
|
+
if (!isRecord2(entry)) {
|
|
6332
|
+
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.pinned[${i}]" must be an object`);
|
|
6333
|
+
}
|
|
6334
|
+
if (typeof entry.docId !== "string" || entry.docId.length === 0 || entry.docId.length > MAX_DOC_ID_LENGTH) {
|
|
6335
|
+
throwInvalid(
|
|
6336
|
+
CONFIG_INVALID,
|
|
6337
|
+
`Invalid SearchPayload: "params.pinned[${i}].docId" must be a string of 1 to ${MAX_DOC_ID_LENGTH} characters`
|
|
6338
|
+
);
|
|
6339
|
+
}
|
|
6340
|
+
if (!isInteger(entry.position) || entry.position < 0 || entry.position > MAX_PINNED_POSITION) {
|
|
6341
|
+
throwInvalid(
|
|
6342
|
+
CONFIG_INVALID,
|
|
6343
|
+
`Invalid SearchPayload: "params.pinned[${i}].position" must be an integer between 0 and ${MAX_PINNED_POSITION}`
|
|
6344
|
+
);
|
|
6345
|
+
}
|
|
6346
|
+
}
|
|
6347
|
+
}
|
|
6348
|
+
function validateModeParam(value) {
|
|
6349
|
+
if (!ALLOWED_MODES.includes(value)) {
|
|
6350
|
+
throwInvalid(
|
|
6351
|
+
SEARCH_INVALID_MODE,
|
|
6352
|
+
`Invalid SearchPayload: "params.mode" must be one of: ${ALLOWED_MODES.join(", ")}`
|
|
6353
|
+
);
|
|
6354
|
+
}
|
|
6355
|
+
}
|
|
6356
|
+
function validateVectorMetricParam(value) {
|
|
6357
|
+
if (!ALLOWED_METRICS.includes(value)) {
|
|
6358
|
+
throwInvalid(
|
|
6359
|
+
SEARCH_INVALID_MODE,
|
|
6360
|
+
`Invalid SearchPayload: "params.vector.metric" must be one of: ${ALLOWED_METRICS.join(", ")}`
|
|
6361
|
+
);
|
|
6362
|
+
}
|
|
6363
|
+
}
|
|
6364
|
+
function validateEfSearchParam(value) {
|
|
6365
|
+
if (!isInteger(value) || value < 1 || value > MAX_EF_SEARCH) {
|
|
6366
|
+
throwInvalid(
|
|
6367
|
+
CONFIG_INVALID,
|
|
6368
|
+
`Invalid SearchPayload: "params.vector.efSearch" must be an integer between 1 and ${MAX_EF_SEARCH}`
|
|
6369
|
+
);
|
|
6370
|
+
}
|
|
6371
|
+
}
|
|
6372
|
+
|
|
6227
6373
|
// src/distribution/query/validators/search.ts
|
|
6228
|
-
var MAX_FACET_SHARD_SIZE =
|
|
6374
|
+
var MAX_FACET_SHARD_SIZE = oversampledShardSize(1e3);
|
|
6229
6375
|
var MAX_VECTOR_DIMENSION = 8192;
|
|
6230
6376
|
var MAX_VECTOR_TEXT_LENGTH = 16384;
|
|
6231
6377
|
var ALLOWED_SCORING = ["local", "dfs", "broadcast"];
|
|
6232
6378
|
var ALLOWED_SORT_DIRECTIONS = ["asc", "desc"];
|
|
6233
|
-
var ALLOWED_HYBRID_STRATEGIES = ["rrf", "linear"];
|
|
6234
6379
|
function validateIndexNameField2(value, fieldLabel) {
|
|
6235
6380
|
if (typeof value !== "string") {
|
|
6236
6381
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "${fieldLabel}" must be a string`);
|
|
@@ -6282,37 +6427,11 @@ function validateVectorParams(value) {
|
|
|
6282
6427
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector.similarity" must be a finite number or null');
|
|
6283
6428
|
}
|
|
6284
6429
|
}
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
if (!isRecord2(value)) {
|
|
6288
|
-
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
6289
|
-
}
|
|
6290
|
-
if (!ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
6291
|
-
throwInvalid(
|
|
6292
|
-
SEARCH_INVALID_MODE,
|
|
6293
|
-
`Invalid SearchPayload: "params.hybrid.strategy" must be one of: ${ALLOWED_HYBRID_STRATEGIES.join(", ")}`
|
|
6294
|
-
);
|
|
6295
|
-
}
|
|
6296
|
-
if (!isInteger(value.k) || value.k < MIN_HYBRID_K || value.k > MAX_HYBRID_K) {
|
|
6297
|
-
throwInvalid(
|
|
6298
|
-
CONFIG_INVALID,
|
|
6299
|
-
`Invalid SearchPayload: "params.hybrid.k" must be an integer between ${MIN_HYBRID_K} and ${MAX_HYBRID_K}`
|
|
6300
|
-
);
|
|
6301
|
-
}
|
|
6302
|
-
if (!isFiniteNumber(value.alpha) || value.alpha < 0 || value.alpha > 1) {
|
|
6303
|
-
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid.alpha" must be a finite number in [0, 1]');
|
|
6430
|
+
if (value.metric !== null) {
|
|
6431
|
+
validateVectorMetricParam(value.metric);
|
|
6304
6432
|
}
|
|
6305
|
-
|
|
6306
|
-
|
|
6307
|
-
if (!isRecord2(value)) {
|
|
6308
|
-
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
|
|
6309
|
-
}
|
|
6310
|
-
validateFieldName(value.field, "params.group.field", SEARCH_INVALID_FIELD);
|
|
6311
|
-
if (!isInteger(value.maxPerGroup) || value.maxPerGroup <= 0 || value.maxPerGroup > MAX_LIMIT) {
|
|
6312
|
-
throwInvalid(
|
|
6313
|
-
CONFIG_INVALID,
|
|
6314
|
-
`Invalid SearchPayload: "params.group.maxPerGroup" must be a positive integer at most ${MAX_LIMIT}`
|
|
6315
|
-
);
|
|
6433
|
+
if (value.efSearch !== null) {
|
|
6434
|
+
validateEfSearchParam(value.efSearch);
|
|
6316
6435
|
}
|
|
6317
6436
|
}
|
|
6318
6437
|
function validateSortParams(value) {
|
|
@@ -6454,6 +6573,24 @@ function validateParams(params) {
|
|
|
6454
6573
|
`Invalid SearchPayload: "params.scoring" must be one of: ${ALLOWED_SCORING.join(", ")}`
|
|
6455
6574
|
);
|
|
6456
6575
|
}
|
|
6576
|
+
if (params.termMatch !== null) {
|
|
6577
|
+
validateTermMatchParam(params.termMatch);
|
|
6578
|
+
}
|
|
6579
|
+
if (params.prefixLength !== null) {
|
|
6580
|
+
validatePrefixLengthParam(params.prefixLength);
|
|
6581
|
+
}
|
|
6582
|
+
if (params.prefix !== null) {
|
|
6583
|
+
validateBooleanParam(params.prefix, "prefix");
|
|
6584
|
+
}
|
|
6585
|
+
if (params.exact !== null) {
|
|
6586
|
+
validateBooleanParam(params.exact, "exact");
|
|
6587
|
+
}
|
|
6588
|
+
if (params.pinned !== null) {
|
|
6589
|
+
validatePinnedParam(params.pinned);
|
|
6590
|
+
}
|
|
6591
|
+
if (params.mode !== null) {
|
|
6592
|
+
validateModeParam(params.mode);
|
|
6593
|
+
}
|
|
6457
6594
|
if (params.vector !== null) {
|
|
6458
6595
|
validateVectorParams(params.vector);
|
|
6459
6596
|
}
|
|
@@ -6704,6 +6841,7 @@ var MAX_SORT_VALUES = 8;
|
|
|
6704
6841
|
var MAX_FACET_FIELDS = 64;
|
|
6705
6842
|
var MAX_FACET_BUCKETS = 1e4;
|
|
6706
6843
|
var MAX_FACET_VALUE_LENGTH = 1024;
|
|
6844
|
+
var MAX_GROUPS_PER_RESPONSE = 65536;
|
|
6707
6845
|
function validateScoredEntry(value, fieldLabel) {
|
|
6708
6846
|
if (!isRecord2(value)) {
|
|
6709
6847
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
@@ -6846,6 +6984,43 @@ function validateFacetErrorBounds(value) {
|
|
|
6846
6984
|
}
|
|
6847
6985
|
}
|
|
6848
6986
|
}
|
|
6987
|
+
function validateGroupEntries(value) {
|
|
6988
|
+
if (!Array.isArray(value)) {
|
|
6989
|
+
throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "groups" must be an array or null');
|
|
6990
|
+
}
|
|
6991
|
+
if (value.length > MAX_GROUPS_PER_RESPONSE) {
|
|
6992
|
+
throwInvalid(
|
|
6993
|
+
CONFIG_INVALID,
|
|
6994
|
+
`Invalid SearchResultPayload: "groups" exceeds maximum length of ${MAX_GROUPS_PER_RESPONSE}`,
|
|
6995
|
+
{ length: value.length, limit: MAX_GROUPS_PER_RESPONSE }
|
|
6996
|
+
);
|
|
6997
|
+
}
|
|
6998
|
+
for (let i = 0; i < value.length; i++) {
|
|
6999
|
+
const entry = value[i];
|
|
7000
|
+
if (!isRecord2(entry) || !isRecord2(entry.values)) {
|
|
7001
|
+
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "groups[${i}]" must hold a values object`);
|
|
7002
|
+
}
|
|
7003
|
+
if (Object.keys(entry.values).length > MAX_FACET_FIELDS) {
|
|
7004
|
+
throwInvalid(
|
|
7005
|
+
CONFIG_INVALID,
|
|
7006
|
+
`Invalid SearchResultPayload: "groups[${i}].values" exceeds maximum field count of ${MAX_FACET_FIELDS}`
|
|
7007
|
+
);
|
|
7008
|
+
}
|
|
7009
|
+
if (!Array.isArray(entry.scored)) {
|
|
7010
|
+
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "groups[${i}].scored" must be an array`);
|
|
7011
|
+
}
|
|
7012
|
+
if (entry.scored.length > MAX_RESULTS_PER_PARTITION) {
|
|
7013
|
+
throwInvalid(
|
|
7014
|
+
CONFIG_INVALID,
|
|
7015
|
+
`Invalid SearchResultPayload: "groups[${i}].scored" exceeds maximum length of ${MAX_RESULTS_PER_PARTITION}`,
|
|
7016
|
+
{ length: entry.scored.length, limit: MAX_RESULTS_PER_PARTITION }
|
|
7017
|
+
);
|
|
7018
|
+
}
|
|
7019
|
+
for (let j = 0; j < entry.scored.length; j++) {
|
|
7020
|
+
validateScoredEntry(entry.scored[j], `groups[${i}].scored[${j}]`);
|
|
7021
|
+
}
|
|
7022
|
+
}
|
|
7023
|
+
}
|
|
6849
7024
|
function validateSearchResultPayload(decoded) {
|
|
6850
7025
|
if (!isRecord2(decoded)) {
|
|
6851
7026
|
throwInvalid(CONFIG_INVALID, "Invalid SearchResultPayload: expected an object");
|
|
@@ -6862,6 +7037,9 @@ function validateSearchResultPayload(decoded) {
|
|
|
6862
7037
|
if (decoded.facetErrorBounds !== null && decoded.facetErrorBounds !== void 0) {
|
|
6863
7038
|
validateFacetErrorBounds(decoded.facetErrorBounds);
|
|
6864
7039
|
}
|
|
7040
|
+
if (decoded.groups !== null && decoded.groups !== void 0) {
|
|
7041
|
+
validateGroupEntries(decoded.groups);
|
|
7042
|
+
}
|
|
6865
7043
|
return decoded;
|
|
6866
7044
|
}
|
|
6867
7045
|
|
|
@@ -7008,21 +7186,52 @@ function wireParamsToLocal(wire, facetShardSize) {
|
|
|
7008
7186
|
tolerance: wire.tolerance ?? void 0,
|
|
7009
7187
|
minScore: wire.threshold ?? void 0,
|
|
7010
7188
|
includeScores: wire.includeScores ?? void 0,
|
|
7189
|
+
termMatch: wire.termMatch ?? void 0,
|
|
7190
|
+
prefixLength: wire.prefixLength ?? void 0,
|
|
7191
|
+
prefix: wire.prefix ?? void 0,
|
|
7192
|
+
exact: wire.exact ?? void 0,
|
|
7193
|
+
pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
|
|
7194
|
+
mode: wire.mode ?? void 0,
|
|
7011
7195
|
limit: wire.limit,
|
|
7012
7196
|
offset: wire.offset,
|
|
7013
7197
|
searchAfter: wire.searchAfter ?? void 0,
|
|
7014
7198
|
sort: convertWireSortToLocal(wire.sort),
|
|
7015
|
-
group: wire.group
|
|
7199
|
+
group: convertWireGroupToLocal(wire.group),
|
|
7016
7200
|
facets: convertWireFacetConfigToLocal(wire.facets, facetShardSize ?? wire.facetSize),
|
|
7017
|
-
vector: wire.vector
|
|
7018
|
-
|
|
7019
|
-
value: wire.vector.value ?? void 0,
|
|
7020
|
-
text: wire.vector.text ?? void 0,
|
|
7021
|
-
similarity: wire.vector.similarity ?? void 0
|
|
7022
|
-
} : void 0,
|
|
7023
|
-
hybrid: wire.hybrid !== null ? { strategy: wire.hybrid.strategy, k: wire.hybrid.k, alpha: wire.hybrid.alpha } : void 0
|
|
7201
|
+
vector: convertWireVectorToLocal(wire.vector),
|
|
7202
|
+
hybrid: convertWireHybridToLocal(wire.hybrid)
|
|
7024
7203
|
};
|
|
7025
7204
|
}
|
|
7205
|
+
function convertWireGroupToLocal(group) {
|
|
7206
|
+
if (group === null) {
|
|
7207
|
+
return void 0;
|
|
7208
|
+
}
|
|
7209
|
+
const local = { fields: [...group.fields], maxPerGroup: group.maxPerGroup };
|
|
7210
|
+
if (group.limit !== null) local.limit = group.limit;
|
|
7211
|
+
return local;
|
|
7212
|
+
}
|
|
7213
|
+
function convertWireVectorToLocal(vector) {
|
|
7214
|
+
if (vector === null) {
|
|
7215
|
+
return void 0;
|
|
7216
|
+
}
|
|
7217
|
+
const local = { field: vector.field };
|
|
7218
|
+
if (vector.value !== null) local.value = vector.value;
|
|
7219
|
+
if (vector.text !== null) local.text = vector.text;
|
|
7220
|
+
if (vector.similarity !== null) local.similarity = vector.similarity;
|
|
7221
|
+
if (vector.metric !== null) local.metric = vector.metric;
|
|
7222
|
+
if (vector.efSearch !== null) local.efSearch = vector.efSearch;
|
|
7223
|
+
return local;
|
|
7224
|
+
}
|
|
7225
|
+
function convertWireHybridToLocal(hybrid) {
|
|
7226
|
+
if (hybrid === null) {
|
|
7227
|
+
return void 0;
|
|
7228
|
+
}
|
|
7229
|
+
const local = {};
|
|
7230
|
+
if (hybrid.strategy !== null) local.strategy = hybrid.strategy;
|
|
7231
|
+
if (hybrid.k !== null) local.k = hybrid.k;
|
|
7232
|
+
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
7233
|
+
return local;
|
|
7234
|
+
}
|
|
7026
7235
|
function convertWireSortToLocal(wireSort) {
|
|
7027
7236
|
if (wireSort === null || wireSort.length === 0) {
|
|
7028
7237
|
return void 0;
|
|
@@ -7056,6 +7265,12 @@ function localParamsToWire(params) {
|
|
|
7056
7265
|
threshold: params.minScore ?? null,
|
|
7057
7266
|
includeScores: params.includeScores ?? null,
|
|
7058
7267
|
scoring: params.scoring ?? "local",
|
|
7268
|
+
termMatch: params.termMatch ?? null,
|
|
7269
|
+
prefixLength: params.prefixLength ?? null,
|
|
7270
|
+
prefix: params.prefix ?? null,
|
|
7271
|
+
exact: params.exact ?? null,
|
|
7272
|
+
pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
|
|
7273
|
+
mode: params.mode ?? null,
|
|
7059
7274
|
vector: convertLocalVectorToWire(params.vector),
|
|
7060
7275
|
hybrid: convertLocalHybridToWire(params.hybrid)
|
|
7061
7276
|
};
|
|
@@ -7083,8 +7298,9 @@ function convertLocalGroupToWire(group) {
|
|
|
7083
7298
|
return null;
|
|
7084
7299
|
}
|
|
7085
7300
|
return {
|
|
7086
|
-
|
|
7087
|
-
maxPerGroup: group.maxPerGroup ?? 1
|
|
7301
|
+
fields: [...group.fields],
|
|
7302
|
+
maxPerGroup: group.maxPerGroup ?? 1,
|
|
7303
|
+
limit: group.limit ?? null
|
|
7088
7304
|
};
|
|
7089
7305
|
}
|
|
7090
7306
|
function convertLocalFacetsToWire(facets) {
|
|
@@ -7101,7 +7317,9 @@ function convertLocalVectorToWire(vector) {
|
|
|
7101
7317
|
field: vector.field,
|
|
7102
7318
|
value: vector.value === void 0 ? null : Array.from(vector.value),
|
|
7103
7319
|
text: vector.text ?? null,
|
|
7104
|
-
similarity: vector.similarity ?? null
|
|
7320
|
+
similarity: vector.similarity ?? null,
|
|
7321
|
+
metric: vector.metric ?? null,
|
|
7322
|
+
efSearch: vector.efSearch ?? null
|
|
7105
7323
|
};
|
|
7106
7324
|
}
|
|
7107
7325
|
function convertLocalHybridToWire(hybrid) {
|
|
@@ -7109,9 +7327,9 @@ function convertLocalHybridToWire(hybrid) {
|
|
|
7109
7327
|
return null;
|
|
7110
7328
|
}
|
|
7111
7329
|
return {
|
|
7112
|
-
strategy: hybrid.strategy ??
|
|
7113
|
-
k: hybrid.k ??
|
|
7114
|
-
alpha: hybrid.alpha ??
|
|
7330
|
+
strategy: hybrid.strategy ?? null,
|
|
7331
|
+
k: hybrid.k ?? null,
|
|
7332
|
+
alpha: hybrid.alpha ?? null
|
|
7115
7333
|
};
|
|
7116
7334
|
}
|
|
7117
7335
|
function convertWireFacetsToLocal(wireFacets, errorBounds) {
|
|
@@ -7133,6 +7351,9 @@ async function handleSearch(message, respond, deps) {
|
|
|
7133
7351
|
const decoded = decode(message.payload);
|
|
7134
7352
|
const payload = validateSearchPayload(decoded);
|
|
7135
7353
|
const queryParams = wireParamsToLocal(payload.params, payload.facetShardSize);
|
|
7354
|
+
if (queryParams.group?.limit !== void 0) {
|
|
7355
|
+
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
7356
|
+
}
|
|
7136
7357
|
const queryResult = await deps.engine.queryPartitions(
|
|
7137
7358
|
payload.indexName,
|
|
7138
7359
|
queryParams,
|
|
@@ -7153,7 +7374,8 @@ async function handleSearch(message, respond, deps) {
|
|
|
7153
7374
|
const resultPayload = {
|
|
7154
7375
|
results,
|
|
7155
7376
|
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
7156
|
-
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets)
|
|
7377
|
+
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
7378
|
+
groups: await convertLocalGroupsToWire(queryResult.groups, sortFields, deps, payload.indexName)
|
|
7157
7379
|
};
|
|
7158
7380
|
await respond({
|
|
7159
7381
|
type: QueryMessageTypes.SEARCH_RESULT,
|
|
@@ -7162,6 +7384,25 @@ async function handleSearch(message, respond, deps) {
|
|
|
7162
7384
|
payload: encode(resultPayload)
|
|
7163
7385
|
});
|
|
7164
7386
|
}
|
|
7387
|
+
async function convertLocalGroupsToWire(groups, sortFields, deps, indexName) {
|
|
7388
|
+
if (groups === void 0) {
|
|
7389
|
+
return null;
|
|
7390
|
+
}
|
|
7391
|
+
const wireGroups = [];
|
|
7392
|
+
for (const group of groups) {
|
|
7393
|
+
const scored = [];
|
|
7394
|
+
for (const hit of group.hits) {
|
|
7395
|
+
let sortValues = null;
|
|
7396
|
+
if (sortFields !== null) {
|
|
7397
|
+
const document = hit.document ?? await deps.engine.get(indexName, hit.id);
|
|
7398
|
+
sortValues = readSortValues(document, sortFields).map(toWireSortValue);
|
|
7399
|
+
}
|
|
7400
|
+
scored.push({ docId: hit.id, score: hit.score ?? null, sortValues });
|
|
7401
|
+
}
|
|
7402
|
+
wireGroups.push({ values: group.values, scored });
|
|
7403
|
+
}
|
|
7404
|
+
return wireGroups;
|
|
7405
|
+
}
|
|
7165
7406
|
async function handleFetch(message, respond, deps) {
|
|
7166
7407
|
const decoded = decode(message.payload);
|
|
7167
7408
|
const payload = validateFetchPayload(decoded);
|
|
@@ -7503,6 +7744,37 @@ var DEFAULT_CAPACITY = {
|
|
|
7503
7744
|
cpuCores: 4,
|
|
7504
7745
|
diskBytes: null
|
|
7505
7746
|
};
|
|
7747
|
+
|
|
7748
|
+
// src/distribution/cluster-node/transport-failure.ts
|
|
7749
|
+
var FAILURE_CODE_BY_TRANSPORT_CODE = {
|
|
7750
|
+
[TransportErrorCodes.CONNECT_FAILED]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
7751
|
+
[TransportErrorCodes.PEER_UNAVAILABLE]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
7752
|
+
[TransportErrorCodes.TIMEOUT]: ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
7753
|
+
[TransportErrorCodes.MESSAGE_TOO_LARGE]: ErrorCodes.QUERY_ROUTING_FAILED,
|
|
7754
|
+
[TransportErrorCodes.DECODE_FAILED]: ErrorCodes.QUERY_ROUTING_FAILED
|
|
7755
|
+
};
|
|
7756
|
+
function narsilErrorFromTransportFailure(error, message, details) {
|
|
7757
|
+
return new NarsilError(FAILURE_CODE_BY_TRANSPORT_CODE[error.code], message, {
|
|
7758
|
+
...details,
|
|
7759
|
+
transportCode: error.code
|
|
7760
|
+
});
|
|
7761
|
+
}
|
|
7762
|
+
async function sendThroughTargets(targets, send, failure) {
|
|
7763
|
+
let lastError;
|
|
7764
|
+
for (const target of targets) {
|
|
7765
|
+
try {
|
|
7766
|
+
return await send(target);
|
|
7767
|
+
} catch (error) {
|
|
7768
|
+
lastError = error;
|
|
7769
|
+
}
|
|
7770
|
+
}
|
|
7771
|
+
if (lastError instanceof TransportError) {
|
|
7772
|
+
throw narsilErrorFromTransportFailure(lastError, failure.message, failure.details ?? {});
|
|
7773
|
+
}
|
|
7774
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
7775
|
+
}
|
|
7776
|
+
|
|
7777
|
+
// src/distribution/cluster-node/write-routing/forwarding.ts
|
|
7506
7778
|
function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
7507
7779
|
const decoded = decode(response.payload);
|
|
7508
7780
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -7560,15 +7832,10 @@ async function forwardRemoveToRemote(indexName, docId, primaryNodeId, deps) {
|
|
|
7560
7832
|
}
|
|
7561
7833
|
async function sendToNode(nodeId, message, deps) {
|
|
7562
7834
|
const targets = await resolveNodeTargets(nodeId, deps);
|
|
7563
|
-
|
|
7564
|
-
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
} catch (error) {
|
|
7568
|
-
lastError = error;
|
|
7569
|
-
}
|
|
7570
|
-
}
|
|
7571
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
7835
|
+
return sendThroughTargets(targets, (target) => deps.transport.send(target, message), {
|
|
7836
|
+
message: `Node '${nodeId}' did not answer a forwarded write`,
|
|
7837
|
+
details: { targetNodeId: nodeId }
|
|
7838
|
+
});
|
|
7572
7839
|
}
|
|
7573
7840
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
7574
7841
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
@@ -7645,24 +7912,15 @@ async function removeFailedReplicasFromInsync(entry, failedReplicas, deps) {
|
|
|
7645
7912
|
}
|
|
7646
7913
|
async function requestInsyncRemovalWithTargets(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, deps) {
|
|
7647
7914
|
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
replicaNodeId,
|
|
7655
|
-
primaryTerm,
|
|
7656
|
-
target,
|
|
7657
|
-
deps.transport,
|
|
7658
|
-
deps.nodeId
|
|
7659
|
-
);
|
|
7660
|
-
return result.accepted;
|
|
7661
|
-
} catch (error) {
|
|
7662
|
-
lastError = error;
|
|
7915
|
+
const result = await sendThroughTargets(
|
|
7916
|
+
targets,
|
|
7917
|
+
(target) => requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, target, deps.transport, deps.nodeId),
|
|
7918
|
+
{
|
|
7919
|
+
message: `Controller '${controllerNodeId}' did not answer an in-sync removal request for index '${indexName}' partition ${partitionId}`,
|
|
7920
|
+
details: { indexName, partitionId, replicaNodeId, controllerNodeId }
|
|
7663
7921
|
}
|
|
7664
|
-
|
|
7665
|
-
|
|
7922
|
+
);
|
|
7923
|
+
return result.accepted;
|
|
7666
7924
|
}
|
|
7667
7925
|
async function replicateEntry(entry, assignment, deps) {
|
|
7668
7926
|
const pending = getPendingAdmissions(deps.catchUp, entry.indexName, entry.partitionId);
|
|
@@ -8908,17 +9166,12 @@ async function resolveNodeTargets2(config, targetNodeId) {
|
|
|
8908
9166
|
}
|
|
8909
9167
|
return targets;
|
|
8910
9168
|
}
|
|
8911
|
-
async function sendToNode2(config, targetNodeId, message) {
|
|
9169
|
+
async function sendToNode2(config, targetNodeId, message, failure) {
|
|
8912
9170
|
const targets = await resolveNodeTargets2(config, targetNodeId);
|
|
8913
|
-
|
|
8914
|
-
|
|
8915
|
-
|
|
8916
|
-
|
|
8917
|
-
} catch (error) {
|
|
8918
|
-
lastError = error;
|
|
8919
|
-
}
|
|
8920
|
-
}
|
|
8921
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
9171
|
+
return sendThroughTargets(targets, (target) => config.transport.send(target, message), {
|
|
9172
|
+
message: failure?.message ?? `Node '${targetNodeId}' did not answer a cluster request`,
|
|
9173
|
+
details: { targetNodeId, ...failure?.details }
|
|
9174
|
+
});
|
|
8922
9175
|
}
|
|
8923
9176
|
function fetchFieldsFor(projection) {
|
|
8924
9177
|
if (projection.kind !== "fields" || projection.include === null) return null;
|
|
@@ -8966,7 +9219,10 @@ async function fetchDistributedDocuments(config, nodeId, engine, indexName, resu
|
|
|
8966
9219
|
},
|
|
8967
9220
|
nodeId
|
|
8968
9221
|
);
|
|
8969
|
-
const response = await sendToNode2(config, targetNodeId, fetchMessage
|
|
9222
|
+
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
9223
|
+
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
9224
|
+
details: { indexName }
|
|
9225
|
+
});
|
|
8970
9226
|
const decoded = decode(response.payload);
|
|
8971
9227
|
const payload = validateFetchResultPayload(decoded);
|
|
8972
9228
|
for (const fetched of payload.documents) {
|
|
@@ -9014,7 +9270,10 @@ async function readDistributedDocuments(config, nodeId, engine, indexName, docId
|
|
|
9014
9270
|
continue;
|
|
9015
9271
|
}
|
|
9016
9272
|
const fetchMessage = createFetchMessage({ indexName, documentIds, fields: null, highlight: null }, nodeId);
|
|
9017
|
-
const response = await sendToNode2(config, targetNodeId, fetchMessage
|
|
9273
|
+
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
9274
|
+
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
9275
|
+
details: { indexName }
|
|
9276
|
+
});
|
|
9018
9277
|
const payload = validateFetchResultPayload(decode(response.payload));
|
|
9019
9278
|
for (const fetched of payload.documents) {
|
|
9020
9279
|
documents.set(fetched.docId, fetched.document);
|
|
@@ -9162,90 +9421,17 @@ function isTimeoutError(error) {
|
|
|
9162
9421
|
return error instanceof TransportError && error.code === "TRANSPORT_TIMEOUT";
|
|
9163
9422
|
}
|
|
9164
9423
|
|
|
9165
|
-
// src/distribution/query/
|
|
9166
|
-
function
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
if (alpha > 1) return 1;
|
|
9170
|
-
return alpha;
|
|
9171
|
-
}
|
|
9172
|
-
function compareFusedEntries(a, b) {
|
|
9173
|
-
const aScore = a.score ?? 0;
|
|
9174
|
-
const bScore = b.score ?? 0;
|
|
9424
|
+
// src/distribution/query/merge.ts
|
|
9425
|
+
function compareByScoreThenId(a, b) {
|
|
9426
|
+
const aScore = a.score ?? Number.NEGATIVE_INFINITY;
|
|
9427
|
+
const bScore = b.score ?? Number.NEGATIVE_INFINITY;
|
|
9175
9428
|
if (aScore !== bScore) {
|
|
9176
9429
|
return bScore - aScore;
|
|
9177
9430
|
}
|
|
9178
9431
|
return compareCodePoints(a.docId, b.docId);
|
|
9179
9432
|
}
|
|
9180
|
-
function
|
|
9181
|
-
|
|
9182
|
-
const scores = /* @__PURE__ */ new Map();
|
|
9183
|
-
for (const list of lists) {
|
|
9184
|
-
for (let rank = 0; rank < list.length; rank++) {
|
|
9185
|
-
const entry = list[rank];
|
|
9186
|
-
const rrfContribution = 1 / (k + rank + 1);
|
|
9187
|
-
scores.set(entry.docId, (scores.get(entry.docId) ?? 0) + rrfContribution);
|
|
9188
|
-
}
|
|
9189
|
-
}
|
|
9190
|
-
const result = [];
|
|
9191
|
-
for (const [docId, score] of scores) {
|
|
9192
|
-
result.push({ docId, score, sortValues: null });
|
|
9193
|
-
}
|
|
9194
|
-
result.sort(compareFusedEntries);
|
|
9195
|
-
return result;
|
|
9196
|
-
}
|
|
9197
|
-
function distributedLinearCombination(textResults, vectorResults, options) {
|
|
9198
|
-
const { alpha } = options;
|
|
9199
|
-
const normalizedText = minMaxNormalizeScoredEntries(textResults);
|
|
9200
|
-
const normalizedVector = minMaxNormalizeScoredEntries(vectorResults);
|
|
9201
|
-
const textScoreMap = /* @__PURE__ */ new Map();
|
|
9202
|
-
for (const entry of normalizedText) {
|
|
9203
|
-
textScoreMap.set(entry.docId, entry.score);
|
|
9204
|
-
}
|
|
9205
|
-
const vectorScoreMap = /* @__PURE__ */ new Map();
|
|
9206
|
-
for (const entry of normalizedVector) {
|
|
9207
|
-
vectorScoreMap.set(entry.docId, entry.score);
|
|
9208
|
-
}
|
|
9209
|
-
const allDocIds = /* @__PURE__ */ new Set();
|
|
9210
|
-
for (const entry of textResults) allDocIds.add(entry.docId);
|
|
9211
|
-
for (const entry of vectorResults) allDocIds.add(entry.docId);
|
|
9212
|
-
const result = [];
|
|
9213
|
-
for (const docId of allDocIds) {
|
|
9214
|
-
const tScore = textScoreMap.get(docId) ?? 0;
|
|
9215
|
-
const vScore = vectorScoreMap.get(docId) ?? 0;
|
|
9216
|
-
const combined = alpha * vScore + (1 - alpha) * tScore;
|
|
9217
|
-
result.push({ docId, score: combined, sortValues: null });
|
|
9218
|
-
}
|
|
9219
|
-
result.sort(compareFusedEntries);
|
|
9220
|
-
return result;
|
|
9221
|
-
}
|
|
9222
|
-
function minMaxNormalizeScoredEntries(entries) {
|
|
9223
|
-
if (entries.length === 0) return [];
|
|
9224
|
-
let min = entries[0].score ?? 0;
|
|
9225
|
-
let max = entries[0].score ?? 0;
|
|
9226
|
-
for (let i = 1; i < entries.length; i++) {
|
|
9227
|
-
const score = entries[i].score ?? 0;
|
|
9228
|
-
if (score < min) min = score;
|
|
9229
|
-
if (score > max) max = score;
|
|
9230
|
-
}
|
|
9231
|
-
const range = max - min;
|
|
9232
|
-
if (range === 0) {
|
|
9233
|
-
return entries.map((e) => ({ docId: e.docId, score: 1 }));
|
|
9234
|
-
}
|
|
9235
|
-
return entries.map((e) => ({ docId: e.docId, score: ((e.score ?? 0) - min) / range }));
|
|
9236
|
-
}
|
|
9237
|
-
|
|
9238
|
-
// src/distribution/query/merge.ts
|
|
9239
|
-
function compareByScoreThenId(a, b) {
|
|
9240
|
-
const aScore = a.score ?? Number.NEGATIVE_INFINITY;
|
|
9241
|
-
const bScore = b.score ?? Number.NEGATIVE_INFINITY;
|
|
9242
|
-
if (aScore !== bScore) {
|
|
9243
|
-
return bScore - aScore;
|
|
9244
|
-
}
|
|
9245
|
-
return compareCodePoints(a.docId, b.docId);
|
|
9246
|
-
}
|
|
9247
|
-
function mergeAndTruncateScoredEntries(arrays, limit) {
|
|
9248
|
-
return mergeAndTruncate(arrays, limit, compareByScoreThenId);
|
|
9433
|
+
function mergeAndTruncateScoredEntries(arrays, limit) {
|
|
9434
|
+
return mergeAndTruncate(arrays, limit, compareByScoreThenId);
|
|
9249
9435
|
}
|
|
9250
9436
|
function mergeAndTruncateSortedEntries(arrays, limit, directions) {
|
|
9251
9437
|
const bySortValues = (a, b) => compareSortValues(a.sortValues ?? [], b.sortValues ?? [], directions) || compareCodePoints(a.docId, b.docId);
|
|
@@ -9396,6 +9582,241 @@ function mergeDistributedFacets(allFacets, allBounds, maxBuckets = DEFAULT_MAX_F
|
|
|
9396
9582
|
return { facets, errorBounds };
|
|
9397
9583
|
}
|
|
9398
9584
|
|
|
9585
|
+
// src/distribution/query/group-merge.ts
|
|
9586
|
+
function mergeDistributedGroups(perNode, fields, maxPerGroup, limit, sortDirections) {
|
|
9587
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
9588
|
+
for (const groups of perNode) {
|
|
9589
|
+
for (const group of groups) {
|
|
9590
|
+
const key = JSON.stringify(fields.map((field) => group.values[field]));
|
|
9591
|
+
const existing = byKey.get(key);
|
|
9592
|
+
if (existing === void 0) {
|
|
9593
|
+
byKey.set(key, { values: group.values, lists: [group.scored] });
|
|
9594
|
+
} else {
|
|
9595
|
+
existing.lists.push(group.scored);
|
|
9596
|
+
}
|
|
9597
|
+
}
|
|
9598
|
+
}
|
|
9599
|
+
const merged = [];
|
|
9600
|
+
for (const entry of byKey.values()) {
|
|
9601
|
+
const scored = sortDirections !== null ? mergeAndTruncateSortedEntries(entry.lists, maxPerGroup, sortDirections) : mergeAndTruncateScoredEntries(entry.lists, maxPerGroup);
|
|
9602
|
+
merged.push({ values: entry.values, scored });
|
|
9603
|
+
}
|
|
9604
|
+
merged.sort((a, b) => {
|
|
9605
|
+
const aScore = a.scored.length > 0 ? a.scored[0].score ?? 0 : 0;
|
|
9606
|
+
const bScore = b.scored.length > 0 ? b.scored[0].score ?? 0 : 0;
|
|
9607
|
+
if (aScore !== bScore) return bScore - aScore;
|
|
9608
|
+
const aId = a.scored.length > 0 ? a.scored[0].docId : "";
|
|
9609
|
+
const bId = b.scored.length > 0 ? b.scored[0].docId : "";
|
|
9610
|
+
return compareCodePoints(aId, bId);
|
|
9611
|
+
});
|
|
9612
|
+
if (limit !== null && merged.length > limit) {
|
|
9613
|
+
merged.length = limit;
|
|
9614
|
+
}
|
|
9615
|
+
return merged;
|
|
9616
|
+
}
|
|
9617
|
+
function mergeGroupsFor(params, allGroups, sortFields) {
|
|
9618
|
+
if (params.group === null || allGroups.length === 0) {
|
|
9619
|
+
return null;
|
|
9620
|
+
}
|
|
9621
|
+
return mergeDistributedGroups(
|
|
9622
|
+
allGroups,
|
|
9623
|
+
params.group.fields,
|
|
9624
|
+
params.group.maxPerGroup,
|
|
9625
|
+
params.group.limit,
|
|
9626
|
+
sortFields !== null ? sortFields.map((field) => field.direction) : null
|
|
9627
|
+
);
|
|
9628
|
+
}
|
|
9629
|
+
|
|
9630
|
+
// src/distribution/query/fusion.ts
|
|
9631
|
+
function clampAlpha(alpha) {
|
|
9632
|
+
if (!Number.isFinite(alpha)) return 0.5;
|
|
9633
|
+
if (alpha < 0) return 0;
|
|
9634
|
+
if (alpha > 1) return 1;
|
|
9635
|
+
return alpha;
|
|
9636
|
+
}
|
|
9637
|
+
function compareFusedEntries(a, b) {
|
|
9638
|
+
const aScore = a.score ?? 0;
|
|
9639
|
+
const bScore = b.score ?? 0;
|
|
9640
|
+
if (aScore !== bScore) {
|
|
9641
|
+
return bScore - aScore;
|
|
9642
|
+
}
|
|
9643
|
+
return compareCodePoints(a.docId, b.docId);
|
|
9644
|
+
}
|
|
9645
|
+
function distributedRRF(lists, options) {
|
|
9646
|
+
const k = options.k > 0 ? options.k : 60;
|
|
9647
|
+
const scores = /* @__PURE__ */ new Map();
|
|
9648
|
+
for (const list of lists) {
|
|
9649
|
+
for (let rank = 0; rank < list.length; rank++) {
|
|
9650
|
+
const entry = list[rank];
|
|
9651
|
+
const rrfContribution = 1 / (k + rank + 1);
|
|
9652
|
+
scores.set(entry.docId, (scores.get(entry.docId) ?? 0) + rrfContribution);
|
|
9653
|
+
}
|
|
9654
|
+
}
|
|
9655
|
+
const result = [];
|
|
9656
|
+
for (const [docId, score] of scores) {
|
|
9657
|
+
result.push({ docId, score, sortValues: null });
|
|
9658
|
+
}
|
|
9659
|
+
result.sort(compareFusedEntries);
|
|
9660
|
+
return result;
|
|
9661
|
+
}
|
|
9662
|
+
function distributedLinearCombination(textResults, vectorResults, options) {
|
|
9663
|
+
const { alpha } = options;
|
|
9664
|
+
const normalizedText = minMaxNormalizeScoredEntries(textResults);
|
|
9665
|
+
const normalizedVector = minMaxNormalizeScoredEntries(vectorResults);
|
|
9666
|
+
const textScoreMap = /* @__PURE__ */ new Map();
|
|
9667
|
+
for (const entry of normalizedText) {
|
|
9668
|
+
textScoreMap.set(entry.docId, entry.score);
|
|
9669
|
+
}
|
|
9670
|
+
const vectorScoreMap = /* @__PURE__ */ new Map();
|
|
9671
|
+
for (const entry of normalizedVector) {
|
|
9672
|
+
vectorScoreMap.set(entry.docId, entry.score);
|
|
9673
|
+
}
|
|
9674
|
+
const allDocIds = /* @__PURE__ */ new Set();
|
|
9675
|
+
for (const entry of textResults) allDocIds.add(entry.docId);
|
|
9676
|
+
for (const entry of vectorResults) allDocIds.add(entry.docId);
|
|
9677
|
+
const result = [];
|
|
9678
|
+
for (const docId of allDocIds) {
|
|
9679
|
+
const tScore = textScoreMap.get(docId) ?? 0;
|
|
9680
|
+
const vScore = vectorScoreMap.get(docId) ?? 0;
|
|
9681
|
+
const combined = alpha * vScore + (1 - alpha) * tScore;
|
|
9682
|
+
result.push({ docId, score: combined, sortValues: null });
|
|
9683
|
+
}
|
|
9684
|
+
result.sort(compareFusedEntries);
|
|
9685
|
+
return result;
|
|
9686
|
+
}
|
|
9687
|
+
function minMaxNormalizeScoredEntries(entries) {
|
|
9688
|
+
if (entries.length === 0) return [];
|
|
9689
|
+
let min = entries[0].score ?? 0;
|
|
9690
|
+
let max = entries[0].score ?? 0;
|
|
9691
|
+
for (let i = 1; i < entries.length; i++) {
|
|
9692
|
+
const score = entries[i].score ?? 0;
|
|
9693
|
+
if (score < min) min = score;
|
|
9694
|
+
if (score > max) max = score;
|
|
9695
|
+
}
|
|
9696
|
+
const range = max - min;
|
|
9697
|
+
if (range === 0) {
|
|
9698
|
+
return entries.map((e) => ({ docId: e.docId, score: 1 }));
|
|
9699
|
+
}
|
|
9700
|
+
return entries.map((e) => ({ docId: e.docId, score: ((e.score ?? 0) - min) / range }));
|
|
9701
|
+
}
|
|
9702
|
+
|
|
9703
|
+
// src/distribution/query/pinning.ts
|
|
9704
|
+
function lastOrganicEntry(page, pinned) {
|
|
9705
|
+
if (pinned === null) {
|
|
9706
|
+
return page.length > 0 ? page[page.length - 1] : void 0;
|
|
9707
|
+
}
|
|
9708
|
+
const pinnedIds = new Set(pinned.map((entry) => entry.docId));
|
|
9709
|
+
for (let i = page.length - 1; i >= 0; i--) {
|
|
9710
|
+
if (!pinnedIds.has(page[i].docId)) return page[i];
|
|
9711
|
+
}
|
|
9712
|
+
return void 0;
|
|
9713
|
+
}
|
|
9714
|
+
function placePinnedEntries(merged, pinned, depth, allMatchesPresent) {
|
|
9715
|
+
const deduped = dedupePinnedEntries(pinned);
|
|
9716
|
+
const pinnedIds = new Set(deduped.map((entry) => entry.docId));
|
|
9717
|
+
const removed = /* @__PURE__ */ new Map();
|
|
9718
|
+
const result = [];
|
|
9719
|
+
for (const entry of merged) {
|
|
9720
|
+
if (pinnedIds.has(entry.docId)) {
|
|
9721
|
+
removed.set(entry.docId, entry);
|
|
9722
|
+
} else {
|
|
9723
|
+
result.push(entry);
|
|
9724
|
+
}
|
|
9725
|
+
}
|
|
9726
|
+
const ordered = deduped.sort((a, b) => a.position - b.position);
|
|
9727
|
+
for (const entry of ordered) {
|
|
9728
|
+
if (!allMatchesPresent && entry.position >= depth) continue;
|
|
9729
|
+
const organic = removed.get(entry.docId);
|
|
9730
|
+
const placed = { docId: entry.docId, score: 0, sortValues: organic?.sortValues ?? null };
|
|
9731
|
+
result.splice(Math.max(0, Math.min(entry.position, result.length)), 0, placed);
|
|
9732
|
+
}
|
|
9733
|
+
return result;
|
|
9734
|
+
}
|
|
9735
|
+
|
|
9736
|
+
// src/distribution/query/hybrid.ts
|
|
9737
|
+
async function executeHybridQuery(indexName, params, limit, offset, facetShardSize, facetSize, totalPartitions, routing, deps, config) {
|
|
9738
|
+
const depth = limit + offset;
|
|
9739
|
+
const textParams = { ...params, vector: null, hybrid: null, mode: null, limit: depth, offset: 0 };
|
|
9740
|
+
const vectorParams = {
|
|
9741
|
+
...params,
|
|
9742
|
+
term: null,
|
|
9743
|
+
hybrid: null,
|
|
9744
|
+
mode: null,
|
|
9745
|
+
group: null,
|
|
9746
|
+
limit: depth,
|
|
9747
|
+
offset: 0
|
|
9748
|
+
};
|
|
9749
|
+
let textGlobalStats = null;
|
|
9750
|
+
if (params.scoring === "dfs") {
|
|
9751
|
+
textGlobalStats = await collectDistributedStats(indexName, textParams, routing.nodeToPartitions, deps, config);
|
|
9752
|
+
}
|
|
9753
|
+
const [textOutcomes, vectorOutcomes] = await Promise.all([
|
|
9754
|
+
fanOutSearch(indexName, textParams, textGlobalStats, facetShardSize, routing.nodeToPartitions, deps, config),
|
|
9755
|
+
fanOutSearch(indexName, vectorParams, null, null, routing.nodeToPartitions, deps, config)
|
|
9756
|
+
]);
|
|
9757
|
+
const coverage = buildCoverage(totalPartitions, routing.unavailablePartitions, textOutcomes, vectorOutcomes);
|
|
9758
|
+
if (!config.allowPartialResults) {
|
|
9759
|
+
const failedCount = coverage.timedOutPartitions + coverage.failedPartitions;
|
|
9760
|
+
if (failedCount > 0) {
|
|
9761
|
+
throw new NarsilError(ErrorCodes.QUERY_PARTIAL_FAILURE, "One or more partitions failed during hybrid query", {
|
|
9762
|
+
coverage
|
|
9763
|
+
});
|
|
9764
|
+
}
|
|
9765
|
+
}
|
|
9766
|
+
const textScored = [];
|
|
9767
|
+
const vectorScored = [];
|
|
9768
|
+
const allFacets = [];
|
|
9769
|
+
const allFacetBounds = [];
|
|
9770
|
+
const allGroups = [];
|
|
9771
|
+
let totalHits = 0;
|
|
9772
|
+
for (const outcome of textOutcomes) {
|
|
9773
|
+
if (outcome.status !== "success" || outcome.results === null) continue;
|
|
9774
|
+
for (const partitionResult of outcome.results.results) {
|
|
9775
|
+
totalHits += partitionResult.totalHits;
|
|
9776
|
+
if (partitionResult.scored.length > 0) {
|
|
9777
|
+
textScored.push(partitionResult.scored);
|
|
9778
|
+
}
|
|
9779
|
+
}
|
|
9780
|
+
if (outcome.results.facets !== null) {
|
|
9781
|
+
allFacets.push(outcome.results.facets);
|
|
9782
|
+
allFacetBounds.push(outcome.results.facetErrorBounds);
|
|
9783
|
+
}
|
|
9784
|
+
if (outcome.results.groups !== null && outcome.results.groups !== void 0) {
|
|
9785
|
+
allGroups.push(outcome.results.groups);
|
|
9786
|
+
}
|
|
9787
|
+
}
|
|
9788
|
+
for (const outcome of vectorOutcomes) {
|
|
9789
|
+
if (outcome.status !== "success" || outcome.results === null) continue;
|
|
9790
|
+
for (const partitionResult of outcome.results.results) {
|
|
9791
|
+
if (partitionResult.scored.length > 0) {
|
|
9792
|
+
vectorScored.push(partitionResult.scored);
|
|
9793
|
+
}
|
|
9794
|
+
}
|
|
9795
|
+
}
|
|
9796
|
+
const mergedText = mergeAndTruncateScoredEntries(textScored, depth);
|
|
9797
|
+
const mergedVector = mergeAndTruncateScoredEntries(vectorScored, depth);
|
|
9798
|
+
const hybrid = params.hybrid;
|
|
9799
|
+
let fused;
|
|
9800
|
+
if ((hybrid?.strategy ?? "rrf") === "rrf") {
|
|
9801
|
+
fused = distributedRRF([mergedText, mergedVector], { k: hybrid?.k ?? 60 });
|
|
9802
|
+
} else {
|
|
9803
|
+
fused = distributedLinearCombination(mergedText, mergedVector, { alpha: clampAlpha(hybrid?.alpha ?? 0.5) });
|
|
9804
|
+
}
|
|
9805
|
+
const allMatchesPresent = coverage.queriedPartitions === coverage.totalPartitions && totalHits <= mergedText.length && mergedVector.length < depth;
|
|
9806
|
+
const fusedWithPinned = params.pinned !== null ? placePinnedEntries(fused, params.pinned, depth, allMatchesPresent) : fused;
|
|
9807
|
+
const truncated = fusedWithPinned.slice(offset, depth);
|
|
9808
|
+
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize) : null;
|
|
9809
|
+
return {
|
|
9810
|
+
scored: truncated,
|
|
9811
|
+
totalHits,
|
|
9812
|
+
facets: mergedFacets?.facets ?? null,
|
|
9813
|
+
facetErrorBounds: mergedFacets?.errorBounds ?? null,
|
|
9814
|
+
groups: mergeGroupsFor(params, allGroups, null),
|
|
9815
|
+
cursor: null,
|
|
9816
|
+
coverage
|
|
9817
|
+
};
|
|
9818
|
+
}
|
|
9819
|
+
|
|
9399
9820
|
// src/distribution/query/types.ts
|
|
9400
9821
|
var DEFAULT_QUERY_CONFIG = {
|
|
9401
9822
|
allowPartialResults: true,
|
|
@@ -9418,16 +9839,17 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
9418
9839
|
...DEFAULT_QUERY_CONFIG,
|
|
9419
9840
|
...config
|
|
9420
9841
|
};
|
|
9842
|
+
const binding = queryBindingOf(wireParamsToLocal(params));
|
|
9421
9843
|
if (params.searchAfter !== null) {
|
|
9422
9844
|
const decoded = decodePageCursor(params.searchAfter);
|
|
9423
|
-
requireMatchingCursor(decoded, params.searchAfter, wireSortSignature(params.sort), true);
|
|
9845
|
+
requireMatchingCursor(decoded, params.searchAfter, wireSortSignature(params.sort), true, binding);
|
|
9424
9846
|
}
|
|
9425
9847
|
const limit = Math.max(params.limit, 0);
|
|
9426
9848
|
const offset = Math.max(params.offset, 0);
|
|
9427
9849
|
requireWithinResultWindow(limit, offset);
|
|
9428
9850
|
const hasFacets = params.facets !== null && params.facets.length > 0;
|
|
9429
9851
|
const facetSize = resolveAndClampFacetSize(params.facetSize, resolvedConfig.defaultFacetSize);
|
|
9430
|
-
const facetShardSize = hasFacets ?
|
|
9852
|
+
const facetShardSize = hasFacets ? oversampledShardSize(facetSize) : null;
|
|
9431
9853
|
const allocationTable = await deps.getAllocation(indexName);
|
|
9432
9854
|
if (allocationTable === null) {
|
|
9433
9855
|
throw new NarsilError(ErrorCodes.QUERY_ROUTING_FAILED, `No allocation table found for index '${indexName}'`, {
|
|
@@ -9440,6 +9862,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
9440
9862
|
totalHits: 0,
|
|
9441
9863
|
facets: null,
|
|
9442
9864
|
facetErrorBounds: null,
|
|
9865
|
+
groups: null,
|
|
9443
9866
|
cursor: null,
|
|
9444
9867
|
coverage: { totalPartitions: 0, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: 0 }
|
|
9445
9868
|
};
|
|
@@ -9453,11 +9876,13 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
9453
9876
|
{ unavailablePartitions: routing.unavailablePartitions }
|
|
9454
9877
|
);
|
|
9455
9878
|
}
|
|
9456
|
-
const isHybrid = params.
|
|
9879
|
+
const isHybrid = params.vector !== null && (params.term !== null || params.mode === "hybrid");
|
|
9457
9880
|
if (isHybrid && params.searchAfter !== null) {
|
|
9458
|
-
throw new NarsilError(
|
|
9459
|
-
|
|
9460
|
-
|
|
9881
|
+
throw new NarsilError(
|
|
9882
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
9883
|
+
"Cursor pagination is not supported for hybrid queries, because fused ranks do not seek",
|
|
9884
|
+
{ indexName }
|
|
9885
|
+
);
|
|
9461
9886
|
}
|
|
9462
9887
|
if (isHybrid && params.sort !== null && params.sort.length > 0) {
|
|
9463
9888
|
throw new NarsilError(
|
|
@@ -9483,6 +9908,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
9483
9908
|
return executeSingleFanOut(
|
|
9484
9909
|
indexName,
|
|
9485
9910
|
params,
|
|
9911
|
+
binding,
|
|
9486
9912
|
limit,
|
|
9487
9913
|
offset,
|
|
9488
9914
|
facetShardSize,
|
|
@@ -9493,7 +9919,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
9493
9919
|
resolvedConfig
|
|
9494
9920
|
);
|
|
9495
9921
|
}
|
|
9496
|
-
async function executeSingleFanOut(indexName, params, limit, offset, facetShardSize, facetSize, totalPartitions, routing, deps, config) {
|
|
9922
|
+
async function executeSingleFanOut(indexName, params, binding, limit, offset, facetShardSize, facetSize, totalPartitions, routing, deps, config) {
|
|
9497
9923
|
let globalStats = null;
|
|
9498
9924
|
if (params.scoring === "dfs") {
|
|
9499
9925
|
globalStats = await collectDistributedStats(indexName, params, routing.nodeToPartitions, deps, config);
|
|
@@ -9525,6 +9951,7 @@ async function executeSingleFanOut(indexName, params, limit, offset, facetShardS
|
|
|
9525
9951
|
const allScored = [];
|
|
9526
9952
|
const allFacets = [];
|
|
9527
9953
|
const allFacetBounds = [];
|
|
9954
|
+
const allGroups = [];
|
|
9528
9955
|
let totalHits = 0;
|
|
9529
9956
|
for (const outcome of outcomes) {
|
|
9530
9957
|
if (outcome.status !== "success" || outcome.results === null) continue;
|
|
@@ -9538,29 +9965,43 @@ async function executeSingleFanOut(indexName, params, limit, offset, facetShardS
|
|
|
9538
9965
|
allFacets.push(outcome.results.facets);
|
|
9539
9966
|
allFacetBounds.push(outcome.results.facetErrorBounds);
|
|
9540
9967
|
}
|
|
9968
|
+
if (outcome.results.groups !== null && outcome.results.groups !== void 0) {
|
|
9969
|
+
allGroups.push(outcome.results.groups);
|
|
9970
|
+
}
|
|
9541
9971
|
}
|
|
9542
9972
|
const merged = sortFields !== null ? mergeAndTruncateSortedEntries(
|
|
9543
9973
|
allScored,
|
|
9544
9974
|
depth,
|
|
9545
9975
|
sortFields.map((field) => field.direction)
|
|
9546
9976
|
) : mergeAndTruncateScoredEntries(allScored, depth);
|
|
9547
|
-
const
|
|
9977
|
+
const allMatchesPresent = coverage.queriedPartitions === coverage.totalPartitions && totalHits <= merged.length;
|
|
9978
|
+
const placed = params.pinned !== null && params.searchAfter === null ? placePinnedEntries(merged, params.pinned, depth, allMatchesPresent) : merged;
|
|
9979
|
+
const mergedScored = placed.slice(offset, depth);
|
|
9548
9980
|
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize) : null;
|
|
9981
|
+
const mergedGroups = mergeGroupsFor(params, allGroups, sortFields);
|
|
9549
9982
|
let cursor = null;
|
|
9550
|
-
|
|
9551
|
-
|
|
9983
|
+
const lastEntry = lastOrganicEntry(mergedScored, params.searchAfter === null ? params.pinned : null);
|
|
9984
|
+
if (lastEntry !== void 0) {
|
|
9552
9985
|
cursor = sortFields !== null ? encodePageCursor({
|
|
9553
9986
|
anchor: lastEntry.docId,
|
|
9554
9987
|
score: null,
|
|
9555
9988
|
sortKey: (lastEntry.sortValues ?? []).map(toComparableSortValue),
|
|
9556
|
-
sortSignature: wireSortSignature(sortFields)
|
|
9557
|
-
|
|
9989
|
+
sortSignature: wireSortSignature(sortFields),
|
|
9990
|
+
binding
|
|
9991
|
+
}) : encodePageCursor({
|
|
9992
|
+
anchor: lastEntry.docId,
|
|
9993
|
+
score: lastEntry.score,
|
|
9994
|
+
sortKey: null,
|
|
9995
|
+
sortSignature: null,
|
|
9996
|
+
binding
|
|
9997
|
+
});
|
|
9558
9998
|
}
|
|
9559
9999
|
return {
|
|
9560
10000
|
scored: mergedScored,
|
|
9561
10001
|
totalHits,
|
|
9562
10002
|
facets: mergedFacets?.facets ?? null,
|
|
9563
10003
|
facetErrorBounds: mergedFacets?.errorBounds ?? null,
|
|
10004
|
+
groups: mergedGroups,
|
|
9564
10005
|
cursor,
|
|
9565
10006
|
coverage
|
|
9566
10007
|
};
|
|
@@ -9575,79 +10016,73 @@ function failOutcomesMissingSortValues(outcomes) {
|
|
|
9575
10016
|
}
|
|
9576
10017
|
}
|
|
9577
10018
|
}
|
|
9578
|
-
|
|
9579
|
-
|
|
9580
|
-
|
|
9581
|
-
const
|
|
9582
|
-
|
|
9583
|
-
if (
|
|
9584
|
-
|
|
9585
|
-
}
|
|
9586
|
-
const [textOutcomes, vectorOutcomes] = await Promise.all([
|
|
9587
|
-
fanOutSearch(indexName, textParams, textGlobalStats, facetShardSize, routing.nodeToPartitions, deps, config),
|
|
9588
|
-
fanOutSearch(indexName, vectorParams, null, null, routing.nodeToPartitions, deps, config)
|
|
9589
|
-
]);
|
|
9590
|
-
const coverage = buildCoverage(totalPartitions, routing.unavailablePartitions, textOutcomes, vectorOutcomes);
|
|
9591
|
-
if (!config.allowPartialResults) {
|
|
9592
|
-
const failedCount = coverage.timedOutPartitions + coverage.failedPartitions;
|
|
9593
|
-
if (failedCount > 0) {
|
|
9594
|
-
throw new NarsilError(ErrorCodes.QUERY_PARTIAL_FAILURE, "One or more partitions failed during hybrid query", {
|
|
9595
|
-
coverage
|
|
9596
|
-
});
|
|
9597
|
-
}
|
|
10019
|
+
|
|
10020
|
+
// src/distribution/cluster-node/reads/groups.ts
|
|
10021
|
+
async function assembleDistributedGroups(deps, indexName, params, distributed, allocation, projection) {
|
|
10022
|
+
const wireGroups = distributed.groups;
|
|
10023
|
+
const group = params.group;
|
|
10024
|
+
if (wireGroups === null || group === void 0) {
|
|
10025
|
+
return void 0;
|
|
9598
10026
|
}
|
|
9599
|
-
const
|
|
9600
|
-
const
|
|
9601
|
-
const
|
|
9602
|
-
const
|
|
9603
|
-
|
|
9604
|
-
|
|
9605
|
-
|
|
9606
|
-
|
|
9607
|
-
|
|
9608
|
-
|
|
9609
|
-
|
|
10027
|
+
const reduce = group.reduce;
|
|
10028
|
+
const needsDocuments = reduce !== void 0 || projection.kind !== "none";
|
|
10029
|
+
const groupDocIds = [...new Set(wireGroups.flatMap((entry) => entry.scored.map((scored) => scored.docId)))];
|
|
10030
|
+
const documents = needsDocuments && groupDocIds.length > 0 ? await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, groupDocIds, allocation) : /* @__PURE__ */ new Map();
|
|
10031
|
+
return wireGroups.map((entry) => {
|
|
10032
|
+
const hits = entry.scored.map((scored) => ({
|
|
10033
|
+
id: scored.docId,
|
|
10034
|
+
score: scored.score ?? void 0,
|
|
10035
|
+
document: projectedGroupDocument(documents.get(scored.docId), projection)
|
|
10036
|
+
}));
|
|
10037
|
+
const result = { values: entry.values, hits };
|
|
10038
|
+
if (reduce !== void 0) {
|
|
10039
|
+
const folded = foldGroupReducer(
|
|
10040
|
+
reduce,
|
|
10041
|
+
entry.scored.map((scored) => ({ document: documents.get(scored.docId), score: scored.score ?? 0 }))
|
|
10042
|
+
);
|
|
10043
|
+
if (folded.reducerError !== void 0) {
|
|
10044
|
+
result.reducerError = folded.reducerError;
|
|
10045
|
+
} else {
|
|
10046
|
+
result.reduced = folded.reduced;
|
|
9610
10047
|
}
|
|
9611
10048
|
}
|
|
9612
|
-
|
|
9613
|
-
|
|
9614
|
-
|
|
9615
|
-
|
|
10049
|
+
return result;
|
|
10050
|
+
});
|
|
10051
|
+
}
|
|
10052
|
+
function projectedGroupDocument(document, projection) {
|
|
10053
|
+
if (document === void 0 || projection.kind === "none") {
|
|
10054
|
+
return {};
|
|
9616
10055
|
}
|
|
9617
|
-
|
|
9618
|
-
|
|
9619
|
-
|
|
9620
|
-
|
|
9621
|
-
|
|
9622
|
-
|
|
10056
|
+
return applyProjection(document, projection);
|
|
10057
|
+
}
|
|
10058
|
+
|
|
10059
|
+
// src/distribution/cluster-node/reads/pinned.ts
|
|
10060
|
+
function splitPinnedByReachability(pinnedIds, allocation) {
|
|
10061
|
+
const partitionCount = allocation.assignments.size;
|
|
10062
|
+
const verifiable = [];
|
|
10063
|
+
const unverifiable = /* @__PURE__ */ new Set();
|
|
10064
|
+
for (const docId of pinnedIds) {
|
|
10065
|
+
const partitionId = resolvePartitionId(docId, partitionCount);
|
|
10066
|
+
const assignment = allocation.assignments.get(partitionId);
|
|
10067
|
+
const nodeId = assignment === void 0 ? null : selectReplica(assignment, void 0, partitionId);
|
|
10068
|
+
if (nodeId === null) {
|
|
10069
|
+
unverifiable.add(docId);
|
|
10070
|
+
} else {
|
|
10071
|
+
verifiable.push(docId);
|
|
9623
10072
|
}
|
|
9624
10073
|
}
|
|
9625
|
-
|
|
9626
|
-
|
|
9627
|
-
|
|
9628
|
-
|
|
9629
|
-
|
|
9630
|
-
fused = distributedRRF([mergedText, mergedVector], { k: hybrid.k });
|
|
9631
|
-
} else if (hybrid !== null) {
|
|
9632
|
-
fused = distributedLinearCombination(mergedText, mergedVector, { alpha: clampAlpha(hybrid.alpha) });
|
|
9633
|
-
} else {
|
|
9634
|
-
fused = mergedText;
|
|
9635
|
-
}
|
|
9636
|
-
const truncated = fused.slice(offset, depth);
|
|
9637
|
-
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize) : null;
|
|
9638
|
-
let cursor = null;
|
|
9639
|
-
if (truncated.length > 0) {
|
|
9640
|
-
const lastEntry = truncated[truncated.length - 1];
|
|
9641
|
-
cursor = encodePageCursor({ anchor: lastEntry.docId, score: lastEntry.score, sortKey: null, sortSignature: null });
|
|
10074
|
+
return { verifiable, unverifiable };
|
|
10075
|
+
}
|
|
10076
|
+
async function dropUnstoredPinnedEntries(deps, indexName, params, distributed, allocation, projection, documents) {
|
|
10077
|
+
if (params.pinned === void 0 || params.searchAfter !== void 0) {
|
|
10078
|
+
return distributed.scored;
|
|
9642
10079
|
}
|
|
9643
|
-
|
|
9644
|
-
|
|
9645
|
-
|
|
9646
|
-
|
|
9647
|
-
|
|
9648
|
-
|
|
9649
|
-
coverage
|
|
9650
|
-
};
|
|
10080
|
+
const pinnedIds = new Set(params.pinned.map((entry) => entry.docId));
|
|
10081
|
+
const { verifiable, unverifiable } = splitPinnedByReachability(pinnedIds, allocation);
|
|
10082
|
+
const stored = projection.kind === "none" ? await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, verifiable, allocation) : documents;
|
|
10083
|
+
return distributed.scored.filter(
|
|
10084
|
+
(entry) => !pinnedIds.has(entry.docId) || unverifiable.has(entry.docId) || stored.has(entry.docId)
|
|
10085
|
+
);
|
|
9651
10086
|
}
|
|
9652
10087
|
function strictScatterGroups(allocation, indexName) {
|
|
9653
10088
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
@@ -9660,29 +10095,10 @@ function strictScatterGroups(allocation, indexName) {
|
|
|
9660
10095
|
}
|
|
9661
10096
|
return Array.from(routing.nodeToPartitions.entries(), ([nodeId, partitionIds]) => ({ nodeId, partitionIds }));
|
|
9662
10097
|
}
|
|
9663
|
-
var READ_FAILURE_BY_TRANSPORT_CODE = {
|
|
9664
|
-
[TransportErrorCodes.CONNECT_FAILED]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
9665
|
-
[TransportErrorCodes.PEER_UNAVAILABLE]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
9666
|
-
[TransportErrorCodes.TIMEOUT]: ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
9667
|
-
[TransportErrorCodes.MESSAGE_TOO_LARGE]: ErrorCodes.QUERY_ROUTING_FAILED,
|
|
9668
|
-
[TransportErrorCodes.DECODE_FAILED]: ErrorCodes.QUERY_ROUTING_FAILED
|
|
9669
|
-
};
|
|
9670
|
-
function readFailure(error, targetNodeId, indexName) {
|
|
9671
|
-
if (error instanceof NarsilError) {
|
|
9672
|
-
return error;
|
|
9673
|
-
}
|
|
9674
|
-
if (error instanceof TransportError) {
|
|
9675
|
-
return new NarsilError(
|
|
9676
|
-
READ_FAILURE_BY_TRANSPORT_CODE[error.code],
|
|
9677
|
-
`Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
9678
|
-
{ indexName, targetNodeId, transportCode: error.code }
|
|
9679
|
-
);
|
|
9680
|
-
}
|
|
9681
|
-
return error;
|
|
9682
|
-
}
|
|
9683
10098
|
async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
|
|
9684
|
-
const response = await sendToNode2(deps.config, targetNodeId, message
|
|
9685
|
-
|
|
10099
|
+
const response = await sendToNode2(deps.config, targetNodeId, message, {
|
|
10100
|
+
message: `Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
10101
|
+
details: { indexName }
|
|
9686
10102
|
});
|
|
9687
10103
|
const decoded = decode(response.payload);
|
|
9688
10104
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -9841,8 +10257,9 @@ async function listCluster(deps, indexName, params) {
|
|
|
9841
10257
|
const startTime = now();
|
|
9842
10258
|
const limit = clampListLimit(params.limit);
|
|
9843
10259
|
const signature = sortSignatureOf(params.sort);
|
|
10260
|
+
const binding = listBindingOf(params.filters);
|
|
9844
10261
|
if (params.cursor !== void 0) {
|
|
9845
|
-
requireMatchingCursor(decodePageCursor(params.cursor), params.cursor, signature, false);
|
|
10262
|
+
requireMatchingCursor(decodePageCursor(params.cursor), params.cursor, signature, false, binding);
|
|
9846
10263
|
}
|
|
9847
10264
|
const normalizedSort = normalizeSort(params.sort);
|
|
9848
10265
|
const sortFieldNames = normalizedSort.length === 0 ? null : normalizedSort.map((entry) => entry.field);
|
|
@@ -9905,7 +10322,8 @@ async function listCluster(deps, indexName, params) {
|
|
|
9905
10322
|
anchor: last.entry.docId,
|
|
9906
10323
|
score: null,
|
|
9907
10324
|
sortKey: sortFieldNames === null ? null : [...last.key ?? []],
|
|
9908
|
-
sortSignature: signature
|
|
10325
|
+
sortSignature: signature,
|
|
10326
|
+
binding
|
|
9909
10327
|
});
|
|
9910
10328
|
}
|
|
9911
10329
|
const documents = page.map((item) => ({
|
|
@@ -10032,6 +10450,7 @@ async function queryCluster(deps, indexName, params) {
|
|
|
10032
10450
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
10033
10451
|
};
|
|
10034
10452
|
const distributed = await distributedQuery(indexName, wireParams, queryDeps, routingConfig(deps.config.query));
|
|
10453
|
+
const projection = resolveProjection(params.document);
|
|
10035
10454
|
const documents = await fetchDistributedDocuments(
|
|
10036
10455
|
deps.config,
|
|
10037
10456
|
deps.nodeId,
|
|
@@ -10039,9 +10458,23 @@ async function queryCluster(deps, indexName, params) {
|
|
|
10039
10458
|
indexName,
|
|
10040
10459
|
distributed,
|
|
10041
10460
|
allocation,
|
|
10042
|
-
|
|
10461
|
+
projection
|
|
10043
10462
|
);
|
|
10044
|
-
|
|
10463
|
+
const scored = await dropUnstoredPinnedEntries(
|
|
10464
|
+
deps,
|
|
10465
|
+
indexName,
|
|
10466
|
+
params,
|
|
10467
|
+
distributed,
|
|
10468
|
+
allocation,
|
|
10469
|
+
projection,
|
|
10470
|
+
documents
|
|
10471
|
+
);
|
|
10472
|
+
const result = distributedResultToLocal({ ...distributed, scored }, documents);
|
|
10473
|
+
const groups = await assembleDistributedGroups(deps, indexName, params, distributed, allocation, projection);
|
|
10474
|
+
if (groups !== void 0) {
|
|
10475
|
+
result.groups = groups;
|
|
10476
|
+
}
|
|
10477
|
+
return result;
|
|
10045
10478
|
}
|
|
10046
10479
|
|
|
10047
10480
|
// src/distribution/cluster-node/clear.ts
|