@delali/narsil 0.2.3-next.34 → 0.2.3-next.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-GN7KF675.mjs → chunk-OOA7OBLA.mjs} +19 -1
- package/dist/{client-DQHB_swQ.d.ts → client-DAJ_UHOa.d.ts} +1 -1
- package/dist/client.d.ts +2 -2
- package/dist/{config-ChZ7lfqc.d.ts → config-CzTJJili.d.ts} +1 -1
- package/dist/distribution.d.ts +28 -4
- package/dist/distribution.mjs +273 -65
- package/dist/{engine-DScGlT2Z.d.ts → engine-DEtRLYpB.d.ts} +1 -1
- package/dist/index.browser.mjs +19 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +1 -1
- package/dist/react.d.ts +3 -3
- package/dist/{search-CQ-9bKxo.d.ts → search-Dt4dkzpf.d.ts} +24 -1
- package/dist/server.d.ts +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -79,7 +79,7 @@ const results = await narsil.query('products', {
|
|
|
79
79
|
})
|
|
80
80
|
```
|
|
81
81
|
|
|
82
|
-
Every hit holds the document, its id, and its BM25 score. `results.count` reports how many documents matched in total,
|
|
82
|
+
Every hit holds the document, its id, and its BM25 score. `results.count` reports how many documents matched in total, `results.elapsed` reports the query time in milliseconds, and `results.coverage` reports how many partitions the search read, which is how you spot a cluster that answered from part of its data.
|
|
83
83
|
|
|
84
84
|
## Features
|
|
85
85
|
|
|
@@ -129,7 +129,7 @@ The [specification](../spec/) defines the `.nrsl` format, the analysis pipeline,
|
|
|
129
129
|
|
|
130
130
|
## Distribution
|
|
131
131
|
|
|
132
|
-
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node creates, drops, clears, writes, updates, searches, lists, counts, and suggests across every partition, and the [cluster example](examples/cluster) runs three
|
|
132
|
+
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node creates, drops, clears, writes, updates, searches, lists, counts, and suggests across every partition, and the [cluster dashboard example](examples/cluster-dashboard) runs three nodes against etcd and shows what each partition does while you cut the links between them. The layer is experimental and its APIs change without notice, so pin an exact version before you depend on it. The [cluster guide](../../docs/cluster.md) covers the surface, and [`packages/spec/distribution`](../spec/distribution) specifies the contract every implementation follows.
|
|
133
133
|
|
|
134
134
|
## Search quality
|
|
135
135
|
|
|
@@ -8342,6 +8342,23 @@ function partitionsFor(manager, partitionIds) {
|
|
|
8342
8342
|
}
|
|
8343
8343
|
return partitions;
|
|
8344
8344
|
}
|
|
8345
|
+
function coverageFor(manager, partitionIds) {
|
|
8346
|
+
if (partitionIds === void 0) {
|
|
8347
|
+
return {
|
|
8348
|
+
totalPartitions: manager.partitionCount,
|
|
8349
|
+
queriedPartitions: manager.partitionCount,
|
|
8350
|
+
timedOutPartitions: 0,
|
|
8351
|
+
failedPartitions: 0
|
|
8352
|
+
};
|
|
8353
|
+
}
|
|
8354
|
+
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
8355
|
+
return {
|
|
8356
|
+
totalPartitions: partitionIds.length,
|
|
8357
|
+
queriedPartitions,
|
|
8358
|
+
timedOutPartitions: 0,
|
|
8359
|
+
failedPartitions: partitionIds.length - queriedPartitions
|
|
8360
|
+
};
|
|
8361
|
+
}
|
|
8345
8362
|
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
8346
8363
|
if (partitionIds === void 0) {
|
|
8347
8364
|
return void 0;
|
|
@@ -8803,7 +8820,8 @@ async function executeQuery(params, context) {
|
|
|
8803
8820
|
elapsed,
|
|
8804
8821
|
cursor: nextCursor,
|
|
8805
8822
|
facets,
|
|
8806
|
-
groups
|
|
8823
|
+
groups,
|
|
8824
|
+
coverage: coverageFor(manager, context.partitionIds)
|
|
8807
8825
|
};
|
|
8808
8826
|
}
|
|
8809
8827
|
async function executePreflight(params, context) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { e as TaskRecord, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-CnMLpfk-.js';
|
|
2
2
|
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions } from './schema-emTQfEiX.js';
|
|
3
|
-
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-
|
|
3
|
+
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-Dt4dkzpf.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This is the `fetch` implementation the client sends every request through.
|
package/dist/client.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-Dt4dkzpf.js';
|
|
2
2
|
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './options-BORJg5XB.js';
|
|
3
3
|
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-CnMLpfk-.js';
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-DAJ_UHOa.js';
|
|
5
5
|
import './schema-emTQfEiX.js';
|
|
6
6
|
import './adapters-79OIa4xm.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult } from './search-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult } from './search-Dt4dkzpf.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument } from './schema-emTQfEiX.js';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
1
|
+
import { N as NarsilConfig } from './config-CzTJJili.js';
|
|
2
2
|
import { A as AllocationTable, N as NodeRole, C as ClusterCoordinator, a as NodeCapacity } from './types-BhdINMtE.js';
|
|
3
3
|
export { b as AllocationConstraints, c as AllocationEvent, d as NodeEvent, e as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-BhdINMtE.js';
|
|
4
4
|
import { N as NodeTransport } from './types-D5OwhtsQ.js';
|
|
5
5
|
export { L as ListenHandler, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-D5OwhtsQ.js';
|
|
6
|
-
import { a as NarsilEventMap, N as Narsil } from './engine-
|
|
7
|
-
import { B as BatchResult, L as ListParams, g as ListResult, i as SuggestParams, j as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './search-
|
|
6
|
+
import { a as NarsilEventMap, N as Narsil } from './engine-DEtRLYpB.js';
|
|
7
|
+
import { B as BatchResult, L as ListParams, g as ListResult, i as SuggestParams, j as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './search-Dt4dkzpf.js';
|
|
8
8
|
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-emTQfEiX.js';
|
|
9
9
|
import './adapters-79OIa4xm.js';
|
|
10
10
|
|
|
@@ -51,9 +51,33 @@ interface ClusterNodeConfig {
|
|
|
51
51
|
engine?: NarsilConfig;
|
|
52
52
|
/** These replication settings override the engine's own, and anything you leave out keeps its value. */
|
|
53
53
|
replication?: Partial<ReplicationConfig>;
|
|
54
|
+
/** These settings govern how this node gathers each search from the cluster, and anything you leave out keeps its default. */
|
|
55
|
+
query?: ClusterQueryConfig;
|
|
54
56
|
/** This receives the failures the node reports while running on its own, away from any call you made. */
|
|
55
57
|
onError?: (error: Error) => void;
|
|
56
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* How this node answers a search when one of the nodes it asked runs slowly or
|
|
61
|
+
* drops out.
|
|
62
|
+
*
|
|
63
|
+
* The defaults let a search answer from the nodes that did reply, so set
|
|
64
|
+
* `allowPartialResults` to false where a caller must never act on an
|
|
65
|
+
* incomplete count. Every search reports what it read in
|
|
66
|
+
* {@link QueryResult.coverage} whichever way you set these.
|
|
67
|
+
*
|
|
68
|
+
* @public
|
|
69
|
+
*/
|
|
70
|
+
interface ClusterQueryConfig {
|
|
71
|
+
/**
|
|
72
|
+
* Set this true and a search returns the hits the reachable nodes gave,
|
|
73
|
+
* counting the rest in {@link QueryResult.coverage}, which is what a node
|
|
74
|
+
* does by default. Set it false and one partition that times out, errors, or
|
|
75
|
+
* has no active copy fails the whole search with `QUERY_PARTIAL_FAILURE`.
|
|
76
|
+
*/
|
|
77
|
+
allowPartialResults?: boolean;
|
|
78
|
+
/** A search waits this many milliseconds for each node before it counts that node's partitions as timed out, which is 5000 milliseconds where you name none. */
|
|
79
|
+
partitionTimeout?: number;
|
|
80
|
+
}
|
|
57
81
|
/**
|
|
58
82
|
* How an index is spread across the cluster when a node creates it.
|
|
59
83
|
*
|
|
@@ -441,4 +465,4 @@ interface ClusterEngineOptions {
|
|
|
441
465
|
*/
|
|
442
466
|
declare function clusterNodeEngine(node: ClusterNode, options?: ClusterEngineOptions): Narsil;
|
|
443
467
|
|
|
444
|
-
export { AllocationTable, ClusterCoordinator, type ClusterEngineOptions, type ClusterNamespace, type ClusterNode, type ClusterNodeConfig, type ClusterNodeInfo, type CreateIndexOptions, DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, NodeCapacity, NodeRole, NodeTransport, type ReplicationConfig, clusterNodeEngine, createClusterNode };
|
|
468
|
+
export { AllocationTable, ClusterCoordinator, type ClusterEngineOptions, type ClusterNamespace, type ClusterNode, type ClusterNodeConfig, type ClusterNodeInfo, type ClusterQueryConfig, type CreateIndexOptions, DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, NodeCapacity, NodeRole, NodeTransport, type ReplicationConfig, clusterNodeEngine, createClusterNode };
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
2
|
-
import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
|
|
1
|
+
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-OOA7OBLA.mjs';
|
|
2
|
+
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
|
|
3
3
|
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-RO4EFWE6.mjs';
|
|
4
4
|
import './chunk-TGP5M5R7.mjs';
|
|
5
5
|
import './chunk-OBXS7OZM.mjs';
|
|
@@ -237,8 +237,18 @@ function countNodeAssignments(assignments) {
|
|
|
237
237
|
}
|
|
238
238
|
return counts;
|
|
239
239
|
}
|
|
240
|
+
function countPrimaryAssignments(assignments) {
|
|
241
|
+
const counts = /* @__PURE__ */ new Map();
|
|
242
|
+
for (const assignment of assignments.values()) {
|
|
243
|
+
if (assignment.primary !== null) {
|
|
244
|
+
counts.set(assignment.primary, (counts.get(assignment.primary) ?? 0) + 1);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return counts;
|
|
248
|
+
}
|
|
240
249
|
function computeNodeWeights(nodes, assignments) {
|
|
241
250
|
const assignmentCounts = countNodeAssignments(assignments);
|
|
251
|
+
const primaryCounts = countPrimaryAssignments(assignments);
|
|
242
252
|
let totalCapacity = 0;
|
|
243
253
|
for (const node of nodes) {
|
|
244
254
|
totalCapacity += node.capacity.memoryBytes;
|
|
@@ -247,12 +257,14 @@ function computeNodeWeights(nodes, assignments) {
|
|
|
247
257
|
const weights = [];
|
|
248
258
|
for (const node of nodes) {
|
|
249
259
|
const partitionCount = assignmentCounts.get(node.nodeId) ?? 0;
|
|
260
|
+
const primaryCount = primaryCounts.get(node.nodeId) ?? 0;
|
|
250
261
|
const normalizedCapacity = averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1;
|
|
251
|
-
const weight = partitionCount / normalizedCapacity;
|
|
252
262
|
weights.push({
|
|
253
263
|
nodeId: node.nodeId,
|
|
254
|
-
weight,
|
|
264
|
+
weight: partitionCount / normalizedCapacity,
|
|
255
265
|
partitionCount,
|
|
266
|
+
primaryWeight: primaryCount / normalizedCapacity,
|
|
267
|
+
primaryCount,
|
|
256
268
|
capacity: node.capacity.memoryBytes
|
|
257
269
|
});
|
|
258
270
|
}
|
|
@@ -271,6 +283,11 @@ function findBestNode(candidates, weights, deciders, context) {
|
|
|
271
283
|
const sorted = [...candidates].sort((a, b) => {
|
|
272
284
|
const wA = weightByNodeId.get(a);
|
|
273
285
|
const wB = weightByNodeId.get(b);
|
|
286
|
+
if (context.role === "primary") {
|
|
287
|
+
const primaryA = wA?.primaryWeight ?? 0;
|
|
288
|
+
const primaryB = wB?.primaryWeight ?? 0;
|
|
289
|
+
if (primaryA !== primaryB) return primaryA - primaryB;
|
|
290
|
+
}
|
|
274
291
|
const weightA = wA?.weight ?? 0;
|
|
275
292
|
const weightB = wB?.weight ?? 0;
|
|
276
293
|
if (weightA !== weightB) return weightA - weightB;
|
|
@@ -380,6 +397,74 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
|
|
|
380
397
|
};
|
|
381
398
|
}
|
|
382
399
|
|
|
400
|
+
// src/distribution/cluster/allocator/leadership.ts
|
|
401
|
+
var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
|
|
402
|
+
function loadOf(nodeId, counts, shares) {
|
|
403
|
+
const share = shares.get(nodeId);
|
|
404
|
+
return (counts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
|
|
405
|
+
}
|
|
406
|
+
function heaviestLeader(shares, counts) {
|
|
407
|
+
let heaviest;
|
|
408
|
+
for (const nodeId of shares.keys()) {
|
|
409
|
+
if (heaviest === void 0 || loadOf(nodeId, counts, shares) > loadOf(heaviest, counts, shares)) {
|
|
410
|
+
heaviest = nodeId;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return heaviest;
|
|
414
|
+
}
|
|
415
|
+
function findSwap(assignments, heaviest, counts, shares) {
|
|
416
|
+
const heaviestLoad = loadOf(heaviest, counts, shares);
|
|
417
|
+
let best = null;
|
|
418
|
+
for (const assignment of assignments.values()) {
|
|
419
|
+
if (assignment.state !== "ACTIVE" || assignment.primary !== heaviest) {
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
for (const candidateNodeId of assignment.inSyncSet) {
|
|
423
|
+
const candidateLoad = loadOf(candidateNodeId, counts, shares);
|
|
424
|
+
if (heaviestLoad - candidateLoad < LEADERSHIP_IMBALANCE_THRESHOLD) {
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
if (best === null || candidateLoad < best.candidateLoad) {
|
|
428
|
+
best = { assignment, candidateNodeId, candidateLoad };
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return best;
|
|
433
|
+
}
|
|
434
|
+
function applySwap(swap, previousPrimary) {
|
|
435
|
+
const { assignment, candidateNodeId } = swap;
|
|
436
|
+
assignment.primary = candidateNodeId;
|
|
437
|
+
assignment.replicas = [...assignment.replicas.filter((nodeId) => nodeId !== candidateNodeId), previousPrimary];
|
|
438
|
+
assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== candidateNodeId);
|
|
439
|
+
assignment.primaryTerm += 1;
|
|
440
|
+
}
|
|
441
|
+
function rebalanceLeadership(assignments, shares) {
|
|
442
|
+
for (let swaps = 0; swaps < assignments.size; swaps++) {
|
|
443
|
+
const counts = countPrimaryAssignments(assignments);
|
|
444
|
+
const heaviest = heaviestLeader(shares, counts);
|
|
445
|
+
if (heaviest === void 0) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
const swap = findSwap(assignments, heaviest, counts, shares);
|
|
449
|
+
if (swap === null) {
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
applySwap(swap, heaviest);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
function capacityShares(nodes) {
|
|
456
|
+
const shares = /* @__PURE__ */ new Map();
|
|
457
|
+
let totalCapacity = 0;
|
|
458
|
+
for (const node of nodes) {
|
|
459
|
+
totalCapacity += node.capacity.memoryBytes;
|
|
460
|
+
}
|
|
461
|
+
const averageCapacity = nodes.length > 0 ? totalCapacity / nodes.length : 0;
|
|
462
|
+
for (const node of nodes) {
|
|
463
|
+
shares.set(node.nodeId, averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1);
|
|
464
|
+
}
|
|
465
|
+
return shares;
|
|
466
|
+
}
|
|
467
|
+
|
|
383
468
|
// src/distribution/cluster/allocator/rebalance.ts
|
|
384
469
|
function cloneAssignments(assignments) {
|
|
385
470
|
const cloned = /* @__PURE__ */ new Map();
|
|
@@ -401,7 +486,27 @@ function markUnassigned(assignment) {
|
|
|
401
486
|
assignment.inSyncSet = [];
|
|
402
487
|
assignment.state = "UNASSIGNED";
|
|
403
488
|
}
|
|
404
|
-
function
|
|
489
|
+
function primaryLoadOf(nodeId, primaryCounts, shares) {
|
|
490
|
+
const share = shares.get(nodeId);
|
|
491
|
+
return (primaryCounts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
|
|
492
|
+
}
|
|
493
|
+
function leastLoadedCandidate(candidates, primaryCounts, shares) {
|
|
494
|
+
let best;
|
|
495
|
+
for (const candidate of candidates) {
|
|
496
|
+
if (best === void 0) {
|
|
497
|
+
best = candidate;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
500
|
+
const bestLoad = primaryLoadOf(best, primaryCounts, shares);
|
|
501
|
+
const candidateLoad = primaryLoadOf(candidate, primaryCounts, shares);
|
|
502
|
+
if (candidateLoad < bestLoad || candidateLoad === bestLoad && candidate < best) {
|
|
503
|
+
best = candidate;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return best;
|
|
507
|
+
}
|
|
508
|
+
function handleLostNodes(assignments, activeNodeIds, shares) {
|
|
509
|
+
const primaryCounts = countPrimaryAssignments(assignments);
|
|
405
510
|
for (const assignment of assignments.values()) {
|
|
406
511
|
const primaryWasLost = assignment.primary !== null && !activeNodeIds.has(assignment.primary);
|
|
407
512
|
if (primaryWasLost) {
|
|
@@ -420,7 +525,12 @@ function handleLostNodes(assignments, activeNodeIds) {
|
|
|
420
525
|
markUnassigned(assignment);
|
|
421
526
|
continue;
|
|
422
527
|
}
|
|
423
|
-
const promoted = inSyncCandidates
|
|
528
|
+
const promoted = leastLoadedCandidate(inSyncCandidates, primaryCounts, shares);
|
|
529
|
+
if (promoted === void 0) {
|
|
530
|
+
markUnassigned(assignment);
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
primaryCounts.set(promoted, (primaryCounts.get(promoted) ?? 0) + 1);
|
|
424
534
|
assignment.primary = promoted;
|
|
425
535
|
assignment.replicas = assignment.replicas.filter((id) => id !== promoted);
|
|
426
536
|
assignment.inSyncSet = assignment.inSyncSet.filter((id) => id !== promoted);
|
|
@@ -453,7 +563,16 @@ function fillReplicaSlots(assignments, sortedNodes, nodeMap, replicationFactor,
|
|
|
453
563
|
}
|
|
454
564
|
}
|
|
455
565
|
}
|
|
456
|
-
function
|
|
566
|
+
function slotWeightOf(nodeId, shares) {
|
|
567
|
+
const share = shares.get(nodeId);
|
|
568
|
+
return 1 / (share === void 0 || share <= 0 ? 1 : share);
|
|
569
|
+
}
|
|
570
|
+
function moveNarrowsGap(mostLoaded, leastLoaded, shares) {
|
|
571
|
+
const gap = mostLoaded.weight - leastLoaded.weight;
|
|
572
|
+
const afterMove = Math.abs(gap - slotWeightOf(mostLoaded.nodeId, shares) - slotWeightOf(leastLoaded.nodeId, shares));
|
|
573
|
+
return afterMove < gap;
|
|
574
|
+
}
|
|
575
|
+
function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares) {
|
|
457
576
|
let totalSlots = 0;
|
|
458
577
|
for (const assignment of assignments.values()) {
|
|
459
578
|
if (assignment.primary !== null) totalSlots++;
|
|
@@ -469,35 +588,60 @@ function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, dec
|
|
|
469
588
|
if (mostLoaded.weight - leastLoaded.weight < REBALANCE_THRESHOLD) {
|
|
470
589
|
break;
|
|
471
590
|
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
if (moved) break;
|
|
475
|
-
const replicaIndex = assignment.replicas.indexOf(mostLoaded.nodeId);
|
|
476
|
-
if (replicaIndex >= 0) {
|
|
477
|
-
const removedReplica = assignment.replicas[replicaIndex];
|
|
478
|
-
assignment.replicas.splice(replicaIndex, 1);
|
|
479
|
-
const nodeAssignmentCounts = countNodeAssignments(assignments);
|
|
480
|
-
const moveContext = {
|
|
481
|
-
partitionId,
|
|
482
|
-
role: "replica",
|
|
483
|
-
currentAssignment: assignment,
|
|
484
|
-
allAssignments: assignments,
|
|
485
|
-
nodeAssignmentCounts,
|
|
486
|
-
nodes: nodeMap,
|
|
487
|
-
constraints
|
|
488
|
-
};
|
|
489
|
-
const target = findBestNode([leastLoaded.nodeId], weights, deciders, moveContext);
|
|
490
|
-
if (target !== null) {
|
|
491
|
-
assignment.replicas.push(target);
|
|
492
|
-
moved = true;
|
|
493
|
-
} else {
|
|
494
|
-
assignment.replicas.splice(replicaIndex, 0, removedReplica);
|
|
495
|
-
}
|
|
496
|
-
}
|
|
591
|
+
if (!moveNarrowsGap(mostLoaded, leastLoaded, shares)) {
|
|
592
|
+
break;
|
|
497
593
|
}
|
|
594
|
+
const move = {
|
|
595
|
+
fromNodeId: mostLoaded.nodeId,
|
|
596
|
+
toNodeId: leastLoaded.nodeId,
|
|
597
|
+
weights,
|
|
598
|
+
nodeMap,
|
|
599
|
+
constraints,
|
|
600
|
+
deciders
|
|
601
|
+
};
|
|
602
|
+
const laggingFirst = moveOneReplica(assignments, move, true);
|
|
603
|
+
const moved = laggingFirst || moveOneReplica(assignments, move, false);
|
|
498
604
|
if (!moved) break;
|
|
499
605
|
}
|
|
500
606
|
}
|
|
607
|
+
function moveOneReplica(assignments, move, laggingOnly) {
|
|
608
|
+
for (const [partitionId, assignment] of assignments) {
|
|
609
|
+
const replicaIndex = assignment.replicas.indexOf(move.fromNodeId);
|
|
610
|
+
if (replicaIndex < 0) {
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
if (laggingOnly && assignment.inSyncSet.includes(move.fromNodeId)) {
|
|
614
|
+
continue;
|
|
615
|
+
}
|
|
616
|
+
const removedReplica = assignment.replicas[replicaIndex];
|
|
617
|
+
assignment.replicas.splice(replicaIndex, 1);
|
|
618
|
+
const moveContext = {
|
|
619
|
+
partitionId,
|
|
620
|
+
role: "replica",
|
|
621
|
+
currentAssignment: assignment,
|
|
622
|
+
allAssignments: assignments,
|
|
623
|
+
nodeAssignmentCounts: countNodeAssignments(assignments),
|
|
624
|
+
nodes: move.nodeMap,
|
|
625
|
+
constraints: move.constraints
|
|
626
|
+
};
|
|
627
|
+
const target = findBestNode([move.toNodeId], move.weights, move.deciders, moveContext);
|
|
628
|
+
if (target === null) {
|
|
629
|
+
assignment.replicas.splice(replicaIndex, 0, removedReplica);
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
assignment.replicas.push(target);
|
|
633
|
+
return true;
|
|
634
|
+
}
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
function pruneInSyncSets(assignments) {
|
|
638
|
+
for (const assignment of assignments.values()) {
|
|
639
|
+
const stale = assignment.inSyncSet.some((nodeId) => !assignment.replicas.includes(nodeId));
|
|
640
|
+
if (stale) {
|
|
641
|
+
assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => assignment.replicas.includes(nodeId));
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
}
|
|
501
645
|
function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
|
|
502
646
|
const sortedNodes = [...nodes].sort((a, b) => {
|
|
503
647
|
if (a.nodeId < b.nodeId) return -1;
|
|
@@ -510,9 +654,12 @@ function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
|
|
|
510
654
|
nodeMap.set(node.nodeId, node);
|
|
511
655
|
}
|
|
512
656
|
const assignments = cloneAssignments(currentTable.assignments);
|
|
513
|
-
|
|
657
|
+
const shares = capacityShares(sortedNodes);
|
|
658
|
+
handleLostNodes(assignments, activeNodeIds, shares);
|
|
514
659
|
fillReplicaSlots(assignments, sortedNodes, nodeMap, currentTable.replicationFactor, constraints, deciders);
|
|
515
|
-
rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders);
|
|
660
|
+
rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares);
|
|
661
|
+
rebalanceLeadership(assignments, shares);
|
|
662
|
+
pruneInSyncSets(assignments);
|
|
516
663
|
const warnings = collectReplicationWarnings(assignments, currentTable.replicationFactor);
|
|
517
664
|
return {
|
|
518
665
|
table: {
|
|
@@ -940,15 +1087,6 @@ async function handleBootstrapCompleteMessage(message, respond, coordinator, con
|
|
|
940
1087
|
});
|
|
941
1088
|
}
|
|
942
1089
|
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
943
|
-
function ensurePrimaryInSyncSet(assignment, inSyncSet) {
|
|
944
|
-
if (assignment.primary === null) {
|
|
945
|
-
return inSyncSet;
|
|
946
|
-
}
|
|
947
|
-
if (inSyncSet.includes(assignment.primary)) {
|
|
948
|
-
return inSyncSet;
|
|
949
|
-
}
|
|
950
|
-
return [assignment.primary, ...inSyncSet];
|
|
951
|
-
}
|
|
952
1090
|
async function processBootstrapComplete(payload, coordinator) {
|
|
953
1091
|
for (let attempt = 0; attempt < BOOTSTRAP_CAS_ATTEMPTS; attempt++) {
|
|
954
1092
|
const table = await coordinator.getAllocation(payload.indexName);
|
|
@@ -968,13 +1106,8 @@ async function processBootstrapComplete(payload, coordinator) {
|
|
|
968
1106
|
if (assignment.state !== "INITIALISING") {
|
|
969
1107
|
return false;
|
|
970
1108
|
}
|
|
971
|
-
const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
|
|
972
1109
|
const updatedAssignments = new Map(table.assignments);
|
|
973
|
-
updatedAssignments.set(payload.partitionId, {
|
|
974
|
-
...assignment,
|
|
975
|
-
state: "ACTIVE",
|
|
976
|
-
inSyncSet: ensurePrimaryInSyncSet(assignment, baseInSyncSet)
|
|
977
|
-
});
|
|
1110
|
+
updatedAssignments.set(payload.partitionId, { ...assignment, state: "ACTIVE" });
|
|
978
1111
|
const written = await coordinator.putAllocation(
|
|
979
1112
|
payload.indexName,
|
|
980
1113
|
{ ...table, version: table.version + 1, assignments: updatedAssignments },
|
|
@@ -1086,7 +1219,7 @@ function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId,
|
|
|
1086
1219
|
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1087
1220
|
});
|
|
1088
1221
|
}
|
|
1089
|
-
function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive) {
|
|
1222
|
+
function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError) {
|
|
1090
1223
|
const decoded = decodeMessage(message);
|
|
1091
1224
|
const payload = validateInsyncAddPayload(decoded);
|
|
1092
1225
|
if (payload === null) {
|
|
@@ -1106,6 +1239,9 @@ function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, is
|
|
|
1106
1239
|
}
|
|
1107
1240
|
const confirmPayload = await handleInsyncAdmission(payload, coordinator);
|
|
1108
1241
|
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1242
|
+
if (confirmPayload.accepted && isActive()) {
|
|
1243
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1244
|
+
}
|
|
1109
1245
|
});
|
|
1110
1246
|
}
|
|
1111
1247
|
function handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive) {
|
|
@@ -1173,7 +1309,7 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
1173
1309
|
if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
|
|
1174
1310
|
handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive);
|
|
1175
1311
|
} else if (message.type === ReplicationMessageTypes.INSYNC_ADD) {
|
|
1176
|
-
handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive);
|
|
1312
|
+
handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError);
|
|
1177
1313
|
} else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
|
|
1178
1314
|
handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive);
|
|
1179
1315
|
}
|
|
@@ -6293,7 +6429,8 @@ function distributedResultToLocal(result, documents = /* @__PURE__ */ new Map())
|
|
|
6293
6429
|
count: result.totalHits,
|
|
6294
6430
|
elapsed: 0,
|
|
6295
6431
|
cursor: result.cursor ?? void 0,
|
|
6296
|
-
facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0
|
|
6432
|
+
facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0,
|
|
6433
|
+
coverage: result.coverage
|
|
6297
6434
|
};
|
|
6298
6435
|
}
|
|
6299
6436
|
function convertLocalSortToWire(sort) {
|
|
@@ -8594,9 +8731,11 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
8594
8731
|
const routing = selectReplicasForQuery(allocationTable, randomSelector);
|
|
8595
8732
|
const totalPartitions = allocationTable.assignments.size;
|
|
8596
8733
|
if (routing.unavailablePartitions.length > 0 && !resolvedConfig.allowPartialResults) {
|
|
8597
|
-
throw new NarsilError(
|
|
8598
|
-
|
|
8599
|
-
|
|
8734
|
+
throw new NarsilError(
|
|
8735
|
+
ErrorCodes.QUERY_PARTIAL_FAILURE,
|
|
8736
|
+
"No active replica serves one or more partitions, and this node refuses partial results",
|
|
8737
|
+
{ unavailablePartitions: routing.unavailablePartitions }
|
|
8738
|
+
);
|
|
8600
8739
|
}
|
|
8601
8740
|
const isHybrid = params.hybrid !== null && params.term !== null && params.vector !== null;
|
|
8602
8741
|
if (isHybrid && params.searchAfter !== null) {
|
|
@@ -8807,17 +8946,20 @@ function worstCaseCoverage(a, b) {
|
|
|
8807
8946
|
failedPartitions: failed
|
|
8808
8947
|
};
|
|
8809
8948
|
}
|
|
8810
|
-
|
|
8811
|
-
const allocation = await deps.config.coordinator.getAllocation(indexName);
|
|
8949
|
+
function servesAnyPartition(allocation) {
|
|
8812
8950
|
if (allocation === null || allocation.assignments.size === 0) {
|
|
8813
|
-
return
|
|
8951
|
+
return false;
|
|
8814
8952
|
}
|
|
8815
8953
|
for (const [, assignment] of allocation.assignments) {
|
|
8816
8954
|
if (assignment.state === "ACTIVE") {
|
|
8817
|
-
return
|
|
8955
|
+
return true;
|
|
8818
8956
|
}
|
|
8819
8957
|
}
|
|
8820
|
-
return
|
|
8958
|
+
return false;
|
|
8959
|
+
}
|
|
8960
|
+
async function activeAllocation(deps, indexName) {
|
|
8961
|
+
const allocation = await deps.config.coordinator.getAllocation(indexName);
|
|
8962
|
+
return servesAnyPartition(allocation) ? allocation : null;
|
|
8821
8963
|
}
|
|
8822
8964
|
function strictScatterGroups(allocation, indexName) {
|
|
8823
8965
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
@@ -8830,8 +8972,30 @@ function strictScatterGroups(allocation, indexName) {
|
|
|
8830
8972
|
}
|
|
8831
8973
|
return Array.from(routing.nodeToPartitions.entries(), ([nodeId, partitionIds]) => ({ nodeId, partitionIds }));
|
|
8832
8974
|
}
|
|
8975
|
+
var READ_FAILURE_BY_TRANSPORT_CODE = {
|
|
8976
|
+
[TransportErrorCodes.CONNECT_FAILED]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
8977
|
+
[TransportErrorCodes.PEER_UNAVAILABLE]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
8978
|
+
[TransportErrorCodes.TIMEOUT]: ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
8979
|
+
[TransportErrorCodes.MESSAGE_TOO_LARGE]: ErrorCodes.QUERY_ROUTING_FAILED,
|
|
8980
|
+
[TransportErrorCodes.DECODE_FAILED]: ErrorCodes.QUERY_ROUTING_FAILED
|
|
8981
|
+
};
|
|
8982
|
+
function readFailure(error, targetNodeId, indexName) {
|
|
8983
|
+
if (error instanceof NarsilError) {
|
|
8984
|
+
return error;
|
|
8985
|
+
}
|
|
8986
|
+
if (error instanceof TransportError) {
|
|
8987
|
+
return new NarsilError(
|
|
8988
|
+
READ_FAILURE_BY_TRANSPORT_CODE[error.code],
|
|
8989
|
+
`Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
8990
|
+
{ indexName, targetNodeId, transportCode: error.code }
|
|
8991
|
+
);
|
|
8992
|
+
}
|
|
8993
|
+
return error;
|
|
8994
|
+
}
|
|
8833
8995
|
async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
|
|
8834
|
-
const response = await sendToNode2(deps.config, targetNodeId, message)
|
|
8996
|
+
const response = await sendToNode2(deps.config, targetNodeId, message).catch((error) => {
|
|
8997
|
+
throw readFailure(error, targetNodeId, indexName);
|
|
8998
|
+
});
|
|
8835
8999
|
const decoded = decode(response.payload);
|
|
8836
9000
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
8837
9001
|
throw new NarsilError(
|
|
@@ -9154,11 +9318,36 @@ async function readClusterDocuments(deps, indexName, docIds) {
|
|
|
9154
9318
|
}
|
|
9155
9319
|
return readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, docIds, allocation);
|
|
9156
9320
|
}
|
|
9321
|
+
function routingConfig(query) {
|
|
9322
|
+
if (query === void 0) {
|
|
9323
|
+
return void 0;
|
|
9324
|
+
}
|
|
9325
|
+
const config = {};
|
|
9326
|
+
if (query.allowPartialResults !== void 0) {
|
|
9327
|
+
config.allowPartialResults = query.allowPartialResults;
|
|
9328
|
+
}
|
|
9329
|
+
if (query.partitionTimeout !== void 0) {
|
|
9330
|
+
config.partitionTimeout = query.partitionTimeout;
|
|
9331
|
+
}
|
|
9332
|
+
return config;
|
|
9333
|
+
}
|
|
9334
|
+
async function queryLocalCopy(deps, indexName, params, allocation) {
|
|
9335
|
+
const result = await deps.engine.query(indexName, params);
|
|
9336
|
+
if (allocation === null || allocation.assignments.size === 0) {
|
|
9337
|
+
return result;
|
|
9338
|
+
}
|
|
9339
|
+
const totalPartitions = allocation.assignments.size;
|
|
9340
|
+
return {
|
|
9341
|
+
...result,
|
|
9342
|
+
coverage: { totalPartitions, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: totalPartitions }
|
|
9343
|
+
};
|
|
9344
|
+
}
|
|
9157
9345
|
async function queryCluster(deps, indexName, params) {
|
|
9158
|
-
const
|
|
9159
|
-
if (
|
|
9160
|
-
return deps
|
|
9346
|
+
const table = await deps.config.coordinator.getAllocation(indexName);
|
|
9347
|
+
if (!servesAnyPartition(table)) {
|
|
9348
|
+
return queryLocalCopy(deps, indexName, params, table);
|
|
9161
9349
|
}
|
|
9350
|
+
const allocation = table;
|
|
9162
9351
|
const wireParams = localParamsToWire(params);
|
|
9163
9352
|
const queryDeps = {
|
|
9164
9353
|
transport: deps.config.transport,
|
|
@@ -9166,7 +9355,7 @@ async function queryCluster(deps, indexName, params) {
|
|
|
9166
9355
|
getAllocation: (idx) => deps.config.coordinator.getAllocation(idx),
|
|
9167
9356
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
9168
9357
|
};
|
|
9169
|
-
const distributed = await distributedQuery(indexName, wireParams, queryDeps);
|
|
9358
|
+
const distributed = await distributedQuery(indexName, wireParams, queryDeps, routingConfig(deps.config.query));
|
|
9170
9359
|
const documents = await fetchDistributedDocuments(
|
|
9171
9360
|
deps.config,
|
|
9172
9361
|
deps.nodeId,
|
|
@@ -9627,6 +9816,25 @@ function validateClusterNodeConfig(config) {
|
|
|
9627
9816
|
);
|
|
9628
9817
|
}
|
|
9629
9818
|
}
|
|
9819
|
+
if (config.query !== void 0) {
|
|
9820
|
+
const { allowPartialResults, partitionTimeout } = config.query;
|
|
9821
|
+
if (allowPartialResults !== void 0 && typeof allowPartialResults !== "boolean") {
|
|
9822
|
+
throw new NarsilError(
|
|
9823
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9824
|
+
"ClusterNodeConfig.query.allowPartialResults must be a boolean",
|
|
9825
|
+
{
|
|
9826
|
+
allowPartialResults
|
|
9827
|
+
}
|
|
9828
|
+
);
|
|
9829
|
+
}
|
|
9830
|
+
if (partitionTimeout !== void 0 && (!Number.isInteger(partitionTimeout) || partitionTimeout <= 0)) {
|
|
9831
|
+
throw new NarsilError(
|
|
9832
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9833
|
+
"ClusterNodeConfig.query.partitionTimeout must be a positive integer",
|
|
9834
|
+
{ partitionTimeout }
|
|
9835
|
+
);
|
|
9836
|
+
}
|
|
9837
|
+
}
|
|
9630
9838
|
}
|
|
9631
9839
|
|
|
9632
9840
|
// src/distribution/cluster-node/index.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { I as IndexInfo, f as IndexStats, P as PartitionStatsResult, B as BatchResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './search-
|
|
2
|
+
import { I as IndexInfo, f as IndexStats, P as PartitionStatsResult, B as BatchResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './search-Dt4dkzpf.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument, a as InsertOptions, P as PartitionConfig } from './schema-emTQfEiX.js';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/index.browser.mjs
CHANGED
|
@@ -22181,6 +22181,23 @@ function partitionsFor(manager, partitionIds) {
|
|
|
22181
22181
|
}
|
|
22182
22182
|
return partitions;
|
|
22183
22183
|
}
|
|
22184
|
+
function coverageFor(manager, partitionIds) {
|
|
22185
|
+
if (partitionIds === void 0) {
|
|
22186
|
+
return {
|
|
22187
|
+
totalPartitions: manager.partitionCount,
|
|
22188
|
+
queriedPartitions: manager.partitionCount,
|
|
22189
|
+
timedOutPartitions: 0,
|
|
22190
|
+
failedPartitions: 0
|
|
22191
|
+
};
|
|
22192
|
+
}
|
|
22193
|
+
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
22194
|
+
return {
|
|
22195
|
+
totalPartitions: partitionIds.length,
|
|
22196
|
+
queriedPartitions,
|
|
22197
|
+
timedOutPartitions: 0,
|
|
22198
|
+
failedPartitions: partitionIds.length - queriedPartitions
|
|
22199
|
+
};
|
|
22200
|
+
}
|
|
22184
22201
|
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
22185
22202
|
if (partitionIds === void 0) {
|
|
22186
22203
|
return void 0;
|
|
@@ -22642,7 +22659,8 @@ async function executeQuery(params, context) {
|
|
|
22642
22659
|
elapsed,
|
|
22643
22660
|
cursor: nextCursor,
|
|
22644
22661
|
facets,
|
|
22645
|
-
groups
|
|
22662
|
+
groups,
|
|
22663
|
+
coverage: coverageFor(manager, context.partitionIds)
|
|
22646
22664
|
};
|
|
22647
22665
|
}
|
|
22648
22666
|
async function executePreflight(params, context) {
|
package/dist/index.d.ts
CHANGED
|
@@ -2,11 +2,11 @@ import { c as StopWordOverride, C as CustomTokenizer } from './schema-emTQfEiX.j
|
|
|
2
2
|
export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, b as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode } from './schema-emTQfEiX.js';
|
|
3
3
|
import { L as LanguageModule } from './language-DB8KG1z5.js';
|
|
4
4
|
export { T as TokenizerConfig } from './language-DB8KG1z5.js';
|
|
5
|
-
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, k as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, l as FacetResult, m as FieldFilter, n as FilterExpression, G as GeoFilter, o as GeoPolygonFilter, p as GeoRadiusFilter, q as GroupConfig, r as GroupReducer, s as GroupResult, H as HighlightConfig, t as HighlightMatch, u as Hit, v as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, w as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, x as PresenceFilter, y as ProcessMemoryReport, Q as QueryParams, e as QueryResult,
|
|
6
|
-
import { N as NarsilConfig } from './config-
|
|
7
|
-
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as InsertContext, b as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, c as StaleAnalysis, U as UpdateContext, W as WorkerConfig, d as WorkerContext } from './config-
|
|
8
|
-
import { N as Narsil } from './engine-
|
|
9
|
-
export { a as NarsilEventMap } from './engine-
|
|
5
|
+
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, k as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, l as FacetResult, m as FieldFilter, n as FilterExpression, G as GeoFilter, o as GeoPolygonFilter, p as GeoRadiusFilter, q as GroupConfig, r as GroupReducer, s as GroupResult, H as HighlightConfig, t as HighlightMatch, u as Hit, v as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, w as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, x as PresenceFilter, y as ProcessMemoryReport, z as QueryCoverage, Q as QueryParams, e as QueryResult, J as ScoreComponents, K as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, O as SortField, R as SortSpec, T as StringFilter, i as SuggestParams, j as SuggestResult, U as TermMatchPolicy, V as VectorMaintenanceResult, W as VectorQueryConfig } from './search-Dt4dkzpf.js';
|
|
6
|
+
import { N as NarsilConfig } from './config-CzTJJili.js';
|
|
7
|
+
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as InsertContext, b as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, c as StaleAnalysis, U as UpdateContext, W as WorkerConfig, d as WorkerContext } from './config-CzTJJili.js';
|
|
8
|
+
import { N as Narsil } from './engine-DEtRLYpB.js';
|
|
9
|
+
export { a as NarsilEventMap } from './engine-DEtRLYpB.js';
|
|
10
10
|
export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
|
|
11
11
|
|
|
12
12
|
/**
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-OOA7OBLA.mjs';
|
|
2
2
|
import './chunk-36ER4WCG.mjs';
|
|
3
3
|
export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache } from './chunk-RO4EFWE6.mjs';
|
|
4
4
|
import './chunk-TGP5M5R7.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createDurableDirectory, writeSegmentedCheckpoint } from '../../chunk-
|
|
1
|
+
import { createDurableDirectory, writeSegmentedCheckpoint } from '../../chunk-OOA7OBLA.mjs';
|
|
2
2
|
import '../../chunk-36ER4WCG.mjs';
|
|
3
3
|
import '../../chunk-RO4EFWE6.mjs';
|
|
4
4
|
import '../../chunk-TGP5M5R7.mjs';
|
package/dist/react.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, Q as QueryParams, h as PreflightResult, e as QueryResult, i as SuggestParams, j as SuggestResult } from './search-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-
|
|
1
|
+
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, Q as QueryParams, h as PreflightResult, e as QueryResult, i as SuggestParams, j as SuggestResult } from './search-Dt4dkzpf.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-Dt4dkzpf.js';
|
|
3
3
|
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-CnMLpfk-.js';
|
|
4
4
|
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-CnMLpfk-.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
|
-
import { N as NarsilClient, I as ImportSource } from './client-
|
|
6
|
+
import { N as NarsilClient, I as ImportSource } from './client-DAJ_UHOa.js';
|
|
7
7
|
import { A as AnyDocument } from './schema-emTQfEiX.js';
|
|
8
8
|
import './adapters-79OIa4xm.js';
|
|
9
9
|
|
|
@@ -229,6 +229,29 @@ interface QueryResult<T = AnyDocument> {
|
|
|
229
229
|
groups?: GroupResult[];
|
|
230
230
|
/** This turns true when the index's terms came from an older analysis than its language module produces now. */
|
|
231
231
|
analysisStale?: boolean;
|
|
232
|
+
/** The search read this much of the index, which is how you spot a degraded answer. */
|
|
233
|
+
coverage: QueryCoverage;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* How much of an index one search read, which is what you check before you
|
|
237
|
+
* trust a count or a ranking.
|
|
238
|
+
*
|
|
239
|
+
* An embedded engine reads every partition it holds, so `queriedPartitions`
|
|
240
|
+
* matches `totalPartitions` there and the other two stay zero. A search across
|
|
241
|
+
* a cluster may find a node slow or unreachable, which it counts here while
|
|
242
|
+
* still returning the hits the other nodes gave it.
|
|
243
|
+
*
|
|
244
|
+
* @public
|
|
245
|
+
*/
|
|
246
|
+
interface QueryCoverage {
|
|
247
|
+
/** The search should have read this many partitions. */
|
|
248
|
+
totalPartitions: number;
|
|
249
|
+
/** The search read this many of them. */
|
|
250
|
+
queriedPartitions: number;
|
|
251
|
+
/** The search counts a partition here when the node holding it took longer than the query allowed. */
|
|
252
|
+
timedOutPartitions: number;
|
|
253
|
+
/** The search counts a partition here when the node holding it returned an error, and when no active replica served it. */
|
|
254
|
+
failedPartitions: number;
|
|
232
255
|
}
|
|
233
256
|
/**
|
|
234
257
|
* One matching document, with the score that ranked it.
|
|
@@ -996,4 +1019,4 @@ interface ListParams {
|
|
|
996
1019
|
document?: DocumentProjection;
|
|
997
1020
|
}
|
|
998
1021
|
|
|
999
|
-
export { type ArrayFilter as A, type BatchResult as B, type ClientErrorCode as C, type DocumentProjection as D, type ErrorCode as E, type FacetConfig as F, type GeoFilter as G, type HighlightConfig as H, type IndexInfo as I, type
|
|
1022
|
+
export { type ArrayFilter as A, type BatchResult as B, type ClientErrorCode as C, type DocumentProjection as D, type ErrorCode as E, type FacetConfig as F, type GeoFilter as G, type HighlightConfig as H, type IndexInfo as I, type ScoreComponents as J, type SearchMode as K, type ListParams as L, type MemoryStats as M, NarsilError as N, type SortField as O, type PartitionStatsResult as P, type QueryParams as Q, type SortSpec as R, type ServerErrorCode as S, type StringFilter as T, type TermMatchPolicy as U, type VectorMaintenanceResult as V, type VectorQueryConfig as W, ClientErrorCodes as a, ErrorCodes as b, type NarsilErrorCode as c, ServerErrorCodes as d, type QueryResult as e, type IndexStats as f, type ListResult as g, type PreflightResult as h, type SuggestParams as i, type SuggestResult as j, type ComparisonFilter as k, type FacetResult as l, type FieldFilter as m, type FilterExpression as n, type GeoPolygonFilter as o, type GeoRadiusFilter as p, type GroupConfig as q, type GroupReducer as r, type GroupResult as s, type HighlightMatch as t, type Hit as u, type HybridConfig as v, type ListedDocument as w, type PresenceFilter as x, type ProcessMemoryReport as y, type QueryCoverage as z };
|
package/dist/server.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { b as ServerOptions, N as NarsilServer } from './options-BORJg5XB.js';
|
|
2
2
|
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, E as ErrorEnvelope, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, c as RequestContext, d as RequestDenial, S as SERVER_CAPABILITIES, e as ServerLimits, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './options-BORJg5XB.js';
|
|
3
|
-
import { N as Narsil } from './engine-
|
|
3
|
+
import { N as Narsil } from './engine-DEtRLYpB.js';
|
|
4
4
|
import { h as TaskStore, e as TaskRecord, d as TaskProgress, b as ImportResult } from './tasks-CnMLpfk-.js';
|
|
5
5
|
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-CnMLpfk-.js';
|
|
6
|
-
export { d as ServerErrorCodes } from './search-
|
|
6
|
+
export { d as ServerErrorCodes } from './search-Dt4dkzpf.js';
|
|
7
7
|
import './adapters-79OIa4xm.js';
|
|
8
8
|
import './schema-emTQfEiX.js';
|
|
9
9
|
|
package/package.json
CHANGED