@delali/narsil 0.2.3-next.33 → 0.2.3-next.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-C6VVLMRX.mjs → chunk-36ER4WCG.mjs} +1 -0
- package/dist/{chunk-MFFAJ2NR.mjs → chunk-GODECMDT.mjs} +1 -1
- package/dist/{chunk-TEAJO7CU.mjs → chunk-OOA7OBLA.mjs} +29 -3
- package/dist/{client-DQHB_swQ.d.ts → client-DAJ_UHOa.d.ts} +1 -1
- package/dist/client.d.ts +2 -2
- package/dist/{config-ChZ7lfqc.d.ts → config-CzTJJili.d.ts} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +94 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +10 -3
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +30 -6
- package/dist/distribution.mjs +1910 -1042
- package/dist/{engine-DScGlT2Z.d.ts → engine-DEtRLYpB.d.ts} +1 -1
- package/dist/index.browser.mjs +19 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/react.d.ts +3 -3
- package/dist/{search-CQ-9bKxo.d.ts → search-Dt4dkzpf.d.ts} +24 -1
- package/dist/server.d.ts +2 -2
- package/dist/{types-Bfhhnh3G.d.ts → types-BhdINMtE.d.ts} +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -79,7 +79,7 @@ const results = await narsil.query('products', {
|
|
|
79
79
|
})
|
|
80
80
|
```
|
|
81
81
|
|
|
82
|
-
Every hit holds the document, its id, and its BM25 score. `results.count` reports how many documents matched in total,
|
|
82
|
+
Every hit holds the document, its id, and its BM25 score. `results.count` reports how many documents matched in total, `results.elapsed` reports the query time in milliseconds, and `results.coverage` reports how many partitions the search read, which is how you spot a cluster that answered from part of its data.
|
|
83
83
|
|
|
84
84
|
## Features
|
|
85
85
|
|
|
@@ -129,7 +129,7 @@ The [specification](../spec/) defines the `.nrsl` format, the analysis pipeline,
|
|
|
129
129
|
|
|
130
130
|
## Distribution
|
|
131
131
|
|
|
132
|
-
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node creates, drops, clears, writes, updates, searches, lists, counts, and suggests across every partition, and the [cluster example](examples/cluster) runs three
|
|
132
|
+
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node creates, drops, clears, writes, updates, searches, lists, counts, and suggests across every partition, and the [cluster dashboard example](examples/cluster-dashboard) runs three nodes against etcd and shows what each partition does while you cut the links between them. The layer is experimental and its APIs change without notice, so pin an exact version before you depend on it. The [cluster guide](../../docs/cluster.md) covers the surface, and [`packages/spec/distribution`](../spec/distribution) specifies the contract every implementation follows.
|
|
133
133
|
|
|
134
134
|
## Search quality
|
|
135
135
|
|
|
@@ -35,6 +35,7 @@ var ReplicationMessageTypes = {
|
|
|
35
35
|
SNAPSHOT_CHUNK: "replication.snapshot_chunk",
|
|
36
36
|
SNAPSHOT_END: "replication.snapshot_end",
|
|
37
37
|
SNAPSHOT_SYNC_REQUEST: "replication.snapshot_sync_request",
|
|
38
|
+
INSYNC_ADD: "replication.insync_add",
|
|
38
39
|
INSYNC_REMOVE: "replication.insync_remove",
|
|
39
40
|
INSYNC_CONFIRM: "replication.insync_confirm"
|
|
40
41
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from './chunk-36ER4WCG.mjs';
|
|
2
2
|
import { encode, decode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
4
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ReplicationMessageTypes } from './chunk-
|
|
1
|
+
import { ReplicationMessageTypes } from './chunk-36ER4WCG.mjs';
|
|
2
2
|
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, clampRowCount, DEFAULT_PAGE_SIZE, validateSchema, validateVectorPromotion, collectQueryTermStats, createPartitionManager, createVectorIndex, flattenSchema, isTextFieldType, requireWithinResultWindow, normalizeSort, sortSignatureOf, requireSortableFields, fanOutQuery, applySorting, applyPagination, resolveProjection, tokenize, decodePageCursor, requireMatchingCursor, encodePageCursor, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, validateRequiredFields, mergePartitionStats, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, createPartitionIndex, fnv1a, createWorkerPool, validateDocument, validateDocumentStrict, createFrozenSegment, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, createSharedFrozenSegment, buildCompactedSegmentPayload, freezeSegmentShared, deserializeMetadata, isCompositePartition, kWayMerge } from './chunk-RO4EFWE6.mjs';
|
|
3
3
|
import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
|
|
4
4
|
import { compareCodePoints, compareComparableKeys } from './chunk-BFIH7OV5.mjs';
|
|
@@ -2565,6 +2565,14 @@ function createForwardBatchResultMessage(payload, sourceId, requestId) {
|
|
|
2565
2565
|
payload: encode(payload)
|
|
2566
2566
|
};
|
|
2567
2567
|
}
|
|
2568
|
+
function createInsyncAddMessage(payload, sourceId) {
|
|
2569
|
+
return {
|
|
2570
|
+
type: ReplicationMessageTypes.INSYNC_ADD,
|
|
2571
|
+
sourceId,
|
|
2572
|
+
requestId: generateId(),
|
|
2573
|
+
payload: encode(payload)
|
|
2574
|
+
};
|
|
2575
|
+
}
|
|
2568
2576
|
function createInsyncRemoveMessage(payload, sourceId) {
|
|
2569
2577
|
return {
|
|
2570
2578
|
type: ReplicationMessageTypes.INSYNC_REMOVE,
|
|
@@ -8334,6 +8342,23 @@ function partitionsFor(manager, partitionIds) {
|
|
|
8334
8342
|
}
|
|
8335
8343
|
return partitions;
|
|
8336
8344
|
}
|
|
8345
|
+
function coverageFor(manager, partitionIds) {
|
|
8346
|
+
if (partitionIds === void 0) {
|
|
8347
|
+
return {
|
|
8348
|
+
totalPartitions: manager.partitionCount,
|
|
8349
|
+
queriedPartitions: manager.partitionCount,
|
|
8350
|
+
timedOutPartitions: 0,
|
|
8351
|
+
failedPartitions: 0
|
|
8352
|
+
};
|
|
8353
|
+
}
|
|
8354
|
+
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
8355
|
+
return {
|
|
8356
|
+
totalPartitions: partitionIds.length,
|
|
8357
|
+
queriedPartitions,
|
|
8358
|
+
timedOutPartitions: 0,
|
|
8359
|
+
failedPartitions: partitionIds.length - queriedPartitions
|
|
8360
|
+
};
|
|
8361
|
+
}
|
|
8337
8362
|
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
8338
8363
|
if (partitionIds === void 0) {
|
|
8339
8364
|
return void 0;
|
|
@@ -8795,7 +8820,8 @@ async function executeQuery(params, context) {
|
|
|
8795
8820
|
elapsed,
|
|
8796
8821
|
cursor: nextCursor,
|
|
8797
8822
|
facets,
|
|
8798
|
-
groups
|
|
8823
|
+
groups,
|
|
8824
|
+
coverage: coverageFor(manager, context.partitionIds)
|
|
8799
8825
|
};
|
|
8800
8826
|
}
|
|
8801
8827
|
async function executePreflight(params, context) {
|
|
@@ -9321,4 +9347,4 @@ async function createNarsil(config) {
|
|
|
9321
9347
|
// src/index.ts
|
|
9322
9348
|
var VERSION = "0.1.0";
|
|
9323
9349
|
|
|
9324
|
-
export { DEFAULT_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, VERSION, WIRE_BATCH_BUDGET, applyDeleteEntry, applyIndexEntry, chunkByBudget, clampLimit, computeEntryChecksum, createAckMessage, createDurableDirectory, createEngineCore, createEngineIndex, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createNarsil, createNarsilFromCore, getIndexMetadata, indexConfigKey, now, putIndexMetadata, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateIndexName, validateInsyncConfirmPayload, validateReplicationEntry, writeSegmentedCheckpoint };
|
|
9350
|
+
export { DEFAULT_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, VERSION, WIRE_BATCH_BUDGET, applyDeleteEntry, applyIndexEntry, chunkByBudget, clampLimit, computeEntryChecksum, createAckMessage, createDurableDirectory, createEngineCore, createEngineIndex, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createNarsil, createNarsilFromCore, getIndexMetadata, indexConfigKey, now, putIndexMetadata, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateIndexName, validateInsyncConfirmPayload, validateReplicationEntry, writeSegmentedCheckpoint };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { e as TaskRecord, b as ImportResult, H as HttpIndexConfig, c as TaskListQuery, T as TaskListPage } from './tasks-CnMLpfk-.js';
|
|
2
2
|
import { P as PartitionConfig, A as AnyDocument, a as InsertOptions } from './schema-emTQfEiX.js';
|
|
3
|
-
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-
|
|
3
|
+
import { V as VectorMaintenanceResult, M as MemoryStats, B as BatchResult, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult } from './search-Dt4dkzpf.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* This is the `fetch` implementation the client sends every request through.
|
package/dist/client.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './search-Dt4dkzpf.js';
|
|
2
2
|
export { A as ASYNC_IMPORT_CAPABILITY, E as ErrorEnvelope, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './options-BORJg5XB.js';
|
|
3
3
|
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, H as HttpIndexConfig, I as ImportError, b as ImportResult, T as TaskListPage, c as TaskListQuery, d as TaskProgress, e as TaskRecord, f as TaskStatus, g as TaskType } from './tasks-CnMLpfk-.js';
|
|
4
|
-
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-
|
|
4
|
+
export { A as AdminOperations, B as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, a as IndexOperations, N as NarsilClient, b as NarsilClientOptions, P as PutResult, R as RequestOptions, S as SearchOperations, c as ServerOperations, d as ServerVersion, T as TaskOperations, W as WaitForTaskOptions, e as createNarsilClient } from './client-DAJ_UHOa.js';
|
|
5
5
|
import './schema-emTQfEiX.js';
|
|
6
6
|
import './adapters-79OIa4xm.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult } from './search-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult } from './search-Dt4dkzpf.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument } from './schema-emTQfEiX.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -39,10 +39,22 @@ function extractSuffix(fullKey, prefix) {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
// src/distribution/coordinator/etcd/leases.ts
|
|
42
|
+
function payloadsMatch(left, right) {
|
|
43
|
+
if (left === null || left.byteLength !== right.byteLength) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
for (let index = 0; index < left.byteLength; index++) {
|
|
47
|
+
if (left[index] !== right[index]) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
42
53
|
var LeaseManager = class {
|
|
43
54
|
leases = /* @__PURE__ */ new Map();
|
|
44
|
-
|
|
45
|
-
|
|
55
|
+
keyLocks = /* @__PURE__ */ new Map();
|
|
56
|
+
track(key, lease, nodeId, payload = null) {
|
|
57
|
+
this.leases.set(key, { lease, nodeId, key, payload });
|
|
46
58
|
}
|
|
47
59
|
get(key) {
|
|
48
60
|
return this.leases.get(key);
|
|
@@ -54,6 +66,10 @@ var LeaseManager = class {
|
|
|
54
66
|
}
|
|
55
67
|
return entry;
|
|
56
68
|
}
|
|
69
|
+
holdsPayload(key, payload) {
|
|
70
|
+
const entry = this.leases.get(key);
|
|
71
|
+
return entry !== void 0 && payloadsMatch(entry.payload, payload);
|
|
72
|
+
}
|
|
57
73
|
remove(key) {
|
|
58
74
|
const entry = this.leases.get(key);
|
|
59
75
|
if (entry !== void 0) {
|
|
@@ -61,6 +77,23 @@ var LeaseManager = class {
|
|
|
61
77
|
}
|
|
62
78
|
return entry;
|
|
63
79
|
}
|
|
80
|
+
runExclusively(key, work) {
|
|
81
|
+
const previous = this.keyLocks.get(key) ?? Promise.resolve();
|
|
82
|
+
const result = previous.then(work, work);
|
|
83
|
+
const settled = result.then(
|
|
84
|
+
() => {
|
|
85
|
+
},
|
|
86
|
+
() => {
|
|
87
|
+
}
|
|
88
|
+
);
|
|
89
|
+
this.keyLocks.set(key, settled);
|
|
90
|
+
void settled.then(() => {
|
|
91
|
+
if (this.keyLocks.get(key) === settled) {
|
|
92
|
+
this.keyLocks.delete(key);
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
64
97
|
async revokeAll() {
|
|
65
98
|
const revokePromises = [];
|
|
66
99
|
for (const [, entry] of this.leases) {
|
|
@@ -113,9 +146,26 @@ function deserializeAllocationTable(data) {
|
|
|
113
146
|
indexName: raw.indexName,
|
|
114
147
|
version: raw.version,
|
|
115
148
|
replicationFactor: typeof raw.replicationFactor === "number" ? raw.replicationFactor : 0,
|
|
116
|
-
assignments:
|
|
149
|
+
assignments: normaliseAssignments(raw.assignments)
|
|
117
150
|
};
|
|
118
151
|
}
|
|
152
|
+
function normaliseAssignments(entries) {
|
|
153
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
154
|
+
for (const pair of entries) {
|
|
155
|
+
if (!Array.isArray(pair) || pair.length < 2) {
|
|
156
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable holds a malformed assignment entry");
|
|
157
|
+
}
|
|
158
|
+
const [partitionId, assignment] = pair;
|
|
159
|
+
if (typeof partitionId !== "number" || assignment === null || typeof assignment !== "object") {
|
|
160
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable holds a malformed partition assignment");
|
|
161
|
+
}
|
|
162
|
+
const candidate = assignment;
|
|
163
|
+
const storedCommitPoint = candidate.commitPoint;
|
|
164
|
+
const commitPoint = typeof storedCommitPoint === "number" && Number.isSafeInteger(storedCommitPoint) && storedCommitPoint >= 0 ? storedCommitPoint : 0;
|
|
165
|
+
assignments.set(partitionId, { ...assignment, commitPoint });
|
|
166
|
+
}
|
|
167
|
+
return assignments;
|
|
168
|
+
}
|
|
119
169
|
function serializeNodeRegistration(reg) {
|
|
120
170
|
return new Uint8Array(encode(reg));
|
|
121
171
|
}
|
|
@@ -136,6 +186,38 @@ function deserializeSchema(data) {
|
|
|
136
186
|
return decode(data);
|
|
137
187
|
}
|
|
138
188
|
|
|
189
|
+
// src/distribution/coordinator/etcd/registration.ts
|
|
190
|
+
function writeNodeRegistration(context, registration) {
|
|
191
|
+
return context.leaseManager.runExclusively(context.key, () => writeUnderLease(context, registration));
|
|
192
|
+
}
|
|
193
|
+
async function writeUnderLease(context, registration) {
|
|
194
|
+
const payload = serializeNodeRegistration(registration);
|
|
195
|
+
const data = Buffer.from(payload);
|
|
196
|
+
const tracked = context.leaseManager.getByNodeId(context.key, registration.nodeId);
|
|
197
|
+
if (tracked !== void 0) {
|
|
198
|
+
try {
|
|
199
|
+
await tracked.lease.keepaliveOnce();
|
|
200
|
+
if (context.leaseManager.holdsPayload(context.key, payload)) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
await tracked.lease.put(context.key).value(data).exec();
|
|
204
|
+
context.leaseManager.track(context.key, tracked.lease, registration.nodeId, payload);
|
|
205
|
+
return;
|
|
206
|
+
} catch (_) {
|
|
207
|
+
context.leaseManager.remove(context.key);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const lease = context.client.lease(context.ttlSeconds);
|
|
211
|
+
await lease.grant();
|
|
212
|
+
await lease.put(context.key).value(data).exec();
|
|
213
|
+
const existing = context.leaseManager.get(context.key);
|
|
214
|
+
if (existing !== void 0) {
|
|
215
|
+
await existing.lease.revoke().catch(() => {
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
context.leaseManager.track(context.key, lease, registration.nodeId, payload);
|
|
219
|
+
}
|
|
220
|
+
|
|
139
221
|
// src/distribution/coordinator/etcd/validation.ts
|
|
140
222
|
var MAX_NODE_ID_LENGTH = 255;
|
|
141
223
|
var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
|
|
@@ -206,18 +288,15 @@ async function createEtcdCoordinator(config) {
|
|
|
206
288
|
async registerNode(registration) {
|
|
207
289
|
assertNotShutdown();
|
|
208
290
|
validateNodeId(registration.nodeId);
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
leaseManager.track(key, lease, registration.nodeId);
|
|
291
|
+
await writeNodeRegistration(
|
|
292
|
+
{
|
|
293
|
+
client,
|
|
294
|
+
leaseManager,
|
|
295
|
+
key: keys.node(registration.nodeId),
|
|
296
|
+
ttlSeconds: resolvedConfig.nodeHeartbeatTtlSeconds
|
|
297
|
+
},
|
|
298
|
+
registration
|
|
299
|
+
);
|
|
221
300
|
},
|
|
222
301
|
async deregisterNode(nodeId) {
|
|
223
302
|
assertNotShutdown();
|
|
@@ -2,6 +2,7 @@ import { compareCodePoints } from '../../chunk-BFIH7OV5.mjs';
|
|
|
2
2
|
|
|
3
3
|
// src/distribution/coordinator/in-memory.ts
|
|
4
4
|
var NODE_HEARTBEAT_PREFIX = "_narsil/node/";
|
|
5
|
+
var NODE_HEARTBEAT_TTL_MS = 3e4;
|
|
5
6
|
function bytesEqual(a, b) {
|
|
6
7
|
if (a.byteLength !== b.byteLength) {
|
|
7
8
|
return false;
|
|
@@ -80,16 +81,19 @@ function createInMemoryCoordinator() {
|
|
|
80
81
|
const coordinator = {
|
|
81
82
|
async registerNode(registration) {
|
|
82
83
|
assertNotShutdown();
|
|
84
|
+
const previous = nodes.get(registration.nodeId);
|
|
83
85
|
nodes.set(registration.nodeId, registration);
|
|
84
86
|
const heartbeatKey = `${NODE_HEARTBEAT_PREFIX}${registration.nodeId}`;
|
|
85
|
-
const defaultTtlMs = 3e4;
|
|
86
87
|
clearLeaseEntry(heartbeatKey);
|
|
87
|
-
const timer = createLeaseTimer(heartbeatKey,
|
|
88
|
+
const timer = createLeaseTimer(heartbeatKey, NODE_HEARTBEAT_TTL_MS);
|
|
88
89
|
leases.set(heartbeatKey, {
|
|
89
90
|
nodeId: registration.nodeId,
|
|
90
|
-
expiresAt: Date.now() +
|
|
91
|
+
expiresAt: Date.now() + NODE_HEARTBEAT_TTL_MS,
|
|
91
92
|
timer
|
|
92
93
|
});
|
|
94
|
+
if (previous !== void 0 && registrationsMatch(previous, registration)) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
93
97
|
emitNodeEvent({ type: "node_joined", nodeId: registration.nodeId, registration });
|
|
94
98
|
},
|
|
95
99
|
async deregisterNode(nodeId) {
|
|
@@ -282,5 +286,8 @@ function createInMemoryCoordinator() {
|
|
|
282
286
|
};
|
|
283
287
|
return coordinator;
|
|
284
288
|
}
|
|
289
|
+
function registrationsMatch(left, right) {
|
|
290
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
291
|
+
}
|
|
285
292
|
|
|
286
293
|
export { createInMemoryCoordinator };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-
|
|
2
|
-
import { TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-
|
|
1
|
+
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-GODECMDT.mjs';
|
|
2
|
+
import { TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-36ER4WCG.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from '../../chunk-EMI63EXP.mjs';
|
|
4
4
|
|
|
5
5
|
// src/distribution/transport/grpc/codec.ts
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-
|
|
1
|
+
import { DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-36ER4WCG.mjs';
|
|
2
2
|
|
|
3
3
|
// src/distribution/transport/in-memory.ts
|
|
4
4
|
function createInMemoryNetwork() {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DEFAULT_TCP_CONFIG, encodeTransportMessage, encodeFrame, FRAME_TYPE_REQUEST, FrameParser, FRAME_TYPE_RESPONSE, decodeTransportMessage, FRAME_TYPE_STREAM_CHUNK, FRAME_TYPE_STREAM_END } from '../../chunk-
|
|
2
|
-
import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from '../../chunk-
|
|
1
|
+
import { DEFAULT_TCP_CONFIG, encodeTransportMessage, encodeFrame, FRAME_TYPE_REQUEST, FrameParser, FRAME_TYPE_RESPONSE, decodeTransportMessage, FRAME_TYPE_STREAM_CHUNK, FRAME_TYPE_STREAM_END } from '../../chunk-GODECMDT.mjs';
|
|
2
|
+
import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from '../../chunk-36ER4WCG.mjs';
|
|
3
3
|
import { connect as connect$1, createServer as createServer$1 } from 'net';
|
|
4
4
|
import { connect, createServer } from 'tls';
|
|
5
5
|
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
2
|
-
import { A as AllocationTable, N as NodeRole, C as ClusterCoordinator, a as NodeCapacity } from './types-
|
|
3
|
-
export { b as AllocationConstraints, c as AllocationEvent, d as NodeEvent, e as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-
|
|
1
|
+
import { N as NarsilConfig } from './config-CzTJJili.js';
|
|
2
|
+
import { A as AllocationTable, N as NodeRole, C as ClusterCoordinator, a as NodeCapacity } from './types-BhdINMtE.js';
|
|
3
|
+
export { b as AllocationConstraints, c as AllocationEvent, d as NodeEvent, e as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-BhdINMtE.js';
|
|
4
4
|
import { N as NodeTransport } from './types-D5OwhtsQ.js';
|
|
5
5
|
export { L as ListenHandler, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-D5OwhtsQ.js';
|
|
6
|
-
import { a as NarsilEventMap, N as Narsil } from './engine-
|
|
7
|
-
import { B as BatchResult, L as ListParams, g as ListResult, i as SuggestParams, j as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './search-
|
|
6
|
+
import { a as NarsilEventMap, N as Narsil } from './engine-DEtRLYpB.js';
|
|
7
|
+
import { B as BatchResult, L as ListParams, g as ListResult, i as SuggestParams, j as SuggestResult, Q as QueryParams, h as PreflightResult, f as IndexStats, P as PartitionStatsResult, M as MemoryStats, e as QueryResult } from './search-Dt4dkzpf.js';
|
|
8
8
|
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-emTQfEiX.js';
|
|
9
9
|
import './adapters-79OIa4xm.js';
|
|
10
10
|
|
|
@@ -51,9 +51,33 @@ interface ClusterNodeConfig {
|
|
|
51
51
|
engine?: NarsilConfig;
|
|
52
52
|
/** These replication settings override the engine's own, and anything you leave out keeps its value. */
|
|
53
53
|
replication?: Partial<ReplicationConfig>;
|
|
54
|
+
/** These settings govern how this node gathers each search from the cluster, and anything you leave out keeps its default. */
|
|
55
|
+
query?: ClusterQueryConfig;
|
|
54
56
|
/** This receives the failures the node reports while running on its own, away from any call you made. */
|
|
55
57
|
onError?: (error: Error) => void;
|
|
56
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* How this node answers a search when one of the nodes it asked runs slowly or
|
|
61
|
+
* drops out.
|
|
62
|
+
*
|
|
63
|
+
* The defaults let a search answer from the nodes that did reply, so set
|
|
64
|
+
* `allowPartialResults` to false where a caller must never act on an
|
|
65
|
+
* incomplete count. Every search reports what it read in
|
|
66
|
+
* {@link QueryResult.coverage} whichever way you set these.
|
|
67
|
+
*
|
|
68
|
+
* @public
|
|
69
|
+
*/
|
|
70
|
+
interface ClusterQueryConfig {
|
|
71
|
+
/**
|
|
72
|
+
* Set this true and a search returns the hits the reachable nodes gave,
|
|
73
|
+
* counting the rest in {@link QueryResult.coverage}, which is what a node
|
|
74
|
+
* does by default. Set it false and one partition that times out, errors, or
|
|
75
|
+
* has no active copy fails the whole search with `QUERY_PARTIAL_FAILURE`.
|
|
76
|
+
*/
|
|
77
|
+
allowPartialResults?: boolean;
|
|
78
|
+
/** A search waits this many milliseconds for each node before it counts that node's partitions as timed out, which is 5000 milliseconds where you name none. */
|
|
79
|
+
partitionTimeout?: number;
|
|
80
|
+
}
|
|
57
81
|
/**
|
|
58
82
|
* How an index is spread across the cluster when a node creates it.
|
|
59
83
|
*
|
|
@@ -441,4 +465,4 @@ interface ClusterEngineOptions {
|
|
|
441
465
|
*/
|
|
442
466
|
declare function clusterNodeEngine(node: ClusterNode, options?: ClusterEngineOptions): Narsil;
|
|
443
467
|
|
|
444
|
-
export { AllocationTable, ClusterCoordinator, type ClusterEngineOptions, type ClusterNamespace, type ClusterNode, type ClusterNodeConfig, type ClusterNodeInfo, type CreateIndexOptions, DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, NodeCapacity, NodeRole, NodeTransport, type ReplicationConfig, clusterNodeEngine, createClusterNode };
|
|
468
|
+
export { AllocationTable, ClusterCoordinator, type ClusterEngineOptions, type ClusterNamespace, type ClusterNode, type ClusterNodeConfig, type ClusterNodeInfo, type ClusterQueryConfig, type CreateIndexOptions, DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, NodeCapacity, NodeRole, NodeTransport, type ReplicationConfig, clusterNodeEngine, createClusterNode };
|