@delali/narsil 0.3.1-next.71 → 0.3.1-next.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-CYeAWYxQ.d.ts} +19 -4
- package/dist/{chunk-3JGWULDF.mjs → chunk-2WLD2NFW.mjs} +5 -5
- package/dist/{chunk-LMO5TJUS.mjs → chunk-3YKWSO54.mjs} +642 -196
- package/dist/{chunk-HWQLTLNG.mjs → chunk-4MQ5F4IL.mjs} +1 -1
- package/dist/{chunk-FVLQ63B7.mjs → chunk-AFKH4ZK3.mjs} +30 -6
- package/dist/{chunk-653BNMAF.mjs → chunk-AK3BLGCI.mjs} +195 -25
- package/dist/{chunk-IOR2K26O.mjs → chunk-CVFAFZSN.mjs} +8000 -4391
- package/dist/{chunk-USKFU7J5.mjs → chunk-I64LXKO3.mjs} +1 -1
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-2STVUTNA.mjs → chunk-LPBWCHBE.mjs} +194 -13
- package/dist/{chunk-YNVIUGF5.mjs → chunk-PU43WDY4.mjs} +4 -2
- package/dist/{chunk-M76V3OGJ.mjs → chunk-THMSFOJO.mjs} +1289 -1405
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-WVDFG45M.mjs} +93 -95
- package/dist/chunk-XW25X6AU.mjs +614 -0
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-ZULSEZCE.mjs} +68 -28
- package/dist/{client-COek5GXK.d.ts → client-D698E5ge.d.ts} +77 -26
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +45 -22
- package/dist/{config-rBpbgTw8.d.ts → config-Bv8etwUZ.d.ts} +11 -4
- package/dist/{config-Dfanyqqt.d.ts → config-QXU3HMpq.d.ts} +26 -2
- package/dist/direct-executor-5SRDFZAS.mjs +13 -0
- package/dist/distribution/coordinator/etcd.d.ts +29 -16
- package/dist/distribution/coordinator/etcd.mjs +163 -76
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +22 -2
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +6 -3
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +40 -16
- package/dist/distribution.d.ts +9 -9
- package/dist/distribution.mjs +682 -388
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-igk0aMpQ.d.ts → engine-D_OzKZ-6.d.ts} +97 -188
- package/dist/index.browser.mjs +11671 -7427
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +13 -14
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-B4jMmygw.d.ts} +279 -27
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -17
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -16
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -1
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +213 -34
- package/dist/{tasks-BZt_zes6.d.ts → tasks-DpgCpW1x.d.ts} +3 -3
- package/dist/{thread-JEVAYETR.mjs → thread-3LNVA43V.mjs} +94 -27
- package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-GIVAGD3Q.mjs +0 -25
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
|
@@ -1,20 +1,31 @@
|
|
|
1
|
-
import { C as ClusterCoordinator } from '../../types-
|
|
2
|
-
import '../../schema-
|
|
1
|
+
import { C as ClusterCoordinator } from '../../types-BiUmaiWT.js';
|
|
2
|
+
import '../../schema-BBOCdhND.js';
|
|
3
3
|
import '../../adapters-79OIa4xm.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
7
|
-
* registration
|
|
6
|
+
* Configures where {@link createEtcdCoordinator} connects, how long etcd keeps
|
|
7
|
+
* a node's registration, and how large a request the coordinator sends.
|
|
8
8
|
*
|
|
9
9
|
* @public
|
|
10
10
|
*/
|
|
11
11
|
interface EtcdCoordinatorConfig {
|
|
12
12
|
/** The coordinator connects to these etcd endpoints. */
|
|
13
13
|
endpoints: string[];
|
|
14
|
-
/** The coordinator writes every key under this prefix,
|
|
14
|
+
/** The coordinator writes every key under this prefix, so that one etcd cluster can store several Narsil clusters. */
|
|
15
15
|
keyPrefix: string;
|
|
16
|
-
/**
|
|
16
|
+
/** etcd keeps a node's registration for this many seconds without a heartbeat, after which peers count the node as gone. */
|
|
17
17
|
nodeHeartbeatTtlSeconds: number;
|
|
18
|
+
/**
|
|
19
|
+
* The coordinator sends etcd requests of at most this many bytes, so set it
|
|
20
|
+
* to the `--max-request-bytes` value of every etcd member. The default of
|
|
21
|
+
* 1,572,864 matches the etcd default. The coordinator raises
|
|
22
|
+
* `CONFIG_INVALID` for a write whose request would be larger, and for a
|
|
23
|
+
* write that etcd reports as too large. When this value is more than
|
|
24
|
+
* 524,288 bytes above the etcd setting, a write in that gap can fail at the
|
|
25
|
+
* gRPC layer of etcd, after which the `etcd3` client stops sending requests
|
|
26
|
+
* to that member for five seconds.
|
|
27
|
+
*/
|
|
28
|
+
maxRequestBytes: number;
|
|
18
29
|
}
|
|
19
30
|
|
|
20
31
|
/**
|
|
@@ -34,19 +45,21 @@ interface EtcdCoordinatorConfig {
|
|
|
34
45
|
declare function validateNodeId(nodeId: string): void;
|
|
35
46
|
|
|
36
47
|
/**
|
|
37
|
-
* Builds a coordinator backed by etcd, which
|
|
38
|
-
*
|
|
48
|
+
* Builds a coordinator backed by etcd, which stores a cluster's shared state
|
|
49
|
+
* across hosts and restarts.
|
|
39
50
|
*
|
|
40
|
-
* etcd
|
|
41
|
-
* a node misses its heartbeats, and compare-and-set writes
|
|
42
|
-
* controllers from
|
|
43
|
-
* because it is an optional peer dependency this call loads on
|
|
51
|
+
* etcd provides the two mechanisms that a cluster uses: leases, which expire
|
|
52
|
+
* once a node misses its heartbeats, and compare-and-set writes, which stop two
|
|
53
|
+
* controllers from writing conflicting state. Install `etcd3` alongside this
|
|
54
|
+
* package, because it is an optional peer dependency that this call loads on
|
|
55
|
+
* demand.
|
|
44
56
|
*
|
|
45
|
-
* @param config -
|
|
46
|
-
*
|
|
47
|
-
* @returns A coordinator you pass as a cluster node's `coordinator`.
|
|
57
|
+
* @param config - The endpoints, key prefix, heartbeat lifetime, and request
|
|
58
|
+
* size to override. Every setting that you leave out keeps its default.
|
|
59
|
+
* @returns A coordinator that you pass as a cluster node's `coordinator`.
|
|
48
60
|
* @throws A `NarsilError` with `COORDINATOR_DEPENDENCY_MISSING` when `etcd3`
|
|
49
|
-
* is not installed
|
|
61
|
+
* is not installed, or with `CONFIG_INVALID` when `maxRequestBytes` is not a
|
|
62
|
+
* whole number from 1,025 to 2,147,483,647.
|
|
50
63
|
*
|
|
51
64
|
* @public
|
|
52
65
|
*/
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
import { MAX_NODE_ID_LENGTH, MAX_WATCHERS } from '../../chunk-ORBLD3EL.mjs';
|
|
2
|
-
import { compareCodePoints } from '../../chunk-
|
|
3
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
2
|
+
import { compareCodePoints } from '../../chunk-LJYTW7GT.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from '../../chunk-I64LXKO3.mjs';
|
|
4
4
|
import { decode, encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
|
+
// src/distribution/coordinator/etcd/constants.ts
|
|
7
|
+
var DEFAULT_ETCD_MAX_REQUEST_BYTES = 1572864;
|
|
8
|
+
var MAX_ETCD_REQUEST_BYTES = 2147483647;
|
|
9
|
+
var ETCD_REQUEST_FRAMING_BYTES = 1024;
|
|
10
|
+
var ETCD_MAX_RESPONSE_BYTES = 2147483647;
|
|
11
|
+
|
|
6
12
|
// src/distribution/coordinator/etcd/types.ts
|
|
7
13
|
var DEFAULT_ETCD_CONFIG = {
|
|
8
14
|
endpoints: ["http://localhost:2379"],
|
|
9
15
|
keyPrefix: "_narsil",
|
|
10
|
-
nodeHeartbeatTtlSeconds: 30
|
|
16
|
+
nodeHeartbeatTtlSeconds: 30,
|
|
17
|
+
maxRequestBytes: DEFAULT_ETCD_MAX_REQUEST_BYTES
|
|
11
18
|
};
|
|
12
19
|
var ETCD_KEY_NODES = "nodes";
|
|
13
20
|
var ETCD_KEY_ALLOCATION = "allocation";
|
|
@@ -211,6 +218,8 @@ function writeNodeRegistration(context, registration) {
|
|
|
211
218
|
async function writeUnderLease(context, registration) {
|
|
212
219
|
const payload = serializeNodeRegistration(registration);
|
|
213
220
|
const data = Buffer.from(payload);
|
|
221
|
+
const write = { key: context.key, value: data, expected: null };
|
|
222
|
+
context.guard.requireFits(write);
|
|
214
223
|
const tracked = context.leaseManager.getByNodeId(context.key, registration.nodeId);
|
|
215
224
|
if (tracked !== void 0) {
|
|
216
225
|
try {
|
|
@@ -227,7 +236,13 @@ async function writeUnderLease(context, registration) {
|
|
|
227
236
|
}
|
|
228
237
|
const lease = context.client.lease(context.ttlSeconds);
|
|
229
238
|
await lease.grant();
|
|
230
|
-
|
|
239
|
+
try {
|
|
240
|
+
await context.guard.send(write, () => lease.put(context.key).value(data).exec());
|
|
241
|
+
} catch (error) {
|
|
242
|
+
await lease.revoke().catch(() => {
|
|
243
|
+
});
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
231
246
|
const existing = context.leaseManager.get(context.key);
|
|
232
247
|
if (existing !== void 0) {
|
|
233
248
|
await existing.lease.revoke().catch(() => {
|
|
@@ -236,6 +251,132 @@ async function writeUnderLease(context, registration) {
|
|
|
236
251
|
context.leaseManager.track(context.key, lease, registration.nodeId, payload);
|
|
237
252
|
}
|
|
238
253
|
|
|
254
|
+
// src/distribution/coordinator/etcd/request-size.ts
|
|
255
|
+
var ETCD_REQUEST_TOO_LARGE = "etcdserver: request is too large";
|
|
256
|
+
var GRPC_MESSAGE_TOO_LARGE = "larger than max";
|
|
257
|
+
function requireEtcdMaxRequestBytes(maxRequestBytes) {
|
|
258
|
+
if (Number.isSafeInteger(maxRequestBytes) && maxRequestBytes > ETCD_REQUEST_FRAMING_BYTES && maxRequestBytes <= MAX_ETCD_REQUEST_BYTES) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
throw new NarsilError(
|
|
262
|
+
ErrorCodes.CONFIG_INVALID,
|
|
263
|
+
`maxRequestBytes must be a whole number above ${ETCD_REQUEST_FRAMING_BYTES} and at most ${MAX_ETCD_REQUEST_BYTES}`,
|
|
264
|
+
{ maxRequestBytes }
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
function requestBytesOf(write) {
|
|
268
|
+
const keyBytes = Buffer.byteLength(write.key);
|
|
269
|
+
return write.value.byteLength + (write.expected?.byteLength ?? 0) + 2 * keyBytes + ETCD_REQUEST_FRAMING_BYTES;
|
|
270
|
+
}
|
|
271
|
+
function refusedForSize(etcd3, error) {
|
|
272
|
+
if (error instanceof etcd3.GRPCInvalidArgumentError) return error.message.includes(ETCD_REQUEST_TOO_LARGE);
|
|
273
|
+
if (error instanceof etcd3.GRPCResourceExhastedError) return error.message.includes(GRPC_MESSAGE_TOO_LARGE);
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
function createEtcdWriteGuard(etcd3, maxRequestBytes) {
|
|
277
|
+
function requireFits(write) {
|
|
278
|
+
const bytes = requestBytesOf(write);
|
|
279
|
+
if (bytes <= maxRequestBytes) return bytes;
|
|
280
|
+
throw new NarsilError(
|
|
281
|
+
ErrorCodes.CONFIG_INVALID,
|
|
282
|
+
`The request to write "${write.key}" would be ${bytes} bytes, more than the maxRequestBytes of ${maxRequestBytes}. Raise maxRequestBytes in createEtcdCoordinator together with the --max-request-bytes value of etcd.`,
|
|
283
|
+
{ key: write.key, bytes, maxRequestBytes }
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
return {
|
|
287
|
+
requireFits,
|
|
288
|
+
async send(write, perform) {
|
|
289
|
+
const bytes = requireFits(write);
|
|
290
|
+
try {
|
|
291
|
+
return await perform();
|
|
292
|
+
} catch (error) {
|
|
293
|
+
if (!refusedForSize(etcd3, error)) throw error;
|
|
294
|
+
throw new NarsilError(
|
|
295
|
+
ErrorCodes.CONFIG_INVALID,
|
|
296
|
+
`etcd reports the ${bytes}-byte request to write "${write.key}" as too large, so its --max-request-bytes value is below the maxRequestBytes of ${maxRequestBytes}. Set both to the same value.`,
|
|
297
|
+
{ key: write.key, bytes, maxRequestBytes }
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// src/distribution/coordinator/etcd/stored-values.ts
|
|
305
|
+
function storedValueMethods(context) {
|
|
306
|
+
const { client, keys, guard, assertNotShutdown, addWatcher, removeWatcher } = context;
|
|
307
|
+
return {
|
|
308
|
+
async get(key) {
|
|
309
|
+
assertNotShutdown();
|
|
310
|
+
const etcdKey = keys.generic(key);
|
|
311
|
+
const buf = await client.get(etcdKey).buffer();
|
|
312
|
+
if (buf === null) {
|
|
313
|
+
return null;
|
|
314
|
+
}
|
|
315
|
+
return new Uint8Array(buf);
|
|
316
|
+
},
|
|
317
|
+
async compareAndSet(key, expected, value) {
|
|
318
|
+
assertNotShutdown();
|
|
319
|
+
const etcdKey = keys.generic(key);
|
|
320
|
+
const newValue = Buffer.from(value);
|
|
321
|
+
return guard.send({ key: etcdKey, value: newValue, expected }, async () => {
|
|
322
|
+
if (expected === null) {
|
|
323
|
+
const txn2 = await client.if(etcdKey, "Create", "==", 0).then(client.put(etcdKey).value(newValue)).commit();
|
|
324
|
+
return txn2.succeeded;
|
|
325
|
+
}
|
|
326
|
+
const expectedBuf = Buffer.from(expected);
|
|
327
|
+
const txn = await client.if(etcdKey, "Value", "==", expectedBuf).then(client.put(etcdKey).value(newValue)).commit();
|
|
328
|
+
return txn.succeeded;
|
|
329
|
+
});
|
|
330
|
+
},
|
|
331
|
+
async getSchema(indexName) {
|
|
332
|
+
assertNotShutdown();
|
|
333
|
+
const key = keys.schema(indexName);
|
|
334
|
+
const buf = await client.get(key).buffer();
|
|
335
|
+
if (buf === null) {
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
return deserializeSchema(buf);
|
|
339
|
+
},
|
|
340
|
+
async putSchema(indexName, schema) {
|
|
341
|
+
assertNotShutdown();
|
|
342
|
+
const key = keys.schema(indexName);
|
|
343
|
+
const value = serializeSchema(schema);
|
|
344
|
+
await guard.send({ key, value, expected: null }, () => client.put(key).value(value).exec());
|
|
345
|
+
},
|
|
346
|
+
async dropSchema(indexName) {
|
|
347
|
+
assertNotShutdown();
|
|
348
|
+
await client.delete().key(keys.schema(indexName)).exec();
|
|
349
|
+
},
|
|
350
|
+
async listSchemas() {
|
|
351
|
+
assertNotShutdown();
|
|
352
|
+
const prefix = keys.schemaPrefix();
|
|
353
|
+
const result = await client.getAll().prefix(prefix).keys();
|
|
354
|
+
return result.map((fullKey) => extractSuffix(fullKey, prefix)).sort(compareCodePoints);
|
|
355
|
+
},
|
|
356
|
+
async watchSchemas(handler) {
|
|
357
|
+
assertNotShutdown();
|
|
358
|
+
const prefix = keys.schemaPrefix();
|
|
359
|
+
const watcher = await client.watch().prefix(prefix).create();
|
|
360
|
+
addWatcher(watcher);
|
|
361
|
+
watcher.on("put", (kv) => {
|
|
362
|
+
try {
|
|
363
|
+
const schema = deserializeSchema(kv.value);
|
|
364
|
+
const fullKey = kv.key.toString();
|
|
365
|
+
const indexName = extractSuffix(fullKey, prefix);
|
|
366
|
+
handler({ type: "schema_created", indexName, schema });
|
|
367
|
+
} catch (_) {
|
|
368
|
+
}
|
|
369
|
+
});
|
|
370
|
+
watcher.on("delete", (kv) => {
|
|
371
|
+
const fullKey = kv.key.toString();
|
|
372
|
+
const indexName = extractSuffix(fullKey, prefix);
|
|
373
|
+
handler({ type: "schema_dropped", indexName, schema: null });
|
|
374
|
+
});
|
|
375
|
+
return () => removeWatcher(watcher);
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
239
380
|
// src/distribution/coordinator/etcd/validation.ts
|
|
240
381
|
var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
|
|
241
382
|
function validateNodeId(nodeId) {
|
|
@@ -274,8 +415,13 @@ async function createEtcdCoordinator(config) {
|
|
|
274
415
|
...DEFAULT_ETCD_CONFIG,
|
|
275
416
|
...config
|
|
276
417
|
};
|
|
418
|
+
requireEtcdMaxRequestBytes(resolvedConfig.maxRequestBytes);
|
|
277
419
|
const etcd3Module = await loadEtcd3Module();
|
|
278
|
-
const client = new etcd3Module.Etcd3({
|
|
420
|
+
const client = new etcd3Module.Etcd3({
|
|
421
|
+
hosts: resolvedConfig.endpoints,
|
|
422
|
+
grpcOptions: { "grpc.max_receive_message_length": ETCD_MAX_RESPONSE_BYTES }
|
|
423
|
+
});
|
|
424
|
+
const guard = createEtcdWriteGuard(etcd3Module, resolvedConfig.maxRequestBytes);
|
|
279
425
|
const leaseManager = new LeaseManager();
|
|
280
426
|
const watchers = /* @__PURE__ */ new Set();
|
|
281
427
|
let isShutdown = false;
|
|
@@ -308,6 +454,7 @@ async function createEtcdCoordinator(config) {
|
|
|
308
454
|
{
|
|
309
455
|
client,
|
|
310
456
|
leaseManager,
|
|
457
|
+
guard,
|
|
311
458
|
key: keys.node(registration.nodeId),
|
|
312
459
|
ttlSeconds: resolvedConfig.nodeHeartbeatTtlSeconds
|
|
313
460
|
},
|
|
@@ -370,13 +517,16 @@ async function createEtcdCoordinator(config) {
|
|
|
370
517
|
assertNotShutdown();
|
|
371
518
|
const key = keys.allocation(indexName);
|
|
372
519
|
const data = Buffer.from(serializeAllocationTable(table));
|
|
520
|
+
const write = { key, value: data, expected: null };
|
|
373
521
|
if (expectedVersion === void 0) {
|
|
374
|
-
await client.put(key).value(data).exec();
|
|
522
|
+
await guard.send(write, () => client.put(key).value(data).exec());
|
|
375
523
|
return true;
|
|
376
524
|
}
|
|
377
525
|
if (expectedVersion === null) {
|
|
378
|
-
|
|
379
|
-
|
|
526
|
+
return guard.send(write, async () => {
|
|
527
|
+
const txn = await client.if(key, "Create", "==", 0).then(client.put(key).value(data)).commit();
|
|
528
|
+
return txn.succeeded;
|
|
529
|
+
});
|
|
380
530
|
}
|
|
381
531
|
const response = await client.get(key).exec();
|
|
382
532
|
if (response.kvs.length === 0) {
|
|
@@ -387,8 +537,10 @@ async function createEtcdCoordinator(config) {
|
|
|
387
537
|
if (currentTable.version !== expectedVersion) {
|
|
388
538
|
return false;
|
|
389
539
|
}
|
|
390
|
-
|
|
391
|
-
|
|
540
|
+
return guard.send(write, async () => {
|
|
541
|
+
const txn = await client.if(key, "Mod", "==", kv.mod_revision).then(client.put(key).value(data)).commit();
|
|
542
|
+
return txn.succeeded;
|
|
543
|
+
});
|
|
392
544
|
},
|
|
393
545
|
async deleteAllocation(indexName) {
|
|
394
546
|
assertNotShutdown();
|
|
@@ -472,72 +624,7 @@ async function createEtcdCoordinator(config) {
|
|
|
472
624
|
await entry.lease.revoke().catch(() => {
|
|
473
625
|
});
|
|
474
626
|
},
|
|
475
|
-
|
|
476
|
-
assertNotShutdown();
|
|
477
|
-
const etcdKey = keys.generic(key);
|
|
478
|
-
const buf = await client.get(etcdKey).buffer();
|
|
479
|
-
if (buf === null) {
|
|
480
|
-
return null;
|
|
481
|
-
}
|
|
482
|
-
return new Uint8Array(buf);
|
|
483
|
-
},
|
|
484
|
-
async compareAndSet(key, expected, value) {
|
|
485
|
-
assertNotShutdown();
|
|
486
|
-
const etcdKey = keys.generic(key);
|
|
487
|
-
const newValue = Buffer.from(value);
|
|
488
|
-
if (expected === null) {
|
|
489
|
-
const txn2 = await client.if(etcdKey, "Create", "==", 0).then(client.put(etcdKey).value(newValue)).commit();
|
|
490
|
-
return txn2.succeeded;
|
|
491
|
-
}
|
|
492
|
-
const expectedBuf = Buffer.from(expected);
|
|
493
|
-
const txn = await client.if(etcdKey, "Value", "==", expectedBuf).then(client.put(etcdKey).value(newValue)).commit();
|
|
494
|
-
return txn.succeeded;
|
|
495
|
-
},
|
|
496
|
-
async getSchema(indexName) {
|
|
497
|
-
assertNotShutdown();
|
|
498
|
-
const key = keys.schema(indexName);
|
|
499
|
-
const buf = await client.get(key).buffer();
|
|
500
|
-
if (buf === null) {
|
|
501
|
-
return null;
|
|
502
|
-
}
|
|
503
|
-
return deserializeSchema(buf);
|
|
504
|
-
},
|
|
505
|
-
async putSchema(indexName, schema) {
|
|
506
|
-
assertNotShutdown();
|
|
507
|
-
const key = keys.schema(indexName);
|
|
508
|
-
await client.put(key).value(serializeSchema(schema)).exec();
|
|
509
|
-
},
|
|
510
|
-
async dropSchema(indexName) {
|
|
511
|
-
assertNotShutdown();
|
|
512
|
-
await client.delete().key(keys.schema(indexName)).exec();
|
|
513
|
-
},
|
|
514
|
-
async listSchemas() {
|
|
515
|
-
assertNotShutdown();
|
|
516
|
-
const prefix = keys.schemaPrefix();
|
|
517
|
-
const result = await client.getAll().prefix(prefix).keys();
|
|
518
|
-
return result.map((fullKey) => extractSuffix(fullKey, prefix)).sort(compareCodePoints);
|
|
519
|
-
},
|
|
520
|
-
async watchSchemas(handler) {
|
|
521
|
-
assertNotShutdown();
|
|
522
|
-
const prefix = keys.schemaPrefix();
|
|
523
|
-
const watcher = await client.watch().prefix(prefix).create();
|
|
524
|
-
addWatcher(watcher);
|
|
525
|
-
watcher.on("put", (kv) => {
|
|
526
|
-
try {
|
|
527
|
-
const schema = deserializeSchema(kv.value);
|
|
528
|
-
const fullKey = kv.key.toString();
|
|
529
|
-
const indexName = extractSuffix(fullKey, prefix);
|
|
530
|
-
handler({ type: "schema_created", indexName, schema });
|
|
531
|
-
} catch (_) {
|
|
532
|
-
}
|
|
533
|
-
});
|
|
534
|
-
watcher.on("delete", (kv) => {
|
|
535
|
-
const fullKey = kv.key.toString();
|
|
536
|
-
const indexName = extractSuffix(fullKey, prefix);
|
|
537
|
-
handler({ type: "schema_dropped", indexName, schema: null });
|
|
538
|
-
});
|
|
539
|
-
return () => removeWatcher(watcher);
|
|
540
|
-
},
|
|
627
|
+
...storedValueMethods({ client, keys, guard, assertNotShutdown, addWatcher, removeWatcher }),
|
|
541
628
|
async getLeaseHolder(key) {
|
|
542
629
|
assertNotShutdown();
|
|
543
630
|
const etcdKey = keys.lease(key);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { NODE_HEARTBEAT_TTL_MS } from '../../chunk-ORBLD3EL.mjs';
|
|
2
|
-
import { compareCodePoints } from '../../chunk-
|
|
2
|
+
import { compareCodePoints } from '../../chunk-LJYTW7GT.mjs';
|
|
3
3
|
|
|
4
4
|
// src/distribution/coordinator/in-memory.ts
|
|
5
5
|
var NODE_HEARTBEAT_PREFIX = "_narsil/node/";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { T as TransportConfig, N as NodeTransport } from '../../types-
|
|
2
|
-
import { a as TlsConfig } from '../../types-
|
|
1
|
+
import { T as TransportConfig, N as NodeTransport } from '../../types-Cb9kiY5P.js';
|
|
2
|
+
import { a as TlsConfig } from '../../types-B3b68nfF.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* How the gRPC transport listens and dials.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-OOSIDHA3.mjs';
|
|
2
2
|
import { MAX_MESSAGE_SIZE_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, GRPC_ENVELOPE_OVERHEAD_BYTES } from '../../chunk-UNEEFTEU.mjs';
|
|
3
3
|
import { TransportError, TransportErrorCodes } from '../../chunk-GCM3JFVA.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from '../../chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../../chunk-I64LXKO3.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/transport/grpc/codec.ts
|
|
7
7
|
var FIELD_ONE_LENGTH_DELIMITED_KEY = 10;
|
|
@@ -170,17 +170,37 @@ var GrpcClientPool = class {
|
|
|
170
170
|
openStreamMethod.responseDeserialize,
|
|
171
171
|
messageBytes,
|
|
172
172
|
this.metadata(),
|
|
173
|
-
|
|
173
|
+
{}
|
|
174
174
|
);
|
|
175
175
|
return new Promise((resolve, reject) => {
|
|
176
176
|
let settled = false;
|
|
177
|
+
const timeoutMs = this.config.snapshotTimeout;
|
|
178
|
+
let idleTimer = null;
|
|
177
179
|
const settle = (fn) => {
|
|
180
|
+
if (idleTimer !== null) clearTimeout(idleTimer);
|
|
178
181
|
if (!settled) {
|
|
179
182
|
settled = true;
|
|
180
183
|
fn();
|
|
181
184
|
}
|
|
182
185
|
};
|
|
186
|
+
const rearm = () => {
|
|
187
|
+
if (idleTimer !== null) clearTimeout(idleTimer);
|
|
188
|
+
idleTimer = setTimeout(() => {
|
|
189
|
+
call.cancel();
|
|
190
|
+
settle(
|
|
191
|
+
() => reject(
|
|
192
|
+
new TransportError(
|
|
193
|
+
TransportErrorCodes.TIMEOUT,
|
|
194
|
+
`No chunk of the stream from '${target}' arrived within ${timeoutMs}ms`,
|
|
195
|
+
{ target, requestId: message.requestId, timeoutMs }
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
);
|
|
199
|
+
}, timeoutMs);
|
|
200
|
+
};
|
|
201
|
+
rearm();
|
|
183
202
|
call.on("data", (chunk) => {
|
|
203
|
+
rearm();
|
|
184
204
|
try {
|
|
185
205
|
handler(chunk);
|
|
186
206
|
} catch (err) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { N as NodeTransport, a as TransportMessage, R as RespondFn, T as TransportConfig } from '../../types-
|
|
2
|
-
export { L as ListenHandler } from '../../types-
|
|
1
|
+
import { N as NodeTransport, a as TransportMessage, R as RespondFn, T as TransportConfig } from '../../types-Cb9kiY5P.js';
|
|
2
|
+
export { L as ListenHandler } from '../../types-Cb9kiY5P.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Receives a streamed response one chunk at a time.
|
|
@@ -133,18 +133,19 @@ function createInMemoryTransport(nodeId, network, config) {
|
|
|
133
133
|
const peer = lookupPeer(target);
|
|
134
134
|
return new Promise((resolve, reject) => {
|
|
135
135
|
let settled = false;
|
|
136
|
-
const
|
|
136
|
+
const expire = () => {
|
|
137
137
|
if (!settled) {
|
|
138
138
|
settled = true;
|
|
139
139
|
reject(
|
|
140
140
|
new TransportError(
|
|
141
141
|
TransportErrorCodes.TIMEOUT,
|
|
142
|
-
`
|
|
142
|
+
`No chunk of the stream from node '${target}' arrived within ${resolvedConfig.snapshotTimeout}ms`,
|
|
143
143
|
{ target, requestId: message.requestId, timeoutMs: resolvedConfig.snapshotTimeout }
|
|
144
144
|
)
|
|
145
145
|
);
|
|
146
146
|
}
|
|
147
|
-
}
|
|
147
|
+
};
|
|
148
|
+
let timeoutId = setTimeout(expire, resolvedConfig.snapshotTimeout);
|
|
148
149
|
const settle = (action) => {
|
|
149
150
|
if (settled) {
|
|
150
151
|
return;
|
|
@@ -158,6 +159,8 @@ function createInMemoryTransport(nodeId, network, config) {
|
|
|
158
159
|
if (settled) {
|
|
159
160
|
return Promise.resolve();
|
|
160
161
|
}
|
|
162
|
+
clearTimeout(timeoutId);
|
|
163
|
+
timeoutId = setTimeout(expire, resolvedConfig.snapshotTimeout);
|
|
161
164
|
try {
|
|
162
165
|
handler(payload);
|
|
163
166
|
} catch (error) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { N as NodeTransport } from '../../types-
|
|
2
|
-
import { T as TcpTransportConfig, a as TlsConfig } from '../../types-
|
|
1
|
+
import { N as NodeTransport } from '../../types-Cb9kiY5P.js';
|
|
2
|
+
import { T as TcpTransportConfig, a as TlsConfig } from '../../types-B3b68nfF.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Builds the TCP transport a cluster node reaches its peers through, with
|
|
@@ -24,6 +24,33 @@ function parseAddress(address) {
|
|
|
24
24
|
return [host, port];
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
// src/distribution/transport/tcp/pending-stream.ts
|
|
28
|
+
function createPendingStream(input) {
|
|
29
|
+
const { target, requestId, timeoutMs, reject } = input;
|
|
30
|
+
const expire = () => {
|
|
31
|
+
input.forget();
|
|
32
|
+
reject(
|
|
33
|
+
new TransportError(
|
|
34
|
+
TransportErrorCodes.TIMEOUT,
|
|
35
|
+
`No chunk of the stream from '${target}' arrived within ${timeoutMs}ms`,
|
|
36
|
+
{ target, requestId, timeoutMs }
|
|
37
|
+
)
|
|
38
|
+
);
|
|
39
|
+
};
|
|
40
|
+
const pending = {
|
|
41
|
+
target,
|
|
42
|
+
handler: input.handler,
|
|
43
|
+
resolve: input.resolve,
|
|
44
|
+
reject,
|
|
45
|
+
timer: setTimeout(expire, timeoutMs),
|
|
46
|
+
rearm: () => {
|
|
47
|
+
clearTimeout(pending.timer);
|
|
48
|
+
pending.timer = setTimeout(expire, timeoutMs);
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
return pending;
|
|
52
|
+
}
|
|
53
|
+
|
|
27
54
|
// src/distribution/transport/tcp/connection.ts
|
|
28
55
|
var TcpConnectionPool = class {
|
|
29
56
|
connections = /* @__PURE__ */ new Map();
|
|
@@ -95,24 +122,19 @@ var TcpConnectionPool = class {
|
|
|
95
122
|
const frame = encodeFrame(FRAME_TYPE_REQUEST, message.requestId, messageBytes);
|
|
96
123
|
const pendingKey = `${target}:${message.requestId}`;
|
|
97
124
|
return new Promise((resolve, reject) => {
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
}
|
|
109
|
-
)
|
|
110
|
-
);
|
|
111
|
-
}, this.config.snapshotTimeout);
|
|
112
|
-
this.pendingStreams.set(pendingKey, { target, handler, resolve, reject, timer });
|
|
125
|
+
const pending = createPendingStream({
|
|
126
|
+
target,
|
|
127
|
+
requestId: message.requestId,
|
|
128
|
+
timeoutMs: this.config.snapshotTimeout,
|
|
129
|
+
handler,
|
|
130
|
+
resolve,
|
|
131
|
+
reject,
|
|
132
|
+
forget: () => this.pendingStreams.delete(pendingKey)
|
|
133
|
+
});
|
|
134
|
+
this.pendingStreams.set(pendingKey, pending);
|
|
113
135
|
socket.write(frame, (err) => {
|
|
114
136
|
if (err) {
|
|
115
|
-
clearTimeout(timer);
|
|
137
|
+
clearTimeout(pending.timer);
|
|
116
138
|
this.pendingStreams.delete(pendingKey);
|
|
117
139
|
reject(
|
|
118
140
|
new TransportError(
|
|
@@ -239,6 +261,7 @@ var TcpConnectionPool = class {
|
|
|
239
261
|
}
|
|
240
262
|
const pendingStream = this.pendingStreams.get(pendingKey);
|
|
241
263
|
if (pendingStream !== void 0) {
|
|
264
|
+
pendingStream.rearm();
|
|
242
265
|
try {
|
|
243
266
|
const message = decodeTransportMessage(frame.data);
|
|
244
267
|
pendingStream.handler(message.payload);
|
|
@@ -259,6 +282,7 @@ var TcpConnectionPool = class {
|
|
|
259
282
|
if (frame.frameType === FRAME_TYPE_STREAM_CHUNK) {
|
|
260
283
|
const pending = this.pendingStreams.get(pendingKey);
|
|
261
284
|
if (pending !== void 0) {
|
|
285
|
+
pending.rearm();
|
|
262
286
|
pending.handler(frame.data);
|
|
263
287
|
}
|
|
264
288
|
return;
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-
|
|
2
|
-
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-
|
|
9
|
-
import './config-
|
|
1
|
+
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-Bv8etwUZ.js';
|
|
2
|
+
export { c as ClusterControllerConfig, d as ClusterNodeInfo, e as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-Bv8etwUZ.js';
|
|
3
|
+
import { B as BatchResult, L as ListParams, h as ListResult, k as SuggestParams, l as SuggestResult, Q as QueryParams, j as PreflightResult, g as IndexStats, P as PartitionStatsResult, M as MemoryStats, m as NarsilEventMap, e as QueryResult } from './memory-B4jMmygw.js';
|
|
4
|
+
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-BBOCdhND.js';
|
|
5
|
+
import { b as NodeCapacity, N as NodeRole } from './types-BiUmaiWT.js';
|
|
6
|
+
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, e as NodeEvent, a as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent, U as UnassignedReason } from './types-BiUmaiWT.js';
|
|
7
|
+
import { N as Narsil } from './engine-D_OzKZ-6.js';
|
|
8
|
+
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-Cb9kiY5P.js';
|
|
9
|
+
import './config-QXU3HMpq.js';
|
|
10
10
|
import './adapters-79OIa4xm.js';
|
|
11
11
|
|
|
12
12
|
/**
|