@helyx/bot 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +68 -0
- package/dist/cloud-connector.d.ts +36 -2
- package/dist/cloud-connector.js +41 -20
- package/dist/cloud-management-module.d.ts +5 -3
- package/dist/cloud-management-module.js +27 -18
- package/dist/command-sync.d.ts +32 -0
- package/dist/command-sync.js +67 -0
- package/dist/control-server.d.ts +10 -0
- package/dist/control-server.js +86 -18
- package/dist/developer-operations-module.d.ts +28 -0
- package/dist/developer-operations-module.js +249 -0
- package/dist/events.d.ts +5 -1
- package/dist/events.js +69 -3
- package/dist/guilds.d.ts +2 -2
- package/dist/guilds.js +17 -4
- package/dist/index.d.ts +2 -0
- package/dist/index.js +111 -10
- package/dist/interactions.d.ts +8 -1
- package/dist/interactions.js +50 -24
- package/dist/shard-cloud-proxy.d.ts +21 -0
- package/dist/shard-cloud-proxy.js +94 -0
- package/dist/shard-contracts.d.ts +751 -0
- package/dist/shard-contracts.js +445 -0
- package/dist/shard-coordinator.d.ts +90 -0
- package/dist/shard-coordinator.js +620 -0
- package/dist/shard-router.d.ts +28 -0
- package/dist/shard-router.js +211 -0
- package/dist/shard-supervisor.d.ts +36 -0
- package/dist/shard-supervisor.js +1230 -0
- package/dist/shard-worker.d.ts +62 -0
- package/dist/shard-worker.js +380 -0
- package/dist/supervisor-cloud-directory.d.ts +11 -0
- package/dist/supervisor-cloud-directory.js +66 -0
- package/package.json +9 -8
|
@@ -0,0 +1,620 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { createClient } from "redis";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { SHARD_CONTROL_MAX_BYTES, assertShardMessageSize, shardCloudRequestSchema, shardCloudResponseSchema, shardControlRequestSchema, shardControlResponseSchema, shardIdentitySchema, shardProcessHealthSchema, } from "./shard-contracts.js";
|
|
5
|
+
const LEASE_TTL_MS = 30_000;
|
|
6
|
+
const HEALTH_TTL_MS = 30_000;
|
|
7
|
+
const GENERATION_TTL_MS = 30_000;
|
|
8
|
+
const RESPONSE_TTL_SECONDS = 20;
|
|
9
|
+
const ROUTE_POLL_MS = 40;
|
|
10
|
+
const generationDescriptorSchema = z
|
|
11
|
+
.object({
|
|
12
|
+
version: z.literal(1),
|
|
13
|
+
applicationId: z.string().regex(/^\d{17,20}$/u),
|
|
14
|
+
environment: z.enum(["development", "test", "production"]),
|
|
15
|
+
generation: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u),
|
|
16
|
+
shardCount: z.number().int().min(1).max(4_096),
|
|
17
|
+
catalogueHash: z.string().regex(/^[a-f0-9]{64}$/u),
|
|
18
|
+
})
|
|
19
|
+
.strict();
|
|
20
|
+
const deploymentOrderSchema = z
|
|
21
|
+
.number()
|
|
22
|
+
.int()
|
|
23
|
+
.positive()
|
|
24
|
+
.max(Number.MAX_SAFE_INTEGER);
|
|
25
|
+
const activeGenerationSchema = generationDescriptorSchema.extend({
|
|
26
|
+
deploymentOrder: deploymentOrderSchema,
|
|
27
|
+
});
|
|
28
|
+
const leaseSchema = z
|
|
29
|
+
.object({
|
|
30
|
+
generation: generationDescriptorSchema.shape.generation,
|
|
31
|
+
deploymentOrder: deploymentOrderSchema,
|
|
32
|
+
supervisorId: z.uuid(),
|
|
33
|
+
shardId: z.number().int().min(0).max(4_095),
|
|
34
|
+
fencingToken: z.number().int().positive(),
|
|
35
|
+
})
|
|
36
|
+
.strict();
|
|
37
|
+
const legacyLeaseSchema = leaseSchema.omit({ deploymentOrder: true }).strict();
|
|
38
|
+
const leaderLeaseSchema = z
|
|
39
|
+
.object({
|
|
40
|
+
generation: generationDescriptorSchema.shape.generation,
|
|
41
|
+
supervisorId: z.uuid(),
|
|
42
|
+
fencingToken: z.number().int().positive(),
|
|
43
|
+
})
|
|
44
|
+
.strict();
|
|
45
|
+
const remoteRequestSchema = z.discriminatedUnion("target", [
|
|
46
|
+
z
|
|
47
|
+
.object({
|
|
48
|
+
version: z.literal(1),
|
|
49
|
+
type: z.literal("helyx.remote.request"),
|
|
50
|
+
target: z.literal("shard"),
|
|
51
|
+
targetShardId: z.number().int().min(0).max(4_095),
|
|
52
|
+
request: shardControlRequestSchema,
|
|
53
|
+
})
|
|
54
|
+
.strict(),
|
|
55
|
+
z
|
|
56
|
+
.object({
|
|
57
|
+
version: z.literal(1),
|
|
58
|
+
type: z.literal("helyx.remote.request"),
|
|
59
|
+
target: z.literal("leader"),
|
|
60
|
+
request: shardControlRequestSchema,
|
|
61
|
+
})
|
|
62
|
+
.strict(),
|
|
63
|
+
z
|
|
64
|
+
.object({
|
|
65
|
+
version: z.literal(1),
|
|
66
|
+
type: z.literal("helyx.remote.request"),
|
|
67
|
+
target: z.literal("leader-cloud"),
|
|
68
|
+
generation: generationDescriptorSchema.shape.generation,
|
|
69
|
+
fencingToken: z.number().int().positive(),
|
|
70
|
+
request: shardCloudRequestSchema,
|
|
71
|
+
})
|
|
72
|
+
.strict(),
|
|
73
|
+
]);
|
|
74
|
+
const storedHealthSchema = z
|
|
75
|
+
.object({
|
|
76
|
+
identity: shardIdentitySchema,
|
|
77
|
+
health: shardProcessHealthSchema,
|
|
78
|
+
})
|
|
79
|
+
.strict();
|
|
80
|
+
export class RedisShardCoordinator {
|
|
81
|
+
supervisorId = randomUUID();
|
|
82
|
+
#command;
|
|
83
|
+
#subscriber;
|
|
84
|
+
#prefix;
|
|
85
|
+
#onError;
|
|
86
|
+
#handler;
|
|
87
|
+
#leaderHandler;
|
|
88
|
+
#cloudHandler;
|
|
89
|
+
constructor(redisUrl, namespace, options = {}) {
|
|
90
|
+
this.#prefix = `helyx:shard:v1:${namespace}`;
|
|
91
|
+
this.#command = createClient({ url: redisUrl });
|
|
92
|
+
this.#subscriber = this.#command.duplicate();
|
|
93
|
+
this.#onError = options.onError ?? (() => undefined);
|
|
94
|
+
this.#command.on("error", (error) => this.#report(error));
|
|
95
|
+
this.#subscriber.on("error", (error) => this.#report(error));
|
|
96
|
+
}
|
|
97
|
+
async start(handler, leaderHandler, cloudHandler) {
|
|
98
|
+
this.#handler = handler;
|
|
99
|
+
this.#leaderHandler = leaderHandler;
|
|
100
|
+
this.#cloudHandler = cloudHandler;
|
|
101
|
+
await Promise.all([this.#command.connect(), this.#subscriber.connect()]);
|
|
102
|
+
await this.#subscriber.subscribe(this.#requestChannel(), (raw) => {
|
|
103
|
+
void this.#handleRemoteRequest(raw).catch((error) => this.#report(error));
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
async checkReady() {
|
|
107
|
+
if (!this.#command.isReady || !this.#subscriber.isReady)
|
|
108
|
+
throw new Error("Shard Redis coordinator is unavailable");
|
|
109
|
+
await this.#command.ping();
|
|
110
|
+
}
|
|
111
|
+
async assertGeneration(descriptor, proposedDeploymentOrder) {
|
|
112
|
+
const accepted = generationDescriptorSchema.parse(descriptor);
|
|
113
|
+
const deploymentOrder = deploymentOrderSchema.parse(proposedDeploymentOrder);
|
|
114
|
+
const active = activeGenerationSchema.parse({
|
|
115
|
+
...accepted,
|
|
116
|
+
deploymentOrder,
|
|
117
|
+
});
|
|
118
|
+
const serialized = JSON.stringify(active);
|
|
119
|
+
const created = await this.#command.set(this.#generationKey(), serialized, {
|
|
120
|
+
NX: true,
|
|
121
|
+
PX: GENERATION_TTL_MS,
|
|
122
|
+
});
|
|
123
|
+
if (created === "OK")
|
|
124
|
+
return deploymentOrder;
|
|
125
|
+
const current = await this.#command.get(this.#generationKey());
|
|
126
|
+
const parsed = activeGenerationSchema.safeParse(parseJson(current ?? ""));
|
|
127
|
+
if (parsed.success) {
|
|
128
|
+
if (JSON.stringify(withoutDeploymentOrder(parsed.data)) !==
|
|
129
|
+
JSON.stringify(accepted))
|
|
130
|
+
throw new Error("Shard coordination namespace is owned by an incompatible generation");
|
|
131
|
+
if (!current ||
|
|
132
|
+
!(await compareAndExpire(this.#command, this.#generationKey(), current, GENERATION_TTL_MS)))
|
|
133
|
+
throw new Error("Shard coordination generation changed while it was being validated");
|
|
134
|
+
return parsed.data.deploymentOrder;
|
|
135
|
+
}
|
|
136
|
+
const legacy = generationDescriptorSchema.safeParse(parseJson(current ?? ""));
|
|
137
|
+
if (!legacy.success ||
|
|
138
|
+
JSON.stringify(legacy.data) !== JSON.stringify(accepted) ||
|
|
139
|
+
!current)
|
|
140
|
+
throw new Error("Shard coordination namespace is owned by an incompatible generation");
|
|
141
|
+
const upgraded = await compareAndReplace(this.#command, this.#generationKey(), current, serialized, GENERATION_TTL_MS);
|
|
142
|
+
if (upgraded)
|
|
143
|
+
return deploymentOrder;
|
|
144
|
+
const winner = activeGenerationSchema.safeParse(parseJson((await this.#command.get(this.#generationKey())) ?? ""));
|
|
145
|
+
if (winner.success &&
|
|
146
|
+
JSON.stringify(withoutDeploymentOrder(winner.data)) ===
|
|
147
|
+
JSON.stringify(accepted))
|
|
148
|
+
return winner.data.deploymentOrder;
|
|
149
|
+
throw new Error("Shard coordination namespace is owned by an incompatible generation");
|
|
150
|
+
}
|
|
151
|
+
async refreshGeneration(descriptor, deploymentOrder) {
|
|
152
|
+
return compareAndExpire(this.#command, this.#generationKey(), JSON.stringify(activeGenerationSchema.parse({
|
|
153
|
+
...generationDescriptorSchema.parse(descriptor),
|
|
154
|
+
deploymentOrder,
|
|
155
|
+
})), GENERATION_TTL_MS);
|
|
156
|
+
}
|
|
157
|
+
async claimShard(descriptor, shardId, deploymentOrder) {
|
|
158
|
+
const accepted = generationDescriptorSchema.parse(descriptor);
|
|
159
|
+
const acceptedDeploymentOrder = deploymentOrderSchema.parse(deploymentOrder);
|
|
160
|
+
if (!Number.isInteger(shardId) ||
|
|
161
|
+
shardId < 0 ||
|
|
162
|
+
shardId >= accepted.shardCount)
|
|
163
|
+
throw new Error("Shard claim ID is outside the active generation");
|
|
164
|
+
const leasePrefix = JSON.stringify({
|
|
165
|
+
generation: accepted.generation,
|
|
166
|
+
deploymentOrder: acceptedDeploymentOrder,
|
|
167
|
+
supervisorId: this.supervisorId,
|
|
168
|
+
shardId,
|
|
169
|
+
}).slice(0, -1);
|
|
170
|
+
const result = await this.#command.eval(`-- atomic-shard-claim
|
|
171
|
+
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
|
172
|
+
if redis.call('EXISTS', KEYS[2]) == 1 then return -1 end
|
|
173
|
+
local fencing_token = redis.call('INCR', KEYS[3])
|
|
174
|
+
local lease = ARGV[2] .. ',"fencingToken":' ..
|
|
175
|
+
tostring(fencing_token) .. '}'
|
|
176
|
+
redis.call('SET', KEYS[2], lease, 'PX', ARGV[3], 'NX')
|
|
177
|
+
return fencing_token`, {
|
|
178
|
+
keys: [
|
|
179
|
+
this.#generationKey(),
|
|
180
|
+
this.#shardLeaseKey(shardId),
|
|
181
|
+
this.#fenceKey(acceptedDeploymentOrder),
|
|
182
|
+
],
|
|
183
|
+
arguments: [
|
|
184
|
+
JSON.stringify({
|
|
185
|
+
...accepted,
|
|
186
|
+
deploymentOrder: acceptedDeploymentOrder,
|
|
187
|
+
}),
|
|
188
|
+
leasePrefix,
|
|
189
|
+
String(LEASE_TTL_MS),
|
|
190
|
+
],
|
|
191
|
+
});
|
|
192
|
+
if (typeof result !== "number" || result < 1)
|
|
193
|
+
return undefined;
|
|
194
|
+
const lease = leaseSchema.parse({
|
|
195
|
+
generation: accepted.generation,
|
|
196
|
+
deploymentOrder: acceptedDeploymentOrder,
|
|
197
|
+
supervisorId: this.supervisorId,
|
|
198
|
+
shardId,
|
|
199
|
+
fencingToken: result,
|
|
200
|
+
});
|
|
201
|
+
return lease;
|
|
202
|
+
}
|
|
203
|
+
renewShard(lease) {
|
|
204
|
+
const accepted = leaseSchema.parse(lease);
|
|
205
|
+
return compareAndExpire(this.#command, this.#shardLeaseKey(accepted.shardId), JSON.stringify(accepted), LEASE_TTL_MS);
|
|
206
|
+
}
|
|
207
|
+
releaseShard(lease) {
|
|
208
|
+
const accepted = leaseSchema.parse(lease);
|
|
209
|
+
return compareAndDelete(this.#command, this.#shardLeaseKey(accepted.shardId), JSON.stringify(accepted));
|
|
210
|
+
}
|
|
211
|
+
async getShardLease(shardId) {
|
|
212
|
+
const raw = await this.#command.get(this.#shardLeaseKey(shardId));
|
|
213
|
+
return parseLease(raw);
|
|
214
|
+
}
|
|
215
|
+
async getShardLeases(shardCount) {
|
|
216
|
+
if (!Number.isInteger(shardCount) || shardCount < 1 || shardCount > 4_096)
|
|
217
|
+
throw new Error("Shard lease count is invalid");
|
|
218
|
+
const values = await this.#command.mGet(Array.from({ length: shardCount }, (_, shardId) => this.#shardLeaseKey(shardId)));
|
|
219
|
+
return values.flatMap((raw) => {
|
|
220
|
+
const lease = parseLease(raw);
|
|
221
|
+
return lease ? [lease] : [];
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
async acquireLeader(descriptor, deploymentOrder) {
|
|
225
|
+
const accepted = generationDescriptorSchema.parse(descriptor);
|
|
226
|
+
const acceptedDeploymentOrder = deploymentOrderSchema.parse(deploymentOrder);
|
|
227
|
+
const leasePrefix = JSON.stringify({
|
|
228
|
+
generation: accepted.generation,
|
|
229
|
+
supervisorId: this.supervisorId,
|
|
230
|
+
}).slice(0, -1);
|
|
231
|
+
const result = await this.#command.eval(`-- atomic-leader-claim
|
|
232
|
+
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
|
233
|
+
if redis.call('EXISTS', KEYS[2]) == 1 then return -1 end
|
|
234
|
+
local fencing_token = redis.call('INCR', KEYS[3])
|
|
235
|
+
local lease = ARGV[2] .. ',"fencingToken":' ..
|
|
236
|
+
tostring(fencing_token) .. '}'
|
|
237
|
+
redis.call('SET', KEYS[2], lease, 'PX', ARGV[3], 'NX')
|
|
238
|
+
return fencing_token`, {
|
|
239
|
+
keys: [
|
|
240
|
+
this.#generationKey(),
|
|
241
|
+
this.#leaderLeaseKey(),
|
|
242
|
+
this.#leaderFenceKey(),
|
|
243
|
+
],
|
|
244
|
+
arguments: [
|
|
245
|
+
JSON.stringify({
|
|
246
|
+
...accepted,
|
|
247
|
+
deploymentOrder: acceptedDeploymentOrder,
|
|
248
|
+
}),
|
|
249
|
+
leasePrefix,
|
|
250
|
+
String(LEASE_TTL_MS),
|
|
251
|
+
],
|
|
252
|
+
});
|
|
253
|
+
if (typeof result !== "number" || result < 1)
|
|
254
|
+
return undefined;
|
|
255
|
+
const lease = leaderLeaseSchema.parse({
|
|
256
|
+
generation: accepted.generation,
|
|
257
|
+
supervisorId: this.supervisorId,
|
|
258
|
+
fencingToken: result,
|
|
259
|
+
});
|
|
260
|
+
return lease;
|
|
261
|
+
}
|
|
262
|
+
renewLeader(lease) {
|
|
263
|
+
const accepted = leaderLeaseSchema.parse(lease);
|
|
264
|
+
return compareAndExpire(this.#command, this.#leaderLeaseKey(), JSON.stringify(accepted), LEASE_TTL_MS);
|
|
265
|
+
}
|
|
266
|
+
releaseLeader(lease) {
|
|
267
|
+
const accepted = leaderLeaseSchema.parse(lease);
|
|
268
|
+
return compareAndDelete(this.#command, this.#leaderLeaseKey(), JSON.stringify(accepted));
|
|
269
|
+
}
|
|
270
|
+
async getLeaderLease() {
|
|
271
|
+
const raw = await this.#command.get(this.#leaderLeaseKey());
|
|
272
|
+
if (!raw)
|
|
273
|
+
return undefined;
|
|
274
|
+
const parsed = leaderLeaseSchema.safeParse(parseJson(raw));
|
|
275
|
+
return parsed.success ? parsed.data : undefined;
|
|
276
|
+
}
|
|
277
|
+
async routeRemote(lease, request) {
|
|
278
|
+
const acceptedLease = leaseSchema.parse(lease);
|
|
279
|
+
const acceptedRequest = shardControlRequestSchema.parse(request);
|
|
280
|
+
if (acceptedRequest.generation !== acceptedLease.generation ||
|
|
281
|
+
acceptedRequest.fencingToken !== acceptedLease.fencingToken)
|
|
282
|
+
throw new Error("Shard route ownership changed");
|
|
283
|
+
const envelope = remoteRequestSchema.parse({
|
|
284
|
+
version: 1,
|
|
285
|
+
type: "helyx.remote.request",
|
|
286
|
+
target: "shard",
|
|
287
|
+
targetShardId: acceptedLease.shardId,
|
|
288
|
+
request: acceptedRequest,
|
|
289
|
+
});
|
|
290
|
+
assertShardMessageSize(envelope);
|
|
291
|
+
const listeners = await this.#command.publish(this.#requestChannel(acceptedLease.supervisorId), JSON.stringify(envelope));
|
|
292
|
+
if (listeners < 1)
|
|
293
|
+
throw new Error("Owning shard supervisor is unavailable");
|
|
294
|
+
const deadline = new Date(acceptedRequest.deadlineAt).getTime();
|
|
295
|
+
const responseKey = this.#responseKey(acceptedRequest.requestId);
|
|
296
|
+
while (Date.now() < deadline) {
|
|
297
|
+
const raw = await this.#command.get(responseKey);
|
|
298
|
+
if (raw) {
|
|
299
|
+
await this.#command.del(responseKey);
|
|
300
|
+
const parsed = shardControlResponseSchema.parse(parseJson(raw));
|
|
301
|
+
assertShardMessageSize(parsed);
|
|
302
|
+
return parsed;
|
|
303
|
+
}
|
|
304
|
+
await wait(ROUTE_POLL_MS);
|
|
305
|
+
}
|
|
306
|
+
throw new Error("Shard control request timed out");
|
|
307
|
+
}
|
|
308
|
+
async routeLeader(lease, request) {
|
|
309
|
+
const acceptedLease = leaderLeaseSchema.parse(lease);
|
|
310
|
+
const acceptedRequest = shardControlRequestSchema.parse(request);
|
|
311
|
+
if (acceptedRequest.generation !== acceptedLease.generation ||
|
|
312
|
+
acceptedRequest.fencingToken !== acceptedLease.fencingToken)
|
|
313
|
+
throw new Error("Shard leader ownership changed");
|
|
314
|
+
const envelope = remoteRequestSchema.parse({
|
|
315
|
+
version: 1,
|
|
316
|
+
type: "helyx.remote.request",
|
|
317
|
+
target: "leader",
|
|
318
|
+
request: acceptedRequest,
|
|
319
|
+
});
|
|
320
|
+
assertShardMessageSize(envelope);
|
|
321
|
+
const listeners = await this.#command.publish(this.#requestChannel(acceptedLease.supervisorId), JSON.stringify(envelope));
|
|
322
|
+
if (listeners < 1)
|
|
323
|
+
throw new Error("Shard leader is unavailable");
|
|
324
|
+
return this.#waitForResponse(acceptedRequest);
|
|
325
|
+
}
|
|
326
|
+
async routeLeaderCloud(lease, request) {
|
|
327
|
+
const acceptedLease = leaderLeaseSchema.parse(lease);
|
|
328
|
+
const acceptedRequest = shardCloudRequestSchema.parse(request);
|
|
329
|
+
const currentLeader = await this.getLeaderLease();
|
|
330
|
+
if (!currentLeader ||
|
|
331
|
+
currentLeader.supervisorId !== acceptedLease.supervisorId ||
|
|
332
|
+
currentLeader.generation !== acceptedLease.generation ||
|
|
333
|
+
currentLeader.fencingToken !== acceptedLease.fencingToken)
|
|
334
|
+
throw new Error("Shard leader ownership changed");
|
|
335
|
+
const envelope = remoteRequestSchema.parse({
|
|
336
|
+
version: 1,
|
|
337
|
+
type: "helyx.remote.request",
|
|
338
|
+
target: "leader-cloud",
|
|
339
|
+
generation: acceptedLease.generation,
|
|
340
|
+
fencingToken: acceptedLease.fencingToken,
|
|
341
|
+
request: acceptedRequest,
|
|
342
|
+
});
|
|
343
|
+
assertShardMessageSize(envelope);
|
|
344
|
+
const listeners = await this.#command.publish(this.#requestChannel(acceptedLease.supervisorId), JSON.stringify(envelope));
|
|
345
|
+
if (listeners < 1)
|
|
346
|
+
throw new Error("Shard leader is unavailable");
|
|
347
|
+
const deadline = new Date(acceptedRequest.deadlineAt).getTime();
|
|
348
|
+
const responseKey = this.#responseKey(acceptedRequest.requestId);
|
|
349
|
+
while (Date.now() < deadline) {
|
|
350
|
+
const raw = await this.#command.get(responseKey);
|
|
351
|
+
if (raw) {
|
|
352
|
+
await this.#command.del(responseKey);
|
|
353
|
+
const parsed = shardCloudResponseSchema.parse(parseJson(raw));
|
|
354
|
+
assertShardMessageSize(parsed);
|
|
355
|
+
return parsed;
|
|
356
|
+
}
|
|
357
|
+
await wait(ROUTE_POLL_MS);
|
|
358
|
+
}
|
|
359
|
+
throw new Error("Shard cloud request timed out");
|
|
360
|
+
}
|
|
361
|
+
async reserveSessionStarts(count, available, resetAfterMs) {
|
|
362
|
+
if (!Number.isInteger(count) ||
|
|
363
|
+
count < 1 ||
|
|
364
|
+
!Number.isInteger(available) ||
|
|
365
|
+
available < 0 ||
|
|
366
|
+
!Number.isFinite(resetAfterMs) ||
|
|
367
|
+
resetAfterMs < 0)
|
|
368
|
+
throw new Error("Discord session-start reservation is invalid");
|
|
369
|
+
const result = await this.#command.eval(`local current = redis.call('GET', KEYS[1])
|
|
370
|
+
if not current then
|
|
371
|
+
current = ARGV[1]
|
|
372
|
+
redis.call('SET', KEYS[1], current, 'PX', ARGV[3], 'NX')
|
|
373
|
+
current = redis.call('GET', KEYS[1])
|
|
374
|
+
elseif tonumber(ARGV[1]) < tonumber(current) then
|
|
375
|
+
current = ARGV[1]
|
|
376
|
+
redis.call('SET', KEYS[1], current, 'PX', ARGV[3], 'XX')
|
|
377
|
+
end
|
|
378
|
+
if tonumber(current) < tonumber(ARGV[2]) then return 0 end
|
|
379
|
+
redis.call('DECRBY', KEYS[1], ARGV[2])
|
|
380
|
+
return 1`, {
|
|
381
|
+
keys: [`${this.#prefix}:session-starts`],
|
|
382
|
+
arguments: [
|
|
383
|
+
String(available),
|
|
384
|
+
String(count),
|
|
385
|
+
String(Math.max(1_000, resetAfterMs)),
|
|
386
|
+
],
|
|
387
|
+
});
|
|
388
|
+
return result === 1;
|
|
389
|
+
}
|
|
390
|
+
async acquireIdentifyPermit(shardId, maxConcurrency, timeoutMs) {
|
|
391
|
+
const bucket = shardId % maxConcurrency;
|
|
392
|
+
const deadline = Date.now() + timeoutMs;
|
|
393
|
+
while (Date.now() < deadline) {
|
|
394
|
+
const acquired = await this.#command.set(`${this.#prefix}:identify:${String(bucket)}`, `${this.supervisorId}:${String(shardId)}`, { NX: true, PX: 5_500 });
|
|
395
|
+
if (acquired === "OK")
|
|
396
|
+
return true;
|
|
397
|
+
await wait(100);
|
|
398
|
+
}
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
async setHealth(identity, health) {
|
|
402
|
+
const value = storedHealthSchema.parse({ identity, health });
|
|
403
|
+
await this.#command.set(this.#healthKey(identity.shardId), JSON.stringify(value), { PX: HEALTH_TTL_MS });
|
|
404
|
+
}
|
|
405
|
+
async getHealth(shardCount) {
|
|
406
|
+
if (!Number.isInteger(shardCount) || shardCount < 1 || shardCount > 4_096)
|
|
407
|
+
throw new Error("Shard health count is invalid");
|
|
408
|
+
const values = await this.#command.mGet(Array.from({ length: shardCount }, (_, shardId) => this.#healthKey(shardId)));
|
|
409
|
+
return values.flatMap((raw) => {
|
|
410
|
+
if (!raw)
|
|
411
|
+
return [];
|
|
412
|
+
const parsed = storedHealthSchema.safeParse(parseJson(raw));
|
|
413
|
+
return parsed.success ? [parsed.data] : [];
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
async stop() {
|
|
417
|
+
await Promise.allSettled([
|
|
418
|
+
this.#subscriber.isOpen
|
|
419
|
+
? this.#subscriber.unsubscribe(this.#requestChannel())
|
|
420
|
+
: Promise.resolve(),
|
|
421
|
+
this.#subscriber.isOpen ? this.#subscriber.quit() : Promise.resolve(),
|
|
422
|
+
this.#command.isOpen ? this.#command.quit() : Promise.resolve(),
|
|
423
|
+
]);
|
|
424
|
+
}
|
|
425
|
+
async #handleRemoteRequest(raw) {
|
|
426
|
+
if (Buffer.byteLength(raw, "utf8") > SHARD_CONTROL_MAX_BYTES)
|
|
427
|
+
return;
|
|
428
|
+
const envelope = remoteRequestSchema.safeParse(parseJson(raw));
|
|
429
|
+
if (!envelope.success || !this.#handler)
|
|
430
|
+
return;
|
|
431
|
+
if (new Date(envelope.data.request.deadlineAt).getTime() <= Date.now())
|
|
432
|
+
return;
|
|
433
|
+
if (envelope.data.target === "leader-cloud") {
|
|
434
|
+
const request = envelope.data.request;
|
|
435
|
+
const leader = await this.getLeaderLease();
|
|
436
|
+
if (!leader ||
|
|
437
|
+
leader.supervisorId !== this.supervisorId ||
|
|
438
|
+
leader.generation !== envelope.data.generation ||
|
|
439
|
+
leader.fencingToken !== envelope.data.fencingToken ||
|
|
440
|
+
!this.#cloudHandler)
|
|
441
|
+
return this.#writeCloudResponse({
|
|
442
|
+
version: 1,
|
|
443
|
+
type: "helyx.cloud.response",
|
|
444
|
+
requestId: request.requestId,
|
|
445
|
+
ok: false,
|
|
446
|
+
message: "Shard leader ownership changed",
|
|
447
|
+
});
|
|
448
|
+
const response = await this.#cloudHandler(request).catch(() => ({
|
|
449
|
+
version: 1,
|
|
450
|
+
type: "helyx.cloud.response",
|
|
451
|
+
requestId: request.requestId,
|
|
452
|
+
ok: false,
|
|
453
|
+
message: "Cloud leader is temporarily unavailable",
|
|
454
|
+
}));
|
|
455
|
+
await this.#writeCloudResponse(response);
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
if (envelope.data.target === "leader") {
|
|
459
|
+
const request = envelope.data.request;
|
|
460
|
+
const leader = await this.getLeaderLease();
|
|
461
|
+
if (!leader ||
|
|
462
|
+
leader.supervisorId !== this.supervisorId ||
|
|
463
|
+
leader.generation !== request.generation ||
|
|
464
|
+
leader.fencingToken !== request.fencingToken ||
|
|
465
|
+
!this.#leaderHandler)
|
|
466
|
+
return this.#writeResponse({
|
|
467
|
+
version: 1,
|
|
468
|
+
type: "helyx.control.response",
|
|
469
|
+
requestId: request.requestId,
|
|
470
|
+
ok: false,
|
|
471
|
+
status: 503,
|
|
472
|
+
message: "Shard leader ownership changed",
|
|
473
|
+
});
|
|
474
|
+
const response = await this.#leaderHandler(request).catch(() => ({
|
|
475
|
+
version: 1,
|
|
476
|
+
type: "helyx.control.response",
|
|
477
|
+
requestId: request.requestId,
|
|
478
|
+
ok: false,
|
|
479
|
+
status: 503,
|
|
480
|
+
message: "Shard leader is temporarily unavailable",
|
|
481
|
+
}));
|
|
482
|
+
await this.#writeResponse(response);
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
const { targetShardId, request } = envelope.data;
|
|
486
|
+
const lease = await this.getShardLease(targetShardId);
|
|
487
|
+
if (!lease ||
|
|
488
|
+
lease.supervisorId !== this.supervisorId ||
|
|
489
|
+
lease.generation !== request.generation ||
|
|
490
|
+
lease.fencingToken !== request.fencingToken)
|
|
491
|
+
return this.#writeResponse({
|
|
492
|
+
version: 1,
|
|
493
|
+
type: "helyx.control.response",
|
|
494
|
+
requestId: request.requestId,
|
|
495
|
+
ok: false,
|
|
496
|
+
status: 503,
|
|
497
|
+
message: "Shard ownership changed",
|
|
498
|
+
});
|
|
499
|
+
const response = await this.#handler(targetShardId, request).catch(() => ({
|
|
500
|
+
version: 1,
|
|
501
|
+
type: "helyx.control.response",
|
|
502
|
+
requestId: request.requestId,
|
|
503
|
+
ok: false,
|
|
504
|
+
status: 503,
|
|
505
|
+
message: "Owning shard is temporarily unavailable",
|
|
506
|
+
}));
|
|
507
|
+
await this.#writeResponse(response);
|
|
508
|
+
}
|
|
509
|
+
async #writeResponse(response) {
|
|
510
|
+
const accepted = shardControlResponseSchema.parse(response);
|
|
511
|
+
assertShardMessageSize(accepted);
|
|
512
|
+
await this.#command.set(this.#responseKey(accepted.requestId), JSON.stringify(accepted), { EX: RESPONSE_TTL_SECONDS });
|
|
513
|
+
}
|
|
514
|
+
async #writeCloudResponse(response) {
|
|
515
|
+
const accepted = shardCloudResponseSchema.parse(response);
|
|
516
|
+
assertShardMessageSize(accepted);
|
|
517
|
+
await this.#command.set(this.#responseKey(accepted.requestId), JSON.stringify(accepted), { EX: RESPONSE_TTL_SECONDS });
|
|
518
|
+
}
|
|
519
|
+
async #waitForResponse(request) {
|
|
520
|
+
const deadline = new Date(request.deadlineAt).getTime();
|
|
521
|
+
const responseKey = this.#responseKey(request.requestId);
|
|
522
|
+
while (Date.now() < deadline) {
|
|
523
|
+
const raw = await this.#command.get(responseKey);
|
|
524
|
+
if (raw) {
|
|
525
|
+
await this.#command.del(responseKey);
|
|
526
|
+
const parsed = shardControlResponseSchema.parse(parseJson(raw));
|
|
527
|
+
assertShardMessageSize(parsed);
|
|
528
|
+
return parsed;
|
|
529
|
+
}
|
|
530
|
+
await wait(ROUTE_POLL_MS);
|
|
531
|
+
}
|
|
532
|
+
throw new Error("Shard control request timed out");
|
|
533
|
+
}
|
|
534
|
+
#generationKey() {
|
|
535
|
+
return `${this.#prefix}:generation`;
|
|
536
|
+
}
|
|
537
|
+
#shardLeaseKey(shardId) {
|
|
538
|
+
return `${this.#prefix}:lease:shard:${String(shardId)}`;
|
|
539
|
+
}
|
|
540
|
+
#fenceKey(deploymentOrder) {
|
|
541
|
+
return `${this.#prefix}:fence:deployment:${String(deploymentOrder)}:claim`;
|
|
542
|
+
}
|
|
543
|
+
#leaderLeaseKey() {
|
|
544
|
+
return `${this.#prefix}:lease:leader`;
|
|
545
|
+
}
|
|
546
|
+
#leaderFenceKey() {
|
|
547
|
+
return `${this.#prefix}:fence:leader`;
|
|
548
|
+
}
|
|
549
|
+
#requestChannel(supervisorId = this.supervisorId) {
|
|
550
|
+
return `${this.#prefix}:request:${supervisorId}`;
|
|
551
|
+
}
|
|
552
|
+
#responseKey(requestId) {
|
|
553
|
+
return `${this.#prefix}:response:${requestId}`;
|
|
554
|
+
}
|
|
555
|
+
#healthKey(shardId) {
|
|
556
|
+
return `${this.#prefix}:health:${String(shardId)}`;
|
|
557
|
+
}
|
|
558
|
+
#report(error) {
|
|
559
|
+
this.#onError(error instanceof Error ? error : new Error("Unknown shard Redis failure"));
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
async function compareAndExpire(client, key, expected, milliseconds) {
|
|
563
|
+
const result = await client.eval(`if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
564
|
+
return redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
|
565
|
+
end
|
|
566
|
+
return 0`, { keys: [key], arguments: [expected, String(milliseconds)] });
|
|
567
|
+
return result === 1;
|
|
568
|
+
}
|
|
569
|
+
async function compareAndDelete(client, key, expected) {
|
|
570
|
+
const result = await client.eval(`if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
571
|
+
return redis.call('DEL', KEYS[1])
|
|
572
|
+
end
|
|
573
|
+
return 0`, { keys: [key], arguments: [expected] });
|
|
574
|
+
return result === 1;
|
|
575
|
+
}
|
|
576
|
+
async function compareAndReplace(client, key, expected, replacement, milliseconds) {
|
|
577
|
+
const result = await client.eval(`if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
578
|
+
redis.call('SET', KEYS[1], ARGV[2], 'PX', ARGV[3])
|
|
579
|
+
return 1
|
|
580
|
+
end
|
|
581
|
+
return 0`, {
|
|
582
|
+
keys: [key],
|
|
583
|
+
arguments: [expected, replacement, String(milliseconds)],
|
|
584
|
+
});
|
|
585
|
+
return result === 1;
|
|
586
|
+
}
|
|
587
|
+
function parseJson(value) {
|
|
588
|
+
try {
|
|
589
|
+
return JSON.parse(value);
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
function parseLease(raw) {
|
|
596
|
+
if (!raw)
|
|
597
|
+
return undefined;
|
|
598
|
+
const value = parseJson(raw);
|
|
599
|
+
const parsed = leaseSchema.safeParse(value);
|
|
600
|
+
if (parsed.success)
|
|
601
|
+
return parsed.data;
|
|
602
|
+
const legacy = legacyLeaseSchema.safeParse(value);
|
|
603
|
+
return legacy.success
|
|
604
|
+
? leaseSchema.parse({ ...legacy.data, deploymentOrder: 1 })
|
|
605
|
+
: undefined;
|
|
606
|
+
}
|
|
607
|
+
function withoutDeploymentOrder(active) {
|
|
608
|
+
return {
|
|
609
|
+
version: active.version,
|
|
610
|
+
applicationId: active.applicationId,
|
|
611
|
+
environment: active.environment,
|
|
612
|
+
generation: active.generation,
|
|
613
|
+
shardCount: active.shardCount,
|
|
614
|
+
catalogueHash: active.catalogueHash,
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
function wait(milliseconds) {
|
|
618
|
+
return new Promise((resolveWait) => setTimeout(resolveWait, milliseconds));
|
|
619
|
+
}
|
|
620
|
+
//# sourceMappingURL=shard-coordinator.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type BotControlHandler } from "./control-server.js";
|
|
2
|
+
import { type ShardControlRequest, type ShardControlResponse } from "./shard-contracts.js";
|
|
3
|
+
import type { ShardCoordinator, ShardLeaderLease, ShardLease } from "./shard-coordinator.js";
|
|
4
|
+
type LocalDispatcher = (shardId: number, request: ShardControlRequest) => Promise<ShardControlResponse>;
|
|
5
|
+
export declare class ShardControlRouter implements BotControlHandler {
|
|
6
|
+
#private;
|
|
7
|
+
private readonly input;
|
|
8
|
+
constructor(input: {
|
|
9
|
+
generation: string;
|
|
10
|
+
shardCount: number;
|
|
11
|
+
localLeases: ReadonlyMap<number, ShardLease>;
|
|
12
|
+
dispatchLocal: LocalDispatcher;
|
|
13
|
+
privacy?: Pick<BotControlHandler, "listPrivacyProviders" | "capturePrivacy" | "erasePrivacy">;
|
|
14
|
+
localLeaderLease?: ShardLeaderLease;
|
|
15
|
+
coordinator?: ShardCoordinator;
|
|
16
|
+
});
|
|
17
|
+
getGuildState(guildId: string): ReturnType<BotControlHandler["getGuildState"]>;
|
|
18
|
+
setModuleStatus(input: Parameters<BotControlHandler["setModuleStatus"]>[0]): ReturnType<BotControlHandler["setModuleStatus"]>;
|
|
19
|
+
updateConfiguration(input: Parameters<BotControlHandler["updateConfiguration"]>[0]): ReturnType<BotControlHandler["updateConfiguration"]>;
|
|
20
|
+
updatePermissions(input: Parameters<BotControlHandler["updatePermissions"]>[0]): ReturnType<BotControlHandler["updatePermissions"]>;
|
|
21
|
+
updateLoggingPolicy(input: Parameters<BotControlHandler["updateLoggingPolicy"]>[0]): ReturnType<BotControlHandler["updateLoggingPolicy"]>;
|
|
22
|
+
executeDashboardAction(input: Parameters<BotControlHandler["executeDashboardAction"]>[0]): ReturnType<BotControlHandler["executeDashboardAction"]>;
|
|
23
|
+
listPrivacyProviders(): ReturnType<BotControlHandler["listPrivacyProviders"]>;
|
|
24
|
+
capturePrivacy(input: Parameters<BotControlHandler["capturePrivacy"]>[0]): ReturnType<BotControlHandler["capturePrivacy"]>;
|
|
25
|
+
erasePrivacy(input: Parameters<BotControlHandler["erasePrivacy"]>[0]): ReturnType<BotControlHandler["erasePrivacy"]>;
|
|
26
|
+
}
|
|
27
|
+
export {};
|
|
28
|
+
//# sourceMappingURL=shard-router.d.ts.map
|