@smithers-orchestrator/server 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/gateway.js ADDED
@@ -0,0 +1,3402 @@
1
+ // @smithers-type-exports-begin
2
+ /** @typedef {import("./EventFrame.js").EventFrame} EventFrame */
3
+ /** @typedef {import("./GatewayDefaults.js").GatewayDefaults} GatewayDefaults */
4
+ /** @typedef {import("./GatewayTokenGrant.js").GatewayTokenGrant} GatewayTokenGrant */
5
+ /** @typedef {import("./HelloResponse.js").HelloResponse} HelloResponse */
6
+ // @smithers-type-exports-end
7
+
8
+ import { createServer } from "node:http";
9
+ import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto";
10
+ import { CronExpressionParser } from "cron-parser";
11
+ import { Effect, Metric } from "effect";
12
+ import { WebSocketServer } from "ws";
13
+ import { runWorkflow } from "@smithers-orchestrator/engine";
14
+ import { approveNode, denyNode } from "@smithers-orchestrator/engine/approvals";
15
+ import { signalRun } from "@smithers-orchestrator/engine/signals";
16
+ import { SmithersDb } from "@smithers-orchestrator/db/adapter";
17
+ import { computeRunStateFromRow } from "@smithers-orchestrator/db/runState";
18
+ import { ensureSmithersTables } from "@smithers-orchestrator/db/ensure";
19
+ import { devtoolsActiveSubscribers, devtoolsBackpressureDisconnectTotal, devtoolsDeltaBuildMs, devtoolsEventBytes, devtoolsEventTotal, devtoolsSnapshotBuildMs, devtoolsSubscribeTotal, gatewayApprovalDecisionsTotal, gatewayAuthEventsTotal, gatewayConnectionsActive, gatewayConnectionsClosedTotal, gatewayConnectionsTotal, gatewayCronTriggersTotal, gatewayErrorsTotal, gatewayHeartbeatTicksTotal, gatewayMessagesReceivedTotal, gatewayMessagesSentTotal, gatewayRpcCallsTotal, gatewayRpcDuration, gatewayRunsCompletedTotal, gatewayRunsStartedTotal, gatewaySignalsTotal, gatewayWebhooksReceivedTotal, gatewayWebhooksRejectedTotal, gatewayWebhooksVerifiedTotal, } from "@smithers-orchestrator/observability/metrics";
20
+ import { runFork, runPromise } from "./smithersRuntime.js";
21
+ import { prometheusContentType, renderPrometheusMetrics } from "@smithers-orchestrator/observability";
22
+ import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
23
+ import { errorToJson } from "@smithers-orchestrator/errors/errorToJson";
24
+ import { isSmithersError } from "@smithers-orchestrator/errors/isSmithersError";
25
+ import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
26
+ import { assertJsonPayloadWithinBounds, assertOptionalStringMaxLength, assertPositiveFiniteInteger, } from "@smithers-orchestrator/db/input-bounds";
27
+ import { loadLatestSnapshot, loadSnapshot } from "@smithers-orchestrator/time-travel/snapshot";
28
+ import { diffRawSnapshots } from "@smithers-orchestrator/time-travel/diff";
29
+ import { getNodeOutputRoute } from "./gatewayRoutes/getNodeOutput.js";
30
+ import { NodeOutputRouteError } from "./gatewayRoutes/NodeOutputRouteError.js";
31
+ import { getNodeDiffRoute } from "./gatewayRoutes/getNodeDiff.js";
32
+ import { DevToolsRouteError, getDevToolsSnapshotRoute, validateFrameNoInput, validateFromSeqInput, validateRunId } from "./gatewayRoutes/getDevToolsSnapshot.js";
33
+ import { streamDevToolsRoute } from "./gatewayRoutes/streamDevTools.js";
34
+ import { jumpToFrameRoute, JumpToFrameError } from "./gatewayRoutes/jumpToFrame.js";
35
+ import { writeRewindAuditRow } from "@smithers-orchestrator/time-travel/writeRewindAuditRow";
36
+ import { recoverInProgressRewindAudits } from "@smithers-orchestrator/time-travel/recoverInProgressRewindAudits";
37
+ /** @typedef {import("./GatewayWebhookRunConfig.js").GatewayWebhookRunConfig} GatewayWebhookRunConfig */
38
+ /** @typedef {import("./GatewayWebhookSignalConfig.js").GatewayWebhookSignalConfig} GatewayWebhookSignalConfig */
39
+ /** @typedef {import("./ConnectRequest.js").ConnectRequest} ConnectRequest */
40
+ /** @typedef {import("./GatewayAuthConfig.js").GatewayAuthConfig} GatewayAuthConfig */
41
+ /** @typedef {import("./GatewayOptions.js").GatewayOptions} GatewayOptions */
42
+ /** @typedef {import("./GatewayWebhookConfig.js").GatewayWebhookConfig} GatewayWebhookConfig */
43
+ /** @typedef {import("node:http").IncomingMessage} IncomingMessage */
44
+ /** @typedef {import("./RequestFrame.js").RequestFrame} RequestFrame */
45
+ /** @typedef {import("./ResponseFrame.js").ResponseFrame} ResponseFrame */
46
+ /** @typedef {import("node:http").ServerResponse} ServerResponse */
47
+ /** @typedef {import("@smithers-orchestrator/components/SmithersWorkflow").SmithersWorkflow<unknown>} SmithersWorkflow */
48
+ /** @typedef {import("@smithers-orchestrator/observability/SmithersEvent").SmithersEvent} SmithersEvent */
49
+ /** @typedef {Record<string, string | number | null | undefined>} GatewayMetricLabels */
50
+ /** @typedef {"ws" | "http"} GatewayTransport */
51
+ /**
52
+ * @typedef {{
53
+ * connectionId?: string;
54
+ * role?: string;
55
+ * scopes?: string[];
56
+ * userId?: string | null;
57
+ * origin?: string;
58
+ * transport?: GatewayTransport;
59
+ * }} GatewayRequestContext
60
+ */
61
+ /**
62
+ * @typedef {{
63
+ * id: string;
64
+ * ws?: unknown;
65
+ * role: string;
66
+ * scopes: string[];
67
+ * userId: string | null;
68
+ * subscribe?: Set<string>;
69
+ * heartbeat?: unknown;
70
+ * lastActivity?: number;
71
+ * closed?: boolean;
72
+ * } & Record<string, unknown>} ConnectionState
73
+ */
74
+ /**
75
+ * @typedef {{
76
+ * role: string;
77
+ * scopes: string[];
78
+ * userId?: string | null;
79
+ * connectionId?: string;
80
+ * }} RunStartAuthContext
81
+ */
82
+ /**
83
+ * @typedef {{
84
+ * workflow: SmithersWorkflow;
85
+ * adapter: SmithersDb;
86
+ * key: string;
87
+ * schedule?: string;
88
+ * webhook?: GatewayWebhookConfig;
89
+ * }} RegisteredWorkflow
90
+ */
91
+ /**
92
+ * @typedef {{
93
+ * runId: string;
94
+ * workflowKey: string;
95
+ * workflow: SmithersWorkflow;
96
+ * adapter: SmithersDb;
97
+ * }} ResolvedRun
98
+ */
99
+
100
+ const DEFAULT_PROTOCOL = 1;
101
+ const DEFAULT_HEARTBEAT_MS = 15_000;
102
+ const DEFAULT_MAX_BODY_BYTES = 1_048_576;
103
+ const DEFAULT_MAX_CONNECTIONS = 1_000;
104
+ export const GATEWAY_RPC_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BODY_BYTES;
105
+ export const GATEWAY_RPC_MAX_DEPTH = 32;
106
+ export const GATEWAY_RPC_MAX_ARRAY_LENGTH = 256;
107
+ export const GATEWAY_RPC_MAX_STRING_LENGTH = 16 * 1024;
108
+ export const GATEWAY_METHOD_NAME_MAX_LENGTH = 64;
109
+ export const GATEWAY_FRAME_ID_MAX_LENGTH = 128;
110
+ export const GATEWAY_RPC_INPUT_MAX_BYTES = GATEWAY_RPC_MAX_PAYLOAD_BYTES;
111
+ export const GATEWAY_RPC_INPUT_MAX_DEPTH = GATEWAY_RPC_MAX_DEPTH;
112
+ const GATEWAY_METHOD_NAME_PATTERN = /^[a-z][a-zA-Z0-9]*(?:\.[a-z][a-zA-Z0-9]*)*$/;
113
+ const ACCESS_RANK = {
114
+ read: 1,
115
+ execute: 2,
116
+ approve: 3,
117
+ admin: 4,
118
+ };
119
+ const METHOD_ACCESS = {
120
+ health: "read",
121
+ "runs.list": "read",
122
+ "runs.get": "read",
123
+ "runs.diff": "read",
124
+ getNodeDiff: "read",
125
+ "devtools.getNodeDiff": "read",
126
+ getNodeOutput: "read",
127
+ "devtools.getNodeOutput": "read",
128
+ getDevToolsSnapshot: "read",
129
+ streamDevTools: "read",
130
+ // jumpToFrame authorizes per-request: owner OR admin role may rewind.
131
+ // Require only `execute` scope so non-admin owners are not pre-blocked
132
+ // at the scope gate; the run handler performs the final auth check.
133
+ jumpToFrame: "execute",
134
+ "devtools.jumpToFrame": "execute",
135
+ "frames.list": "read",
136
+ "frames.get": "read",
137
+ "attempts.list": "read",
138
+ "attempts.get": "read",
139
+ "approvals.list": "read",
140
+ "runs.create": "execute",
141
+ "runs.cancel": "execute",
142
+ "runs.rerun": "execute",
143
+ "signals.send": "execute",
144
+ "approvals.decide": "approve",
145
+ "cron.list": "read",
146
+ "cron.add": "admin",
147
+ "cron.remove": "admin",
148
+ "cron.trigger": "execute",
149
+ };
150
+ /**
151
+ * @template T
152
+ * @param {string | null | undefined} value
153
+ * @returns {T | null}
154
+ */
155
+ function parseJson(value) {
156
+ if (!value)
157
+ return null;
158
+ try {
159
+ return JSON.parse(value);
160
+ }
161
+ catch {
162
+ return null;
163
+ }
164
+ }
165
+ /**
166
+ * @param {unknown} run
167
+ * @returns {string | null}
168
+ */
169
+ function resolveRunOwnerId(run) {
170
+ const config = parseJson(typeof run?.configJson === "string" ? run.configJson : null);
171
+ const auth = asObject(config?.auth);
172
+ const owner = asString(auth?.triggeredBy);
173
+ return owner ? owner : null;
174
+ }
175
+ /**
176
+ * @param {ServerResponse} res
177
+ * @param {number} status
178
+ * @param {unknown} payload
179
+ */
180
+ function sendJson(res, status, payload) {
181
+ res.statusCode = status;
182
+ res.setHeader("Content-Type", "application/json");
183
+ res.setHeader("Cache-Control", "no-store");
184
+ res.setHeader("X-Content-Type-Options", "nosniff");
185
+ res.end(JSON.stringify(payload));
186
+ }
187
+ /**
188
+ * @param {ServerResponse} res
189
+ * @param {number} status
190
+ * @param {string} payload
191
+ */
192
+ function sendText(res, status, payload, contentType = "text/plain; charset=utf-8") {
193
+ res.statusCode = status;
194
+ res.setHeader("Content-Type", contentType);
195
+ res.setHeader("Cache-Control", "no-store");
196
+ res.setHeader("X-Content-Type-Options", "nosniff");
197
+ res.end(payload);
198
+ }
199
+ /**
200
+ * @param {unknown} value
201
+ * @returns {Record<string, unknown> | null}
202
+ */
203
+ function asObject(value) {
204
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
205
+ return null;
206
+ }
207
+ return value;
208
+ }
209
+ /**
210
+ * @param {unknown} value
211
+ * @returns {string | undefined}
212
+ */
213
+ function asString(value) {
214
+ return typeof value === "string" ? value : undefined;
215
+ }
216
+ /**
217
+ * @param {unknown} value
218
+ * @returns {number | undefined}
219
+ */
220
+ function asNumber(value) {
221
+ return typeof value === "number" && Number.isFinite(value) ? value : undefined;
222
+ }
223
+ /**
224
+ * @param {unknown} value
225
+ * @returns {boolean | undefined}
226
+ */
227
+ function asBoolean(value) {
228
+ return typeof value === "boolean" ? value : undefined;
229
+ }
230
+ /**
231
+ * @param {unknown} value
232
+ * @returns {string | undefined}
233
+ */
234
+ function asWebhookString(value) {
235
+ if (typeof value === "string") {
236
+ const trimmed = value.trim();
237
+ return trimmed.length > 0 ? trimmed : undefined;
238
+ }
239
+ if (typeof value === "number" && Number.isFinite(value)) {
240
+ return String(value);
241
+ }
242
+ if (typeof value === "boolean") {
243
+ return value ? "true" : "false";
244
+ }
245
+ return undefined;
246
+ }
247
+ /**
248
+ * @template M
249
+ * @param {M} metric
250
+ * @param {GatewayMetricLabels} [labels]
251
+ * @returns {M}
252
+ */
253
+ function taggedMetric(metric, labels = {}) {
254
+ let tagged = metric;
255
+ for (const [key, value] of Object.entries(labels)) {
256
+ if (value === undefined || value === null) {
257
+ continue;
258
+ }
259
+ tagged = Metric.tagged(tagged, key, String(value));
260
+ }
261
+ return tagged;
262
+ }
263
+ /**
264
+ * @template M
265
+ * @param {M} metric
266
+ * @param {GatewayMetricLabels} [labels]
267
+ */
268
+ function incrementMetric(metric, labels = {}) {
269
+ return Metric.increment(taggedMetric(metric, labels));
270
+ }
271
+ /**
272
+ * @template M
273
+ * @param {M} metric
274
+ * @param {number} value
275
+ * @param {GatewayMetricLabels} [labels]
276
+ */
277
+ function updateMetric(metric, value, labels = {}) {
278
+ return Metric.update(taggedMetric(metric, labels), value);
279
+ }
280
+ /**
281
+ * @param {Effect.Effect<void, never, never>} effect
282
+ */
283
+ function emitGatewayEffect(effect) {
284
+ void runFork(effect);
285
+ }
286
+ /**
287
+ * @param {"debug" | "info" | "warning" | "error"} level
288
+ * @param {string} message
289
+ * @param {Record<string, unknown>} [annotations]
290
+ * @param {string} [span]
291
+ */
292
+ function emitGatewayLog(level, message, annotations, span) {
293
+ let effect = level === "debug"
294
+ ? Effect.logDebug(message)
295
+ : level === "info"
296
+ ? Effect.logInfo(message)
297
+ : level === "warning"
298
+ ? Effect.logWarning(message)
299
+ : Effect.logError(message);
300
+ if (annotations && Object.keys(annotations).length > 0) {
301
+ effect = effect.pipe(Effect.annotateLogs(annotations));
302
+ }
303
+ if (span) {
304
+ effect = effect.pipe(Effect.withLogSpan(span));
305
+ }
306
+ emitGatewayEffect(effect);
307
+ }
308
+ /**
309
+ * @param {GatewayRequestContext} context
310
+ * @returns {Record<string, unknown>}
311
+ */
312
+ function gatewayContextAnnotations(context) {
313
+ return {
314
+ connectionId: context.connectionId,
315
+ transport: context.transport,
316
+ ...(context.userId ? { userId: context.userId } : {}),
317
+ ...(context.role ? { role: context.role } : {}),
318
+ };
319
+ }
320
+ /**
321
+ * @param {Record<string, unknown>} [params]
322
+ * @param {unknown} [payload]
323
+ * @returns {Record<string, unknown>}
324
+ */
325
+ function gatewayRunAnnotations(params, payload) {
326
+ const annotations = {};
327
+ const responsePayload = asObject(payload);
328
+ const runId = asString(params?.runId) ?? asString(responsePayload?.runId);
329
+ const leftRunId = asString(params?.leftRunId);
330
+ const rightRunId = asString(params?.rightRunId);
331
+ if (runId) {
332
+ annotations.runId = runId;
333
+ }
334
+ if (leftRunId) {
335
+ annotations.leftRunId = leftRunId;
336
+ }
337
+ if (rightRunId) {
338
+ annotations.rightRunId = rightRunId;
339
+ }
340
+ return annotations;
341
+ }
342
+ /**
343
+ * @param {string} runId
344
+ * @returns {string}
345
+ */
346
+ function devtoolsRunMetricTag(runId) {
347
+ return createHash("sha1").update(runId).digest("hex").slice(0, 12);
348
+ }
349
+ /**
350
+ * @param {GatewayRequestContext} context
351
+ * @param {Pick<RequestFrame, "id" | "method" | "params">} frame
352
+ * @param {unknown} [payload]
353
+ * @returns {Record<string, unknown>}
354
+ */
355
+ function gatewayRpcAnnotations(context, frame, payload) {
356
+ return {
357
+ ...gatewayContextAnnotations(context),
358
+ frameId: frame.id,
359
+ method: frame.method,
360
+ ...gatewayRunAnnotations(asObject(frame.params) ?? {}, payload),
361
+ };
362
+ }
363
+ /**
364
+ * @param {unknown} error
365
+ * @returns {string}
366
+ */
367
+ function gatewayErrorCode(error) {
368
+ if (error && typeof error === "object") {
369
+ const code = asString(error.code);
370
+ if (code) {
371
+ return code;
372
+ }
373
+ }
374
+ if (error instanceof Error && error.name) {
375
+ return error.name;
376
+ }
377
+ return "UNKNOWN";
378
+ }
379
+ /**
380
+ * @param {unknown} error
381
+ * @returns {Record<string, unknown>}
382
+ */
383
+ function gatewayErrorAnnotations(error) {
384
+ const serialized = asObject(errorToJson(error)) ?? { message: String(error) };
385
+ const summary = asString(serialized.summary);
386
+ const message = asString(serialized.message);
387
+ return {
388
+ errorCode: gatewayErrorCode(error),
389
+ ...(summary ? { errorSummary: summary } : {}),
390
+ ...(message ? { errorMessage: message } : {}),
391
+ error: serialized,
392
+ };
393
+ }
394
+ /**
395
+ * @param {GatewayAuthConfig | undefined} auth
396
+ * @returns {string}
397
+ */
398
+ function gatewayAuthMode(auth) {
399
+ return auth?.mode ?? "none";
400
+ }
401
+ /**
402
+ * @param {string} triggeredBy
403
+ * @returns {string}
404
+ */
405
+ function gatewayTriggerSource(triggeredBy) {
406
+ if (triggeredBy.startsWith("cron:")) {
407
+ return "cron";
408
+ }
409
+ if (triggeredBy.startsWith("webhook:")) {
410
+ return "webhook";
411
+ }
412
+ if (triggeredBy === "gateway") {
413
+ return "gateway";
414
+ }
415
+ return "user";
416
+ }
417
+ /**
418
+ * @param {unknown} value
419
+ * @param {string} field
420
+ * @returns {number | undefined}
421
+ */
422
+ function asOptionalPositiveInt(value, field) {
423
+ if (value === undefined || value === null) {
424
+ return undefined;
425
+ }
426
+ return Math.floor(assertPositiveFiniteInteger(field, Number(value)));
427
+ }
428
+ /**
429
+ * @param {IncomingMessage} req
430
+ * @param {string} name
431
+ * @returns {string | null}
432
+ */
433
+ function headerValue(req, name) {
434
+ const value = req.headers[name.toLowerCase()];
435
+ if (Array.isArray(value)) {
436
+ return value[0] ?? null;
437
+ }
438
+ return typeof value === "string" ? value : null;
439
+ }
440
+ /**
441
+ * @param {IncomingMessage} req
442
+ * @returns {string | null}
443
+ */
444
+ function bearerTokenFromHeaders(req) {
445
+ const smithersKey = headerValue(req, "x-smithers-key");
446
+ if (smithersKey) {
447
+ return smithersKey;
448
+ }
449
+ const authHeader = headerValue(req, "authorization");
450
+ if (!authHeader) {
451
+ return null;
452
+ }
453
+ return authHeader.startsWith("Bearer ") ? authHeader.slice(7) : authHeader;
454
+ }
455
+ /**
456
+ * @param {unknown} value
457
+ * @returns {Record<string, unknown> | null}
458
+ */
459
+ function asStringRecord(value) {
460
+ return asObject(value);
461
+ }
462
+ /**
463
+ * @param {string} id
464
+ * @param {unknown} [payload]
465
+ * @returns {ResponseFrame}
466
+ */
467
+ function responseOk(id, payload) {
468
+ return { type: "res", id, ok: true, payload };
469
+ }
470
+ /**
471
+ * @param {string} id
472
+ * @param {string} code
473
+ * @param {string} message
474
+ * @returns {ResponseFrame}
475
+ */
476
+ function responseError(id, code, message) {
477
+ return { type: "res", id, ok: false, error: { code, message } };
478
+ }
479
+ /**
480
+ * @param {unknown} raw
481
+ * @returns {string}
482
+ */
483
+ function rawDataToUtf8(raw) {
484
+ if (typeof raw === "string") {
485
+ return raw;
486
+ }
487
+ if (Buffer.isBuffer(raw)) {
488
+ return raw.toString("utf8");
489
+ }
490
+ if (Array.isArray(raw)) {
491
+ return Buffer.concat(raw.map((entry) => (Buffer.isBuffer(entry) ? entry : Buffer.from(entry)))).toString("utf8");
492
+ }
493
+ return Buffer.from(raw).toString("utf8");
494
+ }
495
+ /**
496
+ * @param {unknown} method
497
+ * @returns {string}
498
+ */
499
+ export function validateGatewayMethodName(method) {
500
+ if (typeof method !== "string") {
501
+ throw new SmithersError("INVALID_INPUT", "Gateway method name must be a string.", { methodType: typeof method });
502
+ }
503
+ assertOptionalStringMaxLength("method", method, GATEWAY_METHOD_NAME_MAX_LENGTH);
504
+ if (!GATEWAY_METHOD_NAME_PATTERN.test(method)) {
505
+ throw new SmithersError("INVALID_INPUT", "Gateway method name is invalid.", { method });
506
+ }
507
+ return method;
508
+ }
509
+ /**
510
+ * @param {unknown} raw
511
+ * @returns {RequestFrame}
512
+ */
513
+ export function parseGatewayRequestFrame(raw, maxPayloadBytes = GATEWAY_RPC_MAX_PAYLOAD_BYTES) {
514
+ const body = rawDataToUtf8(raw);
515
+ if (Buffer.byteLength(body, "utf8") > maxPayloadBytes) {
516
+ throw new SmithersError("INVALID_INPUT", `Gateway RPC payload exceeds ${maxPayloadBytes} bytes.`, { maxBytes: maxPayloadBytes });
517
+ }
518
+ let parsed;
519
+ try {
520
+ parsed = JSON.parse(body);
521
+ }
522
+ catch (error) {
523
+ throw new SmithersError("INVALID_INPUT", "Gateway RPC payload must be valid JSON.", undefined, { cause: error });
524
+ }
525
+ assertJsonPayloadWithinBounds("gateway frame", parsed, {
526
+ maxArrayLength: GATEWAY_RPC_MAX_ARRAY_LENGTH,
527
+ maxDepth: GATEWAY_RPC_MAX_DEPTH,
528
+ maxStringLength: GATEWAY_RPC_MAX_STRING_LENGTH,
529
+ });
530
+ const frame = asObject(parsed);
531
+ if (!frame || frame.type !== "req") {
532
+ throw new SmithersError("INVALID_INPUT", "Gateway frame must be a request object.");
533
+ }
534
+ if (typeof frame.id !== "string") {
535
+ throw new SmithersError("INVALID_INPUT", "Gateway frame id must be a string.");
536
+ }
537
+ assertOptionalStringMaxLength("id", frame.id, GATEWAY_FRAME_ID_MAX_LENGTH);
538
+ return {
539
+ type: "req",
540
+ id: frame.id,
541
+ method: validateGatewayMethodName(frame.method),
542
+ params: frame.params,
543
+ };
544
+ }
545
+ /**
546
+ * @param {unknown} value
547
+ * @param {number} depth
548
+ * @param {Set<unknown>} seen
549
+ * @returns {number}
550
+ */
551
+ function gatewayInputDepthAt(value, depth, seen) {
552
+ if (!value || typeof value !== "object") {
553
+ return depth;
554
+ }
555
+ if (seen.has(value)) {
556
+ throw new SmithersError("INVALID_INPUT", "Gateway RPC input must not contain circular references.");
557
+ }
558
+ seen.add(value);
559
+ let maxDepth = depth;
560
+ const entries = Array.isArray(value)
561
+ ? value
562
+ : Object.values(value);
563
+ for (const entry of entries) {
564
+ const entryDepth = entry && typeof entry === "object"
565
+ ? gatewayInputDepthAt(entry, depth + 1, seen)
566
+ : depth;
567
+ if (entryDepth > maxDepth) {
568
+ maxDepth = entryDepth;
569
+ }
570
+ }
571
+ seen.delete(value);
572
+ return maxDepth;
573
+ }
574
+ /**
575
+ * @param {unknown} value
576
+ * @returns {number}
577
+ */
578
+ export function getGatewayInputDepth(value) {
579
+ if (!value || typeof value !== "object") {
580
+ return 0;
581
+ }
582
+ return gatewayInputDepthAt(value, 1, new Set());
583
+ }
584
+ /**
585
+ * @param {unknown} value
586
+ * @returns {number}
587
+ */
588
+ export function assertGatewayInputDepthWithinBounds(value, maxDepth = GATEWAY_RPC_INPUT_MAX_DEPTH) {
589
+ const depth = getGatewayInputDepth(value);
590
+ if (depth > maxDepth) {
591
+ throw new SmithersError("INVALID_INPUT", `Gateway RPC input exceeds the maximum nesting depth of ${maxDepth}.`, {
592
+ actualDepth: depth,
593
+ maxDepth,
594
+ });
595
+ }
596
+ return depth;
597
+ }
598
+ /**
599
+ * @param {unknown} input
600
+ * @returns {Record<string, unknown>}
601
+ */
602
+ function validateGatewayRpcInput(input) {
603
+ const normalizedInput = asObject(input) ?? {};
604
+ const inputJson = JSON.stringify(normalizedInput);
605
+ if (inputJson === undefined) {
606
+ throw new SmithersError("INVALID_INPUT", "Gateway RPC input must be JSON-serializable.");
607
+ }
608
+ const inputBytes = Buffer.byteLength(inputJson, "utf8");
609
+ if (inputBytes > GATEWAY_RPC_INPUT_MAX_BYTES) {
610
+ throw new SmithersError("INVALID_INPUT", `Gateway RPC input exceeds ${GATEWAY_RPC_INPUT_MAX_BYTES} bytes.`, {
611
+ actualBytes: inputBytes,
612
+ maxBytes: GATEWAY_RPC_INPUT_MAX_BYTES,
613
+ });
614
+ }
615
+ assertGatewayInputDepthWithinBounds(normalizedInput);
616
+ return normalizedInput;
617
+ }
618
+ /**
619
+ * @param {string | undefined} code
620
+ */
621
+ export function statusForRpcError(code) {
622
+ switch (code) {
623
+ case "UNAUTHORIZED":
624
+ case "Unauthorized":
625
+ return 401;
626
+ case "FORBIDDEN":
627
+ return 403;
628
+ case "NOT_FOUND":
629
+ case "METHOD_NOT_FOUND":
630
+ return 404;
631
+ case "INVALID_REQUEST":
632
+ case "INVALID_FRAME":
633
+ case "INVALID_INPUT":
634
+ case "PROTOCOL_UNSUPPORTED":
635
+ case "InvalidRunId":
636
+ case "InvalidNodeId":
637
+ case "InvalidIteration":
638
+ case "InvalidDelta":
639
+ case "InvalidFrameNo":
640
+ case "ConfirmationRequired":
641
+ case "FrameOutOfRange":
642
+ case "SeqOutOfRange":
643
+ return 400;
644
+ case "RunNotFound":
645
+ case "NodeNotFound":
646
+ case "AttemptNotFound":
647
+ case "IterationNotFound":
648
+ case "NodeHasNoOutput":
649
+ return 404;
650
+ case "AttemptNotFinished":
651
+ case "Busy":
652
+ return 409;
653
+ case "DiffTooLarge":
654
+ case "PayloadTooLarge":
655
+ return 413;
656
+ case "RateLimited":
657
+ case "BackpressureDisconnect":
658
+ return 429;
659
+ case "UnsupportedSandbox":
660
+ return 501;
661
+ case "VcsError":
662
+ case "RewindFailed":
663
+ return 500;
664
+ default:
665
+ return 500;
666
+ }
667
+ }
668
+ /**
669
+ * @param {unknown} payload
670
+ * @returns {string | null}
671
+ */
672
+ function eventRunId(payload) {
673
+ const record = asObject(payload);
674
+ const runId = record ? asString(record.runId) : undefined;
675
+ return runId ?? null;
676
+ }
677
+ /**
678
+ * @param {string} scope
679
+ * @returns {string}
680
+ */
681
+ function normalizeGrantedScope(scope) {
682
+ return scope.trim();
683
+ }
684
+ /**
685
+ * @param {string} method
686
+ * @returns {MethodAccess}
687
+ */
688
+ function accessForMethod(method) {
689
+ return METHOD_ACCESS[method] ?? (method.startsWith("config.") ? "admin" : "read");
690
+ }
691
+ /**
692
+ * @param {string[]} scopes
693
+ * @param {string} method
694
+ * @returns {boolean}
695
+ */
696
+ function hasScope(scopes, method) {
697
+ if (scopes.includes("*")) {
698
+ return true;
699
+ }
700
+ const requiredAccess = accessForMethod(method);
701
+ const grantedLevels = scopes
702
+ .map((scope) => scope.trim())
703
+ .filter((scope) => scope === "read" || scope === "execute" || scope === "approve" || scope === "admin");
704
+ if (grantedLevels.some((level) => ACCESS_RANK[level] >= ACCESS_RANK[requiredAccess])) {
705
+ return true;
706
+ }
707
+ for (const scope of scopes.map(normalizeGrantedScope)) {
708
+ if (!scope)
709
+ continue;
710
+ if (scope === method)
711
+ return true;
712
+ if (scope.endsWith(".*") && method.startsWith(scope.slice(0, -1))) {
713
+ return true;
714
+ }
715
+ }
716
+ return false;
717
+ }
718
+ /**
719
+ * @param {unknown} value
720
+ * @returns {string[]}
721
+ */
722
+ function parseStringArray(value) {
723
+ if (!Array.isArray(value)) {
724
+ return [];
725
+ }
726
+ return value.filter((entry) => typeof entry === "string");
727
+ }
728
+ /**
729
+ * @param {string} value
730
+ * @returns {Record<string, unknown> | null}
731
+ */
732
+ function decodeBase64UrlJson(value) {
733
+ try {
734
+ const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
735
+ const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
736
+ const decoded = Buffer.from(padded, "base64").toString("utf8");
737
+ return asStringRecord(JSON.parse(decoded));
738
+ }
739
+ catch {
740
+ return null;
741
+ }
742
+ }
743
+ /**
744
+ * @param {string} token
745
+ * @param {Extract<GatewayAuthConfig, { mode: "jwt" }>} config
746
+ * @returns {{ ok: true; payload: Record<string, unknown> } | { ok: false; message: string }}
747
+ */
748
+ function verifyJwtToken(token, config) {
749
+ const [encodedHeader, encodedPayload, encodedSignature] = token.split(".");
750
+ if (!encodedHeader || !encodedPayload || !encodedSignature) {
751
+ return { ok: false, message: "JWT must have three segments" };
752
+ }
753
+ const header = decodeBase64UrlJson(encodedHeader);
754
+ const payload = decodeBase64UrlJson(encodedPayload);
755
+ if (!header || !payload) {
756
+ return { ok: false, message: "JWT header or payload was not valid JSON" };
757
+ }
758
+ if (header.alg !== "HS256") {
759
+ return { ok: false, message: "Unsupported JWT algorithm" };
760
+ }
761
+ const expectedSignature = createHmac("sha256", config.secret)
762
+ .update(`${encodedHeader}.${encodedPayload}`)
763
+ .digest("base64url");
764
+ const actualSignature = Buffer.from(encodedSignature, "base64url");
765
+ const expectedSignatureBuffer = Buffer.from(expectedSignature, "base64url");
766
+ if (actualSignature.length !== expectedSignatureBuffer.length ||
767
+ !timingSafeEqual(actualSignature, expectedSignatureBuffer)) {
768
+ return { ok: false, message: "JWT signature verification failed" };
769
+ }
770
+ const now = Math.floor(Date.now() / 1_000);
771
+ const skew = Math.max(0, config.clockSkewSeconds ?? 60);
772
+ const exp = asNumber(payload.exp);
773
+ const nbf = asNumber(payload.nbf);
774
+ const iss = asString(payload.iss);
775
+ const aud = payload.aud;
776
+ if (iss !== config.issuer) {
777
+ return { ok: false, message: "JWT issuer did not match" };
778
+ }
779
+ const audiences = Array.isArray(config.audience) ? config.audience : [config.audience];
780
+ const tokenAudiences = typeof aud === "string" ? [aud] : parseStringArray(aud);
781
+ if (!audiences.some((audience) => tokenAudiences.includes(audience))) {
782
+ return { ok: false, message: "JWT audience did not match" };
783
+ }
784
+ if (typeof exp === "number" && now - skew >= exp) {
785
+ return { ok: false, message: "JWT has expired" };
786
+ }
787
+ if (typeof nbf === "number" && now + skew < nbf) {
788
+ return { ok: false, message: "JWT is not active yet" };
789
+ }
790
+ return { ok: true, payload };
791
+ }
792
+ /**
793
+ * @param {unknown} value
794
+ * @returns {string[]}
795
+ */
796
+ function parseJwtScopes(value) {
797
+ if (typeof value === "string") {
798
+ return value
799
+ .split(/[,\s]+/)
800
+ .map((entry) => entry.trim())
801
+ .filter(Boolean);
802
+ }
803
+ return parseStringArray(value);
804
+ }
805
+ /**
806
+ * @param {unknown} value
807
+ * @param {string | null} fallbackTitle
808
+ * @returns {ApprovalRequestRecord}
809
+ */
810
+ function parseApprovalRequest(value, fallbackTitle) {
811
+ const record = asObject(value);
812
+ const options = Array.isArray(record?.options)
813
+ ? record.options
814
+ .filter((entry) => Boolean(entry && typeof entry === "object" && !Array.isArray(entry)))
815
+ .map((entry) => ({
816
+ key: asString(entry.key) ?? "",
817
+ label: asString(entry.label) ?? "",
818
+ ...(asString(entry.summary) ? { summary: asString(entry.summary) } : {}),
819
+ }))
820
+ .filter((entry) => entry.key.length > 0 && entry.label.length > 0)
821
+ : [];
822
+ const autoApprove = record?.autoApprove && typeof record.autoApprove === "object" && !Array.isArray(record.autoApprove)
823
+ ? record.autoApprove
824
+ : null;
825
+ return {
826
+ mode: record?.mode === "select" || record?.mode === "rank" || record?.mode === "decision"
827
+ ? record.mode
828
+ : "gate",
829
+ title: asString(record?.title) ?? fallbackTitle,
830
+ summary: asString(record?.summary) ?? null,
831
+ options,
832
+ allowedScopes: parseStringArray(record?.allowedScopes),
833
+ allowedUsers: parseStringArray(record?.allowedUsers),
834
+ autoApprove,
835
+ };
836
+ }
837
+ /**
838
+ * @param {ApprovalRequestRecord} request
839
+ * @param {unknown} decision
840
+ */
841
+ function validateApprovalDecision(request, decision) {
842
+ if (request.mode === "select") {
843
+ const payload = asObject(decision);
844
+ const selected = asString(payload?.selected);
845
+ if (!selected) {
846
+ return { ok: false, code: "INVALID_REQUEST", message: "select approvals require decision.selected" };
847
+ }
848
+ if (request.options.length > 0 && !request.options.some((option) => option.key === selected)) {
849
+ return { ok: false, code: "INVALID_REQUEST", message: `Unknown selection: ${selected}` };
850
+ }
851
+ }
852
+ if (request.mode === "rank") {
853
+ const payload = asObject(decision);
854
+ const ranked = parseStringArray(payload?.ranked);
855
+ if (ranked.length === 0) {
856
+ return { ok: false, code: "INVALID_REQUEST", message: "rank approvals require decision.ranked" };
857
+ }
858
+ const allowed = new Set(request.options.map((option) => option.key));
859
+ if (allowed.size > 0 && ranked.some((value) => !allowed.has(value))) {
860
+ return { ok: false, code: "INVALID_REQUEST", message: "rank approval included unknown options" };
861
+ }
862
+ }
863
+ return { ok: true };
864
+ }
865
+ /**
866
+ * @param {string} pattern
867
+ */
868
+ function nextCronRunAtMs(pattern) {
869
+ const interval = CronExpressionParser.parse(pattern);
870
+ return interval.next().getTime();
871
+ }
872
+ /**
873
+ * @param {number} ms
874
+ */
875
+ function delay(ms) {
876
+ return new Promise((resolve) => setTimeout(resolve, ms));
877
+ }
878
+ /**
879
+ * @param {string | null | undefined} value
880
+ * @returns {string | null}
881
+ */
882
+ function normalizeCorrelationId(value) {
883
+ const normalized = typeof value === "string" ? value.trim() : "";
884
+ return normalized.length > 0 ? normalized : null;
885
+ }
886
+ /**
887
+ * @param {string | null} [metaJson]
888
+ */
889
+ function parseWebhookWaitForEventSnapshot(metaJson) {
890
+ if (!metaJson) {
891
+ return null;
892
+ }
893
+ try {
894
+ const parsed = JSON.parse(metaJson);
895
+ const waitForEvent = asObject(asObject(parsed)?.waitForEvent);
896
+ const signalName = asString(waitForEvent?.signalName)?.trim();
897
+ if (!signalName) {
898
+ return null;
899
+ }
900
+ return {
901
+ signalName,
902
+ correlationId: normalizeCorrelationId(asString(waitForEvent?.correlationId) ?? null),
903
+ };
904
+ }
905
+ catch {
906
+ return null;
907
+ }
908
+ }
909
+ /**
910
+ * @param {unknown} source
911
+ * @param {string | undefined} path
912
+ * @returns {unknown}
913
+ */
914
+ function readPathValue(source, path) {
915
+ if (!path) {
916
+ return source;
917
+ }
918
+ const trimmed = path.trim();
919
+ if (!trimmed) {
920
+ return source;
921
+ }
922
+ let current = source;
923
+ for (const segment of trimmed.split(".").filter((entry) => entry.length > 0)) {
924
+ if (Array.isArray(current)) {
925
+ const index = Number(segment);
926
+ if (!Number.isInteger(index) || index < 0) {
927
+ return undefined;
928
+ }
929
+ current = current[index];
930
+ continue;
931
+ }
932
+ const record = asObject(current);
933
+ if (!record) {
934
+ return undefined;
935
+ }
936
+ current = record[segment];
937
+ }
938
+ return current;
939
+ }
940
+ /**
941
+ * @param {Buffer} body
942
+ * @param {string} description
943
+ */
944
+ function parseJsonBuffer(body, description) {
945
+ if (body.length === 0) {
946
+ return {};
947
+ }
948
+ try {
949
+ return JSON.parse(body.toString("utf8"));
950
+ }
951
+ catch (error) {
952
+ throw new SmithersError("INVALID_INPUT", `${description} must be valid JSON.`, undefined, { cause: error });
953
+ }
954
+ }
955
+ /**
956
+ * @param {IncomingMessage} req
957
+ * @param {number} maxBytes
958
+ */
959
+ async function readRawBody(req, maxBytes) {
960
+ const chunks = [];
961
+ let total = 0;
962
+ const lengthHeader = headerValue(req, "content-length");
963
+ const declaredLength = lengthHeader ? Number(lengthHeader) : NaN;
964
+ if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
965
+ throw new SmithersError("INVALID_INPUT", `Gateway request payload exceeds ${maxBytes} bytes.`, { maxBytes });
966
+ }
967
+ for await (const chunk of req) {
968
+ const buffer = Buffer.from(chunk);
969
+ total += buffer.length;
970
+ if (total > maxBytes) {
971
+ throw new SmithersError("INVALID_INPUT", `Gateway request payload exceeds ${maxBytes} bytes.`, { maxBytes });
972
+ }
973
+ chunks.push(buffer);
974
+ }
975
+ if (chunks.length === 0) {
976
+ return Buffer.alloc(0);
977
+ }
978
+ return Buffer.concat(chunks);
979
+ }
980
+ /**
981
+ * @param {IncomingMessage} req
982
+ * @param {number} maxBytes
983
+ */
984
+ async function readBody(req, maxBytes) {
985
+ return parseJsonBuffer(await readRawBody(req, maxBytes), "Gateway RPC payload");
986
+ }
987
+ /**
988
+ * @param {Buffer} rawBody
989
+ * @param {string} secret
990
+ * @param {string} prefix
991
+ */
992
+ function computeWebhookSignature(rawBody, secret, prefix) {
993
+ return `${prefix}${createHmac("sha256", secret).update(rawBody).digest("hex")}`;
994
+ }
995
+ /**
996
+ * @param {string} expected
997
+ * @param {string | null} provided
998
+ */
999
+ function isValidWebhookSignature(expected, provided) {
1000
+ if (!provided) {
1001
+ return false;
1002
+ }
1003
+ const expectedBuffer = Buffer.from(expected);
1004
+ const providedBuffer = Buffer.from(provided);
1005
+ if (expectedBuffer.length !== providedBuffer.length) {
1006
+ return false;
1007
+ }
1008
+ return timingSafeEqual(expectedBuffer, providedBuffer);
1009
+ }
1010
+ /**
1011
+ * @param {unknown} input
1012
+ * @returns {Record<string, unknown>}
1013
+ */
1014
+ function normalizeWebhookRunInput(input) {
1015
+ const normalized = asObject(input) ?? { payload: input ?? null };
1016
+ return validateGatewayRpcInput(normalized);
1017
+ }
1018
+ /**
1019
+ * @param {string} workflowKey
1020
+ */
1021
+ function webhookTriggerUserId(workflowKey) {
1022
+ return `webhook:${workflowKey}`;
1023
+ }
1024
+ /**
1025
+ * @param {string} workflowKey
1026
+ */
1027
+ function cronWorkflowPath(workflowKey) {
1028
+ return `gateway:${workflowKey}`;
1029
+ }
1030
+ /**
1031
+ * @param {string | null | undefined} workflowPath
1032
+ */
1033
+ function workflowKeyFromCronPath(workflowPath) {
1034
+ if (!workflowPath || !workflowPath.startsWith("gateway:")) {
1035
+ return null;
1036
+ }
1037
+ return workflowPath.slice("gateway:".length);
1038
+ }
1039
+ /**
1040
+ * @param {ConnectionState} connection
1041
+ * @param {string | null} runId
1042
+ */
1043
+ function shouldDeliverEvent(connection, runId) {
1044
+ if (!runId)
1045
+ return true;
1046
+ if (!connection.subscribedRuns || connection.subscribedRuns.size === 0) {
1047
+ return true;
1048
+ }
1049
+ return connection.subscribedRuns.has(runId);
1050
+ }
1051
+ export class Gateway {
1052
+ protocol;
1053
+ features;
1054
+ heartbeatMs;
1055
+ maxBodyBytes;
1056
+ maxPayload;
1057
+ maxConnections;
1058
+ auth;
1059
+ defaults;
1060
+ workflows = new Map();
1061
+ connections = new Set();
1062
+ runRegistry = new Map();
1063
+ activeRuns = new Map();
1064
+ inflightRuns = new Map();
1065
+ devtoolsSubscribers = new Map();
1066
+ /** Absolute active subscriber count per runId (gauge source of truth). */
1067
+ devtoolsSubscriberCounts = new Map();
1068
+ /** Flagged subscriber IDs that should force a snapshot on their next emit. */
1069
+ devtoolsInvalidateFlags = new Set();
1070
+ server = null;
1071
+ wsServer = null;
1072
+ schedulerTimer = null;
1073
+ stateVersion = 0;
1074
+ startedAtMs = nowMs();
1075
+ /**
1076
+ * @param {GatewayOptions} [options]
1077
+ */
1078
+ constructor(options = {}) {
1079
+ this.protocol = options.protocol ?? DEFAULT_PROTOCOL;
1080
+ this.features = [...(options.features ?? ["streaming", "runs"])];
1081
+ this.heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
1082
+ this.maxBodyBytes = options.maxBodyBytes === undefined
1083
+ ? DEFAULT_MAX_BODY_BYTES
1084
+ : Math.floor(assertPositiveFiniteInteger("maxBodyBytes", Number(options.maxBodyBytes)));
1085
+ this.maxPayload = options.maxPayload === undefined
1086
+ ? GATEWAY_RPC_MAX_PAYLOAD_BYTES
1087
+ : Math.floor(assertPositiveFiniteInteger("maxPayload", Number(options.maxPayload)));
1088
+ this.maxConnections = options.maxConnections === undefined
1089
+ ? DEFAULT_MAX_CONNECTIONS
1090
+ : Math.floor(assertPositiveFiniteInteger("maxConnections", Number(options.maxConnections)));
1091
+ this.auth = options.auth;
1092
+ this.defaults = options.defaults;
1093
+ }
1094
+ authModeLabel() {
1095
+ return gatewayAuthMode(this.auth);
1096
+ }
1097
+ /**
1098
+ * @param {string} [runId]
1099
+ * @returns {number}
1100
+ */
1101
+ getDevToolsSubscriberCount(runId) {
1102
+ if (!runId) {
1103
+ return this.devtoolsSubscribers.size;
1104
+ }
1105
+ let count = 0;
1106
+ for (const subscriber of this.devtoolsSubscribers.values()) {
1107
+ if (subscriber.runId === runId) {
1108
+ count += 1;
1109
+ }
1110
+ }
1111
+ return count;
1112
+ }
1113
+ /**
1114
+ * Record a single subscribe attempt outcome. Centralised so that invalid
1115
+ * runId, missing run, SeqOutOfRange, etc. still update
1116
+ * `smithers_devtools_subscribe_total{result="error"}`.
1117
+ *
1118
+ * @param {"ok" | "error"} result
1119
+ */
1120
+ recordDevToolsSubscribeAttempt(result) {
1121
+ emitGatewayEffect(Metric.increment(taggedMetric(devtoolsSubscribeTotal, { result })));
1122
+ }
1123
+ /**
1124
+ * Push the absolute active-subscriber count to the Prometheus gauge. The
1125
+ * `runId` is hashed for bounded cardinality.
1126
+ *
1127
+ * @param {string} runId
1128
+ */
1129
+ publishDevToolsActiveSubscribersGauge(runId) {
1130
+ const runMetricLabel = devtoolsRunMetricTag(runId);
1131
+ const value = this.devtoolsSubscriberCounts.get(runId) ?? 0;
1132
+ emitGatewayEffect(Metric.update(taggedMetric(devtoolsActiveSubscribers, { runId: runMetricLabel }), value));
1133
+ }
1134
+ /**
1135
+ * @param {ConnectionState} connection
1136
+ * @param {string} streamId
1137
+ * @param {string} runId
1138
+ * @returns {AbortController}
1139
+ */
1140
+ registerDevToolsSubscriber(connection, streamId, runId) {
1141
+ const abort = new AbortController();
1142
+ if (!connection.devtoolsStreams) {
1143
+ connection.devtoolsStreams = new Map();
1144
+ }
1145
+ connection.devtoolsStreams.set(streamId, {
1146
+ runId,
1147
+ abort,
1148
+ });
1149
+ this.devtoolsSubscribers.set(streamId, {
1150
+ runId,
1151
+ connectionId: connection.connectionId,
1152
+ abort,
1153
+ startedAtMs: Date.now(),
1154
+ });
1155
+ const previous = this.devtoolsSubscriberCounts.get(runId) ?? 0;
1156
+ this.devtoolsSubscriberCounts.set(runId, previous + 1);
1157
+ this.recordDevToolsSubscribeAttempt("ok");
1158
+ this.publishDevToolsActiveSubscribersGauge(runId);
1159
+ return abort;
1160
+ }
1161
+ /**
1162
+ * @param {ConnectionState} connection
1163
+ * @param {string} streamId
1164
+ * @param {Record<string, unknown>} [details]
1165
+ */
1166
+ unregisterDevToolsSubscriber(connection, streamId, details = {}) {
1167
+ const stream = connection.devtoolsStreams?.get(streamId);
1168
+ if (stream) {
1169
+ stream.abort.abort();
1170
+ connection.devtoolsStreams?.delete(streamId);
1171
+ }
1172
+ const subscriber = this.devtoolsSubscribers.get(streamId);
1173
+ if (!subscriber) {
1174
+ return;
1175
+ }
1176
+ this.devtoolsSubscribers.delete(streamId);
1177
+ this.devtoolsInvalidateFlags.delete(streamId);
1178
+ const previous = this.devtoolsSubscriberCounts.get(subscriber.runId) ?? 0;
1179
+ const nextCount = Math.max(0, previous - 1);
1180
+ if (nextCount === 0) {
1181
+ this.devtoolsSubscriberCounts.delete(subscriber.runId);
1182
+ }
1183
+ else {
1184
+ this.devtoolsSubscriberCounts.set(subscriber.runId, nextCount);
1185
+ }
1186
+ this.publishDevToolsActiveSubscribersGauge(subscriber.runId);
1187
+ emitGatewayLog("info", "devtools stream unsubscribed", {
1188
+ runId: subscriber.runId,
1189
+ streamId,
1190
+ durationMs: Date.now() - subscriber.startedAtMs,
1191
+ ...details,
1192
+ }, "gateway:devtools");
1193
+ }
1194
+ /**
1195
+ * Flag every active subscriber for `runId` to rebaseline on its next emit.
1196
+ * Called when the gateway observes `TimeTravelJumped` for that run.
1197
+ *
1198
+ * @param {string} runId
1199
+ */
1200
+ invalidateDevToolsSubscribersForRun(runId) {
1201
+ for (const [streamId, subscriber] of this.devtoolsSubscribers.entries()) {
1202
+ if (subscriber.runId === runId) {
1203
+ this.devtoolsInvalidateFlags.add(streamId);
1204
+ }
1205
+ }
1206
+ }
1207
+ /**
1208
+ * Authorize a devtools request against the connection's `subscribe` set.
1209
+ *
1210
+ * If the client provided a `subscribe` filter at `connect` time, the run
1211
+ * must be in that set before any DB lookup happens.
1212
+ *
1213
+ * @param {ConnectionState | null | undefined} connection
1214
+ * @param {string} runId
1215
+ * @returns {boolean}
1216
+ */
1217
+ isDevToolsRunAuthorized(connection, runId) {
1218
+ if (!connection) {
1219
+ return true;
1220
+ }
1221
+ if (!connection.subscribedRuns || connection.subscribedRuns.size === 0) {
1222
+ return true;
1223
+ }
1224
+ return connection.subscribedRuns.has(runId);
1225
+ }
1226
+ /**
1227
+ * @param {ConnectionState} connection
1228
+ */
1229
+ cleanupDevToolsSubscribers(connection) {
1230
+ const streams = connection.devtoolsStreams;
1231
+ if (!streams || streams.size === 0) {
1232
+ return;
1233
+ }
1234
+ for (const streamId of streams.keys()) {
1235
+ this.unregisterDevToolsSubscriber(connection, streamId, {
1236
+ reason: "connection_closed",
1237
+ });
1238
+ }
1239
+ }
1240
+ /**
1241
+ * @param {GatewayTransport} transport
1242
+ * @param {string} frameType
1243
+ * @param {GatewayMetricLabels} [labels]
1244
+ */
1245
+ recordMessageReceived(transport, frameType, labels = {}) {
1246
+ emitGatewayEffect(incrementMetric(gatewayMessagesReceivedTotal, {
1247
+ transport,
1248
+ frameType,
1249
+ ...labels,
1250
+ }));
1251
+ }
1252
+ /**
1253
+ * @param {GatewayTransport} transport
1254
+ * @param {string} frameType
1255
+ * @param {GatewayMetricLabels} [labels]
1256
+ */
1257
+ recordMessageSent(transport, frameType, labels = {}) {
1258
+ emitGatewayEffect(incrementMetric(gatewayMessagesSentTotal, {
1259
+ transport,
1260
+ frameType,
1261
+ ...labels,
1262
+ }));
1263
+ }
1264
+ /**
1265
+ * @param {GatewayTransport} transport
1266
+ * @param {"success" | "failure"} outcome
1267
+ * @param {GatewayRequestContext} context
1268
+ * @param {Record<string, unknown>} [details]
1269
+ * @param {"debug" | "info" | "warning"} [level]
1270
+ */
1271
+ recordAuthEvent(transport, outcome, context, details = {}, level = outcome === "success" ? "info" : "warning") {
1272
+ const annotations = {
1273
+ ...gatewayContextAnnotations(context),
1274
+ authMode: this.authModeLabel(),
1275
+ outcome,
1276
+ ...details,
1277
+ };
1278
+ const logEffect = level === "debug"
1279
+ ? Effect.logDebug(outcome === "success"
1280
+ ? "Gateway auth succeeded"
1281
+ : "Gateway auth rejected")
1282
+ : level === "info"
1283
+ ? Effect.logInfo("Gateway auth succeeded")
1284
+ : Effect.logWarning("Gateway auth rejected");
1285
+ emitGatewayEffect(Effect.all([
1286
+ incrementMetric(gatewayAuthEventsTotal, {
1287
+ transport,
1288
+ mode: this.authModeLabel(),
1289
+ outcome,
1290
+ }),
1291
+ logEffect.pipe(Effect.annotateLogs(annotations), Effect.withLogSpan("gateway:auth")),
1292
+ ], { discard: true }));
1293
+ }
1294
+ /**
1295
+ * @param {GatewayRequestContext} context
1296
+ * @param {RequestFrame} frame
1297
+ * @param {() => Promise<ResponseFrame>} handler
1298
+ * @returns {Promise<ResponseFrame>}
1299
+ */
1300
+ async executeRpc(context, frame, handler) {
1301
+ const self = this;
1302
+ const start = performance.now();
1303
+ const params = asObject(frame.params) ?? {};
1304
+ const result = await runPromise(Effect.gen(function* () {
1305
+ yield* incrementMetric(gatewayRpcCallsTotal, {
1306
+ transport: context.transport,
1307
+ method: frame.method,
1308
+ });
1309
+ yield* Effect.logDebug("Gateway RPC started");
1310
+ const result = yield* Effect.promise(() => handler()
1311
+ .then((response) => ({ _tag: "success", response }))
1312
+ .catch((error) => ({ _tag: "failure", error })));
1313
+ yield* updateMetric(gatewayRpcDuration, performance.now() - start, {
1314
+ transport: context.transport,
1315
+ method: frame.method,
1316
+ });
1317
+ if (result._tag === "failure") {
1318
+ yield* incrementMetric(gatewayErrorsTotal, {
1319
+ kind: "rpc",
1320
+ transport: context.transport,
1321
+ method: frame.method,
1322
+ code: gatewayErrorCode(result.error),
1323
+ });
1324
+ yield* Effect.logError("Gateway RPC failed").pipe(Effect.annotateLogs(gatewayErrorAnnotations(result.error)));
1325
+ return result;
1326
+ }
1327
+ if (!result.response.ok) {
1328
+ yield* incrementMetric(gatewayErrorsTotal, {
1329
+ kind: "rpc",
1330
+ transport: context.transport,
1331
+ method: frame.method,
1332
+ code: result.response.error?.code ?? "UNKNOWN",
1333
+ });
1334
+ yield* Effect.logWarning("Gateway RPC rejected").pipe(Effect.annotateLogs({
1335
+ ...gatewayRunAnnotations(params, result.response.payload),
1336
+ rpcCode: result.response.error?.code ?? "UNKNOWN",
1337
+ ...(result.response.error?.message
1338
+ ? { rpcMessage: result.response.error.message }
1339
+ : {}),
1340
+ }));
1341
+ }
1342
+ else {
1343
+ yield* Effect.logDebug("Gateway RPC completed").pipe(Effect.annotateLogs(gatewayRunAnnotations(params, result.response.payload)));
1344
+ yield* self.rpcSuccessEffect(context, frame, result.response);
1345
+ }
1346
+ return result;
1347
+ }).pipe(Effect.annotateLogs(gatewayRpcAnnotations(context, frame)), Effect.withLogSpan(`gateway:rpc:${frame.method}`)));
1348
+ if (result._tag === "failure") {
1349
+ throw result.error;
1350
+ }
1351
+ return result.response;
1352
+ }
1353
+ /**
1354
+ * @param {GatewayRequestContext} context
1355
+ * @param {RequestFrame} frame
1356
+ * @param {ResponseFrame} response
1357
+ * @returns {Effect.Effect<void>}
1358
+ */
1359
+ rpcSuccessEffect(context, frame, response) {
1360
+ const params = asObject(frame.params) ?? {};
1361
+ switch (frame.method) {
1362
+ case "approvals.decide": {
1363
+ const approved = asBoolean(params.approved) ?? false;
1364
+ const nodeId = asString(params.nodeId);
1365
+ return Effect.all([
1366
+ incrementMetric(gatewayApprovalDecisionsTotal, {
1367
+ outcome: approved ? "approved" : "denied",
1368
+ }),
1369
+ Effect.logInfo("Gateway approval decision recorded").pipe(Effect.annotateLogs({
1370
+ ...gatewayRpcAnnotations(context, frame, response.payload),
1371
+ ...(nodeId ? { nodeId } : {}),
1372
+ iteration: asNumber(params.iteration) ?? 0,
1373
+ approved,
1374
+ })),
1375
+ ], { discard: true });
1376
+ }
1377
+ case "signals.send": {
1378
+ const signalName = asString(params.signalName);
1379
+ const correlationId = asString(params.correlationId);
1380
+ return Effect.all([
1381
+ incrementMetric(gatewaySignalsTotal, { outcome: "sent" }),
1382
+ Effect.logInfo("Gateway signal sent").pipe(Effect.annotateLogs({
1383
+ ...gatewayRpcAnnotations(context, frame, response.payload),
1384
+ ...(signalName ? { signalName } : {}),
1385
+ ...(correlationId ? { correlationId } : {}),
1386
+ })),
1387
+ ], { discard: true });
1388
+ }
1389
+ case "cron.trigger": {
1390
+ const cronId = asString(params.cronId);
1391
+ const workflow = asString(params.workflow);
1392
+ return Effect.all([
1393
+ incrementMetric(gatewayCronTriggersTotal, { source: "manual" }),
1394
+ Effect.logInfo("Gateway cron trigger requested").pipe(Effect.annotateLogs({
1395
+ ...gatewayRpcAnnotations(context, frame, response.payload),
1396
+ ...(cronId ? { cronId } : {}),
1397
+ ...(workflow ? { workflow } : {}),
1398
+ })),
1399
+ ], { discard: true });
1400
+ }
1401
+ default:
1402
+ return Effect.void;
1403
+ }
1404
+ }
1405
+ /**
1406
+ * @param {ServerResponse} res
1407
+ * @param {number} status
1408
+ * @param {ResponseFrame} response
1409
+ */
1410
+ sendHttpRpcResponse(res, status, response) {
1411
+ this.recordMessageSent("http", "response", {
1412
+ outcome: response.ok ? "ok" : "error",
1413
+ });
1414
+ return sendJson(res, status, response);
1415
+ }
1416
+ /**
1417
+ * @param {SmithersDb} adapter
1418
+ * @param {string} runId
1419
+ * @param {string} signalName
1420
+ * @param {string | null} correlationId
1421
+ */
1422
+ async runWaitsForSignal(adapter, runId, signalName, correlationId) {
1423
+ const nodes = await adapter.listNodes(runId);
1424
+ for (const node of nodes) {
1425
+ if (node.state !== "waiting-event") {
1426
+ continue;
1427
+ }
1428
+ const iteration = node.iteration ?? 0;
1429
+ const attempts = await runPromise(adapter.listAttempts(runId, node.nodeId, iteration));
1430
+ const waitingAttempt = attempts.find((attempt) => attempt.state === "waiting-event") ??
1431
+ attempts[0];
1432
+ const snapshot = parseWebhookWaitForEventSnapshot(waitingAttempt?.metaJson);
1433
+ if (!snapshot) {
1434
+ continue;
1435
+ }
1436
+ if (snapshot.signalName !== signalName) {
1437
+ continue;
1438
+ }
1439
+ if (snapshot.correlationId !== correlationId) {
1440
+ continue;
1441
+ }
1442
+ return true;
1443
+ }
1444
+ return false;
1445
+ }
1446
+ /**
1447
+ * @param {RegisteredWorkflow} entry
1448
+ * @param {string} signalName
1449
+ * @param {string | null} correlationId
1450
+ * @param {string} [explicitRunId]
1451
+ */
1452
+ async findMatchingWebhookRuns(entry, signalName, correlationId, explicitRunId) {
1453
+ const adapter = this.adapterForWorkflow(entry.workflow);
1454
+ const matches = new Set();
1455
+ if (explicitRunId) {
1456
+ const run = await adapter.getRun(explicitRunId);
1457
+ if (run &&
1458
+ run.status !== "finished" &&
1459
+ run.status !== "failed" &&
1460
+ run.status !== "cancelled" &&
1461
+ await this.runWaitsForSignal(adapter, explicitRunId, signalName, correlationId)) {
1462
+ matches.add(explicitRunId);
1463
+ }
1464
+ return [...matches];
1465
+ }
1466
+ const waitingRuns = await adapter.listRuns(1_000, "waiting-event");
1467
+ for (const run of waitingRuns) {
1468
+ if (await this.runWaitsForSignal(adapter, run.runId, signalName, correlationId)) {
1469
+ matches.add(run.runId);
1470
+ }
1471
+ }
1472
+ return [...matches];
1473
+ }
1474
+ /**
1475
+ * @param {IncomingMessage} req
1476
+ * @param {ServerResponse} res
1477
+ * @param {string} workflowKey
1478
+ */
1479
+ async handleWebhook(req, res, workflowKey) {
1480
+ const requestId = headerValue(req, "x-request-id") ?? randomUUID();
1481
+ /**
1482
+ * @param {number} status
1483
+ * @param {unknown} payload
1484
+ */
1485
+ const respond = (status, payload) => {
1486
+ this.recordMessageSent("http", "response", {
1487
+ route: "webhook",
1488
+ workflow: workflowKey,
1489
+ outcome: status < 400 ? "ok" : "error",
1490
+ });
1491
+ return sendJson(res, status, payload);
1492
+ };
1493
+ /**
1494
+ * @param {number} status
1495
+ * @param {string} code
1496
+ * @param {string} message
1497
+ * @param {string} reason
1498
+ * @param {unknown} [error]
1499
+ */
1500
+ const reject = (status, code, message, reason, error) => {
1501
+ emitGatewayEffect(Effect.all([
1502
+ incrementMetric(gatewayWebhooksRejectedTotal, {
1503
+ workflow: workflowKey,
1504
+ reason,
1505
+ }),
1506
+ incrementMetric(gatewayErrorsTotal, {
1507
+ kind: "webhook",
1508
+ workflow: workflowKey,
1509
+ code,
1510
+ }),
1511
+ ], { discard: true }));
1512
+ emitGatewayLog(error && !isSmithersError(error) ? "error" : "warning", "Gateway webhook rejected", {
1513
+ workflow: workflowKey,
1514
+ requestId,
1515
+ reason,
1516
+ errorCode: code,
1517
+ errorMessage: message,
1518
+ ...(error ? gatewayErrorAnnotations(error) : {}),
1519
+ }, "gateway:webhook");
1520
+ return respond(status, { ok: false, error: { code, message } });
1521
+ };
1522
+ this.recordMessageReceived("http", "webhook", { workflow: workflowKey });
1523
+ emitGatewayEffect(incrementMetric(gatewayWebhooksReceivedTotal, {
1524
+ workflow: workflowKey,
1525
+ }));
1526
+ const entry = this.workflows.get(workflowKey);
1527
+ if (!entry) {
1528
+ return reject(404, "NOT_FOUND", `Unknown workflow: ${workflowKey}`, "workflow_not_found");
1529
+ }
1530
+ const webhook = entry.webhook;
1531
+ if (!webhook) {
1532
+ return reject(404, "NOT_FOUND", `Webhook not configured for workflow: ${workflowKey}`, "not_configured");
1533
+ }
1534
+ const secret = webhook.secret.trim();
1535
+ if (!secret) {
1536
+ return reject(500, "SERVER_ERROR", "Webhook secret is not configured", "not_configured");
1537
+ }
1538
+ const signatureHeader = webhook.signatureHeader?.trim().toLowerCase() || "x-hub-signature-256";
1539
+ const signaturePrefix = webhook.signaturePrefix ?? "sha256=";
1540
+ const signalConfig = webhook.signal;
1541
+ const runConfig = webhook.run;
1542
+ const runEnabled = runConfig?.enabled !== false;
1543
+ if (!signalConfig?.name && !runEnabled) {
1544
+ return reject(400, "INVALID_REQUEST", "Webhook config must enable signal delivery or run creation", "misconfigured");
1545
+ }
1546
+ try {
1547
+ const rawBody = await readRawBody(req, this.maxBodyBytes);
1548
+ const providedSignature = headerValue(req, signatureHeader);
1549
+ const expectedSignature = computeWebhookSignature(rawBody, secret, signaturePrefix);
1550
+ if (!isValidWebhookSignature(expectedSignature, providedSignature)) {
1551
+ return reject(401, "UNAUTHORIZED", "Webhook signature verification failed", "invalid_signature");
1552
+ }
1553
+ emitGatewayEffect(incrementMetric(gatewayWebhooksVerifiedTotal, {
1554
+ workflow: workflowKey,
1555
+ }));
1556
+ const payload = parseJsonBuffer(rawBody, "Webhook payload");
1557
+ const adapter = this.adapterForWorkflow(entry.workflow);
1558
+ const explicitRunId = asWebhookString(readPathValue(payload, signalConfig?.runIdPath));
1559
+ const correlationId = normalizeCorrelationId(asWebhookString(readPathValue(payload, signalConfig?.correlationIdPath)) ?? null);
1560
+ const signalPayload = readPathValue(payload, signalConfig?.payloadPath);
1561
+ const matchedRunIds = signalConfig?.name
1562
+ ? await this.findMatchingWebhookRuns(entry, signalConfig.name, correlationId, explicitRunId)
1563
+ : [];
1564
+ const triggeredBy = webhookTriggerUserId(workflowKey);
1565
+ const auth = {
1566
+ triggeredBy,
1567
+ scopes: ["*"],
1568
+ role: "system",
1569
+ };
1570
+ const delivered = [];
1571
+ for (const runId of matchedRunIds) {
1572
+ const signal = await Effect.runPromise(signalRun(adapter, runId, signalConfig.name, signalPayload, {
1573
+ correlationId,
1574
+ receivedBy: triggeredBy,
1575
+ }));
1576
+ delivered.push({
1577
+ runId,
1578
+ seq: signal.seq,
1579
+ signalName: signal.signalName,
1580
+ correlationId: signal.correlationId ?? null,
1581
+ receivedAtMs: signal.receivedAtMs,
1582
+ });
1583
+ await this.resumeRunIfNeeded(runId, workflowKey, adapter, auth);
1584
+ }
1585
+ const started = delivered.length === 0 && runEnabled
1586
+ ? await this.startRun(workflowKey, normalizeWebhookRunInput(readPathValue(payload, runConfig?.inputPath)), auth)
1587
+ : null;
1588
+ emitGatewayLog("info", "Gateway webhook processed", {
1589
+ workflow: workflowKey,
1590
+ requestId,
1591
+ matchedRunCount: matchedRunIds.length,
1592
+ deliveredCount: delivered.length,
1593
+ ...(started ? { startedRunId: started.runId } : {}),
1594
+ }, "gateway:webhook");
1595
+ return respond(200, {
1596
+ ok: true,
1597
+ workflow: workflowKey,
1598
+ verified: true,
1599
+ delivered,
1600
+ matchedRunIds,
1601
+ started,
1602
+ });
1603
+ }
1604
+ catch (error) {
1605
+ if (isSmithersError(error)) {
1606
+ return reject(statusForRpcError(error.code), error.code, error.summary, "invalid_payload", error);
1607
+ }
1608
+ return reject(500, "SERVER_ERROR", error?.message ?? "Gateway webhook failed", "server_error", error);
1609
+ }
1610
+ }
1611
+ /**
1612
+ * @param {string} key
1613
+ * @param {SmithersWorkflow} workflow
1614
+ * @param {{ schedule?: string; webhook?: GatewayWebhookConfig }} [options]
1615
+ * @returns {this}
1616
+ */
1617
+ register(key, workflow, options) {
1618
+ ensureSmithersTables(workflow.db);
1619
+ this.workflows.set(key, {
1620
+ key,
1621
+ workflow,
1622
+ schedule: options?.schedule,
1623
+ webhook: options?.webhook,
1624
+ });
1625
+ // Startup recovery: any audit row left in `in_progress` from a prior
1626
+ // crash is flipped to `partial` and the associated run is flagged as
1627
+ // `needs_attention`. Runs asynchronously; failures are logged and
1628
+ // never block registration.
1629
+ const adapter = new SmithersDb(workflow.db);
1630
+ recoverInProgressRewindAudits(adapter).catch((error) => {
1631
+ emitGatewayLog("warning", "rewind audit recovery failed", {
1632
+ workflow: key,
1633
+ ...gatewayErrorAnnotations(error),
1634
+ }, "gateway:startup-recovery");
1635
+ });
1636
+ return this;
1637
+ }
1638
+ /**
1639
+ * @param {{ port?: number; host?: string }} [options]
1640
+ */
1641
+ async listen(options = {}) {
1642
+ if (this.server) {
1643
+ return this.server;
1644
+ }
1645
+ const wsServer = new WebSocketServer({
1646
+ noServer: true,
1647
+ maxPayload: this.maxPayload,
1648
+ });
1649
+ const server = createServer(async (req, res) => {
1650
+ const url = new URL(req.url ?? "/", "http://127.0.0.1");
1651
+ const webhookMatch = url.pathname.match(/^\/webhooks\/([^/]+)$/);
1652
+ if ((req.method ?? "GET") === "GET" && (req.url ?? "/") === "/health") {
1653
+ return sendJson(res, 200, {
1654
+ ok: true,
1655
+ protocol: this.protocol,
1656
+ features: this.features,
1657
+ stateVersion: this.stateVersion,
1658
+ });
1659
+ }
1660
+ if ((req.method ?? "GET") === "GET" && (req.url ?? "/") === "/metrics") {
1661
+ return sendText(res, 200, renderPrometheusMetrics(), prometheusContentType);
1662
+ }
1663
+ if ((req.method ?? "GET") === "POST" && webhookMatch) {
1664
+ return this.handleWebhook(req, res, decodeURIComponent(webhookMatch[1]));
1665
+ }
1666
+ if ((req.method ?? "GET") === "POST" && (req.url ?? "/") === "/rpc") {
1667
+ return this.handleHttpRpc(req, res);
1668
+ }
1669
+ return sendJson(res, 404, { error: { code: "NOT_FOUND", message: "Route not found" } });
1670
+ });
1671
+ server.on("upgrade", (req, socket, head) => {
1672
+ if (this.connections.size >= this.maxConnections) {
1673
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
1674
+ kind: "connection_limit",
1675
+ transport: "ws",
1676
+ }));
1677
+ emitGatewayLog("warning", "Gateway connection rejected", {
1678
+ transport: "ws",
1679
+ remoteAddress: req.socket.remoteAddress ?? null,
1680
+ maxConnections: this.maxConnections,
1681
+ }, "gateway:connect");
1682
+ const body = "Gateway connection limit reached\n";
1683
+ socket.write("HTTP/1.1 503 Service Unavailable\r\n"
1684
+ + "Connection: close\r\n"
1685
+ + "Content-Type: text/plain; charset=utf-8\r\n"
1686
+ + `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n`
1687
+ + "\r\n"
1688
+ + body);
1689
+ socket.destroy();
1690
+ return;
1691
+ }
1692
+ wsServer.handleUpgrade(req, socket, head, (ws) => {
1693
+ this.handleSocket(ws, req);
1694
+ });
1695
+ });
1696
+ await new Promise((resolve) => {
1697
+ if (options.host === undefined) {
1698
+ server.listen(options.port ?? 7331, () => resolve());
1699
+ return;
1700
+ }
1701
+ server.listen(options.port ?? 7331, options.host, () => resolve());
1702
+ });
1703
+ this.server = server;
1704
+ this.wsServer = wsServer;
1705
+ await this.syncRegisteredSchedules();
1706
+ this.startScheduler();
1707
+ return server;
1708
+ }
1709
+ async close() {
1710
+ const activeRuns = [...this.activeRuns.values()];
1711
+ for (const activeRun of activeRuns) {
1712
+ activeRun.abort.abort();
1713
+ }
1714
+ const inflightRuns = [...this.inflightRuns.values()];
1715
+ if (inflightRuns.length > 0) {
1716
+ await Promise.allSettled(inflightRuns);
1717
+ }
1718
+ for (const connection of this.connections) {
1719
+ if (connection.heartbeatTimer) {
1720
+ clearInterval(connection.heartbeatTimer);
1721
+ }
1722
+ try {
1723
+ connection.ws.close();
1724
+ }
1725
+ catch { }
1726
+ }
1727
+ this.connections.clear();
1728
+ if (this.schedulerTimer) {
1729
+ clearInterval(this.schedulerTimer);
1730
+ this.schedulerTimer = null;
1731
+ }
1732
+ if (this.server) {
1733
+ const server = this.server;
1734
+ this.server = null;
1735
+ await new Promise((resolve) => {
1736
+ let settled = false;
1737
+ const done = () => {
1738
+ if (settled) {
1739
+ return;
1740
+ }
1741
+ settled = true;
1742
+ resolve();
1743
+ };
1744
+ const timeout = setTimeout(done, 250);
1745
+ timeout.unref?.();
1746
+ server.close(() => {
1747
+ clearTimeout(timeout);
1748
+ done();
1749
+ });
1750
+ server.closeIdleConnections?.();
1751
+ server.closeAllConnections?.();
1752
+ });
1753
+ }
1754
+ if (this.wsServer) {
1755
+ this.wsServer.close();
1756
+ this.wsServer = null;
1757
+ }
1758
+ }
1759
+ startScheduler() {
1760
+ if (this.schedulerTimer) {
1761
+ clearInterval(this.schedulerTimer);
1762
+ }
1763
+ const intervalMs = Math.max(1_000, Math.min(this.heartbeatMs, 15_000));
1764
+ this.schedulerTimer = setInterval(() => {
1765
+ void this.processDueCrons();
1766
+ }, intervalMs);
1767
+ }
1768
+ async syncRegisteredSchedules() {
1769
+ for (const entry of this.workflows.values()) {
1770
+ if (!entry.schedule) {
1771
+ continue;
1772
+ }
1773
+ const adapter = this.adapterForWorkflow(entry.workflow);
1774
+ await adapter.upsertCron({
1775
+ cronId: `gateway:${entry.key}`,
1776
+ pattern: entry.schedule,
1777
+ workflowPath: cronWorkflowPath(entry.key),
1778
+ enabled: true,
1779
+ createdAtMs: nowMs(),
1780
+ lastRunAtMs: null,
1781
+ nextRunAtMs: nextCronRunAtMs(entry.schedule),
1782
+ errorJson: null,
1783
+ });
1784
+ }
1785
+ }
1786
+ async processDueCrons() {
1787
+ const now = nowMs();
1788
+ emitGatewayLog("debug", "Gateway cron evaluation tick", {
1789
+ workflowCount: this.workflows.size,
1790
+ }, "gateway:cron");
1791
+ for (const entry of this.workflows.values()) {
1792
+ const adapter = this.adapterForWorkflow(entry.workflow);
1793
+ const crons = await adapter.listCrons(true);
1794
+ for (const cron of crons) {
1795
+ const workflowKey = workflowKeyFromCronPath(cron.workflowPath);
1796
+ if (!workflowKey || workflowKey !== entry.key) {
1797
+ continue;
1798
+ }
1799
+ if (typeof cron.nextRunAtMs === "number" && cron.nextRunAtMs > now) {
1800
+ emitGatewayLog("debug", "Gateway cron skipped", {
1801
+ cronId: cron.cronId,
1802
+ workflow: workflowKey,
1803
+ nextRunAtMs: cron.nextRunAtMs,
1804
+ }, "gateway:cron");
1805
+ continue;
1806
+ }
1807
+ try {
1808
+ const run = await this.startRun(workflowKey, {}, {
1809
+ triggeredBy: "cron:gateway",
1810
+ scopes: ["*"],
1811
+ role: "system",
1812
+ });
1813
+ await adapter.updateCronRunTime(cron.cronId, now, nextCronRunAtMs(cron.pattern), null);
1814
+ emitGatewayEffect(incrementMetric(gatewayCronTriggersTotal, {
1815
+ source: "scheduled",
1816
+ }));
1817
+ emitGatewayLog("info", "Gateway cron triggered", {
1818
+ cronId: cron.cronId,
1819
+ workflow: workflowKey,
1820
+ runId: run.runId,
1821
+ }, "gateway:cron");
1822
+ this.broadcastEvent("cron.triggered", {
1823
+ cronId: cron.cronId,
1824
+ workflow: workflowKey,
1825
+ runId: run.runId,
1826
+ });
1827
+ }
1828
+ catch (error) {
1829
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
1830
+ kind: "cron",
1831
+ code: gatewayErrorCode(error),
1832
+ }));
1833
+ emitGatewayLog("error", "Gateway cron trigger failed", {
1834
+ cronId: cron.cronId,
1835
+ workflow: workflowKey,
1836
+ ...gatewayErrorAnnotations(error),
1837
+ }, "gateway:cron");
1838
+ await adapter.updateCronRunTime(cron.cronId, now, cron.nextRunAtMs ?? now + 60_000, error?.message ?? "cron trigger failed");
1839
+ }
1840
+ }
1841
+ }
1842
+ }
1843
+ /**
1844
+ * @param {string} workflowKey
1845
+ * @param {Record<string, unknown>} input
1846
+ * @param {RunStartAuthContext} auth
1847
+ * @param {string} [runId]
1848
+ * @param {{ resume?: boolean }} [options]
1849
+ */
1850
+ async startRun(workflowKey, input, auth, runId = crypto.randomUUID(), options) {
1851
+ const entry = this.workflows.get(workflowKey);
1852
+ if (!entry) {
1853
+ throw new Error(`Unknown workflow: ${workflowKey}`);
1854
+ }
1855
+ const abort = new AbortController();
1856
+ const record = {
1857
+ workflowKey,
1858
+ workflow: entry.workflow,
1859
+ abort,
1860
+ input,
1861
+ };
1862
+ this.runRegistry.set(runId, record);
1863
+ this.activeRuns.set(runId, record);
1864
+ emitGatewayEffect(Effect.all([
1865
+ incrementMetric(gatewayRunsStartedTotal, {
1866
+ workflow: workflowKey,
1867
+ source: gatewayTriggerSource(auth.triggeredBy),
1868
+ resume: options?.resume ? "true" : "false",
1869
+ }),
1870
+ Effect.logInfo("Gateway run started").pipe(Effect.annotateLogs({
1871
+ workflow: workflowKey,
1872
+ runId,
1873
+ triggeredBy: auth.triggeredBy,
1874
+ source: gatewayTriggerSource(auth.triggeredBy),
1875
+ resume: options?.resume ?? false,
1876
+ ...(auth.subscribeConnection
1877
+ ? gatewayContextAnnotations(auth.subscribeConnection)
1878
+ : {}),
1879
+ }), Effect.withLogSpan("gateway:run")),
1880
+ ], { discard: true }));
1881
+ if (auth.subscribeConnection) {
1882
+ if (!auth.subscribeConnection.subscribedRuns) {
1883
+ auth.subscribeConnection.subscribedRuns = new Set();
1884
+ }
1885
+ auth.subscribeConnection.subscribedRuns.add(runId);
1886
+ }
1887
+ const runPromise = Effect.runPromise(runWorkflow(entry.workflow, {
1888
+ runId,
1889
+ input,
1890
+ resume: options?.resume,
1891
+ signal: abort.signal,
1892
+ onProgress: (event) => this.handleSmithersEvent(event),
1893
+ cliAgentToolsDefault: this.defaults?.cliAgentTools,
1894
+ config: {
1895
+ gatewayWorkflowKey: workflowKey,
1896
+ gatewayTriggeredBy: auth.triggeredBy,
1897
+ },
1898
+ auth: {
1899
+ triggeredBy: auth.triggeredBy,
1900
+ scopes: [...auth.scopes],
1901
+ role: auth.role,
1902
+ createdAt: new Date().toISOString(),
1903
+ },
1904
+ }))
1905
+ .catch((error) => {
1906
+ emitGatewayEffect(Effect.all([
1907
+ incrementMetric(gatewayErrorsTotal, {
1908
+ kind: "run",
1909
+ workflow: workflowKey,
1910
+ code: gatewayErrorCode(error),
1911
+ }),
1912
+ incrementMetric(gatewayRunsCompletedTotal, {
1913
+ workflow: workflowKey,
1914
+ status: "failed",
1915
+ }),
1916
+ Effect.logError("Gateway run failed").pipe(Effect.annotateLogs({
1917
+ workflow: workflowKey,
1918
+ runId,
1919
+ source: gatewayTriggerSource(auth.triggeredBy),
1920
+ ...gatewayErrorAnnotations(error),
1921
+ }), Effect.withLogSpan("gateway:run")),
1922
+ ], { discard: true }));
1923
+ this.broadcastEvent("run.completed", {
1924
+ runId,
1925
+ status: "failed",
1926
+ error: errorToJson(error),
1927
+ });
1928
+ throw error;
1929
+ })
1930
+ .then((result) => {
1931
+ if (result.status === "finished" || result.status === "failed" || result.status === "cancelled") {
1932
+ emitGatewayEffect(Effect.all([
1933
+ incrementMetric(gatewayRunsCompletedTotal, {
1934
+ workflow: workflowKey,
1935
+ status: result.status,
1936
+ }),
1937
+ Effect.logInfo("Gateway run completed").pipe(Effect.annotateLogs({
1938
+ workflow: workflowKey,
1939
+ runId,
1940
+ status: result.status,
1941
+ source: gatewayTriggerSource(auth.triggeredBy),
1942
+ ...(result.error ? { error: result.error } : {}),
1943
+ }), Effect.withLogSpan("gateway:run")),
1944
+ ], { discard: true }));
1945
+ this.broadcastEvent("run.completed", {
1946
+ runId,
1947
+ status: result.status,
1948
+ error: result.error,
1949
+ });
1950
+ }
1951
+ })
1952
+ .finally(() => {
1953
+ this.activeRuns.delete(runId);
1954
+ this.inflightRuns.delete(runId);
1955
+ });
1956
+ this.inflightRuns.set(runId, runPromise.then(() => undefined, () => undefined));
1957
+ return { runId, workflow: workflowKey };
1958
+ }
1959
+ /**
1960
+ * @param {string} runId
1961
+ * @param {string} workflowKey
1962
+ * @param {SmithersDb} adapter
1963
+ * @param {RunStartAuthContext} auth
1964
+ */
1965
+ async resumeRunIfNeeded(runId, workflowKey, adapter, auth) {
1966
+ for (let attempt = 0; attempt < 20; attempt += 1) {
1967
+ if (this.activeRuns.has(runId)) {
1968
+ await delay(25);
1969
+ continue;
1970
+ }
1971
+ const run = await adapter.getRun(runId);
1972
+ if (!run) {
1973
+ return;
1974
+ }
1975
+ if (run.status === "finished" || run.status === "failed" || run.status === "cancelled") {
1976
+ return;
1977
+ }
1978
+ await this.startRun(workflowKey, {}, auth, runId, { resume: true });
1979
+ return;
1980
+ }
1981
+ }
1982
+ /**
1983
+ * @param {WebSocket} ws
1984
+ * @param {IncomingMessage} req
1985
+ */
1986
+ handleSocket(ws, req) {
1987
+ const connection = {
1988
+ connectionId: randomUUID(),
1989
+ transport: "ws",
1990
+ ws,
1991
+ seq: 0,
1992
+ authenticated: false,
1993
+ sessionToken: null,
1994
+ role: null,
1995
+ scopes: [],
1996
+ userId: null,
1997
+ subscribedRuns: null,
1998
+ devtoolsStreams: new Map(),
1999
+ heartbeatTimer: null,
2000
+ };
2001
+ this.connections.add(connection);
2002
+ emitGatewayEffect(Effect.all([
2003
+ incrementMetric(gatewayConnectionsTotal, { transport: "ws" }),
2004
+ updateMetric(gatewayConnectionsActive, 1, { transport: "ws" }),
2005
+ ], { discard: true }));
2006
+ emitGatewayLog("info", "Gateway connection opened", {
2007
+ ...gatewayContextAnnotations(connection),
2008
+ remoteAddress: req.socket.remoteAddress ?? null,
2009
+ activeConnections: this.connections.size,
2010
+ }, "gateway:connect");
2011
+ this.sendEvent(connection, "connect.challenge", {
2012
+ nonce: randomUUID(),
2013
+ ts: nowMs(),
2014
+ });
2015
+ ws.on("message", async (raw) => {
2016
+ this.recordMessageReceived("ws", "request");
2017
+ try {
2018
+ const frame = parseGatewayRequestFrame(raw, this.maxPayload);
2019
+ const response = await this.executeRpc(connection, frame, async () => {
2020
+ if (!connection.authenticated && frame.method !== "connect") {
2021
+ return responseError(frame.id, "UNAUTHORIZED", "Connect first");
2022
+ }
2023
+ if (frame.method === "connect") {
2024
+ return this.handleConnect(connection, req, frame.id, frame.params);
2025
+ }
2026
+ if (!hasScope(connection.scopes, frame.method)) {
2027
+ return responseError(frame.id, "FORBIDDEN", `Missing scope for ${frame.method}`);
2028
+ }
2029
+ return this.routeRequest(connection, frame);
2030
+ });
2031
+ this.sendResponse(connection, response);
2032
+ }
2033
+ catch (error) {
2034
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
2035
+ kind: "frame",
2036
+ transport: "ws",
2037
+ code: gatewayErrorCode(error),
2038
+ }));
2039
+ emitGatewayLog(isSmithersError(error) ? "warning" : "error", "Gateway websocket frame failed", {
2040
+ ...gatewayContextAnnotations(connection),
2041
+ ...gatewayErrorAnnotations(error),
2042
+ }, "gateway:rpc:invalid");
2043
+ if (isSmithersError(error)) {
2044
+ this.sendResponse(connection, responseError("invalid", error.code, error.summary));
2045
+ return;
2046
+ }
2047
+ this.sendResponse(connection, responseError("server", "SERVER_ERROR", error?.message ?? "Gateway request failed"));
2048
+ }
2049
+ });
2050
+ let cleanedUp = false;
2051
+ /**
2052
+ * @param {"close" | "error"} reason
2053
+ * @param {Record<string, unknown>} [details]
2054
+ */
2055
+ const cleanup = (reason, details = {}) => {
2056
+ if (cleanedUp) {
2057
+ return;
2058
+ }
2059
+ cleanedUp = true;
2060
+ if (connection.heartbeatTimer) {
2061
+ clearInterval(connection.heartbeatTimer);
2062
+ }
2063
+ this.connections.delete(connection);
2064
+ this.cleanupDevToolsSubscribers(connection);
2065
+ emitGatewayEffect(Effect.all([
2066
+ updateMetric(gatewayConnectionsActive, -1, { transport: "ws" }),
2067
+ incrementMetric(gatewayConnectionsClosedTotal, {
2068
+ transport: "ws",
2069
+ reason,
2070
+ }),
2071
+ ...(reason === "error"
2072
+ ? [
2073
+ incrementMetric(gatewayErrorsTotal, {
2074
+ kind: "socket",
2075
+ transport: "ws",
2076
+ }),
2077
+ ]
2078
+ : []),
2079
+ ], { discard: true }));
2080
+ emitGatewayLog(reason === "error" ? "warning" : "info", "Gateway connection closed", {
2081
+ ...gatewayContextAnnotations(connection),
2082
+ activeConnections: this.connections.size,
2083
+ closeReason: reason,
2084
+ ...details,
2085
+ }, "gateway:connect");
2086
+ };
2087
+ ws.on("close", (code, reason) => {
2088
+ cleanup("close", {
2089
+ closeCode: code,
2090
+ closeMessage: rawDataToUtf8(reason),
2091
+ });
2092
+ });
2093
+ ws.on("error", (error) => {
2094
+ cleanup("error", gatewayErrorAnnotations(error));
2095
+ });
2096
+ }
2097
+ /**
2098
+ * @param {ConnectionState} connection
2099
+ */
2100
+ startHeartbeat(connection) {
2101
+ if (connection.heartbeatTimer) {
2102
+ clearInterval(connection.heartbeatTimer);
2103
+ }
2104
+ connection.heartbeatTimer = setInterval(() => {
2105
+ emitGatewayEffect(incrementMetric(gatewayHeartbeatTicksTotal));
2106
+ this.sendEvent(connection, "tick", {
2107
+ ts: nowMs(),
2108
+ });
2109
+ }, this.heartbeatMs);
2110
+ }
2111
+ /**
2112
+ * @param {ConnectionState} connection
2113
+ * @param {IncomingMessage} req
2114
+ * @param {string} id
2115
+ * @param {unknown} params
2116
+ * @returns {Promise<ResponseFrame>}
2117
+ */
2118
+ async handleConnect(connection, req, id, params) {
2119
+ const request = asObject(params);
2120
+ if (!request) {
2121
+ return responseError(id, "INVALID_REQUEST", "Connect params must be an object");
2122
+ }
2123
+ if (typeof request.minProtocol !== "number" ||
2124
+ typeof request.maxProtocol !== "number" ||
2125
+ !request.client) {
2126
+ return responseError(id, "INVALID_REQUEST", "Connect request is missing protocol negotiation fields");
2127
+ }
2128
+ try {
2129
+ assertPositiveFiniteInteger("minProtocol", request.minProtocol);
2130
+ assertPositiveFiniteInteger("maxProtocol", request.maxProtocol);
2131
+ }
2132
+ catch (error) {
2133
+ if (error instanceof SmithersError) {
2134
+ return responseError(id, error.code, error.summary);
2135
+ }
2136
+ throw error;
2137
+ }
2138
+ if (request.minProtocol > this.protocol || request.maxProtocol < this.protocol) {
2139
+ return responseError(id, "PROTOCOL_UNSUPPORTED", `Gateway protocol ${this.protocol} is not supported by the client`);
2140
+ }
2141
+ const authResult = await this.authenticate(req, request);
2142
+ if (authResult.ok === false) {
2143
+ this.recordAuthEvent("ws", "failure", connection, {
2144
+ clientId: request.client.id,
2145
+ clientVersion: request.client.version,
2146
+ authCode: authResult.code,
2147
+ authMessage: authResult.message,
2148
+ });
2149
+ return responseError(id, authResult.code, authResult.message);
2150
+ }
2151
+ connection.authenticated = true;
2152
+ connection.sessionToken = randomUUID();
2153
+ connection.role = authResult.role;
2154
+ connection.scopes = [...authResult.scopes];
2155
+ connection.userId = authResult.userId ?? null;
2156
+ connection.subscribedRuns = Array.isArray(request.subscribe)
2157
+ ? new Set(request.subscribe.filter((value) => typeof value === "string"))
2158
+ : null;
2159
+ this.startHeartbeat(connection);
2160
+ this.recordAuthEvent("ws", "success", connection, {
2161
+ clientId: request.client.id,
2162
+ clientVersion: request.client.version,
2163
+ scopeCount: connection.scopes.length,
2164
+ });
2165
+ const hello = {
2166
+ protocol: this.protocol,
2167
+ features: this.features,
2168
+ policy: { heartbeatMs: this.heartbeatMs },
2169
+ auth: {
2170
+ sessionToken: connection.sessionToken,
2171
+ role: authResult.role,
2172
+ scopes: authResult.scopes,
2173
+ userId: authResult.userId ?? null,
2174
+ },
2175
+ snapshot: await this.buildSnapshot(),
2176
+ };
2177
+ return responseOk(id, hello);
2178
+ }
2179
+ /**
2180
+ * @param {IncomingMessage} req
2181
+ * @param {ConnectRequest} request
2182
+ * @returns {Promise< | { ok: true; role: string; scopes: string[]; userId?: string } | { ok: false; code: string; message: string } >}
2183
+ */
2184
+ async authenticate(req, request) {
2185
+ const tokenFromRequest = "token" in (request.auth ?? {}) ? request.auth.token : null;
2186
+ return this.authenticateRequest(req, typeof tokenFromRequest === "string" ? tokenFromRequest : null);
2187
+ }
2188
+ /**
2189
+ * @param {IncomingMessage} req
2190
+ * @param {string | null} token
2191
+ * @returns {Promise< | { ok: true; role: string; scopes: string[]; userId?: string } | { ok: false; code: string; message: string } >}
2192
+ */
2193
+ async authenticateRequest(req, token) {
2194
+ if (!this.auth) {
2195
+ return {
2196
+ ok: true,
2197
+ role: "operator",
2198
+ scopes: ["*"],
2199
+ };
2200
+ }
2201
+ if (this.auth.mode === "token") {
2202
+ if (!token || typeof token !== "string") {
2203
+ return {
2204
+ ok: false,
2205
+ code: "UNAUTHORIZED",
2206
+ message: "A bearer token is required",
2207
+ };
2208
+ }
2209
+ const grant = this.auth.tokens[token];
2210
+ if (!grant) {
2211
+ return {
2212
+ ok: false,
2213
+ code: "UNAUTHORIZED",
2214
+ message: "Invalid token",
2215
+ };
2216
+ }
2217
+ return {
2218
+ ok: true,
2219
+ role: grant.role,
2220
+ scopes: grant.scopes,
2221
+ userId: grant.userId,
2222
+ };
2223
+ }
2224
+ if (this.auth.mode === "jwt") {
2225
+ if (!token || typeof token !== "string") {
2226
+ return {
2227
+ ok: false,
2228
+ code: "UNAUTHORIZED",
2229
+ message: "A bearer token is required",
2230
+ };
2231
+ }
2232
+ const verified = verifyJwtToken(token, this.auth);
2233
+ if (verified.ok === false) {
2234
+ return {
2235
+ ok: false,
2236
+ code: "UNAUTHORIZED",
2237
+ message: verified.message,
2238
+ };
2239
+ }
2240
+ const scopes = parseJwtScopes(verified.payload[this.auth.scopesClaim ?? "scope"]);
2241
+ const role = asString(verified.payload[this.auth.roleClaim ?? "role"]) ??
2242
+ this.auth.defaultRole ??
2243
+ "operator";
2244
+ const userId = asString(verified.payload[this.auth.userClaim ?? "sub"]);
2245
+ return {
2246
+ ok: true,
2247
+ role,
2248
+ scopes: scopes.length > 0 ? scopes : [...(this.auth.defaultScopes ?? [])],
2249
+ userId: userId ?? undefined,
2250
+ };
2251
+ }
2252
+ if (this.auth.mode === "trusted-proxy") {
2253
+ const allowedOrigins = this.auth.allowedOrigins ?? [];
2254
+ const origin = asString(req.headers.origin);
2255
+ if (allowedOrigins.length > 0 && (!origin || !allowedOrigins.includes(origin))) {
2256
+ return {
2257
+ ok: false,
2258
+ code: "UNAUTHORIZED",
2259
+ message: "Origin is not allowed",
2260
+ };
2261
+ }
2262
+ const [userHeader = "x-user-id", scopesHeader = "x-user-scopes", roleHeader = "x-user-role"] = (this.auth.trustedHeaders ?? []).map((value) => value.toLowerCase());
2263
+ const userId = asString(req.headers[userHeader]);
2264
+ const scopesValue = asString(req.headers[scopesHeader]);
2265
+ const role = asString(req.headers[roleHeader]) ?? this.auth.defaultRole ?? "operator";
2266
+ const scopes = scopesValue
2267
+ ? scopesValue.split(/[,\s]+/).map((value) => value.trim()).filter(Boolean)
2268
+ : [...(this.auth.defaultScopes ?? ["*"])];
2269
+ return {
2270
+ ok: true,
2271
+ role,
2272
+ scopes,
2273
+ userId: userId ?? undefined,
2274
+ };
2275
+ }
2276
+ return {
2277
+ ok: false,
2278
+ code: "UNAUTHORIZED",
2279
+ message: "Unsupported auth mode",
2280
+ };
2281
+ }
2282
+ /**
2283
+ * @param {IncomingMessage} req
2284
+ * @param {ServerResponse} res
2285
+ */
2286
+ async handleHttpRpc(req, res) {
2287
+ const requestId = headerValue(req, "x-request-id") ?? randomUUID();
2288
+ const baseContext = {
2289
+ connectionId: `http:${requestId}`,
2290
+ transport: "http",
2291
+ role: null,
2292
+ scopes: [],
2293
+ userId: null,
2294
+ subscribedRuns: null,
2295
+ devtoolsStreams: null,
2296
+ };
2297
+ let context = baseContext;
2298
+ this.recordMessageReceived("http", "request");
2299
+ try {
2300
+ const authResult = await this.authenticateRequest(req, bearerTokenFromHeaders(req));
2301
+ if (authResult.ok === false) {
2302
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
2303
+ kind: "auth",
2304
+ transport: "http",
2305
+ code: authResult.code,
2306
+ }));
2307
+ this.recordAuthEvent("http", "failure", context, {
2308
+ requestId,
2309
+ authCode: authResult.code,
2310
+ authMessage: authResult.message,
2311
+ }, "warning");
2312
+ const response = responseError(requestId, authResult.code, authResult.message);
2313
+ return this.sendHttpRpcResponse(res, statusForRpcError(authResult.code), response);
2314
+ }
2315
+ context = {
2316
+ ...baseContext,
2317
+ role: authResult.role,
2318
+ scopes: [...authResult.scopes],
2319
+ userId: authResult.userId ?? null,
2320
+ };
2321
+ this.recordAuthEvent("http", "success", context, {
2322
+ requestId,
2323
+ scopeCount: authResult.scopes.length,
2324
+ }, "debug");
2325
+ const body = asObject(await readBody(req, this.maxBodyBytes));
2326
+ if (!body) {
2327
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
2328
+ kind: "http",
2329
+ transport: "http",
2330
+ code: "INVALID_REQUEST",
2331
+ }));
2332
+ emitGatewayLog("warning", "Gateway HTTP RPC rejected", {
2333
+ ...gatewayContextAnnotations(context),
2334
+ requestId,
2335
+ errorCode: "INVALID_REQUEST",
2336
+ errorMessage: "RPC body must be a JSON object",
2337
+ }, "gateway:http-rpc");
2338
+ return this.sendHttpRpcResponse(res, 400, responseError(requestId, "INVALID_REQUEST", "RPC body must be a JSON object"));
2339
+ }
2340
+ assertJsonPayloadWithinBounds("gateway frame", body, {
2341
+ maxArrayLength: GATEWAY_RPC_MAX_ARRAY_LENGTH,
2342
+ maxDepth: GATEWAY_RPC_MAX_DEPTH,
2343
+ maxStringLength: GATEWAY_RPC_MAX_STRING_LENGTH,
2344
+ });
2345
+ const method = validateGatewayMethodName(body.method);
2346
+ const bodyId = asString(body.id) ?? requestId;
2347
+ assertOptionalStringMaxLength("id", bodyId, GATEWAY_FRAME_ID_MAX_LENGTH);
2348
+ const frame = {
2349
+ type: "req",
2350
+ id: bodyId,
2351
+ method,
2352
+ params: body.params,
2353
+ };
2354
+ const response = await this.executeRpc(context, frame, async () => {
2355
+ if (!hasScope(context.scopes, method)) {
2356
+ return responseError(bodyId, "FORBIDDEN", `Missing scope for ${method}`);
2357
+ }
2358
+ return this.routeRequest(context, frame);
2359
+ });
2360
+ return this.sendHttpRpcResponse(res, response.ok ? 200 : statusForRpcError(response.error?.code), response);
2361
+ }
2362
+ catch (error) {
2363
+ emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
2364
+ kind: "http",
2365
+ transport: "http",
2366
+ code: gatewayErrorCode(error),
2367
+ }));
2368
+ emitGatewayLog(isSmithersError(error) ? "warning" : "error", "Gateway HTTP RPC failed", {
2369
+ ...gatewayContextAnnotations(context),
2370
+ requestId,
2371
+ ...gatewayErrorAnnotations(error),
2372
+ }, "gateway:http-rpc");
2373
+ if (isSmithersError(error)) {
2374
+ return this.sendHttpRpcResponse(res, statusForRpcError(error.code), responseError(requestId, error.code, error.summary));
2375
+ }
2376
+ const message = error?.message ?? "Gateway request failed";
2377
+ const status = message.includes("valid JSON") ? 400 : message.includes("exceeds") ? 413 : 500;
2378
+ return this.sendHttpRpcResponse(res, status, responseError(requestId, status === 413 ? "PAYLOAD_TOO_LARGE" : status === 400 ? "INVALID_JSON" : "SERVER_ERROR", message));
2379
+ }
2380
+ }
2381
+ /**
2382
+ * @param {ConnectionState} connection
2383
+ * @param {ResponseFrame} frame
2384
+ */
2385
+ sendResponse(connection, frame) {
2386
+ if (connection.ws.readyState !== connection.ws.OPEN) {
2387
+ return;
2388
+ }
2389
+ connection.ws.send(JSON.stringify(frame));
2390
+ this.recordMessageSent("ws", "response", {
2391
+ outcome: frame.ok ? "ok" : "error",
2392
+ });
2393
+ }
2394
+ /**
2395
+ * @param {ConnectionState} connection
2396
+ * @param {string} event
2397
+ * @param {unknown} [payload]
2398
+ */
2399
+ sendEvent(connection, event, payload, stateVersion = this.stateVersion) {
2400
+ if (connection.ws.readyState !== connection.ws.OPEN) {
2401
+ return;
2402
+ }
2403
+ connection.seq += 1;
2404
+ const frame = {
2405
+ type: "event",
2406
+ event,
2407
+ payload,
2408
+ seq: connection.seq,
2409
+ stateVersion,
2410
+ };
2411
+ connection.ws.send(JSON.stringify(frame));
2412
+ this.recordMessageSent("ws", "event", { event });
2413
+ }
2414
+ /**
2415
+ * @param {string} event
2416
+ * @param {unknown} [payload]
2417
+ */
2418
+ broadcastEvent(event, payload) {
2419
+ const runId = eventRunId(payload);
2420
+ this.stateVersion += 1;
2421
+ let recipientCount = 0;
2422
+ for (const connection of this.connections) {
2423
+ if (!connection.authenticated || !shouldDeliverEvent(connection, runId)) {
2424
+ continue;
2425
+ }
2426
+ recipientCount += 1;
2427
+ this.sendEvent(connection, event, payload, this.stateVersion);
2428
+ }
2429
+ emitGatewayLog("debug", "Gateway event broadcast", {
2430
+ event,
2431
+ stateVersion: this.stateVersion,
2432
+ recipientCount,
2433
+ ...(runId ? { runId } : {}),
2434
+ }, "gateway:broadcast");
2435
+ }
2436
+ async buildSnapshot() {
2437
+ const runs = await this.listRunsAcrossWorkflows(1_000);
2438
+ const approvals = await this.listPendingApprovals();
2439
+ return {
2440
+ runs: runs.filter((run) => ["running", "waiting-approval", "waiting-event", "waiting-timer"].includes(run.status)),
2441
+ approvals,
2442
+ stateVersion: this.stateVersion,
2443
+ };
2444
+ }
2445
+ /**
2446
+ * @param {SmithersWorkflow} workflow
2447
+ * @returns {SmithersDb}
2448
+ */
2449
+ adapterForWorkflow(workflow) {
2450
+ return new SmithersDb(workflow.db);
2451
+ }
2452
+ /**
2453
+ * @param {string} [status]
2454
+ */
2455
+ async listRunsAcrossWorkflows(limit = 50, status) {
2456
+ const results = [];
2457
+ for (const entry of this.workflows.values()) {
2458
+ const adapter = this.adapterForWorkflow(entry.workflow);
2459
+ const rows = await adapter.listRuns(limit, status);
2460
+ for (const row of rows) {
2461
+ const config = parseJson(row.configJson);
2462
+ results.push({
2463
+ ...row,
2464
+ workflowKey: asString(config?.gatewayWorkflowKey) ?? entry.key,
2465
+ });
2466
+ }
2467
+ }
2468
+ results.sort((a, b) => (b.createdAtMs ?? 0) - (a.createdAtMs ?? 0));
2469
+ return results.slice(0, limit);
2470
+ }
2471
+ async listPendingApprovals() {
2472
+ const approvals = [];
2473
+ for (const entry of this.workflows.values()) {
2474
+ const adapter = this.adapterForWorkflow(entry.workflow);
2475
+ const runs = await adapter.listRuns(1_000);
2476
+ for (const run of runs) {
2477
+ const pending = await adapter.listPendingApprovals(run.runId);
2478
+ const nodes = await adapter.listNodes(run.runId);
2479
+ const nodeByKey = new Map();
2480
+ for (const node of nodes) {
2481
+ nodeByKey.set(`${node.nodeId}::${node.iteration ?? 0}`, node);
2482
+ }
2483
+ for (const approval of pending) {
2484
+ const node = nodeByKey.get(`${approval.nodeId}::${approval.iteration ?? 0}`);
2485
+ const request = parseApprovalRequest(parseJson(approval.requestJson), node?.label ?? approval.nodeId);
2486
+ approvals.push({
2487
+ runId: approval.runId,
2488
+ nodeId: approval.nodeId,
2489
+ iteration: approval.iteration ?? 0,
2490
+ requestTitle: request.title ?? node?.label ?? approval.nodeId,
2491
+ requestSummary: request.summary,
2492
+ requestedAtMs: approval.requestedAtMs ?? null,
2493
+ approvalMode: request.mode,
2494
+ options: request.options,
2495
+ allowedScopes: request.allowedScopes,
2496
+ allowedUsers: request.allowedUsers,
2497
+ autoApprove: request.autoApprove,
2498
+ });
2499
+ }
2500
+ }
2501
+ }
2502
+ approvals.sort((a, b) => (a.requestedAtMs ?? 0) - (b.requestedAtMs ?? 0));
2503
+ return approvals;
2504
+ }
2505
+ async listCrons() {
2506
+ const rows = [];
2507
+ for (const entry of this.workflows.values()) {
2508
+ const adapter = this.adapterForWorkflow(entry.workflow);
2509
+ const crons = await adapter.listCrons(false);
2510
+ for (const cron of crons) {
2511
+ const workflowKey = workflowKeyFromCronPath(cron.workflowPath) ?? entry.key;
2512
+ rows.push({
2513
+ ...cron,
2514
+ workflow: workflowKey,
2515
+ });
2516
+ }
2517
+ }
2518
+ rows.sort((a, b) => (a.createdAtMs ?? 0) - (b.createdAtMs ?? 0));
2519
+ return rows;
2520
+ }
2521
+ /**
2522
+ * @param {string} cronId
2523
+ */
2524
+ async findCron(cronId) {
2525
+ for (const entry of this.workflows.values()) {
2526
+ const adapter = this.adapterForWorkflow(entry.workflow);
2527
+ const crons = await adapter.listCrons(false);
2528
+ const match = crons.find((cron) => cron.cronId === cronId);
2529
+ if (match) {
2530
+ return {
2531
+ cron: match,
2532
+ workflowKey: workflowKeyFromCronPath(match.workflowPath) ?? entry.key,
2533
+ adapter,
2534
+ };
2535
+ }
2536
+ }
2537
+ return null;
2538
+ }
2539
+ /**
2540
+ * @param {string} runId
2541
+ * @returns {Promise<ResolvedRun | null>}
2542
+ */
2543
+ async resolveRun(runId) {
2544
+ const active = this.runRegistry.get(runId);
2545
+ if (active) {
2546
+ return {
2547
+ workflowKey: active.workflowKey,
2548
+ workflow: active.workflow,
2549
+ adapter: this.adapterForWorkflow(active.workflow),
2550
+ };
2551
+ }
2552
+ for (const entry of this.workflows.values()) {
2553
+ const adapter = this.adapterForWorkflow(entry.workflow);
2554
+ const run = await adapter.getRun(runId);
2555
+ if (run) {
2556
+ return {
2557
+ workflowKey: entry.key,
2558
+ workflow: entry.workflow,
2559
+ adapter,
2560
+ };
2561
+ }
2562
+ }
2563
+ return null;
2564
+ }
2565
+ /**
2566
+ * @param {SmithersEvent} event
2567
+ */
2568
+ handleSmithersEvent(event) {
2569
+ // Invalidate devtools baselines before we broadcast the jump event so
2570
+ // that in-flight streams emit a fresh full snapshot, not a delta rooted
2571
+ // on a stale baseline.
2572
+ if (event.type === "TimeTravelJumped" && typeof event.runId === "string") {
2573
+ this.invalidateDevToolsSubscribersForRun(event.runId);
2574
+ }
2575
+ const mapped = this.mapEvent(event);
2576
+ if (!mapped) {
2577
+ return;
2578
+ }
2579
+ this.broadcastEvent(mapped.event, mapped.payload);
2580
+ }
2581
+ /**
2582
+ * @param {SmithersEvent} event
2583
+ * @returns {{ event: string; payload: unknown } | null}
2584
+ */
2585
+ mapEvent(event) {
2586
+ switch (event.type) {
2587
+ case "NodeStarted":
2588
+ return {
2589
+ event: "node.started",
2590
+ payload: {
2591
+ runId: event.runId,
2592
+ nodeId: event.nodeId,
2593
+ state: "in-progress",
2594
+ },
2595
+ };
2596
+ case "NodeFinished":
2597
+ return {
2598
+ event: "node.finished",
2599
+ payload: {
2600
+ runId: event.runId,
2601
+ nodeId: event.nodeId,
2602
+ state: "finished",
2603
+ },
2604
+ };
2605
+ case "NodeFailed":
2606
+ return {
2607
+ event: "node.failed",
2608
+ payload: {
2609
+ runId: event.runId,
2610
+ nodeId: event.nodeId,
2611
+ state: "failed",
2612
+ error: event.error,
2613
+ },
2614
+ };
2615
+ case "NodeOutput":
2616
+ return {
2617
+ event: "task.output",
2618
+ payload: {
2619
+ runId: event.runId,
2620
+ nodeId: event.nodeId,
2621
+ output: event.text,
2622
+ stream: event.stream,
2623
+ },
2624
+ };
2625
+ case "ApprovalRequested":
2626
+ return {
2627
+ event: "approval.requested",
2628
+ payload: {
2629
+ runId: event.runId,
2630
+ nodeId: event.nodeId,
2631
+ iteration: event.iteration,
2632
+ },
2633
+ };
2634
+ case "ApprovalGranted":
2635
+ return {
2636
+ event: "approval.decided",
2637
+ payload: {
2638
+ runId: event.runId,
2639
+ nodeId: event.nodeId,
2640
+ iteration: event.iteration,
2641
+ approved: true,
2642
+ },
2643
+ };
2644
+ case "ApprovalAutoApproved":
2645
+ return {
2646
+ event: "approval.auto_approved",
2647
+ payload: {
2648
+ runId: event.runId,
2649
+ nodeId: event.nodeId,
2650
+ iteration: event.iteration,
2651
+ },
2652
+ };
2653
+ case "ApprovalDenied":
2654
+ return {
2655
+ event: "approval.decided",
2656
+ payload: {
2657
+ runId: event.runId,
2658
+ nodeId: event.nodeId,
2659
+ iteration: event.iteration,
2660
+ approved: false,
2661
+ },
2662
+ };
2663
+ case "TaskHeartbeat":
2664
+ return {
2665
+ event: "task.heartbeat",
2666
+ payload: {
2667
+ runId: event.runId,
2668
+ nodeId: event.nodeId,
2669
+ iteration: event.iteration,
2670
+ attempt: event.attempt,
2671
+ },
2672
+ };
2673
+ case "TimeTravelJumped":
2674
+ return {
2675
+ event: "run.time_travel_jumped",
2676
+ payload: {
2677
+ runId: event.runId,
2678
+ fromFrameNo: event.fromFrameNo,
2679
+ toFrameNo: event.toFrameNo,
2680
+ timestampMs: event.timestampMs,
2681
+ caller: event.caller ?? null,
2682
+ },
2683
+ };
2684
+ case "RunFinished":
2685
+ return {
2686
+ event: "run.completed",
2687
+ payload: {
2688
+ runId: event.runId,
2689
+ status: "finished",
2690
+ },
2691
+ };
2692
+ case "RunFailed":
2693
+ return {
2694
+ event: "run.completed",
2695
+ payload: {
2696
+ runId: event.runId,
2697
+ status: "failed",
2698
+ error: event.error,
2699
+ },
2700
+ };
2701
+ case "RunCancelled":
2702
+ return {
2703
+ event: "run.completed",
2704
+ payload: {
2705
+ runId: event.runId,
2706
+ status: "cancelled",
2707
+ },
2708
+ };
2709
+ default:
2710
+ return null;
2711
+ }
2712
+ }
2713
+ /**
2714
+ * @param {GatewayRequestContext} connection
2715
+ * @param {RequestFrame} frame
2716
+ * @returns {Promise<ResponseFrame>}
2717
+ */
2718
+ async routeRequest(connection, frame) {
2719
+ const params = asObject(frame.params) ?? {};
2720
+ switch (frame.method) {
2721
+ case "health":
2722
+ return responseOk(frame.id, {
2723
+ protocol: this.protocol,
2724
+ features: this.features,
2725
+ stateVersion: this.stateVersion,
2726
+ uptimeMs: nowMs() - this.startedAtMs,
2727
+ });
2728
+ case "runs.list": {
2729
+ const limit = asOptionalPositiveInt(params.limit, "limit") ?? 50;
2730
+ const status = asString(params.status);
2731
+ return responseOk(frame.id, await this.listRunsAcrossWorkflows(limit, status));
2732
+ }
2733
+ case "runs.create": {
2734
+ const workflowKey = asString(params.workflow);
2735
+ if (!workflowKey) {
2736
+ return responseError(frame.id, "INVALID_REQUEST", "workflow is required");
2737
+ }
2738
+ if (!this.workflows.has(workflowKey)) {
2739
+ return responseError(frame.id, "NOT_FOUND", `Unknown workflow: ${workflowKey}`);
2740
+ }
2741
+ let input;
2742
+ try {
2743
+ input = validateGatewayRpcInput(params.input);
2744
+ }
2745
+ catch (error) {
2746
+ if (isSmithersError(error)) {
2747
+ return responseError(frame.id, error.code, error.summary);
2748
+ }
2749
+ throw error;
2750
+ }
2751
+ return responseOk(frame.id, await this.startRun(workflowKey, input, {
2752
+ triggeredBy: connection.userId ?? "gateway",
2753
+ scopes: [...connection.scopes],
2754
+ role: connection.role ?? "operator",
2755
+ subscribeConnection: connection,
2756
+ }, asString(params.runId) ?? crypto.randomUUID(), { resume: false }));
2757
+ }
2758
+ case "runs.get": {
2759
+ const runId = asString(params.runId);
2760
+ if (!runId) {
2761
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
2762
+ }
2763
+ const resolved = await this.resolveRun(runId);
2764
+ if (!resolved) {
2765
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2766
+ }
2767
+ const run = await resolved.adapter.getRun(runId);
2768
+ if (!run) {
2769
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2770
+ }
2771
+ const summary = await resolved.adapter.countNodesByState(runId);
2772
+ const runState = await computeRunStateFromRow(
2773
+ resolved.adapter,
2774
+ run,
2775
+ ).catch(() => undefined);
2776
+ return responseOk(frame.id, {
2777
+ ...run,
2778
+ workflowKey: resolved.workflowKey,
2779
+ summary: summary.reduce((acc, row) => {
2780
+ acc[row.state] = row.count;
2781
+ return acc;
2782
+ }, {}),
2783
+ ...(runState ? { runState } : {}),
2784
+ });
2785
+ }
2786
+ case "frames.list": {
2787
+ const runId = asString(params.runId);
2788
+ if (!runId) {
2789
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
2790
+ }
2791
+ const resolved = await this.resolveRun(runId);
2792
+ if (!resolved) {
2793
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2794
+ }
2795
+ const limit = asOptionalPositiveInt(params.limit, "limit") ?? 50;
2796
+ const afterFrameNo = asOptionalPositiveInt(params.afterFrameNo, "afterFrameNo");
2797
+ return responseOk(frame.id, await resolved.adapter.listFrames(runId, limit, afterFrameNo));
2798
+ }
2799
+ case "frames.get": {
2800
+ const runId = asString(params.runId);
2801
+ if (!runId) {
2802
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
2803
+ }
2804
+ const resolved = await this.resolveRun(runId);
2805
+ if (!resolved) {
2806
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2807
+ }
2808
+ const frameNo = asOptionalPositiveInt(params.frameNo, "frameNo");
2809
+ const frameRow = frameNo === undefined
2810
+ ? await resolved.adapter.getLastFrame(runId)
2811
+ : (await resolved.adapter.listFrames(runId, Math.max(frameNo + 1, 50))).find((entry) => entry.frameNo === frameNo);
2812
+ if (!frameRow) {
2813
+ return responseError(frame.id, "NOT_FOUND", "Frame not found");
2814
+ }
2815
+ return responseOk(frame.id, frameRow);
2816
+ }
2817
+ case "attempts.list": {
2818
+ const runId = asString(params.runId);
2819
+ if (!runId) {
2820
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
2821
+ }
2822
+ const resolved = await this.resolveRun(runId);
2823
+ if (!resolved) {
2824
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2825
+ }
2826
+ const nodeId = asString(params.nodeId);
2827
+ if (nodeId) {
2828
+ const iteration = asNumber(params.iteration) ?? 0;
2829
+ return responseOk(frame.id, await resolved.adapter.listAttempts(runId, nodeId, iteration));
2830
+ }
2831
+ return responseOk(frame.id, await resolved.adapter.listAttemptsForRun(runId));
2832
+ }
2833
+ case "attempts.get": {
2834
+ const runId = asString(params.runId);
2835
+ const nodeId = asString(params.nodeId);
2836
+ const iteration = asNumber(params.iteration);
2837
+ const attempt = asNumber(params.attempt);
2838
+ if (!runId || !nodeId || iteration === undefined || attempt === undefined) {
2839
+ return responseError(frame.id, "INVALID_REQUEST", "runId, nodeId, iteration, and attempt are required");
2840
+ }
2841
+ const resolved = await this.resolveRun(runId);
2842
+ if (!resolved) {
2843
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
2844
+ }
2845
+ const row = await resolved.adapter.getAttempt(runId, nodeId, iteration, attempt);
2846
+ if (!row) {
2847
+ return responseError(frame.id, "NOT_FOUND", "Attempt not found");
2848
+ }
2849
+ return responseOk(frame.id, row);
2850
+ }
2851
+ case "getNodeOutput":
2852
+ case "devtools.getNodeOutput": {
2853
+ try {
2854
+ const payload = await getNodeOutputRoute({
2855
+ runId: params.runId,
2856
+ nodeId: params.nodeId,
2857
+ iteration: params.iteration,
2858
+ resolveRun: this.resolveRun.bind(this),
2859
+ });
2860
+ return responseOk(frame.id, payload);
2861
+ }
2862
+ catch (error) {
2863
+ if (error instanceof NodeOutputRouteError) {
2864
+ return responseError(frame.id, error.code, error.message);
2865
+ }
2866
+ throw error;
2867
+ }
2868
+ }
2869
+ case "getNodeDiff":
2870
+ case "devtools.getNodeDiff": {
2871
+ const result = await runPromise(Effect.promise(() => getNodeDiffRoute({
2872
+ runId: params.runId,
2873
+ nodeId: params.nodeId,
2874
+ iteration: params.iteration,
2875
+ resolveRun: this.resolveRun.bind(this),
2876
+ })).pipe(Effect.withLogSpan("devtools.getNodeDiff")));
2877
+ if (!result.ok) {
2878
+ return responseError(frame.id, result.error.code, result.error.message);
2879
+ }
2880
+ return responseOk(frame.id, result.payload);
2881
+ }
2882
+ case "getDevToolsSnapshot": {
2883
+ const runId = asString(params.runId);
2884
+ if (!runId) {
2885
+ return responseError(frame.id, "InvalidRunId", "runId is required");
2886
+ }
2887
+ try {
2888
+ // Full route-level validation runs at the gateway boundary
2889
+ // before any DB lookup. Malformed inputs never reach
2890
+ // resolveRun() or the adapter.
2891
+ validateRunId(runId);
2892
+ validateFrameNoInput(params.frameNo);
2893
+ if (!this.isDevToolsRunAuthorized(connection, runId)) {
2894
+ return responseError(frame.id, "Unauthorized", "Connection is not subscribed to this runId.");
2895
+ }
2896
+ const resolved = await this.resolveRun(runId);
2897
+ if (!resolved) {
2898
+ return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
2899
+ }
2900
+ const payload = await getDevToolsSnapshotRoute({
2901
+ adapter: resolved.adapter,
2902
+ runId,
2903
+ frameNo: params.frameNo,
2904
+ onWarning: (warning) => {
2905
+ emitGatewayLog("warning", "devtools snapshot serializer warning", {
2906
+ runId,
2907
+ code: warning.code,
2908
+ path: warning.path,
2909
+ }, "gateway:devtools");
2910
+ },
2911
+ });
2912
+ return responseOk(frame.id, payload);
2913
+ }
2914
+ catch (error) {
2915
+ if (error instanceof DevToolsRouteError) {
2916
+ return responseError(frame.id, error.code, error.message);
2917
+ }
2918
+ throw error;
2919
+ }
2920
+ }
2921
+ case "streamDevTools": {
2922
+ if (connection.transport !== "ws" || !connection.ws) {
2923
+ this.recordDevToolsSubscribeAttempt("error");
2924
+ return responseError(frame.id, "INVALID_REQUEST", "streamDevTools is only supported over websocket connections");
2925
+ }
2926
+ const runId = asString(params.runId);
2927
+ if (!runId) {
2928
+ this.recordDevToolsSubscribeAttempt("error");
2929
+ return responseError(frame.id, "InvalidRunId", "runId is required");
2930
+ }
2931
+ const fromSeq = params.fromSeq;
2932
+ const streamId = randomUUID();
2933
+ try {
2934
+ // Full route-level validation at the gateway boundary so
2935
+ // malformed numeric inputs never reach resolveRun() or
2936
+ // getLastFrame() below.
2937
+ validateRunId(runId);
2938
+ validateFromSeqInput(fromSeq);
2939
+ if (!this.isDevToolsRunAuthorized(connection, runId)) {
2940
+ this.recordDevToolsSubscribeAttempt("error");
2941
+ return responseError(frame.id, "Unauthorized", "Connection is not subscribed to this runId.");
2942
+ }
2943
+ const resolved = await this.resolveRun(runId);
2944
+ if (!resolved) {
2945
+ this.recordDevToolsSubscribeAttempt("error");
2946
+ return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
2947
+ }
2948
+ if (typeof fromSeq === "number") {
2949
+ const latestFrame = await resolved.adapter.getLastFrame(runId);
2950
+ // Zero-frame runs: current seq is 0. fromSeq > 0 is in
2951
+ // the future relative to current seq and must reject.
2952
+ const latestSeq = latestFrame?.frameNo ?? 0;
2953
+ if (fromSeq > latestSeq) {
2954
+ this.recordDevToolsSubscribeAttempt("error");
2955
+ return responseError(frame.id, "SeqOutOfRange", `fromSeq ${fromSeq} is newer than current seq ${latestSeq}`);
2956
+ }
2957
+ }
2958
+ const abort = this.registerDevToolsSubscriber(connection, streamId, runId);
2959
+ emitGatewayLog("info", "devtools stream subscribed", {
2960
+ runId,
2961
+ fromSeq: typeof fromSeq === "number" ? fromSeq : null,
2962
+ streamId,
2963
+ subscriberId: connection.connectionId,
2964
+ }, "gateway:devtools");
2965
+ // Per-subscriber outbound queue gated on actual WS send
2966
+ // pressure. If the WS socket has buffered > limit bytes,
2967
+ // we queue locally up to 1000 events; exceeding that is a
2968
+ // BackpressureDisconnect that tears down only this stream.
2969
+ const outboundQueue = [];
2970
+ const OUTBOUND_QUEUE_LIMIT = 1_000;
2971
+ const WS_BUFFERED_HIGH_WATER_BYTES = 8 * 1024 * 1024;
2972
+ let flushPending = false;
2973
+ const drainOutboundQueue = () => {
2974
+ if (flushPending) return;
2975
+ flushPending = true;
2976
+ queueMicrotask(() => {
2977
+ try {
2978
+ while (outboundQueue.length > 0 && connection.ws.readyState === connection.ws.OPEN) {
2979
+ const ws = connection.ws;
2980
+ if (typeof ws.bufferedAmount === "number" && ws.bufferedAmount > WS_BUFFERED_HIGH_WATER_BYTES) {
2981
+ setTimeout(() => {
2982
+ flushPending = false;
2983
+ drainOutboundQueue();
2984
+ }, 10);
2985
+ return;
2986
+ }
2987
+ const payload = outboundQueue.shift();
2988
+ if (!payload) continue;
2989
+ this.sendEvent(connection, "devtools.event", payload);
2990
+ }
2991
+ }
2992
+ finally {
2993
+ flushPending = false;
2994
+ }
2995
+ });
2996
+ };
2997
+ /**
2998
+ * Send a devtools event through the outbound queue. Applies
2999
+ * WS-level backpressure detection and raises a typed
3000
+ * BackpressureDisconnect if the queue overflows.
3001
+ */
3002
+ const enqueueDevToolsEvent = (payload) => {
3003
+ if (outboundQueue.length >= OUTBOUND_QUEUE_LIMIT) {
3004
+ throw new DevToolsRouteError(
3005
+ "BackpressureDisconnect",
3006
+ `Subscriber outbound queue exceeded ${OUTBOUND_QUEUE_LIMIT} events.`,
3007
+ );
3008
+ }
3009
+ outboundQueue.push(payload);
3010
+ drainOutboundQueue();
3011
+ };
3012
+ void (async () => {
3013
+ let eventsDelivered = 0;
3014
+ try {
3015
+ for await (const event of streamDevToolsRoute({
3016
+ adapter: resolved.adapter,
3017
+ runId,
3018
+ fromSeq: typeof fromSeq === "number" ? fromSeq : undefined,
3019
+ subscriberId: connection.connectionId,
3020
+ signal: abort.signal,
3021
+ invalidateSnapshot: () => {
3022
+ if (this.devtoolsInvalidateFlags.has(streamId)) {
3023
+ this.devtoolsInvalidateFlags.delete(streamId);
3024
+ return true;
3025
+ }
3026
+ return false;
3027
+ },
3028
+ onWarning: (warning) => {
3029
+ emitGatewayLog("warning", "devtools snapshot serializer warning", {
3030
+ runId,
3031
+ code: warning.code,
3032
+ path: warning.path,
3033
+ }, "gateway:devtools");
3034
+ },
3035
+ onLog: (level, message, fields) => {
3036
+ emitGatewayLog(level === "warn" ? "warning" : level, message, fields, "gateway:devtools");
3037
+ },
3038
+ onEvent: (event, stats) => {
3039
+ const kind = event.kind;
3040
+ emitGatewayEffect(Effect.all([
3041
+ Metric.increment(taggedMetric(devtoolsEventTotal, { kind })),
3042
+ Metric.update(devtoolsEventBytes, stats.bytes),
3043
+ ...(kind === "snapshot"
3044
+ ? [Metric.update(devtoolsSnapshotBuildMs, stats.durationMs)]
3045
+ : [Metric.update(devtoolsDeltaBuildMs, stats.durationMs)]),
3046
+ ], { discard: true }));
3047
+ },
3048
+ onClose: ({ errorCode }) => {
3049
+ if (errorCode === "BackpressureDisconnect") {
3050
+ emitGatewayEffect(Metric.increment(devtoolsBackpressureDisconnectTotal));
3051
+ }
3052
+ },
3053
+ })) {
3054
+ eventsDelivered += 1;
3055
+ enqueueDevToolsEvent({
3056
+ streamId,
3057
+ runId,
3058
+ event,
3059
+ });
3060
+ }
3061
+ }
3062
+ catch (error) {
3063
+ const code = error?.code ?? "SERVER_ERROR";
3064
+ if (code === "BackpressureDisconnect") {
3065
+ emitGatewayEffect(Metric.increment(devtoolsBackpressureDisconnectTotal));
3066
+ }
3067
+ emitGatewayLog("error", "devtools stream failed", {
3068
+ runId,
3069
+ streamId,
3070
+ code,
3071
+ message: error?.message ?? "stream failed",
3072
+ }, "gateway:devtools");
3073
+ // Notify ONLY the offending subscriber. The WS
3074
+ // stays open so other subscribers on the same
3075
+ // connection continue to receive events.
3076
+ this.sendEvent(connection, "devtools.error", {
3077
+ streamId,
3078
+ runId,
3079
+ error: {
3080
+ code,
3081
+ message: error?.message ?? "stream failed",
3082
+ },
3083
+ });
3084
+ }
3085
+ finally {
3086
+ this.unregisterDevToolsSubscriber(connection, streamId, {
3087
+ runId,
3088
+ streamId,
3089
+ eventsDelivered,
3090
+ });
3091
+ }
3092
+ })();
3093
+ return responseOk(frame.id, {
3094
+ streamId,
3095
+ runId,
3096
+ fromSeq: typeof fromSeq === "number" ? fromSeq : null,
3097
+ });
3098
+ }
3099
+ catch (error) {
3100
+ this.recordDevToolsSubscribeAttempt("error");
3101
+ if (error instanceof DevToolsRouteError) {
3102
+ return responseError(frame.id, error.code, error.message);
3103
+ }
3104
+ throw error;
3105
+ }
3106
+ }
3107
+ case "jumpToFrame":
3108
+ case "devtools.jumpToFrame": {
3109
+ const runId = asString(params.runId);
3110
+ if (!runId) {
3111
+ return responseError(frame.id, "InvalidRunId", "runId is required");
3112
+ }
3113
+ const frameNo = asNumber(params.frameNo);
3114
+ if (frameNo === undefined) {
3115
+ return responseError(frame.id, "InvalidFrameNo", "frameNo is required");
3116
+ }
3117
+ const confirm = asBoolean(params.confirm);
3118
+ const resolved = await this.resolveRun(runId);
3119
+ if (!resolved) {
3120
+ return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
3121
+ }
3122
+ const run = await resolved.adapter.getRun(runId);
3123
+ if (!run) {
3124
+ return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
3125
+ }
3126
+ const ownerId = resolveRunOwnerId(run);
3127
+ const isAdmin = (connection.role ?? "").toLowerCase() === "admin";
3128
+ const isOwner = Boolean(ownerId && connection.userId && ownerId === connection.userId);
3129
+ if (!isAdmin && !isOwner) {
3130
+ // Record the unauthorized attempt so the audit log contains
3131
+ // every rewind request — successful or not.
3132
+ try {
3133
+ await writeRewindAuditRow(resolved.adapter, {
3134
+ runId,
3135
+ fromFrameNo: -1,
3136
+ toFrameNo: Number.isInteger(frameNo) ? frameNo : -1,
3137
+ caller: connection.userId ?? "gateway",
3138
+ timestampMs: nowMs(),
3139
+ result: "failed",
3140
+ durationMs: 0,
3141
+ });
3142
+ }
3143
+ catch (auditError) {
3144
+ emitGatewayLog("warning", "Gateway jumpToFrame unauthorized audit-write failed", {
3145
+ runId,
3146
+ ...gatewayErrorAnnotations(auditError),
3147
+ }, "gateway:jump-to-frame");
3148
+ }
3149
+ return responseError(frame.id, "Unauthorized", "Only the run owner or an admin may rewind this run.");
3150
+ }
3151
+ const active = this.activeRuns.get(runId);
3152
+ try {
3153
+ const payload = await jumpToFrameRoute({
3154
+ adapter: resolved.adapter,
3155
+ runId,
3156
+ frameNo,
3157
+ confirm,
3158
+ caller: connection.userId ?? "gateway",
3159
+ pauseRunLoop: async () => {
3160
+ if (!active) {
3161
+ return;
3162
+ }
3163
+ active.abort.abort();
3164
+ const timeoutAt = Date.now() + 10_000;
3165
+ while (this.activeRuns.has(runId) && Date.now() < timeoutAt) {
3166
+ await delay(25);
3167
+ }
3168
+ // Hard stop: if the task is still live after the grace
3169
+ // window, abort the rewind rather than mutating the DB
3170
+ // underneath a running task.
3171
+ if (this.activeRuns.has(runId)) {
3172
+ throw new JumpToFrameError("RewindFailed", `Run ${runId} did not stop within 10s of abort; refusing to rewind a live task.`, {
3173
+ details: { runId, stage: "pause" },
3174
+ });
3175
+ }
3176
+ },
3177
+ resumeRunLoop: async () => {
3178
+ await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
3179
+ triggeredBy: connection.userId ?? "gateway",
3180
+ scopes: [...connection.scopes],
3181
+ role: connection.role ?? "operator",
3182
+ subscribeConnection: connection.transport === "ws" ? connection : undefined,
3183
+ });
3184
+ },
3185
+ emitEvent: async (event) => {
3186
+ this.handleSmithersEvent(event);
3187
+ },
3188
+ onLog: async (level, message, fields) => {
3189
+ emitGatewayLog(level === "warn" ? "warning" : level, message, {
3190
+ runId,
3191
+ frameNo,
3192
+ caller: connection.userId ?? "gateway",
3193
+ ...fields,
3194
+ }, "gateway:jump-to-frame");
3195
+ },
3196
+ });
3197
+ return responseOk(frame.id, payload);
3198
+ }
3199
+ catch (error) {
3200
+ if (error instanceof JumpToFrameError) {
3201
+ return responseError(frame.id, error.code, error.message);
3202
+ }
3203
+ throw error;
3204
+ }
3205
+ }
3206
+ case "runs.diff": {
3207
+ const leftRunId = asString(params.leftRunId);
3208
+ const rightRunId = asString(params.rightRunId);
3209
+ if (!leftRunId || !rightRunId) {
3210
+ return responseError(frame.id, "INVALID_REQUEST", "leftRunId and rightRunId are required");
3211
+ }
3212
+ const left = await this.resolveRun(leftRunId);
3213
+ const right = await this.resolveRun(rightRunId);
3214
+ if (!left || !right) {
3215
+ return responseError(frame.id, "NOT_FOUND", "Both runs must exist");
3216
+ }
3217
+ const leftSnapshot = await loadLatestSnapshot(left.adapter, leftRunId);
3218
+ const rightSnapshot = await loadLatestSnapshot(right.adapter, rightRunId);
3219
+ if (!leftSnapshot || !rightSnapshot) {
3220
+ return responseError(frame.id, "NOT_FOUND", "Snapshots not found for both runs");
3221
+ }
3222
+ return responseOk(frame.id, diffRawSnapshots(leftSnapshot, rightSnapshot));
3223
+ }
3224
+ case "approvals.list":
3225
+ return responseOk(frame.id, await this.listPendingApprovals());
3226
+ case "approvals.decide": {
3227
+ const runId = asString(params.runId);
3228
+ const nodeId = asString(params.nodeId);
3229
+ const approved = asBoolean(params.approved);
3230
+ const iteration = asNumber(params.iteration) ?? 0;
3231
+ if (!runId || !nodeId || approved === undefined) {
3232
+ return responseError(frame.id, "INVALID_REQUEST", "runId, nodeId, and approved are required");
3233
+ }
3234
+ const resolved = await this.resolveRun(runId);
3235
+ if (!resolved) {
3236
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
3237
+ }
3238
+ const approval = await resolved.adapter.getApproval(runId, nodeId, iteration);
3239
+ const request = parseApprovalRequest(parseJson(typeof approval?.requestJson === "string" ? approval.requestJson : null), nodeId);
3240
+ if (request.allowedUsers.length > 0 &&
3241
+ (!connection.userId || !request.allowedUsers.includes(connection.userId))) {
3242
+ return responseError(frame.id, "FORBIDDEN", "User is not allowed to decide this approval");
3243
+ }
3244
+ if (request.allowedScopes.length > 0 &&
3245
+ !request.allowedScopes.some((scope) => hasScope(connection.scopes, scope))) {
3246
+ return responseError(frame.id, "FORBIDDEN", "Connection is missing required approval scope");
3247
+ }
3248
+ const decision = params.decision;
3249
+ if (approved) {
3250
+ const validation = validateApprovalDecision(request, decision);
3251
+ if (!validation.ok) {
3252
+ return responseError(frame.id, validation.code, validation.message);
3253
+ }
3254
+ }
3255
+ if (approved) {
3256
+ await Effect.runPromise(approveNode(resolved.adapter, runId, nodeId, iteration, asString(params.note), connection.userId ?? undefined, decision));
3257
+ }
3258
+ else {
3259
+ await Effect.runPromise(denyNode(resolved.adapter, runId, nodeId, iteration, asString(params.note), connection.userId ?? undefined, decision));
3260
+ }
3261
+ await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
3262
+ triggeredBy: connection.userId ?? "gateway",
3263
+ scopes: [...connection.scopes],
3264
+ role: connection.role ?? "operator",
3265
+ subscribeConnection: connection,
3266
+ });
3267
+ return responseOk(frame.id, { runId, nodeId, iteration, approved });
3268
+ }
3269
+ case "signals.send": {
3270
+ const runId = asString(params.runId);
3271
+ const signalName = asString(params.signalName);
3272
+ if (!runId || !signalName) {
3273
+ return responseError(frame.id, "INVALID_REQUEST", "runId and signalName are required");
3274
+ }
3275
+ const resolved = await this.resolveRun(runId);
3276
+ if (!resolved) {
3277
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
3278
+ }
3279
+ const delivered = await Effect.runPromise(signalRun(resolved.adapter, runId, signalName, params.data ?? {}, {
3280
+ correlationId: asString(params.correlationId),
3281
+ receivedBy: connection.userId,
3282
+ }));
3283
+ await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
3284
+ triggeredBy: connection.userId ?? "gateway",
3285
+ scopes: [...connection.scopes],
3286
+ role: connection.role ?? "operator",
3287
+ subscribeConnection: connection,
3288
+ });
3289
+ return responseOk(frame.id, delivered);
3290
+ }
3291
+ case "runs.cancel": {
3292
+ const runId = asString(params.runId);
3293
+ if (!runId) {
3294
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
3295
+ }
3296
+ const active = this.activeRuns.get(runId);
3297
+ if (!active) {
3298
+ return responseError(frame.id, "RUN_NOT_ACTIVE", "Run is not currently active");
3299
+ }
3300
+ active.abort.abort();
3301
+ return responseOk(frame.id, { runId, status: "cancelling" });
3302
+ }
3303
+ case "runs.rerun": {
3304
+ const runId = asString(params.runId);
3305
+ if (!runId) {
3306
+ return responseError(frame.id, "INVALID_REQUEST", "runId is required");
3307
+ }
3308
+ const resolved = await this.resolveRun(runId);
3309
+ if (!resolved) {
3310
+ return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
3311
+ }
3312
+ const client = (resolved.workflow.db.session?.client ?? resolved.workflow.db.$client);
3313
+ const row = client?.query?.("SELECT payload FROM input WHERE run_id = ? LIMIT 1").get(runId);
3314
+ const input = typeof row?.payload === "string"
3315
+ ? parseJson(row.payload) ?? {}
3316
+ : row?.payload ?? {};
3317
+ return this.routeRequest(connection, {
3318
+ type: "req",
3319
+ id: frame.id,
3320
+ method: "runs.create",
3321
+ params: {
3322
+ workflow: resolved.workflowKey,
3323
+ input,
3324
+ runId: asString(params.newRunId),
3325
+ },
3326
+ });
3327
+ }
3328
+ case "cron.list":
3329
+ return responseOk(frame.id, await this.listCrons());
3330
+ case "cron.add": {
3331
+ const workflowKey = asString(params.workflow);
3332
+ const pattern = asString(params.pattern);
3333
+ if (!workflowKey || !pattern) {
3334
+ return responseError(frame.id, "INVALID_REQUEST", "workflow and pattern are required");
3335
+ }
3336
+ const entry = this.workflows.get(workflowKey);
3337
+ if (!entry) {
3338
+ return responseError(frame.id, "NOT_FOUND", `Unknown workflow: ${workflowKey}`);
3339
+ }
3340
+ const cronId = asString(params.cronId) ?? randomUUID();
3341
+ const adapter = this.adapterForWorkflow(entry.workflow);
3342
+ const row = {
3343
+ cronId,
3344
+ pattern,
3345
+ workflowPath: cronWorkflowPath(workflowKey),
3346
+ enabled: asBoolean(params.enabled) ?? true,
3347
+ createdAtMs: nowMs(),
3348
+ lastRunAtMs: null,
3349
+ nextRunAtMs: nextCronRunAtMs(pattern),
3350
+ errorJson: null,
3351
+ };
3352
+ await adapter.upsertCron(row);
3353
+ return responseOk(frame.id, {
3354
+ ...row,
3355
+ workflow: workflowKey,
3356
+ });
3357
+ }
3358
+ case "cron.remove": {
3359
+ const cronId = asString(params.cronId);
3360
+ if (!cronId) {
3361
+ return responseError(frame.id, "INVALID_REQUEST", "cronId is required");
3362
+ }
3363
+ const resolvedCron = await this.findCron(cronId);
3364
+ if (!resolvedCron) {
3365
+ return responseError(frame.id, "NOT_FOUND", `Cron not found: ${cronId}`);
3366
+ }
3367
+ await resolvedCron.adapter.deleteCron(cronId);
3368
+ return responseOk(frame.id, { cronId, removed: true });
3369
+ }
3370
+ case "cron.trigger": {
3371
+ const cronId = asString(params.cronId);
3372
+ const workflowKey = asString(params.workflow);
3373
+ const resolvedCron = cronId ? await this.findCron(cronId) : null;
3374
+ const targetWorkflowKey = resolvedCron?.workflowKey ?? workflowKey;
3375
+ if (!targetWorkflowKey) {
3376
+ return responseError(frame.id, "INVALID_REQUEST", "cronId or workflow is required");
3377
+ }
3378
+ if (resolvedCron) {
3379
+ await resolvedCron.adapter.updateCronRunTime(resolvedCron.cron.cronId, nowMs(), nextCronRunAtMs(resolvedCron.cron.pattern), null);
3380
+ }
3381
+ let input;
3382
+ try {
3383
+ input = validateGatewayRpcInput(params.input);
3384
+ }
3385
+ catch (error) {
3386
+ if (isSmithersError(error)) {
3387
+ return responseError(frame.id, error.code, error.summary);
3388
+ }
3389
+ throw error;
3390
+ }
3391
+ return responseOk(frame.id, await this.startRun(targetWorkflowKey, input, {
3392
+ triggeredBy: connection.userId ?? "gateway",
3393
+ scopes: [...connection.scopes],
3394
+ role: connection.role ?? "operator",
3395
+ subscribeConnection: connection,
3396
+ }, undefined, { resume: false }));
3397
+ }
3398
+ default:
3399
+ return responseError(frame.id, "METHOD_NOT_FOUND", `Unknown method: ${frame.method}`);
3400
+ }
3401
+ }
3402
+ }