@smithers-orchestrator/server 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/package.json +46 -0
- package/src/ConnectRequest.ts +17 -0
- package/src/EventFrame.ts +7 -0
- package/src/GatewayAuthConfig.ts +26 -0
- package/src/GatewayDefaults.ts +3 -0
- package/src/GatewayOptions.ts +13 -0
- package/src/GatewayTokenGrant.ts +5 -0
- package/src/GatewayWebhookConfig.ts +10 -0
- package/src/GatewayWebhookRunConfig.ts +4 -0
- package/src/GatewayWebhookSignalConfig.ts +6 -0
- package/src/HelloResponse.ts +18 -0
- package/src/RequestFrame.ts +6 -0
- package/src/ResponseFrame.ts +10 -0
- package/src/ServeOptions.ts +11 -0
- package/src/ServerOptions.ts +8 -0
- package/src/gateway.js +3402 -0
- package/src/gatewayRoutes/DiffSummary.ts +6 -0
- package/src/gatewayRoutes/GetNodeDiffRouteResult.ts +23 -0
- package/src/gatewayRoutes/NODE_OUTPUT_MAX_BYTES.js +1 -0
- package/src/gatewayRoutes/NODE_OUTPUT_WARN_BYTES.js +1 -0
- package/src/gatewayRoutes/NodeOutputResponse.ts +22 -0
- package/src/gatewayRoutes/NodeOutputRouteError.js +14 -0
- package/src/gatewayRoutes/getDevToolsSnapshot.js +428 -0
- package/src/gatewayRoutes/getNodeDiff.js +609 -0
- package/src/gatewayRoutes/getNodeOutput.js +504 -0
- package/src/gatewayRoutes/jumpToFrame.js +84 -0
- package/src/gatewayRoutes/streamDevTools.js +525 -0
- package/src/index.d.ts +953 -0
- package/src/index.js +1240 -0
- package/src/serve.js +315 -0
- package/src/smithersRuntime.js +63 -0
package/src/gateway.js
ADDED
|
@@ -0,0 +1,3402 @@
|
|
|
1
|
+
// @smithers-type-exports-begin
|
|
2
|
+
/** @typedef {import("./EventFrame.js").EventFrame} EventFrame */
|
|
3
|
+
/** @typedef {import("./GatewayDefaults.js").GatewayDefaults} GatewayDefaults */
|
|
4
|
+
/** @typedef {import("./GatewayTokenGrant.js").GatewayTokenGrant} GatewayTokenGrant */
|
|
5
|
+
/** @typedef {import("./HelloResponse.js").HelloResponse} HelloResponse */
|
|
6
|
+
// @smithers-type-exports-end
|
|
7
|
+
|
|
8
|
+
import { createServer } from "node:http";
|
|
9
|
+
import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto";
|
|
10
|
+
import { CronExpressionParser } from "cron-parser";
|
|
11
|
+
import { Effect, Metric } from "effect";
|
|
12
|
+
import { WebSocketServer } from "ws";
|
|
13
|
+
import { runWorkflow } from "@smithers-orchestrator/engine";
|
|
14
|
+
import { approveNode, denyNode } from "@smithers-orchestrator/engine/approvals";
|
|
15
|
+
import { signalRun } from "@smithers-orchestrator/engine/signals";
|
|
16
|
+
import { SmithersDb } from "@smithers-orchestrator/db/adapter";
|
|
17
|
+
import { computeRunStateFromRow } from "@smithers-orchestrator/db/runState";
|
|
18
|
+
import { ensureSmithersTables } from "@smithers-orchestrator/db/ensure";
|
|
19
|
+
import { devtoolsActiveSubscribers, devtoolsBackpressureDisconnectTotal, devtoolsDeltaBuildMs, devtoolsEventBytes, devtoolsEventTotal, devtoolsSnapshotBuildMs, devtoolsSubscribeTotal, gatewayApprovalDecisionsTotal, gatewayAuthEventsTotal, gatewayConnectionsActive, gatewayConnectionsClosedTotal, gatewayConnectionsTotal, gatewayCronTriggersTotal, gatewayErrorsTotal, gatewayHeartbeatTicksTotal, gatewayMessagesReceivedTotal, gatewayMessagesSentTotal, gatewayRpcCallsTotal, gatewayRpcDuration, gatewayRunsCompletedTotal, gatewayRunsStartedTotal, gatewaySignalsTotal, gatewayWebhooksReceivedTotal, gatewayWebhooksRejectedTotal, gatewayWebhooksVerifiedTotal, } from "@smithers-orchestrator/observability/metrics";
|
|
20
|
+
import { runFork, runPromise } from "./smithersRuntime.js";
|
|
21
|
+
import { prometheusContentType, renderPrometheusMetrics } from "@smithers-orchestrator/observability";
|
|
22
|
+
import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
|
|
23
|
+
import { errorToJson } from "@smithers-orchestrator/errors/errorToJson";
|
|
24
|
+
import { isSmithersError } from "@smithers-orchestrator/errors/isSmithersError";
|
|
25
|
+
import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
|
|
26
|
+
import { assertJsonPayloadWithinBounds, assertOptionalStringMaxLength, assertPositiveFiniteInteger, } from "@smithers-orchestrator/db/input-bounds";
|
|
27
|
+
import { loadLatestSnapshot, loadSnapshot } from "@smithers-orchestrator/time-travel/snapshot";
|
|
28
|
+
import { diffRawSnapshots } from "@smithers-orchestrator/time-travel/diff";
|
|
29
|
+
import { getNodeOutputRoute } from "./gatewayRoutes/getNodeOutput.js";
|
|
30
|
+
import { NodeOutputRouteError } from "./gatewayRoutes/NodeOutputRouteError.js";
|
|
31
|
+
import { getNodeDiffRoute } from "./gatewayRoutes/getNodeDiff.js";
|
|
32
|
+
import { DevToolsRouteError, getDevToolsSnapshotRoute, validateFrameNoInput, validateFromSeqInput, validateRunId } from "./gatewayRoutes/getDevToolsSnapshot.js";
|
|
33
|
+
import { streamDevToolsRoute } from "./gatewayRoutes/streamDevTools.js";
|
|
34
|
+
import { jumpToFrameRoute, JumpToFrameError } from "./gatewayRoutes/jumpToFrame.js";
|
|
35
|
+
import { writeRewindAuditRow } from "@smithers-orchestrator/time-travel/writeRewindAuditRow";
|
|
36
|
+
import { recoverInProgressRewindAudits } from "@smithers-orchestrator/time-travel/recoverInProgressRewindAudits";
|
|
37
|
+
/** @typedef {import("./GatewayWebhookRunConfig.js").GatewayWebhookRunConfig} GatewayWebhookRunConfig */
|
|
38
|
+
/** @typedef {import("./GatewayWebhookSignalConfig.js").GatewayWebhookSignalConfig} GatewayWebhookSignalConfig */
|
|
39
|
+
/** @typedef {import("./ConnectRequest.js").ConnectRequest} ConnectRequest */
|
|
40
|
+
/** @typedef {import("./GatewayAuthConfig.js").GatewayAuthConfig} GatewayAuthConfig */
|
|
41
|
+
/** @typedef {import("./GatewayOptions.js").GatewayOptions} GatewayOptions */
|
|
42
|
+
/** @typedef {import("./GatewayWebhookConfig.js").GatewayWebhookConfig} GatewayWebhookConfig */
|
|
43
|
+
/** @typedef {import("node:http").IncomingMessage} IncomingMessage */
|
|
44
|
+
/** @typedef {import("./RequestFrame.js").RequestFrame} RequestFrame */
|
|
45
|
+
/** @typedef {import("./ResponseFrame.js").ResponseFrame} ResponseFrame */
|
|
46
|
+
/** @typedef {import("node:http").ServerResponse} ServerResponse */
|
|
47
|
+
/** @typedef {import("@smithers-orchestrator/components/SmithersWorkflow").SmithersWorkflow<unknown>} SmithersWorkflow */
|
|
48
|
+
/** @typedef {import("@smithers-orchestrator/observability/SmithersEvent").SmithersEvent} SmithersEvent */
|
|
49
|
+
/** @typedef {Record<string, string | number | null | undefined>} GatewayMetricLabels */
|
|
50
|
+
/** @typedef {"ws" | "http"} GatewayTransport */
|
|
51
|
+
/**
|
|
52
|
+
* @typedef {{
|
|
53
|
+
* connectionId?: string;
|
|
54
|
+
* role?: string;
|
|
55
|
+
* scopes?: string[];
|
|
56
|
+
* userId?: string | null;
|
|
57
|
+
* origin?: string;
|
|
58
|
+
* transport?: GatewayTransport;
|
|
59
|
+
* }} GatewayRequestContext
|
|
60
|
+
*/
|
|
61
|
+
/**
|
|
62
|
+
* @typedef {{
|
|
63
|
+
* id: string;
|
|
64
|
+
* ws?: unknown;
|
|
65
|
+
* role: string;
|
|
66
|
+
* scopes: string[];
|
|
67
|
+
* userId: string | null;
|
|
68
|
+
* subscribe?: Set<string>;
|
|
69
|
+
* heartbeat?: unknown;
|
|
70
|
+
* lastActivity?: number;
|
|
71
|
+
* closed?: boolean;
|
|
72
|
+
* } & Record<string, unknown>} ConnectionState
|
|
73
|
+
*/
|
|
74
|
+
/**
|
|
75
|
+
* @typedef {{
|
|
76
|
+
* role: string;
|
|
77
|
+
* scopes: string[];
|
|
78
|
+
* userId?: string | null;
|
|
79
|
+
* connectionId?: string;
|
|
80
|
+
* }} RunStartAuthContext
|
|
81
|
+
*/
|
|
82
|
+
/**
|
|
83
|
+
* @typedef {{
|
|
84
|
+
* workflow: SmithersWorkflow;
|
|
85
|
+
* adapter: SmithersDb;
|
|
86
|
+
* key: string;
|
|
87
|
+
* schedule?: string;
|
|
88
|
+
* webhook?: GatewayWebhookConfig;
|
|
89
|
+
* }} RegisteredWorkflow
|
|
90
|
+
*/
|
|
91
|
+
/**
|
|
92
|
+
* @typedef {{
|
|
93
|
+
* runId: string;
|
|
94
|
+
* workflowKey: string;
|
|
95
|
+
* workflow: SmithersWorkflow;
|
|
96
|
+
* adapter: SmithersDb;
|
|
97
|
+
* }} ResolvedRun
|
|
98
|
+
*/
|
|
99
|
+
|
|
100
|
+
const DEFAULT_PROTOCOL = 1;
|
|
101
|
+
const DEFAULT_HEARTBEAT_MS = 15_000;
|
|
102
|
+
const DEFAULT_MAX_BODY_BYTES = 1_048_576;
|
|
103
|
+
const DEFAULT_MAX_CONNECTIONS = 1_000;
|
|
104
|
+
export const GATEWAY_RPC_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BODY_BYTES;
|
|
105
|
+
export const GATEWAY_RPC_MAX_DEPTH = 32;
|
|
106
|
+
export const GATEWAY_RPC_MAX_ARRAY_LENGTH = 256;
|
|
107
|
+
export const GATEWAY_RPC_MAX_STRING_LENGTH = 16 * 1024;
|
|
108
|
+
export const GATEWAY_METHOD_NAME_MAX_LENGTH = 64;
|
|
109
|
+
export const GATEWAY_FRAME_ID_MAX_LENGTH = 128;
|
|
110
|
+
export const GATEWAY_RPC_INPUT_MAX_BYTES = GATEWAY_RPC_MAX_PAYLOAD_BYTES;
|
|
111
|
+
export const GATEWAY_RPC_INPUT_MAX_DEPTH = GATEWAY_RPC_MAX_DEPTH;
|
|
112
|
+
const GATEWAY_METHOD_NAME_PATTERN = /^[a-z][a-zA-Z0-9]*(?:\.[a-z][a-zA-Z0-9]*)*$/;
|
|
113
|
+
const ACCESS_RANK = {
|
|
114
|
+
read: 1,
|
|
115
|
+
execute: 2,
|
|
116
|
+
approve: 3,
|
|
117
|
+
admin: 4,
|
|
118
|
+
};
|
|
119
|
+
const METHOD_ACCESS = {
|
|
120
|
+
health: "read",
|
|
121
|
+
"runs.list": "read",
|
|
122
|
+
"runs.get": "read",
|
|
123
|
+
"runs.diff": "read",
|
|
124
|
+
getNodeDiff: "read",
|
|
125
|
+
"devtools.getNodeDiff": "read",
|
|
126
|
+
getNodeOutput: "read",
|
|
127
|
+
"devtools.getNodeOutput": "read",
|
|
128
|
+
getDevToolsSnapshot: "read",
|
|
129
|
+
streamDevTools: "read",
|
|
130
|
+
// jumpToFrame authorizes per-request: owner OR admin role may rewind.
|
|
131
|
+
// Require only `execute` scope so non-admin owners are not pre-blocked
|
|
132
|
+
// at the scope gate; the run handler performs the final auth check.
|
|
133
|
+
jumpToFrame: "execute",
|
|
134
|
+
"devtools.jumpToFrame": "execute",
|
|
135
|
+
"frames.list": "read",
|
|
136
|
+
"frames.get": "read",
|
|
137
|
+
"attempts.list": "read",
|
|
138
|
+
"attempts.get": "read",
|
|
139
|
+
"approvals.list": "read",
|
|
140
|
+
"runs.create": "execute",
|
|
141
|
+
"runs.cancel": "execute",
|
|
142
|
+
"runs.rerun": "execute",
|
|
143
|
+
"signals.send": "execute",
|
|
144
|
+
"approvals.decide": "approve",
|
|
145
|
+
"cron.list": "read",
|
|
146
|
+
"cron.add": "admin",
|
|
147
|
+
"cron.remove": "admin",
|
|
148
|
+
"cron.trigger": "execute",
|
|
149
|
+
};
|
|
150
|
+
/**
|
|
151
|
+
* @template T
|
|
152
|
+
* @param {string | null | undefined} value
|
|
153
|
+
* @returns {T | null}
|
|
154
|
+
*/
|
|
155
|
+
function parseJson(value) {
|
|
156
|
+
if (!value)
|
|
157
|
+
return null;
|
|
158
|
+
try {
|
|
159
|
+
return JSON.parse(value);
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* @param {unknown} run
|
|
167
|
+
* @returns {string | null}
|
|
168
|
+
*/
|
|
169
|
+
function resolveRunOwnerId(run) {
|
|
170
|
+
const config = parseJson(typeof run?.configJson === "string" ? run.configJson : null);
|
|
171
|
+
const auth = asObject(config?.auth);
|
|
172
|
+
const owner = asString(auth?.triggeredBy);
|
|
173
|
+
return owner ? owner : null;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* @param {ServerResponse} res
|
|
177
|
+
* @param {number} status
|
|
178
|
+
* @param {unknown} payload
|
|
179
|
+
*/
|
|
180
|
+
function sendJson(res, status, payload) {
|
|
181
|
+
res.statusCode = status;
|
|
182
|
+
res.setHeader("Content-Type", "application/json");
|
|
183
|
+
res.setHeader("Cache-Control", "no-store");
|
|
184
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
185
|
+
res.end(JSON.stringify(payload));
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* @param {ServerResponse} res
|
|
189
|
+
* @param {number} status
|
|
190
|
+
* @param {string} payload
|
|
191
|
+
*/
|
|
192
|
+
function sendText(res, status, payload, contentType = "text/plain; charset=utf-8") {
|
|
193
|
+
res.statusCode = status;
|
|
194
|
+
res.setHeader("Content-Type", contentType);
|
|
195
|
+
res.setHeader("Cache-Control", "no-store");
|
|
196
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
197
|
+
res.end(payload);
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* @param {unknown} value
|
|
201
|
+
* @returns {Record<string, unknown> | null}
|
|
202
|
+
*/
|
|
203
|
+
function asObject(value) {
|
|
204
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
return value;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* @param {unknown} value
|
|
211
|
+
* @returns {string | undefined}
|
|
212
|
+
*/
|
|
213
|
+
function asString(value) {
|
|
214
|
+
return typeof value === "string" ? value : undefined;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* @param {unknown} value
|
|
218
|
+
* @returns {number | undefined}
|
|
219
|
+
*/
|
|
220
|
+
function asNumber(value) {
|
|
221
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* @param {unknown} value
|
|
225
|
+
* @returns {boolean | undefined}
|
|
226
|
+
*/
|
|
227
|
+
function asBoolean(value) {
|
|
228
|
+
return typeof value === "boolean" ? value : undefined;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* @param {unknown} value
|
|
232
|
+
* @returns {string | undefined}
|
|
233
|
+
*/
|
|
234
|
+
function asWebhookString(value) {
|
|
235
|
+
if (typeof value === "string") {
|
|
236
|
+
const trimmed = value.trim();
|
|
237
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
238
|
+
}
|
|
239
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
240
|
+
return String(value);
|
|
241
|
+
}
|
|
242
|
+
if (typeof value === "boolean") {
|
|
243
|
+
return value ? "true" : "false";
|
|
244
|
+
}
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* @template M
|
|
249
|
+
* @param {M} metric
|
|
250
|
+
* @param {GatewayMetricLabels} [labels]
|
|
251
|
+
* @returns {M}
|
|
252
|
+
*/
|
|
253
|
+
function taggedMetric(metric, labels = {}) {
|
|
254
|
+
let tagged = metric;
|
|
255
|
+
for (const [key, value] of Object.entries(labels)) {
|
|
256
|
+
if (value === undefined || value === null) {
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
tagged = Metric.tagged(tagged, key, String(value));
|
|
260
|
+
}
|
|
261
|
+
return tagged;
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* @template M
|
|
265
|
+
* @param {M} metric
|
|
266
|
+
* @param {GatewayMetricLabels} [labels]
|
|
267
|
+
*/
|
|
268
|
+
function incrementMetric(metric, labels = {}) {
|
|
269
|
+
return Metric.increment(taggedMetric(metric, labels));
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* @template M
|
|
273
|
+
* @param {M} metric
|
|
274
|
+
* @param {number} value
|
|
275
|
+
* @param {GatewayMetricLabels} [labels]
|
|
276
|
+
*/
|
|
277
|
+
function updateMetric(metric, value, labels = {}) {
|
|
278
|
+
return Metric.update(taggedMetric(metric, labels), value);
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* @param {Effect.Effect<void, never, never>} effect
|
|
282
|
+
*/
|
|
283
|
+
function emitGatewayEffect(effect) {
|
|
284
|
+
void runFork(effect);
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* @param {"debug" | "info" | "warning" | "error"} level
|
|
288
|
+
* @param {string} message
|
|
289
|
+
* @param {Record<string, unknown>} [annotations]
|
|
290
|
+
* @param {string} [span]
|
|
291
|
+
*/
|
|
292
|
+
function emitGatewayLog(level, message, annotations, span) {
|
|
293
|
+
let effect = level === "debug"
|
|
294
|
+
? Effect.logDebug(message)
|
|
295
|
+
: level === "info"
|
|
296
|
+
? Effect.logInfo(message)
|
|
297
|
+
: level === "warning"
|
|
298
|
+
? Effect.logWarning(message)
|
|
299
|
+
: Effect.logError(message);
|
|
300
|
+
if (annotations && Object.keys(annotations).length > 0) {
|
|
301
|
+
effect = effect.pipe(Effect.annotateLogs(annotations));
|
|
302
|
+
}
|
|
303
|
+
if (span) {
|
|
304
|
+
effect = effect.pipe(Effect.withLogSpan(span));
|
|
305
|
+
}
|
|
306
|
+
emitGatewayEffect(effect);
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* @param {GatewayRequestContext} context
|
|
310
|
+
* @returns {Record<string, unknown>}
|
|
311
|
+
*/
|
|
312
|
+
function gatewayContextAnnotations(context) {
|
|
313
|
+
return {
|
|
314
|
+
connectionId: context.connectionId,
|
|
315
|
+
transport: context.transport,
|
|
316
|
+
...(context.userId ? { userId: context.userId } : {}),
|
|
317
|
+
...(context.role ? { role: context.role } : {}),
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* @param {Record<string, unknown>} [params]
|
|
322
|
+
* @param {unknown} [payload]
|
|
323
|
+
* @returns {Record<string, unknown>}
|
|
324
|
+
*/
|
|
325
|
+
function gatewayRunAnnotations(params, payload) {
|
|
326
|
+
const annotations = {};
|
|
327
|
+
const responsePayload = asObject(payload);
|
|
328
|
+
const runId = asString(params?.runId) ?? asString(responsePayload?.runId);
|
|
329
|
+
const leftRunId = asString(params?.leftRunId);
|
|
330
|
+
const rightRunId = asString(params?.rightRunId);
|
|
331
|
+
if (runId) {
|
|
332
|
+
annotations.runId = runId;
|
|
333
|
+
}
|
|
334
|
+
if (leftRunId) {
|
|
335
|
+
annotations.leftRunId = leftRunId;
|
|
336
|
+
}
|
|
337
|
+
if (rightRunId) {
|
|
338
|
+
annotations.rightRunId = rightRunId;
|
|
339
|
+
}
|
|
340
|
+
return annotations;
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* @param {string} runId
|
|
344
|
+
* @returns {string}
|
|
345
|
+
*/
|
|
346
|
+
function devtoolsRunMetricTag(runId) {
|
|
347
|
+
return createHash("sha1").update(runId).digest("hex").slice(0, 12);
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* @param {GatewayRequestContext} context
|
|
351
|
+
* @param {Pick<RequestFrame, "id" | "method" | "params">} frame
|
|
352
|
+
* @param {unknown} [payload]
|
|
353
|
+
* @returns {Record<string, unknown>}
|
|
354
|
+
*/
|
|
355
|
+
function gatewayRpcAnnotations(context, frame, payload) {
|
|
356
|
+
return {
|
|
357
|
+
...gatewayContextAnnotations(context),
|
|
358
|
+
frameId: frame.id,
|
|
359
|
+
method: frame.method,
|
|
360
|
+
...gatewayRunAnnotations(asObject(frame.params) ?? {}, payload),
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* @param {unknown} error
|
|
365
|
+
* @returns {string}
|
|
366
|
+
*/
|
|
367
|
+
function gatewayErrorCode(error) {
|
|
368
|
+
if (error && typeof error === "object") {
|
|
369
|
+
const code = asString(error.code);
|
|
370
|
+
if (code) {
|
|
371
|
+
return code;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
if (error instanceof Error && error.name) {
|
|
375
|
+
return error.name;
|
|
376
|
+
}
|
|
377
|
+
return "UNKNOWN";
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* @param {unknown} error
|
|
381
|
+
* @returns {Record<string, unknown>}
|
|
382
|
+
*/
|
|
383
|
+
function gatewayErrorAnnotations(error) {
|
|
384
|
+
const serialized = asObject(errorToJson(error)) ?? { message: String(error) };
|
|
385
|
+
const summary = asString(serialized.summary);
|
|
386
|
+
const message = asString(serialized.message);
|
|
387
|
+
return {
|
|
388
|
+
errorCode: gatewayErrorCode(error),
|
|
389
|
+
...(summary ? { errorSummary: summary } : {}),
|
|
390
|
+
...(message ? { errorMessage: message } : {}),
|
|
391
|
+
error: serialized,
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* @param {GatewayAuthConfig | undefined} auth
|
|
396
|
+
* @returns {string}
|
|
397
|
+
*/
|
|
398
|
+
function gatewayAuthMode(auth) {
|
|
399
|
+
return auth?.mode ?? "none";
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* @param {string} triggeredBy
|
|
403
|
+
* @returns {string}
|
|
404
|
+
*/
|
|
405
|
+
function gatewayTriggerSource(triggeredBy) {
|
|
406
|
+
if (triggeredBy.startsWith("cron:")) {
|
|
407
|
+
return "cron";
|
|
408
|
+
}
|
|
409
|
+
if (triggeredBy.startsWith("webhook:")) {
|
|
410
|
+
return "webhook";
|
|
411
|
+
}
|
|
412
|
+
if (triggeredBy === "gateway") {
|
|
413
|
+
return "gateway";
|
|
414
|
+
}
|
|
415
|
+
return "user";
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* @param {unknown} value
|
|
419
|
+
* @param {string} field
|
|
420
|
+
* @returns {number | undefined}
|
|
421
|
+
*/
|
|
422
|
+
function asOptionalPositiveInt(value, field) {
|
|
423
|
+
if (value === undefined || value === null) {
|
|
424
|
+
return undefined;
|
|
425
|
+
}
|
|
426
|
+
return Math.floor(assertPositiveFiniteInteger(field, Number(value)));
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* @param {IncomingMessage} req
|
|
430
|
+
* @param {string} name
|
|
431
|
+
* @returns {string | null}
|
|
432
|
+
*/
|
|
433
|
+
function headerValue(req, name) {
|
|
434
|
+
const value = req.headers[name.toLowerCase()];
|
|
435
|
+
if (Array.isArray(value)) {
|
|
436
|
+
return value[0] ?? null;
|
|
437
|
+
}
|
|
438
|
+
return typeof value === "string" ? value : null;
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* @param {IncomingMessage} req
|
|
442
|
+
* @returns {string | null}
|
|
443
|
+
*/
|
|
444
|
+
function bearerTokenFromHeaders(req) {
|
|
445
|
+
const smithersKey = headerValue(req, "x-smithers-key");
|
|
446
|
+
if (smithersKey) {
|
|
447
|
+
return smithersKey;
|
|
448
|
+
}
|
|
449
|
+
const authHeader = headerValue(req, "authorization");
|
|
450
|
+
if (!authHeader) {
|
|
451
|
+
return null;
|
|
452
|
+
}
|
|
453
|
+
return authHeader.startsWith("Bearer ") ? authHeader.slice(7) : authHeader;
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* @param {unknown} value
|
|
457
|
+
* @returns {Record<string, unknown> | null}
|
|
458
|
+
*/
|
|
459
|
+
function asStringRecord(value) {
|
|
460
|
+
return asObject(value);
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* @param {string} id
|
|
464
|
+
* @param {unknown} [payload]
|
|
465
|
+
* @returns {ResponseFrame}
|
|
466
|
+
*/
|
|
467
|
+
function responseOk(id, payload) {
|
|
468
|
+
return { type: "res", id, ok: true, payload };
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* @param {string} id
|
|
472
|
+
* @param {string} code
|
|
473
|
+
* @param {string} message
|
|
474
|
+
* @returns {ResponseFrame}
|
|
475
|
+
*/
|
|
476
|
+
function responseError(id, code, message) {
|
|
477
|
+
return { type: "res", id, ok: false, error: { code, message } };
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* @param {unknown} raw
|
|
481
|
+
* @returns {string}
|
|
482
|
+
*/
|
|
483
|
+
function rawDataToUtf8(raw) {
|
|
484
|
+
if (typeof raw === "string") {
|
|
485
|
+
return raw;
|
|
486
|
+
}
|
|
487
|
+
if (Buffer.isBuffer(raw)) {
|
|
488
|
+
return raw.toString("utf8");
|
|
489
|
+
}
|
|
490
|
+
if (Array.isArray(raw)) {
|
|
491
|
+
return Buffer.concat(raw.map((entry) => (Buffer.isBuffer(entry) ? entry : Buffer.from(entry)))).toString("utf8");
|
|
492
|
+
}
|
|
493
|
+
return Buffer.from(raw).toString("utf8");
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* @param {unknown} method
|
|
497
|
+
* @returns {string}
|
|
498
|
+
*/
|
|
499
|
+
export function validateGatewayMethodName(method) {
|
|
500
|
+
if (typeof method !== "string") {
|
|
501
|
+
throw new SmithersError("INVALID_INPUT", "Gateway method name must be a string.", { methodType: typeof method });
|
|
502
|
+
}
|
|
503
|
+
assertOptionalStringMaxLength("method", method, GATEWAY_METHOD_NAME_MAX_LENGTH);
|
|
504
|
+
if (!GATEWAY_METHOD_NAME_PATTERN.test(method)) {
|
|
505
|
+
throw new SmithersError("INVALID_INPUT", "Gateway method name is invalid.", { method });
|
|
506
|
+
}
|
|
507
|
+
return method;
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* @param {unknown} raw
|
|
511
|
+
* @returns {RequestFrame}
|
|
512
|
+
*/
|
|
513
|
+
export function parseGatewayRequestFrame(raw, maxPayloadBytes = GATEWAY_RPC_MAX_PAYLOAD_BYTES) {
|
|
514
|
+
const body = rawDataToUtf8(raw);
|
|
515
|
+
if (Buffer.byteLength(body, "utf8") > maxPayloadBytes) {
|
|
516
|
+
throw new SmithersError("INVALID_INPUT", `Gateway RPC payload exceeds ${maxPayloadBytes} bytes.`, { maxBytes: maxPayloadBytes });
|
|
517
|
+
}
|
|
518
|
+
let parsed;
|
|
519
|
+
try {
|
|
520
|
+
parsed = JSON.parse(body);
|
|
521
|
+
}
|
|
522
|
+
catch (error) {
|
|
523
|
+
throw new SmithersError("INVALID_INPUT", "Gateway RPC payload must be valid JSON.", undefined, { cause: error });
|
|
524
|
+
}
|
|
525
|
+
assertJsonPayloadWithinBounds("gateway frame", parsed, {
|
|
526
|
+
maxArrayLength: GATEWAY_RPC_MAX_ARRAY_LENGTH,
|
|
527
|
+
maxDepth: GATEWAY_RPC_MAX_DEPTH,
|
|
528
|
+
maxStringLength: GATEWAY_RPC_MAX_STRING_LENGTH,
|
|
529
|
+
});
|
|
530
|
+
const frame = asObject(parsed);
|
|
531
|
+
if (!frame || frame.type !== "req") {
|
|
532
|
+
throw new SmithersError("INVALID_INPUT", "Gateway frame must be a request object.");
|
|
533
|
+
}
|
|
534
|
+
if (typeof frame.id !== "string") {
|
|
535
|
+
throw new SmithersError("INVALID_INPUT", "Gateway frame id must be a string.");
|
|
536
|
+
}
|
|
537
|
+
assertOptionalStringMaxLength("id", frame.id, GATEWAY_FRAME_ID_MAX_LENGTH);
|
|
538
|
+
return {
|
|
539
|
+
type: "req",
|
|
540
|
+
id: frame.id,
|
|
541
|
+
method: validateGatewayMethodName(frame.method),
|
|
542
|
+
params: frame.params,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* @param {unknown} value
|
|
547
|
+
* @param {number} depth
|
|
548
|
+
* @param {Set<unknown>} seen
|
|
549
|
+
* @returns {number}
|
|
550
|
+
*/
|
|
551
|
+
function gatewayInputDepthAt(value, depth, seen) {
|
|
552
|
+
if (!value || typeof value !== "object") {
|
|
553
|
+
return depth;
|
|
554
|
+
}
|
|
555
|
+
if (seen.has(value)) {
|
|
556
|
+
throw new SmithersError("INVALID_INPUT", "Gateway RPC input must not contain circular references.");
|
|
557
|
+
}
|
|
558
|
+
seen.add(value);
|
|
559
|
+
let maxDepth = depth;
|
|
560
|
+
const entries = Array.isArray(value)
|
|
561
|
+
? value
|
|
562
|
+
: Object.values(value);
|
|
563
|
+
for (const entry of entries) {
|
|
564
|
+
const entryDepth = entry && typeof entry === "object"
|
|
565
|
+
? gatewayInputDepthAt(entry, depth + 1, seen)
|
|
566
|
+
: depth;
|
|
567
|
+
if (entryDepth > maxDepth) {
|
|
568
|
+
maxDepth = entryDepth;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
seen.delete(value);
|
|
572
|
+
return maxDepth;
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* @param {unknown} value
|
|
576
|
+
* @returns {number}
|
|
577
|
+
*/
|
|
578
|
+
export function getGatewayInputDepth(value) {
|
|
579
|
+
if (!value || typeof value !== "object") {
|
|
580
|
+
return 0;
|
|
581
|
+
}
|
|
582
|
+
return gatewayInputDepthAt(value, 1, new Set());
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* @param {unknown} value
|
|
586
|
+
* @returns {number}
|
|
587
|
+
*/
|
|
588
|
+
export function assertGatewayInputDepthWithinBounds(value, maxDepth = GATEWAY_RPC_INPUT_MAX_DEPTH) {
|
|
589
|
+
const depth = getGatewayInputDepth(value);
|
|
590
|
+
if (depth > maxDepth) {
|
|
591
|
+
throw new SmithersError("INVALID_INPUT", `Gateway RPC input exceeds the maximum nesting depth of ${maxDepth}.`, {
|
|
592
|
+
actualDepth: depth,
|
|
593
|
+
maxDepth,
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
return depth;
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* @param {unknown} input
|
|
600
|
+
* @returns {Record<string, unknown>}
|
|
601
|
+
*/
|
|
602
|
+
function validateGatewayRpcInput(input) {
|
|
603
|
+
const normalizedInput = asObject(input) ?? {};
|
|
604
|
+
const inputJson = JSON.stringify(normalizedInput);
|
|
605
|
+
if (inputJson === undefined) {
|
|
606
|
+
throw new SmithersError("INVALID_INPUT", "Gateway RPC input must be JSON-serializable.");
|
|
607
|
+
}
|
|
608
|
+
const inputBytes = Buffer.byteLength(inputJson, "utf8");
|
|
609
|
+
if (inputBytes > GATEWAY_RPC_INPUT_MAX_BYTES) {
|
|
610
|
+
throw new SmithersError("INVALID_INPUT", `Gateway RPC input exceeds ${GATEWAY_RPC_INPUT_MAX_BYTES} bytes.`, {
|
|
611
|
+
actualBytes: inputBytes,
|
|
612
|
+
maxBytes: GATEWAY_RPC_INPUT_MAX_BYTES,
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
assertGatewayInputDepthWithinBounds(normalizedInput);
|
|
616
|
+
return normalizedInput;
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* @param {string | undefined} code
|
|
620
|
+
*/
|
|
621
|
+
export function statusForRpcError(code) {
|
|
622
|
+
switch (code) {
|
|
623
|
+
case "UNAUTHORIZED":
|
|
624
|
+
case "Unauthorized":
|
|
625
|
+
return 401;
|
|
626
|
+
case "FORBIDDEN":
|
|
627
|
+
return 403;
|
|
628
|
+
case "NOT_FOUND":
|
|
629
|
+
case "METHOD_NOT_FOUND":
|
|
630
|
+
return 404;
|
|
631
|
+
case "INVALID_REQUEST":
|
|
632
|
+
case "INVALID_FRAME":
|
|
633
|
+
case "INVALID_INPUT":
|
|
634
|
+
case "PROTOCOL_UNSUPPORTED":
|
|
635
|
+
case "InvalidRunId":
|
|
636
|
+
case "InvalidNodeId":
|
|
637
|
+
case "InvalidIteration":
|
|
638
|
+
case "InvalidDelta":
|
|
639
|
+
case "InvalidFrameNo":
|
|
640
|
+
case "ConfirmationRequired":
|
|
641
|
+
case "FrameOutOfRange":
|
|
642
|
+
case "SeqOutOfRange":
|
|
643
|
+
return 400;
|
|
644
|
+
case "RunNotFound":
|
|
645
|
+
case "NodeNotFound":
|
|
646
|
+
case "AttemptNotFound":
|
|
647
|
+
case "IterationNotFound":
|
|
648
|
+
case "NodeHasNoOutput":
|
|
649
|
+
return 404;
|
|
650
|
+
case "AttemptNotFinished":
|
|
651
|
+
case "Busy":
|
|
652
|
+
return 409;
|
|
653
|
+
case "DiffTooLarge":
|
|
654
|
+
case "PayloadTooLarge":
|
|
655
|
+
return 413;
|
|
656
|
+
case "RateLimited":
|
|
657
|
+
case "BackpressureDisconnect":
|
|
658
|
+
return 429;
|
|
659
|
+
case "UnsupportedSandbox":
|
|
660
|
+
return 501;
|
|
661
|
+
case "VcsError":
|
|
662
|
+
case "RewindFailed":
|
|
663
|
+
return 500;
|
|
664
|
+
default:
|
|
665
|
+
return 500;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* @param {unknown} payload
|
|
670
|
+
* @returns {string | null}
|
|
671
|
+
*/
|
|
672
|
+
function eventRunId(payload) {
|
|
673
|
+
const record = asObject(payload);
|
|
674
|
+
const runId = record ? asString(record.runId) : undefined;
|
|
675
|
+
return runId ?? null;
|
|
676
|
+
}
|
|
677
|
+
/**
|
|
678
|
+
* @param {string} scope
|
|
679
|
+
* @returns {string}
|
|
680
|
+
*/
|
|
681
|
+
function normalizeGrantedScope(scope) {
|
|
682
|
+
return scope.trim();
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* @param {string} method
|
|
686
|
+
* @returns {MethodAccess}
|
|
687
|
+
*/
|
|
688
|
+
function accessForMethod(method) {
|
|
689
|
+
return METHOD_ACCESS[method] ?? (method.startsWith("config.") ? "admin" : "read");
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* @param {string[]} scopes
|
|
693
|
+
* @param {string} method
|
|
694
|
+
* @returns {boolean}
|
|
695
|
+
*/
|
|
696
|
+
function hasScope(scopes, method) {
|
|
697
|
+
if (scopes.includes("*")) {
|
|
698
|
+
return true;
|
|
699
|
+
}
|
|
700
|
+
const requiredAccess = accessForMethod(method);
|
|
701
|
+
const grantedLevels = scopes
|
|
702
|
+
.map((scope) => scope.trim())
|
|
703
|
+
.filter((scope) => scope === "read" || scope === "execute" || scope === "approve" || scope === "admin");
|
|
704
|
+
if (grantedLevels.some((level) => ACCESS_RANK[level] >= ACCESS_RANK[requiredAccess])) {
|
|
705
|
+
return true;
|
|
706
|
+
}
|
|
707
|
+
for (const scope of scopes.map(normalizeGrantedScope)) {
|
|
708
|
+
if (!scope)
|
|
709
|
+
continue;
|
|
710
|
+
if (scope === method)
|
|
711
|
+
return true;
|
|
712
|
+
if (scope.endsWith(".*") && method.startsWith(scope.slice(0, -1))) {
|
|
713
|
+
return true;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return false;
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* @param {unknown} value
|
|
720
|
+
* @returns {string[]}
|
|
721
|
+
*/
|
|
722
|
+
function parseStringArray(value) {
|
|
723
|
+
if (!Array.isArray(value)) {
|
|
724
|
+
return [];
|
|
725
|
+
}
|
|
726
|
+
return value.filter((entry) => typeof entry === "string");
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* @param {string} value
|
|
730
|
+
* @returns {Record<string, unknown> | null}
|
|
731
|
+
*/
|
|
732
|
+
function decodeBase64UrlJson(value) {
|
|
733
|
+
try {
|
|
734
|
+
const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
|
|
735
|
+
const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
|
|
736
|
+
const decoded = Buffer.from(padded, "base64").toString("utf8");
|
|
737
|
+
return asStringRecord(JSON.parse(decoded));
|
|
738
|
+
}
|
|
739
|
+
catch {
|
|
740
|
+
return null;
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* @param {string} token
|
|
745
|
+
* @param {Extract<GatewayAuthConfig, { mode: "jwt" }>} config
|
|
746
|
+
* @returns {{ ok: true; payload: Record<string, unknown> } | { ok: false; message: string }}
|
|
747
|
+
*/
|
|
748
|
+
function verifyJwtToken(token, config) {
|
|
749
|
+
const [encodedHeader, encodedPayload, encodedSignature] = token.split(".");
|
|
750
|
+
if (!encodedHeader || !encodedPayload || !encodedSignature) {
|
|
751
|
+
return { ok: false, message: "JWT must have three segments" };
|
|
752
|
+
}
|
|
753
|
+
const header = decodeBase64UrlJson(encodedHeader);
|
|
754
|
+
const payload = decodeBase64UrlJson(encodedPayload);
|
|
755
|
+
if (!header || !payload) {
|
|
756
|
+
return { ok: false, message: "JWT header or payload was not valid JSON" };
|
|
757
|
+
}
|
|
758
|
+
if (header.alg !== "HS256") {
|
|
759
|
+
return { ok: false, message: "Unsupported JWT algorithm" };
|
|
760
|
+
}
|
|
761
|
+
const expectedSignature = createHmac("sha256", config.secret)
|
|
762
|
+
.update(`${encodedHeader}.${encodedPayload}`)
|
|
763
|
+
.digest("base64url");
|
|
764
|
+
const actualSignature = Buffer.from(encodedSignature, "base64url");
|
|
765
|
+
const expectedSignatureBuffer = Buffer.from(expectedSignature, "base64url");
|
|
766
|
+
if (actualSignature.length !== expectedSignatureBuffer.length ||
|
|
767
|
+
!timingSafeEqual(actualSignature, expectedSignatureBuffer)) {
|
|
768
|
+
return { ok: false, message: "JWT signature verification failed" };
|
|
769
|
+
}
|
|
770
|
+
const now = Math.floor(Date.now() / 1_000);
|
|
771
|
+
const skew = Math.max(0, config.clockSkewSeconds ?? 60);
|
|
772
|
+
const exp = asNumber(payload.exp);
|
|
773
|
+
const nbf = asNumber(payload.nbf);
|
|
774
|
+
const iss = asString(payload.iss);
|
|
775
|
+
const aud = payload.aud;
|
|
776
|
+
if (iss !== config.issuer) {
|
|
777
|
+
return { ok: false, message: "JWT issuer did not match" };
|
|
778
|
+
}
|
|
779
|
+
const audiences = Array.isArray(config.audience) ? config.audience : [config.audience];
|
|
780
|
+
const tokenAudiences = typeof aud === "string" ? [aud] : parseStringArray(aud);
|
|
781
|
+
if (!audiences.some((audience) => tokenAudiences.includes(audience))) {
|
|
782
|
+
return { ok: false, message: "JWT audience did not match" };
|
|
783
|
+
}
|
|
784
|
+
if (typeof exp === "number" && now - skew >= exp) {
|
|
785
|
+
return { ok: false, message: "JWT has expired" };
|
|
786
|
+
}
|
|
787
|
+
if (typeof nbf === "number" && now + skew < nbf) {
|
|
788
|
+
return { ok: false, message: "JWT is not active yet" };
|
|
789
|
+
}
|
|
790
|
+
return { ok: true, payload };
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* @param {unknown} value
|
|
794
|
+
* @returns {string[]}
|
|
795
|
+
*/
|
|
796
|
+
function parseJwtScopes(value) {
|
|
797
|
+
if (typeof value === "string") {
|
|
798
|
+
return value
|
|
799
|
+
.split(/[,\s]+/)
|
|
800
|
+
.map((entry) => entry.trim())
|
|
801
|
+
.filter(Boolean);
|
|
802
|
+
}
|
|
803
|
+
return parseStringArray(value);
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* @param {unknown} value
|
|
807
|
+
* @param {string | null} fallbackTitle
|
|
808
|
+
* @returns {ApprovalRequestRecord}
|
|
809
|
+
*/
|
|
810
|
+
function parseApprovalRequest(value, fallbackTitle) {
|
|
811
|
+
const record = asObject(value);
|
|
812
|
+
const options = Array.isArray(record?.options)
|
|
813
|
+
? record.options
|
|
814
|
+
.filter((entry) => Boolean(entry && typeof entry === "object" && !Array.isArray(entry)))
|
|
815
|
+
.map((entry) => ({
|
|
816
|
+
key: asString(entry.key) ?? "",
|
|
817
|
+
label: asString(entry.label) ?? "",
|
|
818
|
+
...(asString(entry.summary) ? { summary: asString(entry.summary) } : {}),
|
|
819
|
+
}))
|
|
820
|
+
.filter((entry) => entry.key.length > 0 && entry.label.length > 0)
|
|
821
|
+
: [];
|
|
822
|
+
const autoApprove = record?.autoApprove && typeof record.autoApprove === "object" && !Array.isArray(record.autoApprove)
|
|
823
|
+
? record.autoApprove
|
|
824
|
+
: null;
|
|
825
|
+
return {
|
|
826
|
+
mode: record?.mode === "select" || record?.mode === "rank" || record?.mode === "decision"
|
|
827
|
+
? record.mode
|
|
828
|
+
: "gate",
|
|
829
|
+
title: asString(record?.title) ?? fallbackTitle,
|
|
830
|
+
summary: asString(record?.summary) ?? null,
|
|
831
|
+
options,
|
|
832
|
+
allowedScopes: parseStringArray(record?.allowedScopes),
|
|
833
|
+
allowedUsers: parseStringArray(record?.allowedUsers),
|
|
834
|
+
autoApprove,
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
/**
|
|
838
|
+
* @param {ApprovalRequestRecord} request
|
|
839
|
+
* @param {unknown} decision
|
|
840
|
+
*/
|
|
841
|
+
function validateApprovalDecision(request, decision) {
|
|
842
|
+
if (request.mode === "select") {
|
|
843
|
+
const payload = asObject(decision);
|
|
844
|
+
const selected = asString(payload?.selected);
|
|
845
|
+
if (!selected) {
|
|
846
|
+
return { ok: false, code: "INVALID_REQUEST", message: "select approvals require decision.selected" };
|
|
847
|
+
}
|
|
848
|
+
if (request.options.length > 0 && !request.options.some((option) => option.key === selected)) {
|
|
849
|
+
return { ok: false, code: "INVALID_REQUEST", message: `Unknown selection: ${selected}` };
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
if (request.mode === "rank") {
|
|
853
|
+
const payload = asObject(decision);
|
|
854
|
+
const ranked = parseStringArray(payload?.ranked);
|
|
855
|
+
if (ranked.length === 0) {
|
|
856
|
+
return { ok: false, code: "INVALID_REQUEST", message: "rank approvals require decision.ranked" };
|
|
857
|
+
}
|
|
858
|
+
const allowed = new Set(request.options.map((option) => option.key));
|
|
859
|
+
if (allowed.size > 0 && ranked.some((value) => !allowed.has(value))) {
|
|
860
|
+
return { ok: false, code: "INVALID_REQUEST", message: "rank approval included unknown options" };
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
return { ok: true };
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* @param {string} pattern
|
|
867
|
+
*/
|
|
868
|
+
function nextCronRunAtMs(pattern) {
|
|
869
|
+
const interval = CronExpressionParser.parse(pattern);
|
|
870
|
+
return interval.next().getTime();
|
|
871
|
+
}
|
|
872
|
+
/**
|
|
873
|
+
* @param {number} ms
|
|
874
|
+
*/
|
|
875
|
+
function delay(ms) {
|
|
876
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* @param {string | null | undefined} value
|
|
880
|
+
* @returns {string | null}
|
|
881
|
+
*/
|
|
882
|
+
function normalizeCorrelationId(value) {
|
|
883
|
+
const normalized = typeof value === "string" ? value.trim() : "";
|
|
884
|
+
return normalized.length > 0 ? normalized : null;
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* @param {string | null} [metaJson]
|
|
888
|
+
*/
|
|
889
|
+
function parseWebhookWaitForEventSnapshot(metaJson) {
|
|
890
|
+
if (!metaJson) {
|
|
891
|
+
return null;
|
|
892
|
+
}
|
|
893
|
+
try {
|
|
894
|
+
const parsed = JSON.parse(metaJson);
|
|
895
|
+
const waitForEvent = asObject(asObject(parsed)?.waitForEvent);
|
|
896
|
+
const signalName = asString(waitForEvent?.signalName)?.trim();
|
|
897
|
+
if (!signalName) {
|
|
898
|
+
return null;
|
|
899
|
+
}
|
|
900
|
+
return {
|
|
901
|
+
signalName,
|
|
902
|
+
correlationId: normalizeCorrelationId(asString(waitForEvent?.correlationId) ?? null),
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
catch {
|
|
906
|
+
return null;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* @param {unknown} source
|
|
911
|
+
* @param {string | undefined} path
|
|
912
|
+
* @returns {unknown}
|
|
913
|
+
*/
|
|
914
|
+
function readPathValue(source, path) {
|
|
915
|
+
if (!path) {
|
|
916
|
+
return source;
|
|
917
|
+
}
|
|
918
|
+
const trimmed = path.trim();
|
|
919
|
+
if (!trimmed) {
|
|
920
|
+
return source;
|
|
921
|
+
}
|
|
922
|
+
let current = source;
|
|
923
|
+
for (const segment of trimmed.split(".").filter((entry) => entry.length > 0)) {
|
|
924
|
+
if (Array.isArray(current)) {
|
|
925
|
+
const index = Number(segment);
|
|
926
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
927
|
+
return undefined;
|
|
928
|
+
}
|
|
929
|
+
current = current[index];
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
const record = asObject(current);
|
|
933
|
+
if (!record) {
|
|
934
|
+
return undefined;
|
|
935
|
+
}
|
|
936
|
+
current = record[segment];
|
|
937
|
+
}
|
|
938
|
+
return current;
|
|
939
|
+
}
|
|
940
|
+
/**
|
|
941
|
+
* @param {Buffer} body
|
|
942
|
+
* @param {string} description
|
|
943
|
+
*/
|
|
944
|
+
function parseJsonBuffer(body, description) {
|
|
945
|
+
if (body.length === 0) {
|
|
946
|
+
return {};
|
|
947
|
+
}
|
|
948
|
+
try {
|
|
949
|
+
return JSON.parse(body.toString("utf8"));
|
|
950
|
+
}
|
|
951
|
+
catch (error) {
|
|
952
|
+
throw new SmithersError("INVALID_INPUT", `${description} must be valid JSON.`, undefined, { cause: error });
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
/**
|
|
956
|
+
* @param {IncomingMessage} req
|
|
957
|
+
* @param {number} maxBytes
|
|
958
|
+
*/
|
|
959
|
+
async function readRawBody(req, maxBytes) {
|
|
960
|
+
const chunks = [];
|
|
961
|
+
let total = 0;
|
|
962
|
+
const lengthHeader = headerValue(req, "content-length");
|
|
963
|
+
const declaredLength = lengthHeader ? Number(lengthHeader) : NaN;
|
|
964
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
|
965
|
+
throw new SmithersError("INVALID_INPUT", `Gateway request payload exceeds ${maxBytes} bytes.`, { maxBytes });
|
|
966
|
+
}
|
|
967
|
+
for await (const chunk of req) {
|
|
968
|
+
const buffer = Buffer.from(chunk);
|
|
969
|
+
total += buffer.length;
|
|
970
|
+
if (total > maxBytes) {
|
|
971
|
+
throw new SmithersError("INVALID_INPUT", `Gateway request payload exceeds ${maxBytes} bytes.`, { maxBytes });
|
|
972
|
+
}
|
|
973
|
+
chunks.push(buffer);
|
|
974
|
+
}
|
|
975
|
+
if (chunks.length === 0) {
|
|
976
|
+
return Buffer.alloc(0);
|
|
977
|
+
}
|
|
978
|
+
return Buffer.concat(chunks);
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* @param {IncomingMessage} req
|
|
982
|
+
* @param {number} maxBytes
|
|
983
|
+
*/
|
|
984
|
+
async function readBody(req, maxBytes) {
|
|
985
|
+
return parseJsonBuffer(await readRawBody(req, maxBytes), "Gateway RPC payload");
|
|
986
|
+
}
|
|
987
|
+
/**
|
|
988
|
+
* @param {Buffer} rawBody
|
|
989
|
+
* @param {string} secret
|
|
990
|
+
* @param {string} prefix
|
|
991
|
+
*/
|
|
992
|
+
function computeWebhookSignature(rawBody, secret, prefix) {
|
|
993
|
+
return `${prefix}${createHmac("sha256", secret).update(rawBody).digest("hex")}`;
|
|
994
|
+
}
|
|
995
|
+
/**
|
|
996
|
+
* @param {string} expected
|
|
997
|
+
* @param {string | null} provided
|
|
998
|
+
*/
|
|
999
|
+
function isValidWebhookSignature(expected, provided) {
|
|
1000
|
+
if (!provided) {
|
|
1001
|
+
return false;
|
|
1002
|
+
}
|
|
1003
|
+
const expectedBuffer = Buffer.from(expected);
|
|
1004
|
+
const providedBuffer = Buffer.from(provided);
|
|
1005
|
+
if (expectedBuffer.length !== providedBuffer.length) {
|
|
1006
|
+
return false;
|
|
1007
|
+
}
|
|
1008
|
+
return timingSafeEqual(expectedBuffer, providedBuffer);
|
|
1009
|
+
}
|
|
1010
|
+
/**
|
|
1011
|
+
* @param {unknown} input
|
|
1012
|
+
* @returns {Record<string, unknown>}
|
|
1013
|
+
*/
|
|
1014
|
+
function normalizeWebhookRunInput(input) {
|
|
1015
|
+
const normalized = asObject(input) ?? { payload: input ?? null };
|
|
1016
|
+
return validateGatewayRpcInput(normalized);
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* @param {string} workflowKey
|
|
1020
|
+
*/
|
|
1021
|
+
function webhookTriggerUserId(workflowKey) {
|
|
1022
|
+
return `webhook:${workflowKey}`;
|
|
1023
|
+
}
|
|
1024
|
+
/**
|
|
1025
|
+
* @param {string} workflowKey
|
|
1026
|
+
*/
|
|
1027
|
+
function cronWorkflowPath(workflowKey) {
|
|
1028
|
+
return `gateway:${workflowKey}`;
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* @param {string | null | undefined} workflowPath
|
|
1032
|
+
*/
|
|
1033
|
+
function workflowKeyFromCronPath(workflowPath) {
|
|
1034
|
+
if (!workflowPath || !workflowPath.startsWith("gateway:")) {
|
|
1035
|
+
return null;
|
|
1036
|
+
}
|
|
1037
|
+
return workflowPath.slice("gateway:".length);
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* @param {ConnectionState} connection
|
|
1041
|
+
* @param {string | null} runId
|
|
1042
|
+
*/
|
|
1043
|
+
function shouldDeliverEvent(connection, runId) {
|
|
1044
|
+
if (!runId)
|
|
1045
|
+
return true;
|
|
1046
|
+
if (!connection.subscribedRuns || connection.subscribedRuns.size === 0) {
|
|
1047
|
+
return true;
|
|
1048
|
+
}
|
|
1049
|
+
return connection.subscribedRuns.has(runId);
|
|
1050
|
+
}
|
|
1051
|
+
export class Gateway {
|
|
1052
|
+
protocol;
|
|
1053
|
+
features;
|
|
1054
|
+
heartbeatMs;
|
|
1055
|
+
maxBodyBytes;
|
|
1056
|
+
maxPayload;
|
|
1057
|
+
maxConnections;
|
|
1058
|
+
auth;
|
|
1059
|
+
defaults;
|
|
1060
|
+
workflows = new Map();
|
|
1061
|
+
connections = new Set();
|
|
1062
|
+
runRegistry = new Map();
|
|
1063
|
+
activeRuns = new Map();
|
|
1064
|
+
inflightRuns = new Map();
|
|
1065
|
+
devtoolsSubscribers = new Map();
|
|
1066
|
+
/** Absolute active subscriber count per runId (gauge source of truth). */
|
|
1067
|
+
devtoolsSubscriberCounts = new Map();
|
|
1068
|
+
/** Flagged subscriber IDs that should force a snapshot on their next emit. */
|
|
1069
|
+
devtoolsInvalidateFlags = new Set();
|
|
1070
|
+
server = null;
|
|
1071
|
+
wsServer = null;
|
|
1072
|
+
schedulerTimer = null;
|
|
1073
|
+
stateVersion = 0;
|
|
1074
|
+
startedAtMs = nowMs();
|
|
1075
|
+
/**
|
|
1076
|
+
* @param {GatewayOptions} [options]
|
|
1077
|
+
*/
|
|
1078
|
+
constructor(options = {}) {
|
|
1079
|
+
this.protocol = options.protocol ?? DEFAULT_PROTOCOL;
|
|
1080
|
+
this.features = [...(options.features ?? ["streaming", "runs"])];
|
|
1081
|
+
this.heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
|
|
1082
|
+
this.maxBodyBytes = options.maxBodyBytes === undefined
|
|
1083
|
+
? DEFAULT_MAX_BODY_BYTES
|
|
1084
|
+
: Math.floor(assertPositiveFiniteInteger("maxBodyBytes", Number(options.maxBodyBytes)));
|
|
1085
|
+
this.maxPayload = options.maxPayload === undefined
|
|
1086
|
+
? GATEWAY_RPC_MAX_PAYLOAD_BYTES
|
|
1087
|
+
: Math.floor(assertPositiveFiniteInteger("maxPayload", Number(options.maxPayload)));
|
|
1088
|
+
this.maxConnections = options.maxConnections === undefined
|
|
1089
|
+
? DEFAULT_MAX_CONNECTIONS
|
|
1090
|
+
: Math.floor(assertPositiveFiniteInteger("maxConnections", Number(options.maxConnections)));
|
|
1091
|
+
this.auth = options.auth;
|
|
1092
|
+
this.defaults = options.defaults;
|
|
1093
|
+
}
|
|
1094
|
+
authModeLabel() {
|
|
1095
|
+
return gatewayAuthMode(this.auth);
|
|
1096
|
+
}
|
|
1097
|
+
/**
|
|
1098
|
+
* @param {string} [runId]
|
|
1099
|
+
* @returns {number}
|
|
1100
|
+
*/
|
|
1101
|
+
getDevToolsSubscriberCount(runId) {
|
|
1102
|
+
if (!runId) {
|
|
1103
|
+
return this.devtoolsSubscribers.size;
|
|
1104
|
+
}
|
|
1105
|
+
let count = 0;
|
|
1106
|
+
for (const subscriber of this.devtoolsSubscribers.values()) {
|
|
1107
|
+
if (subscriber.runId === runId) {
|
|
1108
|
+
count += 1;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
return count;
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* Record a single subscribe attempt outcome. Centralised so that invalid
|
|
1115
|
+
* runId, missing run, SeqOutOfRange, etc. still update
|
|
1116
|
+
* `smithers_devtools_subscribe_total{result="error"}`.
|
|
1117
|
+
*
|
|
1118
|
+
* @param {"ok" | "error"} result
|
|
1119
|
+
*/
|
|
1120
|
+
recordDevToolsSubscribeAttempt(result) {
|
|
1121
|
+
emitGatewayEffect(Metric.increment(taggedMetric(devtoolsSubscribeTotal, { result })));
|
|
1122
|
+
}
|
|
1123
|
+
/**
|
|
1124
|
+
* Push the absolute active-subscriber count to the Prometheus gauge. The
|
|
1125
|
+
* `runId` is hashed for bounded cardinality.
|
|
1126
|
+
*
|
|
1127
|
+
* @param {string} runId
|
|
1128
|
+
*/
|
|
1129
|
+
publishDevToolsActiveSubscribersGauge(runId) {
|
|
1130
|
+
const runMetricLabel = devtoolsRunMetricTag(runId);
|
|
1131
|
+
const value = this.devtoolsSubscriberCounts.get(runId) ?? 0;
|
|
1132
|
+
emitGatewayEffect(Metric.update(taggedMetric(devtoolsActiveSubscribers, { runId: runMetricLabel }), value));
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* @param {ConnectionState} connection
|
|
1136
|
+
* @param {string} streamId
|
|
1137
|
+
* @param {string} runId
|
|
1138
|
+
* @returns {AbortController}
|
|
1139
|
+
*/
|
|
1140
|
+
registerDevToolsSubscriber(connection, streamId, runId) {
|
|
1141
|
+
const abort = new AbortController();
|
|
1142
|
+
if (!connection.devtoolsStreams) {
|
|
1143
|
+
connection.devtoolsStreams = new Map();
|
|
1144
|
+
}
|
|
1145
|
+
connection.devtoolsStreams.set(streamId, {
|
|
1146
|
+
runId,
|
|
1147
|
+
abort,
|
|
1148
|
+
});
|
|
1149
|
+
this.devtoolsSubscribers.set(streamId, {
|
|
1150
|
+
runId,
|
|
1151
|
+
connectionId: connection.connectionId,
|
|
1152
|
+
abort,
|
|
1153
|
+
startedAtMs: Date.now(),
|
|
1154
|
+
});
|
|
1155
|
+
const previous = this.devtoolsSubscriberCounts.get(runId) ?? 0;
|
|
1156
|
+
this.devtoolsSubscriberCounts.set(runId, previous + 1);
|
|
1157
|
+
this.recordDevToolsSubscribeAttempt("ok");
|
|
1158
|
+
this.publishDevToolsActiveSubscribersGauge(runId);
|
|
1159
|
+
return abort;
|
|
1160
|
+
}
|
|
1161
|
+
/**
|
|
1162
|
+
* @param {ConnectionState} connection
|
|
1163
|
+
* @param {string} streamId
|
|
1164
|
+
* @param {Record<string, unknown>} [details]
|
|
1165
|
+
*/
|
|
1166
|
+
unregisterDevToolsSubscriber(connection, streamId, details = {}) {
|
|
1167
|
+
const stream = connection.devtoolsStreams?.get(streamId);
|
|
1168
|
+
if (stream) {
|
|
1169
|
+
stream.abort.abort();
|
|
1170
|
+
connection.devtoolsStreams?.delete(streamId);
|
|
1171
|
+
}
|
|
1172
|
+
const subscriber = this.devtoolsSubscribers.get(streamId);
|
|
1173
|
+
if (!subscriber) {
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
this.devtoolsSubscribers.delete(streamId);
|
|
1177
|
+
this.devtoolsInvalidateFlags.delete(streamId);
|
|
1178
|
+
const previous = this.devtoolsSubscriberCounts.get(subscriber.runId) ?? 0;
|
|
1179
|
+
const nextCount = Math.max(0, previous - 1);
|
|
1180
|
+
if (nextCount === 0) {
|
|
1181
|
+
this.devtoolsSubscriberCounts.delete(subscriber.runId);
|
|
1182
|
+
}
|
|
1183
|
+
else {
|
|
1184
|
+
this.devtoolsSubscriberCounts.set(subscriber.runId, nextCount);
|
|
1185
|
+
}
|
|
1186
|
+
this.publishDevToolsActiveSubscribersGauge(subscriber.runId);
|
|
1187
|
+
emitGatewayLog("info", "devtools stream unsubscribed", {
|
|
1188
|
+
runId: subscriber.runId,
|
|
1189
|
+
streamId,
|
|
1190
|
+
durationMs: Date.now() - subscriber.startedAtMs,
|
|
1191
|
+
...details,
|
|
1192
|
+
}, "gateway:devtools");
|
|
1193
|
+
}
|
|
1194
|
+
/**
|
|
1195
|
+
* Flag every active subscriber for `runId` to rebaseline on its next emit.
|
|
1196
|
+
* Called when the gateway observes `TimeTravelJumped` for that run.
|
|
1197
|
+
*
|
|
1198
|
+
* @param {string} runId
|
|
1199
|
+
*/
|
|
1200
|
+
invalidateDevToolsSubscribersForRun(runId) {
|
|
1201
|
+
for (const [streamId, subscriber] of this.devtoolsSubscribers.entries()) {
|
|
1202
|
+
if (subscriber.runId === runId) {
|
|
1203
|
+
this.devtoolsInvalidateFlags.add(streamId);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Authorize a devtools request against the connection's `subscribe` set.
|
|
1209
|
+
*
|
|
1210
|
+
* If the client provided a `subscribe` filter at `connect` time, the run
|
|
1211
|
+
* must be in that set before any DB lookup happens.
|
|
1212
|
+
*
|
|
1213
|
+
* @param {ConnectionState | null | undefined} connection
|
|
1214
|
+
* @param {string} runId
|
|
1215
|
+
* @returns {boolean}
|
|
1216
|
+
*/
|
|
1217
|
+
isDevToolsRunAuthorized(connection, runId) {
|
|
1218
|
+
if (!connection) {
|
|
1219
|
+
return true;
|
|
1220
|
+
}
|
|
1221
|
+
if (!connection.subscribedRuns || connection.subscribedRuns.size === 0) {
|
|
1222
|
+
return true;
|
|
1223
|
+
}
|
|
1224
|
+
return connection.subscribedRuns.has(runId);
|
|
1225
|
+
}
|
|
1226
|
+
/**
|
|
1227
|
+
* @param {ConnectionState} connection
|
|
1228
|
+
*/
|
|
1229
|
+
cleanupDevToolsSubscribers(connection) {
|
|
1230
|
+
const streams = connection.devtoolsStreams;
|
|
1231
|
+
if (!streams || streams.size === 0) {
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
for (const streamId of streams.keys()) {
|
|
1235
|
+
this.unregisterDevToolsSubscriber(connection, streamId, {
|
|
1236
|
+
reason: "connection_closed",
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
/**
|
|
1241
|
+
* @param {GatewayTransport} transport
|
|
1242
|
+
* @param {string} frameType
|
|
1243
|
+
* @param {GatewayMetricLabels} [labels]
|
|
1244
|
+
*/
|
|
1245
|
+
recordMessageReceived(transport, frameType, labels = {}) {
|
|
1246
|
+
emitGatewayEffect(incrementMetric(gatewayMessagesReceivedTotal, {
|
|
1247
|
+
transport,
|
|
1248
|
+
frameType,
|
|
1249
|
+
...labels,
|
|
1250
|
+
}));
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* @param {GatewayTransport} transport
|
|
1254
|
+
* @param {string} frameType
|
|
1255
|
+
* @param {GatewayMetricLabels} [labels]
|
|
1256
|
+
*/
|
|
1257
|
+
recordMessageSent(transport, frameType, labels = {}) {
|
|
1258
|
+
emitGatewayEffect(incrementMetric(gatewayMessagesSentTotal, {
|
|
1259
|
+
transport,
|
|
1260
|
+
frameType,
|
|
1261
|
+
...labels,
|
|
1262
|
+
}));
|
|
1263
|
+
}
|
|
1264
|
+
/**
|
|
1265
|
+
* @param {GatewayTransport} transport
|
|
1266
|
+
* @param {"success" | "failure"} outcome
|
|
1267
|
+
* @param {GatewayRequestContext} context
|
|
1268
|
+
* @param {Record<string, unknown>} [details]
|
|
1269
|
+
* @param {"debug" | "info" | "warning"} [level]
|
|
1270
|
+
*/
|
|
1271
|
+
recordAuthEvent(transport, outcome, context, details = {}, level = outcome === "success" ? "info" : "warning") {
|
|
1272
|
+
const annotations = {
|
|
1273
|
+
...gatewayContextAnnotations(context),
|
|
1274
|
+
authMode: this.authModeLabel(),
|
|
1275
|
+
outcome,
|
|
1276
|
+
...details,
|
|
1277
|
+
};
|
|
1278
|
+
const logEffect = level === "debug"
|
|
1279
|
+
? Effect.logDebug(outcome === "success"
|
|
1280
|
+
? "Gateway auth succeeded"
|
|
1281
|
+
: "Gateway auth rejected")
|
|
1282
|
+
: level === "info"
|
|
1283
|
+
? Effect.logInfo("Gateway auth succeeded")
|
|
1284
|
+
: Effect.logWarning("Gateway auth rejected");
|
|
1285
|
+
emitGatewayEffect(Effect.all([
|
|
1286
|
+
incrementMetric(gatewayAuthEventsTotal, {
|
|
1287
|
+
transport,
|
|
1288
|
+
mode: this.authModeLabel(),
|
|
1289
|
+
outcome,
|
|
1290
|
+
}),
|
|
1291
|
+
logEffect.pipe(Effect.annotateLogs(annotations), Effect.withLogSpan("gateway:auth")),
|
|
1292
|
+
], { discard: true }));
|
|
1293
|
+
}
|
|
1294
|
+
/**
|
|
1295
|
+
* @param {GatewayRequestContext} context
|
|
1296
|
+
* @param {RequestFrame} frame
|
|
1297
|
+
* @param {() => Promise<ResponseFrame>} handler
|
|
1298
|
+
* @returns {Promise<ResponseFrame>}
|
|
1299
|
+
*/
|
|
1300
|
+
async executeRpc(context, frame, handler) {
|
|
1301
|
+
const self = this;
|
|
1302
|
+
const start = performance.now();
|
|
1303
|
+
const params = asObject(frame.params) ?? {};
|
|
1304
|
+
const result = await runPromise(Effect.gen(function* () {
|
|
1305
|
+
yield* incrementMetric(gatewayRpcCallsTotal, {
|
|
1306
|
+
transport: context.transport,
|
|
1307
|
+
method: frame.method,
|
|
1308
|
+
});
|
|
1309
|
+
yield* Effect.logDebug("Gateway RPC started");
|
|
1310
|
+
const result = yield* Effect.promise(() => handler()
|
|
1311
|
+
.then((response) => ({ _tag: "success", response }))
|
|
1312
|
+
.catch((error) => ({ _tag: "failure", error })));
|
|
1313
|
+
yield* updateMetric(gatewayRpcDuration, performance.now() - start, {
|
|
1314
|
+
transport: context.transport,
|
|
1315
|
+
method: frame.method,
|
|
1316
|
+
});
|
|
1317
|
+
if (result._tag === "failure") {
|
|
1318
|
+
yield* incrementMetric(gatewayErrorsTotal, {
|
|
1319
|
+
kind: "rpc",
|
|
1320
|
+
transport: context.transport,
|
|
1321
|
+
method: frame.method,
|
|
1322
|
+
code: gatewayErrorCode(result.error),
|
|
1323
|
+
});
|
|
1324
|
+
yield* Effect.logError("Gateway RPC failed").pipe(Effect.annotateLogs(gatewayErrorAnnotations(result.error)));
|
|
1325
|
+
return result;
|
|
1326
|
+
}
|
|
1327
|
+
if (!result.response.ok) {
|
|
1328
|
+
yield* incrementMetric(gatewayErrorsTotal, {
|
|
1329
|
+
kind: "rpc",
|
|
1330
|
+
transport: context.transport,
|
|
1331
|
+
method: frame.method,
|
|
1332
|
+
code: result.response.error?.code ?? "UNKNOWN",
|
|
1333
|
+
});
|
|
1334
|
+
yield* Effect.logWarning("Gateway RPC rejected").pipe(Effect.annotateLogs({
|
|
1335
|
+
...gatewayRunAnnotations(params, result.response.payload),
|
|
1336
|
+
rpcCode: result.response.error?.code ?? "UNKNOWN",
|
|
1337
|
+
...(result.response.error?.message
|
|
1338
|
+
? { rpcMessage: result.response.error.message }
|
|
1339
|
+
: {}),
|
|
1340
|
+
}));
|
|
1341
|
+
}
|
|
1342
|
+
else {
|
|
1343
|
+
yield* Effect.logDebug("Gateway RPC completed").pipe(Effect.annotateLogs(gatewayRunAnnotations(params, result.response.payload)));
|
|
1344
|
+
yield* self.rpcSuccessEffect(context, frame, result.response);
|
|
1345
|
+
}
|
|
1346
|
+
return result;
|
|
1347
|
+
}).pipe(Effect.annotateLogs(gatewayRpcAnnotations(context, frame)), Effect.withLogSpan(`gateway:rpc:${frame.method}`)));
|
|
1348
|
+
if (result._tag === "failure") {
|
|
1349
|
+
throw result.error;
|
|
1350
|
+
}
|
|
1351
|
+
return result.response;
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* @param {GatewayRequestContext} context
|
|
1355
|
+
* @param {RequestFrame} frame
|
|
1356
|
+
* @param {ResponseFrame} response
|
|
1357
|
+
* @returns {Effect.Effect<void>}
|
|
1358
|
+
*/
|
|
1359
|
+
rpcSuccessEffect(context, frame, response) {
|
|
1360
|
+
const params = asObject(frame.params) ?? {};
|
|
1361
|
+
switch (frame.method) {
|
|
1362
|
+
case "approvals.decide": {
|
|
1363
|
+
const approved = asBoolean(params.approved) ?? false;
|
|
1364
|
+
const nodeId = asString(params.nodeId);
|
|
1365
|
+
return Effect.all([
|
|
1366
|
+
incrementMetric(gatewayApprovalDecisionsTotal, {
|
|
1367
|
+
outcome: approved ? "approved" : "denied",
|
|
1368
|
+
}),
|
|
1369
|
+
Effect.logInfo("Gateway approval decision recorded").pipe(Effect.annotateLogs({
|
|
1370
|
+
...gatewayRpcAnnotations(context, frame, response.payload),
|
|
1371
|
+
...(nodeId ? { nodeId } : {}),
|
|
1372
|
+
iteration: asNumber(params.iteration) ?? 0,
|
|
1373
|
+
approved,
|
|
1374
|
+
})),
|
|
1375
|
+
], { discard: true });
|
|
1376
|
+
}
|
|
1377
|
+
case "signals.send": {
|
|
1378
|
+
const signalName = asString(params.signalName);
|
|
1379
|
+
const correlationId = asString(params.correlationId);
|
|
1380
|
+
return Effect.all([
|
|
1381
|
+
incrementMetric(gatewaySignalsTotal, { outcome: "sent" }),
|
|
1382
|
+
Effect.logInfo("Gateway signal sent").pipe(Effect.annotateLogs({
|
|
1383
|
+
...gatewayRpcAnnotations(context, frame, response.payload),
|
|
1384
|
+
...(signalName ? { signalName } : {}),
|
|
1385
|
+
...(correlationId ? { correlationId } : {}),
|
|
1386
|
+
})),
|
|
1387
|
+
], { discard: true });
|
|
1388
|
+
}
|
|
1389
|
+
case "cron.trigger": {
|
|
1390
|
+
const cronId = asString(params.cronId);
|
|
1391
|
+
const workflow = asString(params.workflow);
|
|
1392
|
+
return Effect.all([
|
|
1393
|
+
incrementMetric(gatewayCronTriggersTotal, { source: "manual" }),
|
|
1394
|
+
Effect.logInfo("Gateway cron trigger requested").pipe(Effect.annotateLogs({
|
|
1395
|
+
...gatewayRpcAnnotations(context, frame, response.payload),
|
|
1396
|
+
...(cronId ? { cronId } : {}),
|
|
1397
|
+
...(workflow ? { workflow } : {}),
|
|
1398
|
+
})),
|
|
1399
|
+
], { discard: true });
|
|
1400
|
+
}
|
|
1401
|
+
default:
|
|
1402
|
+
return Effect.void;
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
/**
|
|
1406
|
+
* @param {ServerResponse} res
|
|
1407
|
+
* @param {number} status
|
|
1408
|
+
* @param {ResponseFrame} response
|
|
1409
|
+
*/
|
|
1410
|
+
sendHttpRpcResponse(res, status, response) {
|
|
1411
|
+
this.recordMessageSent("http", "response", {
|
|
1412
|
+
outcome: response.ok ? "ok" : "error",
|
|
1413
|
+
});
|
|
1414
|
+
return sendJson(res, status, response);
|
|
1415
|
+
}
|
|
1416
|
+
/**
|
|
1417
|
+
* @param {SmithersDb} adapter
|
|
1418
|
+
* @param {string} runId
|
|
1419
|
+
* @param {string} signalName
|
|
1420
|
+
* @param {string | null} correlationId
|
|
1421
|
+
*/
|
|
1422
|
+
async runWaitsForSignal(adapter, runId, signalName, correlationId) {
|
|
1423
|
+
const nodes = await adapter.listNodes(runId);
|
|
1424
|
+
for (const node of nodes) {
|
|
1425
|
+
if (node.state !== "waiting-event") {
|
|
1426
|
+
continue;
|
|
1427
|
+
}
|
|
1428
|
+
const iteration = node.iteration ?? 0;
|
|
1429
|
+
const attempts = await runPromise(adapter.listAttempts(runId, node.nodeId, iteration));
|
|
1430
|
+
const waitingAttempt = attempts.find((attempt) => attempt.state === "waiting-event") ??
|
|
1431
|
+
attempts[0];
|
|
1432
|
+
const snapshot = parseWebhookWaitForEventSnapshot(waitingAttempt?.metaJson);
|
|
1433
|
+
if (!snapshot) {
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
if (snapshot.signalName !== signalName) {
|
|
1437
|
+
continue;
|
|
1438
|
+
}
|
|
1439
|
+
if (snapshot.correlationId !== correlationId) {
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
return true;
|
|
1443
|
+
}
|
|
1444
|
+
return false;
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1447
|
+
* @param {RegisteredWorkflow} entry
|
|
1448
|
+
* @param {string} signalName
|
|
1449
|
+
* @param {string | null} correlationId
|
|
1450
|
+
* @param {string} [explicitRunId]
|
|
1451
|
+
*/
|
|
1452
|
+
async findMatchingWebhookRuns(entry, signalName, correlationId, explicitRunId) {
|
|
1453
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
1454
|
+
const matches = new Set();
|
|
1455
|
+
if (explicitRunId) {
|
|
1456
|
+
const run = await adapter.getRun(explicitRunId);
|
|
1457
|
+
if (run &&
|
|
1458
|
+
run.status !== "finished" &&
|
|
1459
|
+
run.status !== "failed" &&
|
|
1460
|
+
run.status !== "cancelled" &&
|
|
1461
|
+
await this.runWaitsForSignal(adapter, explicitRunId, signalName, correlationId)) {
|
|
1462
|
+
matches.add(explicitRunId);
|
|
1463
|
+
}
|
|
1464
|
+
return [...matches];
|
|
1465
|
+
}
|
|
1466
|
+
const waitingRuns = await adapter.listRuns(1_000, "waiting-event");
|
|
1467
|
+
for (const run of waitingRuns) {
|
|
1468
|
+
if (await this.runWaitsForSignal(adapter, run.runId, signalName, correlationId)) {
|
|
1469
|
+
matches.add(run.runId);
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
return [...matches];
|
|
1473
|
+
}
|
|
1474
|
+
/**
|
|
1475
|
+
* @param {IncomingMessage} req
|
|
1476
|
+
* @param {ServerResponse} res
|
|
1477
|
+
* @param {string} workflowKey
|
|
1478
|
+
*/
|
|
1479
|
+
async handleWebhook(req, res, workflowKey) {
|
|
1480
|
+
const requestId = headerValue(req, "x-request-id") ?? randomUUID();
|
|
1481
|
+
/**
|
|
1482
|
+
* @param {number} status
|
|
1483
|
+
* @param {unknown} payload
|
|
1484
|
+
*/
|
|
1485
|
+
const respond = (status, payload) => {
|
|
1486
|
+
this.recordMessageSent("http", "response", {
|
|
1487
|
+
route: "webhook",
|
|
1488
|
+
workflow: workflowKey,
|
|
1489
|
+
outcome: status < 400 ? "ok" : "error",
|
|
1490
|
+
});
|
|
1491
|
+
return sendJson(res, status, payload);
|
|
1492
|
+
};
|
|
1493
|
+
/**
|
|
1494
|
+
* @param {number} status
|
|
1495
|
+
* @param {string} code
|
|
1496
|
+
* @param {string} message
|
|
1497
|
+
* @param {string} reason
|
|
1498
|
+
* @param {unknown} [error]
|
|
1499
|
+
*/
|
|
1500
|
+
const reject = (status, code, message, reason, error) => {
|
|
1501
|
+
emitGatewayEffect(Effect.all([
|
|
1502
|
+
incrementMetric(gatewayWebhooksRejectedTotal, {
|
|
1503
|
+
workflow: workflowKey,
|
|
1504
|
+
reason,
|
|
1505
|
+
}),
|
|
1506
|
+
incrementMetric(gatewayErrorsTotal, {
|
|
1507
|
+
kind: "webhook",
|
|
1508
|
+
workflow: workflowKey,
|
|
1509
|
+
code,
|
|
1510
|
+
}),
|
|
1511
|
+
], { discard: true }));
|
|
1512
|
+
emitGatewayLog(error && !isSmithersError(error) ? "error" : "warning", "Gateway webhook rejected", {
|
|
1513
|
+
workflow: workflowKey,
|
|
1514
|
+
requestId,
|
|
1515
|
+
reason,
|
|
1516
|
+
errorCode: code,
|
|
1517
|
+
errorMessage: message,
|
|
1518
|
+
...(error ? gatewayErrorAnnotations(error) : {}),
|
|
1519
|
+
}, "gateway:webhook");
|
|
1520
|
+
return respond(status, { ok: false, error: { code, message } });
|
|
1521
|
+
};
|
|
1522
|
+
this.recordMessageReceived("http", "webhook", { workflow: workflowKey });
|
|
1523
|
+
emitGatewayEffect(incrementMetric(gatewayWebhooksReceivedTotal, {
|
|
1524
|
+
workflow: workflowKey,
|
|
1525
|
+
}));
|
|
1526
|
+
const entry = this.workflows.get(workflowKey);
|
|
1527
|
+
if (!entry) {
|
|
1528
|
+
return reject(404, "NOT_FOUND", `Unknown workflow: ${workflowKey}`, "workflow_not_found");
|
|
1529
|
+
}
|
|
1530
|
+
const webhook = entry.webhook;
|
|
1531
|
+
if (!webhook) {
|
|
1532
|
+
return reject(404, "NOT_FOUND", `Webhook not configured for workflow: ${workflowKey}`, "not_configured");
|
|
1533
|
+
}
|
|
1534
|
+
const secret = webhook.secret.trim();
|
|
1535
|
+
if (!secret) {
|
|
1536
|
+
return reject(500, "SERVER_ERROR", "Webhook secret is not configured", "not_configured");
|
|
1537
|
+
}
|
|
1538
|
+
const signatureHeader = webhook.signatureHeader?.trim().toLowerCase() || "x-hub-signature-256";
|
|
1539
|
+
const signaturePrefix = webhook.signaturePrefix ?? "sha256=";
|
|
1540
|
+
const signalConfig = webhook.signal;
|
|
1541
|
+
const runConfig = webhook.run;
|
|
1542
|
+
const runEnabled = runConfig?.enabled !== false;
|
|
1543
|
+
if (!signalConfig?.name && !runEnabled) {
|
|
1544
|
+
return reject(400, "INVALID_REQUEST", "Webhook config must enable signal delivery or run creation", "misconfigured");
|
|
1545
|
+
}
|
|
1546
|
+
try {
|
|
1547
|
+
const rawBody = await readRawBody(req, this.maxBodyBytes);
|
|
1548
|
+
const providedSignature = headerValue(req, signatureHeader);
|
|
1549
|
+
const expectedSignature = computeWebhookSignature(rawBody, secret, signaturePrefix);
|
|
1550
|
+
if (!isValidWebhookSignature(expectedSignature, providedSignature)) {
|
|
1551
|
+
return reject(401, "UNAUTHORIZED", "Webhook signature verification failed", "invalid_signature");
|
|
1552
|
+
}
|
|
1553
|
+
emitGatewayEffect(incrementMetric(gatewayWebhooksVerifiedTotal, {
|
|
1554
|
+
workflow: workflowKey,
|
|
1555
|
+
}));
|
|
1556
|
+
const payload = parseJsonBuffer(rawBody, "Webhook payload");
|
|
1557
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
1558
|
+
const explicitRunId = asWebhookString(readPathValue(payload, signalConfig?.runIdPath));
|
|
1559
|
+
const correlationId = normalizeCorrelationId(asWebhookString(readPathValue(payload, signalConfig?.correlationIdPath)) ?? null);
|
|
1560
|
+
const signalPayload = readPathValue(payload, signalConfig?.payloadPath);
|
|
1561
|
+
const matchedRunIds = signalConfig?.name
|
|
1562
|
+
? await this.findMatchingWebhookRuns(entry, signalConfig.name, correlationId, explicitRunId)
|
|
1563
|
+
: [];
|
|
1564
|
+
const triggeredBy = webhookTriggerUserId(workflowKey);
|
|
1565
|
+
const auth = {
|
|
1566
|
+
triggeredBy,
|
|
1567
|
+
scopes: ["*"],
|
|
1568
|
+
role: "system",
|
|
1569
|
+
};
|
|
1570
|
+
const delivered = [];
|
|
1571
|
+
for (const runId of matchedRunIds) {
|
|
1572
|
+
const signal = await Effect.runPromise(signalRun(adapter, runId, signalConfig.name, signalPayload, {
|
|
1573
|
+
correlationId,
|
|
1574
|
+
receivedBy: triggeredBy,
|
|
1575
|
+
}));
|
|
1576
|
+
delivered.push({
|
|
1577
|
+
runId,
|
|
1578
|
+
seq: signal.seq,
|
|
1579
|
+
signalName: signal.signalName,
|
|
1580
|
+
correlationId: signal.correlationId ?? null,
|
|
1581
|
+
receivedAtMs: signal.receivedAtMs,
|
|
1582
|
+
});
|
|
1583
|
+
await this.resumeRunIfNeeded(runId, workflowKey, adapter, auth);
|
|
1584
|
+
}
|
|
1585
|
+
const started = delivered.length === 0 && runEnabled
|
|
1586
|
+
? await this.startRun(workflowKey, normalizeWebhookRunInput(readPathValue(payload, runConfig?.inputPath)), auth)
|
|
1587
|
+
: null;
|
|
1588
|
+
emitGatewayLog("info", "Gateway webhook processed", {
|
|
1589
|
+
workflow: workflowKey,
|
|
1590
|
+
requestId,
|
|
1591
|
+
matchedRunCount: matchedRunIds.length,
|
|
1592
|
+
deliveredCount: delivered.length,
|
|
1593
|
+
...(started ? { startedRunId: started.runId } : {}),
|
|
1594
|
+
}, "gateway:webhook");
|
|
1595
|
+
return respond(200, {
|
|
1596
|
+
ok: true,
|
|
1597
|
+
workflow: workflowKey,
|
|
1598
|
+
verified: true,
|
|
1599
|
+
delivered,
|
|
1600
|
+
matchedRunIds,
|
|
1601
|
+
started,
|
|
1602
|
+
});
|
|
1603
|
+
}
|
|
1604
|
+
catch (error) {
|
|
1605
|
+
if (isSmithersError(error)) {
|
|
1606
|
+
return reject(statusForRpcError(error.code), error.code, error.summary, "invalid_payload", error);
|
|
1607
|
+
}
|
|
1608
|
+
return reject(500, "SERVER_ERROR", error?.message ?? "Gateway webhook failed", "server_error", error);
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
/**
|
|
1612
|
+
* @param {string} key
|
|
1613
|
+
* @param {SmithersWorkflow} workflow
|
|
1614
|
+
* @param {{ schedule?: string; webhook?: GatewayWebhookConfig }} [options]
|
|
1615
|
+
* @returns {this}
|
|
1616
|
+
*/
|
|
1617
|
+
register(key, workflow, options) {
|
|
1618
|
+
ensureSmithersTables(workflow.db);
|
|
1619
|
+
this.workflows.set(key, {
|
|
1620
|
+
key,
|
|
1621
|
+
workflow,
|
|
1622
|
+
schedule: options?.schedule,
|
|
1623
|
+
webhook: options?.webhook,
|
|
1624
|
+
});
|
|
1625
|
+
// Startup recovery: any audit row left in `in_progress` from a prior
|
|
1626
|
+
// crash is flipped to `partial` and the associated run is flagged as
|
|
1627
|
+
// `needs_attention`. Runs asynchronously; failures are logged and
|
|
1628
|
+
// never block registration.
|
|
1629
|
+
const adapter = new SmithersDb(workflow.db);
|
|
1630
|
+
recoverInProgressRewindAudits(adapter).catch((error) => {
|
|
1631
|
+
emitGatewayLog("warning", "rewind audit recovery failed", {
|
|
1632
|
+
workflow: key,
|
|
1633
|
+
...gatewayErrorAnnotations(error),
|
|
1634
|
+
}, "gateway:startup-recovery");
|
|
1635
|
+
});
|
|
1636
|
+
return this;
|
|
1637
|
+
}
|
|
1638
|
+
/**
|
|
1639
|
+
* @param {{ port?: number; host?: string }} [options]
|
|
1640
|
+
*/
|
|
1641
|
+
async listen(options = {}) {
|
|
1642
|
+
if (this.server) {
|
|
1643
|
+
return this.server;
|
|
1644
|
+
}
|
|
1645
|
+
const wsServer = new WebSocketServer({
|
|
1646
|
+
noServer: true,
|
|
1647
|
+
maxPayload: this.maxPayload,
|
|
1648
|
+
});
|
|
1649
|
+
const server = createServer(async (req, res) => {
|
|
1650
|
+
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
1651
|
+
const webhookMatch = url.pathname.match(/^\/webhooks\/([^/]+)$/);
|
|
1652
|
+
if ((req.method ?? "GET") === "GET" && (req.url ?? "/") === "/health") {
|
|
1653
|
+
return sendJson(res, 200, {
|
|
1654
|
+
ok: true,
|
|
1655
|
+
protocol: this.protocol,
|
|
1656
|
+
features: this.features,
|
|
1657
|
+
stateVersion: this.stateVersion,
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
if ((req.method ?? "GET") === "GET" && (req.url ?? "/") === "/metrics") {
|
|
1661
|
+
return sendText(res, 200, renderPrometheusMetrics(), prometheusContentType);
|
|
1662
|
+
}
|
|
1663
|
+
if ((req.method ?? "GET") === "POST" && webhookMatch) {
|
|
1664
|
+
return this.handleWebhook(req, res, decodeURIComponent(webhookMatch[1]));
|
|
1665
|
+
}
|
|
1666
|
+
if ((req.method ?? "GET") === "POST" && (req.url ?? "/") === "/rpc") {
|
|
1667
|
+
return this.handleHttpRpc(req, res);
|
|
1668
|
+
}
|
|
1669
|
+
return sendJson(res, 404, { error: { code: "NOT_FOUND", message: "Route not found" } });
|
|
1670
|
+
});
|
|
1671
|
+
server.on("upgrade", (req, socket, head) => {
|
|
1672
|
+
if (this.connections.size >= this.maxConnections) {
|
|
1673
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
1674
|
+
kind: "connection_limit",
|
|
1675
|
+
transport: "ws",
|
|
1676
|
+
}));
|
|
1677
|
+
emitGatewayLog("warning", "Gateway connection rejected", {
|
|
1678
|
+
transport: "ws",
|
|
1679
|
+
remoteAddress: req.socket.remoteAddress ?? null,
|
|
1680
|
+
maxConnections: this.maxConnections,
|
|
1681
|
+
}, "gateway:connect");
|
|
1682
|
+
const body = "Gateway connection limit reached\n";
|
|
1683
|
+
socket.write("HTTP/1.1 503 Service Unavailable\r\n"
|
|
1684
|
+
+ "Connection: close\r\n"
|
|
1685
|
+
+ "Content-Type: text/plain; charset=utf-8\r\n"
|
|
1686
|
+
+ `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n`
|
|
1687
|
+
+ "\r\n"
|
|
1688
|
+
+ body);
|
|
1689
|
+
socket.destroy();
|
|
1690
|
+
return;
|
|
1691
|
+
}
|
|
1692
|
+
wsServer.handleUpgrade(req, socket, head, (ws) => {
|
|
1693
|
+
this.handleSocket(ws, req);
|
|
1694
|
+
});
|
|
1695
|
+
});
|
|
1696
|
+
await new Promise((resolve) => {
|
|
1697
|
+
if (options.host === undefined) {
|
|
1698
|
+
server.listen(options.port ?? 7331, () => resolve());
|
|
1699
|
+
return;
|
|
1700
|
+
}
|
|
1701
|
+
server.listen(options.port ?? 7331, options.host, () => resolve());
|
|
1702
|
+
});
|
|
1703
|
+
this.server = server;
|
|
1704
|
+
this.wsServer = wsServer;
|
|
1705
|
+
await this.syncRegisteredSchedules();
|
|
1706
|
+
this.startScheduler();
|
|
1707
|
+
return server;
|
|
1708
|
+
}
|
|
1709
|
+
async close() {
|
|
1710
|
+
const activeRuns = [...this.activeRuns.values()];
|
|
1711
|
+
for (const activeRun of activeRuns) {
|
|
1712
|
+
activeRun.abort.abort();
|
|
1713
|
+
}
|
|
1714
|
+
const inflightRuns = [...this.inflightRuns.values()];
|
|
1715
|
+
if (inflightRuns.length > 0) {
|
|
1716
|
+
await Promise.allSettled(inflightRuns);
|
|
1717
|
+
}
|
|
1718
|
+
for (const connection of this.connections) {
|
|
1719
|
+
if (connection.heartbeatTimer) {
|
|
1720
|
+
clearInterval(connection.heartbeatTimer);
|
|
1721
|
+
}
|
|
1722
|
+
try {
|
|
1723
|
+
connection.ws.close();
|
|
1724
|
+
}
|
|
1725
|
+
catch { }
|
|
1726
|
+
}
|
|
1727
|
+
this.connections.clear();
|
|
1728
|
+
if (this.schedulerTimer) {
|
|
1729
|
+
clearInterval(this.schedulerTimer);
|
|
1730
|
+
this.schedulerTimer = null;
|
|
1731
|
+
}
|
|
1732
|
+
if (this.server) {
|
|
1733
|
+
const server = this.server;
|
|
1734
|
+
this.server = null;
|
|
1735
|
+
await new Promise((resolve) => {
|
|
1736
|
+
let settled = false;
|
|
1737
|
+
const done = () => {
|
|
1738
|
+
if (settled) {
|
|
1739
|
+
return;
|
|
1740
|
+
}
|
|
1741
|
+
settled = true;
|
|
1742
|
+
resolve();
|
|
1743
|
+
};
|
|
1744
|
+
const timeout = setTimeout(done, 250);
|
|
1745
|
+
timeout.unref?.();
|
|
1746
|
+
server.close(() => {
|
|
1747
|
+
clearTimeout(timeout);
|
|
1748
|
+
done();
|
|
1749
|
+
});
|
|
1750
|
+
server.closeIdleConnections?.();
|
|
1751
|
+
server.closeAllConnections?.();
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
if (this.wsServer) {
|
|
1755
|
+
this.wsServer.close();
|
|
1756
|
+
this.wsServer = null;
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
startScheduler() {
|
|
1760
|
+
if (this.schedulerTimer) {
|
|
1761
|
+
clearInterval(this.schedulerTimer);
|
|
1762
|
+
}
|
|
1763
|
+
const intervalMs = Math.max(1_000, Math.min(this.heartbeatMs, 15_000));
|
|
1764
|
+
this.schedulerTimer = setInterval(() => {
|
|
1765
|
+
void this.processDueCrons();
|
|
1766
|
+
}, intervalMs);
|
|
1767
|
+
}
|
|
1768
|
+
async syncRegisteredSchedules() {
|
|
1769
|
+
for (const entry of this.workflows.values()) {
|
|
1770
|
+
if (!entry.schedule) {
|
|
1771
|
+
continue;
|
|
1772
|
+
}
|
|
1773
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
1774
|
+
await adapter.upsertCron({
|
|
1775
|
+
cronId: `gateway:${entry.key}`,
|
|
1776
|
+
pattern: entry.schedule,
|
|
1777
|
+
workflowPath: cronWorkflowPath(entry.key),
|
|
1778
|
+
enabled: true,
|
|
1779
|
+
createdAtMs: nowMs(),
|
|
1780
|
+
lastRunAtMs: null,
|
|
1781
|
+
nextRunAtMs: nextCronRunAtMs(entry.schedule),
|
|
1782
|
+
errorJson: null,
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
async processDueCrons() {
|
|
1787
|
+
const now = nowMs();
|
|
1788
|
+
emitGatewayLog("debug", "Gateway cron evaluation tick", {
|
|
1789
|
+
workflowCount: this.workflows.size,
|
|
1790
|
+
}, "gateway:cron");
|
|
1791
|
+
for (const entry of this.workflows.values()) {
|
|
1792
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
1793
|
+
const crons = await adapter.listCrons(true);
|
|
1794
|
+
for (const cron of crons) {
|
|
1795
|
+
const workflowKey = workflowKeyFromCronPath(cron.workflowPath);
|
|
1796
|
+
if (!workflowKey || workflowKey !== entry.key) {
|
|
1797
|
+
continue;
|
|
1798
|
+
}
|
|
1799
|
+
if (typeof cron.nextRunAtMs === "number" && cron.nextRunAtMs > now) {
|
|
1800
|
+
emitGatewayLog("debug", "Gateway cron skipped", {
|
|
1801
|
+
cronId: cron.cronId,
|
|
1802
|
+
workflow: workflowKey,
|
|
1803
|
+
nextRunAtMs: cron.nextRunAtMs,
|
|
1804
|
+
}, "gateway:cron");
|
|
1805
|
+
continue;
|
|
1806
|
+
}
|
|
1807
|
+
try {
|
|
1808
|
+
const run = await this.startRun(workflowKey, {}, {
|
|
1809
|
+
triggeredBy: "cron:gateway",
|
|
1810
|
+
scopes: ["*"],
|
|
1811
|
+
role: "system",
|
|
1812
|
+
});
|
|
1813
|
+
await adapter.updateCronRunTime(cron.cronId, now, nextCronRunAtMs(cron.pattern), null);
|
|
1814
|
+
emitGatewayEffect(incrementMetric(gatewayCronTriggersTotal, {
|
|
1815
|
+
source: "scheduled",
|
|
1816
|
+
}));
|
|
1817
|
+
emitGatewayLog("info", "Gateway cron triggered", {
|
|
1818
|
+
cronId: cron.cronId,
|
|
1819
|
+
workflow: workflowKey,
|
|
1820
|
+
runId: run.runId,
|
|
1821
|
+
}, "gateway:cron");
|
|
1822
|
+
this.broadcastEvent("cron.triggered", {
|
|
1823
|
+
cronId: cron.cronId,
|
|
1824
|
+
workflow: workflowKey,
|
|
1825
|
+
runId: run.runId,
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
1828
|
+
catch (error) {
|
|
1829
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
1830
|
+
kind: "cron",
|
|
1831
|
+
code: gatewayErrorCode(error),
|
|
1832
|
+
}));
|
|
1833
|
+
emitGatewayLog("error", "Gateway cron trigger failed", {
|
|
1834
|
+
cronId: cron.cronId,
|
|
1835
|
+
workflow: workflowKey,
|
|
1836
|
+
...gatewayErrorAnnotations(error),
|
|
1837
|
+
}, "gateway:cron");
|
|
1838
|
+
await adapter.updateCronRunTime(cron.cronId, now, cron.nextRunAtMs ?? now + 60_000, error?.message ?? "cron trigger failed");
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
/**
|
|
1844
|
+
* @param {string} workflowKey
|
|
1845
|
+
* @param {Record<string, unknown>} input
|
|
1846
|
+
* @param {RunStartAuthContext} auth
|
|
1847
|
+
* @param {string} [runId]
|
|
1848
|
+
* @param {{ resume?: boolean }} [options]
|
|
1849
|
+
*/
|
|
1850
|
+
async startRun(workflowKey, input, auth, runId = crypto.randomUUID(), options) {
|
|
1851
|
+
const entry = this.workflows.get(workflowKey);
|
|
1852
|
+
if (!entry) {
|
|
1853
|
+
throw new Error(`Unknown workflow: ${workflowKey}`);
|
|
1854
|
+
}
|
|
1855
|
+
const abort = new AbortController();
|
|
1856
|
+
const record = {
|
|
1857
|
+
workflowKey,
|
|
1858
|
+
workflow: entry.workflow,
|
|
1859
|
+
abort,
|
|
1860
|
+
input,
|
|
1861
|
+
};
|
|
1862
|
+
this.runRegistry.set(runId, record);
|
|
1863
|
+
this.activeRuns.set(runId, record);
|
|
1864
|
+
emitGatewayEffect(Effect.all([
|
|
1865
|
+
incrementMetric(gatewayRunsStartedTotal, {
|
|
1866
|
+
workflow: workflowKey,
|
|
1867
|
+
source: gatewayTriggerSource(auth.triggeredBy),
|
|
1868
|
+
resume: options?.resume ? "true" : "false",
|
|
1869
|
+
}),
|
|
1870
|
+
Effect.logInfo("Gateway run started").pipe(Effect.annotateLogs({
|
|
1871
|
+
workflow: workflowKey,
|
|
1872
|
+
runId,
|
|
1873
|
+
triggeredBy: auth.triggeredBy,
|
|
1874
|
+
source: gatewayTriggerSource(auth.triggeredBy),
|
|
1875
|
+
resume: options?.resume ?? false,
|
|
1876
|
+
...(auth.subscribeConnection
|
|
1877
|
+
? gatewayContextAnnotations(auth.subscribeConnection)
|
|
1878
|
+
: {}),
|
|
1879
|
+
}), Effect.withLogSpan("gateway:run")),
|
|
1880
|
+
], { discard: true }));
|
|
1881
|
+
if (auth.subscribeConnection) {
|
|
1882
|
+
if (!auth.subscribeConnection.subscribedRuns) {
|
|
1883
|
+
auth.subscribeConnection.subscribedRuns = new Set();
|
|
1884
|
+
}
|
|
1885
|
+
auth.subscribeConnection.subscribedRuns.add(runId);
|
|
1886
|
+
}
|
|
1887
|
+
const runPromise = Effect.runPromise(runWorkflow(entry.workflow, {
|
|
1888
|
+
runId,
|
|
1889
|
+
input,
|
|
1890
|
+
resume: options?.resume,
|
|
1891
|
+
signal: abort.signal,
|
|
1892
|
+
onProgress: (event) => this.handleSmithersEvent(event),
|
|
1893
|
+
cliAgentToolsDefault: this.defaults?.cliAgentTools,
|
|
1894
|
+
config: {
|
|
1895
|
+
gatewayWorkflowKey: workflowKey,
|
|
1896
|
+
gatewayTriggeredBy: auth.triggeredBy,
|
|
1897
|
+
},
|
|
1898
|
+
auth: {
|
|
1899
|
+
triggeredBy: auth.triggeredBy,
|
|
1900
|
+
scopes: [...auth.scopes],
|
|
1901
|
+
role: auth.role,
|
|
1902
|
+
createdAt: new Date().toISOString(),
|
|
1903
|
+
},
|
|
1904
|
+
}))
|
|
1905
|
+
.catch((error) => {
|
|
1906
|
+
emitGatewayEffect(Effect.all([
|
|
1907
|
+
incrementMetric(gatewayErrorsTotal, {
|
|
1908
|
+
kind: "run",
|
|
1909
|
+
workflow: workflowKey,
|
|
1910
|
+
code: gatewayErrorCode(error),
|
|
1911
|
+
}),
|
|
1912
|
+
incrementMetric(gatewayRunsCompletedTotal, {
|
|
1913
|
+
workflow: workflowKey,
|
|
1914
|
+
status: "failed",
|
|
1915
|
+
}),
|
|
1916
|
+
Effect.logError("Gateway run failed").pipe(Effect.annotateLogs({
|
|
1917
|
+
workflow: workflowKey,
|
|
1918
|
+
runId,
|
|
1919
|
+
source: gatewayTriggerSource(auth.triggeredBy),
|
|
1920
|
+
...gatewayErrorAnnotations(error),
|
|
1921
|
+
}), Effect.withLogSpan("gateway:run")),
|
|
1922
|
+
], { discard: true }));
|
|
1923
|
+
this.broadcastEvent("run.completed", {
|
|
1924
|
+
runId,
|
|
1925
|
+
status: "failed",
|
|
1926
|
+
error: errorToJson(error),
|
|
1927
|
+
});
|
|
1928
|
+
throw error;
|
|
1929
|
+
})
|
|
1930
|
+
.then((result) => {
|
|
1931
|
+
if (result.status === "finished" || result.status === "failed" || result.status === "cancelled") {
|
|
1932
|
+
emitGatewayEffect(Effect.all([
|
|
1933
|
+
incrementMetric(gatewayRunsCompletedTotal, {
|
|
1934
|
+
workflow: workflowKey,
|
|
1935
|
+
status: result.status,
|
|
1936
|
+
}),
|
|
1937
|
+
Effect.logInfo("Gateway run completed").pipe(Effect.annotateLogs({
|
|
1938
|
+
workflow: workflowKey,
|
|
1939
|
+
runId,
|
|
1940
|
+
status: result.status,
|
|
1941
|
+
source: gatewayTriggerSource(auth.triggeredBy),
|
|
1942
|
+
...(result.error ? { error: result.error } : {}),
|
|
1943
|
+
}), Effect.withLogSpan("gateway:run")),
|
|
1944
|
+
], { discard: true }));
|
|
1945
|
+
this.broadcastEvent("run.completed", {
|
|
1946
|
+
runId,
|
|
1947
|
+
status: result.status,
|
|
1948
|
+
error: result.error,
|
|
1949
|
+
});
|
|
1950
|
+
}
|
|
1951
|
+
})
|
|
1952
|
+
.finally(() => {
|
|
1953
|
+
this.activeRuns.delete(runId);
|
|
1954
|
+
this.inflightRuns.delete(runId);
|
|
1955
|
+
});
|
|
1956
|
+
this.inflightRuns.set(runId, runPromise.then(() => undefined, () => undefined));
|
|
1957
|
+
return { runId, workflow: workflowKey };
|
|
1958
|
+
}
|
|
1959
|
+
/**
|
|
1960
|
+
* @param {string} runId
|
|
1961
|
+
* @param {string} workflowKey
|
|
1962
|
+
* @param {SmithersDb} adapter
|
|
1963
|
+
* @param {RunStartAuthContext} auth
|
|
1964
|
+
*/
|
|
1965
|
+
async resumeRunIfNeeded(runId, workflowKey, adapter, auth) {
|
|
1966
|
+
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
1967
|
+
if (this.activeRuns.has(runId)) {
|
|
1968
|
+
await delay(25);
|
|
1969
|
+
continue;
|
|
1970
|
+
}
|
|
1971
|
+
const run = await adapter.getRun(runId);
|
|
1972
|
+
if (!run) {
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
if (run.status === "finished" || run.status === "failed" || run.status === "cancelled") {
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
await this.startRun(workflowKey, {}, auth, runId, { resume: true });
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
/**
|
|
1983
|
+
* @param {WebSocket} ws
|
|
1984
|
+
* @param {IncomingMessage} req
|
|
1985
|
+
*/
|
|
1986
|
+
handleSocket(ws, req) {
|
|
1987
|
+
const connection = {
|
|
1988
|
+
connectionId: randomUUID(),
|
|
1989
|
+
transport: "ws",
|
|
1990
|
+
ws,
|
|
1991
|
+
seq: 0,
|
|
1992
|
+
authenticated: false,
|
|
1993
|
+
sessionToken: null,
|
|
1994
|
+
role: null,
|
|
1995
|
+
scopes: [],
|
|
1996
|
+
userId: null,
|
|
1997
|
+
subscribedRuns: null,
|
|
1998
|
+
devtoolsStreams: new Map(),
|
|
1999
|
+
heartbeatTimer: null,
|
|
2000
|
+
};
|
|
2001
|
+
this.connections.add(connection);
|
|
2002
|
+
emitGatewayEffect(Effect.all([
|
|
2003
|
+
incrementMetric(gatewayConnectionsTotal, { transport: "ws" }),
|
|
2004
|
+
updateMetric(gatewayConnectionsActive, 1, { transport: "ws" }),
|
|
2005
|
+
], { discard: true }));
|
|
2006
|
+
emitGatewayLog("info", "Gateway connection opened", {
|
|
2007
|
+
...gatewayContextAnnotations(connection),
|
|
2008
|
+
remoteAddress: req.socket.remoteAddress ?? null,
|
|
2009
|
+
activeConnections: this.connections.size,
|
|
2010
|
+
}, "gateway:connect");
|
|
2011
|
+
this.sendEvent(connection, "connect.challenge", {
|
|
2012
|
+
nonce: randomUUID(),
|
|
2013
|
+
ts: nowMs(),
|
|
2014
|
+
});
|
|
2015
|
+
ws.on("message", async (raw) => {
|
|
2016
|
+
this.recordMessageReceived("ws", "request");
|
|
2017
|
+
try {
|
|
2018
|
+
const frame = parseGatewayRequestFrame(raw, this.maxPayload);
|
|
2019
|
+
const response = await this.executeRpc(connection, frame, async () => {
|
|
2020
|
+
if (!connection.authenticated && frame.method !== "connect") {
|
|
2021
|
+
return responseError(frame.id, "UNAUTHORIZED", "Connect first");
|
|
2022
|
+
}
|
|
2023
|
+
if (frame.method === "connect") {
|
|
2024
|
+
return this.handleConnect(connection, req, frame.id, frame.params);
|
|
2025
|
+
}
|
|
2026
|
+
if (!hasScope(connection.scopes, frame.method)) {
|
|
2027
|
+
return responseError(frame.id, "FORBIDDEN", `Missing scope for ${frame.method}`);
|
|
2028
|
+
}
|
|
2029
|
+
return this.routeRequest(connection, frame);
|
|
2030
|
+
});
|
|
2031
|
+
this.sendResponse(connection, response);
|
|
2032
|
+
}
|
|
2033
|
+
catch (error) {
|
|
2034
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
2035
|
+
kind: "frame",
|
|
2036
|
+
transport: "ws",
|
|
2037
|
+
code: gatewayErrorCode(error),
|
|
2038
|
+
}));
|
|
2039
|
+
emitGatewayLog(isSmithersError(error) ? "warning" : "error", "Gateway websocket frame failed", {
|
|
2040
|
+
...gatewayContextAnnotations(connection),
|
|
2041
|
+
...gatewayErrorAnnotations(error),
|
|
2042
|
+
}, "gateway:rpc:invalid");
|
|
2043
|
+
if (isSmithersError(error)) {
|
|
2044
|
+
this.sendResponse(connection, responseError("invalid", error.code, error.summary));
|
|
2045
|
+
return;
|
|
2046
|
+
}
|
|
2047
|
+
this.sendResponse(connection, responseError("server", "SERVER_ERROR", error?.message ?? "Gateway request failed"));
|
|
2048
|
+
}
|
|
2049
|
+
});
|
|
2050
|
+
let cleanedUp = false;
|
|
2051
|
+
/**
|
|
2052
|
+
* @param {"close" | "error"} reason
|
|
2053
|
+
* @param {Record<string, unknown>} [details]
|
|
2054
|
+
*/
|
|
2055
|
+
const cleanup = (reason, details = {}) => {
|
|
2056
|
+
if (cleanedUp) {
|
|
2057
|
+
return;
|
|
2058
|
+
}
|
|
2059
|
+
cleanedUp = true;
|
|
2060
|
+
if (connection.heartbeatTimer) {
|
|
2061
|
+
clearInterval(connection.heartbeatTimer);
|
|
2062
|
+
}
|
|
2063
|
+
this.connections.delete(connection);
|
|
2064
|
+
this.cleanupDevToolsSubscribers(connection);
|
|
2065
|
+
emitGatewayEffect(Effect.all([
|
|
2066
|
+
updateMetric(gatewayConnectionsActive, -1, { transport: "ws" }),
|
|
2067
|
+
incrementMetric(gatewayConnectionsClosedTotal, {
|
|
2068
|
+
transport: "ws",
|
|
2069
|
+
reason,
|
|
2070
|
+
}),
|
|
2071
|
+
...(reason === "error"
|
|
2072
|
+
? [
|
|
2073
|
+
incrementMetric(gatewayErrorsTotal, {
|
|
2074
|
+
kind: "socket",
|
|
2075
|
+
transport: "ws",
|
|
2076
|
+
}),
|
|
2077
|
+
]
|
|
2078
|
+
: []),
|
|
2079
|
+
], { discard: true }));
|
|
2080
|
+
emitGatewayLog(reason === "error" ? "warning" : "info", "Gateway connection closed", {
|
|
2081
|
+
...gatewayContextAnnotations(connection),
|
|
2082
|
+
activeConnections: this.connections.size,
|
|
2083
|
+
closeReason: reason,
|
|
2084
|
+
...details,
|
|
2085
|
+
}, "gateway:connect");
|
|
2086
|
+
};
|
|
2087
|
+
ws.on("close", (code, reason) => {
|
|
2088
|
+
cleanup("close", {
|
|
2089
|
+
closeCode: code,
|
|
2090
|
+
closeMessage: rawDataToUtf8(reason),
|
|
2091
|
+
});
|
|
2092
|
+
});
|
|
2093
|
+
ws.on("error", (error) => {
|
|
2094
|
+
cleanup("error", gatewayErrorAnnotations(error));
|
|
2095
|
+
});
|
|
2096
|
+
}
|
|
2097
|
+
/**
|
|
2098
|
+
* @param {ConnectionState} connection
|
|
2099
|
+
*/
|
|
2100
|
+
startHeartbeat(connection) {
|
|
2101
|
+
if (connection.heartbeatTimer) {
|
|
2102
|
+
clearInterval(connection.heartbeatTimer);
|
|
2103
|
+
}
|
|
2104
|
+
connection.heartbeatTimer = setInterval(() => {
|
|
2105
|
+
emitGatewayEffect(incrementMetric(gatewayHeartbeatTicksTotal));
|
|
2106
|
+
this.sendEvent(connection, "tick", {
|
|
2107
|
+
ts: nowMs(),
|
|
2108
|
+
});
|
|
2109
|
+
}, this.heartbeatMs);
|
|
2110
|
+
}
|
|
2111
|
+
/**
|
|
2112
|
+
* @param {ConnectionState} connection
|
|
2113
|
+
* @param {IncomingMessage} req
|
|
2114
|
+
* @param {string} id
|
|
2115
|
+
* @param {unknown} params
|
|
2116
|
+
* @returns {Promise<ResponseFrame>}
|
|
2117
|
+
*/
|
|
2118
|
+
async handleConnect(connection, req, id, params) {
|
|
2119
|
+
const request = asObject(params);
|
|
2120
|
+
if (!request) {
|
|
2121
|
+
return responseError(id, "INVALID_REQUEST", "Connect params must be an object");
|
|
2122
|
+
}
|
|
2123
|
+
if (typeof request.minProtocol !== "number" ||
|
|
2124
|
+
typeof request.maxProtocol !== "number" ||
|
|
2125
|
+
!request.client) {
|
|
2126
|
+
return responseError(id, "INVALID_REQUEST", "Connect request is missing protocol negotiation fields");
|
|
2127
|
+
}
|
|
2128
|
+
try {
|
|
2129
|
+
assertPositiveFiniteInteger("minProtocol", request.minProtocol);
|
|
2130
|
+
assertPositiveFiniteInteger("maxProtocol", request.maxProtocol);
|
|
2131
|
+
}
|
|
2132
|
+
catch (error) {
|
|
2133
|
+
if (error instanceof SmithersError) {
|
|
2134
|
+
return responseError(id, error.code, error.summary);
|
|
2135
|
+
}
|
|
2136
|
+
throw error;
|
|
2137
|
+
}
|
|
2138
|
+
if (request.minProtocol > this.protocol || request.maxProtocol < this.protocol) {
|
|
2139
|
+
return responseError(id, "PROTOCOL_UNSUPPORTED", `Gateway protocol ${this.protocol} is not supported by the client`);
|
|
2140
|
+
}
|
|
2141
|
+
const authResult = await this.authenticate(req, request);
|
|
2142
|
+
if (authResult.ok === false) {
|
|
2143
|
+
this.recordAuthEvent("ws", "failure", connection, {
|
|
2144
|
+
clientId: request.client.id,
|
|
2145
|
+
clientVersion: request.client.version,
|
|
2146
|
+
authCode: authResult.code,
|
|
2147
|
+
authMessage: authResult.message,
|
|
2148
|
+
});
|
|
2149
|
+
return responseError(id, authResult.code, authResult.message);
|
|
2150
|
+
}
|
|
2151
|
+
connection.authenticated = true;
|
|
2152
|
+
connection.sessionToken = randomUUID();
|
|
2153
|
+
connection.role = authResult.role;
|
|
2154
|
+
connection.scopes = [...authResult.scopes];
|
|
2155
|
+
connection.userId = authResult.userId ?? null;
|
|
2156
|
+
connection.subscribedRuns = Array.isArray(request.subscribe)
|
|
2157
|
+
? new Set(request.subscribe.filter((value) => typeof value === "string"))
|
|
2158
|
+
: null;
|
|
2159
|
+
this.startHeartbeat(connection);
|
|
2160
|
+
this.recordAuthEvent("ws", "success", connection, {
|
|
2161
|
+
clientId: request.client.id,
|
|
2162
|
+
clientVersion: request.client.version,
|
|
2163
|
+
scopeCount: connection.scopes.length,
|
|
2164
|
+
});
|
|
2165
|
+
const hello = {
|
|
2166
|
+
protocol: this.protocol,
|
|
2167
|
+
features: this.features,
|
|
2168
|
+
policy: { heartbeatMs: this.heartbeatMs },
|
|
2169
|
+
auth: {
|
|
2170
|
+
sessionToken: connection.sessionToken,
|
|
2171
|
+
role: authResult.role,
|
|
2172
|
+
scopes: authResult.scopes,
|
|
2173
|
+
userId: authResult.userId ?? null,
|
|
2174
|
+
},
|
|
2175
|
+
snapshot: await this.buildSnapshot(),
|
|
2176
|
+
};
|
|
2177
|
+
return responseOk(id, hello);
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* @param {IncomingMessage} req
|
|
2181
|
+
* @param {ConnectRequest} request
|
|
2182
|
+
* @returns {Promise< | { ok: true; role: string; scopes: string[]; userId?: string } | { ok: false; code: string; message: string } >}
|
|
2183
|
+
*/
|
|
2184
|
+
async authenticate(req, request) {
|
|
2185
|
+
const tokenFromRequest = "token" in (request.auth ?? {}) ? request.auth.token : null;
|
|
2186
|
+
return this.authenticateRequest(req, typeof tokenFromRequest === "string" ? tokenFromRequest : null);
|
|
2187
|
+
}
|
|
2188
|
+
/**
|
|
2189
|
+
* @param {IncomingMessage} req
|
|
2190
|
+
* @param {string | null} token
|
|
2191
|
+
* @returns {Promise< | { ok: true; role: string; scopes: string[]; userId?: string } | { ok: false; code: string; message: string } >}
|
|
2192
|
+
*/
|
|
2193
|
+
async authenticateRequest(req, token) {
|
|
2194
|
+
if (!this.auth) {
|
|
2195
|
+
return {
|
|
2196
|
+
ok: true,
|
|
2197
|
+
role: "operator",
|
|
2198
|
+
scopes: ["*"],
|
|
2199
|
+
};
|
|
2200
|
+
}
|
|
2201
|
+
if (this.auth.mode === "token") {
|
|
2202
|
+
if (!token || typeof token !== "string") {
|
|
2203
|
+
return {
|
|
2204
|
+
ok: false,
|
|
2205
|
+
code: "UNAUTHORIZED",
|
|
2206
|
+
message: "A bearer token is required",
|
|
2207
|
+
};
|
|
2208
|
+
}
|
|
2209
|
+
const grant = this.auth.tokens[token];
|
|
2210
|
+
if (!grant) {
|
|
2211
|
+
return {
|
|
2212
|
+
ok: false,
|
|
2213
|
+
code: "UNAUTHORIZED",
|
|
2214
|
+
message: "Invalid token",
|
|
2215
|
+
};
|
|
2216
|
+
}
|
|
2217
|
+
return {
|
|
2218
|
+
ok: true,
|
|
2219
|
+
role: grant.role,
|
|
2220
|
+
scopes: grant.scopes,
|
|
2221
|
+
userId: grant.userId,
|
|
2222
|
+
};
|
|
2223
|
+
}
|
|
2224
|
+
if (this.auth.mode === "jwt") {
|
|
2225
|
+
if (!token || typeof token !== "string") {
|
|
2226
|
+
return {
|
|
2227
|
+
ok: false,
|
|
2228
|
+
code: "UNAUTHORIZED",
|
|
2229
|
+
message: "A bearer token is required",
|
|
2230
|
+
};
|
|
2231
|
+
}
|
|
2232
|
+
const verified = verifyJwtToken(token, this.auth);
|
|
2233
|
+
if (verified.ok === false) {
|
|
2234
|
+
return {
|
|
2235
|
+
ok: false,
|
|
2236
|
+
code: "UNAUTHORIZED",
|
|
2237
|
+
message: verified.message,
|
|
2238
|
+
};
|
|
2239
|
+
}
|
|
2240
|
+
const scopes = parseJwtScopes(verified.payload[this.auth.scopesClaim ?? "scope"]);
|
|
2241
|
+
const role = asString(verified.payload[this.auth.roleClaim ?? "role"]) ??
|
|
2242
|
+
this.auth.defaultRole ??
|
|
2243
|
+
"operator";
|
|
2244
|
+
const userId = asString(verified.payload[this.auth.userClaim ?? "sub"]);
|
|
2245
|
+
return {
|
|
2246
|
+
ok: true,
|
|
2247
|
+
role,
|
|
2248
|
+
scopes: scopes.length > 0 ? scopes : [...(this.auth.defaultScopes ?? [])],
|
|
2249
|
+
userId: userId ?? undefined,
|
|
2250
|
+
};
|
|
2251
|
+
}
|
|
2252
|
+
if (this.auth.mode === "trusted-proxy") {
|
|
2253
|
+
const allowedOrigins = this.auth.allowedOrigins ?? [];
|
|
2254
|
+
const origin = asString(req.headers.origin);
|
|
2255
|
+
if (allowedOrigins.length > 0 && (!origin || !allowedOrigins.includes(origin))) {
|
|
2256
|
+
return {
|
|
2257
|
+
ok: false,
|
|
2258
|
+
code: "UNAUTHORIZED",
|
|
2259
|
+
message: "Origin is not allowed",
|
|
2260
|
+
};
|
|
2261
|
+
}
|
|
2262
|
+
const [userHeader = "x-user-id", scopesHeader = "x-user-scopes", roleHeader = "x-user-role"] = (this.auth.trustedHeaders ?? []).map((value) => value.toLowerCase());
|
|
2263
|
+
const userId = asString(req.headers[userHeader]);
|
|
2264
|
+
const scopesValue = asString(req.headers[scopesHeader]);
|
|
2265
|
+
const role = asString(req.headers[roleHeader]) ?? this.auth.defaultRole ?? "operator";
|
|
2266
|
+
const scopes = scopesValue
|
|
2267
|
+
? scopesValue.split(/[,\s]+/).map((value) => value.trim()).filter(Boolean)
|
|
2268
|
+
: [...(this.auth.defaultScopes ?? ["*"])];
|
|
2269
|
+
return {
|
|
2270
|
+
ok: true,
|
|
2271
|
+
role,
|
|
2272
|
+
scopes,
|
|
2273
|
+
userId: userId ?? undefined,
|
|
2274
|
+
};
|
|
2275
|
+
}
|
|
2276
|
+
return {
|
|
2277
|
+
ok: false,
|
|
2278
|
+
code: "UNAUTHORIZED",
|
|
2279
|
+
message: "Unsupported auth mode",
|
|
2280
|
+
};
|
|
2281
|
+
}
|
|
2282
|
+
/**
|
|
2283
|
+
* @param {IncomingMessage} req
|
|
2284
|
+
* @param {ServerResponse} res
|
|
2285
|
+
*/
|
|
2286
|
+
async handleHttpRpc(req, res) {
|
|
2287
|
+
const requestId = headerValue(req, "x-request-id") ?? randomUUID();
|
|
2288
|
+
const baseContext = {
|
|
2289
|
+
connectionId: `http:${requestId}`,
|
|
2290
|
+
transport: "http",
|
|
2291
|
+
role: null,
|
|
2292
|
+
scopes: [],
|
|
2293
|
+
userId: null,
|
|
2294
|
+
subscribedRuns: null,
|
|
2295
|
+
devtoolsStreams: null,
|
|
2296
|
+
};
|
|
2297
|
+
let context = baseContext;
|
|
2298
|
+
this.recordMessageReceived("http", "request");
|
|
2299
|
+
try {
|
|
2300
|
+
const authResult = await this.authenticateRequest(req, bearerTokenFromHeaders(req));
|
|
2301
|
+
if (authResult.ok === false) {
|
|
2302
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
2303
|
+
kind: "auth",
|
|
2304
|
+
transport: "http",
|
|
2305
|
+
code: authResult.code,
|
|
2306
|
+
}));
|
|
2307
|
+
this.recordAuthEvent("http", "failure", context, {
|
|
2308
|
+
requestId,
|
|
2309
|
+
authCode: authResult.code,
|
|
2310
|
+
authMessage: authResult.message,
|
|
2311
|
+
}, "warning");
|
|
2312
|
+
const response = responseError(requestId, authResult.code, authResult.message);
|
|
2313
|
+
return this.sendHttpRpcResponse(res, statusForRpcError(authResult.code), response);
|
|
2314
|
+
}
|
|
2315
|
+
context = {
|
|
2316
|
+
...baseContext,
|
|
2317
|
+
role: authResult.role,
|
|
2318
|
+
scopes: [...authResult.scopes],
|
|
2319
|
+
userId: authResult.userId ?? null,
|
|
2320
|
+
};
|
|
2321
|
+
this.recordAuthEvent("http", "success", context, {
|
|
2322
|
+
requestId,
|
|
2323
|
+
scopeCount: authResult.scopes.length,
|
|
2324
|
+
}, "debug");
|
|
2325
|
+
const body = asObject(await readBody(req, this.maxBodyBytes));
|
|
2326
|
+
if (!body) {
|
|
2327
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
2328
|
+
kind: "http",
|
|
2329
|
+
transport: "http",
|
|
2330
|
+
code: "INVALID_REQUEST",
|
|
2331
|
+
}));
|
|
2332
|
+
emitGatewayLog("warning", "Gateway HTTP RPC rejected", {
|
|
2333
|
+
...gatewayContextAnnotations(context),
|
|
2334
|
+
requestId,
|
|
2335
|
+
errorCode: "INVALID_REQUEST",
|
|
2336
|
+
errorMessage: "RPC body must be a JSON object",
|
|
2337
|
+
}, "gateway:http-rpc");
|
|
2338
|
+
return this.sendHttpRpcResponse(res, 400, responseError(requestId, "INVALID_REQUEST", "RPC body must be a JSON object"));
|
|
2339
|
+
}
|
|
2340
|
+
assertJsonPayloadWithinBounds("gateway frame", body, {
|
|
2341
|
+
maxArrayLength: GATEWAY_RPC_MAX_ARRAY_LENGTH,
|
|
2342
|
+
maxDepth: GATEWAY_RPC_MAX_DEPTH,
|
|
2343
|
+
maxStringLength: GATEWAY_RPC_MAX_STRING_LENGTH,
|
|
2344
|
+
});
|
|
2345
|
+
const method = validateGatewayMethodName(body.method);
|
|
2346
|
+
const bodyId = asString(body.id) ?? requestId;
|
|
2347
|
+
assertOptionalStringMaxLength("id", bodyId, GATEWAY_FRAME_ID_MAX_LENGTH);
|
|
2348
|
+
const frame = {
|
|
2349
|
+
type: "req",
|
|
2350
|
+
id: bodyId,
|
|
2351
|
+
method,
|
|
2352
|
+
params: body.params,
|
|
2353
|
+
};
|
|
2354
|
+
const response = await this.executeRpc(context, frame, async () => {
|
|
2355
|
+
if (!hasScope(context.scopes, method)) {
|
|
2356
|
+
return responseError(bodyId, "FORBIDDEN", `Missing scope for ${method}`);
|
|
2357
|
+
}
|
|
2358
|
+
return this.routeRequest(context, frame);
|
|
2359
|
+
});
|
|
2360
|
+
return this.sendHttpRpcResponse(res, response.ok ? 200 : statusForRpcError(response.error?.code), response);
|
|
2361
|
+
}
|
|
2362
|
+
catch (error) {
|
|
2363
|
+
emitGatewayEffect(incrementMetric(gatewayErrorsTotal, {
|
|
2364
|
+
kind: "http",
|
|
2365
|
+
transport: "http",
|
|
2366
|
+
code: gatewayErrorCode(error),
|
|
2367
|
+
}));
|
|
2368
|
+
emitGatewayLog(isSmithersError(error) ? "warning" : "error", "Gateway HTTP RPC failed", {
|
|
2369
|
+
...gatewayContextAnnotations(context),
|
|
2370
|
+
requestId,
|
|
2371
|
+
...gatewayErrorAnnotations(error),
|
|
2372
|
+
}, "gateway:http-rpc");
|
|
2373
|
+
if (isSmithersError(error)) {
|
|
2374
|
+
return this.sendHttpRpcResponse(res, statusForRpcError(error.code), responseError(requestId, error.code, error.summary));
|
|
2375
|
+
}
|
|
2376
|
+
const message = error?.message ?? "Gateway request failed";
|
|
2377
|
+
const status = message.includes("valid JSON") ? 400 : message.includes("exceeds") ? 413 : 500;
|
|
2378
|
+
return this.sendHttpRpcResponse(res, status, responseError(requestId, status === 413 ? "PAYLOAD_TOO_LARGE" : status === 400 ? "INVALID_JSON" : "SERVER_ERROR", message));
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
/**
|
|
2382
|
+
* @param {ConnectionState} connection
|
|
2383
|
+
* @param {ResponseFrame} frame
|
|
2384
|
+
*/
|
|
2385
|
+
sendResponse(connection, frame) {
|
|
2386
|
+
if (connection.ws.readyState !== connection.ws.OPEN) {
|
|
2387
|
+
return;
|
|
2388
|
+
}
|
|
2389
|
+
connection.ws.send(JSON.stringify(frame));
|
|
2390
|
+
this.recordMessageSent("ws", "response", {
|
|
2391
|
+
outcome: frame.ok ? "ok" : "error",
|
|
2392
|
+
});
|
|
2393
|
+
}
|
|
2394
|
+
/**
|
|
2395
|
+
* @param {ConnectionState} connection
|
|
2396
|
+
* @param {string} event
|
|
2397
|
+
* @param {unknown} [payload]
|
|
2398
|
+
*/
|
|
2399
|
+
sendEvent(connection, event, payload, stateVersion = this.stateVersion) {
|
|
2400
|
+
if (connection.ws.readyState !== connection.ws.OPEN) {
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
connection.seq += 1;
|
|
2404
|
+
const frame = {
|
|
2405
|
+
type: "event",
|
|
2406
|
+
event,
|
|
2407
|
+
payload,
|
|
2408
|
+
seq: connection.seq,
|
|
2409
|
+
stateVersion,
|
|
2410
|
+
};
|
|
2411
|
+
connection.ws.send(JSON.stringify(frame));
|
|
2412
|
+
this.recordMessageSent("ws", "event", { event });
|
|
2413
|
+
}
|
|
2414
|
+
/**
|
|
2415
|
+
* @param {string} event
|
|
2416
|
+
* @param {unknown} [payload]
|
|
2417
|
+
*/
|
|
2418
|
+
broadcastEvent(event, payload) {
|
|
2419
|
+
const runId = eventRunId(payload);
|
|
2420
|
+
this.stateVersion += 1;
|
|
2421
|
+
let recipientCount = 0;
|
|
2422
|
+
for (const connection of this.connections) {
|
|
2423
|
+
if (!connection.authenticated || !shouldDeliverEvent(connection, runId)) {
|
|
2424
|
+
continue;
|
|
2425
|
+
}
|
|
2426
|
+
recipientCount += 1;
|
|
2427
|
+
this.sendEvent(connection, event, payload, this.stateVersion);
|
|
2428
|
+
}
|
|
2429
|
+
emitGatewayLog("debug", "Gateway event broadcast", {
|
|
2430
|
+
event,
|
|
2431
|
+
stateVersion: this.stateVersion,
|
|
2432
|
+
recipientCount,
|
|
2433
|
+
...(runId ? { runId } : {}),
|
|
2434
|
+
}, "gateway:broadcast");
|
|
2435
|
+
}
|
|
2436
|
+
async buildSnapshot() {
|
|
2437
|
+
const runs = await this.listRunsAcrossWorkflows(1_000);
|
|
2438
|
+
const approvals = await this.listPendingApprovals();
|
|
2439
|
+
return {
|
|
2440
|
+
runs: runs.filter((run) => ["running", "waiting-approval", "waiting-event", "waiting-timer"].includes(run.status)),
|
|
2441
|
+
approvals,
|
|
2442
|
+
stateVersion: this.stateVersion,
|
|
2443
|
+
};
|
|
2444
|
+
}
|
|
2445
|
+
/**
|
|
2446
|
+
* @param {SmithersWorkflow} workflow
|
|
2447
|
+
* @returns {SmithersDb}
|
|
2448
|
+
*/
|
|
2449
|
+
adapterForWorkflow(workflow) {
|
|
2450
|
+
return new SmithersDb(workflow.db);
|
|
2451
|
+
}
|
|
2452
|
+
/**
|
|
2453
|
+
* @param {string} [status]
|
|
2454
|
+
*/
|
|
2455
|
+
async listRunsAcrossWorkflows(limit = 50, status) {
|
|
2456
|
+
const results = [];
|
|
2457
|
+
for (const entry of this.workflows.values()) {
|
|
2458
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
2459
|
+
const rows = await adapter.listRuns(limit, status);
|
|
2460
|
+
for (const row of rows) {
|
|
2461
|
+
const config = parseJson(row.configJson);
|
|
2462
|
+
results.push({
|
|
2463
|
+
...row,
|
|
2464
|
+
workflowKey: asString(config?.gatewayWorkflowKey) ?? entry.key,
|
|
2465
|
+
});
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
results.sort((a, b) => (b.createdAtMs ?? 0) - (a.createdAtMs ?? 0));
|
|
2469
|
+
return results.slice(0, limit);
|
|
2470
|
+
}
|
|
2471
|
+
async listPendingApprovals() {
|
|
2472
|
+
const approvals = [];
|
|
2473
|
+
for (const entry of this.workflows.values()) {
|
|
2474
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
2475
|
+
const runs = await adapter.listRuns(1_000);
|
|
2476
|
+
for (const run of runs) {
|
|
2477
|
+
const pending = await adapter.listPendingApprovals(run.runId);
|
|
2478
|
+
const nodes = await adapter.listNodes(run.runId);
|
|
2479
|
+
const nodeByKey = new Map();
|
|
2480
|
+
for (const node of nodes) {
|
|
2481
|
+
nodeByKey.set(`${node.nodeId}::${node.iteration ?? 0}`, node);
|
|
2482
|
+
}
|
|
2483
|
+
for (const approval of pending) {
|
|
2484
|
+
const node = nodeByKey.get(`${approval.nodeId}::${approval.iteration ?? 0}`);
|
|
2485
|
+
const request = parseApprovalRequest(parseJson(approval.requestJson), node?.label ?? approval.nodeId);
|
|
2486
|
+
approvals.push({
|
|
2487
|
+
runId: approval.runId,
|
|
2488
|
+
nodeId: approval.nodeId,
|
|
2489
|
+
iteration: approval.iteration ?? 0,
|
|
2490
|
+
requestTitle: request.title ?? node?.label ?? approval.nodeId,
|
|
2491
|
+
requestSummary: request.summary,
|
|
2492
|
+
requestedAtMs: approval.requestedAtMs ?? null,
|
|
2493
|
+
approvalMode: request.mode,
|
|
2494
|
+
options: request.options,
|
|
2495
|
+
allowedScopes: request.allowedScopes,
|
|
2496
|
+
allowedUsers: request.allowedUsers,
|
|
2497
|
+
autoApprove: request.autoApprove,
|
|
2498
|
+
});
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
}
|
|
2502
|
+
approvals.sort((a, b) => (a.requestedAtMs ?? 0) - (b.requestedAtMs ?? 0));
|
|
2503
|
+
return approvals;
|
|
2504
|
+
}
|
|
2505
|
+
async listCrons() {
|
|
2506
|
+
const rows = [];
|
|
2507
|
+
for (const entry of this.workflows.values()) {
|
|
2508
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
2509
|
+
const crons = await adapter.listCrons(false);
|
|
2510
|
+
for (const cron of crons) {
|
|
2511
|
+
const workflowKey = workflowKeyFromCronPath(cron.workflowPath) ?? entry.key;
|
|
2512
|
+
rows.push({
|
|
2513
|
+
...cron,
|
|
2514
|
+
workflow: workflowKey,
|
|
2515
|
+
});
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
rows.sort((a, b) => (a.createdAtMs ?? 0) - (b.createdAtMs ?? 0));
|
|
2519
|
+
return rows;
|
|
2520
|
+
}
|
|
2521
|
+
/**
|
|
2522
|
+
* @param {string} cronId
|
|
2523
|
+
*/
|
|
2524
|
+
async findCron(cronId) {
|
|
2525
|
+
for (const entry of this.workflows.values()) {
|
|
2526
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
2527
|
+
const crons = await adapter.listCrons(false);
|
|
2528
|
+
const match = crons.find((cron) => cron.cronId === cronId);
|
|
2529
|
+
if (match) {
|
|
2530
|
+
return {
|
|
2531
|
+
cron: match,
|
|
2532
|
+
workflowKey: workflowKeyFromCronPath(match.workflowPath) ?? entry.key,
|
|
2533
|
+
adapter,
|
|
2534
|
+
};
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
return null;
|
|
2538
|
+
}
|
|
2539
|
+
/**
|
|
2540
|
+
* @param {string} runId
|
|
2541
|
+
* @returns {Promise<ResolvedRun | null>}
|
|
2542
|
+
*/
|
|
2543
|
+
async resolveRun(runId) {
|
|
2544
|
+
const active = this.runRegistry.get(runId);
|
|
2545
|
+
if (active) {
|
|
2546
|
+
return {
|
|
2547
|
+
workflowKey: active.workflowKey,
|
|
2548
|
+
workflow: active.workflow,
|
|
2549
|
+
adapter: this.adapterForWorkflow(active.workflow),
|
|
2550
|
+
};
|
|
2551
|
+
}
|
|
2552
|
+
for (const entry of this.workflows.values()) {
|
|
2553
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
2554
|
+
const run = await adapter.getRun(runId);
|
|
2555
|
+
if (run) {
|
|
2556
|
+
return {
|
|
2557
|
+
workflowKey: entry.key,
|
|
2558
|
+
workflow: entry.workflow,
|
|
2559
|
+
adapter,
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
return null;
|
|
2564
|
+
}
|
|
2565
|
+
/**
|
|
2566
|
+
* @param {SmithersEvent} event
|
|
2567
|
+
*/
|
|
2568
|
+
handleSmithersEvent(event) {
|
|
2569
|
+
// Invalidate devtools baselines before we broadcast the jump event so
|
|
2570
|
+
// that in-flight streams emit a fresh full snapshot, not a delta rooted
|
|
2571
|
+
// on a stale baseline.
|
|
2572
|
+
if (event.type === "TimeTravelJumped" && typeof event.runId === "string") {
|
|
2573
|
+
this.invalidateDevToolsSubscribersForRun(event.runId);
|
|
2574
|
+
}
|
|
2575
|
+
const mapped = this.mapEvent(event);
|
|
2576
|
+
if (!mapped) {
|
|
2577
|
+
return;
|
|
2578
|
+
}
|
|
2579
|
+
this.broadcastEvent(mapped.event, mapped.payload);
|
|
2580
|
+
}
|
|
2581
|
+
/**
|
|
2582
|
+
* @param {SmithersEvent} event
|
|
2583
|
+
* @returns {{ event: string; payload: unknown } | null}
|
|
2584
|
+
*/
|
|
2585
|
+
mapEvent(event) {
|
|
2586
|
+
switch (event.type) {
|
|
2587
|
+
case "NodeStarted":
|
|
2588
|
+
return {
|
|
2589
|
+
event: "node.started",
|
|
2590
|
+
payload: {
|
|
2591
|
+
runId: event.runId,
|
|
2592
|
+
nodeId: event.nodeId,
|
|
2593
|
+
state: "in-progress",
|
|
2594
|
+
},
|
|
2595
|
+
};
|
|
2596
|
+
case "NodeFinished":
|
|
2597
|
+
return {
|
|
2598
|
+
event: "node.finished",
|
|
2599
|
+
payload: {
|
|
2600
|
+
runId: event.runId,
|
|
2601
|
+
nodeId: event.nodeId,
|
|
2602
|
+
state: "finished",
|
|
2603
|
+
},
|
|
2604
|
+
};
|
|
2605
|
+
case "NodeFailed":
|
|
2606
|
+
return {
|
|
2607
|
+
event: "node.failed",
|
|
2608
|
+
payload: {
|
|
2609
|
+
runId: event.runId,
|
|
2610
|
+
nodeId: event.nodeId,
|
|
2611
|
+
state: "failed",
|
|
2612
|
+
error: event.error,
|
|
2613
|
+
},
|
|
2614
|
+
};
|
|
2615
|
+
case "NodeOutput":
|
|
2616
|
+
return {
|
|
2617
|
+
event: "task.output",
|
|
2618
|
+
payload: {
|
|
2619
|
+
runId: event.runId,
|
|
2620
|
+
nodeId: event.nodeId,
|
|
2621
|
+
output: event.text,
|
|
2622
|
+
stream: event.stream,
|
|
2623
|
+
},
|
|
2624
|
+
};
|
|
2625
|
+
case "ApprovalRequested":
|
|
2626
|
+
return {
|
|
2627
|
+
event: "approval.requested",
|
|
2628
|
+
payload: {
|
|
2629
|
+
runId: event.runId,
|
|
2630
|
+
nodeId: event.nodeId,
|
|
2631
|
+
iteration: event.iteration,
|
|
2632
|
+
},
|
|
2633
|
+
};
|
|
2634
|
+
case "ApprovalGranted":
|
|
2635
|
+
return {
|
|
2636
|
+
event: "approval.decided",
|
|
2637
|
+
payload: {
|
|
2638
|
+
runId: event.runId,
|
|
2639
|
+
nodeId: event.nodeId,
|
|
2640
|
+
iteration: event.iteration,
|
|
2641
|
+
approved: true,
|
|
2642
|
+
},
|
|
2643
|
+
};
|
|
2644
|
+
case "ApprovalAutoApproved":
|
|
2645
|
+
return {
|
|
2646
|
+
event: "approval.auto_approved",
|
|
2647
|
+
payload: {
|
|
2648
|
+
runId: event.runId,
|
|
2649
|
+
nodeId: event.nodeId,
|
|
2650
|
+
iteration: event.iteration,
|
|
2651
|
+
},
|
|
2652
|
+
};
|
|
2653
|
+
case "ApprovalDenied":
|
|
2654
|
+
return {
|
|
2655
|
+
event: "approval.decided",
|
|
2656
|
+
payload: {
|
|
2657
|
+
runId: event.runId,
|
|
2658
|
+
nodeId: event.nodeId,
|
|
2659
|
+
iteration: event.iteration,
|
|
2660
|
+
approved: false,
|
|
2661
|
+
},
|
|
2662
|
+
};
|
|
2663
|
+
case "TaskHeartbeat":
|
|
2664
|
+
return {
|
|
2665
|
+
event: "task.heartbeat",
|
|
2666
|
+
payload: {
|
|
2667
|
+
runId: event.runId,
|
|
2668
|
+
nodeId: event.nodeId,
|
|
2669
|
+
iteration: event.iteration,
|
|
2670
|
+
attempt: event.attempt,
|
|
2671
|
+
},
|
|
2672
|
+
};
|
|
2673
|
+
case "TimeTravelJumped":
|
|
2674
|
+
return {
|
|
2675
|
+
event: "run.time_travel_jumped",
|
|
2676
|
+
payload: {
|
|
2677
|
+
runId: event.runId,
|
|
2678
|
+
fromFrameNo: event.fromFrameNo,
|
|
2679
|
+
toFrameNo: event.toFrameNo,
|
|
2680
|
+
timestampMs: event.timestampMs,
|
|
2681
|
+
caller: event.caller ?? null,
|
|
2682
|
+
},
|
|
2683
|
+
};
|
|
2684
|
+
case "RunFinished":
|
|
2685
|
+
return {
|
|
2686
|
+
event: "run.completed",
|
|
2687
|
+
payload: {
|
|
2688
|
+
runId: event.runId,
|
|
2689
|
+
status: "finished",
|
|
2690
|
+
},
|
|
2691
|
+
};
|
|
2692
|
+
case "RunFailed":
|
|
2693
|
+
return {
|
|
2694
|
+
event: "run.completed",
|
|
2695
|
+
payload: {
|
|
2696
|
+
runId: event.runId,
|
|
2697
|
+
status: "failed",
|
|
2698
|
+
error: event.error,
|
|
2699
|
+
},
|
|
2700
|
+
};
|
|
2701
|
+
case "RunCancelled":
|
|
2702
|
+
return {
|
|
2703
|
+
event: "run.completed",
|
|
2704
|
+
payload: {
|
|
2705
|
+
runId: event.runId,
|
|
2706
|
+
status: "cancelled",
|
|
2707
|
+
},
|
|
2708
|
+
};
|
|
2709
|
+
default:
|
|
2710
|
+
return null;
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
/**
|
|
2714
|
+
* @param {GatewayRequestContext} connection
|
|
2715
|
+
* @param {RequestFrame} frame
|
|
2716
|
+
* @returns {Promise<ResponseFrame>}
|
|
2717
|
+
*/
|
|
2718
|
+
async routeRequest(connection, frame) {
|
|
2719
|
+
const params = asObject(frame.params) ?? {};
|
|
2720
|
+
switch (frame.method) {
|
|
2721
|
+
case "health":
|
|
2722
|
+
return responseOk(frame.id, {
|
|
2723
|
+
protocol: this.protocol,
|
|
2724
|
+
features: this.features,
|
|
2725
|
+
stateVersion: this.stateVersion,
|
|
2726
|
+
uptimeMs: nowMs() - this.startedAtMs,
|
|
2727
|
+
});
|
|
2728
|
+
case "runs.list": {
|
|
2729
|
+
const limit = asOptionalPositiveInt(params.limit, "limit") ?? 50;
|
|
2730
|
+
const status = asString(params.status);
|
|
2731
|
+
return responseOk(frame.id, await this.listRunsAcrossWorkflows(limit, status));
|
|
2732
|
+
}
|
|
2733
|
+
case "runs.create": {
|
|
2734
|
+
const workflowKey = asString(params.workflow);
|
|
2735
|
+
if (!workflowKey) {
|
|
2736
|
+
return responseError(frame.id, "INVALID_REQUEST", "workflow is required");
|
|
2737
|
+
}
|
|
2738
|
+
if (!this.workflows.has(workflowKey)) {
|
|
2739
|
+
return responseError(frame.id, "NOT_FOUND", `Unknown workflow: ${workflowKey}`);
|
|
2740
|
+
}
|
|
2741
|
+
let input;
|
|
2742
|
+
try {
|
|
2743
|
+
input = validateGatewayRpcInput(params.input);
|
|
2744
|
+
}
|
|
2745
|
+
catch (error) {
|
|
2746
|
+
if (isSmithersError(error)) {
|
|
2747
|
+
return responseError(frame.id, error.code, error.summary);
|
|
2748
|
+
}
|
|
2749
|
+
throw error;
|
|
2750
|
+
}
|
|
2751
|
+
return responseOk(frame.id, await this.startRun(workflowKey, input, {
|
|
2752
|
+
triggeredBy: connection.userId ?? "gateway",
|
|
2753
|
+
scopes: [...connection.scopes],
|
|
2754
|
+
role: connection.role ?? "operator",
|
|
2755
|
+
subscribeConnection: connection,
|
|
2756
|
+
}, asString(params.runId) ?? crypto.randomUUID(), { resume: false }));
|
|
2757
|
+
}
|
|
2758
|
+
case "runs.get": {
|
|
2759
|
+
const runId = asString(params.runId);
|
|
2760
|
+
if (!runId) {
|
|
2761
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
2762
|
+
}
|
|
2763
|
+
const resolved = await this.resolveRun(runId);
|
|
2764
|
+
if (!resolved) {
|
|
2765
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2766
|
+
}
|
|
2767
|
+
const run = await resolved.adapter.getRun(runId);
|
|
2768
|
+
if (!run) {
|
|
2769
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2770
|
+
}
|
|
2771
|
+
const summary = await resolved.adapter.countNodesByState(runId);
|
|
2772
|
+
const runState = await computeRunStateFromRow(
|
|
2773
|
+
resolved.adapter,
|
|
2774
|
+
run,
|
|
2775
|
+
).catch(() => undefined);
|
|
2776
|
+
return responseOk(frame.id, {
|
|
2777
|
+
...run,
|
|
2778
|
+
workflowKey: resolved.workflowKey,
|
|
2779
|
+
summary: summary.reduce((acc, row) => {
|
|
2780
|
+
acc[row.state] = row.count;
|
|
2781
|
+
return acc;
|
|
2782
|
+
}, {}),
|
|
2783
|
+
...(runState ? { runState } : {}),
|
|
2784
|
+
});
|
|
2785
|
+
}
|
|
2786
|
+
case "frames.list": {
|
|
2787
|
+
const runId = asString(params.runId);
|
|
2788
|
+
if (!runId) {
|
|
2789
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
2790
|
+
}
|
|
2791
|
+
const resolved = await this.resolveRun(runId);
|
|
2792
|
+
if (!resolved) {
|
|
2793
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2794
|
+
}
|
|
2795
|
+
const limit = asOptionalPositiveInt(params.limit, "limit") ?? 50;
|
|
2796
|
+
const afterFrameNo = asOptionalPositiveInt(params.afterFrameNo, "afterFrameNo");
|
|
2797
|
+
return responseOk(frame.id, await resolved.adapter.listFrames(runId, limit, afterFrameNo));
|
|
2798
|
+
}
|
|
2799
|
+
case "frames.get": {
|
|
2800
|
+
const runId = asString(params.runId);
|
|
2801
|
+
if (!runId) {
|
|
2802
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
2803
|
+
}
|
|
2804
|
+
const resolved = await this.resolveRun(runId);
|
|
2805
|
+
if (!resolved) {
|
|
2806
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2807
|
+
}
|
|
2808
|
+
const frameNo = asOptionalPositiveInt(params.frameNo, "frameNo");
|
|
2809
|
+
const frameRow = frameNo === undefined
|
|
2810
|
+
? await resolved.adapter.getLastFrame(runId)
|
|
2811
|
+
: (await resolved.adapter.listFrames(runId, Math.max(frameNo + 1, 50))).find((entry) => entry.frameNo === frameNo);
|
|
2812
|
+
if (!frameRow) {
|
|
2813
|
+
return responseError(frame.id, "NOT_FOUND", "Frame not found");
|
|
2814
|
+
}
|
|
2815
|
+
return responseOk(frame.id, frameRow);
|
|
2816
|
+
}
|
|
2817
|
+
case "attempts.list": {
|
|
2818
|
+
const runId = asString(params.runId);
|
|
2819
|
+
if (!runId) {
|
|
2820
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
2821
|
+
}
|
|
2822
|
+
const resolved = await this.resolveRun(runId);
|
|
2823
|
+
if (!resolved) {
|
|
2824
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2825
|
+
}
|
|
2826
|
+
const nodeId = asString(params.nodeId);
|
|
2827
|
+
if (nodeId) {
|
|
2828
|
+
const iteration = asNumber(params.iteration) ?? 0;
|
|
2829
|
+
return responseOk(frame.id, await resolved.adapter.listAttempts(runId, nodeId, iteration));
|
|
2830
|
+
}
|
|
2831
|
+
return responseOk(frame.id, await resolved.adapter.listAttemptsForRun(runId));
|
|
2832
|
+
}
|
|
2833
|
+
case "attempts.get": {
|
|
2834
|
+
const runId = asString(params.runId);
|
|
2835
|
+
const nodeId = asString(params.nodeId);
|
|
2836
|
+
const iteration = asNumber(params.iteration);
|
|
2837
|
+
const attempt = asNumber(params.attempt);
|
|
2838
|
+
if (!runId || !nodeId || iteration === undefined || attempt === undefined) {
|
|
2839
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId, nodeId, iteration, and attempt are required");
|
|
2840
|
+
}
|
|
2841
|
+
const resolved = await this.resolveRun(runId);
|
|
2842
|
+
if (!resolved) {
|
|
2843
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
2844
|
+
}
|
|
2845
|
+
const row = await resolved.adapter.getAttempt(runId, nodeId, iteration, attempt);
|
|
2846
|
+
if (!row) {
|
|
2847
|
+
return responseError(frame.id, "NOT_FOUND", "Attempt not found");
|
|
2848
|
+
}
|
|
2849
|
+
return responseOk(frame.id, row);
|
|
2850
|
+
}
|
|
2851
|
+
case "getNodeOutput":
|
|
2852
|
+
case "devtools.getNodeOutput": {
|
|
2853
|
+
try {
|
|
2854
|
+
const payload = await getNodeOutputRoute({
|
|
2855
|
+
runId: params.runId,
|
|
2856
|
+
nodeId: params.nodeId,
|
|
2857
|
+
iteration: params.iteration,
|
|
2858
|
+
resolveRun: this.resolveRun.bind(this),
|
|
2859
|
+
});
|
|
2860
|
+
return responseOk(frame.id, payload);
|
|
2861
|
+
}
|
|
2862
|
+
catch (error) {
|
|
2863
|
+
if (error instanceof NodeOutputRouteError) {
|
|
2864
|
+
return responseError(frame.id, error.code, error.message);
|
|
2865
|
+
}
|
|
2866
|
+
throw error;
|
|
2867
|
+
}
|
|
2868
|
+
}
|
|
2869
|
+
case "getNodeDiff":
|
|
2870
|
+
case "devtools.getNodeDiff": {
|
|
2871
|
+
const result = await runPromise(Effect.promise(() => getNodeDiffRoute({
|
|
2872
|
+
runId: params.runId,
|
|
2873
|
+
nodeId: params.nodeId,
|
|
2874
|
+
iteration: params.iteration,
|
|
2875
|
+
resolveRun: this.resolveRun.bind(this),
|
|
2876
|
+
})).pipe(Effect.withLogSpan("devtools.getNodeDiff")));
|
|
2877
|
+
if (!result.ok) {
|
|
2878
|
+
return responseError(frame.id, result.error.code, result.error.message);
|
|
2879
|
+
}
|
|
2880
|
+
return responseOk(frame.id, result.payload);
|
|
2881
|
+
}
|
|
2882
|
+
case "getDevToolsSnapshot": {
|
|
2883
|
+
const runId = asString(params.runId);
|
|
2884
|
+
if (!runId) {
|
|
2885
|
+
return responseError(frame.id, "InvalidRunId", "runId is required");
|
|
2886
|
+
}
|
|
2887
|
+
try {
|
|
2888
|
+
// Full route-level validation runs at the gateway boundary
|
|
2889
|
+
// before any DB lookup. Malformed inputs never reach
|
|
2890
|
+
// resolveRun() or the adapter.
|
|
2891
|
+
validateRunId(runId);
|
|
2892
|
+
validateFrameNoInput(params.frameNo);
|
|
2893
|
+
if (!this.isDevToolsRunAuthorized(connection, runId)) {
|
|
2894
|
+
return responseError(frame.id, "Unauthorized", "Connection is not subscribed to this runId.");
|
|
2895
|
+
}
|
|
2896
|
+
const resolved = await this.resolveRun(runId);
|
|
2897
|
+
if (!resolved) {
|
|
2898
|
+
return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
|
|
2899
|
+
}
|
|
2900
|
+
const payload = await getDevToolsSnapshotRoute({
|
|
2901
|
+
adapter: resolved.adapter,
|
|
2902
|
+
runId,
|
|
2903
|
+
frameNo: params.frameNo,
|
|
2904
|
+
onWarning: (warning) => {
|
|
2905
|
+
emitGatewayLog("warning", "devtools snapshot serializer warning", {
|
|
2906
|
+
runId,
|
|
2907
|
+
code: warning.code,
|
|
2908
|
+
path: warning.path,
|
|
2909
|
+
}, "gateway:devtools");
|
|
2910
|
+
},
|
|
2911
|
+
});
|
|
2912
|
+
return responseOk(frame.id, payload);
|
|
2913
|
+
}
|
|
2914
|
+
catch (error) {
|
|
2915
|
+
if (error instanceof DevToolsRouteError) {
|
|
2916
|
+
return responseError(frame.id, error.code, error.message);
|
|
2917
|
+
}
|
|
2918
|
+
throw error;
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
case "streamDevTools": {
|
|
2922
|
+
if (connection.transport !== "ws" || !connection.ws) {
|
|
2923
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
2924
|
+
return responseError(frame.id, "INVALID_REQUEST", "streamDevTools is only supported over websocket connections");
|
|
2925
|
+
}
|
|
2926
|
+
const runId = asString(params.runId);
|
|
2927
|
+
if (!runId) {
|
|
2928
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
2929
|
+
return responseError(frame.id, "InvalidRunId", "runId is required");
|
|
2930
|
+
}
|
|
2931
|
+
const fromSeq = params.fromSeq;
|
|
2932
|
+
const streamId = randomUUID();
|
|
2933
|
+
try {
|
|
2934
|
+
// Full route-level validation at the gateway boundary so
|
|
2935
|
+
// malformed numeric inputs never reach resolveRun() or
|
|
2936
|
+
// getLastFrame() below.
|
|
2937
|
+
validateRunId(runId);
|
|
2938
|
+
validateFromSeqInput(fromSeq);
|
|
2939
|
+
if (!this.isDevToolsRunAuthorized(connection, runId)) {
|
|
2940
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
2941
|
+
return responseError(frame.id, "Unauthorized", "Connection is not subscribed to this runId.");
|
|
2942
|
+
}
|
|
2943
|
+
const resolved = await this.resolveRun(runId);
|
|
2944
|
+
if (!resolved) {
|
|
2945
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
2946
|
+
return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
|
|
2947
|
+
}
|
|
2948
|
+
if (typeof fromSeq === "number") {
|
|
2949
|
+
const latestFrame = await resolved.adapter.getLastFrame(runId);
|
|
2950
|
+
// Zero-frame runs: current seq is 0. fromSeq > 0 is in
|
|
2951
|
+
// the future relative to current seq and must reject.
|
|
2952
|
+
const latestSeq = latestFrame?.frameNo ?? 0;
|
|
2953
|
+
if (fromSeq > latestSeq) {
|
|
2954
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
2955
|
+
return responseError(frame.id, "SeqOutOfRange", `fromSeq ${fromSeq} is newer than current seq ${latestSeq}`);
|
|
2956
|
+
}
|
|
2957
|
+
}
|
|
2958
|
+
const abort = this.registerDevToolsSubscriber(connection, streamId, runId);
|
|
2959
|
+
emitGatewayLog("info", "devtools stream subscribed", {
|
|
2960
|
+
runId,
|
|
2961
|
+
fromSeq: typeof fromSeq === "number" ? fromSeq : null,
|
|
2962
|
+
streamId,
|
|
2963
|
+
subscriberId: connection.connectionId,
|
|
2964
|
+
}, "gateway:devtools");
|
|
2965
|
+
// Per-subscriber outbound queue gated on actual WS send
|
|
2966
|
+
// pressure. If the WS socket has buffered > limit bytes,
|
|
2967
|
+
// we queue locally up to 1000 events; exceeding that is a
|
|
2968
|
+
// BackpressureDisconnect that tears down only this stream.
|
|
2969
|
+
const outboundQueue = [];
|
|
2970
|
+
const OUTBOUND_QUEUE_LIMIT = 1_000;
|
|
2971
|
+
const WS_BUFFERED_HIGH_WATER_BYTES = 8 * 1024 * 1024;
|
|
2972
|
+
let flushPending = false;
|
|
2973
|
+
const drainOutboundQueue = () => {
|
|
2974
|
+
if (flushPending) return;
|
|
2975
|
+
flushPending = true;
|
|
2976
|
+
queueMicrotask(() => {
|
|
2977
|
+
try {
|
|
2978
|
+
while (outboundQueue.length > 0 && connection.ws.readyState === connection.ws.OPEN) {
|
|
2979
|
+
const ws = connection.ws;
|
|
2980
|
+
if (typeof ws.bufferedAmount === "number" && ws.bufferedAmount > WS_BUFFERED_HIGH_WATER_BYTES) {
|
|
2981
|
+
setTimeout(() => {
|
|
2982
|
+
flushPending = false;
|
|
2983
|
+
drainOutboundQueue();
|
|
2984
|
+
}, 10);
|
|
2985
|
+
return;
|
|
2986
|
+
}
|
|
2987
|
+
const payload = outboundQueue.shift();
|
|
2988
|
+
if (!payload) continue;
|
|
2989
|
+
this.sendEvent(connection, "devtools.event", payload);
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
finally {
|
|
2993
|
+
flushPending = false;
|
|
2994
|
+
}
|
|
2995
|
+
});
|
|
2996
|
+
};
|
|
2997
|
+
/**
|
|
2998
|
+
* Send a devtools event through the outbound queue. Applies
|
|
2999
|
+
* WS-level backpressure detection and raises a typed
|
|
3000
|
+
* BackpressureDisconnect if the queue overflows.
|
|
3001
|
+
*/
|
|
3002
|
+
const enqueueDevToolsEvent = (payload) => {
|
|
3003
|
+
if (outboundQueue.length >= OUTBOUND_QUEUE_LIMIT) {
|
|
3004
|
+
throw new DevToolsRouteError(
|
|
3005
|
+
"BackpressureDisconnect",
|
|
3006
|
+
`Subscriber outbound queue exceeded ${OUTBOUND_QUEUE_LIMIT} events.`,
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
outboundQueue.push(payload);
|
|
3010
|
+
drainOutboundQueue();
|
|
3011
|
+
};
|
|
3012
|
+
void (async () => {
|
|
3013
|
+
let eventsDelivered = 0;
|
|
3014
|
+
try {
|
|
3015
|
+
for await (const event of streamDevToolsRoute({
|
|
3016
|
+
adapter: resolved.adapter,
|
|
3017
|
+
runId,
|
|
3018
|
+
fromSeq: typeof fromSeq === "number" ? fromSeq : undefined,
|
|
3019
|
+
subscriberId: connection.connectionId,
|
|
3020
|
+
signal: abort.signal,
|
|
3021
|
+
invalidateSnapshot: () => {
|
|
3022
|
+
if (this.devtoolsInvalidateFlags.has(streamId)) {
|
|
3023
|
+
this.devtoolsInvalidateFlags.delete(streamId);
|
|
3024
|
+
return true;
|
|
3025
|
+
}
|
|
3026
|
+
return false;
|
|
3027
|
+
},
|
|
3028
|
+
onWarning: (warning) => {
|
|
3029
|
+
emitGatewayLog("warning", "devtools snapshot serializer warning", {
|
|
3030
|
+
runId,
|
|
3031
|
+
code: warning.code,
|
|
3032
|
+
path: warning.path,
|
|
3033
|
+
}, "gateway:devtools");
|
|
3034
|
+
},
|
|
3035
|
+
onLog: (level, message, fields) => {
|
|
3036
|
+
emitGatewayLog(level === "warn" ? "warning" : level, message, fields, "gateway:devtools");
|
|
3037
|
+
},
|
|
3038
|
+
onEvent: (event, stats) => {
|
|
3039
|
+
const kind = event.kind;
|
|
3040
|
+
emitGatewayEffect(Effect.all([
|
|
3041
|
+
Metric.increment(taggedMetric(devtoolsEventTotal, { kind })),
|
|
3042
|
+
Metric.update(devtoolsEventBytes, stats.bytes),
|
|
3043
|
+
...(kind === "snapshot"
|
|
3044
|
+
? [Metric.update(devtoolsSnapshotBuildMs, stats.durationMs)]
|
|
3045
|
+
: [Metric.update(devtoolsDeltaBuildMs, stats.durationMs)]),
|
|
3046
|
+
], { discard: true }));
|
|
3047
|
+
},
|
|
3048
|
+
onClose: ({ errorCode }) => {
|
|
3049
|
+
if (errorCode === "BackpressureDisconnect") {
|
|
3050
|
+
emitGatewayEffect(Metric.increment(devtoolsBackpressureDisconnectTotal));
|
|
3051
|
+
}
|
|
3052
|
+
},
|
|
3053
|
+
})) {
|
|
3054
|
+
eventsDelivered += 1;
|
|
3055
|
+
enqueueDevToolsEvent({
|
|
3056
|
+
streamId,
|
|
3057
|
+
runId,
|
|
3058
|
+
event,
|
|
3059
|
+
});
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
catch (error) {
|
|
3063
|
+
const code = error?.code ?? "SERVER_ERROR";
|
|
3064
|
+
if (code === "BackpressureDisconnect") {
|
|
3065
|
+
emitGatewayEffect(Metric.increment(devtoolsBackpressureDisconnectTotal));
|
|
3066
|
+
}
|
|
3067
|
+
emitGatewayLog("error", "devtools stream failed", {
|
|
3068
|
+
runId,
|
|
3069
|
+
streamId,
|
|
3070
|
+
code,
|
|
3071
|
+
message: error?.message ?? "stream failed",
|
|
3072
|
+
}, "gateway:devtools");
|
|
3073
|
+
// Notify ONLY the offending subscriber. The WS
|
|
3074
|
+
// stays open so other subscribers on the same
|
|
3075
|
+
// connection continue to receive events.
|
|
3076
|
+
this.sendEvent(connection, "devtools.error", {
|
|
3077
|
+
streamId,
|
|
3078
|
+
runId,
|
|
3079
|
+
error: {
|
|
3080
|
+
code,
|
|
3081
|
+
message: error?.message ?? "stream failed",
|
|
3082
|
+
},
|
|
3083
|
+
});
|
|
3084
|
+
}
|
|
3085
|
+
finally {
|
|
3086
|
+
this.unregisterDevToolsSubscriber(connection, streamId, {
|
|
3087
|
+
runId,
|
|
3088
|
+
streamId,
|
|
3089
|
+
eventsDelivered,
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
})();
|
|
3093
|
+
return responseOk(frame.id, {
|
|
3094
|
+
streamId,
|
|
3095
|
+
runId,
|
|
3096
|
+
fromSeq: typeof fromSeq === "number" ? fromSeq : null,
|
|
3097
|
+
});
|
|
3098
|
+
}
|
|
3099
|
+
catch (error) {
|
|
3100
|
+
this.recordDevToolsSubscribeAttempt("error");
|
|
3101
|
+
if (error instanceof DevToolsRouteError) {
|
|
3102
|
+
return responseError(frame.id, error.code, error.message);
|
|
3103
|
+
}
|
|
3104
|
+
throw error;
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
case "jumpToFrame":
|
|
3108
|
+
case "devtools.jumpToFrame": {
|
|
3109
|
+
const runId = asString(params.runId);
|
|
3110
|
+
if (!runId) {
|
|
3111
|
+
return responseError(frame.id, "InvalidRunId", "runId is required");
|
|
3112
|
+
}
|
|
3113
|
+
const frameNo = asNumber(params.frameNo);
|
|
3114
|
+
if (frameNo === undefined) {
|
|
3115
|
+
return responseError(frame.id, "InvalidFrameNo", "frameNo is required");
|
|
3116
|
+
}
|
|
3117
|
+
const confirm = asBoolean(params.confirm);
|
|
3118
|
+
const resolved = await this.resolveRun(runId);
|
|
3119
|
+
if (!resolved) {
|
|
3120
|
+
return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
|
|
3121
|
+
}
|
|
3122
|
+
const run = await resolved.adapter.getRun(runId);
|
|
3123
|
+
if (!run) {
|
|
3124
|
+
return responseError(frame.id, "RunNotFound", `Run not found: ${runId}`);
|
|
3125
|
+
}
|
|
3126
|
+
const ownerId = resolveRunOwnerId(run);
|
|
3127
|
+
const isAdmin = (connection.role ?? "").toLowerCase() === "admin";
|
|
3128
|
+
const isOwner = Boolean(ownerId && connection.userId && ownerId === connection.userId);
|
|
3129
|
+
if (!isAdmin && !isOwner) {
|
|
3130
|
+
// Record the unauthorized attempt so the audit log contains
|
|
3131
|
+
// every rewind request — successful or not.
|
|
3132
|
+
try {
|
|
3133
|
+
await writeRewindAuditRow(resolved.adapter, {
|
|
3134
|
+
runId,
|
|
3135
|
+
fromFrameNo: -1,
|
|
3136
|
+
toFrameNo: Number.isInteger(frameNo) ? frameNo : -1,
|
|
3137
|
+
caller: connection.userId ?? "gateway",
|
|
3138
|
+
timestampMs: nowMs(),
|
|
3139
|
+
result: "failed",
|
|
3140
|
+
durationMs: 0,
|
|
3141
|
+
});
|
|
3142
|
+
}
|
|
3143
|
+
catch (auditError) {
|
|
3144
|
+
emitGatewayLog("warning", "Gateway jumpToFrame unauthorized audit-write failed", {
|
|
3145
|
+
runId,
|
|
3146
|
+
...gatewayErrorAnnotations(auditError),
|
|
3147
|
+
}, "gateway:jump-to-frame");
|
|
3148
|
+
}
|
|
3149
|
+
return responseError(frame.id, "Unauthorized", "Only the run owner or an admin may rewind this run.");
|
|
3150
|
+
}
|
|
3151
|
+
const active = this.activeRuns.get(runId);
|
|
3152
|
+
try {
|
|
3153
|
+
const payload = await jumpToFrameRoute({
|
|
3154
|
+
adapter: resolved.adapter,
|
|
3155
|
+
runId,
|
|
3156
|
+
frameNo,
|
|
3157
|
+
confirm,
|
|
3158
|
+
caller: connection.userId ?? "gateway",
|
|
3159
|
+
pauseRunLoop: async () => {
|
|
3160
|
+
if (!active) {
|
|
3161
|
+
return;
|
|
3162
|
+
}
|
|
3163
|
+
active.abort.abort();
|
|
3164
|
+
const timeoutAt = Date.now() + 10_000;
|
|
3165
|
+
while (this.activeRuns.has(runId) && Date.now() < timeoutAt) {
|
|
3166
|
+
await delay(25);
|
|
3167
|
+
}
|
|
3168
|
+
// Hard stop: if the task is still live after the grace
|
|
3169
|
+
// window, abort the rewind rather than mutating the DB
|
|
3170
|
+
// underneath a running task.
|
|
3171
|
+
if (this.activeRuns.has(runId)) {
|
|
3172
|
+
throw new JumpToFrameError("RewindFailed", `Run ${runId} did not stop within 10s of abort; refusing to rewind a live task.`, {
|
|
3173
|
+
details: { runId, stage: "pause" },
|
|
3174
|
+
});
|
|
3175
|
+
}
|
|
3176
|
+
},
|
|
3177
|
+
resumeRunLoop: async () => {
|
|
3178
|
+
await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
|
|
3179
|
+
triggeredBy: connection.userId ?? "gateway",
|
|
3180
|
+
scopes: [...connection.scopes],
|
|
3181
|
+
role: connection.role ?? "operator",
|
|
3182
|
+
subscribeConnection: connection.transport === "ws" ? connection : undefined,
|
|
3183
|
+
});
|
|
3184
|
+
},
|
|
3185
|
+
emitEvent: async (event) => {
|
|
3186
|
+
this.handleSmithersEvent(event);
|
|
3187
|
+
},
|
|
3188
|
+
onLog: async (level, message, fields) => {
|
|
3189
|
+
emitGatewayLog(level === "warn" ? "warning" : level, message, {
|
|
3190
|
+
runId,
|
|
3191
|
+
frameNo,
|
|
3192
|
+
caller: connection.userId ?? "gateway",
|
|
3193
|
+
...fields,
|
|
3194
|
+
}, "gateway:jump-to-frame");
|
|
3195
|
+
},
|
|
3196
|
+
});
|
|
3197
|
+
return responseOk(frame.id, payload);
|
|
3198
|
+
}
|
|
3199
|
+
catch (error) {
|
|
3200
|
+
if (error instanceof JumpToFrameError) {
|
|
3201
|
+
return responseError(frame.id, error.code, error.message);
|
|
3202
|
+
}
|
|
3203
|
+
throw error;
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
case "runs.diff": {
|
|
3207
|
+
const leftRunId = asString(params.leftRunId);
|
|
3208
|
+
const rightRunId = asString(params.rightRunId);
|
|
3209
|
+
if (!leftRunId || !rightRunId) {
|
|
3210
|
+
return responseError(frame.id, "INVALID_REQUEST", "leftRunId and rightRunId are required");
|
|
3211
|
+
}
|
|
3212
|
+
const left = await this.resolveRun(leftRunId);
|
|
3213
|
+
const right = await this.resolveRun(rightRunId);
|
|
3214
|
+
if (!left || !right) {
|
|
3215
|
+
return responseError(frame.id, "NOT_FOUND", "Both runs must exist");
|
|
3216
|
+
}
|
|
3217
|
+
const leftSnapshot = await loadLatestSnapshot(left.adapter, leftRunId);
|
|
3218
|
+
const rightSnapshot = await loadLatestSnapshot(right.adapter, rightRunId);
|
|
3219
|
+
if (!leftSnapshot || !rightSnapshot) {
|
|
3220
|
+
return responseError(frame.id, "NOT_FOUND", "Snapshots not found for both runs");
|
|
3221
|
+
}
|
|
3222
|
+
return responseOk(frame.id, diffRawSnapshots(leftSnapshot, rightSnapshot));
|
|
3223
|
+
}
|
|
3224
|
+
case "approvals.list":
|
|
3225
|
+
return responseOk(frame.id, await this.listPendingApprovals());
|
|
3226
|
+
case "approvals.decide": {
|
|
3227
|
+
const runId = asString(params.runId);
|
|
3228
|
+
const nodeId = asString(params.nodeId);
|
|
3229
|
+
const approved = asBoolean(params.approved);
|
|
3230
|
+
const iteration = asNumber(params.iteration) ?? 0;
|
|
3231
|
+
if (!runId || !nodeId || approved === undefined) {
|
|
3232
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId, nodeId, and approved are required");
|
|
3233
|
+
}
|
|
3234
|
+
const resolved = await this.resolveRun(runId);
|
|
3235
|
+
if (!resolved) {
|
|
3236
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
3237
|
+
}
|
|
3238
|
+
const approval = await resolved.adapter.getApproval(runId, nodeId, iteration);
|
|
3239
|
+
const request = parseApprovalRequest(parseJson(typeof approval?.requestJson === "string" ? approval.requestJson : null), nodeId);
|
|
3240
|
+
if (request.allowedUsers.length > 0 &&
|
|
3241
|
+
(!connection.userId || !request.allowedUsers.includes(connection.userId))) {
|
|
3242
|
+
return responseError(frame.id, "FORBIDDEN", "User is not allowed to decide this approval");
|
|
3243
|
+
}
|
|
3244
|
+
if (request.allowedScopes.length > 0 &&
|
|
3245
|
+
!request.allowedScopes.some((scope) => hasScope(connection.scopes, scope))) {
|
|
3246
|
+
return responseError(frame.id, "FORBIDDEN", "Connection is missing required approval scope");
|
|
3247
|
+
}
|
|
3248
|
+
const decision = params.decision;
|
|
3249
|
+
if (approved) {
|
|
3250
|
+
const validation = validateApprovalDecision(request, decision);
|
|
3251
|
+
if (!validation.ok) {
|
|
3252
|
+
return responseError(frame.id, validation.code, validation.message);
|
|
3253
|
+
}
|
|
3254
|
+
}
|
|
3255
|
+
if (approved) {
|
|
3256
|
+
await Effect.runPromise(approveNode(resolved.adapter, runId, nodeId, iteration, asString(params.note), connection.userId ?? undefined, decision));
|
|
3257
|
+
}
|
|
3258
|
+
else {
|
|
3259
|
+
await Effect.runPromise(denyNode(resolved.adapter, runId, nodeId, iteration, asString(params.note), connection.userId ?? undefined, decision));
|
|
3260
|
+
}
|
|
3261
|
+
await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
|
|
3262
|
+
triggeredBy: connection.userId ?? "gateway",
|
|
3263
|
+
scopes: [...connection.scopes],
|
|
3264
|
+
role: connection.role ?? "operator",
|
|
3265
|
+
subscribeConnection: connection,
|
|
3266
|
+
});
|
|
3267
|
+
return responseOk(frame.id, { runId, nodeId, iteration, approved });
|
|
3268
|
+
}
|
|
3269
|
+
case "signals.send": {
|
|
3270
|
+
const runId = asString(params.runId);
|
|
3271
|
+
const signalName = asString(params.signalName);
|
|
3272
|
+
if (!runId || !signalName) {
|
|
3273
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId and signalName are required");
|
|
3274
|
+
}
|
|
3275
|
+
const resolved = await this.resolveRun(runId);
|
|
3276
|
+
if (!resolved) {
|
|
3277
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
3278
|
+
}
|
|
3279
|
+
const delivered = await Effect.runPromise(signalRun(resolved.adapter, runId, signalName, params.data ?? {}, {
|
|
3280
|
+
correlationId: asString(params.correlationId),
|
|
3281
|
+
receivedBy: connection.userId,
|
|
3282
|
+
}));
|
|
3283
|
+
await this.resumeRunIfNeeded(runId, resolved.workflowKey, resolved.adapter, {
|
|
3284
|
+
triggeredBy: connection.userId ?? "gateway",
|
|
3285
|
+
scopes: [...connection.scopes],
|
|
3286
|
+
role: connection.role ?? "operator",
|
|
3287
|
+
subscribeConnection: connection,
|
|
3288
|
+
});
|
|
3289
|
+
return responseOk(frame.id, delivered);
|
|
3290
|
+
}
|
|
3291
|
+
case "runs.cancel": {
|
|
3292
|
+
const runId = asString(params.runId);
|
|
3293
|
+
if (!runId) {
|
|
3294
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
3295
|
+
}
|
|
3296
|
+
const active = this.activeRuns.get(runId);
|
|
3297
|
+
if (!active) {
|
|
3298
|
+
return responseError(frame.id, "RUN_NOT_ACTIVE", "Run is not currently active");
|
|
3299
|
+
}
|
|
3300
|
+
active.abort.abort();
|
|
3301
|
+
return responseOk(frame.id, { runId, status: "cancelling" });
|
|
3302
|
+
}
|
|
3303
|
+
case "runs.rerun": {
|
|
3304
|
+
const runId = asString(params.runId);
|
|
3305
|
+
if (!runId) {
|
|
3306
|
+
return responseError(frame.id, "INVALID_REQUEST", "runId is required");
|
|
3307
|
+
}
|
|
3308
|
+
const resolved = await this.resolveRun(runId);
|
|
3309
|
+
if (!resolved) {
|
|
3310
|
+
return responseError(frame.id, "NOT_FOUND", `Run not found: ${runId}`);
|
|
3311
|
+
}
|
|
3312
|
+
const client = (resolved.workflow.db.session?.client ?? resolved.workflow.db.$client);
|
|
3313
|
+
const row = client?.query?.("SELECT payload FROM input WHERE run_id = ? LIMIT 1").get(runId);
|
|
3314
|
+
const input = typeof row?.payload === "string"
|
|
3315
|
+
? parseJson(row.payload) ?? {}
|
|
3316
|
+
: row?.payload ?? {};
|
|
3317
|
+
return this.routeRequest(connection, {
|
|
3318
|
+
type: "req",
|
|
3319
|
+
id: frame.id,
|
|
3320
|
+
method: "runs.create",
|
|
3321
|
+
params: {
|
|
3322
|
+
workflow: resolved.workflowKey,
|
|
3323
|
+
input,
|
|
3324
|
+
runId: asString(params.newRunId),
|
|
3325
|
+
},
|
|
3326
|
+
});
|
|
3327
|
+
}
|
|
3328
|
+
case "cron.list":
|
|
3329
|
+
return responseOk(frame.id, await this.listCrons());
|
|
3330
|
+
case "cron.add": {
|
|
3331
|
+
const workflowKey = asString(params.workflow);
|
|
3332
|
+
const pattern = asString(params.pattern);
|
|
3333
|
+
if (!workflowKey || !pattern) {
|
|
3334
|
+
return responseError(frame.id, "INVALID_REQUEST", "workflow and pattern are required");
|
|
3335
|
+
}
|
|
3336
|
+
const entry = this.workflows.get(workflowKey);
|
|
3337
|
+
if (!entry) {
|
|
3338
|
+
return responseError(frame.id, "NOT_FOUND", `Unknown workflow: ${workflowKey}`);
|
|
3339
|
+
}
|
|
3340
|
+
const cronId = asString(params.cronId) ?? randomUUID();
|
|
3341
|
+
const adapter = this.adapterForWorkflow(entry.workflow);
|
|
3342
|
+
const row = {
|
|
3343
|
+
cronId,
|
|
3344
|
+
pattern,
|
|
3345
|
+
workflowPath: cronWorkflowPath(workflowKey),
|
|
3346
|
+
enabled: asBoolean(params.enabled) ?? true,
|
|
3347
|
+
createdAtMs: nowMs(),
|
|
3348
|
+
lastRunAtMs: null,
|
|
3349
|
+
nextRunAtMs: nextCronRunAtMs(pattern),
|
|
3350
|
+
errorJson: null,
|
|
3351
|
+
};
|
|
3352
|
+
await adapter.upsertCron(row);
|
|
3353
|
+
return responseOk(frame.id, {
|
|
3354
|
+
...row,
|
|
3355
|
+
workflow: workflowKey,
|
|
3356
|
+
});
|
|
3357
|
+
}
|
|
3358
|
+
case "cron.remove": {
|
|
3359
|
+
const cronId = asString(params.cronId);
|
|
3360
|
+
if (!cronId) {
|
|
3361
|
+
return responseError(frame.id, "INVALID_REQUEST", "cronId is required");
|
|
3362
|
+
}
|
|
3363
|
+
const resolvedCron = await this.findCron(cronId);
|
|
3364
|
+
if (!resolvedCron) {
|
|
3365
|
+
return responseError(frame.id, "NOT_FOUND", `Cron not found: ${cronId}`);
|
|
3366
|
+
}
|
|
3367
|
+
await resolvedCron.adapter.deleteCron(cronId);
|
|
3368
|
+
return responseOk(frame.id, { cronId, removed: true });
|
|
3369
|
+
}
|
|
3370
|
+
case "cron.trigger": {
|
|
3371
|
+
const cronId = asString(params.cronId);
|
|
3372
|
+
const workflowKey = asString(params.workflow);
|
|
3373
|
+
const resolvedCron = cronId ? await this.findCron(cronId) : null;
|
|
3374
|
+
const targetWorkflowKey = resolvedCron?.workflowKey ?? workflowKey;
|
|
3375
|
+
if (!targetWorkflowKey) {
|
|
3376
|
+
return responseError(frame.id, "INVALID_REQUEST", "cronId or workflow is required");
|
|
3377
|
+
}
|
|
3378
|
+
if (resolvedCron) {
|
|
3379
|
+
await resolvedCron.adapter.updateCronRunTime(resolvedCron.cron.cronId, nowMs(), nextCronRunAtMs(resolvedCron.cron.pattern), null);
|
|
3380
|
+
}
|
|
3381
|
+
let input;
|
|
3382
|
+
try {
|
|
3383
|
+
input = validateGatewayRpcInput(params.input);
|
|
3384
|
+
}
|
|
3385
|
+
catch (error) {
|
|
3386
|
+
if (isSmithersError(error)) {
|
|
3387
|
+
return responseError(frame.id, error.code, error.summary);
|
|
3388
|
+
}
|
|
3389
|
+
throw error;
|
|
3390
|
+
}
|
|
3391
|
+
return responseOk(frame.id, await this.startRun(targetWorkflowKey, input, {
|
|
3392
|
+
triggeredBy: connection.userId ?? "gateway",
|
|
3393
|
+
scopes: [...connection.scopes],
|
|
3394
|
+
role: connection.role ?? "operator",
|
|
3395
|
+
subscribeConnection: connection,
|
|
3396
|
+
}, undefined, { resume: false }));
|
|
3397
|
+
}
|
|
3398
|
+
default:
|
|
3399
|
+
return responseError(frame.id, "METHOD_NOT_FOUND", `Unknown method: ${frame.method}`);
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
}
|