@multiplier-labs/stepflow 0.3.1 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +146 -0
- package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +74 -0
- package/.codekin/reports/comments/2026-04-04_comment-assessment.md +179 -0
- package/.codekin/reports/comments/2026-04-11_comment-assessment.md +161 -0
- package/.codekin/reports/comments/2026-04-14_comment-assessment.md +34 -0
- package/.codekin/reports/complexity/2026-04-03_complexity-report.md +152 -0
- package/.codekin/reports/complexity/2026-04-11_complexity-report.md +152 -0
- package/.codekin/reports/complexity/2026-04-14_complexity-report.md +34 -0
- package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +94 -0
- package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +93 -0
- package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +10 -0
- package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +138 -0
- package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +121 -0
- package/.codekin/reports/security/2026-04-03_security-audit.md +208 -0
- package/.codekin/reports/security/2026-04-11_security-audit.md +297 -0
- package/.github/CODEOWNERS +15 -0
- package/.github/dependabot.yml +23 -0
- package/.github/workflows/ci.yml +38 -0
- package/.github/workflows/publish.yml +17 -5
- package/.github/workflows/security.yml +29 -0
- package/CHANGELOG.md +34 -0
- package/README.md +24 -0
- package/SECURITY.md +40 -0
- package/dist/{chunk-FQCT3I2A.js → chunk-5XJBAPXF.js} +527 -551
- package/dist/chunk-5XJBAPXF.js.map +1 -0
- package/dist/{chunk-U5QGN3RT.js → chunk-FMPJBK2N.js} +67 -55
- package/dist/chunk-FMPJBK2N.js.map +1 -0
- package/dist/{chunk-GXY66V7Q.js → chunk-JPXPB7U5.js} +52 -2
- package/dist/chunk-JPXPB7U5.js.map +1 -0
- package/dist/{chunk-XCF4PLN7.js → chunk-U3PCVK6O.js} +276 -61
- package/dist/chunk-U3PCVK6O.js.map +1 -0
- package/dist/events/index.d.ts +60 -7
- package/dist/events/index.js +1 -1
- package/dist/{index-CPfGsmrp.d.ts → index-Cd0IZ40L.d.ts} +56 -17
- package/dist/index.d.ts +15 -23
- package/dist/index.js +784 -547
- package/dist/index.js.map +1 -1
- package/dist/scheduler/index.d.ts +4 -4
- package/dist/scheduler/index.js +2 -2
- package/dist/storage/index.d.ts +322 -56
- package/dist/storage/index.js +4 -2
- package/dist/{types-BlwFEGzj.d.ts → types-DSEjiLMj.d.ts} +16 -15
- package/dist/{types-DzkEqnQJ.d.ts → types-DZUdWmCc.d.ts} +2 -2
- package/dist/{types-BVShq7zT.d.ts → types-K5Gjk3H_.d.ts} +4 -4
- package/docs/stepflow-api-reference.md +5 -4
- package/package.json +12 -8
- package/src/core/engine/concurrency.ts +229 -0
- package/src/core/engine/lifecycle.ts +176 -0
- package/src/core/engine.test.ts +536 -522
- package/src/core/engine.ts +116 -235
- package/src/core/index.ts +3 -3
- package/src/core/orchestrator/persistence.ts +165 -0
- package/src/core/orchestrator/plan-resolution.ts +140 -0
- package/src/core/orchestrator/state-transitions.ts +109 -0
- package/src/core/orchestrator/step-runner.ts +427 -0
- package/src/core/orchestrator.ts +57 -395
- package/src/core/types.ts +7 -29
- package/src/events/index.ts +4 -4
- package/src/events/memory.test.ts +39 -41
- package/src/events/memory.ts +7 -10
- package/src/events/socketio.test.ts +112 -162
- package/src/events/socketio.ts +20 -27
- package/src/events/types.ts +16 -19
- package/src/events/webhook.test.ts +489 -235
- package/src/events/webhook.ts +355 -139
- package/src/index.ts +19 -19
- package/src/planning/evaluate-condition.test.ts +153 -0
- package/src/planning/evaluate-condition.ts +105 -0
- package/src/planning/index.ts +6 -3
- package/src/planning/planner.ts +45 -152
- package/src/planning/planning.test.ts +303 -366
- package/src/planning/registry.test.ts +204 -251
- package/src/planning/registry.ts +20 -106
- package/src/planning/types.ts +23 -27
- package/src/scheduler/cron.test.ts +149 -150
- package/src/scheduler/cron.ts +58 -69
- package/src/scheduler/index.ts +4 -14
- package/src/scheduler/postgres-persistence.ts +89 -105
- package/src/scheduler/sqlite-persistence.test.ts +96 -96
- package/src/scheduler/sqlite-persistence.ts +40 -22
- package/src/scheduler/types.ts +4 -9
- package/src/storage/index.ts +5 -5
- package/src/storage/memory.test.ts +133 -183
- package/src/storage/memory.ts +21 -42
- package/src/storage/postgres-core.ts +550 -0
- package/src/storage/postgres-migration.test.ts +69 -0
- package/src/storage/postgres-transaction-pagination.test.ts +107 -0
- package/src/storage/postgres.ts +222 -937
- package/src/storage/sqlite-core.ts +151 -0
- package/src/storage/sqlite.test.ts +330 -281
- package/src/storage/sqlite.ts +225 -170
- package/src/storage/types.ts +33 -58
- package/src/utils/errors.test.ts +78 -80
- package/src/utils/errors.ts +27 -57
- package/src/utils/id.ts +3 -3
- package/src/utils/index.ts +6 -4
- package/src/utils/logger.test.ts +52 -58
- package/src/utils/logger.ts +4 -4
- package/src/utils/pinned-fetch.test.ts +84 -0
- package/src/utils/pinned-fetch.ts +118 -0
- package/src/utils/postgres-deps.ts +20 -4
- package/src/utils/retry.test.ts +47 -67
- package/src/utils/retry.ts +9 -9
- package/src/utils/safe-json.test.ts +166 -0
- package/src/utils/safe-json.ts +107 -0
- package/src/utils/webhook-signature.test.ts +111 -0
- package/src/utils/webhook-signature.ts +74 -0
- package/dist/chunk-FQCT3I2A.js.map +0 -1
- package/dist/chunk-GXY66V7Q.js.map +0 -1
- package/dist/chunk-U5QGN3RT.js.map +0 -1
- package/dist/chunk-XCF4PLN7.js.map +0 -1
- package/src/scheduler/postgres-persistence.test.ts +0 -906
package/src/events/webhook.ts
CHANGED
|
@@ -5,15 +5,11 @@
|
|
|
5
5
|
* Supports multiple webhooks with optional filtering by event type.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { promises as dns } from
|
|
9
|
-
import
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
WorkflowEvent,
|
|
14
|
-
WorkflowEventType,
|
|
15
|
-
} from "./types";
|
|
16
|
-
import type { Logger } from "../core/types";
|
|
8
|
+
import { promises as dns } from 'node:dns';
|
|
9
|
+
import { isIPv4, isIPv6 } from 'node:net';
|
|
10
|
+
import { pinnedFetch } from '../utils/pinned-fetch';
|
|
11
|
+
import type { EventTransport, EventCallback, Unsubscribe, WorkflowEvent, WorkflowEventType } from './types';
|
|
12
|
+
import type { Logger } from '../core/types';
|
|
17
13
|
|
|
18
14
|
/**
|
|
19
15
|
* Webhook endpoint configuration.
|
|
@@ -84,8 +80,27 @@ export interface WebhookEventTransportConfig {
|
|
|
84
80
|
*/
|
|
85
81
|
maxConcurrentRequests?: number;
|
|
86
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Maximum number of queued webhook deliveries waiting for a concurrency slot
|
|
85
|
+
* (default: 1000, env override: `STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH`).
|
|
86
|
+
*
|
|
87
|
+
* When the queue is full, incoming `emit()` deliveries are dropped to
|
|
88
|
+
* protect the process from memory exhaustion under burst traffic. Drops
|
|
89
|
+
* are logged via `logger.warn` with the endpoint id, current queue depth,
|
|
90
|
+
* and a recommended `retryAfterMs`. A value of `0` disables the cap (not
|
|
91
|
+
* recommended in production).
|
|
92
|
+
*/
|
|
93
|
+
maxQueueDepth?: number;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Suggested back-off in milliseconds reported on dropped deliveries
|
|
97
|
+
* (default: 1000). Surfaced in the warning log so external orchestrators
|
|
98
|
+
* (sidecars, sweeper jobs) can respect it like an HTTP `Retry-After`.
|
|
99
|
+
*/
|
|
100
|
+
queueDropRetryAfterMs?: number;
|
|
101
|
+
|
|
87
102
|
/** Logger for webhook transport errors (default: console-based) */
|
|
88
|
-
logger?: import(
|
|
103
|
+
logger?: import('../core/types').Logger;
|
|
89
104
|
}
|
|
90
105
|
|
|
91
106
|
/**
|
|
@@ -97,7 +112,7 @@ export interface WebhookPayload {
|
|
|
97
112
|
* The `Date` is converted to a string before delivery so that JSON consumers
|
|
98
113
|
* receive a portable, timezone-aware representation without needing Date parsing.
|
|
99
114
|
*/
|
|
100
|
-
event: Omit<WorkflowEvent,
|
|
115
|
+
event: Omit<WorkflowEvent, 'timestamp'> & { timestamp: string };
|
|
101
116
|
/** ISO 8601 timestamp of when this payload was delivered. */
|
|
102
117
|
deliveredAt: string;
|
|
103
118
|
/** ID of the webhook endpoint that received this delivery. */
|
|
@@ -136,13 +151,29 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
136
151
|
private defaultRetries: number;
|
|
137
152
|
private retryDelay: number;
|
|
138
153
|
private fetchFn: typeof fetch;
|
|
154
|
+
/**
|
|
155
|
+
* Whether `fetchFn` was caller-supplied. When `false` the default fetch is
|
|
156
|
+
* replaced with `pinnedFetch` so the TCP connection can target the IP that
|
|
157
|
+
* was just validated (closes the DNS-rebinding TOCTOU window). When `true`
|
|
158
|
+
* the custom fetch is used as-is so test injection still works.
|
|
159
|
+
*/
|
|
160
|
+
private hasCustomFetch: boolean;
|
|
139
161
|
private allowInsecureUrls: boolean;
|
|
140
162
|
private maxPayloadBytes: number;
|
|
141
163
|
private maxConcurrentRequests: number;
|
|
164
|
+
private maxQueueDepth: number;
|
|
165
|
+
private queueDropRetryAfterMs: number;
|
|
142
166
|
private activeRequests = 0;
|
|
143
167
|
private requestQueue: Array<() => void> = [];
|
|
168
|
+
private droppedRequests = 0;
|
|
144
169
|
private logger: Logger;
|
|
145
170
|
|
|
171
|
+
// DNS resolution cache with TTL — stores `family` so the pin can reuse it
|
|
172
|
+
// without a second lookup.
|
|
173
|
+
private dnsCache = new Map<string, { address: string; family: 4 | 6; expiresAt: number }>();
|
|
174
|
+
private static readonly DNS_CACHE_TTL_MS = 30_000; // 30 seconds
|
|
175
|
+
private static readonly DNS_TIMEOUT_MS = 5_000; // 5 seconds
|
|
176
|
+
|
|
146
177
|
// Local subscribers for server-side subscriptions
|
|
147
178
|
private runSubscribers = new Map<string, Set<EventCallback>>();
|
|
148
179
|
private globalSubscribers = new Set<EventCallback>();
|
|
@@ -151,19 +182,30 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
151
182
|
this.defaultTimeout = config.defaultTimeout ?? 5000;
|
|
152
183
|
this.defaultRetries = config.defaultRetries ?? 3;
|
|
153
184
|
this.retryDelay = config.retryDelay ?? 1000;
|
|
185
|
+
this.hasCustomFetch = config.fetchFn !== undefined;
|
|
154
186
|
this.fetchFn = config.fetchFn ?? globalThis.fetch;
|
|
155
187
|
this.allowInsecureUrls = config.allowInsecureUrls ?? false;
|
|
156
188
|
this.maxPayloadBytes = config.maxPayloadBytes ?? 1_048_576; // 1 MB
|
|
157
189
|
this.maxConcurrentRequests = config.maxConcurrentRequests ?? 50;
|
|
190
|
+
if (this.maxConcurrentRequests <= 0) {
|
|
191
|
+
throw new Error('maxConcurrentRequests must be greater than 0');
|
|
192
|
+
}
|
|
193
|
+
this.maxQueueDepth = resolveMaxQueueDepth(config.maxQueueDepth);
|
|
194
|
+
if (this.maxQueueDepth < 0) {
|
|
195
|
+
throw new Error('maxQueueDepth must be >= 0 (0 disables the cap)');
|
|
196
|
+
}
|
|
197
|
+
this.queueDropRetryAfterMs = config.queueDropRetryAfterMs ?? 1000;
|
|
158
198
|
this.logger = config.logger ?? {
|
|
159
199
|
debug() {},
|
|
160
200
|
info() {},
|
|
161
201
|
warn() {},
|
|
162
|
-
error(message: string, ...args: unknown[]) {
|
|
163
|
-
console.error(message, ...args);
|
|
164
|
-
},
|
|
202
|
+
error(message: string, ...args: unknown[]) { console.error(message, ...args); },
|
|
165
203
|
};
|
|
166
204
|
|
|
205
|
+
if (this.allowInsecureUrls) {
|
|
206
|
+
this.logger.warn('WebhookEventTransport: allowInsecureUrls is enabled. Webhook payloads will be sent over HTTP. Do not use this setting in production.');
|
|
207
|
+
}
|
|
208
|
+
|
|
167
209
|
// Register initial endpoints (validates URLs)
|
|
168
210
|
if (config.endpoints) {
|
|
169
211
|
for (const endpoint of config.endpoints) {
|
|
@@ -217,7 +259,7 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
217
259
|
try {
|
|
218
260
|
callback(event);
|
|
219
261
|
} catch (error) {
|
|
220
|
-
this.logger.error(
|
|
262
|
+
this.logger.error('Event callback error:', error);
|
|
221
263
|
}
|
|
222
264
|
}
|
|
223
265
|
}
|
|
@@ -226,7 +268,7 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
226
268
|
try {
|
|
227
269
|
callback(event);
|
|
228
270
|
} catch (error) {
|
|
229
|
-
this.logger.error(
|
|
271
|
+
this.logger.error('Event callback error:', error);
|
|
230
272
|
}
|
|
231
273
|
}
|
|
232
274
|
|
|
@@ -235,14 +277,43 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
235
277
|
if (!endpoint.enabled) continue;
|
|
236
278
|
if (!this.matchesFilter(event, endpoint)) continue;
|
|
237
279
|
|
|
238
|
-
this.enqueueRequest(
|
|
239
|
-
|
|
240
|
-
this.
|
|
241
|
-
|
|
280
|
+
const accepted = this.enqueueRequest(
|
|
281
|
+
() =>
|
|
282
|
+
this.sendWebhook(endpoint, event).catch((error) => {
|
|
283
|
+
this.logger.error(`Webhook ${endpoint.id} failed:`, error);
|
|
284
|
+
}),
|
|
285
|
+
endpoint.id
|
|
242
286
|
);
|
|
287
|
+
|
|
288
|
+
if (!accepted) {
|
|
289
|
+
// Queue overflow — equivalent to a server returning 429 with Retry-After.
|
|
290
|
+
// The delivery is dropped to protect the process from memory exhaustion.
|
|
291
|
+
this.droppedRequests++;
|
|
292
|
+
this.logger.warn(
|
|
293
|
+
`Webhook ${endpoint.id} delivery dropped: queue full ` +
|
|
294
|
+
`(depth ${this.requestQueue.length}/${this.maxQueueDepth}, ` +
|
|
295
|
+
`active ${this.activeRequests}/${this.maxConcurrentRequests}, ` +
|
|
296
|
+
`total drops ${this.droppedRequests}, retryAfterMs ${this.queueDropRetryAfterMs})`
|
|
297
|
+
);
|
|
298
|
+
}
|
|
243
299
|
}
|
|
244
300
|
}
|
|
245
301
|
|
|
302
|
+
/**
|
|
303
|
+
* Number of webhook deliveries dropped due to queue overflow since startup.
|
|
304
|
+
* Useful for surfacing back-pressure to operators (metrics/health checks).
|
|
305
|
+
*/
|
|
306
|
+
getDroppedRequestCount(): number {
|
|
307
|
+
return this.droppedRequests;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Current depth of the pending request queue.
|
|
312
|
+
*/
|
|
313
|
+
getQueueDepth(): number {
|
|
314
|
+
return this.requestQueue.length;
|
|
315
|
+
}
|
|
316
|
+
|
|
246
317
|
/**
|
|
247
318
|
* Subscribe to events for a specific run (server-side).
|
|
248
319
|
*/
|
|
@@ -284,10 +355,7 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
284
355
|
/**
|
|
285
356
|
* Check if an event matches an endpoint's filters.
|
|
286
357
|
*/
|
|
287
|
-
private matchesFilter(
|
|
288
|
-
event: WorkflowEvent,
|
|
289
|
-
endpoint: WebhookEndpoint,
|
|
290
|
-
): boolean {
|
|
358
|
+
private matchesFilter(event: WorkflowEvent, endpoint: WebhookEndpoint): boolean {
|
|
291
359
|
// Check event type filter
|
|
292
360
|
if (endpoint.eventTypes && endpoint.eventTypes.length > 0) {
|
|
293
361
|
if (!endpoint.eventTypes.includes(event.eventType)) {
|
|
@@ -308,10 +376,7 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
308
376
|
/**
|
|
309
377
|
* Send a webhook with retry support.
|
|
310
378
|
*/
|
|
311
|
-
private async sendWebhook(
|
|
312
|
-
endpoint: WebhookEndpoint,
|
|
313
|
-
event: WorkflowEvent,
|
|
314
|
-
): Promise<void> {
|
|
379
|
+
private async sendWebhook(endpoint: WebhookEndpoint, event: WorkflowEvent): Promise<void> {
|
|
315
380
|
const timeout = endpoint.timeout ?? this.defaultTimeout;
|
|
316
381
|
const maxRetries = endpoint.retries ?? this.defaultRetries;
|
|
317
382
|
|
|
@@ -329,39 +394,50 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
329
394
|
// Enforce payload size limit to prevent excessive memory use and network abuse
|
|
330
395
|
if (body.length > this.maxPayloadBytes) {
|
|
331
396
|
throw new Error(
|
|
332
|
-
`Webhook payload exceeds maximum size (${body.length} bytes > ${this.maxPayloadBytes} bytes)
|
|
397
|
+
`Webhook payload exceeds maximum size (${body.length} bytes > ${this.maxPayloadBytes} bytes)`
|
|
333
398
|
);
|
|
334
399
|
}
|
|
335
400
|
|
|
336
401
|
const headers: Record<string, string> = {
|
|
337
|
-
|
|
402
|
+
'Content-Type': 'application/json',
|
|
338
403
|
...endpoint.headers,
|
|
339
404
|
};
|
|
340
405
|
|
|
341
406
|
// Sign payload if secret is provided
|
|
342
407
|
if (endpoint.secret) {
|
|
343
408
|
const signature = await this.signPayload(body, endpoint.secret);
|
|
344
|
-
headers[
|
|
409
|
+
headers['X-Webhook-Signature'] = signature;
|
|
345
410
|
}
|
|
346
411
|
|
|
347
|
-
// Resolve hostname and validate the resolved IP is not private/reserved (SSRF protection).
|
|
348
|
-
// This prevents DNS rebinding attacks where a hostname initially resolves to a public IP
|
|
349
|
-
// but later resolves to a private IP.
|
|
350
|
-
await this.validateResolvedHost(endpoint.url);
|
|
351
|
-
|
|
352
412
|
let lastError: Error | undefined;
|
|
353
413
|
|
|
354
414
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
355
415
|
try {
|
|
416
|
+
// Resolve the hostname once and validate the resulting IP is not
|
|
417
|
+
// private/reserved (SSRF protection). When the default fetch is used,
|
|
418
|
+
// the resolved IP is then pinned for the connection so a hostile DNS
|
|
419
|
+
// server cannot rebind the hostname to a different address between
|
|
420
|
+
// validation and connect (M1: rebinding TOCTOU). The lookup runs
|
|
421
|
+
// inside the retry loop so DNS failures are retried along with
|
|
422
|
+
// network errors.
|
|
423
|
+
const pinned = await this.validateResolvedHost(endpoint.url);
|
|
424
|
+
|
|
356
425
|
const controller = new AbortController();
|
|
357
426
|
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
358
427
|
|
|
359
|
-
const response =
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
428
|
+
const response = this.hasCustomFetch || !pinned
|
|
429
|
+
? await this.fetchFn(endpoint.url, {
|
|
430
|
+
method: 'POST',
|
|
431
|
+
headers,
|
|
432
|
+
body,
|
|
433
|
+
signal: controller.signal,
|
|
434
|
+
})
|
|
435
|
+
: await pinnedFetch(
|
|
436
|
+
endpoint.url,
|
|
437
|
+
{ method: 'POST', headers, body, signal: controller.signal },
|
|
438
|
+
pinned.address,
|
|
439
|
+
pinned.family,
|
|
440
|
+
);
|
|
365
441
|
|
|
366
442
|
clearTimeout(timeoutId);
|
|
367
443
|
|
|
@@ -369,9 +445,7 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
369
445
|
return; // Success
|
|
370
446
|
}
|
|
371
447
|
|
|
372
|
-
lastError = new Error(
|
|
373
|
-
`Webhook returned ${response.status}: ${response.statusText}`,
|
|
374
|
-
);
|
|
448
|
+
lastError = new Error(`Webhook returned ${response.status}: ${response.statusText}`);
|
|
375
449
|
} catch (error) {
|
|
376
450
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
377
451
|
}
|
|
@@ -395,18 +469,16 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
395
469
|
const data = encoder.encode(payload);
|
|
396
470
|
|
|
397
471
|
const key = await crypto.subtle.importKey(
|
|
398
|
-
|
|
472
|
+
'raw',
|
|
399
473
|
keyData,
|
|
400
|
-
{ name:
|
|
474
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
401
475
|
false,
|
|
402
|
-
[
|
|
476
|
+
['sign']
|
|
403
477
|
);
|
|
404
478
|
|
|
405
|
-
const signature = await crypto.subtle.sign(
|
|
479
|
+
const signature = await crypto.subtle.sign('HMAC', key, data);
|
|
406
480
|
const hashArray = Array.from(new Uint8Array(signature));
|
|
407
|
-
const hashHex = hashArray
|
|
408
|
-
.map((b) => b.toString(16).padStart(2, "0"))
|
|
409
|
-
.join("");
|
|
481
|
+
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
|
|
410
482
|
|
|
411
483
|
return `sha256=${hashHex}`;
|
|
412
484
|
}
|
|
@@ -415,39 +487,83 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
415
487
|
* Sleep helper.
|
|
416
488
|
*/
|
|
417
489
|
private sleep(ms: number): Promise<void> {
|
|
418
|
-
return new Promise(
|
|
490
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
419
491
|
}
|
|
420
492
|
|
|
421
493
|
/**
|
|
422
|
-
* Resolve
|
|
423
|
-
*
|
|
494
|
+
* Resolve a webhook URL's hostname exactly once, verify the resolved IP is
|
|
495
|
+
* not private/reserved, and return the address + family so the caller can
|
|
496
|
+
* pin the connection to it.
|
|
497
|
+
*
|
|
498
|
+
* Returns `null` for raw-IP URLs (already validated by `validateWebhookUrl`).
|
|
499
|
+
*
|
|
500
|
+
* Pinning the resolved IP closes the DNS rebinding TOCTOU window (audit M1):
|
|
501
|
+
* with the original implementation `fetch()` re-resolved the hostname,
|
|
502
|
+
* giving a hostile DNS server a chance to flip the answer to an internal
|
|
503
|
+
* address between SSRF validation and connect.
|
|
504
|
+
*
|
|
505
|
+
* Uses a short-TTL cache and per-call timeout to avoid blocking the event loop.
|
|
424
506
|
*/
|
|
425
|
-
private async validateResolvedHost(
|
|
507
|
+
private async validateResolvedHost(
|
|
508
|
+
url: string,
|
|
509
|
+
): Promise<{ address: string; family: 4 | 6 } | null> {
|
|
426
510
|
const parsed = new URL(url);
|
|
427
511
|
const hostname = parsed.hostname.toLowerCase();
|
|
428
512
|
|
|
429
|
-
// Skip DNS resolution for raw IP
|
|
430
|
-
|
|
431
|
-
|
|
513
|
+
// Skip DNS resolution for raw IP literals — already checked by isBlockedHost in validateWebhookUrl.
|
|
514
|
+
// Node's URL parser wraps IPv6 literals in brackets (e.g., "[2001:db8::1]").
|
|
515
|
+
const stripped = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname;
|
|
516
|
+
if (isIPv4(stripped) || isIPv6(stripped)) {
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// Check cache first
|
|
521
|
+
const cached = this.dnsCache.get(hostname);
|
|
522
|
+
if (cached && cached.expiresAt > Date.now()) {
|
|
523
|
+
if (isBlockedIp(cached.address)) {
|
|
524
|
+
throw new Error(
|
|
525
|
+
`Webhook URL hostname "${hostname}" resolves to blocked IP ${cached.address}`
|
|
526
|
+
);
|
|
527
|
+
}
|
|
528
|
+
return { address: cached.address, family: cached.family };
|
|
432
529
|
}
|
|
433
530
|
|
|
434
531
|
try {
|
|
435
|
-
const
|
|
532
|
+
const ac = new AbortController();
|
|
533
|
+
const timeoutId = setTimeout(() => ac.abort(), WebhookEventTransport.DNS_TIMEOUT_MS);
|
|
534
|
+
|
|
535
|
+
let address: string;
|
|
536
|
+
let family: 4 | 6;
|
|
537
|
+
try {
|
|
538
|
+
const result = await dns.lookup(hostname, { signal: ac.signal } as any);
|
|
539
|
+
address = result.address;
|
|
540
|
+
family = (result.family === 6 ? 6 : 4) as 4 | 6;
|
|
541
|
+
} finally {
|
|
542
|
+
clearTimeout(timeoutId);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// Cache the result
|
|
546
|
+
this.dnsCache.set(hostname, {
|
|
547
|
+
address,
|
|
548
|
+
family,
|
|
549
|
+
expiresAt: Date.now() + WebhookEventTransport.DNS_CACHE_TTL_MS,
|
|
550
|
+
});
|
|
551
|
+
|
|
436
552
|
if (isBlockedIp(address)) {
|
|
437
553
|
throw new Error(
|
|
438
|
-
`Webhook URL hostname "${hostname}" resolves to blocked IP ${address}
|
|
554
|
+
`Webhook URL hostname "${hostname}" resolves to blocked IP ${address}`
|
|
439
555
|
);
|
|
440
556
|
}
|
|
557
|
+
|
|
558
|
+
return { address, family };
|
|
441
559
|
} catch (error) {
|
|
442
|
-
if (
|
|
443
|
-
error instanceof Error &&
|
|
444
|
-
error.message.includes("resolves to blocked IP")
|
|
445
|
-
) {
|
|
560
|
+
if (error instanceof Error && error.message.includes('resolves to blocked IP')) {
|
|
446
561
|
throw error;
|
|
447
562
|
}
|
|
448
|
-
|
|
449
|
-
`
|
|
450
|
-
|
|
563
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
564
|
+
throw new Error(`DNS lookup for webhook URL hostname "${hostname}" timed out`);
|
|
565
|
+
}
|
|
566
|
+
throw new Error(`Failed to resolve webhook URL hostname "${hostname}": ${error}`);
|
|
451
567
|
}
|
|
452
568
|
}
|
|
453
569
|
|
|
@@ -465,31 +581,31 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
465
581
|
}
|
|
466
582
|
|
|
467
583
|
// Scheme check
|
|
468
|
-
if (!this.allowInsecureUrls && parsed.protocol !==
|
|
584
|
+
if (!this.allowInsecureUrls && parsed.protocol !== 'https:') {
|
|
469
585
|
throw new Error(
|
|
470
|
-
`Webhook URL must use HTTPS (got ${parsed.protocol}). Set allowInsecureUrls for development
|
|
586
|
+
`Webhook URL must use HTTPS (got ${parsed.protocol}). Set allowInsecureUrls for development.`
|
|
471
587
|
);
|
|
472
588
|
}
|
|
473
589
|
|
|
474
|
-
if (parsed.protocol !==
|
|
475
|
-
throw new Error(
|
|
476
|
-
`Webhook URL must use HTTP(S) scheme (got ${parsed.protocol})`,
|
|
477
|
-
);
|
|
590
|
+
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
|
|
591
|
+
throw new Error(`Webhook URL must use HTTP(S) scheme (got ${parsed.protocol})`);
|
|
478
592
|
}
|
|
479
593
|
|
|
480
594
|
// Block dangerous hostnames
|
|
481
595
|
const hostname = parsed.hostname.toLowerCase();
|
|
482
596
|
if (isBlockedHost(hostname)) {
|
|
483
|
-
throw new Error(
|
|
484
|
-
`Webhook URL hostname is blocked (private/reserved): ${hostname}`,
|
|
485
|
-
);
|
|
597
|
+
throw new Error(`Webhook URL hostname is blocked (private/reserved): ${hostname}`);
|
|
486
598
|
}
|
|
487
599
|
}
|
|
488
600
|
|
|
489
601
|
/**
|
|
490
|
-
* Enqueue a webhook request with concurrency limiting.
|
|
602
|
+
* Enqueue a webhook request with concurrency limiting and a queue-depth cap.
|
|
603
|
+
* Returns `true` if the request was accepted (executed or queued), or
|
|
604
|
+
* `false` if the queue is full and the request was rejected. The caller is
|
|
605
|
+
* responsible for logging the drop and incrementing any drop metrics, since
|
|
606
|
+
* it has the endpoint context.
|
|
491
607
|
*/
|
|
492
|
-
private enqueueRequest(fn: () => Promise<void
|
|
608
|
+
private enqueueRequest(fn: () => Promise<void>, _endpointId: string): boolean {
|
|
493
609
|
const execute = () => {
|
|
494
610
|
this.activeRequests++;
|
|
495
611
|
fn().finally(() => {
|
|
@@ -501,60 +617,54 @@ export class WebhookEventTransport implements EventTransport {
|
|
|
501
617
|
|
|
502
618
|
if (this.activeRequests < this.maxConcurrentRequests) {
|
|
503
619
|
execute();
|
|
504
|
-
|
|
505
|
-
|
|
620
|
+
return true;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
if (this.maxQueueDepth > 0 && this.requestQueue.length >= this.maxQueueDepth) {
|
|
624
|
+
return false;
|
|
506
625
|
}
|
|
626
|
+
|
|
627
|
+
this.requestQueue.push(execute);
|
|
628
|
+
return true;
|
|
507
629
|
}
|
|
508
630
|
}
|
|
509
631
|
|
|
510
632
|
/**
|
|
511
|
-
*
|
|
512
|
-
*
|
|
633
|
+
* Resolve the effective `maxQueueDepth` from explicit config and the
|
|
634
|
+
* `STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH` environment variable.
|
|
635
|
+
*
|
|
636
|
+
* Precedence: explicit config value > env var > default (1000).
|
|
513
637
|
*/
|
|
514
|
-
function
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
hostname === "[::1]"
|
|
521
|
-
) {
|
|
522
|
-
return true;
|
|
638
|
+
function resolveMaxQueueDepth(configured: number | undefined): number {
|
|
639
|
+
if (typeof configured === 'number') return configured;
|
|
640
|
+
const env = typeof process !== 'undefined' ? process.env?.STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH : undefined;
|
|
641
|
+
if (env !== undefined && env !== '') {
|
|
642
|
+
const parsed = Number(env);
|
|
643
|
+
if (Number.isFinite(parsed) && parsed >= 0) return parsed;
|
|
523
644
|
}
|
|
645
|
+
return 1000;
|
|
646
|
+
}
|
|
524
647
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
648
|
+
/**
|
|
649
|
+
* Check if a hostname (as it appears in a URL) is blocked.
|
|
650
|
+
* Handles bare hostnames, IPv4 literals, and bracketed IPv6 literals.
|
|
651
|
+
*/
|
|
652
|
+
function isBlockedHost(hostname: string): boolean {
|
|
653
|
+
if (hostname === 'localhost') return true;
|
|
529
654
|
|
|
530
|
-
//
|
|
531
|
-
if (hostname
|
|
532
|
-
|
|
655
|
+
// IPv6 literals are wrapped in brackets by Node's URL parser, e.g. "[::1]".
|
|
656
|
+
if (hostname.startsWith('[') && hostname.endsWith(']')) {
|
|
657
|
+
const bare = hostname.slice(1, -1);
|
|
658
|
+
return isIPv6(bare) ? isBlockedIPv6(bare) : true;
|
|
533
659
|
}
|
|
534
660
|
|
|
535
|
-
//
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
);
|
|
539
|
-
if (ipv4Match) {
|
|
540
|
-
const [, a, b] = ipv4Match.map(Number);
|
|
541
|
-
// 10.0.0.0/8
|
|
542
|
-
if (a === 10) return true;
|
|
543
|
-
// 172.16.0.0/12
|
|
544
|
-
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
545
|
-
// 192.168.0.0/16
|
|
546
|
-
if (a === 192 && b === 168) return true;
|
|
547
|
-
// 127.0.0.0/8
|
|
548
|
-
if (a === 127) return true;
|
|
549
|
-
// 169.254.0.0/16 (link-local)
|
|
550
|
-
if (a === 169 && b === 254) return true;
|
|
551
|
-
// 0.0.0.0
|
|
552
|
-
if (a === 0 && b === 0) return true;
|
|
661
|
+
// Some callers (or legacy code paths) may pass an unwrapped IPv6 literal.
|
|
662
|
+
if (isIPv6(hostname)) {
|
|
663
|
+
return isBlockedIPv6(hostname);
|
|
553
664
|
}
|
|
554
665
|
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
return true;
|
|
666
|
+
if (isIPv4(hostname)) {
|
|
667
|
+
return isBlockedIPv4(hostname);
|
|
558
668
|
}
|
|
559
669
|
|
|
560
670
|
return false;
|
|
@@ -565,27 +675,133 @@ function isBlockedHost(hostname: string): boolean {
|
|
|
565
675
|
* Used for runtime DNS resolution validation to prevent SSRF via DNS rebinding.
|
|
566
676
|
*/
|
|
567
677
|
function isBlockedIp(ip: string): boolean {
|
|
568
|
-
|
|
569
|
-
if (
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
678
|
+
if (isIPv4(ip)) return isBlockedIPv4(ip);
|
|
679
|
+
if (isIPv6(ip)) return isBlockedIPv6(ip);
|
|
680
|
+
return false;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/**
|
|
684
|
+
* Block IPv4 addresses in private, loopback, link-local, and cloud-metadata ranges.
|
|
685
|
+
*/
|
|
686
|
+
function isBlockedIPv4(ip: string): boolean {
|
|
687
|
+
const match = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
688
|
+
if (!match) return false;
|
|
689
|
+
const [, a, b] = match.map(Number);
|
|
690
|
+
return isBlockedIPv4Octets(a, b);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
function isBlockedIPv4Octets(a: number, b: number): boolean {
|
|
694
|
+
// 0.0.0.0/8 — "this network"
|
|
695
|
+
if (a === 0) return true;
|
|
696
|
+
// 10.0.0.0/8 — RFC 1918 private
|
|
697
|
+
if (a === 10) return true;
|
|
698
|
+
// 100.64.0.0/10 — CGNAT
|
|
699
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
700
|
+
// 127.0.0.0/8 — loopback
|
|
701
|
+
if (a === 127) return true;
|
|
702
|
+
// 169.254.0.0/16 — link-local (covers cloud metadata 169.254.169.254)
|
|
703
|
+
if (a === 169 && b === 254) return true;
|
|
704
|
+
// 172.16.0.0/12 — RFC 1918 private
|
|
705
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
706
|
+
// 192.168.0.0/16 — RFC 1918 private
|
|
707
|
+
if (a === 192 && b === 168) return true;
|
|
708
|
+
// 224.0.0.0/4 multicast and 240.0.0.0/4 reserved (incl. 255.255.255.255 broadcast)
|
|
709
|
+
if (a >= 224) return true;
|
|
710
|
+
return false;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
/**
|
|
714
|
+
* Block IPv6 addresses in reserved, private, loopback, link-local, IPv4-mapped,
|
|
715
|
+
* IPv4-translated, 6to4, and Teredo ranges. For mapped/embedded IPv4 forms,
|
|
716
|
+
* the embedded IPv4 is decoded and rechecked against the IPv4 blocklist.
|
|
717
|
+
*/
|
|
718
|
+
function isBlockedIPv6(ip: string): boolean {
|
|
719
|
+
const groups = expandIPv6(ip);
|
|
720
|
+
if (!groups) return false;
|
|
721
|
+
|
|
722
|
+
// ::1 loopback and :: unspecified
|
|
723
|
+
const allButLastZero = groups.slice(0, 7).every(g => g === 0);
|
|
724
|
+
if (allButLastZero && groups[7] === 1) return true;
|
|
725
|
+
if (groups.every(g => g === 0)) return true;
|
|
726
|
+
|
|
727
|
+
// ::ffff:0:0/96 — IPv4-mapped IPv6 (decode embedded IPv4)
|
|
728
|
+
if (groups.slice(0, 5).every(g => g === 0) && groups[5] === 0xffff) {
|
|
729
|
+
return isBlockedIPv4Octets((groups[6] >> 8) & 0xff, groups[6] & 0xff);
|
|
583
730
|
}
|
|
584
731
|
|
|
585
|
-
// IPv6
|
|
586
|
-
if (
|
|
587
|
-
|
|
588
|
-
|
|
732
|
+
// 64:ff9b::/96 — IPv4/IPv6 translation (RFC 6052) and 64:ff9b:1::/48 (RFC 8215)
|
|
733
|
+
if (groups[0] === 0x64 && groups[1] === 0xff9b && groups.slice(2, 6).every(g => g === 0)) {
|
|
734
|
+
return isBlockedIPv4Octets((groups[6] >> 8) & 0xff, groups[6] & 0xff);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// 2002::/16 — 6to4 (next 32 bits are the embedded IPv4)
|
|
738
|
+
if (groups[0] === 0x2002) {
|
|
739
|
+
return isBlockedIPv4Octets((groups[1] >> 8) & 0xff, groups[1] & 0xff);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// 2001::/32 — Teredo. Block conservatively; embedded server/client v4 fields
|
|
743
|
+
// can mask private destinations and Teredo isn't a normal webhook target.
|
|
744
|
+
if (groups[0] === 0x2001 && groups[1] === 0) return true;
|
|
745
|
+
|
|
746
|
+
// 100::/64 — RFC 6666 Discard prefix
|
|
747
|
+
if (groups[0] === 0x0100 && groups.slice(1, 4).every(g => g === 0)) return true;
|
|
748
|
+
|
|
749
|
+
// fc00::/7 — Unique Local Addresses (fc00:: through fdff::)
|
|
750
|
+
if ((groups[0] & 0xfe00) === 0xfc00) return true;
|
|
751
|
+
|
|
752
|
+
// fe80::/10 — Link-local (fe80:: through febf::)
|
|
753
|
+
if ((groups[0] & 0xffc0) === 0xfe80) return true;
|
|
754
|
+
|
|
755
|
+
// ff00::/8 — Multicast
|
|
756
|
+
if ((groups[0] & 0xff00) === 0xff00) return true;
|
|
589
757
|
|
|
590
758
|
return false;
|
|
591
759
|
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* Expand an IPv6 string into an array of 8 numeric hextets (0–0xffff).
|
|
763
|
+
* Handles "::" abbreviation, embedded IPv4 dotted-quad, and zone IDs.
|
|
764
|
+
* Returns null if the input is not a valid IPv6 address.
|
|
765
|
+
*/
|
|
766
|
+
function expandIPv6(addr: string): number[] | null {
|
|
767
|
+
// Strip zone identifier (e.g., "fe80::1%eth0")
|
|
768
|
+
const noZone = addr.split('%')[0];
|
|
769
|
+
if (!isIPv6(noZone)) return null;
|
|
770
|
+
|
|
771
|
+
let normalized = noZone.toLowerCase();
|
|
772
|
+
|
|
773
|
+
// Convert trailing dotted-quad (e.g., "::ffff:192.168.0.1") into two hextets.
|
|
774
|
+
const v4Tail = normalized.match(/^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
775
|
+
if (v4Tail) {
|
|
776
|
+
const a = parseInt(v4Tail[2], 10);
|
|
777
|
+
const b = parseInt(v4Tail[3], 10);
|
|
778
|
+
const c = parseInt(v4Tail[4], 10);
|
|
779
|
+
const d = parseInt(v4Tail[5], 10);
|
|
780
|
+
if ([a, b, c, d].some(n => n > 255)) return null;
|
|
781
|
+
const hi = ((a << 8) | b).toString(16);
|
|
782
|
+
const lo = ((c << 8) | d).toString(16);
|
|
783
|
+
normalized = `${v4Tail[1]}${hi}:${lo}`;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const halves = normalized.split('::');
|
|
787
|
+
let parts: string[];
|
|
788
|
+
if (halves.length === 1) {
|
|
789
|
+
parts = halves[0].split(':');
|
|
790
|
+
if (parts.length !== 8) return null;
|
|
791
|
+
} else if (halves.length === 2) {
|
|
792
|
+
const head = halves[0] ? halves[0].split(':') : [];
|
|
793
|
+
const tail = halves[1] ? halves[1].split(':') : [];
|
|
794
|
+
const missing = 8 - head.length - tail.length;
|
|
795
|
+
if (missing < 0) return null;
|
|
796
|
+
parts = [...head, ...Array(missing).fill('0'), ...tail];
|
|
797
|
+
} else {
|
|
798
|
+
return null;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
const groups: number[] = [];
|
|
802
|
+
for (const part of parts) {
|
|
803
|
+
if (!/^[0-9a-f]{1,4}$/.test(part)) return null;
|
|
804
|
+
groups.push(parseInt(part, 16));
|
|
805
|
+
}
|
|
806
|
+
return groups;
|
|
807
|
+
}
|