@multiplier-labs/stepflow 0.3.1 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +146 -0
  2. package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +74 -0
  3. package/.codekin/reports/comments/2026-04-04_comment-assessment.md +179 -0
  4. package/.codekin/reports/comments/2026-04-11_comment-assessment.md +161 -0
  5. package/.codekin/reports/comments/2026-04-14_comment-assessment.md +34 -0
  6. package/.codekin/reports/complexity/2026-04-03_complexity-report.md +152 -0
  7. package/.codekin/reports/complexity/2026-04-11_complexity-report.md +152 -0
  8. package/.codekin/reports/complexity/2026-04-14_complexity-report.md +34 -0
  9. package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +94 -0
  10. package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +93 -0
  11. package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +10 -0
  12. package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +138 -0
  13. package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +121 -0
  14. package/.codekin/reports/security/2026-04-03_security-audit.md +208 -0
  15. package/.codekin/reports/security/2026-04-11_security-audit.md +297 -0
  16. package/.github/CODEOWNERS +15 -0
  17. package/.github/dependabot.yml +23 -0
  18. package/.github/workflows/ci.yml +38 -0
  19. package/.github/workflows/publish.yml +17 -5
  20. package/.github/workflows/security.yml +29 -0
  21. package/CHANGELOG.md +34 -0
  22. package/README.md +24 -0
  23. package/SECURITY.md +40 -0
  24. package/dist/{chunk-FQCT3I2A.js → chunk-5XJBAPXF.js} +527 -551
  25. package/dist/chunk-5XJBAPXF.js.map +1 -0
  26. package/dist/{chunk-U5QGN3RT.js → chunk-FMPJBK2N.js} +67 -55
  27. package/dist/chunk-FMPJBK2N.js.map +1 -0
  28. package/dist/{chunk-GXY66V7Q.js → chunk-JPXPB7U5.js} +52 -2
  29. package/dist/chunk-JPXPB7U5.js.map +1 -0
  30. package/dist/{chunk-XCF4PLN7.js → chunk-U3PCVK6O.js} +276 -61
  31. package/dist/chunk-U3PCVK6O.js.map +1 -0
  32. package/dist/events/index.d.ts +60 -7
  33. package/dist/events/index.js +1 -1
  34. package/dist/{index-CPfGsmrp.d.ts → index-Cd0IZ40L.d.ts} +56 -17
  35. package/dist/index.d.ts +15 -23
  36. package/dist/index.js +784 -547
  37. package/dist/index.js.map +1 -1
  38. package/dist/scheduler/index.d.ts +4 -4
  39. package/dist/scheduler/index.js +2 -2
  40. package/dist/storage/index.d.ts +322 -56
  41. package/dist/storage/index.js +4 -2
  42. package/dist/{types-BlwFEGzj.d.ts → types-DSEjiLMj.d.ts} +16 -15
  43. package/dist/{types-DzkEqnQJ.d.ts → types-DZUdWmCc.d.ts} +2 -2
  44. package/dist/{types-BVShq7zT.d.ts → types-K5Gjk3H_.d.ts} +4 -4
  45. package/docs/stepflow-api-reference.md +5 -4
  46. package/package.json +12 -8
  47. package/src/core/engine/concurrency.ts +229 -0
  48. package/src/core/engine/lifecycle.ts +176 -0
  49. package/src/core/engine.test.ts +536 -522
  50. package/src/core/engine.ts +116 -235
  51. package/src/core/index.ts +3 -3
  52. package/src/core/orchestrator/persistence.ts +165 -0
  53. package/src/core/orchestrator/plan-resolution.ts +140 -0
  54. package/src/core/orchestrator/state-transitions.ts +109 -0
  55. package/src/core/orchestrator/step-runner.ts +427 -0
  56. package/src/core/orchestrator.ts +57 -395
  57. package/src/core/types.ts +7 -29
  58. package/src/events/index.ts +4 -4
  59. package/src/events/memory.test.ts +39 -41
  60. package/src/events/memory.ts +7 -10
  61. package/src/events/socketio.test.ts +112 -162
  62. package/src/events/socketio.ts +20 -27
  63. package/src/events/types.ts +16 -19
  64. package/src/events/webhook.test.ts +489 -235
  65. package/src/events/webhook.ts +355 -139
  66. package/src/index.ts +19 -19
  67. package/src/planning/evaluate-condition.test.ts +153 -0
  68. package/src/planning/evaluate-condition.ts +105 -0
  69. package/src/planning/index.ts +6 -3
  70. package/src/planning/planner.ts +45 -152
  71. package/src/planning/planning.test.ts +303 -366
  72. package/src/planning/registry.test.ts +204 -251
  73. package/src/planning/registry.ts +20 -106
  74. package/src/planning/types.ts +23 -27
  75. package/src/scheduler/cron.test.ts +149 -150
  76. package/src/scheduler/cron.ts +58 -69
  77. package/src/scheduler/index.ts +4 -14
  78. package/src/scheduler/postgres-persistence.ts +89 -105
  79. package/src/scheduler/sqlite-persistence.test.ts +96 -96
  80. package/src/scheduler/sqlite-persistence.ts +40 -22
  81. package/src/scheduler/types.ts +4 -9
  82. package/src/storage/index.ts +5 -5
  83. package/src/storage/memory.test.ts +133 -183
  84. package/src/storage/memory.ts +21 -42
  85. package/src/storage/postgres-core.ts +550 -0
  86. package/src/storage/postgres-migration.test.ts +69 -0
  87. package/src/storage/postgres-transaction-pagination.test.ts +107 -0
  88. package/src/storage/postgres.ts +222 -937
  89. package/src/storage/sqlite-core.ts +151 -0
  90. package/src/storage/sqlite.test.ts +330 -281
  91. package/src/storage/sqlite.ts +225 -170
  92. package/src/storage/types.ts +33 -58
  93. package/src/utils/errors.test.ts +78 -80
  94. package/src/utils/errors.ts +27 -57
  95. package/src/utils/id.ts +3 -3
  96. package/src/utils/index.ts +6 -4
  97. package/src/utils/logger.test.ts +52 -58
  98. package/src/utils/logger.ts +4 -4
  99. package/src/utils/pinned-fetch.test.ts +84 -0
  100. package/src/utils/pinned-fetch.ts +118 -0
  101. package/src/utils/postgres-deps.ts +20 -4
  102. package/src/utils/retry.test.ts +47 -67
  103. package/src/utils/retry.ts +9 -9
  104. package/src/utils/safe-json.test.ts +166 -0
  105. package/src/utils/safe-json.ts +107 -0
  106. package/src/utils/webhook-signature.test.ts +111 -0
  107. package/src/utils/webhook-signature.ts +74 -0
  108. package/dist/chunk-FQCT3I2A.js.map +0 -1
  109. package/dist/chunk-GXY66V7Q.js.map +0 -1
  110. package/dist/chunk-U5QGN3RT.js.map +0 -1
  111. package/dist/chunk-XCF4PLN7.js.map +0 -1
  112. package/src/scheduler/postgres-persistence.test.ts +0 -906
@@ -5,15 +5,11 @@
5
5
  * Supports multiple webhooks with optional filtering by event type.
6
6
  */
7
7
 
8
- import { promises as dns } from "node:dns";
9
- import type {
10
- EventTransport,
11
- EventCallback,
12
- Unsubscribe,
13
- WorkflowEvent,
14
- WorkflowEventType,
15
- } from "./types";
16
- import type { Logger } from "../core/types";
8
+ import { promises as dns } from 'node:dns';
9
+ import { isIPv4, isIPv6 } from 'node:net';
10
+ import { pinnedFetch } from '../utils/pinned-fetch';
11
+ import type { EventTransport, EventCallback, Unsubscribe, WorkflowEvent, WorkflowEventType } from './types';
12
+ import type { Logger } from '../core/types';
17
13
 
18
14
  /**
19
15
  * Webhook endpoint configuration.
@@ -84,8 +80,27 @@ export interface WebhookEventTransportConfig {
84
80
  */
85
81
  maxConcurrentRequests?: number;
86
82
 
83
+ /**
84
+ * Maximum number of queued webhook deliveries waiting for a concurrency slot
85
+ * (default: 1000, env override: `STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH`).
86
+ *
87
+ * When the queue is full, incoming `emit()` deliveries are dropped to
88
+ * protect the process from memory exhaustion under burst traffic. Drops
89
+ * are logged via `logger.warn` with the endpoint id, current queue depth,
90
+ * and a recommended `retryAfterMs`. A value of `0` disables the cap (not
91
+ * recommended in production).
92
+ */
93
+ maxQueueDepth?: number;
94
+
95
+ /**
96
+ * Suggested back-off in milliseconds reported on dropped deliveries
97
+ * (default: 1000). Surfaced in the warning log so external orchestrators
98
+ * (sidecars, sweeper jobs) can respect it like an HTTP `Retry-After`.
99
+ */
100
+ queueDropRetryAfterMs?: number;
101
+
87
102
  /** Logger for webhook transport errors (default: console-based) */
88
- logger?: import("../core/types").Logger;
103
+ logger?: import('../core/types').Logger;
89
104
  }
90
105
 
91
106
  /**
@@ -97,7 +112,7 @@ export interface WebhookPayload {
97
112
  * The `Date` is converted to a string before delivery so that JSON consumers
98
113
  * receive a portable, timezone-aware representation without needing Date parsing.
99
114
  */
100
- event: Omit<WorkflowEvent, "timestamp"> & { timestamp: string };
115
+ event: Omit<WorkflowEvent, 'timestamp'> & { timestamp: string };
101
116
  /** ISO 8601 timestamp of when this payload was delivered. */
102
117
  deliveredAt: string;
103
118
  /** ID of the webhook endpoint that received this delivery. */
@@ -136,13 +151,29 @@ export class WebhookEventTransport implements EventTransport {
136
151
  private defaultRetries: number;
137
152
  private retryDelay: number;
138
153
  private fetchFn: typeof fetch;
154
+ /**
155
+ * Whether `fetchFn` was caller-supplied. When `false` the default fetch is
156
+ * replaced with `pinnedFetch` so the TCP connection can target the IP that
157
+ * was just validated (closes the DNS-rebinding TOCTOU window). When `true`
158
+ * the custom fetch is used as-is so test injection still works.
159
+ */
160
+ private hasCustomFetch: boolean;
139
161
  private allowInsecureUrls: boolean;
140
162
  private maxPayloadBytes: number;
141
163
  private maxConcurrentRequests: number;
164
+ private maxQueueDepth: number;
165
+ private queueDropRetryAfterMs: number;
142
166
  private activeRequests = 0;
143
167
  private requestQueue: Array<() => void> = [];
168
+ private droppedRequests = 0;
144
169
  private logger: Logger;
145
170
 
171
+ // DNS resolution cache with TTL — stores `family` so the pin can reuse it
172
+ // without a second lookup.
173
+ private dnsCache = new Map<string, { address: string; family: 4 | 6; expiresAt: number }>();
174
+ private static readonly DNS_CACHE_TTL_MS = 30_000; // 30 seconds
175
+ private static readonly DNS_TIMEOUT_MS = 5_000; // 5 seconds
176
+
146
177
  // Local subscribers for server-side subscriptions
147
178
  private runSubscribers = new Map<string, Set<EventCallback>>();
148
179
  private globalSubscribers = new Set<EventCallback>();
@@ -151,19 +182,30 @@ export class WebhookEventTransport implements EventTransport {
151
182
  this.defaultTimeout = config.defaultTimeout ?? 5000;
152
183
  this.defaultRetries = config.defaultRetries ?? 3;
153
184
  this.retryDelay = config.retryDelay ?? 1000;
185
+ this.hasCustomFetch = config.fetchFn !== undefined;
154
186
  this.fetchFn = config.fetchFn ?? globalThis.fetch;
155
187
  this.allowInsecureUrls = config.allowInsecureUrls ?? false;
156
188
  this.maxPayloadBytes = config.maxPayloadBytes ?? 1_048_576; // 1 MB
157
189
  this.maxConcurrentRequests = config.maxConcurrentRequests ?? 50;
190
+ if (this.maxConcurrentRequests <= 0) {
191
+ throw new Error('maxConcurrentRequests must be greater than 0');
192
+ }
193
+ this.maxQueueDepth = resolveMaxQueueDepth(config.maxQueueDepth);
194
+ if (this.maxQueueDepth < 0) {
195
+ throw new Error('maxQueueDepth must be >= 0 (0 disables the cap)');
196
+ }
197
+ this.queueDropRetryAfterMs = config.queueDropRetryAfterMs ?? 1000;
158
198
  this.logger = config.logger ?? {
159
199
  debug() {},
160
200
  info() {},
161
201
  warn() {},
162
- error(message: string, ...args: unknown[]) {
163
- console.error(message, ...args);
164
- },
202
+ error(message: string, ...args: unknown[]) { console.error(message, ...args); },
165
203
  };
166
204
 
205
+ if (this.allowInsecureUrls) {
206
+ this.logger.warn('WebhookEventTransport: allowInsecureUrls is enabled. Webhook payloads will be sent over HTTP. Do not use this setting in production.');
207
+ }
208
+
167
209
  // Register initial endpoints (validates URLs)
168
210
  if (config.endpoints) {
169
211
  for (const endpoint of config.endpoints) {
@@ -217,7 +259,7 @@ export class WebhookEventTransport implements EventTransport {
217
259
  try {
218
260
  callback(event);
219
261
  } catch (error) {
220
- this.logger.error("Event callback error:", error);
262
+ this.logger.error('Event callback error:', error);
221
263
  }
222
264
  }
223
265
  }
@@ -226,7 +268,7 @@ export class WebhookEventTransport implements EventTransport {
226
268
  try {
227
269
  callback(event);
228
270
  } catch (error) {
229
- this.logger.error("Event callback error:", error);
271
+ this.logger.error('Event callback error:', error);
230
272
  }
231
273
  }
232
274
 
@@ -235,14 +277,43 @@ export class WebhookEventTransport implements EventTransport {
235
277
  if (!endpoint.enabled) continue;
236
278
  if (!this.matchesFilter(event, endpoint)) continue;
237
279
 
238
- this.enqueueRequest(() =>
239
- this.sendWebhook(endpoint, event).catch((error) => {
240
- this.logger.error(`Webhook ${endpoint.id} failed:`, error);
241
- }),
280
+ const accepted = this.enqueueRequest(
281
+ () =>
282
+ this.sendWebhook(endpoint, event).catch((error) => {
283
+ this.logger.error(`Webhook ${endpoint.id} failed:`, error);
284
+ }),
285
+ endpoint.id
242
286
  );
287
+
288
+ if (!accepted) {
289
+ // Queue overflow — equivalent to a server returning 429 with Retry-After.
290
+ // The delivery is dropped to protect the process from memory exhaustion.
291
+ this.droppedRequests++;
292
+ this.logger.warn(
293
+ `Webhook ${endpoint.id} delivery dropped: queue full ` +
294
+ `(depth ${this.requestQueue.length}/${this.maxQueueDepth}, ` +
295
+ `active ${this.activeRequests}/${this.maxConcurrentRequests}, ` +
296
+ `total drops ${this.droppedRequests}, retryAfterMs ${this.queueDropRetryAfterMs})`
297
+ );
298
+ }
243
299
  }
244
300
  }
245
301
 
302
+ /**
303
+ * Number of webhook deliveries dropped due to queue overflow since startup.
304
+ * Useful for surfacing back-pressure to operators (metrics/health checks).
305
+ */
306
+ getDroppedRequestCount(): number {
307
+ return this.droppedRequests;
308
+ }
309
+
310
+ /**
311
+ * Current depth of the pending request queue.
312
+ */
313
+ getQueueDepth(): number {
314
+ return this.requestQueue.length;
315
+ }
316
+
246
317
  /**
247
318
  * Subscribe to events for a specific run (server-side).
248
319
  */
@@ -284,10 +355,7 @@ export class WebhookEventTransport implements EventTransport {
284
355
  /**
285
356
  * Check if an event matches an endpoint's filters.
286
357
  */
287
- private matchesFilter(
288
- event: WorkflowEvent,
289
- endpoint: WebhookEndpoint,
290
- ): boolean {
358
+ private matchesFilter(event: WorkflowEvent, endpoint: WebhookEndpoint): boolean {
291
359
  // Check event type filter
292
360
  if (endpoint.eventTypes && endpoint.eventTypes.length > 0) {
293
361
  if (!endpoint.eventTypes.includes(event.eventType)) {
@@ -308,10 +376,7 @@ export class WebhookEventTransport implements EventTransport {
308
376
  /**
309
377
  * Send a webhook with retry support.
310
378
  */
311
- private async sendWebhook(
312
- endpoint: WebhookEndpoint,
313
- event: WorkflowEvent,
314
- ): Promise<void> {
379
+ private async sendWebhook(endpoint: WebhookEndpoint, event: WorkflowEvent): Promise<void> {
315
380
  const timeout = endpoint.timeout ?? this.defaultTimeout;
316
381
  const maxRetries = endpoint.retries ?? this.defaultRetries;
317
382
 
@@ -329,39 +394,50 @@ export class WebhookEventTransport implements EventTransport {
329
394
  // Enforce payload size limit to prevent excessive memory use and network abuse
330
395
  if (body.length > this.maxPayloadBytes) {
331
396
  throw new Error(
332
- `Webhook payload exceeds maximum size (${body.length} bytes > ${this.maxPayloadBytes} bytes)`,
397
+ `Webhook payload exceeds maximum size (${body.length} bytes > ${this.maxPayloadBytes} bytes)`
333
398
  );
334
399
  }
335
400
 
336
401
  const headers: Record<string, string> = {
337
- "Content-Type": "application/json",
402
+ 'Content-Type': 'application/json',
338
403
  ...endpoint.headers,
339
404
  };
340
405
 
341
406
  // Sign payload if secret is provided
342
407
  if (endpoint.secret) {
343
408
  const signature = await this.signPayload(body, endpoint.secret);
344
- headers["X-Webhook-Signature"] = signature;
409
+ headers['X-Webhook-Signature'] = signature;
345
410
  }
346
411
 
347
- // Resolve hostname and validate the resolved IP is not private/reserved (SSRF protection).
348
- // This prevents DNS rebinding attacks where a hostname initially resolves to a public IP
349
- // but later resolves to a private IP.
350
- await this.validateResolvedHost(endpoint.url);
351
-
352
412
  let lastError: Error | undefined;
353
413
 
354
414
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
355
415
  try {
416
+ // Resolve the hostname once and validate the resulting IP is not
417
+ // private/reserved (SSRF protection). When the default fetch is used,
418
+ // the resolved IP is then pinned for the connection so a hostile DNS
419
+ // server cannot rebind the hostname to a different address between
420
+ // validation and connect (M1: rebinding TOCTOU). The lookup runs
421
+ // inside the retry loop so DNS failures are retried along with
422
+ // network errors.
423
+ const pinned = await this.validateResolvedHost(endpoint.url);
424
+
356
425
  const controller = new AbortController();
357
426
  const timeoutId = setTimeout(() => controller.abort(), timeout);
358
427
 
359
- const response = await this.fetchFn(endpoint.url, {
360
- method: "POST",
361
- headers,
362
- body,
363
- signal: controller.signal,
364
- });
428
+ const response = this.hasCustomFetch || !pinned
429
+ ? await this.fetchFn(endpoint.url, {
430
+ method: 'POST',
431
+ headers,
432
+ body,
433
+ signal: controller.signal,
434
+ })
435
+ : await pinnedFetch(
436
+ endpoint.url,
437
+ { method: 'POST', headers, body, signal: controller.signal },
438
+ pinned.address,
439
+ pinned.family,
440
+ );
365
441
 
366
442
  clearTimeout(timeoutId);
367
443
 
@@ -369,9 +445,7 @@ export class WebhookEventTransport implements EventTransport {
369
445
  return; // Success
370
446
  }
371
447
 
372
- lastError = new Error(
373
- `Webhook returned ${response.status}: ${response.statusText}`,
374
- );
448
+ lastError = new Error(`Webhook returned ${response.status}: ${response.statusText}`);
375
449
  } catch (error) {
376
450
  lastError = error instanceof Error ? error : new Error(String(error));
377
451
  }
@@ -395,18 +469,16 @@ export class WebhookEventTransport implements EventTransport {
395
469
  const data = encoder.encode(payload);
396
470
 
397
471
  const key = await crypto.subtle.importKey(
398
- "raw",
472
+ 'raw',
399
473
  keyData,
400
- { name: "HMAC", hash: "SHA-256" },
474
+ { name: 'HMAC', hash: 'SHA-256' },
401
475
  false,
402
- ["sign"],
476
+ ['sign']
403
477
  );
404
478
 
405
- const signature = await crypto.subtle.sign("HMAC", key, data);
479
+ const signature = await crypto.subtle.sign('HMAC', key, data);
406
480
  const hashArray = Array.from(new Uint8Array(signature));
407
- const hashHex = hashArray
408
- .map((b) => b.toString(16).padStart(2, "0"))
409
- .join("");
481
+ const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
410
482
 
411
483
  return `sha256=${hashHex}`;
412
484
  }
@@ -415,39 +487,83 @@ export class WebhookEventTransport implements EventTransport {
415
487
  * Sleep helper.
416
488
  */
417
489
  private sleep(ms: number): Promise<void> {
418
- return new Promise((resolve) => setTimeout(resolve, ms));
490
+ return new Promise(resolve => setTimeout(resolve, ms));
419
491
  }
420
492
 
421
493
  /**
422
- * Resolve the hostname from a URL and verify the resolved IP is not private/reserved.
423
- * Prevents DNS rebinding attacks where a public hostname resolves to a private IP at send time.
494
+ * Resolve a webhook URL's hostname exactly once, verify the resolved IP is
495
+ * not private/reserved, and return the address + family so the caller can
496
+ * pin the connection to it.
497
+ *
498
+ * Returns `null` for raw-IP URLs (already validated by `validateWebhookUrl`).
499
+ *
500
+ * Pinning the resolved IP closes the DNS rebinding TOCTOU window (audit M1):
501
+ * with the original implementation `fetch()` re-resolved the hostname,
502
+ * giving a hostile DNS server a chance to flip the answer to an internal
503
+ * address between SSRF validation and connect.
504
+ *
505
+ * Uses a short-TTL cache and per-call timeout to avoid blocking the event loop.
424
506
  */
425
- private async validateResolvedHost(url: string): Promise<void> {
507
+ private async validateResolvedHost(
508
+ url: string,
509
+ ): Promise<{ address: string; family: 4 | 6 } | null> {
426
510
  const parsed = new URL(url);
427
511
  const hostname = parsed.hostname.toLowerCase();
428
512
 
429
- // Skip DNS resolution for raw IP addresses — already checked by isBlockedHost in validateWebhookUrl
430
- if (/^(\d{1,3}\.){3}\d{1,3}$/.test(hostname)) {
431
- return;
513
+ // Skip DNS resolution for raw IP literals — already checked by isBlockedHost in validateWebhookUrl.
514
+ // Node's URL parser wraps IPv6 literals in brackets (e.g., "[2001:db8::1]").
515
+ const stripped = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname;
516
+ if (isIPv4(stripped) || isIPv6(stripped)) {
517
+ return null;
518
+ }
519
+
520
+ // Check cache first
521
+ const cached = this.dnsCache.get(hostname);
522
+ if (cached && cached.expiresAt > Date.now()) {
523
+ if (isBlockedIp(cached.address)) {
524
+ throw new Error(
525
+ `Webhook URL hostname "${hostname}" resolves to blocked IP ${cached.address}`
526
+ );
527
+ }
528
+ return { address: cached.address, family: cached.family };
432
529
  }
433
530
 
434
531
  try {
435
- const { address } = await dns.lookup(hostname);
532
+ const ac = new AbortController();
533
+ const timeoutId = setTimeout(() => ac.abort(), WebhookEventTransport.DNS_TIMEOUT_MS);
534
+
535
+ let address: string;
536
+ let family: 4 | 6;
537
+ try {
538
+ const result = await dns.lookup(hostname, { signal: ac.signal } as any);
539
+ address = result.address;
540
+ family = (result.family === 6 ? 6 : 4) as 4 | 6;
541
+ } finally {
542
+ clearTimeout(timeoutId);
543
+ }
544
+
545
+ // Cache the result
546
+ this.dnsCache.set(hostname, {
547
+ address,
548
+ family,
549
+ expiresAt: Date.now() + WebhookEventTransport.DNS_CACHE_TTL_MS,
550
+ });
551
+
436
552
  if (isBlockedIp(address)) {
437
553
  throw new Error(
438
- `Webhook URL hostname "${hostname}" resolves to blocked IP ${address}`,
554
+ `Webhook URL hostname "${hostname}" resolves to blocked IP ${address}`
439
555
  );
440
556
  }
557
+
558
+ return { address, family };
441
559
  } catch (error) {
442
- if (
443
- error instanceof Error &&
444
- error.message.includes("resolves to blocked IP")
445
- ) {
560
+ if (error instanceof Error && error.message.includes('resolves to blocked IP')) {
446
561
  throw error;
447
562
  }
448
- throw new Error(
449
- `Failed to resolve webhook URL hostname "${hostname}": ${error}`,
450
- );
563
+ if (error instanceof Error && error.name === 'AbortError') {
564
+ throw new Error(`DNS lookup for webhook URL hostname "${hostname}" timed out`);
565
+ }
566
+ throw new Error(`Failed to resolve webhook URL hostname "${hostname}": ${error}`);
451
567
  }
452
568
  }
453
569
 
@@ -465,31 +581,31 @@ export class WebhookEventTransport implements EventTransport {
465
581
  }
466
582
 
467
583
  // Scheme check
468
- if (!this.allowInsecureUrls && parsed.protocol !== "https:") {
584
+ if (!this.allowInsecureUrls && parsed.protocol !== 'https:') {
469
585
  throw new Error(
470
- `Webhook URL must use HTTPS (got ${parsed.protocol}). Set allowInsecureUrls for development.`,
586
+ `Webhook URL must use HTTPS (got ${parsed.protocol}). Set allowInsecureUrls for development.`
471
587
  );
472
588
  }
473
589
 
474
- if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
475
- throw new Error(
476
- `Webhook URL must use HTTP(S) scheme (got ${parsed.protocol})`,
477
- );
590
+ if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
591
+ throw new Error(`Webhook URL must use HTTP(S) scheme (got ${parsed.protocol})`);
478
592
  }
479
593
 
480
594
  // Block dangerous hostnames
481
595
  const hostname = parsed.hostname.toLowerCase();
482
596
  if (isBlockedHost(hostname)) {
483
- throw new Error(
484
- `Webhook URL hostname is blocked (private/reserved): ${hostname}`,
485
- );
597
+ throw new Error(`Webhook URL hostname is blocked (private/reserved): ${hostname}`);
486
598
  }
487
599
  }
488
600
 
489
601
  /**
490
- * Enqueue a webhook request with concurrency limiting.
602
+ * Enqueue a webhook request with concurrency limiting and a queue-depth cap.
603
+ * Returns `true` if the request was accepted (executed or queued), or
604
+ * `false` if the queue is full and the request was rejected. The caller is
605
+ * responsible for logging the drop and incrementing any drop metrics, since
606
+ * it has the endpoint context.
491
607
  */
492
- private enqueueRequest(fn: () => Promise<void>): void {
608
+ private enqueueRequest(fn: () => Promise<void>, _endpointId: string): boolean {
493
609
  const execute = () => {
494
610
  this.activeRequests++;
495
611
  fn().finally(() => {
@@ -501,60 +617,54 @@ export class WebhookEventTransport implements EventTransport {
501
617
 
502
618
  if (this.activeRequests < this.maxConcurrentRequests) {
503
619
  execute();
504
- } else {
505
- this.requestQueue.push(execute);
620
+ return true;
621
+ }
622
+
623
+ if (this.maxQueueDepth > 0 && this.requestQueue.length >= this.maxQueueDepth) {
624
+ return false;
506
625
  }
626
+
627
+ this.requestQueue.push(execute);
628
+ return true;
507
629
  }
508
630
  }
509
631
 
510
632
  /**
511
- * Check if a hostname resolves to a blocked (private/reserved) address.
512
- * Blocks loopback, RFC 1918 private ranges, link-local, and cloud metadata IPs.
633
+ * Resolve the effective `maxQueueDepth` from explicit config and the
634
+ * `STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH` environment variable.
635
+ *
636
+ * Precedence: explicit config value > env var > default (1000).
513
637
  */
514
- function isBlockedHost(hostname: string): boolean {
515
- // Loopback (Node's URL parser wraps IPv6 in brackets, e.g. "[::1]")
516
- if (
517
- hostname === "localhost" ||
518
- hostname === "127.0.0.1" ||
519
- hostname === "::1" ||
520
- hostname === "[::1]"
521
- ) {
522
- return true;
638
+ function resolveMaxQueueDepth(configured: number | undefined): number {
639
+ if (typeof configured === 'number') return configured;
640
+ const env = typeof process !== 'undefined' ? process.env?.STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH : undefined;
641
+ if (env !== undefined && env !== '') {
642
+ const parsed = Number(env);
643
+ if (Number.isFinite(parsed) && parsed >= 0) return parsed;
523
644
  }
645
+ return 1000;
646
+ }
524
647
 
525
- // Block all IPv6 address literals (wrapped in brackets by URL parser)
526
- if (hostname.startsWith("[")) {
527
- return true;
528
- }
648
+ /**
649
+ * Check if a hostname (as it appears in a URL) is blocked.
650
+ * Handles bare hostnames, IPv4 literals, and bracketed IPv6 literals.
651
+ */
652
+ function isBlockedHost(hostname: string): boolean {
653
+ if (hostname === 'localhost') return true;
529
654
 
530
- // Cloud metadata endpoint
531
- if (hostname === "169.254.169.254") {
532
- return true;
655
+ // IPv6 literals are wrapped in brackets by Node's URL parser, e.g. "[::1]".
656
+ if (hostname.startsWith('[') && hostname.endsWith(']')) {
657
+ const bare = hostname.slice(1, -1);
658
+ return isIPv6(bare) ? isBlockedIPv6(bare) : true;
533
659
  }
534
660
 
535
- // Check for IP addresses in private/reserved ranges
536
- const ipv4Match = hostname.match(
537
- /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/,
538
- );
539
- if (ipv4Match) {
540
- const [, a, b] = ipv4Match.map(Number);
541
- // 10.0.0.0/8
542
- if (a === 10) return true;
543
- // 172.16.0.0/12
544
- if (a === 172 && b >= 16 && b <= 31) return true;
545
- // 192.168.0.0/16
546
- if (a === 192 && b === 168) return true;
547
- // 127.0.0.0/8
548
- if (a === 127) return true;
549
- // 169.254.0.0/16 (link-local)
550
- if (a === 169 && b === 254) return true;
551
- // 0.0.0.0
552
- if (a === 0 && b === 0) return true;
661
+ // Some callers (or legacy code paths) may pass an unwrapped IPv6 literal.
662
+ if (isIPv6(hostname)) {
663
+ return isBlockedIPv6(hostname);
553
664
  }
554
665
 
555
- // IPv6 link-local (fe80::)
556
- if (hostname.startsWith("fe80:") || hostname.startsWith("[fe80:")) {
557
- return true;
666
+ if (isIPv4(hostname)) {
667
+ return isBlockedIPv4(hostname);
558
668
  }
559
669
 
560
670
  return false;
@@ -565,27 +675,133 @@ function isBlockedHost(hostname: string): boolean {
565
675
  * Used for runtime DNS resolution validation to prevent SSRF via DNS rebinding.
566
676
  */
567
677
  function isBlockedIp(ip: string): boolean {
568
- const ipv4Match = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
569
- if (ipv4Match) {
570
- const [, a, b] = ipv4Match.map(Number);
571
- // 127.0.0.0/8 (loopback)
572
- if (a === 127) return true;
573
- // 10.0.0.0/8
574
- if (a === 10) return true;
575
- // 172.16.0.0/12
576
- if (a === 172 && b >= 16 && b <= 31) return true;
577
- // 192.168.0.0/16
578
- if (a === 192 && b === 168) return true;
579
- // 169.254.0.0/16 (link-local, including cloud metadata 169.254.169.254)
580
- if (a === 169 && b === 254) return true;
581
- // 0.0.0.0
582
- if (a === 0 && b === 0) return true;
678
+ if (isIPv4(ip)) return isBlockedIPv4(ip);
679
+ if (isIPv6(ip)) return isBlockedIPv6(ip);
680
+ return false;
681
+ }
682
+
683
+ /**
684
+ * Block IPv4 addresses in private, loopback, link-local, and cloud-metadata ranges.
685
+ */
686
+ function isBlockedIPv4(ip: string): boolean {
687
+ const match = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
688
+ if (!match) return false;
689
+ const [, a, b] = match.map(Number);
690
+ return isBlockedIPv4Octets(a, b);
691
+ }
692
+
693
+ function isBlockedIPv4Octets(a: number, b: number): boolean {
694
+ // 0.0.0.0/8 — "this network"
695
+ if (a === 0) return true;
696
+ // 10.0.0.0/8 — RFC 1918 private
697
+ if (a === 10) return true;
698
+ // 100.64.0.0/10 — CGNAT
699
+ if (a === 100 && b >= 64 && b <= 127) return true;
700
+ // 127.0.0.0/8 — loopback
701
+ if (a === 127) return true;
702
+ // 169.254.0.0/16 — link-local (covers cloud metadata 169.254.169.254)
703
+ if (a === 169 && b === 254) return true;
704
+ // 172.16.0.0/12 — RFC 1918 private
705
+ if (a === 172 && b >= 16 && b <= 31) return true;
706
+ // 192.168.0.0/16 — RFC 1918 private
707
+ if (a === 192 && b === 168) return true;
708
+ // 224.0.0.0/4 multicast and 240.0.0.0/4 reserved (incl. 255.255.255.255 broadcast)
709
+ if (a >= 224) return true;
710
+ return false;
711
+ }
712
+
713
+ /**
714
+ * Block IPv6 addresses in reserved, private, loopback, link-local, IPv4-mapped,
715
+ * IPv4-translated, 6to4, and Teredo ranges. For mapped/embedded IPv4 forms,
716
+ * the embedded IPv4 is decoded and rechecked against the IPv4 blocklist.
717
+ */
718
+ function isBlockedIPv6(ip: string): boolean {
719
+ const groups = expandIPv6(ip);
720
+ if (!groups) return false;
721
+
722
+ // ::1 loopback and :: unspecified
723
+ const allButLastZero = groups.slice(0, 7).every(g => g === 0);
724
+ if (allButLastZero && groups[7] === 1) return true;
725
+ if (groups.every(g => g === 0)) return true;
726
+
727
+ // ::ffff:0:0/96 — IPv4-mapped IPv6 (decode embedded IPv4)
728
+ if (groups.slice(0, 5).every(g => g === 0) && groups[5] === 0xffff) {
729
+ return isBlockedIPv4Octets((groups[6] >> 8) & 0xff, groups[6] & 0xff);
583
730
  }
584
731
 
585
- // IPv6 loopback
586
- if (ip === "::1") return true;
587
- // IPv6 link-local
588
- if (ip.startsWith("fe80:")) return true;
732
+ // 64:ff9b::/96 — IPv4/IPv6 translation (RFC 6052) and 64:ff9b:1::/48 (RFC 8215)
733
+ if (groups[0] === 0x64 && groups[1] === 0xff9b && groups.slice(2, 6).every(g => g === 0)) {
734
+ return isBlockedIPv4Octets((groups[6] >> 8) & 0xff, groups[6] & 0xff);
735
+ }
736
+
737
+ // 2002::/16 — 6to4 (next 32 bits are the embedded IPv4)
738
+ if (groups[0] === 0x2002) {
739
+ return isBlockedIPv4Octets((groups[1] >> 8) & 0xff, groups[1] & 0xff);
740
+ }
741
+
742
+ // 2001::/32 — Teredo. Block conservatively; embedded server/client v4 fields
743
+ // can mask private destinations and Teredo isn't a normal webhook target.
744
+ if (groups[0] === 0x2001 && groups[1] === 0) return true;
745
+
746
+ // 100::/64 — RFC 6666 Discard prefix
747
+ if (groups[0] === 0x0100 && groups.slice(1, 4).every(g => g === 0)) return true;
748
+
749
+ // fc00::/7 — Unique Local Addresses (fc00:: through fdff::)
750
+ if ((groups[0] & 0xfe00) === 0xfc00) return true;
751
+
752
+ // fe80::/10 — Link-local (fe80:: through febf::)
753
+ if ((groups[0] & 0xffc0) === 0xfe80) return true;
754
+
755
+ // ff00::/8 — Multicast
756
+ if ((groups[0] & 0xff00) === 0xff00) return true;
589
757
 
590
758
  return false;
591
759
  }
760
+
761
+ /**
762
+ * Expand an IPv6 string into an array of 8 numeric hextets (0–0xffff).
763
+ * Handles "::" abbreviation, embedded IPv4 dotted-quad, and zone IDs.
764
+ * Returns null if the input is not a valid IPv6 address.
765
+ */
766
+ function expandIPv6(addr: string): number[] | null {
767
+ // Strip zone identifier (e.g., "fe80::1%eth0")
768
+ const noZone = addr.split('%')[0];
769
+ if (!isIPv6(noZone)) return null;
770
+
771
+ let normalized = noZone.toLowerCase();
772
+
773
+ // Convert trailing dotted-quad (e.g., "::ffff:192.168.0.1") into two hextets.
774
+ const v4Tail = normalized.match(/^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
775
+ if (v4Tail) {
776
+ const a = parseInt(v4Tail[2], 10);
777
+ const b = parseInt(v4Tail[3], 10);
778
+ const c = parseInt(v4Tail[4], 10);
779
+ const d = parseInt(v4Tail[5], 10);
780
+ if ([a, b, c, d].some(n => n > 255)) return null;
781
+ const hi = ((a << 8) | b).toString(16);
782
+ const lo = ((c << 8) | d).toString(16);
783
+ normalized = `${v4Tail[1]}${hi}:${lo}`;
784
+ }
785
+
786
+ const halves = normalized.split('::');
787
+ let parts: string[];
788
+ if (halves.length === 1) {
789
+ parts = halves[0].split(':');
790
+ if (parts.length !== 8) return null;
791
+ } else if (halves.length === 2) {
792
+ const head = halves[0] ? halves[0].split(':') : [];
793
+ const tail = halves[1] ? halves[1].split(':') : [];
794
+ const missing = 8 - head.length - tail.length;
795
+ if (missing < 0) return null;
796
+ parts = [...head, ...Array(missing).fill('0'), ...tail];
797
+ } else {
798
+ return null;
799
+ }
800
+
801
+ const groups: number[] = [];
802
+ for (const part of parts) {
803
+ if (!/^[0-9a-f]{1,4}$/.test(part)) return null;
804
+ groups.push(parseInt(part, 16));
805
+ }
806
+ return groups;
807
+ }