@hatchet-dev/typescript-sdk 1.35.1 → 1.37.0

package/v1/client/features/streams.js

@@ -34,6 +34,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamsClient = void 0;
 const crypto_1 = require("crypto");
+const axios_1 = require("axios");
 const nice_grpc_1 = require("nice-grpc");
 const grpc_helpers_1 = require("../../../util/grpc-helpers");
 const grpc_error_1 = require("../../../util/grpc-error");
@@ -58,7 +59,27 @@ const permanentSubscribeErrors = new Set([
 ]);
 const resubscribeMinDelayMs = 250;
 const resubscribeMaxDelayMs = 10000;
+// the server's limit for a payload sent over gRPC (MaxStreamMessagePayloadBytes)
+const maxInlinePayloadBytes = 4 * 1024 * 1024 - 5 * 1024;
+// the server's limit for an uploaded payload (MaxStreamUploadedPayloadBytes)
+const maxUploadedPayloadBytes = 64 * 1024 * 1024;
+const payloadRequestAttempts = 3;
+function httpStatus(err) {
+    var _a;
+    return (0, axios_1.isAxiosError)(err) ? (_a = err.response) === null || _a === void 0 ? void 0 : _a.status : undefined;
+}
+// a payload fetch that got no response, a 5xx or a 429 may succeed later
+function isTemporaryHttpError(err) {
+    if (!(0, axios_1.isAxiosError)(err)) {
+        return false;
+    }
+    const status = httpStatus(err);
+    return status === undefined || status >= 500 || status === 429;
+}
 function shouldResubscribe(err, attempt) {
+    if (isTemporaryHttpError(err)) {
+        return true;
+    }
     const code = (0, grpc_error_1.getGrpcErrorCode)(err);
     if (code === undefined || permanentSubscribeErrors.has(code)) {
         return false;
@@ -82,6 +103,7 @@ class StreamsClient {
         // stored promise always resolves, so one failure doesn't block later calls
         this.publishChains = new Map();
         this._config = client.config;
+        this._client = client;
     }
     get grpc() {
         if (!this._grpc) {
@@ -90,21 +112,46 @@ class StreamsClient {
         }
         return this._grpc;
     }
-    publishOrdered(key_1, namespace_1, topic_1, payload_1) {
-        return __awaiter(this, arguments, void 0, function* (key, namespace, topic, payload, retryGap = true) {
+    // retries a 5xx or network failure; a 4xx won't succeed on retry
+    static withRetries(request) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (let attempt = 1;; attempt += 1) {
+                try {
+                    return yield request();
+                }
+                catch (err) {
+                    const status = httpStatus(err);
+                    if ((status !== undefined && status < 500) || attempt >= payloadRequestAttempts) {
+                        throw err;
+                    }
+                }
+            }
+        });
+    }
+    uploadPayload(payload) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield StreamsClient.withRetries(() => this._client.api.v1StreamPayloadUpload(this._config.tenant_id, {
+                payload: new Blob([payload]),
+            }));
+            return res.data.ref;
+        });
+    }
+    fetchPayload(payloadRef, signal) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield StreamsClient.withRetries(() => this._client.api.v1StreamPayloadGet(this._config.tenant_id, { ref: payloadRef }, { format: 'arraybuffer', signal }));
+            return new Uint8Array(res.data);
+        });
+    }
+    publishOrdered(key_1, namespace_1, topic_1, body_1) {
+        return __awaiter(this, arguments, void 0, function* (key, namespace, topic, body, retryGap = true) {
             let producer = this.producers.get(key);
             if (!producer) {
                 producer = { producerId: (0, crypto_1.randomUUID)(), seq: 0 };
                 this.producers.set(key, producer);
             }
             try {
-                yield this.grpc.publish({
-                    namespace,
-                    topic,
-                    payload,
-                    producerId: producer.producerId,
-                    producerSeq: producer.seq,
-                });
+                yield this.grpc.publish(Object.assign(Object.assign({ namespace,
+                    topic }, body), { producerId: producer.producerId, producerSeq: producer.seq }));
             }
             catch (err) {
                 // it may still land, and a different payload under its seq would be dropped as a duplicate
@@ -113,7 +160,7 @@ class StreamsClient {
                 }
                 // a gap stored nothing (e.g. the watermark passed cursor retention), so resend as the new producer
                 if (retryGap && (0, grpc_error_1.getGrpcErrorCode)(err) === nice_grpc_1.Status.FAILED_PRECONDITION) {
-                    return this.publishOrdered(key, namespace, topic, payload, false);
+                    return this.publishOrdered(key, namespace, topic, body, false);
                 }
                 throw err;
             }
@@ -133,12 +180,41 @@ class StreamsClient {
             const payload = typeof message === 'string' ? new TextEncoder().encode(message) : message;
             const namespace = (_a = options === null || options === void 0 ? void 0 : options.namespace) !== null && _a !== void 0 ? _a : '';
             const key = `${namespace}\u0000${topic}`;
+            if (payload.byteLength > maxUploadedPayloadBytes) {
+                throw new Error(`payload exceeds maximum size of ${maxUploadedPayloadBytes} bytes`);
+            }
+            // uploaded before taking its place in the topic's order, so uploads overlap
+            const uploaded = payload.byteLength > maxInlinePayloadBytes ? this.uploadPayload(payload) : undefined;
+            // awaited in order below; this only keeps an early failure from reporting as unhandled
+            uploaded === null || uploaded === void 0 ? void 0 : uploaded.catch(() => { });
             const previous = (_b = this.publishChains.get(key)) !== null && _b !== void 0 ? _b : Promise.resolve();
-            const publishPromise = previous.then(() => this.publishOrdered(key, namespace, topic, payload));
+            const publishPromise = previous.then(() => __awaiter(this, void 0, void 0, function* () {
+                return this.publishOrdered(key, namespace, topic, uploaded ? { payloadRef: yield uploaded } : { payload });
+            }));
             this.publishChains.set(key, publishPromise.catch(() => { }));
             return publishPromise;
         });
     }
+    /**
+     * Describes topic's retained messages. Rejects with a NOT_FOUND gRPC error
+     * for a topic nothing was published to.
+     * @param topic - the topic to describe
+     * @param options - optional namespace override
+     */
+    topicMetadata(topic, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const res = yield this.grpc.getTopicMetadata({ namespace: (_a = options === null || options === void 0 ? void 0 : options.namespace) !== null && _a !== void 0 ? _a : '', topic });
+            return {
+                namespace: res.namespace,
+                topic: res.topic,
+                tenantId: res.tenantId,
+                messageCount: res.messageCount,
+                latestCursor: res.latestCursor,
+                lastPublishedAt: res.lastPublishedAt,
+            };
+        });
+    }
     /**
      * Yields topic's messages after options.cursor, or from the oldest retained.
      * If the connection drops, it resubscribes from the last message yielded, so
@@ -167,11 +243,15 @@ class StreamsClient {
                             if (msg.hangup) {
                                 return yield __await(void 0);
                             }
-                            failures = 0;
                             for (const entry of msg.entries) {
+                                const payload = entry.payloadRef
+                                    ? yield __await(this.fetchPayload(entry.payloadRef, signal))
+                                    : entry.payload;
                                 ({ cursor } = entry);
+                                // reset only on delivery, so a payload fetch failing on every reconnect still backs off
+                                failures = 0;
                                 yield yield __await({
-                                    payload: entry.payload,
+                                    payload,
                                     cursor: entry.cursor,
                                     createdAt: entry.createdAt,
                                 });

package/v1/client/worker/engine-version.d.ts

@@ -1,5 +1,7 @@
 export declare const MinEngineVersion: {
     readonly SLOT_CONFIG: "v0.78.23";
     readonly DURABLE_EVICTION: "v0.80.0";
+    readonly RETRY_AFTER: "v0.111.0";
 };
+export declare function supportsRetryAfter(engineVersion: string | undefined): boolean;
 export declare function supportsEviction(engineVersion: string | undefined): boolean;

package/v1/client/worker/engine-version.js

@@ -1,12 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MinEngineVersion = void 0;
+exports.supportsRetryAfter = supportsRetryAfter;
 exports.supportsEviction = supportsEviction;
 const deprecation_1 = require("./deprecated/deprecation");
 exports.MinEngineVersion = {
     SLOT_CONFIG: 'v0.78.23',
     DURABLE_EVICTION: 'v0.80.0',
+    RETRY_AFTER: 'v0.111.0',
 };
+function supportsRetryAfter(engineVersion) {
+    if (!engineVersion)
+        return false;
+    return !(0, deprecation_1.semverLessThan)(engineVersion, exports.MinEngineVersion.RETRY_AFTER);
+}
 function supportsEviction(engineVersion) {
     if (!engineVersion)
         return false;

package/v1/client/worker/worker-internal.d.ts

@@ -69,6 +69,7 @@ export declare class InternalWorker {
      * Keep the signature in sync with the instrumentor wrapper.
      */
     handleStartStepRun(action: Action): Promise<Error | undefined>;
+    private retryAfterMs;
     getStepActionEvent(action: Action, eventType: StepActionEventType, shouldNotRetry: boolean, payload?: any, retryCount?: number): StepActionEvent;
     /**
      * Handles a START_BATCH action: invokes the registered batch task handler once with
@@ -88,6 +89,7 @@ export declare class InternalWorker {
     getBatchActionEvent(action: Action, eventType: StepActionEventType, items: {
         taskRunExternalId: string;
         eventPayload: string;
+        retryAfterMs?: number;
     }[]): BatchActionEvent;
     getGroupKeyActionEvent(action: Action, eventType: GroupKeyActionEventType, payload?: any): GroupKeyActionEvent;
     /**

package/v1/client/worker/worker-internal.js

@@ -349,6 +349,7 @@ class InternalWorker {
                 });
                 const failure = (error) => __awaiter(this, void 0, void 0, function* () {
                     const shouldNotRetry = error instanceof task_1.NonRetryableError;
+                    const retryAfterMs = this.retryAfterMs(error, actionId);
                     try {
                         if (context.cancelled) {
                             return;
@@ -366,6 +367,7 @@ class InternalWorker {
                             message: error === null || error === void 0 ? void 0 : error.message,
                             stack: error === null || error === void 0 ? void 0 : error.stack,
                         }, action.retryCount);
+                        event.retryAfterMs = retryAfterMs;
                         yield this.client.dispatcher.sendStepActionEvent(event);
                     }
                     catch (e) {
@@ -423,6 +425,18 @@ class InternalWorker {
             }
         });
     }
+    retryAfterMs(error, actionId) {
+        var _a;
+        if (!(error instanceof task_1.RetryAfterError)) {
+            return undefined;
+        }
+        if (!(0, engine_version_1.supportsRetryAfter)(this.engineVersion)) {
+            this.logger.warn(`RetryAfterError requires engine >= ${engine_version_1.MinEngineVersion.RETRY_AFTER} ` +
+                `(connected: ${(_a = this.engineVersion) !== null && _a !== void 0 ? _a : 'unknown'}). Task ${actionId} threw it, so the engine ` +
+                `will handle the failure with the task's retry policy instead of the requested delay.`);
+        }
+        return error.afterMs;
+    }
     getStepActionEvent(action, eventType, shouldNotRetry, payload = '', retryCount = 0) {
         return {
             workerId: this.name,
@@ -518,7 +532,12 @@ class InternalWorker {
     sendBatchFailureForAll(action, memberIds, error) {
         return __awaiter(this, void 0, void 0, function* () {
             const payload = JSON.stringify({ message: error === null || error === void 0 ? void 0 : error.message, stack: error === null || error === void 0 ? void 0 : error.stack });
-            const items = memberIds.map((id) => ({ taskRunExternalId: id, eventPayload: payload }));
+            const retryAfterMs = this.retryAfterMs(error, action.actionId);
+            const items = memberIds.map((id) => ({
+                taskRunExternalId: id,
+                eventPayload: payload,
+                retryAfterMs,
+            }));
             try {
                 yield this.client.dispatcher.sendBatchActionEvent(this.getBatchActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, items));
             }

package/v1/examples/durable-streams/benchmark.d.ts

@@ -0,0 +1,34 @@
+import { HatchetClient } from '../..';
+import { StreamEvent } from '../../client/features/streams';
+export declare const MiB: number;
+export declare const maxInlinePayloadBytes: number;
+export declare const describeBenchmark: jest.Describe;
+export type LatencyStats = {
+    count: number;
+    avgMs: number;
+    p50Ms: number;
+    p99Ms: number;
+    maxMs: number;
+};
+export declare function latencyStats(durationsMs: number[]): LatencyStats;
+export declare function formatLatency(stats: LatencyStats, digits?: number): string;
+export declare function formatSize(bytes: number): string;
+export type PublishLatencyStats = LatencyStats & {
+    throughputPerSec: number;
+};
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+export declare function measurePublishLatency(hatchet: HatchetClient, topic: string, payload: Uint8Array | string, count: number, warmupCount?: number): Promise<PublishLatencyStats>;
+export declare function formatPublishLatency(stats: PublishLatencyStats): string;
+export type LiveSubscription = {
+    /** resolves with the next message delivered after the call */
+    nextEvent: () => Promise<StreamEvent>;
+    close: () => Promise<void>;
+};
+/**
+ * Subscribes to topic and resolves once the subscription is live, so a
+ * benchmark's timings don't include subscribe setup.
+ */
+export declare function openLiveSubscription(hatchet: HatchetClient, topic: string): Promise<LiveSubscription>;

package/v1/examples/durable-streams/benchmark.js

@@ -0,0 +1,120 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.describeBenchmark = exports.maxInlinePayloadBytes = exports.MiB = void 0;
+exports.latencyStats = latencyStats;
+exports.formatLatency = formatLatency;
+exports.formatSize = formatSize;
+exports.measurePublishLatency = measurePublishLatency;
+exports.formatPublishLatency = formatPublishLatency;
+exports.openLiveSubscription = openLiveSubscription;
+const perf_hooks_1 = require("perf_hooks");
+exports.MiB = 1024 * 1024;
+// the server's per-message gRPC limit (MaxStreamMessagePayloadBytes); larger payloads are uploaded
+exports.maxInlinePayloadBytes = 4 * exports.MiB - 5 * 1024;
+// benchmarks run only when asked for (test_durable_streams.sh --bench)
+exports.describeBenchmark = process.env.HATCHET_E2E_BENCHMARKS ? describe : describe.skip;
+function percentile(sorted, p) {
+    return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))];
+}
+function latencyStats(durationsMs) {
+    const sorted = [...durationsMs].sort((a, b) => a - b);
+    return {
+        count: durationsMs.length,
+        avgMs: durationsMs.reduce((sum, d) => sum + d, 0) / durationsMs.length,
+        p50Ms: percentile(sorted, 0.5),
+        p99Ms: percentile(sorted, 0.99),
+        maxMs: sorted[sorted.length - 1],
+    };
+}
+function formatLatency(stats, digits = 2) {
+    const fmt = (ms) => ms.toFixed(digits);
+    return `avg=${fmt(stats.avgMs)}ms p50=${fmt(stats.p50Ms)}ms p99=${fmt(stats.p99Ms)}ms max=${fmt(stats.maxMs)}ms`;
+}
+function formatSize(bytes) {
+    if (bytes % exports.MiB === 0)
+        return `${bytes / exports.MiB}MiB`;
+    if (bytes % 1024 === 0)
+        return `${bytes / 1024}KiB`;
+    return `${bytes}B`;
+}
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+function measurePublishLatency(hatchet_1, topic_1, payload_1, count_1) {
+    return __awaiter(this, arguments, void 0, function* (hatchet, topic, payload, count, warmupCount = 5) {
+        for (let i = 0; i < warmupCount; i += 1) {
+            yield hatchet.streams.publish(topic, payload);
+        }
+        const durationsMs = [];
+        const start = perf_hooks_1.performance.now();
+        for (let i = 0; i < count; i += 1) {
+            const before = perf_hooks_1.performance.now();
+            yield hatchet.streams.publish(topic, payload);
+            durationsMs.push(perf_hooks_1.performance.now() - before);
+        }
+        const totalMs = perf_hooks_1.performance.now() - start;
+        return Object.assign(Object.assign({}, latencyStats(durationsMs)), { throughputPerSec: (count / totalMs) * 1000 });
+    });
+}
+function formatPublishLatency(stats) {
+    return `${formatLatency(stats)} throughput=${stats.throughputPerSec.toFixed(2)}/s`;
+}
+/**
+ * Subscribes to topic and resolves once the subscription is live, so a
+ * benchmark's timings don't include subscribe setup.
+ */
+function openLiveSubscription(hatchet, topic) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const abort = new AbortController();
+        let onEvent;
+        const consumer = (() => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            try {
+                for (var _d = true, _e = __asyncValues(hatchet.streams.events(topic, { signal: abort.signal })), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const ev = _c;
+                    onEvent === null || onEvent === void 0 ? void 0 : onEvent(ev);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        }))();
+        const nextEvent = () => new Promise((resolve) => {
+            onEvent = resolve;
+        });
+        // the subscription is live once a first message comes through
+        const ready = nextEvent();
+        yield hatchet.streams.publish(topic, 'ready');
+        yield ready;
+        return {
+            nextEvent,
+            close: () => __awaiter(this, void 0, void 0, function* () {
+                abort.abort();
+                yield consumer;
+            }),
+        };
+    });
+}

package/v1/examples/e2e-worker.js

@@ -35,21 +35,22 @@ const workflow_14 = require("./durable_callback_ordering/workflow");
 const workflow_15 = require("./durable_sleep/workflow");
 const workflow_16 = require("./logger/workflow");
 const workflow_17 = require("./non_retryable/workflow");
-const workflow_18 = require("./on_failure/workflow");
-const workflow_19 = require("./idempotency/workflow");
-const workflow_20 = require("./on_event/workflow");
-const workflow_21 = require("./return_exceptions/workflow");
-const workflow_22 = require("./run_details/workflow");
+const workflow_18 = require("./retry_after/workflow");
+const workflow_19 = require("./on_failure/workflow");
+const workflow_20 = require("./idempotency/workflow");
+const workflow_21 = require("./on_event/workflow");
+const workflow_22 = require("./return_exceptions/workflow");
+const workflow_23 = require("./run_details/workflow");
 const e2e_workflows_1 = require("./simple/e2e-workflows");
-const workflow_23 = require("./batch_assign/workflow");
-const workflow_24 = require("./streaming/workflow");
-const workflow_25 = require("./subscribe_to_stream/workflow");
-const workflow_26 = require("./timeout/workflow");
-const workflow_27 = require("./webhooks/workflow");
-const workflow_28 = require("./child_index/workflow");
-const workflow_29 = require("./support_agent/workflow");
-const workflow_30 = require("./welcome_email/workflow");
-const workflow_31 = require("./pdf_pipeline/workflow");
+const workflow_24 = require("./batch_assign/workflow");
+const workflow_25 = require("./streaming/workflow");
+const workflow_26 = require("./subscribe_to_stream/workflow");
+const workflow_27 = require("./timeout/workflow");
+const workflow_28 = require("./webhooks/workflow");
+const workflow_29 = require("./child_index/workflow");
+const workflow_30 = require("./support_agent/workflow");
+const workflow_31 = require("./welcome_email/workflow");
+const workflow_32 = require("./pdf_pipeline/workflow");
 const workflows = [
     workflow_1.bulkChild,
     workflow_1.bulkParentWorkflow,
@@ -101,43 +102,45 @@ const workflows = [
     workflow_13.evictableChildBulkSpawn,
     (0, workflow_16.createLoggingWorkflow)(hatchet_client_1.hatchet),
     workflow_17.nonRetryableWorkflow,
-    workflow_18.failureWorkflow,
-    workflow_19.idempotentTask,
-    workflow_19.idempotentTaskShortWindow,
-    workflow_20.lower,
-    workflow_21.returnExceptionsTask,
-    workflow_22.runDetailTestWorkflow,
+    workflow_18.retryAfterUpstreamDelay,
+    workflow_18.retryAfterExponentialBackoff,
+    workflow_19.failureWorkflow,
+    workflow_20.idempotentTask,
+    workflow_20.idempotentTaskShortWindow,
+    workflow_21.lower,
+    workflow_22.returnExceptionsTask,
+    workflow_23.runDetailTestWorkflow,
     e2e_workflows_1.helloWorld,
     e2e_workflows_1.helloWorldDurable,
-    workflow_24.streamingTask,
-    workflow_25.dagStream,
-    workflow_25.longStream,
-    workflow_26.timeoutTask,
-    workflow_26.refreshTimeoutTask,
-    workflow_27.webhookWorkflow,
-    workflow_28.childIndexChild,
-    workflow_28.childIndexParent,
-    workflow_28.scenarioTask,
-    workflow_28.orchestratorTask,
-    workflow_29.supportAgent,
-    workflow_29.triageTicket,
-    workflow_29.generateReply,
-    workflow_29.escalateTicket,
-    workflow_30.welcomeEmail,
-    workflow_31.pdfPipeline,
-    workflow_23.batchSimple,
-    workflow_23.batchKeyed,
-    workflow_23.batchKeyedFailable,
-    workflow_23.batchKeyedInterval,
-    workflow_23.batchLarge,
-    workflow_23.batchSingle,
-    workflow_23.batchOrdered,
-    workflow_23.batchBroadcast,
-    workflow_23.batchCancel,
-    workflow_23.child,
-    workflow_23.childBatch,
-    workflow_23.batchChildSpawn,
-    workflow_23.batchChildBatchSpawn,
+    workflow_25.streamingTask,
+    workflow_26.dagStream,
+    workflow_26.longStream,
+    workflow_27.timeoutTask,
+    workflow_27.refreshTimeoutTask,
+    workflow_28.webhookWorkflow,
+    workflow_29.childIndexChild,
+    workflow_29.childIndexParent,
+    workflow_29.scenarioTask,
+    workflow_29.orchestratorTask,
+    workflow_30.supportAgent,
+    workflow_30.triageTicket,
+    workflow_30.generateReply,
+    workflow_30.escalateTicket,
+    workflow_31.welcomeEmail,
+    workflow_32.pdfPipeline,
+    workflow_24.batchSimple,
+    workflow_24.batchKeyed,
+    workflow_24.batchKeyedFailable,
+    workflow_24.batchKeyedInterval,
+    workflow_24.batchLarge,
+    workflow_24.batchSingle,
+    workflow_24.batchOrdered,
+    workflow_24.batchBroadcast,
+    workflow_24.batchCancel,
+    workflow_24.child,
+    workflow_24.childBatch,
+    workflow_24.batchChildSpawn,
+    workflow_24.batchChildBatchSpawn,
 ];
 function main() {
     return __awaiter(this, void 0, void 0, function* () {

package/v1/examples/retry_after/run.d.ts

@@ -0,0 +1 @@
+export {};

package/v1/examples/retry_after/run.js

@@ -0,0 +1,21 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workflow_1 = require("./workflow");
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const res = yield workflow_1.retryAfterUpstreamDelay.run({ failingAttempts: 2 });
+        console.log(res);
+    });
+}
+if (require.main === module) {
+    main();
+}

package/v1/examples/retry_after/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/v1/examples/retry_after/worker.js

@@ -0,0 +1,24 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const hatchet_client_1 = require("../hatchet-client");
+const workflow_1 = require("./workflow");
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const worker = yield hatchet_client_1.hatchet.worker('retry-after-worker', {
+            workflows: [workflow_1.retryAfterUpstreamDelay, workflow_1.retryAfterExponentialBackoff],
+        });
+        yield worker.start();
+    });
+}
+if (require.main === module) {
+    main();
+}

package/v1/examples/retry_after/workflow.d.ts

@@ -0,0 +1,9 @@
+export type UpstreamInput = {
+    failingAttempts: number;
+};
+export declare const retryAfterUpstreamDelay: import("../..").TaskWorkflowDeclaration<UpstreamInput, {
+    attempt: number;
+}, {}, {}, {}, {}>;
+export declare const retryAfterExponentialBackoff: import("../..").TaskWorkflowDeclaration<UpstreamInput, {
+    attempt: number;
+}, {}, {}, {}, {}>;

package/v1/examples/retry_after/workflow.js

@@ -0,0 +1,40 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retryAfterExponentialBackoff = exports.retryAfterUpstreamDelay = void 0;
+const task_1 = require("../../task");
+const hatchet_client_1 = require("../hatchet-client");
+// > Retry after an upstream-provided delay
+exports.retryAfterUpstreamDelay = hatchet_client_1.hatchet.task({
+    name: 'retry-after-upstream-delay',
+    retries: 5,
+    fn: (input, ctx) => __awaiter(void 0, void 0, void 0, function* () {
+        if (ctx.retryCount() < input.failingAttempts) {
+            throw new task_1.RetryAfterError('upstream rate limited', { after: '2s' });
+        }
+        return { attempt: ctx.retryCount() };
+    }),
+});
+// !!
+// > Exponential backoff with full jitter
+exports.retryAfterExponentialBackoff = hatchet_client_1.hatchet.task({
+    name: 'retry-after-exponential-backoff',
+    retries: 3,
+    fn: (input, ctx) => __awaiter(void 0, void 0, void 0, function* () {
+        if (ctx.retryCount() < input.failingAttempts) {
+            throw new task_1.RetryAfterError('upstream unavailable', {
+                after: Math.random() * Math.min(2 ** ctx.retryCount(), 60) * 1000,
+            });
+        }
+        return { attempt: ctx.retryCount() };
+    }),
+});
+// !!