@workflow/core 5.0.0-beta.4 → 5.0.0-beta.40

package/dist/runtime/replay-budget.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Per-invocation accounting of the *non-step* portion of a workflow
+ * handler run: deterministic event-log replay, workflow-VM execution
+ * between step boundaries, suspension handling, queue round-trips, etc.
+ * Inline step bodies (`"use step"` functions invoked via `executeStep`)
+ * are intentionally excluded — they are bounded by the platform's
+ * function `maxDuration` and the `NO_INLINE_REPLAY_AFTER_MS` early-return
+ * guard.
+ *
+ * Usage:
+ *
+ * ```ts
+ * const budget = new ReplayBudget();
+ * // …non-step work happens here, accumulates against the budget…
+ * budget.pause();
+ * try {
+ *   await executeStep(...); // not charged
+ * } finally {
+ *   budget.resume();
+ * }
+ * // back to charging
+ * if (budget.isExhausted()) { ... }
+ * ```
+ *
+ * Implementation notes:
+ *
+ * - `pause()` and `resume()` are idempotent: calling `pause()` while
+ *   already paused (or `resume()` while already resumed) is a no-op.
+ *   This protects against double-counting in future refactors that nest
+ *   step execution or take an early-return path between a `pause()` and
+ *   the matching `resume()`.
+ * - `isExhausted()` is checked at loop boundaries by the caller — the
+ *   budget itself does not arm any timers. This means an in-flight
+ *   pathological `runWorkflow` call (e.g. a huge event-log replay) can
+ *   overshoot the budget by up to one iteration's worth of work before
+ *   the next check fires. In practice the 20s headroom built into
+ *   `MAX_REPLAY_TIMEOUT_MS` (and the function `maxDuration` ceiling)
+ *   gives us slack; the old `setTimeout`-based approach also ultimately
+ *   relied on the platform SIGTERM as the hard backstop.
+ */
+export declare class ReplayBudget {
+    private readonly limitMs;
+    private elapsedMs;
+    private intervalStart;
+    constructor(limitMs?: number);
+    /**
+     * The configured replay-timeout limit, in ms. Useful for log messages.
+     */
+    get configuredLimitMs(): number;
+    /**
+     * Total non-step time accumulated so far, in ms. Includes the
+     * currently-active interval if the budget is not paused.
+     */
+    elapsed(): number;
+    /**
+     * Stop counting elapsed time toward the budget. Idempotent — safe to
+     * call multiple times in a row; subsequent calls are no-ops until
+     * `resume()` reopens an interval.
+     */
+    pause(): void;
+    /**
+     * Resume counting elapsed time toward the budget. Idempotent — safe to
+     * call multiple times in a row; subsequent calls re-anchor the
+     * interval start to `now()`, which is fine because no time accrues
+     * between back-to-back `resume()` calls.
+     */
+    resume(): void;
+    /**
+     * True if the budget has been exhausted (`elapsed() >= limitMs`).
+     * Callers should invoke `handleExhausted(...)` afterward and return
+     * from the handler.
+     */
+    isExhausted(): boolean;
+}
+/**
+ * Fail the run (or retry, on early attempts) when the replay budget is
+ * exhausted. The handling depends on whether the underlying World
+ * supports `process.exit(1)` as a queue redelivery signal (see
+ * `World.processExitTriggersQueueRedelivery`):
+ *
+ * - **Managed-platform Worlds** (`world-vercel`): on attempts <=
+ *   `REPLAY_TIMEOUT_MAX_RETRIES` exit the process so the platform fails
+ *   the invocation and the queue redelivers; on the next attempt write
+ *   `run_failed` with `RUN_ERROR_CODES.REPLAY_TIMEOUT` and exit.
+ *
+ * - **In-process Worlds** (`world-local`, dev servers): calling
+ *   `process.exit()` would terminate the host (e.g. `pnpm dev`), so
+ *   instead log a warning, write `run_failed` best-effort, and return.
+ *   The framework completes the request normally.
+ */
+export declare function handleReplayBudgetExhausted(args: {
+    runId: string;
+    workflowName: string;
+    requestId: string | undefined;
+    attempt: number;
+    limitMs: number;
+}): Promise<void>;
+//# sourceMappingURL=replay-budget.d.ts.map

package/dist/runtime/replay-budget.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay-budget.d.ts","sourceRoot":"","sources":["../../src/runtime/replay-budget.ts"],"names":[],"mappings":"AASA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAgB;gBAEzB,OAAO,GAAE,MAA6B;IAKlD;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED;;;OAGG;IACH,OAAO,IAAI,MAAM;IAMjB;;;;OAIG;IACH,KAAK,IAAI,IAAI;IAMb;;;;;OAKG;IACH,MAAM,IAAI,IAAI;IAId;;;;OAIG;IACH,WAAW,IAAI,OAAO;CAGvB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,2BAA2B,CAAC,IAAI,EAAE;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkHhB"}

package/dist/runtime/replay-budget.js

@@ -0,0 +1,192 @@
+import { FatalError, RUN_ERROR_CODES } from '@workflow/errors';
+import { SPEC_VERSION_CURRENT } from '@workflow/world';
+import { describeError } from '../describe-error.js';
+import { runtimeLogger } from '../logger.js';
+import { dehydrateRunError } from '../serialization.js';
+import { getReplayTimeoutMaxRetries, getReplayTimeoutMs } from './constants.js';
+import { memoizeEncryptionKey } from './helpers.js';
+import { getWorld } from './world.js';
+/**
+ * Per-invocation accounting of the *non-step* portion of a workflow
+ * handler run: deterministic event-log replay, workflow-VM execution
+ * between step boundaries, suspension handling, queue round-trips, etc.
+ * Inline step bodies (`"use step"` functions invoked via `executeStep`)
+ * are intentionally excluded — they are bounded by the platform's
+ * function `maxDuration` and the `NO_INLINE_REPLAY_AFTER_MS` early-return
+ * guard.
+ *
+ * Usage:
+ *
+ * ```ts
+ * const budget = new ReplayBudget();
+ * // …non-step work happens here, accumulates against the budget…
+ * budget.pause();
+ * try {
+ *   await executeStep(...); // not charged
+ * } finally {
+ *   budget.resume();
+ * }
+ * // back to charging
+ * if (budget.isExhausted()) { ... }
+ * ```
+ *
+ * Implementation notes:
+ *
+ * - `pause()` and `resume()` are idempotent: calling `pause()` while
+ *   already paused (or `resume()` while already resumed) is a no-op.
+ *   This protects against double-counting in future refactors that nest
+ *   step execution or take an early-return path between a `pause()` and
+ *   the matching `resume()`.
+ * - `isExhausted()` is checked at loop boundaries by the caller — the
+ *   budget itself does not arm any timers. This means an in-flight
+ *   pathological `runWorkflow` call (e.g. a huge event-log replay) can
+ *   overshoot the budget by up to one iteration's worth of work before
+ *   the next check fires. In practice the 20s headroom built into
+ *   `MAX_REPLAY_TIMEOUT_MS` (and the function `maxDuration` ceiling)
+ *   gives us slack; the old `setTimeout`-based approach also ultimately
+ *   relied on the platform SIGTERM as the hard backstop.
+ */
+export class ReplayBudget {
+    limitMs;
+    elapsedMs = 0;
+    intervalStart;
+    constructor(limitMs = getReplayTimeoutMs()) {
+        this.limitMs = limitMs;
+        this.intervalStart = Date.now();
+    }
+    /**
+     * The configured replay-timeout limit, in ms. Useful for log messages.
+     */
+    get configuredLimitMs() {
+        return this.limitMs;
+    }
+    /**
+     * Total non-step time accumulated so far, in ms. Includes the
+     * currently-active interval if the budget is not paused.
+     */
+    elapsed() {
+        const open = this.intervalStart === null ? 0 : Date.now() - this.intervalStart;
+        return this.elapsedMs + open;
+    }
+    /**
+     * Stop counting elapsed time toward the budget. Idempotent — safe to
+     * call multiple times in a row; subsequent calls are no-ops until
+     * `resume()` reopens an interval.
+     */
+    pause() {
+        if (this.intervalStart === null)
+            return;
+        this.elapsedMs += Date.now() - this.intervalStart;
+        this.intervalStart = null;
+    }
+    /**
+     * Resume counting elapsed time toward the budget. Idempotent — safe to
+     * call multiple times in a row; subsequent calls re-anchor the
+     * interval start to `now()`, which is fine because no time accrues
+     * between back-to-back `resume()` calls.
+     */
+    resume() {
+        this.intervalStart = Date.now();
+    }
+    /**
+     * True if the budget has been exhausted (`elapsed() >= limitMs`).
+     * Callers should invoke `handleExhausted(...)` afterward and return
+     * from the handler.
+     */
+    isExhausted() {
+        return this.elapsed() >= this.limitMs;
+    }
+}
+/**
+ * Fail the run (or retry, on early attempts) when the replay budget is
+ * exhausted. The handling depends on whether the underlying World
+ * supports `process.exit(1)` as a queue redelivery signal (see
+ * `World.processExitTriggersQueueRedelivery`):
+ *
+ * - **Managed-platform Worlds** (`world-vercel`): on attempts <=
+ *   `REPLAY_TIMEOUT_MAX_RETRIES` exit the process so the platform fails
+ *   the invocation and the queue redelivers; on the next attempt write
+ *   `run_failed` with `RUN_ERROR_CODES.REPLAY_TIMEOUT` and exit.
+ *
+ * - **In-process Worlds** (`world-local`, dev servers): calling
+ *   `process.exit()` would terminate the host (e.g. `pnpm dev`), so
+ *   instead log a warning, write `run_failed` best-effort, and return.
+ *   The framework completes the request normally.
+ */
+export async function handleReplayBudgetExhausted(args) {
+    const { runId, workflowName, requestId, attempt, limitMs } = args;
+    const runLogger = runtimeLogger.forRun(runId, workflowName);
+    const world = await getWorld();
+    const canExitForRedelivery = world.processExitTriggersQueueRedelivery === true;
+    // Worlds without managed-platform redelivery (e.g. world-local, custom
+    // in-process worlds) must not have us exit the process — that would
+    // kill the user's host (`pnpm dev`, CLI, etc.) without producing a
+    // retry. Surface the failure via the event log if we can, then return.
+    if (!canExitForRedelivery) {
+        runLogger.warn('Workflow replay exceeded timeout; current World does not support process exit for redelivery — failing the run and returning', { timeoutMs: limitMs, attempt });
+        try {
+            const getEncryptionKey = memoizeEncryptionKey(world, runId);
+            const timeoutErr = new FatalError(`Workflow replay exceeded maximum duration (${limitMs / 1000}s)`);
+            await world.events.create(runId, {
+                eventType: 'run_failed',
+                specVersion: SPEC_VERSION_CURRENT,
+                eventData: {
+                    error: await dehydrateRunError(timeoutErr, runId, await getEncryptionKey()),
+                    errorCode: RUN_ERROR_CODES.REPLAY_TIMEOUT,
+                },
+            }, { requestId });
+        }
+        catch (err) {
+            runLogger.warn('Unable to mark run as failed', {
+                attempt,
+                errorName: err instanceof Error ? err.name : 'UnknownError',
+                errorMessage: err instanceof Error ? err.message : String(err),
+            });
+        }
+        return;
+    }
+    const maxRetries = getReplayTimeoutMaxRetries();
+    if (attempt <= maxRetries) {
+        runLogger.warn('Workflow replay exceeded timeout but will be re-attempted (attempt < maxRetries)', {
+            timeoutMs: limitMs,
+            attempt,
+            maxRetries,
+        });
+        process.exit(1);
+    }
+    const replayTimeoutDescription = describeError(undefined, RUN_ERROR_CODES.REPLAY_TIMEOUT);
+    runLogger.error('Workflow replay exceeded timeout and max retries exceeded. Failing the run', {
+        timeoutMs: limitMs,
+        attempt,
+        maxRetries,
+        errorCode: replayTimeoutDescription.errorCode,
+        errorAttribution: replayTimeoutDescription.attribution,
+    });
+    try {
+        const getEncryptionKey = memoizeEncryptionKey(world, runId);
+        const timeoutErr = new FatalError(`Workflow replay exceeded maximum duration (${limitMs / 1000}s) after ${attempt} attempts`);
+        await world.events.create(runId, {
+            eventType: 'run_failed',
+            specVersion: SPEC_VERSION_CURRENT,
+            eventData: {
+                error: await dehydrateRunError(timeoutErr, runId, await getEncryptionKey()),
+                errorCode: RUN_ERROR_CODES.REPLAY_TIMEOUT,
+            },
+        }, { requestId });
+    }
+    catch (err) {
+        // Best effort — process exits regardless. Surface why so operators
+        // can diagnose repeat timeouts against the backend.
+        runLogger.warn('Unable to mark run as failed. The queue will continue to retry', {
+            attempt,
+            errorName: err instanceof Error ? err.name : 'UnknownError',
+            errorMessage: err instanceof Error ? err.message : String(err),
+            errorStack: err instanceof Error ? err.stack : undefined,
+        });
+    }
+    // Note that this also prevents the runtime from acking the queue
+    // message, so the queue will call back once, after which a 410 will
+    // get it to exit early.
+    process.exit(1);
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVwbGF5LWJ1ZGdldC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9ydW50aW1lL3JlcGxheS1idWRnZXQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFVBQVUsRUFBRSxlQUFlLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUMvRCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDckQsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUM3QyxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUN4RCxPQUFPLEVBQUUsMEJBQTBCLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUNoRixPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFDcEQsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLFlBQVksQ0FBQztBQUV0Qzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBdUNHO0FBQ0gsTUFBTSxPQUFPLFlBQVk7SUFDTixPQUFPLENBQVM7SUFDekIsU0FBUyxHQUFHLENBQUMsQ0FBQztJQUNkLGFBQWEsQ0FBZ0I7SUFFckMsWUFBWSxVQUFrQixrQkFBa0IsRUFBRTtRQUNoRCxJQUFJLENBQUMsT0FBTyxHQUFHLE9BQU8sQ0FBQztRQUN2QixJQUFJLENBQUMsYUFBYSxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztJQUNsQyxDQUFDO0lBRUQ7O09BRUc7SUFDSCxJQUFJLGlCQUFpQjtRQUNuQixPQUFPLElBQUksQ0FBQyxPQUFPLENBQUM7SUFDdEIsQ0FBQztJQUVEOzs7T0FHRztJQUNILE9BQU87UUFDTCxNQUFNLElBQUksR0FDUixJQUFJLENBQUMsYUFBYSxLQUFLLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLEdBQUcsSUFBSSxDQUFDLGFBQWEsQ0FBQztRQUNwRSxPQUFPLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDO0lBQy9CLENBQUM7SUFFRDs7OztPQUlHO0lBQ0gsS0FBSztRQUNILElBQUksSUFBSSxDQUFDLGFBQWEsS0FBSyxJQUFJO1lBQUUsT0FBTztRQUN4QyxJQUFJLENBQUMsU0FBUyxJQUFJLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxJQUFJLENBQUMsYUFBYSxDQUFDO1FBQ2xELElBQUksQ0FBQyxhQUFhLEdBQUcsSUFBSSxDQUFDO0lBQzVCLENBQUM7SUFFRDs7Ozs7T0FLRztJQUNILE1BQU07UUFDSixJQUFJLENBQUMsYUFBYSxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztJQUNsQyxDQUFDO0lBRUQ7Ozs7T0FJRztJQUNILFdBQVc7UUFDVCxPQUFPLElBQUksQ0FBQyxPQUFPLEVBQUUsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDO0lBQ3hDLENBQUM7Q0FDRjtBQUVEOzs7Ozs7Ozs7Ozs7Ozs7R0FlRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsMkJBQTJCLENBQUMsSUFNakQ7SUFDQyxNQUFNLEVBQUUsS0FBSyxFQUFFLFlBQVksRUFBRSxTQUFTLEVBQUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxHQUFHLElBQUksQ0FBQztJQUNsRSxNQUFNLFNBQVMsR0FBRyxhQUFhLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxZQUFZLENBQUMsQ0FBQztJQUU1RCxNQUFNLEtBQUssR0FBRyxNQUFNLFFBQVEsRUFBRSxDQUFDO0lBQy9CLE1BQU0sb0JBQW9CLEdBQ3hCLEtBQUssQ0FBQyxrQ0FBa0MsS0FBSyxJQUFJLENBQUM7SUFFcEQsdUVBQXVFO0lBQ3ZFLG9FQUFvRTtJQUNwRSxtRUFBbUU7SUFDbkUsdUVBQXVFO0lBQ3ZFLElBQUksQ0FBQyxvQkFBb0IsRUFBRSxDQUFDO1FBQzFCLFNBQVMsQ0FBQyxJQUFJLENBQ1osOEhBQThILEVBQzlILEVBQUUsU0FBUyxFQUFFLE9BQU8sRUFBRSxPQUFPLEVBQUUsQ0FDaEMsQ0FBQztRQUNGLElBQUksQ0FBQztZQUNILE1BQU0sZ0JBQWdCLEdBQUcsb0JBQW9CLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxDQUFDO1lBQzVELE1BQU0sVUFBVSxHQUFHLElBQUksVUFBVSxDQUMvQiw4Q0FBOEMsT0FBTyxHQUFHLElBQUksSUFBSSxDQUNqRSxDQUFDO1lBQ0YsTUFBTSxLQUFLLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FDdkIsS0FBSyxFQUNMO2dCQUNFLFNBQVMsRUFBRSxZQUFZO2dCQUN2QixXQUFXLEVBQUUsb0JBQW9CO2dCQUNqQyxTQUFTLEVBQUU7b0JBQ1QsS0FBSyxFQUFFLE1BQU0saUJBQWlCLENBQzVCLFVBQVUsRUFDVixLQUFLLEVBQ0wsTUFBTSxnQkFBZ0IsRUFBRSxDQUN6QjtvQkFDRCxTQUFTLEVBQUUsZUFBZSxDQUFDLGNBQWM7aUJBQzFDO2FBQ0YsRUFDRCxFQUFFLFNBQVMsRUFBRSxDQUNkLENBQUM7UUFDSixDQUFDO1FBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztZQUNiLFNBQVMsQ0FBQyxJQUFJLENBQUMsOEJBQThCLEVBQUU7Z0JBQzdDLE9BQU87Z0JBQ1AsU0FBUyxFQUFFLEdBQUcsWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLGNBQWM7Z0JBQzNELFlBQVksRUFBRSxHQUFHLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDO2FBQy9ELENBQUMsQ0FBQztRQUNMLENBQUM7UUFDRCxPQUFPO0lBQ1QsQ0FBQztJQUVELE1BQU0sVUFBVSxHQUFHLDBCQUEwQixFQUFFLENBQUM7SUFDaEQsSUFBSSxPQUFPLElBQUksVUFBVSxFQUFFLENBQUM7UUFDMUIsU0FBUyxDQUFDLElBQUksQ0FDWixrRkFBa0YsRUFDbEY7WUFDRSxTQUFTLEVBQUUsT0FBTztZQUNsQixPQUFPO1lBQ1AsVUFBVTtTQUNYLENBQ0YsQ0FBQztRQUNGLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDbEIsQ0FBQztJQUVELE1BQU0sd0JBQXdCLEdBQUcsYUFBYSxDQUM1QyxTQUFTLEVBQ1QsZUFBZSxDQUFDLGNBQWMsQ0FDL0IsQ0FBQztJQUNGLFNBQVMsQ0FBQyxLQUFLLENBQ2IsNEVBQTRFLEVBQzVFO1FBQ0UsU0FBUyxFQUFFLE9BQU87UUFDbEIsT0FBTztRQUNQLFVBQVU7UUFDVixTQUFTLEVBQUUsd0JBQXdCLENBQUMsU0FBUztRQUM3QyxnQkFBZ0IsRUFBRSx3QkFBd0IsQ0FBQyxXQUFXO0tBQ3ZELENBQ0YsQ0FBQztJQUVGLElBQUksQ0FBQztRQUNILE1BQU0sZ0JBQWdCLEdBQUcsb0JBQW9CLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxDQUFDO1FBQzVELE1BQU0sVUFBVSxHQUFHLElBQUksVUFBVSxDQUMvQiw4Q0FBOEMsT0FBTyxHQUFHLElBQUksWUFBWSxPQUFPLFdBQVcsQ0FDM0YsQ0FBQztRQUNGLE1BQU0sS0FBSyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQ3ZCLEtBQUssRUFDTDtZQUNFLFNBQVMsRUFBRSxZQUFZO1lBQ3ZCLFdBQVcsRUFBRSxvQkFBb0I7WUFDakMsU0FBUyxFQUFFO2dCQUNULEtBQUssRUFBRSxNQUFNLGlCQUFpQixDQUM1QixVQUFVLEVBQ1YsS0FBSyxFQUNMLE1BQU0sZ0JBQWdCLEVBQUUsQ0FDekI7Z0JBQ0QsU0FBUyxFQUFFLGVBQWUsQ0FBQyxjQUFjO2FBQzFDO1NBQ0YsRUFDRCxFQUFFLFNBQVMsRUFBRSxDQUNkLENBQUM7SUFDSixDQUFDO0lBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztRQUNiLG1FQUFtRTtRQUNuRSxvREFBb0Q7UUFDcEQsU0FBUyxDQUFDLElBQUksQ0FDWixnRUFBZ0UsRUFDaEU7WUFDRSxPQUFPO1lBQ1AsU0FBUyxFQUFFLEdBQUcsWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLGNBQWM7WUFDM0QsWUFBWSxFQUFFLEdBQUcsWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUM7WUFDOUQsVUFBVSxFQUFFLEdBQUcsWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLFNBQVM7U0FDekQsQ0FDRixDQUFDO0lBQ0osQ0FBQztJQUNELGlFQUFpRTtJQUNqRSxvRUFBb0U7SUFDcEUsd0JBQXdCO0lBQ3hCLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDbEIsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCB7IEZhdGFsRXJyb3IsIFJVTl9FUlJPUl9DT0RFUyB9IGZyb20gJ0B3b3JrZmxvdy9lcnJvcnMnO1xuaW1wb3J0IHsgU1BFQ19WRVJTSU9OX0NVUlJFTlQgfSBmcm9tICdAd29ya2Zsb3cvd29ybGQnO1xuaW1wb3J0IHsgZGVzY3JpYmVFcnJvciB9IGZyb20gJy4uL2Rlc2NyaWJlLWVycm9yLmpzJztcbmltcG9ydCB7IHJ1bnRpbWVMb2dnZXIgfSBmcm9tICcuLi9sb2dnZXIuanMnO1xuaW1wb3J0IHsgZGVoeWRyYXRlUnVuRXJyb3IgfSBmcm9tICcuLi9zZXJpYWxpemF0aW9uLmpzJztcbmltcG9ydCB7IGdldFJlcGxheVRpbWVvdXRNYXhSZXRyaWVzLCBnZXRSZXBsYXlUaW1lb3V0TXMgfSBmcm9tICcuL2NvbnN0YW50cy5qcyc7XG5pbXBvcnQgeyBtZW1vaXplRW5jcnlwdGlvbktleSB9IGZyb20gJy4vaGVscGVycy5qcyc7XG5pbXBvcnQgeyBnZXRXb3JsZCB9IGZyb20gJy4vd29ybGQuanMnO1xuXG4vKipcbiAqIFBlci1pbnZvY2F0aW9uIGFjY291bnRpbmcgb2YgdGhlICpub24tc3RlcCogcG9ydGlvbiBvZiBhIHdvcmtmbG93XG4gKiBoYW5kbGVyIHJ1bjogZGV0ZXJtaW5pc3RpYyBldmVudC1sb2cgcmVwbGF5LCB3b3JrZmxvdy1WTSBleGVjdXRpb25cbiAqIGJldHdlZW4gc3RlcCBib3VuZGFyaWVzLCBzdXNwZW5zaW9uIGhhbmRsaW5nLCBxdWV1ZSByb3VuZC10cmlwcywgZXRjLlxuICogSW5saW5lIHN0ZXAgYm9kaWVzIChgXCJ1c2Ugc3RlcFwiYCBmdW5jdGlvbnMgaW52b2tlZCB2aWEgYGV4ZWN1dGVTdGVwYClcbiAqIGFyZSBpbnRlbnRpb25hbGx5IGV4Y2x1ZGVkIOKAlCB0aGV5IGFyZSBib3VuZGVkIGJ5IHRoZSBwbGF0Zm9ybSdzXG4gKiBmdW5jdGlvbiBgbWF4RHVyYXRpb25gIGFuZCB0aGUgYE5PX0lOTElORV9SRVBMQVlfQUZURVJfTVNgIGVhcmx5LXJldHVyblxuICogZ3VhcmQuXG4gKlxuICogVXNhZ2U6XG4gKlxuICogYGBgdHNcbiAqIGNvbnN0IGJ1ZGdldCA9IG5ldyBSZXBsYXlCdWRnZXQoKTtcbiAqIC8vIOKApm5vbi1zdGVwIHdvcmsgaGFwcGVucyBoZXJlLCBhY2N1bXVsYXRlcyBhZ2FpbnN0IHRoZSBidWRnZXTigKZcbiAqIGJ1ZGdldC5wYXVzZSgpO1xuICogdHJ5IHtcbiAqICAgYXdhaXQgZXhlY3V0ZVN0ZXAoLi4uKTsgLy8gbm90IGNoYXJnZWRcbiAqIH0gZmluYWxseSB7XG4gKiAgIGJ1ZGdldC5yZXN1bWUoKTtcbiAqIH1cbiAqIC8vIGJhY2sgdG8gY2hhcmdpbmdcbiAqIGlmIChidWRnZXQuaXNFeGhhdXN0ZWQoKSkgeyAuLi4gfVxuICogYGBgXG4gKlxuICogSW1wbGVtZW50YXRpb24gbm90ZXM6XG4gKlxuICogLSBgcGF1c2UoKWAgYW5kIGByZXN1bWUoKWAgYXJlIGlkZW1wb3RlbnQ6IGNhbGxpbmcgYHBhdXNlKClgIHdoaWxlXG4gKiAgIGFscmVhZHkgcGF1c2VkIChvciBgcmVzdW1lKClgIHdoaWxlIGFscmVhZHkgcmVzdW1lZCkgaXMgYSBuby1vcC5cbiAqICAgVGhpcyBwcm90ZWN0cyBhZ2FpbnN0IGRvdWJsZS1jb3VudGluZyBpbiBmdXR1cmUgcmVmYWN0b3JzIHRoYXQgbmVzdFxuICogICBzdGVwIGV4ZWN1dGlvbiBvciB0YWtlIGFuIGVhcmx5LXJldHVybiBwYXRoIGJldHdlZW4gYSBgcGF1c2UoKWAgYW5kXG4gKiAgIHRoZSBtYXRjaGluZyBgcmVzdW1lKClgLlxuICogLSBgaXNFeGhhdXN0ZWQoKWAgaXMgY2hlY2tlZCBhdCBsb29wIGJvdW5kYXJpZXMgYnkgdGhlIGNhbGxlciDigJQgdGhlXG4gKiAgIGJ1ZGdldCBpdHNlbGYgZG9lcyBub3QgYXJtIGFueSB0aW1lcnMuIFRoaXMgbWVhbnMgYW4gaW4tZmxpZ2h0XG4gKiAgIHBhdGhvbG9naWNhbCBgcnVuV29ya2Zsb3dgIGNhbGwgKGUuZy4gYSBodWdlIGV2ZW50LWxvZyByZXBsYXkpIGNhblxuICogICBvdmVyc2hvb3QgdGhlIGJ1ZGdldCBieSB1cCB0byBvbmUgaXRlcmF0aW9uJ3Mgd29ydGggb2Ygd29yayBiZWZvcmVcbiAqICAgdGhlIG5leHQgY2hlY2sgZmlyZXMuIEluIHByYWN0aWNlIHRoZSAyMHMgaGVhZHJvb20gYnVpbHQgaW50b1xuICogICBgTUFYX1JFUExBWV9USU1FT1VUX01TYCAoYW5kIHRoZSBmdW5jdGlvbiBgbWF4RHVyYXRpb25gIGNlaWxpbmcpXG4gKiAgIGdpdmVzIHVzIHNsYWNrOyB0aGUgb2xkIGBzZXRUaW1lb3V0YC1iYXNlZCBhcHByb2FjaCBhbHNvIHVsdGltYXRlbHlcbiAqICAgcmVsaWVkIG9uIHRoZSBwbGF0Zm9ybSBTSUdURVJNIGFzIHRoZSBoYXJkIGJhY2tzdG9wLlxuICovXG5leHBvcnQgY2xhc3MgUmVwbGF5QnVkZ2V0IHtcbiAgcHJpdmF0ZSByZWFkb25seSBsaW1pdE1zOiBudW1iZXI7XG4gIHByaXZhdGUgZWxhcHNlZE1zID0gMDtcbiAgcHJpdmF0ZSBpbnRlcnZhbFN0YXJ0OiBudW1iZXIgfCBudWxsO1xuXG4gIGNvbnN0cnVjdG9yKGxpbWl0TXM6IG51bWJlciA9IGdldFJlcGxheVRpbWVvdXRNcygpKSB7XG4gICAgdGhpcy5saW1pdE1zID0gbGltaXRNcztcbiAgICB0aGlzLmludGVydmFsU3RhcnQgPSBEYXRlLm5vdygpO1xuICB9XG5cbiAgLyoqXG4gICAqIFRoZSBjb25maWd1cmVkIHJlcGxheS10aW1lb3V0IGxpbWl0LCBpbiBtcy4gVXNlZnVsIGZvciBsb2cgbWVzc2FnZXMuXG4gICAqL1xuICBnZXQgY29uZmlndXJlZExpbWl0TXMoKTogbnVtYmVyIHtcbiAgICByZXR1cm4gdGhpcy5saW1pdE1zO1xuICB9XG5cbiAgLyoqXG4gICAqIFRvdGFsIG5vbi1zdGVwIHRpbWUgYWNjdW11bGF0ZWQgc28gZmFyLCBpbiBtcy4gSW5jbHVkZXMgdGhlXG4gICAqIGN1cnJlbnRseS1hY3RpdmUgaW50ZXJ2YWwgaWYgdGhlIGJ1ZGdldCBpcyBub3QgcGF1c2VkLlxuICAgKi9cbiAgZWxhcHNlZCgpOiBudW1iZXIge1xuICAgIGNvbnN0IG9wZW4gPVxuICAgICAgdGhpcy5pbnRlcnZhbFN0YXJ0ID09PSBudWxsID8gMCA6IERhdGUubm93KCkgLSB0aGlzLmludGVydmFsU3RhcnQ7XG4gICAgcmV0dXJuIHRoaXMuZWxhcHNlZE1zICsgb3BlbjtcbiAgfVxuXG4gIC8qKlxuICAgKiBTdG9wIGNvdW50aW5nIGVsYXBzZWQgdGltZSB0b3dhcmQgdGhlIGJ1ZGdldC4gSWRlbXBvdGVudCDigJQgc2FmZSB0b1xuICAgKiBjYWxsIG11bHRpcGxlIHRpbWVzIGluIGEgcm93OyBzdWJzZXF1ZW50IGNhbGxzIGFyZSBuby1vcHMgdW50aWxcbiAgICogYHJlc3VtZSgpYCByZW9wZW5zIGFuIGludGVydmFsLlxuICAgKi9cbiAgcGF1c2UoKTogdm9pZCB7XG4gICAgaWYgKHRoaXMuaW50ZXJ2YWxTdGFydCA9PT0gbnVsbCkgcmV0dXJuO1xuICAgIHRoaXMuZWxhcHNlZE1zICs9IERhdGUubm93KCkgLSB0aGlzLmludGVydmFsU3RhcnQ7XG4gICAgdGhpcy5pbnRlcnZhbFN0YXJ0ID0gbnVsbDtcbiAgfVxuXG4gIC8qKlxuICAgKiBSZXN1bWUgY291bnRpbmcgZWxhcHNlZCB0aW1lIHRvd2FyZCB0aGUgYnVkZ2V0LiBJZGVtcG90ZW50IOKAlCBzYWZlIHRvXG4gICAqIGNhbGwgbXVsdGlwbGUgdGltZXMgaW4gYSByb3c7IHN1YnNlcXVlbnQgY2FsbHMgcmUtYW5jaG9yIHRoZVxuICAgKiBpbnRlcnZhbCBzdGFydCB0byBgbm93KClgLCB3aGljaCBpcyBmaW5lIGJlY2F1c2Ugbm8gdGltZSBhY2NydWVzXG4gICAqIGJldHdlZW4gYmFjay10by1iYWNrIGByZXN1bWUoKWAgY2FsbHMuXG4gICAqL1xuICByZXN1bWUoKTogdm9pZCB7XG4gICAgdGhpcy5pbnRlcnZhbFN0YXJ0ID0gRGF0ZS5ub3coKTtcbiAgfVxuXG4gIC8qKlxuICAgKiBUcnVlIGlmIHRoZSBidWRnZXQgaGFzIGJlZW4gZXhoYXVzdGVkIChgZWxhcHNlZCgpID49IGxpbWl0TXNgKS5cbiAgICogQ2FsbGVycyBzaG91bGQgaW52b2tlIGBoYW5kbGVFeGhhdXN0ZWQoLi4uKWAgYWZ0ZXJ3YXJkIGFuZCByZXR1cm5cbiAgICogZnJvbSB0aGUgaGFuZGxlci5cbiAgICovXG4gIGlzRXhoYXVzdGVkKCk6IGJvb2xlYW4ge1xuICAgIHJldHVybiB0aGlzLmVsYXBzZWQoKSA+PSB0aGlzLmxpbWl0TXM7XG4gIH1cbn1cblxuLyoqXG4gKiBGYWlsIHRoZSBydW4gKG9yIHJldHJ5LCBvbiBlYXJseSBhdHRlbXB0cykgd2hlbiB0aGUgcmVwbGF5IGJ1ZGdldCBpc1xuICogZXhoYXVzdGVkLiBUaGUgaGFuZGxpbmcgZGVwZW5kcyBvbiB3aGV0aGVyIHRoZSB1bmRlcmx5aW5nIFdvcmxkXG4gKiBzdXBwb3J0cyBgcHJvY2Vzcy5leGl0KDEpYCBhcyBhIHF1ZXVlIHJlZGVsaXZlcnkgc2lnbmFsIChzZWVcbiAqIGBXb3JsZC5wcm9jZXNzRXhpdFRyaWdnZXJzUXVldWVSZWRlbGl2ZXJ5YCk6XG4gKlxuICogLSAqKk1hbmFnZWQtcGxhdGZvcm0gV29ybGRzKiogKGB3b3JsZC12ZXJjZWxgKTogb24gYXR0ZW1wdHMgPD1cbiAqICAgYFJFUExBWV9USU1FT1VUX01BWF9SRVRSSUVTYCBleGl0IHRoZSBwcm9jZXNzIHNvIHRoZSBwbGF0Zm9ybSBmYWlsc1xuICogICB0aGUgaW52b2NhdGlvbiBhbmQgdGhlIHF1ZXVlIHJlZGVsaXZlcnM7IG9uIHRoZSBuZXh0IGF0dGVtcHQgd3JpdGVcbiAqICAgYHJ1bl9mYWlsZWRgIHdpdGggYFJVTl9FUlJPUl9DT0RFUy5SRVBMQVlfVElNRU9VVGAgYW5kIGV4aXQuXG4gKlxuICogLSAqKkluLXByb2Nlc3MgV29ybGRzKiogKGB3b3JsZC1sb2NhbGAsIGRldiBzZXJ2ZXJzKTogY2FsbGluZ1xuICogICBgcHJvY2Vzcy5leGl0KClgIHdvdWxkIHRlcm1pbmF0ZSB0aGUgaG9zdCAoZS5nLiBgcG5wbSBkZXZgKSwgc29cbiAqICAgaW5zdGVhZCBsb2cgYSB3YXJuaW5nLCB3cml0ZSBgcnVuX2ZhaWxlZGAgYmVzdC1lZmZvcnQsIGFuZCByZXR1cm4uXG4gKiAgIFRoZSBmcmFtZXdvcmsgY29tcGxldGVzIHRoZSByZXF1ZXN0IG5vcm1hbGx5LlxuICovXG5leHBvcnQgYXN5bmMgZnVuY3Rpb24gaGFuZGxlUmVwbGF5QnVkZ2V0RXhoYXVzdGVkKGFyZ3M6IHtcbiAgcnVuSWQ6IHN0cmluZztcbiAgd29ya2Zsb3dOYW1lOiBzdHJpbmc7XG4gIHJlcXVlc3RJZDogc3RyaW5nIHwgdW5kZWZpbmVkO1xuICBhdHRlbXB0OiBudW1iZXI7XG4gIGxpbWl0TXM6IG51bWJlcjtcbn0pOiBQcm9taXNlPHZvaWQ+IHtcbiAgY29uc3QgeyBydW5JZCwgd29ya2Zsb3dOYW1lLCByZXF1ZXN0SWQsIGF0dGVtcHQsIGxpbWl0TXMgfSA9IGFyZ3M7XG4gIGNvbnN0IHJ1bkxvZ2dlciA9IHJ1bnRpbWVMb2dnZXIuZm9yUnVuKHJ1bklkLCB3b3JrZmxvd05hbWUpO1xuXG4gIGNvbnN0IHdvcmxkID0gYXdhaXQgZ2V0V29ybGQoKTtcbiAgY29uc3QgY2FuRXhpdEZvclJlZGVsaXZlcnkgPVxuICAgIHdvcmxkLnByb2Nlc3NFeGl0VHJpZ2dlcnNRdWV1ZVJlZGVsaXZlcnkgPT09IHRydWU7XG5cbiAgLy8gV29ybGRzIHdpdGhvdXQgbWFuYWdlZC1wbGF0Zm9ybSByZWRlbGl2ZXJ5IChlLmcuIHdvcmxkLWxvY2FsLCBjdXN0b21cbiAgLy8gaW4tcHJvY2VzcyB3b3JsZHMpIG11c3Qgbm90IGhhdmUgdXMgZXhpdCB0aGUgcHJvY2VzcyDigJQgdGhhdCB3b3VsZFxuICAvLyBraWxsIHRoZSB1c2VyJ3MgaG9zdCAoYHBucG0gZGV2YCwgQ0xJLCBldGMuKSB3aXRob3V0IHByb2R1Y2luZyBhXG4gIC8vIHJldHJ5LiBTdXJmYWNlIHRoZSBmYWlsdXJlIHZpYSB0aGUgZXZlbnQgbG9nIGlmIHdlIGNhbiwgdGhlbiByZXR1cm4uXG4gIGlmICghY2FuRXhpdEZvclJlZGVsaXZlcnkpIHtcbiAgICBydW5Mb2dnZXIud2FybihcbiAgICAgICdXb3JrZmxvdyByZXBsYXkgZXhjZWVkZWQgdGltZW91dDsgY3VycmVudCBXb3JsZCBkb2VzIG5vdCBzdXBwb3J0IHByb2Nlc3MgZXhpdCBmb3IgcmVkZWxpdmVyeSDigJQgZmFpbGluZyB0aGUgcnVuIGFuZCByZXR1cm5pbmcnLFxuICAgICAgeyB0aW1lb3V0TXM6IGxpbWl0TXMsIGF0dGVtcHQgfVxuICAgICk7XG4gICAgdHJ5IHtcbiAgICAgIGNvbnN0IGdldEVuY3J5cHRpb25LZXkgPSBtZW1vaXplRW5jcnlwdGlvbktleSh3b3JsZCwgcnVuSWQpO1xuICAgICAgY29uc3QgdGltZW91dEVyciA9IG5ldyBGYXRhbEVycm9yKFxuICAgICAgICBgV29ya2Zsb3cgcmVwbGF5IGV4Y2VlZGVkIG1heGltdW0gZHVyYXRpb24gKCR7bGltaXRNcyAvIDEwMDB9cylgXG4gICAgICApO1xuICAgICAgYXdhaXQgd29ybGQuZXZlbnRzLmNyZWF0ZShcbiAgICAgICAgcnVuSWQsXG4gICAgICAgIHtcbiAgICAgICAgICBldmVudFR5cGU6ICdydW5fZmFpbGVkJyxcbiAgICAgICAgICBzcGVjVmVyc2lvbjogU1BFQ19WRVJTSU9OX0NVUlJFTlQsXG4gICAgICAgICAgZXZlbnREYXRhOiB7XG4gICAgICAgICAgICBlcnJvcjogYXdhaXQgZGVoeWRyYXRlUnVuRXJyb3IoXG4gICAgICAgICAgICAgIHRpbWVvdXRFcnIsXG4gICAgICAgICAgICAgIHJ1bklkLFxuICAgICAgICAgICAgICBhd2FpdCBnZXRFbmNyeXB0aW9uS2V5KClcbiAgICAgICAgICAgICksXG4gICAgICAgICAgICBlcnJvckNvZGU6IFJVTl9FUlJPUl9DT0RFUy5SRVBMQVlfVElNRU9VVCxcbiAgICAgICAgICB9LFxuICAgICAgICB9LFxuICAgICAgICB7IHJlcXVlc3RJZCB9XG4gICAgICApO1xuICAgIH0gY2F0Y2ggKGVycikge1xuICAgICAgcnVuTG9nZ2VyLndhcm4oJ1VuYWJsZSB0byBtYXJrIHJ1biBhcyBmYWlsZWQnLCB7XG4gICAgICAgIGF0dGVtcHQsXG4gICAgICAgIGVycm9yTmFtZTogZXJyIGluc3RhbmNlb2YgRXJyb3IgPyBlcnIubmFtZSA6ICdVbmtub3duRXJyb3InLFxuICAgICAgICBlcnJvck1lc3NhZ2U6IGVyciBpbnN0YW5jZW9mIEVycm9yID8gZXJyLm1lc3NhZ2UgOiBTdHJpbmcoZXJyKSxcbiAgICAgIH0pO1xuICAgIH1cbiAgICByZXR1cm47XG4gIH1cblxuICBjb25zdCBtYXhSZXRyaWVzID0gZ2V0UmVwbGF5VGltZW91dE1heFJldHJpZXMoKTtcbiAgaWYgKGF0dGVtcHQgPD0gbWF4UmV0cmllcykge1xuICAgIHJ1bkxvZ2dlci53YXJuKFxuICAgICAgJ1dvcmtmbG93IHJlcGxheSBleGNlZWRlZCB0aW1lb3V0IGJ1dCB3aWxsIGJlIHJlLWF0dGVtcHRlZCAoYXR0ZW1wdCA8IG1heFJldHJpZXMpJyxcbiAgICAgIHtcbiAgICAgICAgdGltZW91dE1zOiBsaW1pdE1zLFxuICAgICAgICBhdHRlbXB0LFxuICAgICAgICBtYXhSZXRyaWVzLFxuICAgICAgfVxuICAgICk7XG4gICAgcHJvY2Vzcy5leGl0KDEpO1xuICB9XG5cbiAgY29uc3QgcmVwbGF5VGltZW91dERlc2NyaXB0aW9uID0gZGVzY3JpYmVFcnJvcihcbiAgICB1bmRlZmluZWQsXG4gICAgUlVOX0VSUk9SX0NPREVTLlJFUExBWV9USU1FT1VUXG4gICk7XG4gIHJ1bkxvZ2dlci5lcnJvcihcbiAgICAnV29ya2Zsb3cgcmVwbGF5IGV4Y2VlZGVkIHRpbWVvdXQgYW5kIG1heCByZXRyaWVzIGV4Y2VlZGVkLiBGYWlsaW5nIHRoZSBydW4nLFxuICAgIHtcbiAgICAgIHRpbWVvdXRNczogbGltaXRNcyxcbiAgICAgIGF0dGVtcHQsXG4gICAgICBtYXhSZXRyaWVzLFxuICAgICAgZXJyb3JDb2RlOiByZXBsYXlUaW1lb3V0RGVzY3JpcHRpb24uZXJyb3JDb2RlLFxuICAgICAgZXJyb3JBdHRyaWJ1dGlvbjogcmVwbGF5VGltZW91dERlc2NyaXB0aW9uLmF0dHJpYnV0aW9uLFxuICAgIH1cbiAgKTtcblxuICB0cnkge1xuICAgIGNvbnN0IGdldEVuY3J5cHRpb25LZXkgPSBtZW1vaXplRW5jcnlwdGlvbktleSh3b3JsZCwgcnVuSWQpO1xuICAgIGNvbnN0IHRpbWVvdXRFcnIgPSBuZXcgRmF0YWxFcnJvcihcbiAgICAgIGBXb3JrZmxvdyByZXBsYXkgZXhjZWVkZWQgbWF4aW11bSBkdXJhdGlvbiAoJHtsaW1pdE1zIC8gMTAwMH1zKSBhZnRlciAke2F0dGVtcHR9IGF0dGVtcHRzYFxuICAgICk7XG4gICAgYXdhaXQgd29ybGQuZXZlbnRzLmNyZWF0ZShcbiAgICAgIHJ1bklkLFxuICAgICAge1xuICAgICAgICBldmVudFR5cGU6ICdydW5fZmFpbGVkJyxcbiAgICAgICAgc3BlY1ZlcnNpb246IFNQRUNfVkVSU0lPTl9DVVJSRU5ULFxuICAgICAgICBldmVudERhdGE6IHtcbiAgICAgICAgICBlcnJvcjogYXdhaXQgZGVoeWRyYXRlUnVuRXJyb3IoXG4gICAgICAgICAgICB0aW1lb3V0RXJyLFxuICAgICAgICAgICAgcnVuSWQsXG4gICAgICAgICAgICBhd2FpdCBnZXRFbmNyeXB0aW9uS2V5KClcbiAgICAgICAgICApLFxuICAgICAgICAgIGVycm9yQ29kZTogUlVOX0VSUk9SX0NPREVTLlJFUExBWV9USU1FT1VULFxuICAgICAgICB9LFxuICAgICAgfSxcbiAgICAgIHsgcmVxdWVzdElkIH1cbiAgICApO1xuICB9IGNhdGNoIChlcnIpIHtcbiAgICAvLyBCZXN0IGVmZm9ydCDigJQgcHJvY2VzcyBleGl0cyByZWdhcmRsZXNzLiBTdXJmYWNlIHdoeSBzbyBvcGVyYXRvcnNcbiAgICAvLyBjYW4gZGlhZ25vc2UgcmVwZWF0IHRpbWVvdXRzIGFnYWluc3QgdGhlIGJhY2tlbmQuXG4gICAgcnVuTG9nZ2VyLndhcm4oXG4gICAgICAnVW5hYmxlIHRvIG1hcmsgcnVuIGFzIGZhaWxlZC4gVGhlIHF1ZXVlIHdpbGwgY29udGludWUgdG8gcmV0cnknLFxuICAgICAge1xuICAgICAgICBhdHRlbXB0LFxuICAgICAgICBlcnJvck5hbWU6IGVyciBpbnN0YW5jZW9mIEVycm9yID8gZXJyLm5hbWUgOiAnVW5rbm93bkVycm9yJyxcbiAgICAgICAgZXJyb3JNZXNzYWdlOiBlcnIgaW5zdGFuY2VvZiBFcnJvciA/IGVyci5tZXNzYWdlIDogU3RyaW5nKGVyciksXG4gICAgICAgIGVycm9yU3RhY2s6IGVyciBpbnN0YW5jZW9mIEVycm9yID8gZXJyLnN0YWNrIDogdW5kZWZpbmVkLFxuICAgICAgfVxuICAgICk7XG4gIH1cbiAgLy8gTm90ZSB0aGF0IHRoaXMgYWxzbyBwcmV2ZW50cyB0aGUgcnVudGltZSBmcm9tIGFja2luZyB0aGUgcXVldWVcbiAgLy8gbWVzc2FnZSwgc28gdGhlIHF1ZXVlIHdpbGwgY2FsbCBiYWNrIG9uY2UsIGFmdGVyIHdoaWNoIGEgNDEwIHdpbGxcbiAgLy8gZ2V0IGl0IHRvIGV4aXQgZWFybHkuXG4gIHByb2Nlc3MuZXhpdCgxKTtcbn1cbiJdfQ==

package/dist/runtime/replay-recovery-reporter.d.ts

@@ -0,0 +1,36 @@
+import type { CreateEventParams } from '@workflow/world';
+/**
+ * Invocation-scoped, one-shot reporter for a replay-divergence recovery.
+ *
+ * The reporter stays dormant until replay reaches a valid deterministic
+ * boundary. The first subsequent event write carries the telemetry for free.
+ * A recovered invocation with no natural event write intentionally reports
+ * nothing.
+ */
+export declare class ReplayRecoveryReporter {
+    private readonly divergenceCount;
+    private active;
+    private reported;
+    constructor(divergenceCount: number);
+    /**
+     * A reporter for an invocation that never diverged: `activate()` cannot arm
+     * it, so `withEventCreate` always passes writes through untouched. Lets every
+     * call site hold a non-optional reporter instead of branching on one.
+     *
+     * Structurally inert rather than a zero-count reporter on purpose — the
+     * server rejects a count below 1 as out-of-range and emits a
+     * `telemetry_rejected` metric, so a reporter that *can* report zero is a
+     * live footgun.
+     */
+    static inert(): ReplayRecoveryReporter;
+    activate(): void;
+    /**
+     * Decorate one natural event write with recovery telemetry.
+     *
+     * The claim is taken synchronously, so concurrent suspension writes cannot
+     * all carry the same count. A failed request releases it for a later natural
+     * write in the same invocation.
+     */
+    withEventCreate<T>(params: CreateEventParams | undefined, create: (params: CreateEventParams | undefined) => Promise<T>): Promise<T>;
+}
+//# sourceMappingURL=replay-recovery-reporter.d.ts.map

package/dist/runtime/replay-recovery-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay-recovery-reporter.d.ts","sourceRoot":"","sources":["../../src/runtime/replay-recovery-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD;;;;;;;GAOG;AACH,qBAAa,sBAAsB;IAIrB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAH5C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;gBAEI,eAAe,EAAE,MAAM;IAEpD;;;;;;;;;OASG;IACH,MAAM,CAAC,KAAK,IAAI,sBAAsB;IAItC,QAAQ,IAAI,IAAI;IAIhB;;;;;;OAMG;IACG,eAAe,CAAC,CAAC,EACrB,MAAM,EAAE,iBAAiB,GAAG,SAAS,EACrC,MAAM,EAAE,CAAC,MAAM,EAAE,iBAAiB,GAAG,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5D,OAAO,CAAC,CAAC,CAAC;CAcd"}

package/dist/runtime/replay-recovery-reporter.js

@@ -0,0 +1,64 @@
+/**
+ * Invocation-scoped, one-shot reporter for a replay-divergence recovery.
+ *
+ * The reporter stays dormant until replay reaches a valid deterministic
+ * boundary. The first subsequent event write carries the telemetry for free.
+ * A recovered invocation with no natural event write intentionally reports
+ * nothing.
+ */
+export class ReplayRecoveryReporter {
+    divergenceCount;
+    active = false;
+    reported = false;
+    constructor(divergenceCount) {
+        this.divergenceCount = divergenceCount;
+    }
+    /**
+     * A reporter for an invocation that never diverged: `activate()` cannot arm
+     * it, so `withEventCreate` always passes writes through untouched. Lets every
+     * call site hold a non-optional reporter instead of branching on one.
+     *
+     * Structurally inert rather than a zero-count reporter on purpose — the
+     * server rejects a count below 1 as out-of-range and emits a
+     * `telemetry_rejected` metric, so a reporter that *can* report zero is a
+     * live footgun.
+     */
+    static inert() {
+        return new InertReplayRecoveryReporter();
+    }
+    activate() {
+        this.active = true;
+    }
+    /**
+     * Decorate one natural event write with recovery telemetry.
+     *
+     * The claim is taken synchronously, so concurrent suspension writes cannot
+     * all carry the same count. A failed request releases it for a later natural
+     * write in the same invocation.
+     */
+    async withEventCreate(params, create) {
+        if (!this.active || this.reported)
+            return create(params);
+        this.reported = true;
+        try {
+            return await create({
+                ...params,
+                replayDivergenceCount: this.divergenceCount,
+            });
+        }
+        catch (error) {
+            this.reported = false;
+            throw error;
+        }
+    }
+}
+/** See {@link ReplayRecoveryReporter.inert}. */
+class InertReplayRecoveryReporter extends ReplayRecoveryReporter {
+    constructor() {
+        super(0);
+    }
+    activate() {
+        // Deliberately empty: nothing to report, so nothing can arm.
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVwbGF5LXJlY292ZXJ5LXJlcG9ydGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3J1bnRpbWUvcmVwbGF5LXJlY292ZXJ5LXJlcG9ydGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBOzs7Ozs7O0dBT0c7QUFDSCxNQUFNLE9BQU8sc0JBQXNCO0lBSUo7SUFIckIsTUFBTSxHQUFHLEtBQUssQ0FBQztJQUNmLFFBQVEsR0FBRyxLQUFLLENBQUM7SUFFekIsWUFBNkIsZUFBdUI7UUFBdkIsb0JBQWUsR0FBZixlQUFlLENBQVE7SUFBRyxDQUFDO0lBRXhEOzs7Ozs7Ozs7T0FTRztJQUNILE1BQU0sQ0FBQyxLQUFLO1FBQ1YsT0FBTyxJQUFJLDJCQUEyQixFQUFFLENBQUM7SUFDM0MsQ0FBQztJQUVELFFBQVE7UUFDTixJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksQ0FBQztJQUNyQixDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0gsS0FBSyxDQUFDLGVBQWUsQ0FDbkIsTUFBcUMsRUFDckMsTUFBNkQ7UUFFN0QsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLElBQUksSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFPLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUV6RCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUM7WUFDSCxPQUFPLE1BQU0sTUFBTSxDQUFDO2dCQUNsQixHQUFHLE1BQU07Z0JBQ1QscUJBQXFCLEVBQUUsSUFBSSxDQUFDLGVBQWU7YUFDNUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQztRQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7WUFDZixJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztZQUN0QixNQUFNLEtBQUssQ0FBQztRQUNkLENBQUM7SUFDSCxDQUFDO0NBQ0Y7QUFFRCxnREFBZ0Q7QUFDaEQsTUFBTSwyQkFBNEIsU0FBUSxzQkFBc0I7SUFDOUQ7UUFDRSxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDWCxDQUFDO0lBRVEsUUFBUTtRQUNmLDZEQUE2RDtJQUMvRCxDQUFDO0NBQ0YiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgdHlwZSB7IENyZWF0ZUV2ZW50UGFyYW1zIH0gZnJvbSAnQHdvcmtmbG93L3dvcmxkJztcblxuLyoqXG4gKiBJbnZvY2F0aW9uLXNjb3BlZCwgb25lLXNob3QgcmVwb3J0ZXIgZm9yIGEgcmVwbGF5LWRpdmVyZ2VuY2UgcmVjb3ZlcnkuXG4gKlxuICogVGhlIHJlcG9ydGVyIHN0YXlzIGRvcm1hbnQgdW50aWwgcmVwbGF5IHJlYWNoZXMgYSB2YWxpZCBkZXRlcm1pbmlzdGljXG4gKiBib3VuZGFyeS4gVGhlIGZpcnN0IHN1YnNlcXVlbnQgZXZlbnQgd3JpdGUgY2FycmllcyB0aGUgdGVsZW1ldHJ5IGZvciBmcmVlLlxuICogQSByZWNvdmVyZWQgaW52b2NhdGlvbiB3aXRoIG5vIG5hdHVyYWwgZXZlbnQgd3JpdGUgaW50ZW50aW9uYWxseSByZXBvcnRzXG4gKiBub3RoaW5nLlxuICovXG5leHBvcnQgY2xhc3MgUmVwbGF5UmVjb3ZlcnlSZXBvcnRlciB7XG4gIHByaXZhdGUgYWN0aXZlID0gZmFsc2U7XG4gIHByaXZhdGUgcmVwb3J0ZWQgPSBmYWxzZTtcblxuICBjb25zdHJ1Y3Rvcihwcml2YXRlIHJlYWRvbmx5IGRpdmVyZ2VuY2VDb3VudDogbnVtYmVyKSB7fVxuXG4gIC8qKlxuICAgKiBBIHJlcG9ydGVyIGZvciBhbiBpbnZvY2F0aW9uIHRoYXQgbmV2ZXIgZGl2ZXJnZWQ6IGBhY3RpdmF0ZSgpYCBjYW5ub3QgYXJtXG4gICAqIGl0LCBzbyBgd2l0aEV2ZW50Q3JlYXRlYCBhbHdheXMgcGFzc2VzIHdyaXRlcyB0aHJvdWdoIHVudG91Y2hlZC4gTGV0cyBldmVyeVxuICAgKiBjYWxsIHNpdGUgaG9sZCBhIG5vbi1vcHRpb25hbCByZXBvcnRlciBpbnN0ZWFkIG9mIGJyYW5jaGluZyBvbiBvbmUuXG4gICAqXG4gICAqIFN0cnVjdHVyYWxseSBpbmVydCByYXRoZXIgdGhhbiBhIHplcm8tY291bnQgcmVwb3J0ZXIgb24gcHVycG9zZSDigJQgdGhlXG4gICAqIHNlcnZlciByZWplY3RzIGEgY291bnQgYmVsb3cgMSBhcyBvdXQtb2YtcmFuZ2UgYW5kIGVtaXRzIGFcbiAgICogYHRlbGVtZXRyeV9yZWplY3RlZGAgbWV0cmljLCBzbyBhIHJlcG9ydGVyIHRoYXQgKmNhbiogcmVwb3J0IHplcm8gaXMgYVxuICAgKiBsaXZlIGZvb3RndW4uXG4gICAqL1xuICBzdGF0aWMgaW5lcnQoKTogUmVwbGF5UmVjb3ZlcnlSZXBvcnRlciB7XG4gICAgcmV0dXJuIG5ldyBJbmVydFJlcGxheVJlY292ZXJ5UmVwb3J0ZXIoKTtcbiAgfVxuXG4gIGFjdGl2YXRlKCk6IHZvaWQge1xuICAgIHRoaXMuYWN0aXZlID0gdHJ1ZTtcbiAgfVxuXG4gIC8qKlxuICAgKiBEZWNvcmF0ZSBvbmUgbmF0dXJhbCBldmVudCB3cml0ZSB3aXRoIHJlY292ZXJ5IHRlbGVtZXRyeS5cbiAgICpcbiAgICogVGhlIGNsYWltIGlzIHRha2VuIHN5bmNocm9ub3VzbHksIHNvIGNvbmN1cnJlbnQgc3VzcGVuc2lvbiB3cml0ZXMgY2Fubm90XG4gICAqIGFsbCBjYXJyeSB0aGUgc2FtZSBjb3VudC4gQSBmYWlsZWQgcmVxdWVzdCByZWxlYXNlcyBpdCBmb3IgYSBsYXRlciBuYXR1cmFsXG4gICAqIHdyaXRlIGluIHRoZSBzYW1lIGludm9jYXRpb24uXG4gICAqL1xuICBhc3luYyB3aXRoRXZlbnRDcmVhdGU8VD4oXG4gICAgcGFyYW1zOiBDcmVhdGVFdmVudFBhcmFtcyB8IHVuZGVmaW5lZCxcbiAgICBjcmVhdGU6IChwYXJhbXM6IENyZWF0ZUV2ZW50UGFyYW1zIHwgdW5kZWZpbmVkKSA9PiBQcm9taXNlPFQ+XG4gICk6IFByb21pc2U8VD4ge1xuICAgIGlmICghdGhpcy5hY3RpdmUgfHwgdGhpcy5yZXBvcnRlZCkgcmV0dXJuIGNyZWF0ZShwYXJhbXMpO1xuXG4gICAgdGhpcy5yZXBvcnRlZCA9IHRydWU7XG4gICAgdHJ5IHtcbiAgICAgIHJldHVybiBhd2FpdCBjcmVhdGUoe1xuICAgICAgICAuLi5wYXJhbXMsXG4gICAgICAgIHJlcGxheURpdmVyZ2VuY2VDb3VudDogdGhpcy5kaXZlcmdlbmNlQ291bnQsXG4gICAgICB9KTtcbiAgICB9IGNhdGNoIChlcnJvcikge1xuICAgICAgdGhpcy5yZXBvcnRlZCA9IGZhbHNlO1xuICAgICAgdGhyb3cgZXJyb3I7XG4gICAgfVxuICB9XG59XG5cbi8qKiBTZWUge0BsaW5rIFJlcGxheVJlY292ZXJ5UmVwb3J0ZXIuaW5lcnR9LiAqL1xuY2xhc3MgSW5lcnRSZXBsYXlSZWNvdmVyeVJlcG9ydGVyIGV4dGVuZHMgUmVwbGF5UmVjb3ZlcnlSZXBvcnRlciB7XG4gIGNvbnN0cnVjdG9yKCkge1xuICAgIHN1cGVyKDApO1xuICB9XG5cbiAgb3ZlcnJpZGUgYWN0aXZhdGUoKTogdm9pZCB7XG4gICAgLy8gRGVsaWJlcmF0ZWx5IGVtcHR5OiBub3RoaW5nIHRvIHJlcG9ydCwgc28gbm90aGluZyBjYW4gYXJtLlxuICB9XG59XG4iXX0=

package/dist/runtime/resume-hook.d.ts

@@ -1,13 +1,36 @@
 import { type Hook } from '@workflow/world';
-import { type CryptoKey } from '../encryption.js';
+import { type PayloadKey } from '../serialization.js';
 /**
  * Get the hook by token to find the associated workflow run,
  * and hydrate the `metadata` property if it was set from within
  * the workflow run.
  *
+ * A Hook kept by minimum retention remains available here after its run ends,
+ * but cannot be resumed.
+ *
  * @param token - The unique token identifying the hook
  */
 export declare function getHookByToken(token: string): Promise<Hook>;
+/**
+ * The result of {@link resumeHook}: a {@link Hook} augmented with an optional
+ * resilience signal.
+ *
+ * On the parallel fast path, `resumeHook()` writes the `hook_received` event and
+ * dispatches the workflow queue message concurrently. When the direct event
+ * write fails *transiently* — a 429/5xx, a transport error, or an expected
+ * `(runId, resumeId)` conflict with its own re-ensuring consumer — but the queue
+ * dispatch succeeds, the resume is still guaranteed: the queue consumer
+ * idempotently materializes the `hook_received` event from the payload carried
+ * on the message before replay. In that recovered case the returned hook carries
+ * `resilientResume: true`.
+ *
+ * On the happy path (the direct write landed) and on the sequential fallback
+ * path, the flag is absent (`undefined`). Callers that don't care about the
+ * distinction can treat the result as a plain {@link Hook}.
+ */
+export type ResumedHook = Hook & {
+    resilientResume?: boolean;
+};
 /**
  * Resumes a workflow run by sending a payload to a hook identified by its token.
  *
@@ -16,8 +39,8 @@ export declare function getHookByToken(token: string): Promise<Hook>;
  *
  * @param tokenOrHook - The unique token identifying the hook, or the hook object itself
  * @param payload - The data payload to send to the hook
- * @returns Promise resolving to the hook
- * @throws Error if the hook is not found or if there's an error during the process
+ * @returns Promise resolving to the {@link ResumedHook}
+ * @throws {HookNotFoundError} If the Hook does not exist or its run has ended
  *
  * @example
  *
@@ -37,7 +60,7 @@ export declare function getHookByToken(token: string): Promise<Hook>;
  * }
  * ```
  */
-export declare function resumeHook<T = any>(tokenOrHook: string | Hook, payload: T, encryptionKeyOverride?: CryptoKey): Promise<Hook>;
+export declare function resumeHook<T = any>(tokenOrHook: string | Hook, payload: T, encryptionKeyOverride?: PayloadKey): Promise<ResumedHook>;
 /**
  * Resumes a webhook by sending a {@link https://developer.mozilla.org/en-US/docs/Web/API/Request | Request}
  * object to a hook identified by its token.

package/dist/runtime/resume-hook.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resume-hook.d.ts","sourceRoot":"","sources":["../../src/runtime/resume-hook.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,IAAI,EAMV,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,KAAK,SAAS,EAAa,MAAM,kBAAkB,CAAC;AAqC7D;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGjE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,UAAU,CAAC,CAAC,GAAG,GAAG,EACtC,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,OAAO,EAAE,CAAC,EACV,qBAAqB,CAAC,EAAE,SAAS,GAChC,OAAO,CAAC,IAAI,CAAC,CAoHf;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,QAAQ,CAAC,CA0DnB"}
+{"version":3,"file":"resume-hook.d.ts","sourceRoot":"","sources":["../../src/runtime/resume-hook.ts"],"names":[],"mappings":"AAOA,OAAO,EAGL,KAAK,IAAI,EAUV,MAAM,iBAAiB,CAAC;AAQzB,OAAO,EAGL,KAAK,UAAU,EAGhB,MAAM,qBAAqB,CAAC;AAwJ7B;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGjE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,WAAW,GAAG,IAAI,GAAG;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,UAAU,CAAC,CAAC,GAAG,GAAG,EACtC,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,OAAO,EAAE,CAAC,EACV,qBAAqB,CAAC,EAAE,UAAU,GACjC,OAAO,CAAC,WAAW,CAAC,CAUtB;AA8ZD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,QAAQ,CAAC,CA+DnB"}