eve 0.29.5 → 0.30.0

package/dist/src/compiled/@workflow/core/runtime/constants.d.ts

@@ -201,4 +201,22 @@ export declare const REPLAY_DIVERGENCE_MAX_RETRIES = 3;
  * `WORKFLOW_REPLAY_DIVERGENCE_MAX_RETRIES`.
  */
 export declare function getReplayDivergenceMaxRetries(): number;
+export declare const PRECONDITION_MAX_INPROCESS_RESTARTS = 3;
+/**
+ * Effective in-process replay-restart budget for stale-snapshot rejections.
+ * Override via `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`.
+ */
+export declare function getPreconditionMaxInProcessRestarts(): number;
+export declare const PRECONDITION_MAX_REINVOCATIONS = 5;
+/**
+ * Effective per-run budget for re-invocations caused by stale-snapshot
+ * rejections. Override via `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS`.
+ */
+export declare function getPreconditionMaxReinvocations(): number;
+export declare const PRECONDITION_REINVOKE_DELAY_SECONDS = 2;
+/**
+ * Effective delay before a precondition re-invocation. Override via
+ * `WORKFLOW_PRECONDITION_REINVOKE_DELAY_SECONDS`.
+ */
+export declare function getPreconditionReinvokeDelaySeconds(): number;
 //# sourceMappingURL=constants.d.ts.map

package/dist/src/compiled/@workflow/core/runtime/helpers.d.ts

@@ -1,4 +1,4 @@
-import type { Event, HealthCheckPayload, ValidQueueName, WorkflowRun, World } from '#compiled/@workflow/world/index.js';
+import type { CreateEventParams, CreateEventRequest, Event, EventResult, HealthCheckPayload, ValidQueueName, WorkflowRun, World } from '#compiled/@workflow/world/index.js';
 import { type PayloadKey } from '../serialization/encryption.js';
 /**
  * Validates a workflow name and returns the corresponding queue name.
@@ -34,6 +34,18 @@ export interface HealthCheckResult {
      * key-lookup request.
      */
     encryptionPublicKey?: string;
+    /**
+     * The responding deployment's `HOOK_RESUME_INPUT_VERSION` — the protocol
+     * version at which the *consumer* (queue-message target) re-ensures the
+     * `hook_received` event from `hookInput` on replay. A cross-deployment
+     * `start()` stamps the *target's* value (not the caller's) into the new
+     * run's `executionContext.hookResumeInputVersion` so that `resumeHook()`
+     * only takes the parallel path when the deployment that will actually
+     * consume the queue message is known to honor `hookInput`. Omitted when the
+     * responding deployment predates this field (an older consumer that ignores
+     * `hookInput`), which fails the gate closed.
+     */
+    hookResumeInputVersion?: number;
 }
 /**
  * Checks if the given message is a health check payload.
@@ -76,8 +88,29 @@ export declare function healthCheck(world: World, options?: HealthCheckOptions):
  *
  * Pass the IDs currently present in `target` when appending repeatedly to the
  * same array. The set is updated alongside `target`.
+ *
+ * Events are appended in the order the World returned them, and are not
+ * re-sorted: a World's canonical order is its own, and the runtime cannot
+ * reproduce it from event ids alone. `world-vercel` orders by event id, while
+ * `world-local` orders by `(createdAt, eventId)` and deliberately re-mints keys
+ * so that the two diverge. Every append source is already in canonical order
+ * relative to the tail (a cursor-delimited page, or a write-response delta), so
+ * receipt order is the order to keep. Nothing downstream may assume the tail is
+ * the newest event — see {@link latestEventStateUpdatedAt}.
  */
 export declare function appendUniqueEvents(target: Event[], events: readonly Event[], targetIds?: Set<string>): void;
+/**
+ * Inserts `event` into `target` at the position that keeps `target` ordered by
+ * ascending `eventId`, or no-ops if an event with the same `eventId` is already
+ * present (idempotent).
+ *
+ * `preloadedEvents` is loaded `sortOrder: 'asc'` and is never re-sorted
+ * client-side, so a `hook_received` spliced in by the lazy-resume consumer must
+ * land in `eventId` order — a plain `push` would place a late-committing
+ * earlier event after events that sort before it, corrupting replay. Event IDs
+ * are ULIDs, so lexicographic string order matches commit order.
+ */
+export declare function insertEventByEventId(target: Event[], event: Event): void;
 /**
  * Loads workflow run events by iterating through all pages of paginated
  * results. Events are returned in chronological (ascending) order for
@@ -94,18 +127,11 @@ export declare function loadWorkflowRunEvents(runId: string, afterCursor?: strin
     cursor: string | null;
 }>;
 /**
- * Maximum number of times a replay-context event creation will reload the
- * event log and retry after the backend rejects it as stale (412). After this
- * many failed reloads the precondition error propagates so the run is
- * re-invoked from the queue with a fresh replay.
- */
-export declare const PRECONDITION_MAX_RELOAD_RETRIES = 2;
-/**
- * A mutable view of the runtime's in-memory event log. `withPreconditionRetry`
- * appends freshly-loaded events to `events` (in place) and advances `cursor`
- * when it reloads, so the caller's loaded snapshot stays current.
+ * The runtime's loaded event-log snapshot: the events replayed so far and the
+ * cursor positioned after them. Handed to helpers that derive the precondition
+ * snapshot from it; they do not mutate it.
  */
-export interface MutableEventLog {
+export interface LoadedEventLog {
     events: Event[];
     cursor: string | null;
 }
@@ -120,36 +146,82 @@ export interface MutableEventLog {
 export declare function isPreconditionGuardEnabled(): boolean;
 /**
  * The `stateUpdatedAt` value to send with a replay-context event creation: the
- * ULID time (epoch ms) of the latest event the runtime has loaded. Events are
- * stored in ascending order, so the last one is the newest. Returns `undefined`
- * when there are no events or the latest id is not a decodable ULID.
+ * *maximum* ULID time (epoch ms) over the events the runtime has loaded. Returns
+ * `undefined` when there are no events or that id is not a decodable ULID.
+ *
+ * It is the maximum rather than the tail's because the loaded log is in the
+ * World's canonical order, which is not necessarily event-id order (see
+ * {@link appendUniqueEvents}). The maximum is what lets the count sent alongside
+ * it be read as "events at or below this watermark": every loaded event is at or
+ * below it, so the count is exactly `events.length`. Reading the tail instead
+ * would understate the watermark on a World whose order is not id-ordered, which
+ * is safe (it can only weaken detection) but needlessly imprecise.
+ *
+ * The maximum is found by lexicographic id comparison, decoding only once: the
+ * 26-character Crockford ULID encodes its timestamp in the leading 10
+ * characters, so the greatest id also carries the greatest time.
  *
  * Granularity: snapshots are epoch-milliseconds, and the backend allows an
  * equal-timestamp snapshot (an up-to-date client must not be rejected). Two
  * out-of-band events landing in the same millisecond where only the first was
- * loaded therefore pass the guard undetected — the guard is best-effort by
- * design, and fails open rather than livelocking.
+ * loaded therefore pass this half of the guard undetected — that is exactly
+ * the hole `stateEventCount` closes, since the count of events at or below the
+ * watermark differs even when the watermarks are equal.
  */
 export declare function latestEventStateUpdatedAt(events: Event[]): number | undefined;
 /**
- * The `stateUpdatedAt` to attach to a replay-context event creation:
- * the loaded snapshot's ULID time when the precondition guard is enabled,
- * `undefined` (no guard, backend behaves as before) otherwise.
+ * The precondition snapshot a replay-context event creation sends, describing
+ * the event log the replay derived the event from.
+ *
+ * The three fields are one indivisible unit: the backend reads the count only
+ * relative to the watermark, and returns its inline delta only relative to the
+ * cursor. Passing them as a single object is what keeps them from drifting
+ * apart at a call site.
+ */
+export interface PreconditionSnapshotParams {
+    stateUpdatedAt?: number;
+    stateEventCount?: number;
+    stateCursor?: string;
+}
+/**
+ * Build the precondition snapshot to attach to a replay-context event creation.
+ *
+ * Returns an empty object — no guard, backend behaves as before — when the
+ * guard is disabled or the watermark is not derivable. All three fields fail
+ * open together: a count without a watermark is meaningless to the backend, and
+ * a cursor without either would invite a delta nobody asked for.
+ *
+ * `stateEventCount` is `events.length` because the watermark is the log's
+ * *maximum* ULID time, so every loaded event is at or below it regardless of the
+ * order the World returned them in.
+ *
+ * Both fields are therefore invariant under permutation of the log: a maximum is
+ * order-independent, and the length is set cardinality once `appendUniqueEvents`
+ * has deduped by event id. Two replays that consume the same events in different
+ * orders send an identical snapshot, so this guard detects that a log is missing
+ * an event and can never detect that a replay consumed one in a different order.
  */
-export declare function stateUpdatedAtForCreate(events: Event[]): number | undefined;
+export declare function preconditionSnapshotParams(events: Event[], cursor?: string | null): PreconditionSnapshotParams;
 /**
- * Runs a replay-context event creation with the optimistic-concurrency guard.
+ * The events a rejecting World attached to a `PreconditionFailedError`, when it
+ * returned the ones the client's snapshot was missing inline.
+ *
+ * Returns `null` for anything else — no details, a World that did not implement
+ * this, or a payload that does not narrow cleanly. Callers fall back to
+ * reloading the event log, which is always correct; this is untrusted-shaped
+ * data on a failure path, so nothing here is repaired.
  *
- * `op` receives the current `stateUpdatedAt` (the ULID time of the latest
- * loaded event) to pass to `world.events.create`. If the backend rejects the
- * creation as stale (`PreconditionFailedError` / 412), the event log is
- * reloaded to completion from the last cursor, merged into `log` in place, and
- * `op` is retried with the now-newer snapshot — up to
- * `PRECONDITION_MAX_RELOAD_RETRIES` times. If it still fails, the error is
- * rethrown so the run falls back to a queue re-invocation. Non-precondition
- * errors are rethrown immediately.
+ * `runId` is the caller's run. Every event must belong to it: the delta is
+ * merged straight into the replay's log, and one foreign event there is a
+ * corrupt log rather than a corrected one — the replay would consume a
+ * correlation id for an event that does not exist on this run.
  */
-export declare function withPreconditionRetry<T>(runId: string, log: MutableEventLog, op: (stateUpdatedAt: number | undefined) => Promise<T>): Promise<T>;
+export declare function preconditionEventDelta(error: unknown, runId: string): {
+    events: Event[];
+    cursor: string | null;
+} | null;
+/** Creates one event on a bound run, carrying replay-recovery telemetry. */
+export type EventCreator = (data: CreateEventRequest, params?: CreateEventParams) => Promise<EventResult>;
 /**
  * Wraps a request/response handler and adds a health check "mode"
  * based on the presence of a `__health` query parameter.

package/dist/src/compiled/@workflow/core/runtime/replay-recovery-reporter.d.ts

@@ -0,0 +1,36 @@
+import type { CreateEventParams } from '#compiled/@workflow/world/index.js';
+/**
+ * Invocation-scoped, one-shot reporter for a replay-divergence recovery.
+ *
+ * The reporter stays dormant until replay reaches a valid deterministic
+ * boundary. The first subsequent event write carries the telemetry for free.
+ * A recovered invocation with no natural event write intentionally reports
+ * nothing.
+ */
+export declare class ReplayRecoveryReporter {
+    private readonly divergenceCount;
+    private active;
+    private reported;
+    constructor(divergenceCount: number);
+    /**
+     * A reporter for an invocation that never diverged: `activate()` cannot arm
+     * it, so `withEventCreate` always passes writes through untouched. Lets every
+     * call site hold a non-optional reporter instead of branching on one.
+     *
+     * Structurally inert rather than a zero-count reporter on purpose — the
+     * server rejects a count below 1 as out-of-range and emits a
+     * `telemetry_rejected` metric, so a reporter that *can* report zero is a
+     * live footgun.
+     */
+    static inert(): ReplayRecoveryReporter;
+    activate(): void;
+    /**
+     * Decorate one natural event write with recovery telemetry.
+     *
+     * The claim is taken synchronously, so concurrent suspension writes cannot
+     * all carry the same count. A failed request releases it for a later natural
+     * write in the same invocation.
+     */
+    withEventCreate<T>(params: CreateEventParams | undefined, create: (params: CreateEventParams | undefined) => Promise<T>): Promise<T>;
+}
+//# sourceMappingURL=replay-recovery-reporter.d.ts.map

package/dist/src/compiled/@workflow/core/runtime/resume-hook.d.ts

@@ -11,6 +11,26 @@ import { type PayloadKey } from '../serialization.js';
  * @param token - The unique token identifying the hook
  */
 export declare function getHookByToken(token: string): Promise<Hook>;
+/**
+ * The result of {@link resumeHook}: a {@link Hook} augmented with an optional
+ * resilience signal.
+ *
+ * On the parallel fast path, `resumeHook()` writes the `hook_received` event and
+ * dispatches the workflow queue message concurrently. When the direct event
+ * write fails *transiently* — a 429/5xx, a transport error, or an expected
+ * `(runId, resumeId)` conflict with its own re-ensuring consumer — but the queue
+ * dispatch succeeds, the resume is still guaranteed: the queue consumer
+ * idempotently materializes the `hook_received` event from the payload carried
+ * on the message before replay. In that recovered case the returned hook carries
+ * `resilientResume: true`.
+ *
+ * On the happy path (the direct write landed) and on the sequential fallback
+ * path, the flag is absent (`undefined`). Callers that don't care about the
+ * distinction can treat the result as a plain {@link Hook}.
+ */
+export type ResumedHook = Hook & {
+    resilientResume?: boolean;
+};
 /**
  * Resumes a workflow run by sending a payload to a hook identified by its token.
  *
@@ -19,7 +39,7 @@ export declare function getHookByToken(token: string): Promise<Hook>;
  *
  * @param tokenOrHook - The unique token identifying the hook, or the hook object itself
  * @param payload - The data payload to send to the hook
- * @returns Promise resolving to the hook
+ * @returns Promise resolving to the {@link ResumedHook}
  * @throws {HookNotFoundError} If the Hook does not exist or its run has ended
  *
  * @example
@@ -40,7 +60,7 @@ export declare function getHookByToken(token: string): Promise<Hook>;
  * }
  * ```
  */
-export declare function resumeHook<T = any>(tokenOrHook: string | Hook, payload: T, encryptionKeyOverride?: PayloadKey): Promise<Hook>;
+export declare function resumeHook<T = any>(tokenOrHook: string | Hook, payload: T, encryptionKeyOverride?: PayloadKey): Promise<ResumedHook>;
 /**
  * Resumes a webhook by sending a {@link https://developer.mozilla.org/en-US/docs/Web/API/Request | Request}
  * object to a hook identified by its token.

package/dist/src/compiled/@workflow/core/runtime/step-executor.d.ts

@@ -1,5 +1,7 @@
 import type { Event, SerializedData, World } from '#compiled/@workflow/world/index.js';
 import type { PayloadKey } from '../serialization/encryption.js';
+import { type PreconditionSnapshotParams } from './helpers.js';
+import { ReplayRecoveryReporter } from './replay-recovery-reporter.js';
 import { type StepLatencyTracking } from './step-latency.js';
 export declare const DEFAULT_STEP_MAX_RETRIES = 3;
 export interface StepExecutorParams {
@@ -57,20 +59,20 @@ export interface StepExecutorParams {
      */
     inlineDeltaSinceCursor?: string;
     /**
-     * Precondition-guard snapshot (epoch ms of the latest event the caller's
-     * replay loaded) to attach to this step's `step_started` claim. On the lazy
-     * inline path the claim is the step's FIRST durable write (its
-     * `step_created` is deferred), so without this the claim would bypass the
-     * optimistic-concurrency guard entirely: a replay working from a stale view
-     * could claim — and then commit — a step scheduled without observing an
-     * out-of-band event. A guard-enforcing World rejects a stale claim with
-     * `PreconditionFailedError` (412); executeStep does NOT translate that
-     * rejection (re-claiming in place would still commit the stale schedule),
-     * so it propagates for the caller to abandon the batch and force a fresh
-     * replay. Undefined when the guard is disabled or the caller has no
-     * snapshot; Worlds that don't enforce the guard ignore it.
+     * Precondition-guard snapshot of the event log the caller's replay loaded, to
+     * attach to this step's `step_started` claim. On the lazy inline path the
+     * claim is the step's FIRST durable write (its `step_created` is deferred),
+     * so without this the claim would bypass the optimistic-concurrency guard
+     * entirely: a replay working from a stale view could claim — and then commit
+     * — a step scheduled without observing an event it never loaded. A
+     * guard-enforcing World rejects a stale claim with `PreconditionFailedError`
+     * (412); executeStep does NOT translate that rejection (re-claiming in place
+     * would still commit the stale schedule), so it propagates for the caller to
+     * abandon the batch and restart its replay. Undefined when the guard is
+     * disabled or the caller has no snapshot; Worlds that don't enforce the guard
+     * ignore it.
      */
-    stateUpdatedAt?: number;
+    preconditionSnapshot?: PreconditionSnapshotParams;
     /**
      * Suppress optimistic inline start for this step regardless of
      * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the
@@ -120,6 +122,8 @@ export interface StepExecutorParams {
      *   (`metadata.attempt`), which increments on every redelivery.
      */
     authoritativeAttempt?: number;
+    /** One-shot recovery telemetry activated by the orchestrator replay. */
+    replayRecoveryReporter?: ReplayRecoveryReporter;
 }
 /**
  * Inline-delta returned by a step-terminal write when the caller passed

package/dist/src/compiled/@workflow/core/runtime/suspension-handler.d.ts

@@ -1,7 +1,8 @@
 import type { Span } from '#compiled/@opentelemetry/api/index.js';
 import { type SerializedData, type WorkflowRun, type World } from '#compiled/@workflow/world/index.js';
 import type { StepInvocationQueueItem, WorkflowSuspension } from '../global.js';
-import { type MutableEventLog } from './helpers.js';
+import { type LoadedEventLog } from './helpers.js';
+import { ReplayRecoveryReporter } from './replay-recovery-reporter.js';
 export interface SuspensionHandlerParams {
     suspension: WorkflowSuspension;
     world: World;
@@ -9,13 +10,15 @@ export interface SuspensionHandlerParams {
     span?: Span;
     requestId?: string;
     /**
-     * The runtime's loaded event log. Each event creation is sent with this
-     * snapshot's `stateUpdatedAt` and, if the backend rejects it as stale (412),
-     * the log is reloaded in place and the create is retried — see
-     * `withPreconditionRetry`. Guarding per-create (rather than the whole
-     * suspension) ensures a retry never re-issues an already-created event.
+     * The runtime's loaded event log. Every event creation this suspension makes
+     * is sent with the precondition snapshot derived from it, so a backend that
+     * has recorded an event the replay did not see rejects the write with a 412
+     * instead of accepting a divergent event. The rejection is not retried here:
+     * the event's correlation id was minted by *this* replay's seeded sequence,
+     * so re-committing it against a corrected log would persist an event no
+     * correct replay produces. The caller restarts the replay instead.
      */
-    eventLog?: MutableEventLog;
+    eventLog?: LoadedEventLog;
     /**
      * Turbo mode only: a promise that resolves once the backgrounded
      * `run_started` has landed (the run exists). When present, every world write
@@ -26,6 +29,8 @@ export interface SuspensionHandlerParams {
      * where `run_started` was already awaited up front.
      */
     runReadyBarrier?: Promise<unknown>;
+    /** One-shot telemetry reporter, activated only after replay has recovered. */
+    replayRecoveryReporter?: ReplayRecoveryReporter;
 }
 /**
  * Result of handling a suspension. Returns pending step items so the caller
@@ -103,5 +108,5 @@ export interface SuspensionHandlerResult {
  * 1. Hooks are processed first to prevent race conditions with webhook receivers
  * 2. Step events and wait events are created in parallel
  */
-export declare function handleSuspension({ suspension, world, run, span, requestId, eventLog, runReadyBarrier, }: SuspensionHandlerParams): Promise<SuspensionHandlerResult>;
+export declare function handleSuspension({ suspension, world, run, span, requestId, eventLog, runReadyBarrier, replayRecoveryReporter, }: SuspensionHandlerParams): Promise<SuspensionHandlerResult>;
 //# sourceMappingURL=suspension-handler.d.ts.map

package/dist/src/compiled/@workflow/core/runtime.d.ts

@@ -2,7 +2,7 @@ import { type Event, type WorkflowRun } from '#compiled/@workflow/world/index.js
 export type { Event, WorkflowRun };
 export { WorkflowSuspension } from './global.js';
 export { type HealthCheckOptions, type HealthCheckResult, healthCheck, } from './runtime/helpers.js';
-export { getHookByToken, resumeHook, resumeWebhook, } from './runtime/resume-hook.js';
+export { getHookByToken, type ResumedHook, resumeHook, resumeWebhook, } from './runtime/resume-hook.js';
 export { getRun, Run, type WorkflowReadableStream, type WorkflowReadableStreamOptions, } from './runtime/run.js';
 export { type CancelRunOptions, cancelRun, listStreams, type ReadStreamOptions, type RecreateRunOptions, type ReenqueueRunOptions, readStream, recreateRunFromExisting, reenqueueRun, type StopSleepOptions, type StopSleepResult, wakeUpRun, } from './runtime/runs.js';
 export { type StartOptions, type StartOptionsBase, type StartOptionsWithDeploymentId, type StartOptionsWithoutDeploymentId, start, } from './runtime/start.js';