@redflow/client 0.0.1

package/src/worker.ts

@@ -0,0 +1,1042 @@
+import type { RedisClient } from "bun";
+import { RedisClient as BunRedisClient, redis as defaultRedis } from "bun";
+import { Cron } from "croner";
+import {
+  computeRetryDelayMs,
+  createClient,
+  defaultPrefix,
+  isRetryableError,
+  makeErrorJson,
+  validateInputWithSchema,
+  type RedflowClient,
+} from "./client";
+import { CanceledError, OutputSerializationError, TimeoutError, UnknownWorkflowError } from "./internal/errors";
+import { keys } from "./internal/keys";
+import { safeJsonParse, safeJsonStringify, safeJsonTryParse } from "./internal/json";
+import { sleep } from "./internal/sleep";
+import { nowMs } from "./internal/time";
+import { getDefaultRegistry, type WorkflowRegistry } from "./registry";
+import type { OnFailureContext, RunStatus, StepApi, StepStatus } from "./types";
+
+export type StartWorkerOptions = {
+  redis?: RedisClient;
+  url?: string;
+  prefix?: string;
+  registry?: WorkflowRegistry;
+
+  /** If omitted, derived from workflows' `queue` options. */
+  queues?: string[];
+  concurrency?: number;
+
+  /** Runtime tuning. Most users don't need this. */
+  runtime?: {
+    /** Lease TTL in ms for crash recovery. Default: 5000 */
+    leaseMs?: number;
+    /** BLMOVE block timeout in seconds. Default: 1 */
+    blmoveTimeoutSec?: number;
+    /** How often reaper scans processing lists. Default: 500 */
+    reaperIntervalMs?: number;
+  };
+};
+
+export type WorkerHandle = {
+  stop(): Promise<void>;
+};
+
+const UNLOCK_LUA = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("del", KEYS[1])
+else
+  return 0
+end
+`;
+
+const RENEW_LEASE_LUA = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("set", KEYS[1], ARGV[1], "PX", ARGV[2])
+else
+  return nil
+end
+`;
+
+const CLAIM_QUEUED_RUN_FOR_INLINE_LUA = `
+if redis.call("hget", KEYS[3], "status") ~= "queued" then
+  return 0
+end
+
+if redis.call("lrem", KEYS[1], 1, ARGV[1]) <= 0 then
+  return 0
+end
+
+redis.call("lpush", KEYS[2], ARGV[1])
+return 1
+`;
+
+export async function startWorker(options?: StartWorkerOptions): Promise<WorkerHandle> {
+  const registry = options?.registry ?? getDefaultRegistry();
+  const prefix = options?.prefix ?? defaultPrefix();
+  const ownsBaseRedis = !options?.redis && !!options?.url;
+  const baseRedis = options?.redis ?? (options?.url ? new BunRedisClient(options.url) : defaultRedis);
+  const syncClient = createClient({ redis: baseRedis, prefix });
+
+  const queues = options?.queues ?? deriveQueuesFromRegistry(registry);
+  const concurrency = Math.max(1, options?.concurrency ?? 1);
+  const leaseMs = Math.max(100, options?.runtime?.leaseMs ?? 5000);
+  const blmoveTimeoutSec = options?.runtime?.blmoveTimeoutSec ?? 1;
+  const reaperIntervalMs = options?.runtime?.reaperIntervalMs ?? 500;
+
+  const abort = new AbortController();
+  const tasks: Promise<void>[] = [];
+  const ownedRedis: RedisClient[] = [];
+
+  const closeOwnedResources = async (): Promise<void> => {
+    abort.abort();
+    await Promise.allSettled(tasks);
+    for (const r of ownedRedis) {
+      try {
+        r.close();
+      } catch {
+        // Ignore close errors during shutdown.
+      }
+    }
+    if (ownsBaseRedis) {
+      try {
+        baseRedis.close();
+      } catch {
+        // Ignore close errors during shutdown.
+      }
+    }
+  };
+
+  try {
+    await syncClient.syncRegistry(registry);
+
+    // Worker loops (blocking BLMOVE). Use dedicated connections per slot.
+    for (let i = 0; i < concurrency; i++) {
+      const r = await baseRedis.duplicate();
+      const client = createClient({ redis: r, prefix });
+      ownedRedis.push(r);
+      tasks.push(workerLoop({
+        redis: r,
+        client,
+        registry,
+        prefix,
+        queues,
+        blmoveTimeoutSec,
+        leaseMs,
+        signal: abort.signal,
+      }));
+    }
+
+    // Background loops use dedicated Redis connections to avoid response interleaving.
+    const promoterRedis = await baseRedis.duplicate();
+    const promoterClient = createClient({ redis: promoterRedis, prefix });
+    ownedRedis.push(promoterRedis);
+    tasks.push(scheduledPromoterLoop({ redis: promoterRedis, client: promoterClient, prefix, queues, signal: abort.signal }));
+
+    const reaperRedis = await baseRedis.duplicate();
+    ownedRedis.push(reaperRedis);
+    tasks.push(
+      reaperLoop({
+        redis: reaperRedis,
+        prefix,
+        queues,
+        intervalMs: reaperIntervalMs,
+        signal: abort.signal,
+      }),
+    );
+
+    const cronRedis = await baseRedis.duplicate();
+    const cronClient = createClient({ redis: cronRedis, prefix });
+    ownedRedis.push(cronRedis);
+    tasks.push(cronSchedulerLoop({ redis: cronRedis, client: cronClient, prefix, signal: abort.signal }));
+
+    return {
+      async stop() {
+        await closeOwnedResources();
+      },
+    };
+  } catch (err) {
+    await closeOwnedResources();
+    throw err;
+  }
+}
+
+function deriveQueuesFromRegistry(registry: WorkflowRegistry): string[] {
+  const set = new Set<string>();
+  for (const def of registry.list()) set.add(def.options.queue ?? "default");
+  return [...set].sort();
+}
+
+function parseRedisScore(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string" && value.trim() !== "") {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+function parseZPopMinEntry(value: unknown): { member: string; score: number } | null {
+  if (!Array.isArray(value) || value.length === 0) return null;
+
+  if (value.length === 1 && Array.isArray(value[0])) {
+    return parseZPopMinEntry(value[0]);
+  }
+
+  const member = value[0];
+  const score = parseRedisScore(value[1]);
+  if (typeof member !== "string" || score == null) return null;
+  return { member, score };
+}
+
+/** Parse ZPOPMIN response with count > 1.
+ *  Bun returns [[member, score], [member, score], ...] for batch zpopmin. */
+function parseZPopMinBatch(value: unknown): Array<{ member: string; score: number }> {
+  if (!Array.isArray(value) || value.length === 0) return [];
+
+  const results: Array<{ member: string; score: number }> = [];
+
+  for (const item of value) {
+    // Each item is [member, score]
+    const entry = parseZPopMinEntry(item);
+    if (entry) {
+      results.push(entry);
+      continue;
+    }
+
+    // Fallback: flat format [member1, score1, member2, score2, ...]
+    // Try parsing value itself as a single entry
+    if (typeof item === "string" && results.length === 0) {
+      const flat = parseZPopMinEntry(value);
+      if (flat) return [flat];
+      break;
+    }
+  }
+  return results;
+}
+
+function encodeIdempotencyPart(value: string): string {
+  return `${value.length}:${value}`;
+}
+
+function defaultStepWorkflowIdempotencyKey(parentRunId: string, stepName: string, childWorkflowName: string): string {
+  return `stepwf:${encodeIdempotencyPart(parentRunId)}:${encodeIdempotencyPart(stepName)}:${encodeIdempotencyPart(childWorkflowName)}`;
+}
+
+function defaultStepEventIdempotencyKey(parentRunId: string, stepName: string, eventName: string): string {
+  return `stepev:${encodeIdempotencyPart(parentRunId)}:${encodeIdempotencyPart(stepName)}:${encodeIdempotencyPart(eventName)}`;
+}
+
+async function claimQueuedRunForInlineExecution(args: {
+  redis: RedisClient;
+  prefix: string;
+  queue: string;
+  runId: string;
+}): Promise<boolean> {
+  const { redis, prefix, queue, runId } = args;
+  const result = await redis.send("EVAL", [
+    CLAIM_QUEUED_RUN_FOR_INLINE_LUA,
+    "3",
+    keys.queueReady(prefix, queue),
+    keys.queueProcessing(prefix, queue),
+    keys.run(prefix, runId),
+    runId,
+  ]);
+
+  return result === 1 || result === "1";
+}
+
+async function workerLoop(args: {
+  redis: RedisClient;
+  client: RedflowClient;
+  registry: WorkflowRegistry;
+  prefix: string;
+  queues: string[];
+  blmoveTimeoutSec: number;
+  leaseMs: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  const { redis, queues, blmoveTimeoutSec, signal } = args;
+  let queueCursor = 0;
+
+  while (!signal.aborted) {
+    try {
+      if (queues.length === 0) {
+        await sleep(25);
+        continue;
+      }
+
+      let claim: { queue: string; processingKey: string; runId: string } | null = null;
+
+      // First probe all queues without blocking so busy queues don't wait behind empty ones.
+      for (let i = 0; i < queues.length; i++) {
+        if (signal.aborted) break;
+        const idx = (queueCursor + i) % queues.length;
+        const queue = queues[idx]!;
+        const readyKey = keys.queueReady(args.prefix, queue);
+        const processingKey = keys.queueProcessing(args.prefix, queue);
+        const runId = await redis.lmove(readyKey, processingKey, "RIGHT", "LEFT");
+        if (!runId) continue;
+
+        claim = { queue, processingKey, runId };
+        queueCursor = (idx + 1) % queues.length;
+        break;
+      }
+
+      // Nothing ready right now - block on one queue, rotating the queue every time.
+      if (!claim && !signal.aborted) {
+        const idx = queueCursor % queues.length;
+        const queue = queues[idx]!;
+        queueCursor = (idx + 1) % queues.length;
+
+        const readyKey = keys.queueReady(args.prefix, queue);
+        const processingKey = keys.queueProcessing(args.prefix, queue);
+        const runId = await redis.blmove(readyKey, processingKey, "RIGHT", "LEFT", blmoveTimeoutSec);
+        if (runId) {
+          claim = { queue, processingKey, runId };
+        }
+      }
+
+      if (!claim) {
+        await sleep(10);
+        continue;
+      }
+
+      try {
+        await processRun({ ...args, queue: claim.queue, runId: claim.runId, processingKey: claim.processingKey });
+      } catch {
+        // Defensive: never let a single job crash the loop.
+      }
+    } catch {
+      if (signal.aborted) break;
+      await sleep(50);
+    }
+  }
+}
+
+async function processRun(args: {
+  redis: RedisClient;
+  client: RedflowClient;
+  registry: WorkflowRegistry;
+  prefix: string;
+  queues: string[];
+  queue: string;
+  runId: string;
+  processingKey: string;
+  leaseMs: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  const { redis, client, registry, prefix, queues, queue, runId, processingKey, leaseMs, signal } = args;
+  const runKey = keys.run(prefix, runId);
+  const stepsKey = keys.runSteps(prefix, runId);
+  const leaseKey = keys.runLease(prefix, runId);
+  const leaseToken = `lease_${crypto.randomUUID()}`;
+
+  const leaseOk = await redis.set(leaseKey, leaseToken, "NX", "PX", String(leaseMs));
+  if (leaseOk === null) {
+    // Someone else owns this run. Remove the duplicate processing entry.
+    await redis.lrem(processingKey, 1, runId);
+    return;
+  }
+
+  const runAbort = new AbortController();
+  let leaseLost = false;
+  const markLeaseLost = () => {
+    if (leaseLost) return;
+    leaseLost = true;
+    runAbort.abort();
+  };
+
+  let leaseInterval: Timer | null = null;
+  let renewFailures = 0;
+  leaseInterval = setInterval(() => {
+    // Best-effort renewal; if we lose ownership, abort this attempt so another
+    // worker can safely recover the run.
+    redis
+      .send("EVAL", [RENEW_LEASE_LUA, "1", leaseKey, leaseToken, String(leaseMs)])
+      .then((renewed) => {
+        if (renewed == null) {
+          markLeaseLost();
+          return;
+        }
+
+        renewFailures = 0;
+      })
+      .catch(() => {
+        renewFailures += 1;
+        if (renewFailures >= 2) {
+          markLeaseLost();
+        }
+      });
+  }, Math.floor(leaseMs / 2));
+
+  try {
+    const run = await redis.hgetall(runKey);
+    if (!run || Object.keys(run).length === 0) {
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    const currentStatus = (run.status as RunStatus) ?? "queued";
+    if (currentStatus === "succeeded" || currentStatus === "failed" || currentStatus === "canceled") {
+      // Defensive: if a terminal run is accidentally re-queued, do not re-execute.
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    if (currentStatus === "scheduled") {
+      // Defensive: scheduled runs are promoted by the promoter loop only.
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    if (currentStatus !== "queued" && currentStatus !== "running") {
+      // Unknown status, do not execute the run.
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    const workflowName = run.workflow ?? "";
+    const maxAttempts = Number(run.maxAttempts ?? "1");
+    const cancelRequestedAt = run.cancelRequestedAt ? Number(run.cancelRequestedAt) : 0;
+    if (cancelRequestedAt > 0) {
+      await client.finalizeRun(runId, { status: "canceled", finishedAt: nowMs() });
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    const startedAt = run.startedAt && run.startedAt !== "" ? Number(run.startedAt) : nowMs();
+
+    if (currentStatus === "queued") {
+      const movedToRunning = await client.transitionRunStatusIfCurrent(runId, "queued", "running", startedAt);
+      if (!movedToRunning) {
+        // Most likely canceled between dequeue and start transition.
+        await redis.lrem(processingKey, 0, runId);
+        return;
+      }
+    }
+
+    if (!run.startedAt || run.startedAt === "") {
+      await redis.hset(runKey, { startedAt: String(startedAt) });
+    }
+
+    // Only increment attempt on fresh pick-up (queued → running).
+    // On crash recovery the status is already "running" and attempt was
+    // already incremented before the previous worker crashed.
+    const attempt = currentStatus === "queued"
+      ? await redis.hincrby(runKey, "attempt", 1)
+      : Math.max(1, Number(run.attempt ?? "0"));
+
+    if (!workflowName) {
+      const errorJson = makeErrorJson(new UnknownWorkflowError(""));
+      await client.finalizeRun(runId, { status: "failed", errorJson, finishedAt: nowMs() });
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    const def = registry.get(workflowName);
+    if (!def) {
+      const errorJson = makeErrorJson(new UnknownWorkflowError(workflowName));
+      await client.finalizeRun(runId, { status: "failed", errorJson, finishedAt: nowMs() });
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    let input: unknown;
+    try {
+      const inputRaw = safeJsonParse(run.inputJson ?? "null");
+      input = def.options.schema ? validateInputWithSchema(def.options.schema, inputRaw) : inputRaw;
+    } catch (err) {
+      // Input is not valid for this workflow. This is deterministic, so don't retry.
+      await invokeOnFailure(def.options.onFailure, {
+        error: err, input: undefined, run: { id: runId, workflow: workflowName, queue, attempt, maxAttempts },
+      });
+      await client.finalizeRun(runId, { status: "failed", errorJson: makeErrorJson(err), finishedAt: nowMs() });
+      await redis.lrem(processingKey, 0, runId);
+      return;
+    }
+
+    const onWorkerAbort = () => {
+      runAbort.abort();
+    };
+    // Use { once: true } so the listener auto-removes, and add it
+    // unconditionally — if the signal is already aborted, manually abort.
+    signal.addEventListener("abort", onWorkerAbort, { once: true });
+    if (signal.aborted) {
+      runAbort.abort();
+    }
+
+    const cancelPoll = setInterval(() => {
+      // Best-effort: if cancellation is requested mid-step, abort the run.
+      redis
+        .hget(runKey, "cancelRequestedAt")
+        .then((cancel) => {
+          if (cancel && cancel !== "") runAbort.abort();
+        })
+        .catch(() => {});
+    }, 100);
+
+    try {
+      const ensureNotCanceled = async () => {
+        if (runAbort.signal.aborted) throw new CanceledError();
+        const cancel = await redis.hget(runKey, "cancelRequestedAt");
+        if (cancel && cancel !== "") {
+          runAbort.abort();
+          throw new CanceledError();
+        }
+      };
+
+      const existingStepsRaw = await redis.hgetall(stepsKey);
+      let stepSeq = 0;
+      for (const raw of Object.values(existingStepsRaw)) {
+        try {
+          const parsed = safeJsonParse<any>(raw);
+          const seq = parsed?.seq;
+          if (typeof seq === "number" && Number.isFinite(seq) && seq > stepSeq) {
+            stepSeq = seq;
+          }
+        } catch {
+          // Ignore malformed cached step data.
+        }
+      }
+
+      // Track step names within a single execution to catch accidental
+      // duplicates. On retries cached steps are already in Redis, so we
+      // only guard against re-use within the *current* handler invocation.
+      const usedStepNames = new Set<string>();
+
+      const runStep: StepApi["run"] = async <T>(
+        options: { name: string; timeoutMs?: number },
+        fn: (ctx: { signal: AbortSignal }) => Promise<T>,
+      ): Promise<T> => {
+        await ensureNotCanceled();
+
+        if (usedStepNames.has(options.name)) {
+          throw new Error(`Duplicate step name '${options.name}' in workflow '${workflowName}'. Step names must be unique within a run.`);
+        }
+        usedStepNames.add(options.name);
+
+        const existingRaw = await redis.hget(stepsKey, options.name);
+        let seq = 0;
+        if (existingRaw) {
+          const existing = safeJsonParse<any>(existingRaw);
+          const existingSeq = typeof existing.seq === "number" && Number.isFinite(existing.seq) ? existing.seq : undefined;
+          if (existingSeq != null && existingSeq > stepSeq) {
+            stepSeq = existingSeq;
+          }
+
+          if (existing.status === "succeeded") {
+            return safeJsonTryParse(existing.outputJson ?? null) as T;
+          }
+
+          seq = existingSeq ?? (stepSeq += 1);
+        } else {
+          seq = (stepSeq += 1);
+        }
+
+        const stepStartedAt = nowMs();
+        await redis.hset(stepsKey, {
+          [options.name]: safeJsonStringify({
+            status: "running" satisfies StepStatus,
+            seq,
+            startedAt: stepStartedAt,
+          }),
+        });
+
+        const stepAbort = new AbortController();
+        let timeout: Timer | null = null;
+        let runAbortListener: (() => void) | null = null;
+
+        try {
+          const fnPromise = fn({ signal: stepAbort.signal });
+
+          const cancelPromise = new Promise<never>((_, reject) => {
+            const onAbort = () => {
+              stepAbort.abort();
+              reject(new CanceledError());
+            };
+
+            if (runAbort.signal.aborted) return onAbort();
+            runAbortListener = onAbort;
+            runAbort.signal.addEventListener("abort", onAbort);
+          });
+
+          const timeoutPromise =
+            options.timeoutMs && options.timeoutMs > 0
+              ? new Promise<never>((_, reject) => {
+                  timeout = setTimeout(() => {
+                    stepAbort.abort();
+                    reject(new TimeoutError(`Step timed out: ${options.name}`));
+                  }, options.timeoutMs);
+                })
+              : null;
+
+          const value = (await Promise.race([fnPromise, cancelPromise, timeoutPromise].filter(Boolean))) as T;
+          const stepFinishedAt = nowMs();
+          let outputJson: string;
+          try {
+            outputJson = safeJsonStringify(value);
+          } catch (err) {
+            throw new OutputSerializationError(`Step output is not JSON-serializable: ${options.name}`, err);
+          }
+
+          await redis.hset(stepsKey, {
+            [options.name]: safeJsonStringify({
+              status: "succeeded" satisfies StepStatus,
+              seq,
+              startedAt: stepStartedAt,
+              finishedAt: stepFinishedAt,
+              outputJson,
+            }),
+          });
+          return value;
+        } catch (err) {
+          const stepFinishedAt = nowMs();
+          const error = err;
+          await redis.hset(stepsKey, {
+            [options.name]: safeJsonStringify({
+              status: "failed" satisfies StepStatus,
+              seq,
+              startedAt: stepStartedAt,
+              finishedAt: stepFinishedAt,
+              errorJson: makeErrorJson(error),
+            }),
+          });
+          throw error;
+        } finally {
+          if (runAbortListener) runAbort.signal.removeEventListener("abort", runAbortListener);
+          if (timeout) clearTimeout(timeout);
+        }
+      };
+
+      const runWorkflowStep: StepApi["runWorkflow"] = async (options, workflow, workflowInput) => {
+        const idempotencyKey =
+          options.idempotencyKey ?? defaultStepWorkflowIdempotencyKey(runId, options.name, workflow.name);
+
+        const waitForRunResultWithInlineAssist = async <TOutput>(
+          childRunId: string,
+          resultOptions: { timeoutMs?: number; pollMs?: number } | undefined,
+          stepSignal: AbortSignal,
+        ): Promise<TOutput> => {
+          const timeoutMs = resultOptions?.timeoutMs ?? 30_000;
+          const pollMs = resultOptions?.pollMs ?? 250;
+          const deadline = nowMs() + timeoutMs;
+          const missingGraceMs = Math.max(250, Math.min(2000, pollMs * 4));
+
+          let missingSince: number | null = null;
+          let seenState = false;
+
+          while (true) {
+            if (stepSignal.aborted) throw new CanceledError();
+
+            const state = await client.getRun(childRunId);
+            if (!state) {
+              const t = nowMs();
+              if (missingSince == null) missingSince = t;
+
+              if (t - missingSince >= missingGraceMs) {
+                if (seenState) {
+                  throw new Error(`Run state unavailable: ${childRunId}`);
+                }
+                throw new Error(`Run not found: ${childRunId}`);
+              }
+
+              if (t > deadline) throw new TimeoutError(`Timed out waiting for run result (${childRunId})`);
+              await sleep(pollMs);
+              continue;
+            }
+
+            seenState = true;
+            missingSince = null;
+
+            if (state.status === "succeeded") return state.output as TOutput;
+            if (state.status === "failed") throw new Error(`Run failed: ${safeJsonStringify(state.error)}`);
+            if (state.status === "canceled") {
+              throw new CanceledError(state.cancelReason ? `Run canceled: ${state.cancelReason}` : "Run canceled");
+            }
+
+            if (state.status === "queued" && queues.includes(state.queue)) {
+              const claimed = await claimQueuedRunForInlineExecution({
+                redis,
+                prefix,
+                queue: state.queue,
+                runId: childRunId,
+              });
+
+              if (claimed) {
+                await processRun({
+                  redis,
+                  client,
+                  registry,
+                  prefix,
+                  queues,
+                  queue: state.queue,
+                  runId: childRunId,
+                  processingKey: keys.queueProcessing(prefix, state.queue),
+                  leaseMs,
+                  signal,
+                });
+                continue;
+              }
+            }
+
+            if (nowMs() > deadline) throw new TimeoutError(`Timed out waiting for run result (${childRunId})`);
+            await sleep(pollMs);
+          }
+        };
+
+        return await runStep(
+          { name: options.name, timeoutMs: options.timeoutMs },
+          async ({ signal: stepSignal }) => {
+            const handle = await workflow.run(workflowInput, {
+              ...(options.run ?? {}),
+              idempotencyKey,
+            });
+
+            return await waitForRunResultWithInlineAssist(handle.id, options.result, stepSignal);
+          },
+        );
+      };
+
+      const emitEventStep: StepApi["emitEvent"] = async (options, payload) => {
+        const idempotencyKey =
+          options.idempotencyKey ?? defaultStepEventIdempotencyKey(runId, options.name, options.event);
+
+        return await runStep(
+          { name: options.name, timeoutMs: options.timeoutMs },
+          async () => {
+            return await client.emitEvent(options.event, payload, {
+              ...(options.emit ?? {}),
+              idempotencyKey,
+            });
+          },
+        );
+      };
+
+      const scheduleEventStep: StepApi["scheduleEvent"] = async (options, payload) => {
+        const idempotencyKey =
+          options.idempotencyKey ?? defaultStepEventIdempotencyKey(runId, options.name, options.event);
+
+        return await runStep(
+          { name: options.name, timeoutMs: options.timeoutMs },
+          async () => {
+            return await client.scheduleEvent(options.event, payload, {
+              ...(options.schedule ?? {}),
+              idempotencyKey,
+            });
+          },
+        );
+      };
+
+      const step: StepApi = {
+        run: runStep,
+        runWorkflow: runWorkflowStep,
+        emitEvent: emitEventStep,
+        scheduleEvent: scheduleEventStep,
+      };
+
+      let outputJson: string;
+      try {
+        await ensureNotCanceled();
+        let runAbortListener: (() => void) | null = null;
+        const cancelPromise = new Promise<never>((_, reject) => {
+          const onAbort = () => reject(new CanceledError());
+          if (runAbort.signal.aborted) return onAbort();
+          runAbortListener = onAbort;
+          runAbort.signal.addEventListener("abort", onAbort);
+        });
+
+        let output: unknown;
+        try {
+          output = await Promise.race([
+            def.handler({
+              input,
+              run: {
+                id: runId,
+                workflow: workflowName,
+                queue,
+                attempt,
+                maxAttempts,
+              },
+              step,
+              signal: runAbort.signal,
+            }),
+            cancelPromise,
+          ]);
+        } finally {
+          if (runAbortListener) {
+            runAbort.signal.removeEventListener("abort", runAbortListener);
+          }
+        }
+
+        await ensureNotCanceled();
+
+        try {
+          outputJson = safeJsonStringify(output);
+        } catch (err) {
+          throw new OutputSerializationError("Workflow output is not JSON-serializable", err);
+        }
+      } catch (err) {
+        if (err instanceof CanceledError) {
+          if (leaseLost) {
+            // Lease ownership moved away from this worker. Do not finalize or
+            // mutate queue state here - recovery worker/reaper will continue.
+            return;
+          }
+          await client.finalizeRun(runId, { status: "canceled", finishedAt: nowMs() });
+          await redis.lrem(processingKey, 0, runId);
+          return;
+        }
+
+        const retryable = isRetryableError(err);
+        const hasAttemptsLeft = attempt < maxAttempts;
+        const finishedAt = nowMs();
+
+        if (retryable && hasAttemptsLeft) {
+          const delay = computeRetryDelayMs(attempt);
+          const nextAt = finishedAt + delay;
+          await client.scheduleRetry(runId, {
+            queue,
+            nextAt,
+            errorJson: makeErrorJson(err),
+            updatedAt: finishedAt,
+          });
+          await redis.lrem(processingKey, 0, runId);
+          return;
+        }
+
+        await invokeOnFailure(def.options.onFailure, {
+          error: err, input, run: { id: runId, workflow: workflowName, queue, attempt, maxAttempts },
+        });
+        await client.finalizeRun(runId, { status: "failed", errorJson: makeErrorJson(err), finishedAt });
+        await redis.lrem(processingKey, 0, runId);
+        return;
+      }
+
+      await client.finalizeRun(runId, { status: "succeeded", outputJson, finishedAt: nowMs() });
+      await redis.lrem(processingKey, 0, runId);
+    } finally {
+      clearInterval(cancelPoll);
+      signal.removeEventListener("abort", onWorkerAbort);
+    }
+  } finally {
+    if (leaseInterval) clearInterval(leaseInterval);
+    await redis.send("EVAL", [UNLOCK_LUA, "1", leaseKey, leaseToken]).catch(() => {});
+  }
+}
+
+async function scheduledPromoterLoop(args: {
+  redis: RedisClient;
+  client: RedflowClient;
+  prefix: string;
+  queues: string[];
+  signal: AbortSignal;
+}): Promise<void> {
+  const { redis, prefix, queues, signal } = args;
+
+  while (!signal.aborted) {
+    try {
+      const now = nowMs();
+      let nextWake: number | null = null;
+
+      for (const queue of queues) {
+        if (signal.aborted) break;
+        const schedKey = keys.queueScheduled(prefix, queue);
+        const readyKey = keys.queueReady(prefix, queue);
+
+        // Pop up to 100 entries at a time to avoid starvation under burst load.
+        const batch = parseZPopMinBatch(await redis.send("ZPOPMIN", [schedKey, "100"]));
+        if (batch.length === 0) continue;
+
+        for (const { member: runId, score } of batch) {
+          if (signal.aborted) break;
+
+          if (score <= now) {
+            const queued = await args.client.transitionRunStatusIfCurrent(runId, "scheduled", "queued", now);
+            if (queued) {
+              await redis.lpush(readyKey, runId);
+            }
+          } else {
+            // Not yet due — put it back and compute next wake time.
+            await redis.zadd(schedKey, score, runId);
+            nextWake = nextWake == null ? score : Math.min(nextWake, score);
+          }
+        }
+      }
+
+      if (nextWake != null) {
+        await sleep(Math.min(1000, Math.max(10, nextWake - nowMs())));
+      } else {
+        await sleep(50);
+      }
+    } catch {
+      if (signal.aborted) break;
+      await sleep(100);
+    }
+  }
+}
+
+async function reaperLoop(args: {
+  redis: RedisClient;
+  prefix: string;
+  queues: string[];
+  intervalMs: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  const { redis, prefix, queues, intervalMs, signal } = args;
+
+  while (!signal.aborted) {
+    try {
+      for (const queue of queues) {
+        if (signal.aborted) break;
+        const processingKey = keys.queueProcessing(prefix, queue);
+        const readyKey = keys.queueReady(prefix, queue);
+        const runIds = await redis.lrange(processingKey, 0, -1);
+
+        for (const runId of runIds) {
+          const leaseKey = keys.runLease(prefix, runId);
+          const lease = await redis.get(leaseKey);
+          if (lease) continue;
+          // Not leased => assume worker crashed.
+          const removed = await redis.lrem(processingKey, 1, runId);
+          if (removed > 0) await redis.lpush(readyKey, runId);
+        }
+      }
+
+      await sleep(intervalMs);
+    } catch {
+      if (signal.aborted) break;
+      await sleep(Math.max(100, intervalMs));
+    }
+  }
+}
+
+async function cronSchedulerLoop(args: {
+  redis: RedisClient;
+  client: RedflowClient;
+  prefix: string;
+  signal: AbortSignal;
+}): Promise<void> {
+  const { redis, client, prefix, signal } = args;
+  const lockKey = keys.lockCron(prefix);
+  const token = `cronlock_${crypto.randomUUID()}`;
+  const lockMs = 2000;
+
+  while (!signal.aborted) {
+    try {
+      const acquired = await redis.set(lockKey, token, "NX", "PX", String(lockMs));
+      if (acquired === null) {
+        await sleep(250);
+        continue;
+      }
+
+      // We are the active scheduler until we stop renewing.
+      while (!signal.aborted) {
+        try {
+          // Renew lock (best-effort, safe).
+          const renewed = await redis.send("EVAL", [
+            `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("set", KEYS[1], ARGV[1], "PX", ARGV[2]) else return nil end`,
+            "1",
+            lockKey,
+            token,
+            String(lockMs),
+          ]);
+          if (renewed == null) break;
+
+          const now = nowMs();
+          const next = parseZPopMinEntry(await redis.zpopmin(keys.cronNext(prefix)));
+          if (!next) {
+            await sleep(250);
+            continue;
+          }
+          const { member: cronId, score } = next;
+          if (score > now) {
+            await redis.zadd(keys.cronNext(prefix), score, cronId);
+            await sleep(Math.min(1000, Math.max(10, score - now)));
+            continue;
+          }
+
+          let requeueCurrent = true;
+          try {
+            const defRaw = await redis.hget(keys.cronDef(prefix), cronId);
+            if (!defRaw) {
+              requeueCurrent = false;
+              continue;
+            }
+
+            let def: any;
+            try {
+              def = safeJsonParse<any>(defRaw);
+            } catch {
+              // Corrupted definition, drop it to avoid a tight error loop.
+              requeueCurrent = false;
+              continue;
+            }
+
+            const workflow = def.workflow as string;
+            const queue = def.queue as string;
+
+            let input: unknown;
+            try {
+              input = safeJsonParse(def.inputJson);
+            } catch {
+              // Corrupted payload encoding, drop it.
+              requeueCurrent = false;
+              continue;
+            }
+
+            await client.runByName(workflow, input, { queueOverride: queue });
+
+            // Schedule next run.
+            let nextAt: number | null = null;
+            try {
+              const job = new Cron(def.expression, { timezone: def.timezone, catch: true });
+              // Reschedule from current time so stale cron scores do not create
+              // catch-up bursts after clock drift/temporary pauses.
+              const next = job.nextRun(new Date(nowMs()));
+              job.stop();
+              nextAt = next ? next.getTime() : null;
+            } catch {
+              // Invalid expression in metadata. Drop this trigger.
+              requeueCurrent = false;
+              continue;
+            }
+
+            if (nextAt != null) {
+              await redis.zadd(keys.cronNext(prefix), nextAt, cronId);
+            }
+            requeueCurrent = false;
+          } catch {
+            // Keep the current cron id alive on transient failures.
+          } finally {
+            if (requeueCurrent) {
+              const retryAt = Math.max(score, nowMs() + 250);
+              await redis.zadd(keys.cronNext(prefix), retryAt, cronId).catch(() => {});
+            }
+          }
+        } catch {
+          if (signal.aborted) break;
+          await sleep(250);
+          break;
+        }
+      }
+    } catch {
+      if (signal.aborted) break;
+      await sleep(250);
+    }
+  }
+}
+
+/** Best-effort onFailure invocation — errors are swallowed so they never break finalization. */
+async function invokeOnFailure(
+  handler: ((ctx: OnFailureContext) => void | Promise<void>) | undefined,
+  ctx: OnFailureContext,
+): Promise<void> {
+  if (!handler) return;
+  try {
+    await handler(ctx);
+  } catch {
+    // onFailure must not interfere with the run lifecycle.
+  }
+}