@voyantjs/workflows 0.0.0

package/src/runtime/determinism.ts

@@ -0,0 +1,75 @@
+// Deterministic clock and RNG used by the workflow body.
+//
+// `ctx.now()` must return the timestamp of the event currently being
+// consumed from the journal during replay (§4.5 of docs/design.md).
+// `ctx.random()` / `ctx.randomUUID()` are seeded from the run id so
+// replays produce the same values.
+
+export interface ClockState {
+  /** Base wall-clock time recorded at run start. */
+  readonly baseWallClock: number;
+  /** Offset from baseWallClock at which ctx.now() should return — set by the executor when replaying journaled events. */
+  offset: number;
+}
+
+export function createClock(runStartedAt: number): ClockState {
+  return { baseWallClock: runStartedAt, offset: 0 };
+}
+
+export function now(clock: ClockState): number {
+  return clock.baseWallClock + clock.offset;
+}
+
+/** Advance the clock to the event currently being replayed. */
+export function advanceClockTo(clock: ClockState, eventAt: number): void {
+  clock.offset = eventAt - clock.baseWallClock;
+}
+
+/**
+ * Mulberry32 PRNG — fast, fine for workflow-determinism use. Seeded
+ * from a 32-bit hash of the run id. Not cryptographic.
+ */
+export function seededRandom(seed: number): () => number {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let t = state;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+export function hashSeed(runId: string): number {
+  // FNV-1a 32-bit
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < runId.length; i++) {
+    hash ^= runId.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return hash >>> 0;
+}
+
+export function createRandom(runId: string): () => number {
+  return seededRandom(hashSeed(runId));
+}
+
+const HEX = "0123456789abcdef";
+
+export function createRandomUUID(rng: () => number): () => string {
+  // v4-shaped deterministic UUID. Not cryptographically random; matches
+  // the style of crypto.randomUUID but is reproducible across replays.
+  return () => {
+    const bytes = new Uint8Array(16);
+    for (let i = 0; i < 16; i++) bytes[i] = Math.floor(rng() * 256);
+    bytes[6] = (bytes[6]! & 0x0f) | 0x40; // version 4 (high nibble of byte 6)
+    bytes[8] = (bytes[8]! & 0x3f) | 0x80; // variant RFC 4122 (high bits of byte 8)
+    let s = "";
+    for (let i = 0; i < 16; i++) {
+      const b = bytes[i]!;
+      s += HEX[b >>> 4]! + HEX[b & 0x0f]!;
+    }
+    // s is 32 hex chars; slice into 8-4-4-4-12 groups.
+    return `${s.slice(0, 8)}-${s.slice(8, 12)}-${s.slice(12, 16)}-${s.slice(16, 20)}-${s.slice(20, 32)}`;
+  };
+}

package/src/runtime/errors.ts

@@ -0,0 +1,58 @@
+// Internal runtime signal errors. Distinct from user-facing `@voyantjs/workflows-errors`:
+// these are thrown inside the executor to unwind the workflow body on
+// waitpoint yield or run cancellation. They must not be caught by user
+// code (the executor re-throws if it observes one being swallowed).
+
+const WAITPOINT_PENDING = Symbol.for("voyant.workflows.waitpointPending");
+const RUN_CANCELLED = Symbol.for("voyant.workflows.runCancelled");
+const COMPENSATE_REQUESTED = Symbol.for("voyant.workflows.compensateRequested");
+
+export class WaitpointPendingSignal extends Error {
+  readonly [WAITPOINT_PENDING] = true as const;
+  readonly waitpointId: string;
+  constructor(waitpointId: string) {
+    super(`waitpoint pending: ${waitpointId}`);
+    this.name = "WaitpointPendingSignal";
+    this.waitpointId = waitpointId;
+  }
+}
+
+export class RunCancelledSignal extends Error {
+  readonly [RUN_CANCELLED] = true as const;
+  constructor(reason?: string) {
+    super(reason ?? "run cancelled");
+    this.name = "RunCancelledSignal";
+  }
+}
+
+export function isWaitpointPending(err: unknown): err is WaitpointPendingSignal {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { [WAITPOINT_PENDING]?: true })[WAITPOINT_PENDING] === true
+  );
+}
+
+export function isRunCancelled(err: unknown): err is RunCancelledSignal {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { [RUN_CANCELLED]?: true })[RUN_CANCELLED] === true
+  );
+}
+
+export class CompensateRequestedSignal extends Error {
+  readonly [COMPENSATE_REQUESTED] = true as const;
+  constructor() {
+    super("compensate requested");
+    this.name = "CompensateRequestedSignal";
+  }
+}
+
+export function isCompensateRequested(err: unknown): err is CompensateRequestedSignal {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { [COMPENSATE_REQUESTED]?: true })[COMPENSATE_REQUESTED] === true
+  );
+}

package/src/runtime/executor.ts

@@ -0,0 +1,427 @@
+// The workflow-step executor. Takes a registered workflow + journal and
+// runs the body exactly once, yielding a response envelope.
+//
+// Corresponds to the tenant-side handler of POST /__voyant/workflow-step
+// described in docs/runtime-protocol.md §2.1.
+
+import type { RunTrigger, RunStatus, WaitpointKind } from "../types.js";
+import type { SerializedError } from "../protocol/index.js";
+import type { WorkflowDefinition, StepOptions } from "../workflow.js";
+import type { JournalSlice, StepJournalEntry } from "./journal.js";
+import { createClock, createRandom } from "./determinism.js";
+import { buildCtx, type RuntimeCallbacks, type RuntimeEnvironment } from "./ctx.js";
+import {
+  WaitpointPendingSignal,
+  RunCancelledSignal,
+  isWaitpointPending,
+  isRunCancelled,
+  isCompensateRequested,
+} from "./errors.js";
+import { durationToMs, type RateLimiter } from "../rate-limit/index.js";
+
+export interface StepRunner {
+  /**
+   * Executes a step body and returns the journal entry to record.
+   *
+   * In-process runners (the default edge runner, local-dev passthrough)
+   * call `fn(stepCtx)` directly. Dispatching runners (e.g. the CF
+   * Container runner) ignore `fn` — they can't serialize a closure —
+   * and use the run/workflow identity + step options to address the
+   * remote container and POST the required context.
+   */
+  (args: {
+    stepId: string;
+    attempt: number;
+    input: unknown;
+    fn: (stepCtx: import("../workflow.js").StepContext) => Promise<unknown>;
+    stepCtx: import("../workflow.js").StepContext;
+    /** Identity of the run — used by dispatching runners. */
+    runId: string;
+    workflowId: string;
+    workflowVersion: string;
+    /** Project / organization id from the runtime environment — used by
+     *  dispatching runners to resolve per-tenant bundle storage keys. */
+    projectId: string;
+    organizationId: string;
+    /** Merged step options (runtime, machine, timeout, …). */
+    options: import("../workflow.js").StepOptions<unknown>;
+    /**
+     * Current journal slice at dispatch time — steps already completed,
+     * waitpoints already resolved, etc. Dispatching runners pass this
+     * to the remote executor so body replay there short-circuits on
+     * cached steps, and the container can stop cleanly after the
+     * target step runs.
+     */
+    journal: JournalSlice;
+  }): Promise<StepJournalEntry>;
+}
+
+export interface WaitpointRegistration {
+  clientWaitpointId: string;
+  kind: WaitpointKind;
+  meta: Record<string, unknown>;
+  timeoutMs?: number;
+}
+
+export interface MetadataMutation {
+  op: "set" | "increment" | "append" | "remove";
+  key: string;
+  value?: unknown;
+  target?: "self" | "parent" | "root";
+}
+
+export interface CompensationReport {
+  stepId: string;
+  status: "ok" | "err";
+  error?: SerializedError;
+  durationMs: number;
+}
+
+export interface StreamChunk {
+  streamId: string;
+  seq: number;
+  encoding: "text" | "json" | "base64";
+  chunk: unknown;
+  final: boolean;
+  at: number;
+}
+
+export interface ExecuteWorkflowStepRequest {
+  runId: string;
+  workflowId: string;
+  workflowVersion: string;
+  input: unknown;
+  journal: JournalSlice;
+  invocationCount: number;
+  environment: RuntimeEnvironment;
+  triggeredBy: RunTrigger;
+  runStartedAt: number;
+  tags: string[];
+  abortSignal?: AbortSignal;
+  /**
+   * Default step executor (the "edge" runtime) — runs step bodies
+   * in-process. Used for any step whose `options.runtime` is unset or
+   * explicitly `"edge"`.
+   */
+  stepRunner: StepRunner;
+  /**
+   * Optional runner for steps declared with `options.runtime === "node"`.
+   * Typical impl dispatches to a Cloudflare Container sized for the
+   * step (or, in local dev, an in-process passthrough).
+   *
+   * If a step requests `"node"` and this is unset, the step fails with
+   * `NODE_RUNTIME_UNAVAILABLE` — declaring a runtime and then silently
+   * falling back to edge would hide deployment bugs.
+   */
+  nodeStepRunner?: StepRunner;
+  /**
+   * Optional rate limiter. When a step declares `options.rateLimit`,
+   * the executor calls `rateLimiter.acquire(...)` before invoking the
+   * step runner. Without a limiter, a step that declares `rateLimit`
+   * fails with `RATE_LIMITER_MISSING` — declaring a limit and not
+   * enforcing it would be silently dangerous.
+   */
+  rateLimiter?: RateLimiter;
+  /** `() => number` used for compensation durations. Defaults to Date.now. */
+  now?: () => number;
+  /**
+   * Optional per-chunk callback fired synchronously from
+   * `ctx.stream.*` as each chunk is produced. Enables live streaming
+   * (dashboards, queues) in-process before the invocation completes.
+   * Chunks are still accumulated in the response's `streamChunks`
+   * array so the at-end delivery keeps working.
+   */
+  onStreamChunk?: (chunk: StreamChunk) => void;
+}
+
+export type ExecuteWorkflowStepResponse =
+  | {
+      status: "completed";
+      output: unknown;
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      streamChunks: StreamChunk[];
+    }
+  | {
+      status: "failed";
+      error: SerializedError;
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      streamChunks: StreamChunk[];
+    }
+  | {
+      status: "cancelled";
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      compensations: CompensationReport[];
+      streamChunks: StreamChunk[];
+    }
+  | {
+      status: "waiting";
+      waitpoints: WaitpointRegistration[];
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      streamChunks: StreamChunk[];
+    }
+  | {
+      status: "compensated";
+      /** Only set when compensation was triggered by an uncaught body error. */
+      error?: SerializedError;
+      compensations: CompensationReport[];
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      streamChunks: StreamChunk[];
+    }
+  | {
+      status: "compensation_failed";
+      error?: SerializedError;
+      compensations: CompensationReport[];
+      metadataUpdates: MetadataMutation[];
+      journal: JournalSlice;
+      streamChunks: StreamChunk[];
+    };
+
+interface Compensable {
+  stepId: string;
+  output: unknown;
+  compensate: (output: unknown) => Promise<void>;
+}
+
+export async function executeWorkflowStep(
+  def: WorkflowDefinition,
+  req: ExecuteWorkflowStepRequest,
+): Promise<ExecuteWorkflowStepResponse> {
+  const abortSignal = req.abortSignal ?? new AbortController().signal;
+  const now = req.now ?? (() => Date.now());
+  const clock = createClock(req.runStartedAt);
+  const random = createRandom(req.runId);
+  const waitpoints: WaitpointRegistration[] = [];
+  const metadataUpdates: MetadataMutation[] = [];
+  const compensable: Compensable[] = [];
+  const streamChunks: StreamChunk[] = [];
+  const retryOverride: { current: import("../types.js").RetryPolicy | undefined } = { current: def.config.retry };
+
+  const callbacks: RuntimeCallbacks = {
+    invocationCount: req.invocationCount,
+    abortSignal,
+    async runStep(args) {
+      if (args.options.rateLimit) {
+        await acquireRateLimit({
+          spec: args.options.rateLimit,
+          stepId: args.stepId,
+          input: req.input,
+          runId: req.runId,
+          projectId: req.environment.project.id,
+          limiter: req.rateLimiter,
+          signal: abortSignal,
+        });
+      }
+      const runtime = args.options.runtime ?? "edge";
+      const runner = runtime === "node" ? req.nodeStepRunner : req.stepRunner;
+      if (!runner) {
+        const e = new Error(
+          `step "${args.stepId}" declared runtime="node" but the handler has no nodeStepRunner wired; ` +
+            `pass { nodeStepRunner } to createStepHandler() or remove options.runtime`,
+        );
+        (e as Error & { code?: string }).code = "NODE_RUNTIME_UNAVAILABLE";
+        throw e;
+      }
+      const entry = await runner({
+        stepId: args.stepId,
+        attempt: args.attempt,
+        input: args.input,
+        fn: args.fn,
+        stepCtx: args.stepCtx,
+        runId: req.runId,
+        workflowId: req.workflowId,
+        workflowVersion: req.workflowVersion,
+        projectId: req.environment.project.id,
+        organizationId: req.environment.organization.id,
+        options: args.options,
+        journal: req.journal,
+      });
+      // Stamp the runtime on the journal entry so downstream consumers
+      // (journal persistence, dashboard events) can report where each
+      // step actually ran.
+      entry.runtime = runtime;
+      return entry;
+    },
+    registerWaitpoint(args) {
+      waitpoints.push(args);
+    },
+    pushMetadata(op) {
+      metadataUpdates.push(op);
+    },
+    recordCompensable(args) {
+      compensable.push(args);
+    },
+    compensableLength(): number {
+      return compensable.length;
+    },
+    spliceCompensable(fromIndex: number): Compensable[] {
+      return compensable.splice(fromIndex);
+    },
+    pushStreamChunk(args) {
+      const chunk = { ...args, at: now() };
+      streamChunks.push(chunk);
+      // Fire the live hook synchronously. Errors are swallowed — a
+      // misbehaving subscriber must not break the workflow body.
+      if (req.onStreamChunk) {
+        try {
+          req.onStreamChunk(chunk);
+        } catch {
+          /* ignore */
+        }
+      }
+    },
+  };
+
+  const ctx = buildCtx({
+    env: req.environment,
+    journal: req.journal,
+    callbacks,
+    clock,
+    random,
+    retryOverride,
+  });
+
+  try {
+    const output = await def.config.run(req.input, ctx);
+    // If the body registered a waitpoint but a user try/catch swallowed
+    // the internal yield signal, honour the waitpoint — the body can't
+    // both register a waitpoint and complete in the same invocation.
+    if (waitpoints.length > 0) {
+      return { status: "waiting", waitpoints, metadataUpdates, journal: req.journal, streamChunks };
+    }
+    return { status: "completed", output, metadataUpdates, journal: req.journal, streamChunks };
+  } catch (err) {
+    if (isWaitpointPending(err)) {
+      return { status: "waiting", waitpoints, metadataUpdates, journal: req.journal, streamChunks };
+    }
+    // Same guard for the error path: a swallowed signal shouldn't let
+    // the body claim failure while waitpoints are pending.
+    if (waitpoints.length > 0 && !isRunCancelled(err) && !isCompensateRequested(err)) {
+      return { status: "waiting", waitpoints, metadataUpdates, journal: req.journal, streamChunks };
+    }
+    if (isRunCancelled(err)) {
+      // Default: compensate on cancel. Terminal status stays `cancelled`
+      // to reflect why the run ended.
+      const compensations = await runCompensations(compensable, now);
+      return { status: "cancelled", metadataUpdates, journal: req.journal, compensations, streamChunks };
+    }
+    if (isCompensateRequested(err)) {
+      const compensations = await runCompensations(compensable, now);
+      const anyErr = compensations.some((c) => c.status === "err");
+      return {
+        status: anyErr ? "compensation_failed" : "compensated",
+        compensations,
+        metadataUpdates,
+        journal: req.journal,
+        streamChunks,
+      };
+    }
+
+    // Uncaught user error. Run compensations LIFO; adjust terminal status
+    // based on whether compensations were registered and all succeeded.
+    const compensations = await runCompensations(compensable, now);
+    const serialized = serializeError(err);
+
+    if (compensations.length === 0) {
+      return {
+        status: "failed",
+        error: serialized,
+        metadataUpdates,
+        journal: req.journal,
+        streamChunks,
+      };
+    }
+
+    const anyErr = compensations.some((c) => c.status === "err");
+    return {
+      status: anyErr ? "compensation_failed" : "compensated",
+      error: serialized,
+      compensations,
+      metadataUpdates,
+      journal: req.journal,
+      streamChunks,
+    };
+  }
+}
+
+async function runCompensations(
+  compensable: readonly Compensable[],
+  now: () => number,
+): Promise<CompensationReport[]> {
+  const reports: CompensationReport[] = [];
+  // Reverse order: last-completed compensates first.
+  for (let i = compensable.length - 1; i >= 0; i--) {
+    const c = compensable[i]!;
+    const startedAt = now();
+    try {
+      await c.compensate(c.output);
+      reports.push({ stepId: c.stepId, status: "ok", durationMs: now() - startedAt });
+    } catch (err) {
+      reports.push({
+        stepId: c.stepId,
+        status: "err",
+        error: serializeError(err),
+        durationMs: now() - startedAt,
+      });
+      // Continue with remaining compensations; don't abort on one failure.
+    }
+  }
+  return reports;
+}
+
+function serializeError(err: unknown): SerializedError {
+  if (err instanceof Error) {
+    const code = (err as { code?: string }).code;
+    const cause = (err as { cause?: unknown }).cause;
+    return {
+      category: "USER_ERROR",
+      code: typeof code === "string" ? code : "UNKNOWN",
+      message: err.message,
+      name: err.name,
+      stack: err.stack,
+      cause: cause !== undefined ? serializeError(cause) : undefined,
+    };
+  }
+  return { category: "USER_ERROR", code: "UNKNOWN", message: String(err) };
+}
+
+async function acquireRateLimit(args: {
+  spec: NonNullable<StepOptions<unknown>["rateLimit"]>;
+  stepId: string;
+  input: unknown;
+  runId: string;
+  projectId: string;
+  limiter: RateLimiter | undefined;
+  signal: AbortSignal;
+}): Promise<void> {
+  const ctx = { run: { id: args.runId }, project: { id: args.projectId } };
+  const key =
+    typeof args.spec.key === "function" ? args.spec.key(args.input, ctx) : args.spec.key;
+  const limit =
+    typeof args.spec.limit === "function" ? args.spec.limit(args.input) : args.spec.limit;
+  const units =
+    args.spec.units === undefined
+      ? 1
+      : typeof args.spec.units === "function"
+        ? args.spec.units(args.input)
+        : args.spec.units;
+  const windowMs = durationToMs(args.spec.window);
+  const onLimit = args.spec.onLimit ?? "queue";
+
+  if (!args.limiter) {
+    const e = new Error(
+      `step "${args.stepId}" declared options.rateLimit but the handler has no rateLimiter wired; ` +
+        `pass { rateLimiter } to createStepHandler()`,
+    );
+    (e as Error & { code?: string }).code = "RATE_LIMITER_MISSING";
+    throw e;
+  }
+  await args.limiter.acquire({ key, limit, units, windowMs, onLimit, signal: args.signal });
+}
+
+export { WaitpointPendingSignal, RunCancelledSignal };
+export type { RunStatus };

package/src/runtime/journal.ts

@@ -0,0 +1,79 @@
+// Tenant-side view of the journal sent by the orchestrator on every
+// `/__voyant/workflow-step` invocation.
+//
+// Wire shape defined in docs/runtime-protocol.md §2.1 (JournalSlice).
+
+import type { WaitpointKind } from "../types.js";
+import type { SerializedError } from "../protocol/index.js";
+
+export interface StepJournalEntry {
+  attempt: number;
+  status: "ok" | "err";
+  output?: unknown;
+  error?: SerializedError;
+  startedAt: number;
+  finishedAt: number;
+  /**
+   * Which runtime actually executed the step. Set by the executor when
+   * routing between `stepRunner` (edge) and `nodeStepRunner`.
+   * Informational only — doesn't affect replay.
+   */
+  runtime?: "edge" | "node";
+}
+
+export interface WaitpointResolutionEntry {
+  kind: WaitpointKind;
+  resolvedAt: number;
+  matchedEventId?: string;
+  payload?: unknown;
+  source: "live" | "inbox" | "replay";
+  /** Populated for RUN waitpoints when the child run ended in a failure state. */
+  error?: SerializedError;
+}
+
+export interface CompensationJournalEntry {
+  status: "ok" | "err";
+  finishedAt: number;
+  error?: SerializedError;
+}
+
+export interface JournalSlice {
+  stepResults: Record<string, StepJournalEntry>;
+  waitpointsResolved: Record<string, WaitpointResolutionEntry>;
+  compensationsRun: Record<string, CompensationJournalEntry>;
+  metadataState: Record<string, unknown>;
+  /**
+   * Stream ids whose generator has already been consumed in a prior
+   * invocation. The orchestrator already has the chunks on the run
+   * record; replaying the body must skip re-iterating the source
+   * (generators often have side effects — LLM calls, file reads,
+   * billable API usage).
+   */
+  streamsCompleted: Record<string, { chunkCount: number }>;
+}
+
+export function emptyJournal(): JournalSlice {
+  return {
+    stepResults: {},
+    waitpointsResolved: {},
+    compensationsRun: {},
+    metadataState: {},
+    streamsCompleted: {},
+  };
+}
+
+/**
+ * Clone the journal into a slice that the body sees. Each step /
+ * waitpoint the body replays is *consumed* from the slice so we can
+ * detect leftover journal entries that the code no longer produces
+ * (which is a versioning hazard — see §6.7 of design.md).
+ */
+export function forkJournal(j: JournalSlice): JournalSlice {
+  return {
+    stepResults: { ...j.stepResults },
+    waitpointsResolved: { ...j.waitpointsResolved },
+    compensationsRun: { ...j.compensationsRun },
+    metadataState: { ...j.metadataState },
+    streamsCompleted: { ...j.streamsCompleted },
+  };
+}