bazis 0.127.0 → 0.128.0

package/core/workflow/runtime_module/background/WorkflowWorker.ts

@@ -1,53 +1,46 @@
 import { BackgroundService, delay } from "../../../background";
 import type { ServiceProvider } from "../../../di";
+import { WorkflowClock } from "../../contracts/WorkflowClock";
 import type { ConfigView } from "../../../kernel";
 import type { workflowRuntimeConfig } from "../config/Runtime.config";
-import { Schedules } from "../internal/Schedules";
-import { TimerStep } from "../internal/TimerStep";
-import { ToolStep } from "../internal/ToolStep";
-import { WorkflowEngine } from "../internal/WorkflowEngine";
-import { RuntimeDbContext } from "../model/Runtime.dbContext";
-import type { WfJob } from "../model/WfJob.model";
+import { WorkflowJobs } from "../internal/WorkflowJobs";
 
-/** How long a claimed job belongs to this worker; if it dies, another one takes the job after that. */
-const LEASE_MS = 5 * 60_000;
-/** A job that keeps failing outside its Tool (the database, a lost version) fails its step after this many tries. */
-const MAX_JOB_ATTEMPTS = 5;
 /** How often the scheduled starts are matched against the published versions. */
 const RECONCILE_MS = 60_000;
 
-
 /**
  * Takes due jobs of `bazis_wf_run_jobs` every `worker.intervalMs`, up to
- * `worker.batchSize` at a time, with `FOR UPDATE SKIP LOCKED` and a lease, so
- * application instances never take one job twice. A full batch is followed by
- * the next one at once. Every job runs in its own DI scope.
+ * `worker.batchSize` at a time ({@link WorkflowJobs}). A full batch is
+ * followed by the next one at once.
  */
 export class WorkflowWorker extends BackgroundService {
+  private readonly jobs: WorkflowJobs;
+
   constructor(
     private readonly services: ServiceProvider,
     private readonly config: ConfigView<WorkflowSettings>,
   ) {
     super();
+    this.jobs = new WorkflowJobs(services, (message, error, fields) => this.reportFailure(message, error, fields));
   }
 
   protected override async execute(signal: AbortSignal): Promise<void> {
-    if (!this.config.get("worker.enabled")) return;
+    if (!this.config.get("worker.enabled") || this.services.resolve(WorkflowClock).manual === true) return;
     const intervalMs = this.config.get("worker.intervalMs");
     const batchSize = this.config.get("worker.batchSize");
     let reconciledAt = 0;
     while (!signal.aborted) {
       if (Date.now() - reconciledAt >= RECONCILE_MS) {
         reconciledAt = Date.now();
-        await this.reconcile();
+        await this.jobs.reconcile();
       }
       let claimed = 0;
       try {
-        const jobs = await this.claim(batchSize);
+        const jobs = await this.jobs.claim(batchSize);
         claimed = jobs.length;
         for (const job of jobs) {
           if (signal.aborted) return;
-          await this.run(job, signal);
+          await this.jobs.run(job, signal);
         }
       } catch (error) {
         if (signal.aborted) return;
@@ -56,76 +49,6 @@ export class WorkflowWorker extends BackgroundService {
       if (claimed < batchSize) await delay(intervalMs, signal);
     }
   }
-
-  private async run(job: WfJob, signal: AbortSignal): Promise<void> {
-    const scope = this.services.createScope();
-    try {
-      if (job.kind === "timer") await scope.resolve(TimerStep).run(job);
-      else if (job.kind === "schedule") await scope.resolve(Schedules).fire(job);
-      else await scope.resolve(ToolStep).run(job, signal);
-    } catch (error) {
-      this.reportFailure("job failed", error, { job: job.id, instance: job.instanceId, attempts: job.attempts });
-      await this.giveUpOrDelay(job, error).catch((failure: unknown) => this.reportFailure("job failure handling failed", failure, { job: job.id }));
-    } finally {
-      await scope.dispose();
-    }
-  }
-
-  private async reconcile(): Promise<void> {
-    const scope = this.services.createScope();
-    try {
-      await scope.resolve(Schedules).reconcile();
-    } catch (error) {
-      this.reportFailure("scheduled starts could not be planned", error);
-    } finally {
-      await scope.dispose();
-    }
-  }
-
-  /** A job that failed outside its Tool waits a minute per try, then fails its step for a person to retry. */
-  private async giveUpOrDelay(job: WfJob, error: unknown): Promise<void> {
-    const message = error instanceof Error ? error.message : String(error);
-    const scope = this.services.createScope();
-    try {
-      const db = scope.resolve(RuntimeDbContext);
-      if (job.attempts < MAX_JOB_ATTEMPTS || job.instanceId === null || job.tokenId === null) {
-        await db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeUpdate({ dueAt: new Date(Date.now() + job.attempts * 60_000), lockedUntil: null, lastError: message.slice(0, 1000) });
-        return;
-      }
-      const engine = scope.resolve(WorkflowEngine);
-      await engine.inStep(job.instanceId, async (step) => {
-        await db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
-        const token = step.tokens.find((t) => t.id === job.tokenId);
-        if (token && token.status === "waiting" && step.instance.status === "running") engine.fail(step, token, "job_failed", message.slice(0, 1000));
-      });
-    } finally {
-      await scope.dispose();
-    }
-  }
-
-  private async claim(batchSize: number): Promise<WfJob[]> {
-    const scope = this.services.createScope();
-    try {
-      const db = scope.resolve(RuntimeDbContext);
-      return await db.transactionScope(async () => {
-        const now = new Date();
-        const jobs = await db.jobs
-          .where((j) => j.kind.in(["tool", "timer", "schedule"]).and(j.dueAt.lte(now)).and(j.lockedUntil.isNull().or(j.lockedUntil.lt(now))))
-          .orderBy((j) => j.id)
-          .take(batchSize)
-          .forUpdate({ skipLocked: true })
-          .toList();
-        for (const job of jobs) {
-          job.lockedUntil = new Date(now.getTime() + LEASE_MS);
-          job.attempts += 1;
-        }
-        await db.saveChanges();
-        return jobs;
-      });
-    } finally {
-      await scope.dispose();
-    }
-  }
 }
 
 /** The resolved settings of {@link workflowRuntimeConfig}. */

package/core/workflow/runtime_module/internal/Schedules.ts

@@ -22,7 +22,7 @@ export class Schedules {
   ) {}
 
   /** Adds jobs for new scheduled versions and drops those of replaced or archived ones. */
-  async reconcile(now = new Date()): Promise<void> {
+  async reconcile(now = this.engine.now()): Promise<void> {
     const versions = (await this.definitions.published()).filter((version) => scheduleNodes(version).length > 0);
     const wanted = new Map(versions.map((version) => [version.id, version]));
     const added: WfJob[] = [];
@@ -58,7 +58,9 @@ export class Schedules {
     const due = new Date(Math.floor(job.dueAt.getTime() / 60_000) * 60_000);
     const graph = this.engine.cache(latest);
     await this.db.transactionScope(async () => {
-      const next = nextFiring(latest, due);
+      // Planned from now, not from the missed firing: a stopped worker does not catch up.
+      const now = this.engine.now();
+      const next = nextFiring(latest, now > due ? now : due);
       if (next === undefined) await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
       else await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeUpdate({ dueAt: next, lockedUntil: null, attempts: 0, lastError: null });
       for (const [nodeId, cron] of scheduleNodes(latest)) {

package/core/workflow/runtime_module/internal/ToolStep.ts

@@ -44,7 +44,7 @@ export class ToolStep {
     await this.engine.inStep(instance.id, async (step) => {
       const current = step.tokens.find((t) => t.id === token.id);
       const retryAt = current?.status === "waiting" && step.instance.status === "running" && result.status === "error" && reason === undefined
-        ? nextAttempt(node?.config?.retry, job.attempts)
+        ? nextAttempt(node?.config?.retry, job.attempts, this.engine.now())
         : undefined;
       if (retryAt !== undefined) {
         // The same job runs again later: its attempts keep counting.
@@ -87,10 +87,10 @@ function reasonOf(result: AgentToolResult): string | undefined {
  * `undefined` when the attempts are used up. The pause doubles from `backoff`
  * (10 seconds by default) and stops growing at a day.
  */
-export function nextAttempt(policy: unknown, attempts: number): Date | undefined {
+export function nextAttempt(policy: unknown, attempts: number, now: Date): Date | undefined {
   if (typeof policy !== "object" || policy === null) return undefined;
   const { maxAttempts, backoff } = policy as { maxAttempts?: unknown; backoff?: unknown };
   if (typeof maxAttempts !== "number" || attempts >= maxAttempts) return undefined;
   const base = typeof backoff === "string" ? durationMs(backoff) ?? 10_000 : 10_000;
-  return new Date(Date.now() + Math.min(base * 2 ** Math.max(attempts - 1, 0), 86_400_000));
+  return new Date(now.getTime() + Math.min(base * 2 ** Math.max(attempts - 1, 0), 86_400_000));
 }

package/core/workflow/runtime_module/internal/WorkflowEngine.ts

@@ -2,6 +2,7 @@ import type { ServiceResolver } from "../../../di";
 import { EventBus, type EventToken, type Logger } from "../../../kernel";
 import { durationMs, evaluateCondition, nodeKindOf, readPath, type FormSpec, type InstanceData, type ProcessFlow, type ProcessNode } from "../../../../library/workflow";
 import type { WorkflowActor } from "../../contracts/types";
+import type { WorkflowClock } from "../../contracts/WorkflowClock";
 import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
 import { WORKFLOW_INSTANCE_FINISHED, WORKFLOW_TASK_REQUESTED, WORKFLOW_TOKEN_CLOSED } from "../events/Runtime.events";
 import type { RuntimeDbContext } from "../model/Runtime.dbContext";
@@ -53,8 +54,14 @@ export class WorkflowEngine {
     private readonly events: EventBus,
     private readonly scope: ServiceResolver,
     private readonly logger: Logger | undefined,
+    private readonly clock: WorkflowClock,
   ) {}
 
+  /** The engine's time ({@link WorkflowClock}). */
+  now(): Date {
+    return this.clock.now();
+  }
+
   /** The graph of a version. Versions are immutable, so graphs are cached. */
   async graphOf(versionId: string): Promise<ProcessGraph> {
     const cached = this.graphs.get(versionId);
@@ -168,7 +175,7 @@ export class WorkflowEngine {
     const token = step.tokens.find((t) => t.id === tokenId);
     if (!token || token.status !== "waiting" || step.instance.status !== "running") return false;
     const node = step.graph.document.nodes[token.nodeId];
-    const firedAt = new Date().toISOString();
+    const firedAt = this.now().toISOString();
     if (node?.kind === "timerCatch") {
       this.history(step, token, token.nodeId, "timer", { firedAt });
       return await this.complete(step, tokenId, { firedAt });
@@ -239,12 +246,12 @@ export class WorkflowEngine {
     const open = step.tokens.filter((t) => LIVE.has(t.status));
     for (const token of open) {
       token.status = "cancelled";
-      token.endedAt = new Date();
+      token.endedAt = this.now();
     }
     await this.db.jobs.asNoTracking().where((j) => j.instanceId.eq(step.instance.id)).executeDelete();
     await this.db.subscriptions.asNoTracking().where((s) => s.instanceId.eq(step.instance.id)).executeDelete();
     step.instance.status = "cancelled";
-    step.instance.endedAt = new Date();
+    step.instance.endedAt = this.now();
     this.history(step, null, null, "cancelled", { reason }, actor);
     for (const token of open) {
       if (token.nodeId !== "" && step.graph.document.nodes[token.nodeId]?.kind === "userTask") {
@@ -278,7 +285,7 @@ export class WorkflowEngine {
     }
     if (step.instance.status === "running" && !step.tokens.some((t) => LIVE.has(t.status))) {
       step.instance.status = "completed";
-      step.instance.endedAt = new Date();
+      step.instance.endedAt = this.now();
       this.history(step, null, null, "completed", null);
       this.finished(step, "completed");
     }
@@ -319,7 +326,7 @@ export class WorkflowEngine {
   }
 
   private waitForTimer(step: Step, token: WfToken, node: ProcessNode): WfToken[] {
-    const dueAt = timerDue(node.config ?? {}, dataOf(step.instance));
+    const dueAt = timerDue(node.config ?? {}, dataOf(step.instance), this.now());
     if (dueAt === undefined) {
       this.fail(step, token, "bad_timer", `the timer of ${token.nodeId} has no valid date`);
       return [];
@@ -348,7 +355,7 @@ export class WorkflowEngine {
         continue;
       }
       if (node.kind !== "boundaryTimer") continue;
-      const dueAt = timerDue(node.config ?? {}, dataOf(step.instance));
+      const dueAt = timerDue(node.config ?? {}, dataOf(step.instance), this.now());
       if (dueAt === undefined) {
         this.fail(step, host, "bad_timer", `the timer of ${id} has no valid date`);
         return [];
@@ -393,7 +400,7 @@ export class WorkflowEngine {
 
   private waitForTool(step: Step, token: WfToken): WfToken[] {
     token.status = "waiting";
-    this.db.jobs.add(Object.assign(new WfJob(), { id: Bun.randomUUIDv7(), kind: "tool" as const, instanceId: step.instance.id, tokenId: token.id, dueAt: new Date() }));
+    this.db.jobs.add(Object.assign(new WfJob(), { id: Bun.randomUUIDv7(), kind: "tool" as const, instanceId: step.instance.id, tokenId: token.id, dueAt: this.now() }));
     return [];
   }
 
@@ -412,7 +419,7 @@ export class WorkflowEngine {
         title: node.title,
         assignee: config.assignee as { role: string } | { user: string },
         form: (config.form as FormSpec | undefined) ?? null,
-        dueAt: dueIn === undefined ? null : new Date(Date.now() + dueIn),
+        dueAt: dueIn === undefined ? null : new Date(this.now().getTime() + dueIn),
       },
     });
     return [];
@@ -467,7 +474,7 @@ export class WorkflowEngine {
 
   private close(token: WfToken, status: "done" | "cancelled"): void {
     token.status = status;
-    token.endedAt = new Date();
+    token.endedAt = this.now();
   }
 
   private spawn(step: Step, nodeId: string, viaFlowId: string | null): WfToken {
@@ -505,10 +512,10 @@ export class WorkflowEngine {
 }
 
 /** When a timer fires: `duration` from now, or the date at the `at` path; `undefined` if neither is valid. */
-export function timerDue(config: Readonly<Record<string, unknown>>, data: InstanceData): Date | undefined {
+export function timerDue(config: Readonly<Record<string, unknown>>, data: InstanceData, now: Date): Date | undefined {
   if (typeof config.duration === "string") {
     const ms = durationMs(config.duration);
-    return ms === undefined ? undefined : new Date(Date.now() + ms);
+    return ms === undefined ? undefined : new Date(now.getTime() + ms);
   }
   if (typeof config.at === "string") {
     const value = readPath(data, config.at);

package/core/workflow/runtime_module/internal/WorkflowJobs.ts

@@ -0,0 +1,100 @@
+import type { ServiceProvider } from "../../../di";
+import { WorkflowClock } from "../../contracts/WorkflowClock";
+import { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfJob } from "../model/WfJob.model";
+import { Schedules } from "./Schedules";
+import { TimerStep } from "./TimerStep";
+import { ToolStep } from "./ToolStep";
+import { WorkflowEngine } from "./WorkflowEngine";
+
+/** How long a claimed job belongs to its worker; if it dies, another one takes the job after that. */
+const LEASE_MS = 5 * 60_000;
+/** A job that keeps failing outside its Tool (the database, a lost version) fails its step after this many tries. */
+const MAX_JOB_ATTEMPTS = 5;
+
+/** Where a failure goes: the worker's log. */
+export type JobFailureReport = (message: string, error: unknown, fields?: Readonly<Record<string, unknown>>) => void;
+
+/**
+ * Claiming and running jobs of `bazis_wf_run_jobs`: the work of the worker, and
+ * of `drain` in tests. Every job runs in its own DI scope.
+ */
+export class WorkflowJobs {
+  constructor(
+    private readonly services: ServiceProvider,
+    private readonly report: JobFailureReport,
+  ) {}
+
+  /** Takes up to `batchSize` due jobs with `FOR UPDATE SKIP LOCKED` and a lease, so instances never take one job twice. */
+  async claim(batchSize: number): Promise<WfJob[]> {
+    const scope = this.services.createScope();
+    try {
+      const db = scope.resolve(RuntimeDbContext);
+      const now = scope.resolve(WorkflowClock).now();
+      return await db.transactionScope(async () => {
+        const jobs = await db.jobs
+          .where((j) => j.kind.in(["tool", "timer", "schedule"]).and(j.dueAt.lte(now)).and(j.lockedUntil.isNull().or(j.lockedUntil.lt(now))))
+          .orderBy((j) => j.id)
+          .take(batchSize)
+          .forUpdate({ skipLocked: true })
+          .toList();
+        for (const job of jobs) {
+          job.lockedUntil = new Date(now.getTime() + LEASE_MS);
+          job.attempts += 1;
+        }
+        await db.saveChanges();
+        return jobs;
+      });
+    } finally {
+      await scope.dispose();
+    }
+  }
+
+  async run(job: WfJob, signal: AbortSignal): Promise<void> {
+    const scope = this.services.createScope();
+    try {
+      if (job.kind === "timer") await scope.resolve(TimerStep).run(job);
+      else if (job.kind === "schedule") await scope.resolve(Schedules).fire(job);
+      else await scope.resolve(ToolStep).run(job, signal);
+    } catch (error) {
+      this.report("job failed", error, { job: job.id, instance: job.instanceId, attempts: job.attempts });
+      await this.giveUpOrDelay(job, error).catch((failure: unknown) => this.report("job failure handling failed", failure, { job: job.id }));
+    } finally {
+      await scope.dispose();
+    }
+  }
+
+  /** Matches the scheduled starts against the published versions. */
+  async reconcile(): Promise<void> {
+    const scope = this.services.createScope();
+    try {
+      await scope.resolve(Schedules).reconcile();
+    } catch (error) {
+      this.report("scheduled starts could not be planned", error);
+    } finally {
+      await scope.dispose();
+    }
+  }
+
+  /** A job that failed outside its Tool waits a minute per try, then fails its step for a person to retry. */
+  private async giveUpOrDelay(job: WfJob, error: unknown): Promise<void> {
+    const message = error instanceof Error ? error.message : String(error);
+    const scope = this.services.createScope();
+    try {
+      const db = scope.resolve(RuntimeDbContext);
+      if (job.attempts < MAX_JOB_ATTEMPTS || job.instanceId === null || job.tokenId === null) {
+        const now = scope.resolve(WorkflowClock).now();
+        await db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeUpdate({ dueAt: new Date(now.getTime() + job.attempts * 60_000), lockedUntil: null, lastError: message.slice(0, 1000) });
+        return;
+      }
+      const engine = scope.resolve(WorkflowEngine);
+      await engine.inStep(job.instanceId, async (step) => {
+        await db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+        const token = step.tokens.find((t) => t.id === job.tokenId);
+        if (token && token.status === "waiting" && step.instance.status === "running") engine.fail(step, token, "job_failed", message.slice(0, 1000));
+      });
+    } finally {
+      await scope.dispose();
+    }
+  }
+}

package/core/workflow/tasks_module/WorkflowTasks.module.ts

@@ -1,4 +1,6 @@
 import { DI, Module, scoped } from "../../di";
+import { WorkflowClock } from "../contracts/WorkflowClock";
+import { WorkflowDefinitionsModule } from "../definitions_module/WorkflowDefinitions.module";
 import { withEventSubscribers } from "../../kernel";
 import { IWorkflowRuntime } from "../runtime_module/services/IWorkflowRuntime.service";
 import { WorkflowRuntimeModule } from "../runtime_module/WorkflowRuntime.module";
@@ -15,10 +17,10 @@ import { WorkflowTasks } from "./services/WorkflowTasks.service";
  * application's codegen.
  */
 @Module({
-  imports: [WorkflowRuntimeModule],
+  imports: [WorkflowDefinitionsModule, WorkflowRuntimeModule],
   ormBazis: { context: TasksDbContext, entities: [WfTask], ownedStore: tasksStore, registerRepositories: false },
   providers: [
-    scoped(WorkflowTasks, WorkflowTasks, [TasksDbContext, IWorkflowRuntime] as const),
+    scoped(WorkflowTasks, WorkflowTasks, [TasksDbContext, IWorkflowRuntime, WorkflowClock] as const),
     DI.scoped(DI.factoryProviderWithResolver(IWorkflowTasks, [], (resolver) => resolver.resolve(WorkflowTasks))),
     scoped(TaskEvents, TaskEvents, [WorkflowTasks] as const),
     ...withEventSubscribers(TaskEvents),

package/core/workflow/tasks_module/services/WorkflowTasks.service.ts

@@ -1,5 +1,6 @@
 import { UniqueViolationError } from "../../../orm";
 import { isUuid, pagingOf, type WorkflowActor, type WorkflowPage } from "../../contracts/types";
+import type { WorkflowClock } from "../../contracts/WorkflowClock";
 import { WorkflowError } from "../../contracts/WorkflowError";
 import type { WorkflowInstancesDeleted, WorkflowTaskRequested, WorkflowTokenClosed } from "../../runtime_module/events/Runtime.events";
 import type { IWorkflowRuntime } from "../../runtime_module/services/IWorkflowRuntime.service";
@@ -13,6 +14,7 @@ export class WorkflowTasks extends IWorkflowTasks {
   constructor(
     private readonly db: TasksDbContext,
     private readonly runtime: IWorkflowRuntime,
+    private readonly clock: WorkflowClock,
   ) {
     super();
   }
@@ -49,7 +51,7 @@ export class WorkflowTasks extends IWorkflowTasks {
       if (task.status !== "open") throw new WorkflowError("wrong_status", `The task is ${task.status}`);
       task.status = "claimed";
       task.claimedById = actor.id;
-      task.claimedAt = new Date();
+      task.claimedAt = this.clock.now();
       await this.db.saveChanges();
       return detailsOf(task);
     });
@@ -83,7 +85,7 @@ export class WorkflowTasks extends IWorkflowTasks {
     await this.db.tasks.asNoTracking().where((t) => t.id.eq(task.id)).executeUpdate({
       status: "completed",
       completedById: actor.id,
-      completedAt: new Date(),
+      completedAt: this.clock.now(),
       output: typeof saved === "object" && saved !== null && !Array.isArray(saved) ? saved as Record<string, unknown> : null,
     });
     return await this.get(task.id);

package/core/workflow/testing/index.ts

@@ -0,0 +1,162 @@
+import { DI } from "../../di";
+import { startTestApp, type TestApp, type TestAppOptions } from "../../testing";
+import { durationMs } from "../../../library/workflow";
+import { WorkflowClock } from "../contracts/WorkflowClock";
+import type { WorkflowActor } from "../contracts/types";
+import { WorkflowJobs } from "../runtime_module/internal/WorkflowJobs";
+import { IWorkflowRuntime, type EmitOptions, type EmitResult, type HistoryEntry, type InstanceDetails, type StartInstance } from "../runtime_module/services/IWorkflowRuntime.service";
+import { TasksDbContext } from "../tasks_module/model/Tasks.dbContext";
+import { IWorkflowTasks } from "../tasks_module/services/IWorkflowTasks.service";
+
+/** Rounds of `drain` before it gives up: a process that keeps creating due jobs never settles. */
+const MAX_ROUNDS = 1000;
+const BATCH = 50;
+const TEST_ACTOR: WorkflowActor = { kind: "system", id: "test" };
+
+/** A clock that stands still until the test moves it (WORKFLOW-001 §14). */
+export class ManualWorkflowClock extends WorkflowClock {
+  override readonly manual = true;
+  private current: number;
+
+  constructor(start: Date | string = new Date()) {
+    super();
+    this.current = toTime(start, "now");
+  }
+
+  now(): Date {
+    return new Date(this.current);
+  }
+
+  /** Moves the clock forward by an ISO duration (`PT2H`, `P1D`) or milliseconds. */
+  advance(by: string | number): Date {
+    const ms = typeof by === "number" ? by : durationMs(by);
+    if (ms === undefined || !Number.isFinite(ms) || ms < 0) throw new Error(`advance takes an ISO duration without months and years or milliseconds ≥ 0, not ${JSON.stringify(by)}`);
+    this.current += ms;
+    return this.now();
+  }
+
+  /** Moves the clock forward to a moment. */
+  set(to: Date | string): Date {
+    const time = toTime(to, "set");
+    if (time < this.current) throw new Error(`The workflow clock only moves forward: it is ${this.now().toISOString()}`);
+    this.current = time;
+    return this.now();
+  }
+}
+
+export interface TestWorkflowOptions extends TestAppOptions {
+  /** Where the workflow clock starts; the current time by default. */
+  readonly now?: Date | string;
+  /** On whose behalf `start`, `emit` and `complete` act; `{ kind: "system", id: "test" }` by default. */
+  readonly actor?: WorkflowActor;
+}
+
+/** A started application whose processes run only when the test says so. */
+export interface TestWorkflow {
+  readonly app: TestApp;
+  readonly clock: ManualWorkflowClock;
+  now(): Date;
+  /** Starts the latest version of a process from its manual start; jobs wait for {@link drain}. */
+  start(processKey: string, options?: Partial<StartInstance>): Promise<InstanceDetails>;
+  emit(eventKey: string, payload?: unknown, options?: EmitOptions): Promise<EmitResult>;
+  /** Completes the user task waiting at `nodeId` (through its task when the Tasks part is there). */
+  complete(instanceId: string, nodeId: string, output?: unknown, actor?: WorkflowActor): Promise<InstanceDetails>;
+  /** Plans scheduled starts and runs every job due by the workflow clock until none is left; returns how many ran. */
+  drain(): Promise<number>;
+  /** Moves the clock forward (ISO duration or milliseconds), then {@link drain}. */
+  advance(by: string | number): Promise<number>;
+  instance(instanceId: string): Promise<InstanceDetails>;
+  history(instanceId: string): Promise<readonly HistoryEntry[]>;
+  stop(): Promise<void>;
+}
+
+/**
+ * Starts the application like `startTestApp`, with a {@link ManualWorkflowClock}
+ * and no background worker: jobs run only in `drain`, so a test of a process is
+ * deterministic. Needs PostgreSQL, as the workflow module does.
+ *
+ * ```ts
+ * const wf = await testWorkflow(AppModule, { now: "2026-11-02T09:00:00Z" });
+ * try {
+ *   const order = await wf.start("order-approval", { input: { amount: 5000 } });
+ *   await wf.drain();
+ *   await wf.complete(order.id, "approve", { approved: true });
+ *   await wf.advance("PT2H");
+ *   expect((await wf.instance(order.id)).status).toBe("completed");
+ * } finally {
+ *   await wf.stop();
+ * }
+ * ```
+ */
+export async function testWorkflow(root: Parameters<typeof startTestApp>[0], options: TestWorkflowOptions = {}): Promise<TestWorkflow> {
+  const { now, actor = TEST_ACTOR, overrides = [], ...appOptions } = options;
+  const clock = new ManualWorkflowClock(now);
+  const app = await startTestApp(root, { ...appOptions, overrides: [...overrides, DI.singleton(DI.valueProvider(WorkflowClock, clock))] });
+  const failures: unknown[] = [];
+  const jobs = new WorkflowJobs(app.container, (message, error, fields) => failures.push({ message, error, ...fields }));
+  const signal = new AbortController().signal;
+
+  const inScope = async <T>(work: (scope: ReturnType<TestApp["container"]["createScope"]>) => Promise<T>): Promise<T> => {
+    const scope = app.container.createScope();
+    try {
+      return await work(scope);
+    } finally {
+      await scope.dispose();
+    }
+  };
+
+  const drain = async (): Promise<number> => {
+    await jobs.reconcile();
+    let ran = 0;
+    for (let round = 0; round < MAX_ROUNDS; round += 1) {
+      const due = await jobs.claim(BATCH);
+      if (due.length === 0) return ran;
+      for (const job of due) await jobs.run(job, signal);
+      ran += due.length;
+      // A failure outside a Tool (a failing Tool is a step result) is a bug the test should see.
+      if (failures.length > 0) {
+        const found = failures.splice(0);
+        throw new AggregateError(found.map((failure) => (failure as { error: unknown }).error), `drain: ${found.length} job(s) failed outside their Tools: ${found.map((failure) => { const { message, error } = failure as { message: string; error: unknown }; return `${message}: ${error instanceof Error ? error.message : String(error)}`; }).join("; ")}`);
+      }
+    }
+    throw new Error(`drain did not settle after ${MAX_ROUNDS} rounds: a process keeps creating due jobs`);
+  };
+
+  return {
+    app,
+    clock,
+    now: () => clock.now(),
+    start: (processKey, start = {}) => inScope((scope) => scope.resolve(IWorkflowRuntime).start(processKey, { ...start, actor: start.actor ?? actor })),
+    emit: (eventKey, payload, emitOptions = {}) => inScope((scope) => scope.resolve(IWorkflowRuntime).emit(eventKey, payload, { ...emitOptions, actor: emitOptions.actor ?? actor })),
+    complete: (instanceId, nodeId, output = {}, by = actor) => inScope(async (scope) => {
+      const runtime = scope.resolve(IWorkflowRuntime);
+      const instance = await runtime.instance(instanceId);
+      const token = instance.tokens.find((t) => t.nodeId === nodeId && t.status === "waiting");
+      if (!token) {
+        const waiting = instance.tokens.filter((t) => t.status === "waiting").map((t) => t.nodeId);
+        throw new Error(`No step waits at ${nodeId} in instance ${instanceId} (${instance.status}); waiting: ${waiting.length === 0 ? "none" : waiting.join(", ")}`);
+      }
+      const tasks = scope.tryResolve(IWorkflowTasks);
+      const task = tasks === undefined ? undefined : await scope.resolve(TasksDbContext).tasks.asNoTracking().firstOrDefault((t) => t.tokenId.eq(token.id).and(t.status.in(["open", "claimed"])));
+      if (tasks !== undefined && task) {
+        await tasks.complete(task.id, output, by);
+        return await runtime.instance(instanceId);
+      }
+      return await runtime.completeToken(token.id, output, by);
+    }),
+    drain,
+    advance: async (by) => {
+      clock.advance(by);
+      return await drain();
+    },
+    instance: (instanceId) => inScope((scope) => scope.resolve(IWorkflowRuntime).instance(instanceId)),
+    history: (instanceId) => inScope((scope) => scope.resolve(IWorkflowRuntime).history(instanceId)),
+    stop: () => app.stop(),
+  };
+}
+
+function toTime(value: Date | string, what: string): number {
+  const time = value instanceof Date ? value.getTime() : Date.parse(value);
+  if (!Number.isFinite(time)) throw new Error(`The workflow clock ${what} must be a date, not ${JSON.stringify(value)}`);
+  return time;
+}