bazis 0.132.0 → 0.134.0

package/core/workflow/runtime_module/internal/ChildCalls.ts

@@ -0,0 +1,186 @@
+import { inputSpecOf, resolveInput } from "../../../../library/workflow";
+import type { ServiceProvider } from "../../../di";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import { WorkflowInputValidator } from "../../contracts/WorkflowInputValidator";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import type { WorkflowLanguages } from "../../internal/WorkflowLanguages";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfInstance } from "../model/WfInstance.model";
+import type { WfJob } from "../model/WfJob.model";
+import type { WfToken } from "../model/WfToken.model";
+import { dataOf, runActorOf, type StepError, type WorkflowEngine } from "./WorkflowEngine";
+
+/** How deep processes may call each other: a process that calls itself has to stop somewhere. */
+const MAX_DEPTH = 16;
+
+/**
+ * `callProcess`: the jobs between a caller and the process it called. Each runs in
+ * the transaction of one instance, so caller and child never wait for each other.
+ * The child runs on behalf of the caller's actor, in its partition.
+ */
+export class ChildCalls {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+    /** The scope: the input validator is asked here, so that the application's own registration replaces the default. */
+    private readonly services: ServiceProvider,
+    private readonly languages: WorkflowLanguages,
+  ) {}
+
+  /**
+   * A `call` job: the step starts its process. If it has called already (a retry, a
+   * repeat after a crash), it goes on with that child: a failed one is retried, so
+   * that its finished steps do not run twice. A cancelled or timed out one is called
+   * anew only when the step was entered again (`fresh`).
+   */
+  async call(job: WfJob): Promise<void> {
+    const instance = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    const token = job.tokenId === null ? undefined : await this.db.tokens.asNoTracking().firstOrDefault((t) => t.id.eq(job.tokenId!));
+    if (!instance || !token || instance.status !== "running" || token.status !== "waiting") {
+      await this.dropJob(job);
+      return;
+    }
+    const child = await this.lastChild(instance.id, token.id);
+    if (child?.status === "running" || child?.status === "pending") {
+      await this.dropJob(job);
+      return;
+    }
+    if (child?.status === "failed") {
+      const actor = runActorOf((await this.engine.graphOf(instance.versionId)).document, instance);
+      await this.engine.inStep(child.id, async (step) => {
+        await this.dropJob(job);
+        if (step.instance.status === "failed") await this.engine.retry(step, actor);
+      });
+      return;
+    }
+    if (child?.status === "completed") {
+      await this.engine.inStep(instance.id, async (step) => {
+        await this.dropJob(job);
+        await this.engine.complete(step, token.id, child.output ?? {});
+      });
+      return;
+    }
+    if (child && (job.payload as { fresh?: boolean } | null)?.fresh !== true) {
+      await this.applyEnd(job, instance.id, token.id, child);
+      return;
+    }
+    await this.start(job, instance, token);
+  }
+
+  private async start(job: WfJob, instance: WfInstance, token: WfToken): Promise<void> {
+    const graph = await this.engine.graphOf(instance.versionId);
+    const config = graph.document.nodes[token.nodeId]?.config ?? {};
+    const key = String(config.process ?? "");
+    const depth = (typeof instance.trigger?.depth === "number" ? instance.trigger.depth : 0) + 1;
+    const actor = runActorOf(graph.document, instance);
+    const refuse = (error: StepError) => this.engine.inStep(instance.id, async (step) => {
+      await this.dropJob(job);
+      const current = step.tokens.find((t) => t.id === token.id);
+      if (current?.status === "waiting" && step.instance.status === "running") await this.engine.failStep(step, current, error);
+    });
+    if (depth > MAX_DEPTH) {
+      await refuse({ code: "call_depth", message: `Processes call each other deeper than ${MAX_DEPTH} levels` });
+      return;
+    }
+    let version: VersionDetails;
+    try {
+      version = await this.definitions.latest(key);
+    } catch (error) {
+      if (!(error instanceof WorkflowError)) throw error;
+      await refuse({ code: "process_unavailable", message: error.message, reason: error.code });
+      return;
+    }
+    const start = Object.entries(version.document.nodes).find(([, node]) => node.kind === "manualStart");
+    if (!start) {
+      await refuse({ code: "process_unavailable", message: `Process ${key} has no manual start` });
+      return;
+    }
+    const input = resolveInput(config.input ?? {}, dataOf(instance, this.engine.now(), token));
+    const { value, issues } = await this.services.resolve(WorkflowInputValidator).validate(inputSpecOf(version.document), input, { purpose: "input", processKey: key, partition: instance.partition, language: this.languages.default, actor });
+    if (issues.length > 0) {
+      await refuse({ code: "invalid_input", message: `The input of process ${key} is invalid: ${issues.map((issue) => `${issue.field}: ${issue.message}`).join("; ")}` });
+      return;
+    }
+    const childGraph = this.engine.cache(version);
+    await this.engine.inStep(instance.id, async (step) => {
+      await this.dropJob(job);
+      const current = step.tokens.find((t) => t.id === token.id);
+      if (current?.status !== "waiting" || step.instance.status !== "running") return;
+      const { id } = await this.engine.begin(childGraph, start[0], {
+        input: value,
+        partition: step.instance.partition,
+        businessKey: null,
+        startedVia: "callProcess",
+        actor,
+        parent: { instanceId: step.instance.id, tokenId: current.id, depth },
+        cause: { instanceId: step.instance.id, processKey: step.instance.processKey, nodeId: current.nodeId },
+      });
+      this.engine.history(step, current, current.nodeId, "called", { process: key, version: version.version, instanceId: id });
+    });
+  }
+
+  /** A `returned` job: a child stopped. If its caller no longer waits, the job drops; a caller that is itself stopped looks again on its retry. */
+  async returned(job: WfJob): Promise<void> {
+    const { parentInstanceId, parentTokenId } = job.payload as { parentInstanceId: string; parentTokenId: string };
+    const child = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (!child) {
+      await this.dropJob(job);
+      return;
+    }
+    await this.applyEnd(job, parentInstanceId, parentTokenId, child);
+  }
+
+  private async applyEnd(job: WfJob, instanceId: string, tokenId: string, child: WfInstance): Promise<void> {
+    const done = await this.engine.inStep(instanceId, async (step) => {
+      await this.dropJob(job);
+      const token = step.tokens.find((t) => t.id === tokenId);
+      if (token?.status !== "waiting" || step.instance.status !== "running") return;
+      if (child.status === "completed") {
+        this.engine.history(step, token, token.nodeId, "returned", { instanceId: child.id });
+        await this.engine.complete(step, tokenId, child.output ?? {});
+        return;
+      }
+      const error = errorOf(child);
+      if (error) await this.engine.failStep(step, token, error);
+    });
+    if (done === null) await this.dropJob(job);
+  }
+
+  /** A `cancelChildren` job: the step that called is closed, so is what it called. */
+  async cancelChildren(job: WfJob): Promise<void> {
+    const { tokenId, reason } = job.payload as { tokenId: string; reason: string };
+    const parent = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (parent) {
+      const children = await this.db.instances.asNoTracking()
+        .where((i) => i.parentInstanceId.eq(parent.id).and(i.parentTokenId.eq(tokenId)).and(i.status.in(["pending", "running", "failed"])))
+        .toList();
+      const actor = { kind: "system" as const, id: `process:${parent.processKey}` };
+      for (const child of children) {
+        await this.engine.inStep(child.id, async (step) => {
+          if (step.instance.status === "pending" || step.instance.status === "running" || step.instance.status === "failed") await this.engine.cancel(step, reason, actor);
+        });
+      }
+    }
+    await this.dropJob(job);
+  }
+
+  /** The newest child of a step. */
+  private async lastChild(instanceId: string, tokenId: string): Promise<WfInstance | undefined> {
+    const children = await this.db.instances.asNoTracking().where((i) => i.parentInstanceId.eq(instanceId).and(i.parentTokenId.eq(tokenId))).orderBy((i) => i.id).toList();
+    return children.at(-1);
+  }
+
+  private async dropJob(job: WfJob): Promise<void> {
+    await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
+}
+
+/** The failure of a step whose child stopped otherwise than by completing; `undefined` while it runs again. */
+function errorOf(child: WfInstance): StepError | undefined {
+  const about = `Instance ${child.id} of process ${child.processKey}`;
+  if (child.status === "failed") return { code: "child_failed", message: `${about} failed: ${child.error?.message ?? ""}`, ...(child.error ? { reason: child.error.code } : {}) };
+  if (child.status === "cancelled") return { code: "child_cancelled", message: `${about} was cancelled` };
+  if (child.status === "timedOut") return { code: "child_timed_out", message: `${about} took too long`, reason: "timeout" };
+  return undefined;
+}

package/core/workflow/runtime_module/internal/EventDelivery.ts

@@ -0,0 +1,234 @@
+import { createHash } from "node:crypto";
+import { durationMs, evaluateExpression, inputSpecOf, isTruthy, type Expr, type FormIssue, type InstanceData, type ProcessNode } from "../../../../library/workflow";
+import { LOGGER } from "../../../kernel";
+import type { ServiceProvider } from "../../../di";
+import { isUuid, type WorkflowActor, type WorkflowRequest } from "../../contracts/types";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import { WorkflowInputValidator } from "../../contracts/WorkflowInputValidator";
+import { WorkflowPartitions } from "../../contracts/WorkflowPartitions";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import { canonicalJson } from "../../internal/canonicalJson";
+import type { WorkflowLanguages } from "../../internal/WorkflowLanguages";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import { WfJob } from "../model/WfJob.model";
+import type { EmitCause, EmitResult } from "../services/IWorkflowRuntime.service";
+import type { WorkflowEngine } from "./WorkflowEngine";
+
+/** An event on its way: what `emit` got, with the defaults in place. */
+export interface EventMessage {
+  readonly eventKey: string;
+  readonly payload: Record<string, unknown>;
+  readonly correlation: string | null;
+  readonly partition: string | null;
+  readonly actor: WorkflowActor;
+  readonly request: WorkflowRequest | null;
+  readonly cause: EmitCause | null;
+}
+
+interface EventStartConfig {
+  readonly filter?: Expr;
+  readonly debounce?: { readonly key: Expr; readonly window: string };
+  readonly delay?: string;
+  readonly ignoreOwn?: boolean;
+}
+
+/** A process the event is to start, with its checked input. */
+interface PlannedStart {
+  readonly version: VersionDetails;
+  readonly nodeId: string;
+  readonly config: EventStartConfig;
+  readonly input: Record<string, unknown>;
+}
+
+/**
+ * Takes an event to whoever waits for it and to the processes it starts
+ * (WORKFLOW-002 §5.1, §5.2). An `event` job (`emitLater`) turns into an
+ * `eventStart` job per process in one transaction, and each of those starts its
+ * instance in one transaction with its own removal: a repeat after a crash starts
+ * nothing twice. Waits are safe as they are: a subscription leaves with its step.
+ */
+export class EventDelivery {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+    private readonly languages: WorkflowLanguages,
+    /** The scope: the application's own ports replace the defaults. */
+    private readonly services: ServiceProvider,
+  ) {}
+
+  /** Moves the waits and starts the processes, now or, under `debounce` and `delay`, by a job. */
+  async emit(message: EventMessage): Promise<EmitResult> {
+    const continued = await this.continueWaits(message);
+    const { planned, rejected } = await this.plan(message);
+    const started: string[] = [];
+    const skipped: { processKey: string; instanceId: string }[] = [];
+    const deferred: { processKey: string }[] = [];
+    for (const start of planned) {
+      if (start.config.debounce !== undefined || start.config.delay !== undefined) {
+        await this.defer(start, message);
+        deferred.push({ processKey: start.version.key });
+        continue;
+      }
+      const begun = await this.begin(start.version, start.nodeId, start.input, message);
+      if (begun.skipped) skipped.push({ processKey: start.version.key, instanceId: begun.id });
+      else started.push(begun.id);
+    }
+    return { started, skipped, deferred, continued, rejected };
+  }
+
+  /** An `event` job: the waits move, and the job becomes an `eventStart` job per process it starts. */
+  async deliverLater(job: WfJob): Promise<void> {
+    const message = job.payload as EventMessage;
+    await this.continueWaits(message);
+    const { planned, rejected } = await this.plan(message);
+    for (const { processKey, issues } of rejected) this.services.tryResolve(LOGGER)?.warn("workflow event did not start a process", { event: message.eventKey, processKey, issues });
+    await this.db.transactionScope(async () => {
+      await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+      for (const start of planned) await this.defer(start, message);
+    });
+  }
+
+  /** An `eventStart` job: its instance starts together with the removal of the job. An archived process takes nothing. */
+  async startLater(job: WfJob): Promise<void> {
+    const { message, input } = job.payload as { readonly message: EventMessage; readonly input: Record<string, unknown> };
+    const version = job.versionId === null ? undefined : await this.definitions.versionById(job.versionId);
+    if (version === undefined || job.nodeId === null || !await this.available(version.key)) {
+      await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+      return;
+    }
+    const graph = this.engine.cache(version);
+    await this.db.transactionScope(async () => {
+      await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+      await this.engine.begin(graph, job.nodeId!, this.beginOptions(input, message));
+    });
+  }
+
+  private async available(processKey: string): Promise<boolean> {
+    try {
+      await this.definitions.latest(processKey);
+      return true;
+    } catch (error) {
+      if (error instanceof WorkflowError) return false;
+      throw error;
+    }
+  }
+
+  /** Every wait of the event: `eventWait.until` may leave a step waiting. */
+  private async continueWaits(message: EventMessage): Promise<string[]> {
+    const { eventKey, correlation, partition, payload } = message;
+    let waits = this.db.subscriptions.asNoTracking().where((s) => s.eventKey.eq(eventKey));
+    waits = correlation === null ? waits.where((s) => s.correlation.isNull()) : waits.where((s) => s.correlation.isNull().or(s.correlation.eq(correlation)));
+    if (partition !== null) waits = waits.where((s) => s.partition.eq(partition));
+    const continued = new Set<string>();
+    for (const wait of await waits.orderBy((s) => s.id).toList()) {
+      const moved = await this.engine.inStep(wait.instanceId, async (step) => {
+        if (!this.engine.eventAccepted(step, wait.tokenId, payload)) return false;
+        await this.db.subscriptions.asNoTracking().where((s) => s.id.eq(wait.id)).executeDelete();
+        return await this.engine.eventReceived(step, wait.tokenId, eventKey, payload);
+      });
+      if (moved === true) continued.add(wait.instanceId);
+    }
+    return [...continued];
+  }
+
+  /** The `eventStart` nodes that take the event: connected to its partition, past `ignoreOwn` and `filter`, with a valid input. */
+  private async plan(message: EventMessage): Promise<{ readonly planned: PlannedStart[]; readonly rejected: { processKey: string; issues: readonly FormIssue[] }[] }> {
+    const { eventKey, payload, partition, actor } = message;
+    const planned: PlannedStart[] = [];
+    const rejected: { processKey: string; issues: readonly FormIssue[] }[] = [];
+    const data = eventData(message, this.engine.now());
+    const own = await this.causeProcess(message.cause);
+    for (const version of await this.definitions.published()) {
+      for (const [nodeId, node] of Object.entries(version.document.nodes)) {
+        if (node.kind !== "eventStart" || node.config?.event !== eventKey) continue;
+        const config = eventStartConfig(node);
+        if (!await this.connected(version.key, partition)) continue;
+        if (config.ignoreOwn === true && own === version.key) continue;
+        if (config.filter !== undefined && !isTruthy(evaluateExpression(config.filter, data))) continue;
+        // An event carries more than a process needs: fields the input does not declare are dropped, not refused.
+        const { value, issues: all } = await this.services.resolve(WorkflowInputValidator).validate(inputSpecOf(version.document), payload, { purpose: "input", processKey: version.key, partition, language: this.languages.default, actor });
+        const issues = all.filter((issue) => issue.code !== "unknown");
+        if (issues.length > 0) rejected.push({ processKey: version.key, issues });
+        else planned.push({ version, nodeId, config, input: value });
+      }
+    }
+    return { planned, rejected };
+  }
+
+  /** Whether the process works in the partition of the event; an event without a partition is for processes without them. */
+  private async connected(processKey: string, partition: string | null): Promise<boolean> {
+    const partitions = await this.services.resolve(WorkflowPartitions).of(processKey, { partition, language: this.languages.default, actor: null });
+    return partitions.some((item) => item.enabled && item.partition === partition);
+  }
+
+  /** The process of the instance that caused the event. */
+  private async causeProcess(cause: EmitCause | null): Promise<string | undefined> {
+    const id = cause?.instanceId;
+    if (!isUuid(id)) return undefined;
+    const row = await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(id));
+    return row?.processKey;
+  }
+
+  private async begin(version: VersionDetails, nodeId: string, input: Record<string, unknown>, message: EventMessage) {
+    return await this.engine.begin(this.engine.cache(version), nodeId, this.beginOptions(input, message));
+  }
+
+  private beginOptions(input: Record<string, unknown>, message: EventMessage) {
+    return {
+      input,
+      partition: message.partition,
+      businessKey: message.correlation,
+      startedVia: "event" as const,
+      actor: message.actor,
+      request: message.request,
+      event: { key: message.eventKey, payload: message.payload },
+      ...(message.cause === null ? {} : { cause: message.cause }),
+    };
+  }
+
+  /**
+   * An `eventStart` job due after `delay` and the `debounce` window. The window runs
+   * from its first event and later ones replace what the job carries, so a steady
+   * stream of events still starts a process once per window. A job a worker holds
+   * takes no more events: the next one opens a new window.
+   */
+  private async defer(start: PlannedStart, message: EventMessage): Promise<void> {
+    const { version, nodeId, config, input } = start;
+    const now = this.engine.now();
+    const dueAt = new Date(now.getTime() + (config.debounce === undefined ? 0 : durationMs(config.debounce.window) ?? 0) + (config.delay === undefined ? 0 : durationMs(config.delay) ?? 0));
+    const payload = { message, input };
+    const dedupeKey = config.debounce === undefined ? null : debounceKey(version.key, nodeId, message, config.debounce.key, now);
+    const job = Object.assign(new WfJob(), { id: Bun.randomUUIDv7(), kind: "eventStart" as const, versionId: version.id, nodeId, partition: message.partition, dueAt, payload, dedupeKey, createdAt: now });
+    if (dedupeKey === null) {
+      await this.db.jobs.insertIfAbsent(job, { conflictBy: (j) => [j.id] });
+      return;
+    }
+    await this.db.transactionScope(async () => {
+      await this.db.database.querySqlRaw("SELECT pg_advisory_xact_lock(hashtext({0}))", dedupeKey);
+      const merged = await this.db.jobs.asNoTracking().where((j) => j.dedupeKey.eq(dedupeKey).and(j.lockedUntil.isNull())).executeUpdate({ payload, versionId: version.id });
+      if (merged.affectedRows === 0) await this.db.jobs.insertIfAbsent(job, { conflictBy: (j) => [j.id] });
+    });
+  }
+}
+
+function eventStartConfig(node: ProcessNode): EventStartConfig {
+  return (node.config ?? {}) as unknown as EventStartConfig;
+}
+
+/** What the expressions of an `eventStart` read: there is no instance yet. */
+function eventData(message: EventMessage, now: Date): InstanceData {
+  return {
+    input: {},
+    vars: {},
+    nodes: {},
+    event: message.payload,
+    trigger: { via: "event", at: now.toISOString(), actor: message.actor, eventKey: message.eventKey, ...(message.cause === null ? {} : { cause: message.cause }) },
+    now: now.toISOString(),
+  };
+}
+
+function debounceKey(processKey: string, nodeId: string, message: EventMessage, key: Expr, now: Date): string {
+  const value = evaluateExpression(key, eventData(message, now)) ?? null;
+  return createHash("sha256").update(canonicalJson([processKey, nodeId, message.partition, value])).digest("hex");
+}

package/core/workflow/runtime_module/internal/GraphCache.ts

@@ -10,6 +10,8 @@ export interface ProcessGraph {
   readonly incoming: ReadonlyMap<string, readonly (readonly [string, ProcessFlow])[]>;
   /** Boundary nodes by the step they are attached to. */
   readonly boundaries: ReadonlyMap<string, readonly (readonly [string, ProcessNode])[]>;
+  /** The node an iteration begins with, by loop. */
+  readonly entries: ReadonlyMap<string, string>;
 }
 
 export function buildGraph(versionId: string, processKey: string, version: number, document: ProcessDocument): ProcessGraph {
@@ -23,7 +25,11 @@ export function buildGraph(versionId: string, processKey: string, version: numbe
   for (const [id, node] of Object.entries(document.nodes)) {
     if (node.attachedTo !== undefined) boundaries.set(node.attachedTo, [...boundaries.get(node.attachedTo) ?? [], [id, node]]);
   }
-  return { versionId, processKey, version, document, outgoing, incoming, boundaries };
+  const entries = new Map<string, string>();
+  for (const [id, node] of Object.entries(document.nodes)) {
+    if (node.parent !== undefined && node.attachedTo === undefined && !incoming.has(id)) entries.set(node.parent, id);
+  }
+  return { versionId, processKey, version, document, outgoing, incoming, boundaries, entries };
 }
 
 /** Graphs of versions in memory. Versions are immutable, so entries never go stale; the oldest go first past the limit. */

package/core/workflow/runtime_module/internal/RunLimits.ts

@@ -0,0 +1,134 @@
+import { inputSpecOf, validateForm } from "../../../../library/workflow";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfJob } from "../model/WfJob.model";
+import { queuesSchedule, type WorkflowEngine } from "./WorkflowEngine";
+
+/** How many waiting instances of a process one pass looks at. */
+const RELEASE_BATCH = 1000;
+
+/** What a whole run is limited by and what follows its failure: `runTimeout`, `maxConcurrent`, `sameKey: "queue"`, `overlap: "queue"`, `errorHandler`. */
+export class RunLimits {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+  ) {}
+
+  /** The run's time is out. A `failed` instance waits for a person; its retry plans the job again. */
+  async runTimeout(job: WfJob): Promise<void> {
+    const done = job.instanceId === null ? null : await this.engine.inStep(job.instanceId, async (step) => {
+      await this.dropJob(job);
+      if (step.instance.status === "running") await this.engine.timeOut(step);
+      return true;
+    });
+    if (done === null) await this.dropJob(job);
+  }
+
+  /** A place may have come free: instances of the process that wait for one start, oldest first. */
+  async release(job: WfJob): Promise<void> {
+    await this.releaseProcess((job.payload as { processKey: string }).processKey);
+    await this.dropJob(job);
+  }
+
+  /** Every process with waiting instances: the safety net for what no ending instance tells, such as a new version without a limit. */
+  async releaseWaiting(): Promise<void> {
+    const rows = await this.db.instances.asNoTracking().where((i) => i.status.eq("pending")).select((i) => ({ processKey: i.processKey })).toList();
+    for (const key of new Set(rows.map((row) => row.processKey))) await this.releaseProcess(key);
+  }
+
+  private async releaseProcess(processKey: string): Promise<void> {
+    const waiting = await this.db.instances.asNoTracking().where((i) => i.processKey.eq(processKey).and(i.status.eq("pending"))).orderBy((i) => i.id).take(RELEASE_BATCH).toList();
+    if (waiting.length === 0) return;
+    // Graphs live in the Definitions store, out of reach inside the transaction; `inStep` finds them cached.
+    const graphs = new Map(await Promise.all(waiting.map(async (row) => [row.id, await this.engine.graphOf(row.versionId)] as const)));
+    await this.db.transactionScope(async () => {
+      await this.engine.lockProcess(processKey);
+      let running = await this.engine.runningOf(processKey);
+      for (const row of waiting) {
+        const { document } = graphs.get(row.id)!;
+        const { maxConcurrent, sameKey } = document.settings ?? {};
+        if (maxConcurrent !== undefined && running >= maxConcurrent) continue;
+        if (sameKey === "queue" && row.businessKey !== null && await this.keyTaken(processKey, row.businessKey, row.id)) continue;
+        if (row.startedVia === "schedule" && queuesSchedule(document) && await this.scheduleTaken(processKey, row.partition, row.id)) continue;
+        const released = await this.engine.inStep(row.id, async (step) => {
+          if (step.instance.status !== "pending") return false;
+          await this.engine.release(step);
+          return true;
+        });
+        if (released === true) running += 1;
+      }
+    });
+  }
+
+  /** Whether an instance older than this one still works on the key: it goes first. */
+  private async keyTaken(processKey: string, businessKey: string, id: string): Promise<boolean> {
+    const same = await this.db.instances.asNoTracking()
+      .where((i) => i.processKey.eq(processKey).and(i.businessKey.eq(businessKey)).and(i.status.in(["pending", "running", "failed"])).and(i.id.lt(id)))
+      .count();
+    return same > 0;
+  }
+
+  /** Whether a scheduled run of the partition goes on, or waits since before this one. */
+  private async scheduleTaken(processKey: string, partition: string | null, id: string): Promise<boolean> {
+    const same = await this.db.instances.asNoTracking()
+      .where((i) => i.processKey.eq(processKey).and(i.startedVia.eq("schedule")).and(partition === null ? i.partition.isNull() : i.partition.eq(partition)).and(i.status.eq("running").or(i.status.eq("pending").and(i.id.lt(id)))))
+      .count();
+    return same > 0;
+  }
+
+  /** An `errorHandler` job: the handler starts with `$event` telling what happened. What keeps it from starting goes to the history of the instance that failed. */
+  async errorHandler(job: WfJob): Promise<void> {
+    const failed = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (!failed) {
+      await this.dropJob(job);
+      return;
+    }
+    const process = (job.payload as { process: string }).process;
+    const note = (data: Record<string, unknown>) => this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      this.engine.history(step, null, null, "handler", { process, ...data });
+    });
+    let version: VersionDetails;
+    try {
+      version = await this.definitions.latest(process);
+    } catch (error) {
+      if (!(error instanceof WorkflowError)) throw error;
+      await note({ error: error.message });
+      return;
+    }
+    const start = Object.entries(version.document.nodes).find(([, node]) => node.kind === "manualStart");
+    if (!start) {
+      await note({ error: "the process has no manual start" });
+      return;
+    }
+    const { value, issues } = validateForm(inputSpecOf(version.document), {});
+    if (issues.length > 0) {
+      await note({ error: "the process needs input", issues });
+      return;
+    }
+    const graph = this.engine.cache(version);
+    // In one transaction with the job, so that a repeat of the job does not start a second handler.
+    await this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      const { id } = await this.engine.begin(graph, start[0], {
+        input: value,
+        partition: failed.partition,
+        businessKey: null,
+        startedVia: "errorHandler",
+        actor: { kind: "system", id: `process:${failed.processKey}` },
+        event: {
+          key: "workflow.instance-stopped",
+          payload: { instanceId: failed.id, processKey: failed.processKey, version: failed.version, partition: failed.partition, businessKey: failed.businessKey, status: failed.status, node: failed.error?.node ?? null, code: failed.error?.code ?? null, message: failed.error?.message ?? null },
+        },
+        cause: { instanceId: failed.id, processKey: failed.processKey },
+      });
+      this.engine.history(step, null, null, "handler", { process, instanceId: id });
+    });
+  }
+
+  private async dropJob(job: WfJob): Promise<void> {
+    await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
+}

package/core/workflow/runtime_module/internal/RuntimeProcessUsage.ts

@@ -0,0 +1,13 @@
+import { WorkflowProcessUsage } from "../../contracts/WorkflowProcessUsage";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+
+/** Unfinished instances of a process: they stop it from being deleted. */
+export class RuntimeProcessUsage extends WorkflowProcessUsage {
+  constructor(private readonly db: RuntimeDbContext) {
+    super();
+  }
+
+  async liveInstances(processKey: string): Promise<number> {
+    return await this.db.instances.asNoTracking().where((i) => i.processKey.eq(processKey).and(i.status.in(["pending", "running", "failed"]))).count();
+  }
+}