bazis 0.132.0 → 0.133.0

package/core/infra/connectors/codex/CodexAppServerClient.ts

@@ -232,7 +232,7 @@ export class CodexAppServerClient implements CodexClient {
         if (method === "account/updated") this.modelCache = undefined;
       }, () => { this.clearLogin(); this.modelCache = undefined; });
       const initialized = await client.request("initialize", {
-        clientInfo: { name: "bazis", title: "Bazis", version: "0.132.0" }, capabilities: { experimentalApi: true },
+        clientInfo: { name: "bazis", title: "Bazis", version: "0.133.0" }, capabilities: { experimentalApi: true },
       }, 15_000, signal);
       if (resolve(text(initialized.codexHome, 4096)) !== this.paths.home) throw new CodexError("PROTOCOL_ERROR");
       client.notify("initialized");

package/core/workflow/definitions_module/code/CodeProcessPublisher.ts

@@ -10,8 +10,8 @@ import { definitionProblems, WorkflowSetupError } from "./codeProcesses";
 /**
  * At start, after the tables exist: brings over the catalog of the old store
  * (WORKFLOW-002 §9), then publishes the processes of `@Module({ processes })`
- * (WORKFLOW-001 §14). A document with an unknown Tool or an `extensions` namespace
- * nobody checks stops the start with every finding of every process; the same
+ * (WORKFLOW-001 §14). A document with an unknown Tool or process, or an `extensions`
+ * namespace nobody checks, stops the start with every finding of every process; the same
  * document again gives no new version.
  */
 export class CodeProcessPublisher implements HostedService {
@@ -30,10 +30,11 @@ export class CodeProcessPublisher implements HostedService {
       const tools = this.services.resolve(WorkflowTools);
       const definitions = scope.resolve(IWorkflowDefinitions);
       const problems: string[] = [];
+      const processes = entries.map(({ process }) => process.key);
       for (const { process, module } of entries) {
         const found = [...definitionProblems(process, (name) => tools.has(name))];
-        for (const issue of await definitions.validate(process.document)) {
-          if (issue.severity === "error" && (issue.code === "extension_unknown" || issue.code === "extension_invalid")) found.push(`${issue.message} (${issue.code})`);
+        for (const issue of await definitions.validate(process.document, { processes })) {
+          if (issue.severity === "error" && (issue.code === "extension_unknown" || issue.code === "extension_invalid" || issue.code === "unknown_process")) found.push(`${issue.message} (${issue.code})`);
         }
         if (found.length > 0) problems.push(`${module}: process "${process.key}":\n${found.map((line) => `  - ${line}`).join("\n")}`);
       }

package/core/workflow/definitions_module/http/Processes.controller.ts

@@ -122,7 +122,8 @@ export class ProcessesController {
   async validate(ctx: HttpContext) {
     const request = await body(ctx);
     const { actor, language } = await callerOf(ctx);
-    return { issues: await this.definitions.validate(request.document, { actor, language }) };
+    const key = optionalString(request.key, "key", 64);
+    return { issues: await this.definitions.validate(request.document, { actor, language, ...(key === undefined ? {} : { processes: [key] }) }) };
   }
 
   /** What a designer builds its forms from (WORKFLOW-002 §4.2). */

package/core/workflow/definitions_module/services/IWorkflowDefinitions.service.ts

@@ -76,6 +76,8 @@ export interface ProcessListQuery extends WorkflowPaging {
 export interface ValidateOptions {
   readonly language?: string;
   readonly actor?: WorkflowActor;
+  /** Keys that count as published besides the catalog: the process itself, processes of the code about to be published. */
+  readonly processes?: readonly string[];
 }
 
 export interface SaveDraftOptions extends WorkflowAuditOptions, Pick<ValidateOptions, "language"> {

package/core/workflow/definitions_module/services/WorkflowDefinitions.service.ts

@@ -86,7 +86,9 @@ export class WorkflowDefinitions extends IWorkflowDefinitions {
   async validate(document: unknown, options: ValidateOptions = {}): Promise<readonly ProcessIssue[]> {
     const port = this.services.resolve(WorkflowExtensions);
     const namespaces = new Set(port.namespaces());
-    const issues = validateProcess(document, { tools: (name) => this.workflowTools.has(name), extensions: (namespace) => namespaces.has(namespace) });
+    const published = await this.db.processes.asNoTracking().where((p) => p.archivedAt.isNull().and(p.latestVersion.isNotNull())).select((p) => ({ key: p.key })).toList();
+    const processes = new Set([...published.map((row) => row.key), ...options.processes ?? []]);
+    const issues = validateProcess(document, { tools: (name) => this.workflowTools.has(name), processes: (key) => processes.has(key), extensions: (namespace) => namespaces.has(namespace) });
     if (namespaces.size === 0 || issues.some((issue) => issue.code === "too_large" || issue.code === "schema")) return issues;
     const context: WorkflowPortContext = { partition: null, language: options.language ?? this.languages.default, actor: options.actor ?? null };
     const found: ProcessIssue[] = [...issues];
@@ -178,7 +180,7 @@ export class WorkflowDefinitions extends IWorkflowDefinitions {
       if (!row) throw notFound(key);
       this.assertNotCode(key);
       if (row.archivedAt !== null) throw new WorkflowError("archived", `Process ${key} is archived`);
-      const issues = await this.validate(row.draft, { actor, ...(options.language === undefined ? {} : { language: options.language }) });
+      const issues = await this.validate(row.draft, { actor, processes: [key], ...(options.language === undefined ? {} : { language: options.language }) });
       if (hasErrors(issues)) throw new WorkflowError("invalid_document", `Process ${key} has errors and cannot be published`, issues);
       const { version, created } = await this.publishDraft(row, actor);
       if (created) transaction.afterCommit(() => this.changed("published", key, actor, options, version.version));

package/core/workflow/generated/deps.ts

@@ -22,7 +22,9 @@ import { WorkflowRetention } from "../runtime_module/background/WorkflowRetentio
 import { WorkflowWorker } from "../runtime_module/background/WorkflowWorker";
 import { EventsController } from "../runtime_module/http/Events.controller";
 import { InstancesController } from "../runtime_module/http/Instances.controller";
+import { ChildCalls } from "../runtime_module/internal/ChildCalls";
 import { GraphCache } from "../runtime_module/internal/GraphCache";
+import { RunLimits } from "../runtime_module/internal/RunLimits";
 import { Schedules } from "../runtime_module/internal/Schedules";
 import { TimerStep } from "../runtime_module/internal/TimerStep";
 import { ToolStep } from "../runtime_module/internal/ToolStep";
@@ -38,6 +40,7 @@ import { WorkflowTasks } from "../tasks_module/services/WorkflowTasks.service";
 
 registerFrameworkClassDeps([
   [CatalogImport, [DefinitionsDbContext, WorkflowLanguages]],
+  [ChildCalls, [RuntimeDbContext, WorkflowEngine, IWorkflowDefinitions, ServiceProvider, WorkflowLanguages]],
   [CodeProcessCatalog, [ServiceProvider]],
   [CodeProcessPublisher, [ServiceProvider]],
   [DefinitionsDbContext, [DbContextOptions]],
@@ -47,6 +50,7 @@ registerFrameworkClassDeps([
   [ProcessesController, [IWorkflowDefinitions]],
   [ProcessFolderLookup, [DefinitionsDbContext]],
   [ProcessReferences, [DefinitionsDbContext]],
+  [RunLimits, [RuntimeDbContext, WorkflowEngine, IWorkflowDefinitions]],
   [RuntimeDbContext, [DbContextOptions]],
   [Schedules, [RuntimeDbContext, WorkflowEngine, IWorkflowDefinitions]],
   [TaskEvents, [TasksDbContext]],

package/core/workflow/runtime_module/WorkflowRuntime.module.ts

@@ -7,7 +7,9 @@ import { WorkflowRetention } from "./background/WorkflowRetention";
 import { WorkflowWorker } from "./background/WorkflowWorker";
 import { EventsController } from "./http/Events.controller";
 import { InstancesController } from "./http/Instances.controller";
+import { ChildCalls } from "./internal/ChildCalls";
 import { GraphCache } from "./internal/GraphCache";
+import { RunLimits } from "./internal/RunLimits";
 import { Schedules } from "./internal/Schedules";
 import { TimerStep } from "./internal/TimerStep";
 import { ToolStep } from "./internal/ToolStep";
@@ -31,6 +33,8 @@ import { WorkflowRuntime } from "./services/WorkflowRuntime.service";
     scoped(ToolStep),
     scoped(TimerStep),
     scoped(Schedules),
+    scoped(ChildCalls),
+    scoped(RunLimits),
     singleton(WorkflowInputValidator, DefaultWorkflowInputValidator),
     scoped(IWorkflowRuntime, WorkflowRuntime),
   ],

package/core/workflow/runtime_module/background/WorkflowRetention.ts

@@ -10,7 +10,7 @@ import type { WorkflowSettings } from "../../contracts/Workflow.config";
 const DAY_MS = 86_400_000;
 
 /**
- * Deletes `completed` and `cancelled` instances older than their process's
+ * Deletes `completed`, `cancelled` and `timedOut` instances older than their process's
  * `retentionDays`, with tokens, jobs, subscriptions and history (by cascade),
  * `retention.batchSize` at a time. `failed` instances wait for a person and stay.
  * Each deleted batch is published as `WORKFLOW_INSTANCES_DELETED`, so parts
@@ -52,7 +52,7 @@ export class WorkflowRetention extends BackgroundService {
         for (;;) {
           if (signal?.aborted) return deleted;
           const ids = (await db.instances.asNoTracking()
-            .where((i) => i.processKey.eq(policy.key).and(i.status.in(["completed", "cancelled"])).and(i.endedAt.lt(cutoff)))
+            .where((i) => i.processKey.eq(policy.key).and(i.status.in(["completed", "cancelled", "timedOut"])).and(i.endedAt.lt(cutoff)))
             .orderBy((i) => i.id)
             .take(batchSize)
             .select((i) => ({ id: i.id }))

package/core/workflow/runtime_module/events/Runtime.events.ts

@@ -21,16 +21,16 @@ export interface WorkflowTaskRequested {
 export interface WorkflowTokenClosed {
   readonly instanceId: string;
   readonly tokenId: string;
-  /** `cancelled` — the instance was cancelled; `interrupted` — a boundary timer took over the step. */
-  readonly reason: "cancelled" | "interrupted";
+  /** `cancelled` — the instance was cancelled; `interrupted` — a boundary event or a skipped iteration closed the step; `timedOut` — the run took too long. */
+  readonly reason: "cancelled" | "interrupted" | "timedOut";
 }
 
-/** An instance finished: completed or cancelled. */
+/** An instance finished: completed, cancelled or out of time. */
 export interface WorkflowInstanceFinished {
   readonly instanceId: string;
   readonly processKey: string;
   readonly partition: string | null;
-  readonly status: Extract<InstanceStatus, "completed" | "cancelled">;
+  readonly status: Extract<InstanceStatus, "completed" | "cancelled" | "timedOut">;
 }
 
 /** Retention deleted these finished instances with their tokens and history. */

package/core/workflow/runtime_module/http/Instances.controller.ts

@@ -5,7 +5,7 @@ import { buildListDocument } from "../../../../library/jsonapi";
 import { body, callerOf, filterOf, listQuery, optionalString, pagingOfList, param, requiredString } from "../../internal/http";
 import type { InstanceStatus } from "../model/WfInstance.model";
 import { IWorkflowDefinitions } from "../../definitions_module/services/IWorkflowDefinitions.service";
-import { IWorkflowRuntime, type InstanceSummary } from "../services/IWorkflowRuntime.service";
+import { IWorkflowRuntime, type InstanceSummary, type StartManyItem } from "../services/IWorkflowRuntime.service";
 
 /** Instances over HTTP (WORKFLOW-001 §4); rights are `workflow.instances.*` (WORKFLOW-002 §6.2). */
 @Controller("workflow/instances")
@@ -33,12 +33,23 @@ export class InstancesController {
     return Created(`${ctx.path}/${instance.id}`, instance);
   }
 
+  /** A start per element, for an action over many records (WORKFLOW-002 §5.1). */
+  @Post("bulk")
+  @Authorize(can("workflow.instances.start"))
+  async startMany(ctx: HttpContext) {
+    const request = await body(ctx);
+    const processKey = requiredString(request.processKey, "processKey", 64);
+    const partition = optionalString(request.partition, "partition") ?? null;
+    const { actor, request: from, language } = await callerOf(ctx, "workflow.instances.start", { scope: partition, process: processKey, folder: await this.definitions.folderOf(processKey) });
+    return { data: await this.runtime.startMany(processKey, { items: request.items as readonly StartManyItem[], partition, actor, request: from, language }) };
+  }
+
   @Get()
   @Authorize(can("workflow.instances.read"))
   async list(ctx: HttpContext) {
     const { access } = await callerOf(ctx);
     const list = listQuery(ctx, {
-      filter: { partition: ["eq"], processKey: ["eq"], status: ["eq"], businessKey: ["eq"], startedById: ["eq"] },
+      filter: { partition: ["eq"], processKey: ["eq"], status: ["eq"], businessKey: ["eq"], startedById: ["eq"], parentInstanceId: ["eq"] },
       sort: ["createdAt", "endedAt", "status", "processKey"],
     });
     const page = await this.runtime.instances({
@@ -49,6 +60,7 @@ export class InstancesController {
       status: filterOf(list, "status") as InstanceStatus | undefined,
       businessKey: filterOf(list, "businessKey"),
       startedById: filterOf(list, "startedById"),
+      parentInstanceId: filterOf(list, "parentInstanceId"),
     });
     return buildListDocument(page.items, list, page.total, { basePath: ctx.path });
   }

package/core/workflow/runtime_module/internal/ChildCalls.ts

@@ -0,0 +1,186 @@
+import { inputSpecOf, resolveInput } from "../../../../library/workflow";
+import type { ServiceProvider } from "../../../di";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import { WorkflowInputValidator } from "../../contracts/WorkflowInputValidator";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import type { WorkflowLanguages } from "../../internal/WorkflowLanguages";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfInstance } from "../model/WfInstance.model";
+import type { WfJob } from "../model/WfJob.model";
+import type { WfToken } from "../model/WfToken.model";
+import { dataOf, runActorOf, type StepError, type WorkflowEngine } from "./WorkflowEngine";
+
+/** How deep processes may call each other: a process that calls itself has to stop somewhere. */
+const MAX_DEPTH = 16;
+
+/**
+ * `callProcess`: the jobs between a caller and the process it called. Each runs in
+ * the transaction of one instance, so caller and child never wait for each other.
+ * The child runs on behalf of the caller's actor, in its partition.
+ */
+export class ChildCalls {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+    /** The scope: the input validator is asked here, so that the application's own registration replaces the default. */
+    private readonly services: ServiceProvider,
+    private readonly languages: WorkflowLanguages,
+  ) {}
+
+  /**
+   * A `call` job: the step starts its process. If it has called already (a retry, a
+   * repeat after a crash), it goes on with that child: a failed one is retried, so
+   * that its finished steps do not run twice. A cancelled or timed out one is called
+   * anew only when the step was entered again (`fresh`).
+   */
+  async call(job: WfJob): Promise<void> {
+    const instance = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    const token = job.tokenId === null ? undefined : await this.db.tokens.asNoTracking().firstOrDefault((t) => t.id.eq(job.tokenId!));
+    if (!instance || !token || instance.status !== "running" || token.status !== "waiting") {
+      await this.dropJob(job);
+      return;
+    }
+    const child = await this.lastChild(instance.id, token.id);
+    if (child?.status === "running" || child?.status === "pending") {
+      await this.dropJob(job);
+      return;
+    }
+    if (child?.status === "failed") {
+      const actor = runActorOf((await this.engine.graphOf(instance.versionId)).document, instance);
+      await this.engine.inStep(child.id, async (step) => {
+        await this.dropJob(job);
+        if (step.instance.status === "failed") await this.engine.retry(step, actor);
+      });
+      return;
+    }
+    if (child?.status === "completed") {
+      await this.engine.inStep(instance.id, async (step) => {
+        await this.dropJob(job);
+        await this.engine.complete(step, token.id, child.output ?? {});
+      });
+      return;
+    }
+    if (child && (job.payload as { fresh?: boolean } | null)?.fresh !== true) {
+      await this.applyEnd(job, instance.id, token.id, child);
+      return;
+    }
+    await this.start(job, instance, token);
+  }
+
+  private async start(job: WfJob, instance: WfInstance, token: WfToken): Promise<void> {
+    const graph = await this.engine.graphOf(instance.versionId);
+    const config = graph.document.nodes[token.nodeId]?.config ?? {};
+    const key = String(config.process ?? "");
+    const depth = (typeof instance.trigger?.depth === "number" ? instance.trigger.depth : 0) + 1;
+    const actor = runActorOf(graph.document, instance);
+    const refuse = (error: StepError) => this.engine.inStep(instance.id, async (step) => {
+      await this.dropJob(job);
+      const current = step.tokens.find((t) => t.id === token.id);
+      if (current?.status === "waiting" && step.instance.status === "running") await this.engine.failStep(step, current, error);
+    });
+    if (depth > MAX_DEPTH) {
+      await refuse({ code: "call_depth", message: `Processes call each other deeper than ${MAX_DEPTH} levels` });
+      return;
+    }
+    let version: VersionDetails;
+    try {
+      version = await this.definitions.latest(key);
+    } catch (error) {
+      if (!(error instanceof WorkflowError)) throw error;
+      await refuse({ code: "process_unavailable", message: error.message, reason: error.code });
+      return;
+    }
+    const start = Object.entries(version.document.nodes).find(([, node]) => node.kind === "manualStart");
+    if (!start) {
+      await refuse({ code: "process_unavailable", message: `Process ${key} has no manual start` });
+      return;
+    }
+    const input = resolveInput(config.input ?? {}, dataOf(instance, this.engine.now(), token));
+    const { value, issues } = await this.services.resolve(WorkflowInputValidator).validate(inputSpecOf(version.document), input, { purpose: "input", processKey: key, partition: instance.partition, language: this.languages.default, actor });
+    if (issues.length > 0) {
+      await refuse({ code: "invalid_input", message: `The input of process ${key} is invalid: ${issues.map((issue) => `${issue.field}: ${issue.message}`).join("; ")}` });
+      return;
+    }
+    const childGraph = this.engine.cache(version);
+    await this.engine.inStep(instance.id, async (step) => {
+      await this.dropJob(job);
+      const current = step.tokens.find((t) => t.id === token.id);
+      if (current?.status !== "waiting" || step.instance.status !== "running") return;
+      const { id } = await this.engine.begin(childGraph, start[0], {
+        input: value,
+        partition: step.instance.partition,
+        businessKey: null,
+        startedVia: "callProcess",
+        actor,
+        parent: { instanceId: step.instance.id, tokenId: current.id, depth },
+        cause: { instanceId: step.instance.id, processKey: step.instance.processKey, nodeId: current.nodeId },
+      });
+      this.engine.history(step, current, current.nodeId, "called", { process: key, version: version.version, instanceId: id });
+    });
+  }
+
+  /** A `returned` job: a child stopped. If its caller no longer waits, the job drops; a caller that is itself stopped looks again on its retry. */
+  async returned(job: WfJob): Promise<void> {
+    const { parentInstanceId, parentTokenId } = job.payload as { parentInstanceId: string; parentTokenId: string };
+    const child = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (!child) {
+      await this.dropJob(job);
+      return;
+    }
+    await this.applyEnd(job, parentInstanceId, parentTokenId, child);
+  }
+
+  private async applyEnd(job: WfJob, instanceId: string, tokenId: string, child: WfInstance): Promise<void> {
+    const done = await this.engine.inStep(instanceId, async (step) => {
+      await this.dropJob(job);
+      const token = step.tokens.find((t) => t.id === tokenId);
+      if (token?.status !== "waiting" || step.instance.status !== "running") return;
+      if (child.status === "completed") {
+        this.engine.history(step, token, token.nodeId, "returned", { instanceId: child.id });
+        await this.engine.complete(step, tokenId, child.output ?? {});
+        return;
+      }
+      const error = errorOf(child);
+      if (error) await this.engine.failStep(step, token, error);
+    });
+    if (done === null) await this.dropJob(job);
+  }
+
+  /** A `cancelChildren` job: the step that called is closed, so is what it called. */
+  async cancelChildren(job: WfJob): Promise<void> {
+    const { tokenId, reason } = job.payload as { tokenId: string; reason: string };
+    const parent = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (parent) {
+      const children = await this.db.instances.asNoTracking()
+        .where((i) => i.parentInstanceId.eq(parent.id).and(i.parentTokenId.eq(tokenId)).and(i.status.in(["pending", "running", "failed"])))
+        .toList();
+      const actor = { kind: "system" as const, id: `process:${parent.processKey}` };
+      for (const child of children) {
+        await this.engine.inStep(child.id, async (step) => {
+          if (step.instance.status === "pending" || step.instance.status === "running" || step.instance.status === "failed") await this.engine.cancel(step, reason, actor);
+        });
+      }
+    }
+    await this.dropJob(job);
+  }
+
+  /** The newest child of a step. */
+  private async lastChild(instanceId: string, tokenId: string): Promise<WfInstance | undefined> {
+    const children = await this.db.instances.asNoTracking().where((i) => i.parentInstanceId.eq(instanceId).and(i.parentTokenId.eq(tokenId))).orderBy((i) => i.id).toList();
+    return children.at(-1);
+  }
+
+  private async dropJob(job: WfJob): Promise<void> {
+    await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
+}
+
+/** The failure of a step whose child stopped otherwise than by completing; `undefined` while it runs again. */
+function errorOf(child: WfInstance): StepError | undefined {
+  const about = `Instance ${child.id} of process ${child.processKey}`;
+  if (child.status === "failed") return { code: "child_failed", message: `${about} failed: ${child.error?.message ?? ""}`, ...(child.error ? { reason: child.error.code } : {}) };
+  if (child.status === "cancelled") return { code: "child_cancelled", message: `${about} was cancelled` };
+  if (child.status === "timedOut") return { code: "child_timed_out", message: `${about} took too long`, reason: "timeout" };
+  return undefined;
+}

package/core/workflow/runtime_module/internal/GraphCache.ts

@@ -10,6 +10,8 @@ export interface ProcessGraph {
   readonly incoming: ReadonlyMap<string, readonly (readonly [string, ProcessFlow])[]>;
   /** Boundary nodes by the step they are attached to. */
   readonly boundaries: ReadonlyMap<string, readonly (readonly [string, ProcessNode])[]>;
+  /** The node an iteration begins with, by loop. */
+  readonly entries: ReadonlyMap<string, string>;
 }
 
 export function buildGraph(versionId: string, processKey: string, version: number, document: ProcessDocument): ProcessGraph {
@@ -23,7 +25,11 @@ export function buildGraph(versionId: string, processKey: string, version: numbe
   for (const [id, node] of Object.entries(document.nodes)) {
     if (node.attachedTo !== undefined) boundaries.set(node.attachedTo, [...boundaries.get(node.attachedTo) ?? [], [id, node]]);
   }
-  return { versionId, processKey, version, document, outgoing, incoming, boundaries };
+  const entries = new Map<string, string>();
+  for (const [id, node] of Object.entries(document.nodes)) {
+    if (node.parent !== undefined && node.attachedTo === undefined && !incoming.has(id)) entries.set(node.parent, id);
+  }
+  return { versionId, processKey, version, document, outgoing, incoming, boundaries, entries };
 }
 
 /** Graphs of versions in memory. Versions are immutable, so entries never go stale; the oldest go first past the limit. */

package/core/workflow/runtime_module/internal/RunLimits.ts

@@ -0,0 +1,124 @@
+import { inputSpecOf, validateForm } from "../../../../library/workflow";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfJob } from "../model/WfJob.model";
+import type { WorkflowEngine } from "./WorkflowEngine";
+
+/** How many waiting instances of a process one pass looks at. */
+const RELEASE_BATCH = 1000;
+
+/** What a whole run is limited by and what follows its failure: `runTimeout`, `maxConcurrent`, `sameKey: "queue"`, `errorHandler`. */
+export class RunLimits {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+  ) {}
+
+  /** The run's time is out. A `failed` instance waits for a person; its retry plans the job again. */
+  async runTimeout(job: WfJob): Promise<void> {
+    const done = job.instanceId === null ? null : await this.engine.inStep(job.instanceId, async (step) => {
+      await this.dropJob(job);
+      if (step.instance.status === "running") await this.engine.timeOut(step);
+      return true;
+    });
+    if (done === null) await this.dropJob(job);
+  }
+
+  /** A place may have come free: instances of the process that wait for one start, oldest first. */
+  async release(job: WfJob): Promise<void> {
+    await this.releaseProcess((job.payload as { processKey: string }).processKey);
+    await this.dropJob(job);
+  }
+
+  /** Every process with waiting instances: the safety net for what no ending instance tells, such as a new version without a limit. */
+  async releaseWaiting(): Promise<void> {
+    const rows = await this.db.instances.asNoTracking().where((i) => i.status.eq("pending")).select((i) => ({ processKey: i.processKey })).toList();
+    for (const key of new Set(rows.map((row) => row.processKey))) await this.releaseProcess(key);
+  }
+
+  private async releaseProcess(processKey: string): Promise<void> {
+    const waiting = await this.db.instances.asNoTracking().where((i) => i.processKey.eq(processKey).and(i.status.eq("pending"))).orderBy((i) => i.id).take(RELEASE_BATCH).toList();
+    if (waiting.length === 0) return;
+    // Graphs live in the Definitions store, out of reach inside the transaction; `inStep` finds them cached.
+    const graphs = new Map(await Promise.all(waiting.map(async (row) => [row.id, await this.engine.graphOf(row.versionId)] as const)));
+    await this.db.transactionScope(async () => {
+      await this.engine.lockProcess(processKey);
+      let running = await this.engine.runningOf(processKey);
+      for (const row of waiting) {
+        const { maxConcurrent, sameKey } = graphs.get(row.id)!.document.settings ?? {};
+        if (maxConcurrent !== undefined && running >= maxConcurrent) continue;
+        if (sameKey === "queue" && row.businessKey !== null && await this.keyTaken(processKey, row.businessKey, row.id)) continue;
+        const released = await this.engine.inStep(row.id, async (step) => {
+          if (step.instance.status !== "pending") return false;
+          await this.engine.release(step);
+          return true;
+        });
+        if (released === true) running += 1;
+      }
+    });
+  }
+
+  /** Whether an instance older than this one still works on the key: it goes first. */
+  private async keyTaken(processKey: string, businessKey: string, id: string): Promise<boolean> {
+    const same = await this.db.instances.asNoTracking()
+      .where((i) => i.processKey.eq(processKey).and(i.businessKey.eq(businessKey)).and(i.status.in(["pending", "running", "failed"])).and(i.id.lt(id)))
+      .count();
+    return same > 0;
+  }
+
+  /** An `errorHandler` job: the handler starts with `$event` telling what happened. What keeps it from starting goes to the history of the instance that failed. */
+  async errorHandler(job: WfJob): Promise<void> {
+    const failed = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (!failed) {
+      await this.dropJob(job);
+      return;
+    }
+    const process = (job.payload as { process: string }).process;
+    const note = (data: Record<string, unknown>) => this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      this.engine.history(step, null, null, "handler", { process, ...data });
+    });
+    let version: VersionDetails;
+    try {
+      version = await this.definitions.latest(process);
+    } catch (error) {
+      if (!(error instanceof WorkflowError)) throw error;
+      await note({ error: error.message });
+      return;
+    }
+    const start = Object.entries(version.document.nodes).find(([, node]) => node.kind === "manualStart");
+    if (!start) {
+      await note({ error: "the process has no manual start" });
+      return;
+    }
+    const { value, issues } = validateForm(inputSpecOf(version.document), {});
+    if (issues.length > 0) {
+      await note({ error: "the process needs input", issues });
+      return;
+    }
+    const graph = this.engine.cache(version);
+    // In one transaction with the job, so that a repeat of the job does not start a second handler.
+    await this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      const { id } = await this.engine.begin(graph, start[0], {
+        input: value,
+        partition: failed.partition,
+        businessKey: null,
+        startedVia: "errorHandler",
+        actor: { kind: "system", id: `process:${failed.processKey}` },
+        event: {
+          key: "workflow.instance-stopped",
+          payload: { instanceId: failed.id, processKey: failed.processKey, version: failed.version, partition: failed.partition, businessKey: failed.businessKey, status: failed.status, node: failed.error?.node ?? null, code: failed.error?.code ?? null, message: failed.error?.message ?? null },
+        },
+        cause: { instanceId: failed.id, processKey: failed.processKey },
+      });
+      this.engine.history(step, null, null, "handler", { process, instanceId: id });
+    });
+  }
+
+  private async dropJob(job: WfJob): Promise<void> {
+    await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
+}

package/core/workflow/runtime_module/internal/TimerStep.ts

@@ -2,7 +2,7 @@ import type { RuntimeDbContext } from "../model/Runtime.dbContext";
 import type { WfJob } from "../model/WfJob.model";
 import type { WorkflowEngine } from "./WorkflowEngine";
 
-/** One `timer` job: a `timerCatch` step or a boundary timer is due. A closed token drops it. */
+/** One `timer` job: a `timerCatch` step or a boundary timer is due. A closed token drops it. A `loop` job works the same way. */
 export class TimerStep {
   constructor(
     private readonly db: RuntimeDbContext,
@@ -18,4 +18,14 @@ export class TimerStep {
       });
     if (fired === null) await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
   }
+
+  async continueLoop(job: WfJob): Promise<void> {
+    const went = job.instanceId === null || job.tokenId === null
+      ? null
+      : await this.engine.inStep(job.instanceId, async (step) => {
+        await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+        return await this.engine.continueLoop(step, job.tokenId!);
+      });
+    if (went === null) await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
 }

package/core/workflow/runtime_module/internal/ToolStep.ts

@@ -4,13 +4,13 @@ import type { WorkflowToolMetadata } from "../../contracts/types";
 import type { WorkflowTools } from "../../internal/WorkflowTools";
 import type { RuntimeDbContext } from "../model/Runtime.dbContext";
 import type { WfJob } from "../model/WfJob.model";
-import { dataOf, type WorkflowEngine } from "./WorkflowEngine";
+import { dataOf, runActorOf, type WorkflowEngine } from "./WorkflowEngine";
 
 /**
  * One `tool` job: call the service task's Tool and apply the result. The Tool
  * runs outside a transaction, the result is applied in a step transaction; a
  * token closed meanwhile (a cancel) drops it. The Tool runs on behalf of who
- * started the instance, with the token id as its idempotency key: after a crash
+ * started the instance or of the process (`runAs`), with the token id as its idempotency key: after a crash
  * between the call and the step the call may repeat (TOOLS-001 §7).
  */
 /** How long a claimed job belongs to its worker; if it dies, another one takes the job after that. */
@@ -35,7 +35,7 @@ export class ToolStep {
     const graph = await this.engine.graphOf(instance.versionId);
     const node = graph.document.nodes[token.nodeId];
     const name = String(node?.config?.tool ?? "");
-    const input = resolveInput(node?.config?.input ?? {}, dataOf(instance, this.engine.now()));
+    const input = resolveInput(node?.config?.input ?? {}, dataOf(instance, this.engine.now(), token));
     const timeoutMs = stepTimeoutMs(node?.config?.timeout, graph.document);
     // A step may run longer than the lease: another worker must not take the job and call the Tool again meanwhile.
     if (timeoutMs !== undefined && timeoutMs >= LEASE_MS) {
@@ -46,7 +46,7 @@ export class ToolStep {
       ? { callId: token.id, name, status: "error", error: { code: "TOOL_NOT_FOUND", message: `Tool ${name} is not available to processes` }, metadata: {} } as AgentToolResult
       : await runner.run(name, input, {
         idempotencyKey: token.id,
-        actor: { kind: instance.startedByKind, id: instance.startedById },
+        actor: runActorOf(graph.document, instance),
         metadata: { instanceId: instance.id, processKey: instance.processKey, nodeId: token.nodeId, partition: instance.partition, test: false } satisfies WorkflowToolMetadata,
         signal,
         ...(timeoutMs === undefined ? {} : { timeoutMs }),