bazis 0.131.0 → 0.133.0

package/core/workflow/runtime_module/internal/RunLimits.ts

@@ -0,0 +1,124 @@
+import { inputSpecOf, validateForm } from "../../../../library/workflow";
+import { WorkflowError } from "../../contracts/WorkflowError";
+import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
+import type { RuntimeDbContext } from "../model/Runtime.dbContext";
+import type { WfJob } from "../model/WfJob.model";
+import type { WorkflowEngine } from "./WorkflowEngine";
+
+/** How many waiting instances of a process one pass looks at. */
+const RELEASE_BATCH = 1000;
+
+/** What a whole run is limited by and what follows its failure: `runTimeout`, `maxConcurrent`, `sameKey: "queue"`, `errorHandler`. */
+export class RunLimits {
+  constructor(
+    private readonly db: RuntimeDbContext,
+    private readonly engine: WorkflowEngine,
+    private readonly definitions: IWorkflowDefinitions,
+  ) {}
+
+  /** The run's time is out. A `failed` instance waits for a person; its retry plans the job again. */
+  async runTimeout(job: WfJob): Promise<void> {
+    const done = job.instanceId === null ? null : await this.engine.inStep(job.instanceId, async (step) => {
+      await this.dropJob(job);
+      if (step.instance.status === "running") await this.engine.timeOut(step);
+      return true;
+    });
+    if (done === null) await this.dropJob(job);
+  }
+
+  /** A place may have come free: instances of the process that wait for one start, oldest first. */
+  async release(job: WfJob): Promise<void> {
+    await this.releaseProcess((job.payload as { processKey: string }).processKey);
+    await this.dropJob(job);
+  }
+
+  /** Every process with waiting instances: the safety net for what no ending instance tells, such as a new version without a limit. */
+  async releaseWaiting(): Promise<void> {
+    const rows = await this.db.instances.asNoTracking().where((i) => i.status.eq("pending")).select((i) => ({ processKey: i.processKey })).toList();
+    for (const key of new Set(rows.map((row) => row.processKey))) await this.releaseProcess(key);
+  }
+
+  private async releaseProcess(processKey: string): Promise<void> {
+    const waiting = await this.db.instances.asNoTracking().where((i) => i.processKey.eq(processKey).and(i.status.eq("pending"))).orderBy((i) => i.id).take(RELEASE_BATCH).toList();
+    if (waiting.length === 0) return;
+    // Graphs live in the Definitions store, out of reach inside the transaction; `inStep` finds them cached.
+    const graphs = new Map(await Promise.all(waiting.map(async (row) => [row.id, await this.engine.graphOf(row.versionId)] as const)));
+    await this.db.transactionScope(async () => {
+      await this.engine.lockProcess(processKey);
+      let running = await this.engine.runningOf(processKey);
+      for (const row of waiting) {
+        const { maxConcurrent, sameKey } = graphs.get(row.id)!.document.settings ?? {};
+        if (maxConcurrent !== undefined && running >= maxConcurrent) continue;
+        if (sameKey === "queue" && row.businessKey !== null && await this.keyTaken(processKey, row.businessKey, row.id)) continue;
+        const released = await this.engine.inStep(row.id, async (step) => {
+          if (step.instance.status !== "pending") return false;
+          await this.engine.release(step);
+          return true;
+        });
+        if (released === true) running += 1;
+      }
+    });
+  }
+
+  /** Whether an instance older than this one still works on the key: it goes first. */
+  private async keyTaken(processKey: string, businessKey: string, id: string): Promise<boolean> {
+    const same = await this.db.instances.asNoTracking()
+      .where((i) => i.processKey.eq(processKey).and(i.businessKey.eq(businessKey)).and(i.status.in(["pending", "running", "failed"])).and(i.id.lt(id)))
+      .count();
+    return same > 0;
+  }
+
+  /** An `errorHandler` job: the handler starts with `$event` telling what happened. What keeps it from starting goes to the history of the instance that failed. */
+  async errorHandler(job: WfJob): Promise<void> {
+    const failed = job.instanceId === null ? undefined : await this.db.instances.asNoTracking().firstOrDefault((i) => i.id.eq(job.instanceId!));
+    if (!failed) {
+      await this.dropJob(job);
+      return;
+    }
+    const process = (job.payload as { process: string }).process;
+    const note = (data: Record<string, unknown>) => this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      this.engine.history(step, null, null, "handler", { process, ...data });
+    });
+    let version: VersionDetails;
+    try {
+      version = await this.definitions.latest(process);
+    } catch (error) {
+      if (!(error instanceof WorkflowError)) throw error;
+      await note({ error: error.message });
+      return;
+    }
+    const start = Object.entries(version.document.nodes).find(([, node]) => node.kind === "manualStart");
+    if (!start) {
+      await note({ error: "the process has no manual start" });
+      return;
+    }
+    const { value, issues } = validateForm(inputSpecOf(version.document), {});
+    if (issues.length > 0) {
+      await note({ error: "the process needs input", issues });
+      return;
+    }
+    const graph = this.engine.cache(version);
+    // In one transaction with the job, so that a repeat of the job does not start a second handler.
+    await this.engine.inStep(failed.id, async (step) => {
+      await this.dropJob(job);
+      const { id } = await this.engine.begin(graph, start[0], {
+        input: value,
+        partition: failed.partition,
+        businessKey: null,
+        startedVia: "errorHandler",
+        actor: { kind: "system", id: `process:${failed.processKey}` },
+        event: {
+          key: "workflow.instance-stopped",
+          payload: { instanceId: failed.id, processKey: failed.processKey, version: failed.version, partition: failed.partition, businessKey: failed.businessKey, status: failed.status, node: failed.error?.node ?? null, code: failed.error?.code ?? null, message: failed.error?.message ?? null },
+        },
+        cause: { instanceId: failed.id, processKey: failed.processKey },
+      });
+      this.engine.history(step, null, null, "handler", { process, instanceId: id });
+    });
+  }
+
+  private async dropJob(job: WfJob): Promise<void> {
+    await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
+}

package/core/workflow/runtime_module/internal/Schedules.ts

@@ -1,4 +1,4 @@
-import { parseCron } from "../../../../library/workflow";
+import { inputSpecOf, parseCron, validateForm } from "../../../../library/workflow";
 import type { IWorkflowDefinitions, VersionDetails } from "../../definitions_module/services/IWorkflowDefinitions.service";
 import type { RuntimeDbContext } from "../model/Runtime.dbContext";
 import { WfJob } from "../model/WfJob.model";
@@ -65,7 +65,7 @@ export class Schedules {
       else await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeUpdate({ dueAt: next, lockedUntil: null, attempts: 0, lastError: null });
       for (const [nodeId, cron] of scheduleNodes(latest)) {
         if (!cron.matches(due)) continue;
-        await this.engine.begin(graph, nodeId, { input: {}, partition: null, businessKey: null, startedVia: "schedule", actor: SCHEDULE_ACTOR });
+        await this.engine.begin(graph, nodeId, { input: validateForm(inputSpecOf(latest.document), {}).value, partition: null, businessKey: null, startedVia: "schedule", actor: SCHEDULE_ACTOR });
       }
     });
   }

package/core/workflow/runtime_module/internal/TimerStep.ts

@@ -2,7 +2,7 @@ import type { RuntimeDbContext } from "../model/Runtime.dbContext";
 import type { WfJob } from "../model/WfJob.model";
 import type { WorkflowEngine } from "./WorkflowEngine";
 
-/** One `timer` job: a `timerCatch` step or a boundary timer is due. A closed token drops it. */
+/** One `timer` job: a `timerCatch` step or a boundary timer is due. A closed token drops it. A `loop` job works the same way. */
 export class TimerStep {
   constructor(
     private readonly db: RuntimeDbContext,
@@ -18,4 +18,14 @@ export class TimerStep {
       });
     if (fired === null) await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
   }
+
+  async continueLoop(job: WfJob): Promise<void> {
+    const went = job.instanceId === null || job.tokenId === null
+      ? null
+      : await this.engine.inStep(job.instanceId, async (step) => {
+        await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+        return await this.engine.continueLoop(step, job.tokenId!);
+      });
+    if (went === null) await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeDelete();
+  }
 }

package/core/workflow/runtime_module/internal/ToolStep.ts

@@ -1,18 +1,23 @@
-import type { AgentToolResult } from "../../../agent";
-import { durationMs, resolveInput } from "../../../../library/workflow";
+import type { AgentSchemaContract, AgentToolResult } from "../../../agent";
+import { durationMs, maskByMapping, maskBySchema, masksPersonal, personalPaths, resolveInput, type ProcessDocument } from "../../../../library/workflow";
 import type { WorkflowToolMetadata } from "../../contracts/types";
 import type { WorkflowTools } from "../../internal/WorkflowTools";
 import type { RuntimeDbContext } from "../model/Runtime.dbContext";
 import type { WfJob } from "../model/WfJob.model";
-import { dataOf, type WorkflowEngine } from "./WorkflowEngine";
+import { dataOf, runActorOf, type WorkflowEngine } from "./WorkflowEngine";
 
 /**
  * One `tool` job: call the service task's Tool and apply the result. The Tool
  * runs outside a transaction, the result is applied in a step transaction; a
  * token closed meanwhile (a cancel) drops it. The Tool runs on behalf of who
- * started the instance, with the token id as its idempotency key: after a crash
+ * started the instance or of the process (`runAs`), with the token id as its idempotency key: after a crash
  * between the call and the step the call may repeat (TOOLS-001 §7).
  */
+/** How long a claimed job belongs to its worker; if it dies, another one takes the job after that. */
+export const LEASE_MS = 5 * 60_000;
+
+const TIMEOUT_CODES: ReadonlySet<string> = new Set(["TOOL_TIMEOUT", "TOOL_TIMEOUT_OUTCOME_UNKNOWN"]);
+
 export class ToolStep {
   constructor(
     private readonly db: RuntimeDbContext,
@@ -30,21 +35,29 @@ export class ToolStep {
     const graph = await this.engine.graphOf(instance.versionId);
     const node = graph.document.nodes[token.nodeId];
     const name = String(node?.config?.tool ?? "");
-    const input = resolveInput(node?.config?.input ?? {}, dataOf(instance));
+    const input = resolveInput(node?.config?.input ?? {}, dataOf(instance, this.engine.now(), token));
+    const timeoutMs = stepTimeoutMs(node?.config?.timeout, graph.document);
+    // A step may run longer than the lease: another worker must not take the job and call the Tool again meanwhile.
+    if (timeoutMs !== undefined && timeoutMs >= LEASE_MS) {
+      await this.db.jobs.asNoTracking().where((j) => j.id.eq(job.id)).executeUpdate({ lockedUntil: new Date(this.engine.now().getTime() + timeoutMs + LEASE_MS) });
+    }
     const runner = this.tools.bound;
     const result: AgentToolResult = runner === undefined
       ? { callId: token.id, name, status: "error", error: { code: "TOOL_NOT_FOUND", message: `Tool ${name} is not available to processes` }, metadata: {} } as AgentToolResult
       : await runner.run(name, input, {
         idempotencyKey: token.id,
-        actor: { kind: instance.startedByKind, id: instance.startedById },
+        actor: runActorOf(graph.document, instance),
         metadata: { instanceId: instance.id, processKey: instance.processKey, nodeId: token.nodeId, partition: instance.partition, test: false } satisfies WorkflowToolMetadata,
         signal,
+        ...(timeoutMs === undefined ? {} : { timeoutMs }),
       });
 
-    const reason = reasonOf(result);
+    const timedOut = TIMEOUT_CODES.has(result.error?.code ?? "");
+    const reason = timedOut ? "timeout" : reasonOf(result);
+    const payloads = graph.document.settings?.log?.toolPayloads === true ? this.payloads(graph.document, name, node?.config?.input, input, result) : {};
     await this.engine.inStep(instance.id, async (step) => {
       const current = step.tokens.find((t) => t.id === token.id);
-      const retryAt = current?.status === "waiting" && step.instance.status === "running" && result.status === "error" && reason === undefined
+      const retryAt = current?.status === "waiting" && step.instance.status === "running" && result.status === "error" && (reason === undefined || timedOut)
         ? nextAttempt(node?.config?.retry, job.attempts, this.engine.now())
         : undefined;
       if (retryAt !== undefined) {
@@ -59,6 +72,7 @@ export class ToolStep {
         status: result.status,
         attempt: job.attempts,
         durationMs: result.durationMs ?? null,
+        ...payloads,
         ...(result.error ? { error: { code: result.error.code ?? null, message: result.error.message, ...(reason === undefined ? {} : { reason }) } } : {}),
         ...(retryAt === undefined ? {} : { retryAt: retryAt.toISOString() }),
       });
@@ -74,6 +88,15 @@ export class ToolStep {
       }
     });
   }
+
+  /** The input and output for the history under `log.toolPayloads`; personal data is masked unless `log.maskPersonal` is `false`. */
+  private payloads(document: ProcessDocument, name: string, mapping: unknown, input: unknown, result: AgentToolResult): Record<string, unknown> {
+    if (!masksPersonal(document)) return { input, output: result.output ?? null };
+    const contract = this.tools.bound?.describe(name);
+    const schemaOf = (side: AgentSchemaContract | undefined) => (side?.kind === "json-schema" ? side.schema : undefined);
+    const masked = maskByMapping(mapping ?? {}, maskBySchema(input, schemaOf(contract?.input)), personalPaths(document));
+    return { input: masked, output: maskBySchema(result.output ?? null, schemaOf(contract?.output)) };
+  }
 }
 
 /** The business reason of a `ToolError`, if the Tool gave one. */
@@ -83,15 +106,23 @@ function reasonOf(result: AgentToolResult): string | undefined {
   return typeof reason === "string" ? reason : undefined;
 }
 
+/** How long a step's Tool may run: its `timeout`, else `settings.stepTimeout`; `undefined` leaves the Tool's own limit. */
+function stepTimeoutMs(timeout: unknown, document: ProcessDocument): number | undefined {
+  const value = typeof timeout === "string" ? timeout : document.settings?.stepTimeout;
+  return value === undefined ? undefined : durationMs(value);
+}
+
 /**
  * When to try a failed call again under the node's `retry` policy, or
- * `undefined` when the attempts are used up. The pause doubles from `backoff`
- * (10 seconds by default) and stops growing at a day.
+ * `undefined` when the attempts are used up. The pause starts at `backoff` (10
+ * seconds by default), grows by `factor` (2 by default) unless `growth` is
+ * `fixed`, and stops growing at a day.
  */
 export function nextAttempt(policy: unknown, attempts: number, now: Date): Date | undefined {
   if (typeof policy !== "object" || policy === null) return undefined;
-  const { maxAttempts, backoff } = policy as { maxAttempts?: unknown; backoff?: unknown };
+  const { maxAttempts, backoff, growth, factor } = policy as { maxAttempts?: unknown; backoff?: unknown; growth?: unknown; factor?: unknown };
   if (typeof maxAttempts !== "number" || attempts >= maxAttempts) return undefined;
   const base = typeof backoff === "string" ? durationMs(backoff) ?? 10_000 : 10_000;
-  return new Date(now.getTime() + Math.min(base * 2 ** Math.max(attempts - 1, 0), 86_400_000));
+  const scale = growth === "fixed" ? 1 : (typeof factor === "number" ? factor : 2) ** Math.max(attempts - 1, 0);
+  return new Date(now.getTime() + Math.min(base * scale, 86_400_000));
 }