create-windy 0.6.11 → 0.6.14

package/template/apps/server/src/agent-workflow/scheduler-effect-result.ts

@@ -0,0 +1,205 @@
+import type { AgentWorkflowNodeRun } from "@southwind-ai/shared";
+import type { AgentWorkflowAdapterDispatchResult } from "./adapter-dispatch.js";
+import { handleScheduledNodeFailure } from "./scheduler-failure.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { releaseNodeReservation } from "./scheduler-cancellation-support.js";
+import { convergeFailedReservation } from "./scheduler-reservation-convergence.js";
+import { transitionAgentWorkflowSideEffect } from "./side-effect-state.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export function dispatchResultAt(
+  baseline: string,
+  result: AgentWorkflowAdapterDispatchResult | { readonly outcome: "pending" },
+) {
+  return monotonicAt(
+    baseline,
+    result.outcome === "committed" ? result.receipt.committedAt : undefined,
+    result.outcome === "committed" ? result.nodeResult.recordedAt : undefined,
+  );
+}
+
+export async function finishScheduledDispatch(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  commandId: string,
+  result: AgentWorkflowAdapterDispatchResult,
+  at: string,
+) {
+  snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  )!;
+  if (result.outcome === "waiting") {
+    return waitScheduledNode(
+      dependencies,
+      snapshot,
+      nodeRun,
+      result.adapter,
+      at,
+    );
+  }
+  if (result.outcome === "failed") {
+    const command = snapshot.sideEffects.find(
+      (item) => item.commandId === commandId,
+    );
+    const budget = await convergeFailedReservation({
+      service: dependencies.service,
+      command,
+      cumulativeUsage: result.cumulativeUsage,
+      safeCode: result.safeCode,
+      at,
+    });
+    if (command?.budgetReservationId && !budget.settled) {
+      throw new Error("BUDGET_RESERVATION_CONVERGENCE_PENDING");
+    }
+    await failScheduledEffect(dependencies, commandId, result.safeCode, at);
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    )!;
+    return failNode(
+      dependencies,
+      snapshot,
+      nodeRun,
+      result.safeCode,
+      at,
+      budget.settled ? undefined : result.cumulativeUsage,
+    );
+  }
+  const command = snapshot.sideEffects.find(
+    (item) => item.commandId === commandId,
+  );
+  if (result.cumulativeUsage && command?.budgetReservationId) {
+    const ledger = await dependencies.service.readBudget(snapshot.run.runId);
+    const reservation = ledger?.activeReservations.find(
+      (item) => item.reservationId === command.budgetReservationId,
+    );
+    if (!reservation) {
+      return failNode(
+        dependencies,
+        snapshot,
+        nodeRun,
+        "BUDGET_RESERVATION_MISSING",
+        at,
+      );
+    }
+    const settled = await dependencies.service.settleBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: snapshot.run.runId,
+      expectedReservationVersion: reservation.version,
+      consumed: result.cumulativeUsage,
+      usageReceiptId: result.receipt.receiptId,
+      authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+      runPolicyDigest: snapshot.run.executionPolicyDigest,
+      settledAt: at,
+    });
+    if ("safeCode" in settled) {
+      return failNode(dependencies, snapshot, nodeRun, settled.safeCode, at);
+    }
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    )!;
+  }
+  const settledByLedger = Boolean(
+    result.cumulativeUsage && command?.budgetReservationId,
+  );
+  return dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedRunVersion:
+      result.cumulativeUsage && !settledByLedger
+        ? snapshot.run.version
+        : undefined,
+    expectedSequence: snapshot.headSequence,
+    command: {
+      type: "complete",
+      receipt: result.nodeResult,
+      outputArtifacts: result.outputArtifacts,
+    },
+    cumulativeUsage: settledByLedger ? undefined : result.cumulativeUsage,
+    effectCommandId: commandId,
+    idempotencyKey: `scheduler:effect-complete:${nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+export async function failScheduledEffect(
+  dependencies: SchedulerExecutionDependencies,
+  commandId: string,
+  safeCode: string,
+  at: string,
+) {
+  const command = await dependencies.repository.getSideEffect(commandId);
+  if (
+    !command ||
+    ["failed", "cancelled", "reconciled"].includes(command.status)
+  ) {
+    return;
+  }
+  const released = await releaseNodeReservation({
+    service: dependencies.service,
+    command,
+    at,
+    reasonCode: "DELEGATION_EFFECT_FAILED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  await dependencies.repository.updateSideEffect(
+    transitionAgentWorkflowSideEffect({
+      command,
+      expectedVersion: command.version,
+      transition: { type: "fail", safeCode },
+      at: monotonicAt(at, command.updatedAt),
+    }),
+    command.version,
+  );
+}
+
+export function waitScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  adapter: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+  at: string,
+) {
+  return dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "wait", adapter },
+    idempotencyKey: `scheduler:wait:${nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+async function failNode(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: import("@southwind-ai/shared").AgentWorkflowUsage,
+) {
+  const release = await dependencies.requireRelease(snapshot.run.releaseId);
+  return handleScheduledNodeFailure(
+    dependencies,
+    snapshot,
+    release.definition.nodes.find(({ id }) => id === nodeRun.nodeId)!,
+    nodeRun,
+    safeCode,
+    at,
+    cumulativeUsage,
+  );
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(...values.map((value) => Date.parse(value ?? "") || 0)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-execution-convergence.ts

@@ -0,0 +1,174 @@
+import type {
+  AgentWorkflowNodeRun,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type {
+  AgentWorkflowAdapterDispatchResult,
+  AgentWorkflowNodeAdapter,
+} from "./adapter-dispatch.js";
+import { deadlineCompletionAt } from "./scheduler-deadline.js";
+import {
+  finishScheduledDispatch,
+  waitScheduledNode,
+} from "./scheduler-effect-result.js";
+import type {
+  executeScheduledNode,
+  SchedulerExecutionDependencies,
+} from "./scheduler-execution.js";
+import { handleScheduledNodeFailure } from "./scheduler-failure.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import { releaseActiveNodeReservations } from "./scheduler-cancellation-support.js";
+
+export async function currentScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+) {
+  const snapshot = await dependencies.requireSnapshot(runId);
+  const current = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === nodeRun.nodeRunId,
+  );
+  if (!current) throw new Error("Node Run 不存在");
+  return { snapshot, nodeRun: current };
+}
+
+export async function convergeWait(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  adapter: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+  at: string,
+) {
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return waitScheduledNode(
+    dependencies,
+    latest.snapshot,
+    latest.nodeRun,
+    adapter,
+    at,
+  );
+}
+
+export async function convergeFailure(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const latest = await currentScheduledNode(dependencies, input.runId, nodeRun);
+  return handleScheduledNodeFailure(
+    dependencies,
+    latest.snapshot,
+    input.node,
+    latest.nodeRun,
+    safeCode,
+    at,
+    cumulativeUsage,
+  );
+}
+
+export async function convergeTimeout(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const released = await releaseActiveNodeReservations({
+    service: dependencies.service,
+    runId,
+    nodeRunId: nodeRun.nodeRunId,
+    at,
+    reasonCode: "NODE_DEADLINE_EXCEEDED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return dependencies.service.transitionNode({
+    runId,
+    nodeRunId: latest.nodeRun.nodeRunId,
+    expectedNodeVersion: latest.nodeRun.version,
+    expectedRunVersion: cumulativeUsage
+      ? latest.snapshot.run.version
+      : undefined,
+    expectedSequence: latest.snapshot.headSequence,
+    command: { type: "timeout" },
+    cumulativeUsage,
+    idempotencyKey: `scheduler:timeout:${latest.nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+export async function convergeDispatch(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  commandId: string,
+  result: AgentWorkflowAdapterDispatchResult,
+  at: string,
+) {
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return finishScheduledDispatch(
+    dependencies,
+    latest.snapshot,
+    latest.nodeRun,
+    commandId,
+    result,
+    at,
+  );
+}
+
+export async function convergeTimedOutPreparation(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+  snapshot: Awaited<
+    ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+  >,
+  running: AgentWorkflowNodeRun,
+  adapter: AgentWorkflowNodeAdapter,
+  deadlineAt: string,
+) {
+  let cancellation:
+    | Awaited<ReturnType<NonNullable<AgentWorkflowNodeAdapter["cancel"]>>>
+    | undefined;
+  if (!adapter.cancel) cancellation = { cancelled: true };
+  try {
+    if (adapter.cancel) {
+      cancellation = await adapter.cancel({
+        context: schedulerAdapterContext({
+          snapshot,
+          release: input.release,
+          nodeRun: running,
+          node: input.node,
+          signal: AbortSignal.timeout(5_000),
+        }),
+      });
+    }
+  } catch {
+    cancellation = undefined;
+  }
+  const at = deadlineCompletionAt(deadlineAt, running.updatedAt);
+  return dependencies.serializeMutation(() =>
+    cancellation?.cancelled
+      ? convergeTimeout(
+          dependencies,
+          input.runId,
+          running,
+          at,
+          cancellation.cumulativeUsage,
+        )
+      : convergeWait(
+          dependencies,
+          input.runId,
+          running,
+          requiredAdapter(running),
+          at,
+        ),
+  );
+}
+
+function requiredAdapter(nodeRun: AgentWorkflowNodeRun) {
+  if (!nodeRun.adapter) throw new Error("Node Run Adapter coordinate 缺失");
+  return nodeRun.adapter;
+}

package/template/apps/server/src/agent-workflow/scheduler-execution.ts

@@ -0,0 +1,211 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type {
+  AgentWorkflowNodeAdapter,
+  AgentWorkflowNodeAdapterRegistry,
+} from "./adapter-dispatch.js";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { WorkflowEffectCommandFencePort } from "./effect-command-fence.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  awaitScheduledNode,
+  SchedulerDeadlineExceeded,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+import {
+  convergeFailure,
+  convergeTimedOutPreparation,
+  convergeWait,
+  currentScheduledNode,
+} from "./scheduler-execution-convergence.js";
+import { executePreparedSideEffect } from "./scheduler-side-effect-execution.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export interface SchedulerExecutionDependencies {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly effectFence: WorkflowEffectCommandFencePort;
+  readonly adapters: AgentWorkflowNodeAdapterRegistry;
+  readonly requireSnapshot: (runId: string) => Promise<Snapshot>;
+  readonly requireRelease: (releaseId: string) => Promise<AgentWorkflowRelease>;
+  readonly insertReady: (
+    snapshot: Snapshot,
+    nodes: readonly AgentWorkflowNode[],
+    source: AgentWorkflowNodeRun,
+    at: string,
+  ) => Promise<unknown>;
+  readonly serializeMutation: <T>(mutation: () => Promise<T>) => Promise<T>;
+}
+
+export async function executeScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly node: AgentWorkflowNode;
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+) {
+  const running = await claimScheduledNode(dependencies, input);
+  return executeClaimedScheduledNode(dependencies, {
+    ...input,
+    nodeRun: running,
+  });
+}
+
+export async function claimScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  await dependencies.service.transitionNode({
+    runId: input.runId,
+    nodeRunId: input.nodeRun.nodeRunId,
+    expectedNodeVersion: input.nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "start" },
+    idempotencyKey: `scheduler:start:${input.nodeRun.nodeRunId}`,
+    at: input.at,
+  });
+  const claimed = await dependencies.requireSnapshot(input.runId);
+  return claimed.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  )!;
+}
+
+export async function executeClaimedScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  const running = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  )!;
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: snapshot.run,
+    node: input.node,
+    nodeRun: running,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot,
+    release: input.release,
+    nodeRun: running,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  const adapter = dependencies.adapters.require(input.node.type);
+  let preparation: Awaited<ReturnType<AgentWorkflowNodeAdapter["prepare"]>>;
+  try {
+    preparation = await awaitScheduledNode(adapter.prepare(context), deadline);
+  } catch (error) {
+    if (input.signal.aborted) return { outcome: "aborted" as const };
+    if (error instanceof SchedulerDeadlineExceeded) {
+      return convergeTimedOutPreparation(
+        dependencies,
+        input,
+        snapshot,
+        running,
+        adapter,
+        deadlineAt,
+      );
+    }
+    return dependencies.serializeMutation(() =>
+      convergeFailure(
+        dependencies,
+        input,
+        running,
+        "ADAPTER_PREPARATION_FAILED",
+        monotonicAt(input.at, running.updatedAt),
+      ),
+    );
+  }
+  const preparedAt = monotonicAt(
+    input.at,
+    running.updatedAt,
+    preparation.outcome === "direct"
+      ? preparation.result.recordedAt
+      : undefined,
+  );
+  if (preparation.outcome === "direct") {
+    return dependencies.serializeMutation(async () => {
+      const latest = await currentScheduledNode(
+        dependencies,
+        input.runId,
+        running,
+      );
+      return dependencies.service.transitionNode({
+        runId: input.runId,
+        nodeRunId: latest.nodeRun.nodeRunId,
+        expectedNodeVersion: latest.nodeRun.version,
+        expectedRunVersion: preparation.cumulativeUsage
+          ? latest.snapshot.run.version
+          : undefined,
+        expectedSequence: latest.snapshot.headSequence,
+        command: {
+          type: "complete",
+          receipt: preparation.result,
+          outputArtifacts: preparation.outputArtifacts,
+        },
+        cumulativeUsage: preparation.cumulativeUsage,
+        idempotencyKey: `scheduler:complete:${latest.nodeRun.nodeRunId}`,
+        at: preparedAt,
+      });
+    });
+  }
+  if (preparation.outcome === "waiting") {
+    return dependencies.serializeMutation(() =>
+      convergeWait(
+        dependencies,
+        input.runId,
+        running,
+        preparation.adapter,
+        preparedAt,
+      ),
+    );
+  }
+  if (preparation.outcome === "failed") {
+    return dependencies.serializeMutation(() =>
+      convergeFailure(
+        dependencies,
+        input,
+        running,
+        preparation.safeCode,
+        preparedAt,
+        preparation.cumulativeUsage,
+      ),
+    );
+  }
+  return executePreparedSideEffect(dependencies, {
+    input,
+    snapshot,
+    running,
+    context,
+    adapter,
+    preparation,
+    preparedAt,
+    deadlineAt,
+    deadline,
+  });
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-failure.ts

@@ -0,0 +1,102 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failureDecision } from "./scheduler-support.js";
+import { releaseActiveNodeReservations } from "./scheduler-cancellation-support.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export async function handleScheduledNodeFailure(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  node: AgentWorkflowNode,
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const released = await releaseActiveNodeReservations({
+    service: dependencies.service,
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    at,
+    reasonCode: "NODE_EXECUTION_FAILED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  )!;
+  const decision = [
+    "MODEL_USAGE_UNMEASURED",
+    "USAGE_UNKNOWN_OR_EXCEEDED",
+  ].includes(safeCode)
+    ? ({ type: "fail-run" } as const)
+    : failureDecision(node, nodeRun);
+  if (decision.type === "skip") {
+    return dependencies.service.transitionNode({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedSequence: snapshot.headSequence,
+      command: { type: "skip" },
+      expectedRunVersion: cumulativeUsage ? snapshot.run.version : undefined,
+      cumulativeUsage,
+      idempotencyKey: `scheduler:skip:${nodeRun.nodeRunId}`,
+      at,
+    });
+  }
+  const failed = await dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "fail", safeCode },
+    expectedRunVersion: cumulativeUsage ? snapshot.run.version : undefined,
+    cumulativeUsage,
+    idempotencyKey: `scheduler:fail:${nodeRun.nodeRunId}`,
+    at,
+  });
+  if (decision.type === "fallback") {
+    const latest = await dependencies.requireSnapshot(snapshot.run.runId);
+    const release = await dependencies.requireRelease(snapshot.run.releaseId);
+    const fallback = release.definition.nodes.find(
+      ({ id }) => id === decision.nodeId,
+    );
+    if (!fallback) throw new Error("Fallback Node 不存在");
+    return dependencies.insertReady(latest, [fallback], nodeRun, at);
+  }
+  if (decision.type === "retry") {
+    const scheduled = await dependencies.service.scheduleRetry({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version + 1,
+      expectedSequence: failed.record.headSequence,
+      safeCode,
+      idempotencyKey: `scheduler:retry:${nodeRun.nodeRunId}:${nodeRun.attempt + 1}`,
+      at,
+    });
+    return {
+      outcome: "retry-after" as const,
+      scheduled,
+      retryAt: new Date(
+        Date.parse(at) + decision.backoffSeconds * 1_000,
+      ).toISOString(),
+    };
+  }
+  const latest = await dependencies.requireSnapshot(snapshot.run.runId);
+  return dependencies.service.transitionRun({
+    runId: snapshot.run.runId,
+    expectedVersion: latest.run.version,
+    expectedSequence: failed.record.headSequence,
+    command: { type: "fail", safeCode },
+    idempotencyKey: `scheduler:fail-run:${nodeRun.nodeRunId}`,
+    at,
+  });
+}