create-windy 0.6.12 → 0.6.15

package/template/apps/server/src/agent-workflow/scheduler-node-timeout.ts

@@ -0,0 +1,139 @@
+import type {
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import {
+  deadlineCompletionAt,
+  scheduledNodeDeadlineAt,
+} from "./scheduler-deadline.js";
+import { convergeTimeout } from "./scheduler-execution-convergence.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failScheduledEffect } from "./scheduler-effect-result.js";
+import { recoverPersistedEffect } from "./scheduler-recovery-effect.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+
+export function firstOverdueNode(input: {
+  readonly release: AgentWorkflowRelease;
+  readonly snapshot: Awaited<
+    ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+  >;
+  readonly at: string;
+}) {
+  return input.snapshot.nodeRuns.find((nodeRun) => {
+    if (
+      !nodeRun.startedAt ||
+      !["running", "waiting"].includes(nodeRun.status)
+    ) {
+      return false;
+    }
+    const node = input.release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    return Boolean(
+      node &&
+      Date.parse(input.at) >=
+        Date.parse(
+          scheduledNodeDeadlineAt({
+            run: input.snapshot.run,
+            node,
+            nodeRun,
+          }),
+        ),
+    );
+  });
+}
+
+export async function expireScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+) {
+  let snapshot = await dependencies.requireSnapshot(input.runId);
+  let nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun || !["running", "waiting"].includes(nodeRun.status)) {
+    return { outcome: "already-converged" as const };
+  }
+  const node = input.release.definition.nodes.find(
+    ({ id }) => id === nodeRun!.nodeId,
+  );
+  if (!node) throw new Error("Node Run 不属于 immutable Release");
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: snapshot.run,
+    node,
+    nodeRun,
+  });
+  const at = deadlineCompletionAt(deadlineAt, nodeRun.updatedAt);
+  const command = snapshot.sideEffects.find(
+    (effect) => effect.nodeRunId === nodeRun!.nodeRunId,
+  );
+  if (command && !["failed", "cancelled"].includes(command.status)) {
+    const recovered = await recoverPersistedEffect(dependencies, {
+      snapshot,
+      release: input.release,
+      node,
+      nodeRun,
+      command,
+      authorizationFence: input.authorizationFence,
+      at,
+      signal: AbortSignal.timeout(5_000),
+    });
+    if (recovered.outcome === "completed") return recovered;
+    if (recovered.outcome === "failed" && !recovered.policyEligible) {
+      return recovered;
+    }
+  }
+  snapshot = await dependencies.requireSnapshot(input.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun || !["running", "waiting"].includes(nodeRun.status)) {
+    return { outcome: "already-converged" as const };
+  }
+  const latestCommand = snapshot.sideEffects.find(
+    (effect) => effect.nodeRunId === nodeRun!.nodeRunId,
+  );
+  const adapter = dependencies.adapters.require(node.type);
+  let cancelled = latestCommand?.status === "prepared";
+  let cumulativeUsage: AgentWorkflowUsage | undefined;
+  if (!latestCommand && !adapter.cancel) cancelled = true;
+  if (!cancelled) {
+    try {
+      const result = await adapter.cancel?.({
+        context: schedulerAdapterContext({
+          snapshot,
+          release: input.release,
+          nodeRun,
+          node,
+          signal: AbortSignal.timeout(5_000),
+        }),
+        command: latestCommand,
+      });
+      cancelled = Boolean(result?.cancelled);
+      cumulativeUsage = result?.cumulativeUsage;
+    } catch {
+      cancelled = false;
+    }
+  }
+  if (!cancelled) return { outcome: "timeout-reconciliation-pending" as const };
+  if (latestCommand) {
+    await failScheduledEffect(
+      dependencies,
+      latestCommand.commandId,
+      "NODE_DEADLINE_EXCEEDED",
+      at,
+    );
+  }
+  return dependencies.serializeMutation(() =>
+    convergeTimeout(dependencies, input.runId, nodeRun!, at, cumulativeUsage),
+  );
+}

package/template/apps/server/src/agent-workflow/scheduler-parallel-dispatch.ts

@@ -0,0 +1,99 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import {
+  claimScheduledNode,
+  executeClaimedScheduledNode,
+  type SchedulerExecutionDependencies,
+} from "./scheduler-execution.js";
+
+export async function dispatchReadyNodeBatch(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly ready: readonly AgentWorkflowNodeRun[];
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+    readonly maxParallelism: number;
+  },
+) {
+  const initial = await dependencies.requireSnapshot(input.runId);
+  const available = Math.max(
+    0,
+    input.maxParallelism - activeExecutionCount(initial.nodeRuns),
+  );
+  if (!available) return { outcome: "parallelism-blocked" as const };
+  const candidates = [...input.ready]
+    .sort((left, right) =>
+      `${left.createdAt}:${left.nodeRunId}`.localeCompare(
+        `${right.createdAt}:${right.nodeRunId}`,
+      ),
+    )
+    .slice(0, available);
+  const claimed: Array<{
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly node: AgentWorkflowNode;
+  }> = [];
+  for (const candidate of candidates) {
+    if (input.signal.aborted) break;
+    const snapshot = await dependencies.requireSnapshot(input.runId);
+    const current = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === candidate.nodeRunId,
+    );
+    if (!current || current.status !== "ready") continue;
+    if (activeExecutionCount(snapshot.nodeRuns) >= input.maxParallelism) break;
+    const node = input.release.definition.nodes.find(
+      ({ id }) => id === current.nodeId,
+    );
+    if (!node) throw new Error("Ready Node 不属于 immutable Release");
+    try {
+      const nodeRun = await claimScheduledNode(dependencies, {
+        ...input,
+        node,
+        nodeRun: current,
+      });
+      claimed.push({ nodeRun, node });
+    } catch (error) {
+      const latest = await dependencies.requireSnapshot(input.runId);
+      const latestNode = latest.nodeRuns.find(
+        ({ nodeRunId }) => nodeRunId === current.nodeRunId,
+      );
+      const stateChanged =
+        latest.headSequence !== snapshot.headSequence ||
+        latestNode?.version !== current.version ||
+        latestNode?.status !== "ready";
+      if (!stateChanged) throw error;
+    }
+  }
+  if (!claimed.length) return { outcome: "parallelism-blocked" as const };
+  const settled = await Promise.allSettled(
+    claimed.map(({ nodeRun, node }) =>
+      executeClaimedScheduledNode(dependencies, {
+        ...input,
+        nodeRun,
+        node,
+      }),
+    ),
+  );
+  return {
+    outcome: "batch-dispatched" as const,
+    claimed: claimed.map(({ nodeRun }) => nodeRun.nodeRunId),
+    rejected: settled.flatMap((item, index) =>
+      item.status === "rejected" ? [claimed[index]!.nodeRun.nodeRunId] : [],
+    ),
+    results: settled.flatMap((item) =>
+      item.status === "fulfilled" ? [item.value] : [],
+    ),
+  };
+}
+
+function activeExecutionCount(nodeRuns: readonly AgentWorkflowNodeRun[]) {
+  return nodeRuns.filter(({ status }) =>
+    ["running", "waiting"].includes(status),
+  ).length;
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-activation.ts

@@ -0,0 +1,63 @@
+import type { AgentWorkflowRelease } from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export async function activatePendingWorkflowNode(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly service: AgentWorkflowService;
+  readonly at: string;
+}) {
+  const entry = input.snapshot.nodeRuns.find(
+    ({ nodeId, status, previousNodeRunId }) =>
+      nodeId === input.release.definition.entryNodeId &&
+      status === "pending" &&
+      !previousNodeRunId,
+  );
+  if (entry) {
+    return input.service.transitionNode({
+      runId: input.snapshot.run.runId,
+      nodeRunId: entry.nodeRunId,
+      expectedNodeVersion: entry.version,
+      expectedSequence: input.snapshot.headSequence,
+      command: { type: "ready" },
+      idempotencyKey: `scheduler:entry-ready:${entry.nodeRunId}`,
+      at: input.at,
+    });
+  }
+  const retry = input.snapshot.nodeRuns.find(
+    ({ status, previousNodeRunId }) =>
+      status === "pending" && previousNodeRunId,
+  );
+  if (!retry) return undefined;
+  const previous = input.snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === retry.previousNodeRunId,
+  );
+  const node = input.release.definition.nodes.find(
+    ({ id }) => id === retry.nodeId,
+  );
+  if (!previous?.finishedAt || !node) {
+    throw new Error("Retry durable backoff coordinate 不完整");
+  }
+  const backoff =
+    node.failurePolicy.mode === "retry" ? node.failurePolicy.backoffSeconds : 0;
+  if (
+    Date.parse(input.at) <
+    Date.parse(previous.finishedAt) + backoff * 1_000
+  ) {
+    return { outcome: "backoff" as const };
+  }
+  return input.service.transitionNode({
+    runId: input.snapshot.run.runId,
+    nodeRunId: retry.nodeRunId,
+    expectedNodeVersion: retry.version,
+    expectedSequence: input.snapshot.headSequence,
+    command: { type: "ready" },
+    idempotencyKey: `scheduler:retry-ready:${retry.nodeRunId}`,
+    at: input.at,
+  });
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-budget.ts

@@ -0,0 +1,41 @@
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+
+export async function releaseRecoveryReservation(
+  dependencies: SchedulerExecutionDependencies,
+  request: { readonly runId: string; readonly at: string },
+  reservationId: string,
+) {
+  const snapshot = await dependencies.requireSnapshot(request.runId);
+  const ledger = await dependencies.service.readBudget(request.runId);
+  const reservation = ledger?.activeReservations.find(
+    (item) => item.reservationId === reservationId,
+  );
+  if (!reservation) return true;
+  const released = await dependencies.service.releaseBudget({
+    reservationId,
+    parentRunId: request.runId,
+    expectedReservationVersion: reservation.version,
+    reasonCode: "NODE_EXECUTION_FAILED",
+    authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+    runPolicyDigest: snapshot.run.executionPolicyDigest,
+    releasedAt: monotonicAt(request.at, snapshot.run.updatedAt),
+  });
+  return released.outcome === "released" || released.outcome === "replayed";
+}
+
+export async function activeRecoveryReservationCount(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+) {
+  const ledger = await dependencies.service.readBudget(runId);
+  return ledger?.activeReservations.length;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-effect.ts

@@ -0,0 +1,331 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+  AgentWorkflowSideEffectCommand,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { AgentWorkflowAdapterDispatchResult } from "./adapter-dispatch.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { convergeTimeout } from "./scheduler-execution-convergence.js";
+import {
+  dispatchResultAt,
+  failScheduledEffect,
+  waitScheduledNode,
+} from "./scheduler-effect-result.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  awaitScheduledNode,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export type SchedulerRecoveryStep =
+  | { readonly outcome: "completed" }
+  | { readonly outcome: "waiting" }
+  | {
+      readonly outcome: "failed";
+      readonly safeCode: string;
+      readonly policyEligible: boolean;
+    };
+
+export async function recoverPersistedEffect(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly release: AgentWorkflowRelease;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly command: AgentWorkflowSideEffectCommand;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+): Promise<SchedulerRecoveryStep> {
+  if (input.command.status === "failed") {
+    if (input.command.safeErrorCode === "NODE_DEADLINE_EXCEEDED") {
+      await dependencies.serializeMutation(() =>
+        convergeTimeout(
+          dependencies,
+          input.snapshot.run.runId,
+          input.nodeRun,
+          monotonicAt(input.at, input.command.updatedAt),
+        ),
+      );
+      return { outcome: "completed" };
+    }
+    return {
+      outcome: "failed",
+      safeCode: input.command.safeErrorCode ?? "ADAPTER_DISPATCH_FAILED",
+      policyEligible: true,
+    };
+  }
+  const adapter = dependencies.adapters.require(input.node.type);
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: input.snapshot.run,
+    node: input.node,
+    nodeRun: input.nodeRun,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot: input.snapshot,
+    release: input.release,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  try {
+    const receipt = await dependencies.effectFence.reconcile(
+      input.command.commandId,
+    );
+    if (
+      !receipt &&
+      input.command.status === "prepared" &&
+      (deadline.expired() || Date.parse(input.at) >= Date.parse(deadlineAt))
+    ) {
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        "NODE_DEADLINE_EXCEEDED",
+        monotonicAt(input.at, deadlineAt),
+      );
+      await dependencies.serializeMutation(() =>
+        convergeTimeout(
+          dependencies,
+          input.snapshot.run.runId,
+          input.nodeRun,
+          monotonicAt(input.at, deadlineAt),
+        ),
+      );
+      return { outcome: "completed" };
+    }
+    const claimed = receipt
+      ? { outcome: "replayed" as const, receipt }
+      : input.command.status === "prepared"
+        ? await dependencies.effectFence.claim({
+            command: input.command,
+            expectedCommandVersion: input.command.version,
+            authorizationFence: input.authorizationFence,
+            budgetReservationId: input.command.budgetReservationId,
+            deadlineAt,
+          })
+        : await dependencies.effectFence.claimReconciliation({
+            command: input.command,
+            expectedCommandVersion: input.command.version,
+          });
+    if (claimed.outcome === "conflict") {
+      return { outcome: "waiting" };
+    }
+    if (claimed.outcome === "authorization-stale") {
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        claimed.safeCode,
+        monotonicAt(input.at),
+      );
+      return {
+        outcome: "failed",
+        safeCode: claimed.safeCode,
+        policyEligible: true,
+      };
+    }
+    if (!("fence" in claimed) && !("receipt" in claimed)) {
+      throw new Error("EFFECT_CLAIM_OUTCOME_UNHANDLED");
+    }
+    const result =
+      input.command.status === "prepared" && claimed.outcome === "claimed"
+        ? adapter.dispatch
+          ? await awaitScheduledNode(
+              adapter.dispatch({
+                context,
+                command: input.command,
+                fence: claimed.fence,
+              }),
+              deadline,
+            )
+          : { outcome: "failed" as const, safeCode: "ADAPTER_DISPATCH_MISSING" }
+        : adapter.reconcile
+          ? await adapter.reconcile({
+              context: { ...context, signal: input.signal },
+              command: input.command,
+              authorizationFence: input.authorizationFence,
+              ...("receipt" in claimed
+                ? { committedReceipt: claimed.receipt }
+                : { fence: claimed.fence }),
+            })
+          : { outcome: "pending" as const };
+    const at = monotonicAt(input.at, dispatchResultAt(input.at, result));
+    if (result.outcome === "pending" || result.outcome === "waiting") {
+      if (
+        result.outcome === "waiting" &&
+        input.nodeRun.status === "waiting" &&
+        !sameAdapter(input.nodeRun.adapter, result.adapter)
+      ) {
+        return {
+          outcome: "failed",
+          safeCode: "ADAPTER_COORDINATE_DRIFT",
+          policyEligible: false,
+        };
+      }
+      if (input.nodeRun.status === "running") {
+        await waitScheduledNode(
+          dependencies,
+          input.snapshot,
+          input.nodeRun,
+          result.outcome === "waiting" ? result.adapter : input.command.adapter,
+          at,
+        );
+      }
+      return { outcome: "waiting" };
+    }
+    if (result.outcome === "failed") {
+      if (
+        receipt ||
+        input.command.status === "committed" ||
+        input.command.status === "reconciled"
+      ) {
+        return {
+          outcome: "failed",
+          safeCode: result.safeCode,
+          policyEligible: false,
+        };
+      }
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        result.safeCode,
+        at,
+      );
+      return {
+        outcome: "failed",
+        safeCode: result.safeCode,
+        policyEligible: true,
+      };
+    }
+    if (claimed.outcome === "claimed") {
+      const committed = await dependencies.effectFence.commit({
+        fence: claimed.fence,
+        receipt: result.receipt,
+      });
+      if (committed.outcome === "conflict") {
+        return { outcome: "waiting" };
+      }
+    }
+    return completeRecoveredEffect(dependencies, input, result, at);
+  } catch {
+    if (input.signal.aborted) return { outcome: "waiting" };
+    return { outcome: "waiting" };
+  }
+}
+
+async function completeRecoveredEffect(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof recoverPersistedEffect>[1],
+  result: Extract<AgentWorkflowAdapterDispatchResult, { outcome: "committed" }>,
+  at: string,
+): Promise<SchedulerRecoveryStep> {
+  let snapshot = await dependencies.requireSnapshot(input.snapshot.run.runId);
+  let nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun)
+    return {
+      outcome: "failed",
+      safeCode: "NODE_RUN_MISSING",
+      policyEligible: false,
+    };
+  let usage = result.cumulativeUsage;
+  if (usage && input.command.budgetReservationId) {
+    const ledger = await dependencies.service.readBudget(snapshot.run.runId);
+    const reservation = ledger?.activeReservations.find(
+      ({ reservationId }) =>
+        reservationId === input.command.budgetReservationId,
+    );
+    if (!reservation) {
+      return {
+        outcome: "failed",
+        safeCode: "BUDGET_RESERVATION_MISSING",
+        policyEligible: false,
+      };
+    }
+    const settled = await dependencies.service.settleBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: snapshot.run.runId,
+      expectedReservationVersion: reservation.version,
+      consumed: usage,
+      usageReceiptId: result.receipt.receiptId,
+      authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+      runPolicyDigest: snapshot.run.executionPolicyDigest,
+      settledAt: at,
+    });
+    if ("safeCode" in settled) {
+      return {
+        outcome: "failed",
+        safeCode: settled.safeCode,
+        policyEligible: false,
+      };
+    }
+    usage = undefined;
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+    );
+    if (!nodeRun)
+      return {
+        outcome: "failed",
+        safeCode: "NODE_RUN_MISSING",
+        policyEligible: false,
+      };
+  }
+  try {
+    await dependencies.service.transitionNode({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedRunVersion: usage ? snapshot.run.version : undefined,
+      expectedSequence: snapshot.headSequence,
+      command: {
+        type: "complete",
+        receipt: result.nodeResult,
+        outputArtifacts: result.outputArtifacts,
+      },
+      cumulativeUsage: usage,
+      effectCommandId: input.command.commandId,
+      idempotencyKey: `scheduler:recovery:complete:${input.command.commandId}`,
+      at,
+    });
+    return { outcome: "completed" };
+  } catch {
+    const current = await dependencies.requireSnapshot(snapshot.run.runId);
+    const latest = current.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+    );
+    return latest?.status === "completed"
+      ? { outcome: "completed" }
+      : {
+          outcome: "failed",
+          safeCode: "NODE_RESULT_COMMIT_FAILED",
+          policyEligible: false,
+        };
+  }
+}
+
+function sameAdapter(
+  left: AgentWorkflowNodeRun["adapter"],
+  right: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+) {
+  return left?.kind === right.kind && left.coordinate === right.coordinate;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-event-time.ts

@@ -0,0 +1,26 @@
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+
+export async function recoveryEventAt(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  at: string,
+) {
+  const snapshot = await dependencies.requireSnapshot(runId);
+  const tail = await dependencies.repository.readEvents({
+    runId,
+    afterSequence: Math.max(0, snapshot.headSequence - 1),
+    limit: 1,
+  });
+  return new Date(
+    Math.max(
+      Date.now(),
+      Date.parse(at) || 0,
+      Date.parse(snapshot.run.updatedAt) || 0,
+      ...snapshot.nodeRuns.map(({ updatedAt }) => Date.parse(updatedAt) || 0),
+      ...snapshot.sideEffects.map(
+        ({ updatedAt }) => Date.parse(updatedAt) || 0,
+      ),
+      ...tail.map(({ occurredAt }) => Date.parse(occurredAt) || 0),
+    ),
+  ).toISOString();
+}