create-windy 0.6.12 → 0.6.15

package/template/apps/server/src/agent-workflow/drizzle-effect-command-fence.ts

@@ -0,0 +1,333 @@
+import { randomBytes } from "node:crypto";
+import {
+  agentWorkflowBudgetReservations,
+  agentWorkflowDelegations,
+  agentWorkflowNodeRuns,
+  agentWorkflowRuns,
+  agentWorkflowSideEffectCommands,
+  agentWorkflowSideEffectReceipts,
+} from "@southwind-ai/database";
+import type { AgentWorkflowSideEffectReceipt } from "@southwind-ai/shared";
+import { and, eq } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { assertWorkflowAuthorizationFence } from "./budget-validation.js";
+import {
+  assertEffectCommandReceipt,
+  type WorkflowEffectCommandFencePort,
+} from "./effect-command-fence.js";
+import {
+  sideEffectReceiptRow,
+  toSideEffect,
+  toSideEffectReceipt,
+} from "./drizzle-repository-mappers.js";
+import { samePreparedSideEffect } from "./side-effect-state.js";
+import {
+  claimEffectForCancellation,
+  effectFenceConflict as conflict,
+  effectFenceDigest as digest,
+  lockEffectCommand as lockCommand,
+  sameEffectReceipt as sameReceipt,
+} from "./drizzle-cancellation-reconcile.js";
+
+export class DrizzleWorkflowEffectCommandFence implements WorkflowEffectCommandFencePort {
+  constructor(private readonly db: NodePgDatabase) {}
+
+  claim(input: Parameters<WorkflowEffectCommandFencePort["claim"]>[0]) {
+    return this.db.transaction(async (tx) => {
+      const now = new Date();
+      if (Date.parse(input.deadlineAt) <= now.getTime()) {
+        return conflict("WORKFLOW_DEADLINE_EXCEEDED");
+      }
+      await lockCommand(tx, input.command.commandId);
+      const [commandRow] = await tx
+        .select()
+        .from(agentWorkflowSideEffectCommands)
+        .where(
+          eq(
+            agentWorkflowSideEffectCommands.commandId,
+            input.command.commandId,
+          ),
+        )
+        .limit(1);
+      if (!commandRow) return conflict("EFFECT_COMMAND_NOT_FOUND");
+      if (!samePreparedSideEffect(toSideEffect(commandRow), input.command)) {
+        return conflict("EFFECT_COMMAND_CONFLICT");
+      }
+      const [receipt] = await tx
+        .select()
+        .from(agentWorkflowSideEffectReceipts)
+        .where(
+          eq(agentWorkflowSideEffectReceipts.commandId, commandRow.commandId),
+        )
+        .limit(1);
+      if (receipt) {
+        return {
+          outcome: "replayed" as const,
+          receipt: toSideEffectReceipt(receipt),
+        };
+      }
+      const [run] = await tx
+        .select()
+        .from(agentWorkflowRuns)
+        .where(eq(agentWorkflowRuns.runId, commandRow.runId))
+        .for("update")
+        .limit(1);
+      if (
+        !run ||
+        ["completed", "failed", "cancelled", "timed-out"].includes(run.status)
+      ) {
+        return conflict("WORKFLOW_RUN_NOT_ACTIVE");
+      }
+      if (run.cancellationRequestedAt) {
+        return conflict("WORKFLOW_CANCELLATION_REQUESTED");
+      }
+      if (
+        input.deadlineAt !== new Date(run.deadlineAt).toISOString() ||
+        run.deadlineAt <= now.getTime()
+      ) {
+        return conflict("WORKFLOW_DEADLINE_EXCEEDED");
+      }
+      const [nodeRun] = await tx
+        .select()
+        .from(agentWorkflowNodeRuns)
+        .where(eq(agentWorkflowNodeRuns.nodeRunId, commandRow.nodeRunId))
+        .for("update")
+        .limit(1);
+      if (
+        !nodeRun ||
+        nodeRun.runId !== run.runId ||
+        !["running", "waiting"].includes(nodeRun.status)
+      ) {
+        return conflict("WORKFLOW_NODE_NOT_ACTIVE");
+      }
+      try {
+        assertWorkflowAuthorizationFence(
+          input.authorizationFence,
+          run.actorId,
+          now,
+        );
+      } catch {
+        return {
+          outcome: "authorization-stale" as const,
+          safeCode: "AUTHORIZATION_FENCE_STALE",
+        };
+      }
+      if (
+        input.authorizationFence.authorizationSnapshotDigest !==
+          run.authorizationSnapshotDigest ||
+        input.authorizationFence.runPolicyDigest !== run.executionPolicyDigest
+      ) {
+        return {
+          outcome: "authorization-stale" as const,
+          safeCode: "RUN_POLICY_STALE",
+        };
+      }
+      if (
+        (input.budgetReservationId ?? null) !== commandRow.budgetReservationId
+      ) {
+        return conflict("BUDGET_RESERVATION_COORDINATE_CONFLICT");
+      }
+      if (
+        commandRow.effectKind === "subagent" &&
+        (!input.budgetReservationId ||
+          !commandRow.delegationId ||
+          input.budgetReservationId !== commandRow.budgetReservationId)
+      ) {
+        return conflict("SUBAGENT_BUDGET_RESERVATION_REQUIRED");
+      }
+      if (
+        !(await validReservation(
+          tx,
+          commandRow.budgetReservationId ?? undefined,
+          run.runId,
+          commandRow.nodeRunId,
+          commandRow.delegationId,
+        ))
+      ) {
+        return conflict("BUDGET_RESERVATION_INVALID");
+      }
+      if (
+        commandRow.version !== input.expectedCommandVersion ||
+        !["prepared", "dispatched"].includes(commandRow.status)
+      ) {
+        return conflict("EFFECT_COMMAND_STALE");
+      }
+      const fenceToken = randomBytes(32).toString("base64url");
+      const fenceTokenDigest = digest(fenceToken);
+      const generation = commandRow.generation + 1;
+      const claimedAt = now.toISOString();
+      const [updated] = await tx
+        .update(agentWorkflowSideEffectCommands)
+        .set({
+          generation,
+          fenceTokenDigest,
+          authorizationSnapshotDigest:
+            input.authorizationFence.authorizationSnapshotDigest,
+          runPolicyDigest: input.authorizationFence.runPolicyDigest,
+          budgetReservationId: commandRow.budgetReservationId,
+          status: "dispatched",
+          safeErrorCode: null,
+          claimedAt: now.getTime(),
+          updatedAt: now.getTime(),
+          version: commandRow.version + 1,
+        })
+        .where(
+          and(
+            eq(agentWorkflowSideEffectCommands.commandId, commandRow.commandId),
+            eq(agentWorkflowSideEffectCommands.version, commandRow.version),
+            eq(
+              agentWorkflowSideEffectCommands.generation,
+              commandRow.generation,
+            ),
+          ),
+        )
+        .returning();
+      if (!updated) return conflict("EFFECT_COMMAND_STALE");
+      return {
+        outcome: "claimed" as const,
+        fence: {
+          commandId: updated.commandId,
+          runId: updated.runId,
+          nodeRunId: updated.nodeRunId,
+          effectKind: updated.effectKind,
+          adapter: input.command.adapter,
+          generation,
+          fenceToken,
+          fenceTokenDigest,
+          requestDigest: updated.requestDigest,
+          authorizationSnapshotDigest:
+            input.authorizationFence.authorizationSnapshotDigest,
+          runPolicyDigest: input.authorizationFence.runPolicyDigest,
+          ...(commandRow.budgetReservationId
+            ? { budgetReservationId: commandRow.budgetReservationId }
+            : {}),
+          ...(commandRow.delegationId
+            ? { delegationId: commandRow.delegationId }
+            : {}),
+          claimedAt,
+        },
+      };
+    });
+  }
+
+  claimReconciliation(
+    input: Parameters<WorkflowEffectCommandFencePort["claimReconciliation"]>[0],
+  ) {
+    return claimEffectForCancellation(this.db, input);
+  }
+
+  commit(input: Parameters<WorkflowEffectCommandFencePort["commit"]>[0]) {
+    return this.db.transaction(async (tx) => {
+      assertEffectCommandReceipt(input);
+      await lockCommand(tx, input.fence.commandId);
+      const [existing] = await tx
+        .select()
+        .from(agentWorkflowSideEffectReceipts)
+        .where(
+          eq(agentWorkflowSideEffectReceipts.commandId, input.fence.commandId),
+        )
+        .limit(1);
+      if (existing) {
+        const receipt = toSideEffectReceipt(existing);
+        return sameReceipt(receipt, input.receipt)
+          ? { outcome: "replayed" as const, receipt }
+          : conflict("EFFECT_RECEIPT_CONFLICT");
+      }
+      const [command] = await tx
+        .select()
+        .from(agentWorkflowSideEffectCommands)
+        .where(
+          eq(agentWorkflowSideEffectCommands.commandId, input.fence.commandId),
+        )
+        .limit(1);
+      const reconciliation =
+        "purpose" in input.fence && input.fence.purpose === "reconciliation";
+      if (
+        !command ||
+        (reconciliation
+          ? !input.fence.fenceToken.startsWith("reconciliation.")
+          : input.fence.fenceToken.startsWith("reconciliation.")) ||
+        (reconciliation
+          ? !["dispatched", "cancellation-requested"].includes(command.status)
+          : command.status !== "dispatched") ||
+        command.runId !== input.fence.runId ||
+        command.nodeRunId !== input.fence.nodeRunId ||
+        command.effectKind !== input.fence.effectKind ||
+        command.requestDigest !== input.fence.requestDigest ||
+        command.generation !== input.fence.generation ||
+        command.fenceTokenDigest !== input.fence.fenceTokenDigest ||
+        command.fenceTokenDigest !== digest(input.fence.fenceToken) ||
+        command.authorizationSnapshotDigest !==
+          input.fence.authorizationSnapshotDigest ||
+        command.runPolicyDigest !== input.fence.runPolicyDigest
+      ) {
+        return conflict("EFFECT_FENCE_STALE");
+      }
+      const receipt = input.receipt as AgentWorkflowSideEffectReceipt;
+      const committedAt = Date.parse(receipt.committedAt);
+      await tx
+        .insert(agentWorkflowSideEffectReceipts)
+        .values(sideEffectReceiptRow(receipt));
+      const [updated] = await tx
+        .update(agentWorkflowSideEffectCommands)
+        .set({
+          status: reconciliation ? "reconciled" : "committed",
+          committedAt,
+          updatedAt: committedAt,
+          version: command.version + 1,
+        })
+        .where(
+          and(
+            eq(agentWorkflowSideEffectCommands.commandId, command.commandId),
+            eq(agentWorkflowSideEffectCommands.version, command.version),
+            eq(agentWorkflowSideEffectCommands.generation, command.generation),
+          ),
+        )
+        .returning();
+      if (!updated) return conflict("EFFECT_FENCE_STALE");
+      return { outcome: "committed" as const, receipt: input.receipt };
+    });
+  }
+
+  reconcile(commandId: string) {
+    return this.db
+      .select()
+      .from(agentWorkflowSideEffectReceipts)
+      .where(eq(agentWorkflowSideEffectReceipts.commandId, commandId))
+      .limit(1)
+      .then(([row]) => (row ? toSideEffectReceipt(row) : undefined));
+  }
+}
+
+async function validReservation(
+  tx: Transaction,
+  reservationId: string | undefined,
+  runId: string,
+  nodeRunId: string,
+  delegationId: string | null,
+) {
+  if (!reservationId) return true;
+  const [reservation] = await tx
+    .select()
+    .from(agentWorkflowBudgetReservations)
+    .where(eq(agentWorkflowBudgetReservations.reservationId, reservationId))
+    .for("update")
+    .limit(1);
+  if (reservation?.parentRunId !== runId || reservation.status !== "active") {
+    return false;
+  }
+  const [delegation] = await tx
+    .select()
+    .from(agentWorkflowDelegations)
+    .where(eq(agentWorkflowDelegations.delegationId, reservation.delegationId))
+    .for("update")
+    .limit(1);
+  return (
+    delegation?.parentRunId === runId &&
+    delegation.parentNodeRunId === nodeRunId &&
+    delegation.delegationId === reservation.delegationId &&
+    delegation.delegationId === delegationId
+  );
+}
+
+type Transaction = Parameters<Parameters<NodePgDatabase["transaction"]>[0]>[0];

package/template/apps/server/src/agent-workflow/drizzle-evaluation-repository.ts

@@ -0,0 +1,214 @@
+import { agentWorkflowEvaluations } from "@southwind-ai/database";
+import type { AgentWorkflowEvaluationRecord } from "@southwind-ai/shared";
+import { and, asc, eq, sql } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import {
+  AgentWorkflowEvaluationRepositoryConflict,
+  assertEvaluationRecordIntegrity,
+  canonicalEvaluationJson,
+  evaluationTrialCoordinate,
+  immutableEvaluationClone,
+  type AgentWorkflowEvaluationOwner,
+  type AgentWorkflowEvaluationRepository,
+  type AgentWorkflowEvaluationTrialCoordinate,
+} from "./evaluation-repository.js";
+
+export class DrizzleAgentWorkflowEvaluationRepository implements AgentWorkflowEvaluationRepository {
+  constructor(private readonly db: NodePgDatabase) {}
+
+  getById(evaluationId: string, owner: AgentWorkflowEvaluationOwner) {
+    return this.db
+      .select()
+      .from(agentWorkflowEvaluations)
+      .where(
+        and(
+          eq(agentWorkflowEvaluations.evaluationId, evaluationId),
+          ownerWhere(owner),
+        ),
+      )
+      .limit(1)
+      .then(([row]) => (row ? toRecord(row) : undefined));
+  }
+
+  findByTrial(coordinate: AgentWorkflowEvaluationTrialCoordinate) {
+    return this.db
+      .select()
+      .from(agentWorkflowEvaluations)
+      .where(trialWhere(coordinate))
+      .limit(1)
+      .then(([row]) => (row ? toRecord(row) : undefined));
+  }
+
+  listBySet(
+    evaluationSetKey: string,
+    evaluationSetVersion: string,
+    owner: AgentWorkflowEvaluationOwner,
+  ) {
+    return this.db
+      .select()
+      .from(agentWorkflowEvaluations)
+      .where(
+        and(
+          eq(agentWorkflowEvaluations.evaluationSetKey, evaluationSetKey),
+          eq(
+            agentWorkflowEvaluations.evaluationSetVersion,
+            evaluationSetVersion,
+          ),
+          ownerWhere(owner),
+        ),
+      )
+      .orderBy(
+        asc(sql`${agentWorkflowEvaluations.identityJson}->>'caseKey'`),
+        asc(sql`${agentWorkflowEvaluations.identityJson}->>'caseVersion'`),
+        asc(
+          sql`(${agentWorkflowEvaluations.identityJson}->>'trialIndex')::int`,
+        ),
+        asc(sql`${agentWorkflowEvaluations.identityJson}->>'trialKey'`),
+      )
+      .then((rows) => rows.map(toRecord));
+  }
+
+  appendTerminal(record: AgentWorkflowEvaluationRecord) {
+    assertEvaluationRecordIntegrity(record);
+    if (record.status === "pending") {
+      throw new AgentWorkflowEvaluationRepositoryConflict(
+        "PENDING_NOT_PERSISTABLE",
+      );
+    }
+    return this.db.transaction(async (tx) => {
+      await tx.execute(
+        sql`select pg_advisory_xact_lock(hashtextextended(${trialLock(record)}, 0))`,
+      );
+      const [byId] = await tx
+        .select()
+        .from(agentWorkflowEvaluations)
+        .where(eq(agentWorkflowEvaluations.evaluationId, record.evaluationId))
+        .limit(1);
+      if (byId)
+        return sameRecord(toRecord(byId), record)
+          ? toRecord(byId)
+          : evaluationConflict("EVALUATION_ID_CONFLICT");
+      const [byTrial] = await tx
+        .select()
+        .from(agentWorkflowEvaluations)
+        .where(trialWhere(evaluationTrialCoordinate(record)))
+        .limit(1);
+      if (byTrial)
+        return sameRecord(toRecord(byTrial), record)
+          ? toRecord(byTrial)
+          : evaluationConflict("TRIAL_CONFLICT");
+      const [created] = await tx
+        .insert(agentWorkflowEvaluations)
+        .values(toRow(record))
+        .returning();
+      if (!created) throw new Error("Workflow Evaluation insert 未返回记录");
+      return toRecord(created);
+    });
+  }
+}
+
+function trialWhere(coordinate: AgentWorkflowEvaluationTrialCoordinate) {
+  return and(
+    eq(agentWorkflowEvaluations.tenantId, coordinate.tenantId),
+    eq(agentWorkflowEvaluations.createdBy, coordinate.createdBy),
+    eq(agentWorkflowEvaluations.evaluationSetKey, coordinate.evaluationSetKey),
+    eq(
+      agentWorkflowEvaluations.evaluationSetVersion,
+      coordinate.evaluationSetVersion,
+    ),
+    eq(agentWorkflowEvaluations.identityDigest, coordinate.identityDigest),
+    sql`${agentWorkflowEvaluations.identityJson}->>'caseKey' = ${coordinate.caseKey}`,
+    sql`${agentWorkflowEvaluations.identityJson}->>'caseVersion' = ${coordinate.caseVersion}`,
+    sql`${agentWorkflowEvaluations.identityJson}->>'trialKey' = ${coordinate.trialKey}`,
+    sql`(${agentWorkflowEvaluations.identityJson}->>'trialIndex')::int = ${coordinate.trialIndex}`,
+  );
+}
+
+function toRecord(
+  row: typeof agentWorkflowEvaluations.$inferSelect,
+): AgentWorkflowEvaluationRecord {
+  const record: AgentWorkflowEvaluationRecord = {
+    evaluationId: row.evaluationId,
+    tenantId: row.tenantId,
+    createdBy: row.createdBy,
+    identity: row.identityJson,
+    identityDigest: row.identityDigest,
+    singleAgent: row.singleArmJson,
+    multiAgent: row.multiArmJson,
+    status: row.status as AgentWorkflowEvaluationRecord["status"],
+    ...(row.dispositionReason
+      ? { dispositionReason: row.dispositionReason }
+      : {}),
+    createdAt: new Date(row.createdAt).toISOString(),
+    ...(row.completedAt
+      ? { completedAt: new Date(row.completedAt).toISOString() }
+      : {}),
+  };
+  assertEvaluationRecordIntegrity(record);
+  return immutableEvaluationClone(record);
+}
+
+function toRow(record: AgentWorkflowEvaluationRecord) {
+  return {
+    evaluationId: record.evaluationId,
+    tenantId: record.tenantId,
+    createdBy: record.createdBy,
+    evaluationSetKey: record.identity.evaluationSetKey,
+    evaluationSetVersion: record.identity.evaluationSetVersion,
+    identityDigest: record.identityDigest,
+    identityJson: record.identity,
+    singleArmJson: record.singleAgent,
+    multiArmJson: record.multiAgent,
+    status: record.status,
+    dispositionReason: record.dispositionReason ?? null,
+    createdAt: Date.parse(record.createdAt),
+    completedAt: record.completedAt ? Date.parse(record.completedAt) : null,
+  };
+}
+
+function sameRecord(
+  left: AgentWorkflowEvaluationRecord,
+  right: AgentWorkflowEvaluationRecord,
+) {
+  const stable = (record: AgentWorkflowEvaluationRecord) => {
+    const {
+      createdAt: _createdAt,
+      completedAt: _completedAt,
+      ...value
+    } = structuredClone(record);
+    return value;
+  };
+  return (
+    canonicalEvaluationJson(stable(left)) ===
+    canonicalEvaluationJson(stable(right))
+  );
+}
+
+function trialLock(record: AgentWorkflowEvaluationRecord) {
+  return canonicalEvaluationJson({
+    tenantId: record.tenantId,
+    createdBy: record.createdBy,
+    evaluationSetKey: record.identity.evaluationSetKey,
+    evaluationSetVersion: record.identity.evaluationSetVersion,
+    caseKey: record.identity.caseKey,
+    caseVersion: record.identity.caseVersion,
+    trialKey: record.identity.trialKey,
+    trialIndex: record.identity.trialIndex,
+    identityDigest: record.identityDigest,
+  });
+}
+
+function ownerWhere(owner: AgentWorkflowEvaluationOwner) {
+  return and(
+    eq(agentWorkflowEvaluations.tenantId, owner.tenantId),
+    eq(agentWorkflowEvaluations.createdBy, owner.createdBy),
+  );
+}
+
+function evaluationConflict(
+  code: ConstructorParameters<
+    typeof AgentWorkflowEvaluationRepositoryConflict
+  >[0],
+): never {
+  throw new AgentWorkflowEvaluationRepositoryConflict(code);
+}

package/template/apps/server/src/agent-workflow/drizzle-event-validation.ts

@@ -0,0 +1,43 @@
+import { agentWorkflowEvents } from "@southwind-ai/database";
+import { asc, eq } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import {
+  assertCanonicalWorkflowEventHistory,
+  assertParentChildEventCausality,
+} from "./event-validation.js";
+import { eventRow, toEvent } from "./drizzle-repository-mappers.js";
+import type { PendingAgentWorkflowEvent } from "./repository.js";
+
+export async function assertDrizzleEventAppend(input: {
+  readonly tx: Transaction;
+  readonly runId: string;
+  readonly events: readonly PendingAgentWorkflowEvent[];
+}) {
+  const rows = await input.tx
+    .select()
+    .from(agentWorkflowEvents)
+    .where(eq(agentWorkflowEvents.runId, input.runId))
+    .orderBy(asc(agentWorkflowEvents.sequence));
+  const existing = rows.map(toEvent);
+  const proposed = input.events.map((event, index) =>
+    toEvent(eventRow(input.runId, existing.length + index + 1, event)),
+  );
+  assertCanonicalWorkflowEventHistory([...existing, ...proposed]);
+  for (const event of proposed.filter(({ parentEventId }) => parentEventId)) {
+    await assertParentChildEventCausality({
+      event,
+      lookup: {
+        findCanonicalEvent: async (eventId) => {
+          const [row] = await input.tx
+            .select()
+            .from(agentWorkflowEvents)
+            .where(eq(agentWorkflowEvents.eventId, eventId))
+            .limit(1);
+          return row ? toEvent(row) : undefined;
+        },
+      },
+    });
+  }
+}
+
+type Transaction = Parameters<Parameters<NodePgDatabase["transaction"]>[0]>[0];

package/template/apps/server/src/agent-workflow/drizzle-recovery-admission.ts

@@ -0,0 +1,115 @@
+import {
+  agentWorkflowBudgetReservations,
+  agentWorkflowEvents,
+  agentWorkflowRuns,
+} from "@southwind-ai/database";
+import { and, desc, eq, sql } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { calculateRemainingBudget } from "./budget-validation.js";
+import { assertDrizzleEventAppend } from "./drizzle-event-validation.js";
+import { eventRow, toEvent, toRun } from "./drizzle-repository-mappers.js";
+import {
+  assertPendingEvent,
+  conflict,
+  mutationResult,
+} from "./in-memory-repository-support.js";
+import type {
+  AgentWorkflowRecoveryAdmissionInput,
+  AgentWorkflowRunRecord,
+} from "./repository.js";
+
+export async function admitDrizzleRecovery(
+  db: NodePgDatabase,
+  input: AgentWorkflowRecoveryAdmissionInput,
+) {
+  return db.transaction(async (tx) => {
+    await tx.execute(
+      sql`select 1 from ${agentWorkflowRuns} where ${agentWorkflowRuns.runId} = ${input.runId} for update`,
+    );
+    const [run] = await tx
+      .select()
+      .from(agentWorkflowRuns)
+      .where(eq(agentWorkflowRuns.runId, input.runId))
+      .limit(1);
+    if (!run) conflict("RUN_NOT_FOUND");
+    if (
+      ["completed", "failed", "cancelled", "timed-out"].includes(run.status) ||
+      run.actorId !== input.actorId ||
+      run.scopeDigest !== input.scopeDigest ||
+      run.authorizationSnapshotDigest !== input.authorizationSnapshotDigest ||
+      run.toolPolicyDigest !== input.toolPolicyDigest ||
+      run.workspacePolicyDigest !== input.workspacePolicyDigest ||
+      run.executionPolicyDigest !== input.executionPolicyDigest ||
+      run.recoveryCount !== input.expectedRecoveryCount
+    ) {
+      conflict("RECOVERY_ADMISSION_REJECTED");
+    }
+    const reservations = await tx
+      .select()
+      .from(agentWorkflowBudgetReservations)
+      .where(
+        and(
+          eq(agentWorkflowBudgetReservations.parentRunId, input.runId),
+          eq(agentWorkflowBudgetReservations.status, "active"),
+        ),
+      )
+      .for("update");
+    const remainingBudget = calculateRemainingBudget({
+      budget: run.budgetJson,
+      consumed: run.usageJson,
+      activeReservations: reservations.map(({ reservedJson }) => ({
+        requested: reservedJson,
+        status: "active" as const,
+      })),
+    });
+    const record = await runRecord(tx, run);
+    if (record.headSequence !== input.expectedSequence)
+      conflict("SEQUENCE_CONFLICT");
+    assertPendingEvent(input.event, input.runId, record.headSequence + 1);
+    await assertDrizzleEventAppend({
+      tx,
+      runId: input.runId,
+      events: [input.event],
+    });
+    const [updated] = await tx
+      .update(agentWorkflowRuns)
+      .set({ recoveryCount: input.expectedRecoveryCount + 1 })
+      .where(
+        and(
+          eq(agentWorkflowRuns.runId, input.runId),
+          eq(agentWorkflowRuns.recoveryCount, input.expectedRecoveryCount),
+        ),
+      )
+      .returning();
+    if (!updated) conflict("VERSION_CONFLICT");
+    const [event] = await tx
+      .insert(agentWorkflowEvents)
+      .values(eventRow(input.runId, record.headSequence + 1, input.event))
+      .returning();
+    if (!event) throw new Error("Recovery Started Event 未返回记录");
+    return {
+      ...mutationResult(await runRecord(tx, updated), [toEvent(event)], false),
+      remainingBudget,
+    };
+  });
+}
+
+async function runRecord(
+  tx: Transaction,
+  row: typeof agentWorkflowRuns.$inferSelect,
+): Promise<AgentWorkflowRunRecord> {
+  const [head] = await tx
+    .select({ sequence: agentWorkflowEvents.sequence })
+    .from(agentWorkflowEvents)
+    .where(eq(agentWorkflowEvents.runId, row.runId))
+    .orderBy(desc(agentWorkflowEvents.sequence))
+    .limit(1);
+  return {
+    run: toRun(row),
+    idempotencyKey: row.idempotencyKey,
+    headSequence: head?.sequence ?? 0,
+    recoveryCount: row.recoveryCount,
+  };
+}
+
+type Transaction = Parameters<Parameters<NodePgDatabase["transaction"]>[0]>[0];