create-windy 0.6.11 → 0.6.14

package/template/apps/server/src/agent-workflow/service-mutation-identity.ts

@@ -0,0 +1,94 @@
+import type {
+  AgentWorkflowNodeRun,
+  AgentWorkflowRun,
+} from "@southwind-ai/shared";
+import { canonicalJson } from "./canonical-json.js";
+import { AgentWorkflowRepositoryConflict } from "./repository.js";
+
+export function assertImmutableRunIdentity(
+  current: AgentWorkflowRun,
+  next: AgentWorkflowRun,
+) {
+  const left = immutableRunIdentity(current);
+  const right = immutableRunIdentity(next);
+  if (canonicalJson(left) !== canonicalJson(right)) conflict();
+}
+
+export function assertImmutableNodeRunIdentity(
+  current: AgentWorkflowNodeRun,
+  next: AgentWorkflowNodeRun,
+) {
+  const left = immutableNodeIdentity(current);
+  const right = immutableNodeIdentity(next);
+  if (canonicalJson(left) !== canonicalJson(right)) conflict();
+  if (
+    current.adapter &&
+    canonicalJson(current.adapter) !== canonicalJson(next.adapter)
+  ) {
+    conflict();
+  }
+}
+
+export function assertNodeInsertLineage(
+  nodeRun: AgentWorkflowNodeRun,
+  current: ReadonlyMap<string, AgentWorkflowNodeRun>,
+) {
+  if (!nodeRun.previousNodeRunId) {
+    if (nodeRun.attempt !== 1) conflict();
+    return;
+  }
+  const previous = current.get(nodeRun.previousNodeRunId);
+  if (
+    !previous ||
+    previous.runId !== nodeRun.runId ||
+    previous.nodeId !== nodeRun.nodeId ||
+    previous.nodeType !== nodeRun.nodeType ||
+    previous.attempt + 1 !== nodeRun.attempt ||
+    previous.maxAttempts !== nodeRun.maxAttempts
+  ) {
+    conflict();
+  }
+}
+
+function immutableRunIdentity(run: AgentWorkflowRun) {
+  return {
+    runId: run.runId,
+    releaseId: run.releaseId,
+    tenantId: run.tenantId,
+    actorId: run.actorId,
+    routeKey: run.routeKey,
+    inputDigest: run.inputDigest,
+    inputReceipt: run.inputReceipt,
+    scopeDigest: run.scopeDigest,
+    authorizationSnapshotDigest: run.authorizationSnapshotDigest,
+    toolPolicyDigest: run.toolPolicyDigest,
+    workspacePolicyDigest: run.workspacePolicyDigest,
+    executionPolicyDigest: run.executionPolicyDigest,
+    budget: run.budget,
+    deadlineAt: run.deadlineAt,
+    sessionId: run.sessionId,
+    parentRunId: run.parentRunId,
+    parentNodeId: run.parentNodeId,
+    parentNodeRunId: run.parentNodeRunId,
+    createdAt: run.createdAt,
+  };
+}
+
+function immutableNodeIdentity(nodeRun: AgentWorkflowNodeRun) {
+  return {
+    nodeRunId: nodeRun.nodeRunId,
+    runId: nodeRun.runId,
+    nodeId: nodeRun.nodeId,
+    previousNodeRunId: nodeRun.previousNodeRunId,
+    nodeType: nodeRun.nodeType,
+    attempt: nodeRun.attempt,
+    maxAttempts: nodeRun.maxAttempts,
+    idempotencyKey: nodeRun.idempotencyKey,
+    inputArtifacts: nodeRun.inputArtifacts,
+    createdAt: nodeRun.createdAt,
+  };
+}
+
+function conflict(): never {
+  throw new AgentWorkflowRepositoryConflict("VERSION_CONFLICT");
+}

package/template/apps/server/src/agent-workflow/service-node-admission.ts

@@ -0,0 +1,36 @@
+import type { AgentWorkflowRun } from "@southwind-ai/shared";
+import type { AgentWorkflowNodeRunCommand } from "./state-machine.js";
+import { AgentWorkflowStateError } from "./state-machine.js";
+
+export function assertNodeCommandAdmitted(input: {
+  readonly run: AgentWorkflowRun;
+  readonly command: Exclude<
+    AgentWorkflowNodeRunCommand,
+    { type: "schedule-retry" }
+  >;
+  readonly effectCommandId?: string;
+  readonly at: string;
+}) {
+  const reconciliation =
+    input.command.type === "complete" && Boolean(input.effectCommandId);
+  if (
+    input.run.cancellationRequestedAt &&
+    !["cancel", "timeout"].includes(input.command.type) &&
+    !reconciliation
+  ) {
+    throw new AgentWorkflowStateError(
+      "CANCELLATION_REQUESTED",
+      "Cancellation Requested 后禁止新的 Node Run 行为",
+    );
+  }
+  if (
+    Date.parse(input.at) >= Date.parse(input.run.deadlineAt) &&
+    !["cancel", "timeout"].includes(input.command.type) &&
+    !reconciliation
+  ) {
+    throw new AgentWorkflowStateError(
+      "DEADLINE_EXCEEDED",
+      "Workflow Run Deadline 后禁止新的 Node Run 行为",
+    );
+  }
+}

package/template/apps/server/src/agent-workflow/service-node-run-update.ts

@@ -0,0 +1,55 @@
+import type {
+  AgentWorkflowRun,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import { transitionAgentWorkflowRun } from "./state-machine.js";
+import type { AgentWorkflowNodeRunCommand } from "./state-machine.js";
+import { recordAgentWorkflowUsage } from "./usage.js";
+
+export function buildNodeCommandRunUpdate(input: {
+  readonly run: AgentWorkflowRun;
+  readonly command: Exclude<
+    AgentWorkflowNodeRunCommand,
+    { type: "schedule-retry" }
+  >;
+  readonly cumulativeUsage?: AgentWorkflowUsage;
+  readonly expectedRunVersion?: number;
+  readonly at: string;
+}) {
+  const runLifecycle =
+    input.command.type === "wait" && input.run.status === "running"
+      ? ({ type: "wait" } as const)
+      : ["ready", "start", "resume"].includes(input.command.type) &&
+          input.run.status === "waiting"
+        ? ({ type: "resume" } as const)
+        : undefined;
+  if (runLifecycle && input.cumulativeUsage) {
+    throw new Error("Run lifecycle 与 Usage 更新必须分步提交");
+  }
+  if (input.cumulativeUsage) {
+    if (input.expectedRunVersion === undefined) {
+      throw new Error("expectedRunVersion 必填");
+    }
+    return {
+      expectedVersion: input.expectedRunVersion,
+      value: recordAgentWorkflowUsage(
+        input.run,
+        input.cumulativeUsage,
+        input.expectedRunVersion,
+        input.at,
+        ["fail", "cancel", "timeout"].includes(input.command.type),
+      ),
+    };
+  }
+  return runLifecycle
+    ? {
+        expectedVersion: input.run.version,
+        value: transitionAgentWorkflowRun({
+          run: input.run,
+          expectedVersion: input.run.version,
+          command: runLifecycle,
+          at: input.at,
+        }),
+      }
+    : undefined;
+}

package/template/apps/server/src/agent-workflow/service-recovery-events.ts

@@ -0,0 +1,38 @@
+import type { AgentWorkflowRepository } from "./repository.js";
+import { workflowEvent } from "./service-support.js";
+
+export async function appendRecoveryFact(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly runId: string;
+  readonly recoveryNumber: number;
+  readonly outcome: "waiting" | "failed";
+  readonly commandId?: string;
+  readonly unresolvedCount?: number;
+  readonly at: string;
+}) {
+  const current = await input.repository.getSnapshot(input.runId);
+  if (!current) throw new Error("Workflow Run 不存在");
+  return input.repository.commit({
+    runId: input.runId,
+    expectedSequence: current.headSequence,
+    events: [
+      workflowEvent({
+        runId: input.runId,
+        idempotencyKey: `scheduler:recovery:${input.recoveryNumber}:${input.outcome}:${input.commandId ?? "active"}`,
+        at: input.at,
+        data: {
+          type:
+            input.outcome === "waiting"
+              ? "workflow.recovery-waiting"
+              : "workflow.recovery-failed",
+          recoveryNumber: input.recoveryNumber,
+          safeCode:
+            input.outcome === "waiting"
+              ? "EFFECT_RECONCILIATION_PENDING"
+              : "EFFECT_RECONCILIATION_FAILED",
+          unresolvedCount: input.unresolvedCount ?? 1,
+        },
+      }),
+    ],
+  });
+}

package/template/apps/server/src/agent-workflow/service-recovery.ts

@@ -0,0 +1,264 @@
+import {
+  isAgentWorkflowNodeTerminal,
+  isAgentWorkflowTerminal,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import {
+  activeRecoveryReservationCount,
+  releaseRecoveryReservation,
+} from "./scheduler-recovery-budget.js";
+import { recoverPersistedEffect } from "./scheduler-recovery-effect.js";
+import { recoveryEventAt } from "./scheduler-recovery-event-time.js";
+import {
+  failNodeAndRunForRecovery,
+  failRunForRecovery,
+} from "./scheduler-recovery-failed-policy.js";
+import { recoverUncommandedNode } from "./scheduler-recovery-node.js";
+import {
+  appendRecoveryFailure,
+  applyRecoveryFailurePolicy,
+  convergeRecoveryFailedPolicies,
+} from "./scheduler-recovery-policy-convergence.js";
+import { failureDecision } from "./scheduler-support.js";
+import { appendRecoveryFact } from "./service-recovery-events.js";
+import { workflowEvent } from "./service-support.js";
+
+export interface RecoverAgentWorkflowInput {
+  readonly runId: string;
+  readonly authorizationFence: WorkflowAuthorizationFence;
+  readonly at: string;
+  readonly signal: AbortSignal;
+}
+
+export async function recoverAgentWorkflow(input: {
+  readonly dependencies: SchedulerExecutionDependencies;
+  readonly request: RecoverAgentWorkflowInput;
+}) {
+  const { dependencies, request } = input;
+  const initial = await dependencies.requireSnapshot(request.runId);
+  if (isAgentWorkflowTerminal(initial.run.status)) {
+    throw new Error("Workflow Run 已终态，不能 Recovery");
+  }
+  const recoveryNumber = initial.recoveryCount + 1;
+  const recoveryStartedAt = await recoveryEventAt(
+    dependencies,
+    request.runId,
+    request.at,
+  );
+  const admission = await dependencies.repository.admitRecovery({
+    runId: request.runId,
+    expectedSequence: initial.headSequence,
+    expectedRecoveryCount: initial.recoveryCount,
+    actorId: request.authorizationFence.actorId,
+    scopeDigest: initial.run.scopeDigest,
+    authorizationSnapshotDigest:
+      request.authorizationFence.authorizationSnapshotDigest,
+    toolPolicyDigest: initial.run.toolPolicyDigest,
+    workspacePolicyDigest: initial.run.workspacePolicyDigest,
+    executionPolicyDigest: request.authorizationFence.runPolicyDigest,
+    event: workflowEvent({
+      runId: request.runId,
+      idempotencyKey: `scheduler:recovery:${recoveryNumber}:started`,
+      at: recoveryStartedAt,
+      data: { type: "workflow.recovery-started", recoveryNumber },
+    }),
+  });
+  const release = await dependencies.requireRelease(initial.run.releaseId);
+  const candidates = (
+    await dependencies.requireSnapshot(request.runId)
+  ).nodeRuns
+    .filter(
+      ({ status, resultReceipt }) =>
+        ["running", "waiting"].includes(status) && !resultReceipt,
+    )
+    .map(({ nodeRunId }) => nodeRunId);
+  for (const nodeRunId of candidates) {
+    const snapshot = await dependencies.requireSnapshot(request.runId);
+    if (isAgentWorkflowTerminal(snapshot.run.status)) {
+      return { outcome: "failed" as const, admission };
+    }
+    const nodeRun = snapshot.nodeRuns.find(
+      (item) => item.nodeRunId === nodeRunId,
+    );
+    if (!nodeRun || isAgentWorkflowNodeTerminal(nodeRun.status)) continue;
+    const node = release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    if (!node) {
+      const stable = await appendRecoveryFailure(
+        dependencies,
+        request,
+        recoveryNumber,
+      );
+      await failRunForRecovery(dependencies, {
+        runId: request.runId,
+        safeCode: "NODE_DEFINITION_MISSING",
+        at: request.at,
+      });
+      return { outcome: "failed" as const, admission, stable };
+    }
+    const command = snapshot.sideEffects.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    );
+    if (!command && snapshot.run.cancellationRequestedAt) {
+      const stable = await appendRecoveryFact({
+        repository: dependencies.repository,
+        runId: request.runId,
+        recoveryNumber,
+        outcome: "waiting",
+        at: await recoveryEventAt(dependencies, request.runId, request.at),
+      });
+      return { outcome: "waiting" as const, admission, stable };
+    }
+    const step = command
+      ? await recoverPersistedEffect(dependencies, {
+          snapshot,
+          release,
+          node,
+          nodeRun,
+          command,
+          authorizationFence: request.authorizationFence,
+          at: request.at,
+          signal: request.signal,
+        })
+      : await recoverUncommandedNode(dependencies, {
+          snapshot,
+          release,
+          node,
+          nodeRun,
+          authorizationFence: request.authorizationFence,
+          at: request.at,
+          signal: request.signal,
+        });
+    if (step.outcome === "waiting") {
+      const stable = await appendRecoveryFact({
+        repository: dependencies.repository,
+        runId: request.runId,
+        recoveryNumber,
+        outcome: "waiting",
+        commandId: command?.commandId,
+        at: await recoveryEventAt(dependencies, request.runId, request.at),
+      });
+      return { outcome: "waiting" as const, admission, stable };
+    }
+    if (step.outcome === "failed") {
+      const latestCommand = command
+        ? await dependencies.repository.getSideEffect(command.commandId)
+        : undefined;
+      const policyEligible =
+        step.policyEligible &&
+        !(
+          latestCommand?.effectKind === "subagent" &&
+          latestCommand.generation > 0 &&
+          ![
+            "ADAPTER_DISPATCH_MISSING",
+            "WORKFLOW_AUTHORIZATION_STALE",
+          ].includes(step.safeCode)
+        );
+      if (
+        policyEligible &&
+        command?.budgetReservationId &&
+        !(await releaseRecoveryReservation(
+          dependencies,
+          request,
+          command.budgetReservationId,
+        ))
+      ) {
+        const stable = await appendRecoveryFact({
+          repository: dependencies.repository,
+          runId: request.runId,
+          recoveryNumber,
+          outcome: "waiting",
+          commandId: command.commandId,
+          at: await recoveryEventAt(dependencies, request.runId, request.at),
+        });
+        return { outcome: "waiting" as const, admission, stable };
+      }
+      const decision = failureDecision(node, nodeRun);
+      if (!policyEligible || decision.type === "fail-run") {
+        const stable = await appendRecoveryFailure(
+          dependencies,
+          request,
+          recoveryNumber,
+          command?.commandId,
+        );
+        if (policyEligible) {
+          await applyRecoveryFailurePolicy(
+            dependencies,
+            request.runId,
+            nodeRunId,
+            step.safeCode,
+            request.at,
+          );
+        } else {
+          await failNodeAndRunForRecovery(dependencies, {
+            runId: request.runId,
+            nodeRunId,
+            safeCode: step.safeCode,
+            at: request.at,
+          });
+        }
+        return { outcome: "failed" as const, admission, stable };
+      }
+      await applyRecoveryFailurePolicy(
+        dependencies,
+        request.runId,
+        nodeRunId,
+        step.safeCode,
+        request.at,
+      );
+    }
+  }
+  const failedPolicy = await convergeRecoveryFailedPolicies(
+    dependencies,
+    request,
+    recoveryNumber,
+    admission,
+  );
+  if (failedPolicy) return failedPolicy;
+  const converged = await dependencies.requireSnapshot(request.runId);
+  const unresolvedNodes = converged.nodeRuns.filter(
+    ({ status, resultReceipt }) =>
+      ["running", "waiting"].includes(status) && !resultReceipt,
+  );
+  const unresolvedEffects = converged.sideEffects.filter(({ status }) =>
+    ["prepared", "dispatched", "cancellation-requested"].includes(status),
+  );
+  const unresolvedReservations =
+    (await activeRecoveryReservationCount(dependencies, request.runId)) ?? 1;
+  if (
+    unresolvedNodes.length ||
+    unresolvedEffects.length ||
+    unresolvedReservations
+  ) {
+    const stable = await appendRecoveryFact({
+      repository: dependencies.repository,
+      runId: request.runId,
+      recoveryNumber,
+      outcome: "waiting",
+      unresolvedCount:
+        unresolvedNodes.length +
+        unresolvedEffects.length +
+        unresolvedReservations,
+      at: await recoveryEventAt(dependencies, request.runId, request.at),
+    });
+    return { outcome: "waiting" as const, admission, stable };
+  }
+  if (isAgentWorkflowTerminal(converged.run.status)) {
+    return { outcome: "failed" as const, admission };
+  }
+  const completed = await dependencies.repository.commit({
+    runId: request.runId,
+    expectedSequence: converged.headSequence,
+    events: [
+      workflowEvent({
+        runId: request.runId,
+        idempotencyKey: `scheduler:recovery:${recoveryNumber}:completed`,
+        at: await recoveryEventAt(dependencies, request.runId, request.at),
+        data: { type: "workflow.recovery-completed", recoveryNumber },
+      }),
+    ],
+  });
+  return { outcome: "completed" as const, admission, completed };
+}

package/template/apps/server/src/agent-workflow/service-retry.ts

@@ -0,0 +1,86 @@
+import {
+  createAgentWorkflowRetry,
+  transitionAgentWorkflowNodeRun,
+  transitionAgentWorkflowRun,
+} from "./state-machine.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import { nodeEventData, workflowEvent, workflowId } from "./service-support.js";
+
+export interface ScheduleAgentWorkflowRetryInput {
+  readonly runId: string;
+  readonly nodeRunId: string;
+  readonly expectedNodeVersion: number;
+  readonly expectedSequence: number;
+  readonly safeCode: string;
+  readonly idempotencyKey: string;
+  readonly at: string;
+}
+
+export async function scheduleAgentWorkflowRetry(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly request: ScheduleAgentWorkflowRetryInput;
+}) {
+  const request = input.request;
+  const snapshot = await input.repository.getSnapshot(request.runId);
+  if (!snapshot) throw new Error("Workflow Run 不存在");
+  const source = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === request.nodeRunId,
+  );
+  if (!source) throw new Error("Node Run 不存在");
+  const command = {
+    type: "schedule-retry" as const,
+    safeCode: request.safeCode,
+  };
+  const scheduled = transitionAgentWorkflowNodeRun({
+    nodeRun: source,
+    expectedVersion: request.expectedNodeVersion,
+    command,
+    at: request.at,
+  });
+  const retry = createAgentWorkflowRetry({
+    source: scheduled,
+    nodeRunId: workflowId(
+      "workflow_node_run",
+      source.nodeRunId,
+      String(source.attempt + 1),
+      request.idempotencyKey,
+    ),
+    idempotencyKey: workflowId(
+      "workflow_node_attempt",
+      source.runId,
+      source.nodeId,
+      String(source.attempt + 1),
+      request.idempotencyKey,
+    ),
+    createdAt: request.at,
+  });
+  const runUpdate =
+    snapshot.run.status === "running"
+      ? {
+          expectedVersion: snapshot.run.version,
+          value: transitionAgentWorkflowRun({
+            run: snapshot.run,
+            expectedVersion: snapshot.run.version,
+            command: { type: "wait" },
+            at: request.at,
+          }),
+        }
+      : undefined;
+  return input.repository.commit({
+    runId: request.runId,
+    expectedSequence: request.expectedSequence,
+    nodeUpdates: [
+      { expectedVersion: request.expectedNodeVersion, value: scheduled },
+    ],
+    nodeInserts: [retry],
+    ...(runUpdate ? { runUpdate } : {}),
+    events: [
+      workflowEvent({
+        runId: request.runId,
+        idempotencyKey: request.idempotencyKey,
+        at: request.at,
+        data: nodeEventData(source, command),
+      }),
+    ],
+  });
+}