create-windy 0.6.12 → 0.6.15

package/template/apps/server/src/agent-workflow/scheduler-recovery-failed-policy.ts

@@ -0,0 +1,129 @@
+import {
+  isAgentWorkflowNodeTerminal,
+  isAgentWorkflowTerminal,
+  type AgentWorkflowNode,
+  type AgentWorkflowNodeRun,
+} from "@southwind-ai/shared";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failureDecision } from "./scheduler-support.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export async function convergeFailedNodePolicy(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly at: string;
+  },
+) {
+  if (input.nodeRun.status !== "failed")
+    return { outcome: "converged" as const };
+  const decision = failureDecision(input.node, input.nodeRun);
+  const safeCode = input.nodeRun.safeErrorCode ?? "NODE_EXECUTION_FAILED";
+  if (decision.type === "retry") {
+    const existing = input.snapshot.nodeRuns.some(
+      ({ previousNodeRunId }) => previousNodeRunId === input.nodeRun.nodeRunId,
+    );
+    if (!existing) {
+      await dependencies.service.scheduleRetry({
+        runId: input.snapshot.run.runId,
+        nodeRunId: input.nodeRun.nodeRunId,
+        expectedNodeVersion: input.nodeRun.version,
+        expectedSequence: input.snapshot.headSequence,
+        safeCode,
+        idempotencyKey: `scheduler:recovery:retry:${input.nodeRun.nodeRunId}:${input.nodeRun.attempt + 1}`,
+        at: monotonicAt(input.at, input.nodeRun.updatedAt),
+      });
+    }
+    return { outcome: "converged" as const };
+  }
+  if (decision.type === "fallback") {
+    const release = await dependencies.requireRelease(
+      input.snapshot.run.releaseId,
+    );
+    const fallback = release.definition.nodes.find(
+      ({ id }) => id === decision.nodeId,
+    );
+    if (!fallback) {
+      return {
+        outcome: "fail-run" as const,
+        safeCode: "FALLBACK_NODE_MISSING",
+      };
+    }
+    if (!input.snapshot.nodeRuns.some(({ nodeId }) => nodeId === fallback.id)) {
+      await dependencies.insertReady(
+        input.snapshot,
+        [fallback],
+        input.nodeRun,
+        monotonicAt(input.at, input.nodeRun.updatedAt),
+      );
+    }
+    return { outcome: "converged" as const };
+  }
+  return {
+    outcome: "fail-run" as const,
+    safeCode:
+      decision.type === "skip" ? "FAILED_SKIP_POLICY_INCONSISTENT" : safeCode,
+  };
+}
+
+export async function failRunForRecovery(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly runId: string;
+    readonly safeCode: string;
+    readonly at: string;
+  },
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  return dependencies.service.transitionRun({
+    runId: input.runId,
+    expectedVersion: snapshot.run.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "fail", safeCode: input.safeCode },
+    idempotencyKey: `scheduler:recovery:fail-run:${input.runId}:${snapshot.recoveryCount}`,
+    at: monotonicAt(input.at, snapshot.run.updatedAt),
+  });
+}
+
+export async function failNodeAndRunForRecovery(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly runId: string;
+    readonly nodeRunId: string;
+    readonly safeCode: string;
+    readonly at: string;
+  },
+) {
+  let snapshot = await dependencies.requireSnapshot(input.runId);
+  const nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRunId,
+  );
+  if (nodeRun && !isAgentWorkflowNodeTerminal(nodeRun.status)) {
+    await dependencies.service.transitionNode({
+      runId: input.runId,
+      nodeRunId: input.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedSequence: snapshot.headSequence,
+      command: { type: "fail", safeCode: input.safeCode },
+      idempotencyKey: `scheduler:recovery:unsafe-fail:${input.nodeRunId}`,
+      at: monotonicAt(input.at, nodeRun.updatedAt),
+    });
+  }
+  snapshot = await dependencies.requireSnapshot(input.runId);
+  if (isAgentWorkflowTerminal(snapshot.run.status)) return;
+  await failRunForRecovery(dependencies, input);
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-node.ts

@@ -0,0 +1,208 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { AgentWorkflowNodeAdapter } from "./adapter-dispatch.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { waitScheduledNode } from "./scheduler-effect-result.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  recoverPersistedEffect,
+  type SchedulerRecoveryStep,
+} from "./scheduler-recovery-effect.js";
+import { prepareAgentWorkflowSideEffect } from "./side-effect-state.js";
+import { workflowId } from "./service-support.js";
+import {
+  awaitScheduledNode,
+  SchedulerDeadlineExceeded,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+import { convergeTimedOutPreparation } from "./scheduler-execution-convergence.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export async function recoverUncommandedNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly release: AgentWorkflowRelease;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+): Promise<SchedulerRecoveryStep> {
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: input.snapshot.run,
+    node: input.node,
+    nodeRun: input.nodeRun,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot: input.snapshot,
+    release: input.release,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  const adapter = dependencies.adapters.require(input.node.type);
+  let preparation: Awaited<ReturnType<AgentWorkflowNodeAdapter["prepare"]>>;
+  try {
+    preparation = await awaitScheduledNode(adapter.prepare(context), deadline);
+  } catch (error) {
+    if (input.signal.aborted) return { outcome: "waiting" };
+    if (error instanceof SchedulerDeadlineExceeded) {
+      await convergeTimedOutPreparation(
+        dependencies,
+        {
+          release: input.release,
+          nodeRun: input.nodeRun,
+          node: input.node,
+          runId: input.snapshot.run.runId,
+          authorizationFence: input.authorizationFence,
+          at: input.at,
+          signal: input.signal,
+        },
+        input.snapshot,
+        input.nodeRun,
+        adapter,
+        deadlineAt,
+      );
+      const current = await dependencies.requireSnapshot(
+        input.snapshot.run.runId,
+      );
+      return current.nodeRuns.some(
+        ({ nodeRunId, status }) =>
+          nodeRunId === input.nodeRun.nodeRunId && status === "timed-out",
+      )
+        ? { outcome: "completed" }
+        : { outcome: "waiting" };
+    }
+    return {
+      outcome: "failed",
+      safeCode: "ADAPTER_PREPARATION_FAILED",
+      policyEligible: true,
+    };
+  }
+  const at = monotonicAt(
+    input.at,
+    input.nodeRun.updatedAt,
+    preparation.outcome === "direct"
+      ? preparation.result.recordedAt
+      : undefined,
+  );
+  if (preparation.outcome === "failed") {
+    return {
+      outcome: "failed",
+      safeCode: preparation.safeCode,
+      policyEligible: true,
+    };
+  }
+  if (preparation.outcome === "direct") {
+    try {
+      await dependencies.service.transitionNode({
+        runId: input.snapshot.run.runId,
+        nodeRunId: input.nodeRun.nodeRunId,
+        expectedNodeVersion: input.nodeRun.version,
+        expectedRunVersion: preparation.cumulativeUsage
+          ? input.snapshot.run.version
+          : undefined,
+        expectedSequence: input.snapshot.headSequence,
+        command: {
+          type: "complete",
+          receipt: preparation.result,
+          outputArtifacts: preparation.outputArtifacts,
+        },
+        cumulativeUsage: preparation.cumulativeUsage,
+        idempotencyKey: `scheduler:recovery:complete:${input.nodeRun.nodeRunId}`,
+        at,
+      });
+      return { outcome: "completed" };
+    } catch {
+      const current = await dependencies.requireSnapshot(
+        input.snapshot.run.runId,
+      );
+      const latest = current.nodeRuns.find(
+        ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+      );
+      return latest?.status === "completed"
+        ? { outcome: "completed" }
+        : {
+            outcome: "failed",
+            safeCode: "NODE_RESULT_COMMIT_FAILED",
+            policyEligible: true,
+          };
+    }
+  }
+  if (preparation.outcome === "waiting") {
+    if (
+      input.nodeRun.status === "waiting" &&
+      !sameAdapter(input.nodeRun.adapter, preparation.adapter)
+    ) {
+      return {
+        outcome: "failed",
+        safeCode: "ADAPTER_COORDINATE_DRIFT",
+        policyEligible: true,
+      };
+    }
+    if (input.nodeRun.status === "running") {
+      await waitScheduledNode(
+        dependencies,
+        input.snapshot,
+        input.nodeRun,
+        preparation.adapter,
+        at,
+      );
+    }
+    return { outcome: "waiting" };
+  }
+  const command = prepareAgentWorkflowSideEffect({
+    commandId: workflowId(
+      "workflow_effect",
+      input.nodeRun.nodeRunId,
+      preparation.effectKey,
+    ),
+    runId: input.snapshot.run.runId,
+    nodeRunId: input.nodeRun.nodeRunId,
+    effectKind: preparation.effectKind,
+    effectKey: preparation.effectKey,
+    adapter: preparation.adapter,
+    idempotencyKey: workflowId(
+      "workflow_effect_idem",
+      input.nodeRun.idempotencyKey,
+      preparation.effectKey,
+    ),
+    requestDigest: preparation.requestDigest,
+    budgetReservationId: preparation.budgetReservationId,
+    delegationId: preparation.delegationId,
+    createdAt: at,
+  });
+  const persisted = await dependencies.repository.prepareSideEffect(command);
+  return recoverPersistedEffect(dependencies, {
+    ...input,
+    command: persisted.command,
+    at,
+  });
+}
+
+function sameAdapter(
+  left: AgentWorkflowNodeRun["adapter"],
+  right: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+) {
+  return left?.kind === right.kind && left.coordinate === right.coordinate;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-policy-convergence.ts

@@ -0,0 +1,140 @@
+import { isAgentWorkflowNodeTerminal } from "@southwind-ai/shared";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { handleScheduledNodeFailure } from "./scheduler-failure.js";
+import {
+  convergeFailedNodePolicy,
+  failRunForRecovery,
+} from "./scheduler-recovery-failed-policy.js";
+import { recoveryEventAt } from "./scheduler-recovery-event-time.js";
+import { appendRecoveryFact } from "./service-recovery-events.js";
+
+interface RecoveryCoordinate {
+  readonly runId: string;
+  readonly at: string;
+}
+
+export async function convergeRecoveryFailedPolicies(
+  dependencies: SchedulerExecutionDependencies,
+  request: RecoveryCoordinate,
+  recoveryNumber: number,
+  admission: unknown,
+) {
+  const snapshot = await dependencies.requireSnapshot(request.runId);
+  const release = await dependencies.requireRelease(snapshot.run.releaseId);
+  for (const nodeRun of snapshot.nodeRuns.filter(
+    ({ status }) => status === "failed",
+  )) {
+    const node = release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    if (!node) {
+      const stable = await appendRecoveryFailure(
+        dependencies,
+        request,
+        recoveryNumber,
+      );
+      await failRunForRecovery(dependencies, {
+        runId: request.runId,
+        safeCode: "NODE_DEFINITION_MISSING",
+        at: request.at,
+      });
+      return { outcome: "failed" as const, admission, stable };
+    }
+    const current = await dependencies.requireSnapshot(request.runId);
+    const command = current.sideEffects.find(
+      ({ nodeRunId }) => nodeRunId === nodeRun.nodeRunId,
+    );
+    const reservations =
+      (await dependencies.service.readBudget(request.runId))
+        ?.activeReservations ?? [];
+    if (
+      command?.effectKind === "subagent" &&
+      command.generation > 0 &&
+      reservations.some(
+        ({ reservationId }) => reservationId === command.budgetReservationId,
+      )
+    ) {
+      const stable = await appendRecoveryFailure(
+        dependencies,
+        request,
+        recoveryNumber,
+        command.commandId,
+      );
+      await failRunForRecovery(dependencies, {
+        runId: request.runId,
+        safeCode: "SUBAGENT_USAGE_UNRECONCILED",
+        at: request.at,
+      });
+      return { outcome: "failed" as const, admission, stable };
+    }
+    const result = await convergeFailedNodePolicy(dependencies, {
+      snapshot: current,
+      node,
+      nodeRun,
+      at: request.at,
+    });
+    if (result.outcome !== "fail-run") continue;
+    const stable = await appendRecoveryFailure(
+      dependencies,
+      request,
+      recoveryNumber,
+    );
+    await failRunForRecovery(dependencies, {
+      runId: request.runId,
+      safeCode: result.safeCode,
+      at: request.at,
+    });
+    return { outcome: "failed" as const, admission, stable };
+  }
+  return undefined;
+}
+
+export async function applyRecoveryFailurePolicy(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRunId: string,
+  safeCode: string,
+  at: string,
+) {
+  const snapshot = await dependencies.requireSnapshot(runId);
+  const nodeRun = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRunId,
+  );
+  if (!nodeRun || isAgentWorkflowNodeTerminal(nodeRun.status)) return;
+  const release = await dependencies.requireRelease(snapshot.run.releaseId);
+  const node = release.definition.nodes.find(({ id }) => id === nodeRun.nodeId);
+  if (!node) throw new Error("Workflow Node Definition 不存在");
+  await handleScheduledNodeFailure(
+    dependencies,
+    snapshot,
+    node,
+    nodeRun,
+    safeCode,
+    monotonicAt(at, nodeRun.updatedAt),
+  );
+}
+
+export async function appendRecoveryFailure(
+  dependencies: SchedulerExecutionDependencies,
+  request: RecoveryCoordinate,
+  recoveryNumber: number,
+  commandId?: string,
+) {
+  return appendRecoveryFact({
+    repository: dependencies.repository,
+    runId: request.runId,
+    recoveryNumber,
+    outcome: "failed",
+    commandId,
+    at: await recoveryEventAt(dependencies, request.runId, request.at),
+  });
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-reservation-convergence.ts

@@ -0,0 +1,40 @@
+import type {
+  AgentWorkflowSideEffectCommand,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowService } from "./service.js";
+
+export async function convergeFailedReservation(input: {
+  readonly service: AgentWorkflowService;
+  readonly command: AgentWorkflowSideEffectCommand | undefined;
+  readonly cumulativeUsage?: AgentWorkflowUsage;
+  readonly safeCode: string;
+  readonly at: string;
+}) {
+  if (!input.command?.budgetReservationId) return { settled: false };
+  if (!input.cumulativeUsage) return { settled: false };
+  const ledger = await input.service.readBudget(input.command.runId);
+  const reservation = ledger?.activeReservations.find(
+    ({ reservationId }) => reservationId === input.command!.budgetReservationId,
+  );
+  if (!reservation) return { settled: true };
+  const settled = await input.service.settleBudget({
+    reservationId: reservation.reservationId,
+    parentRunId: reservation.parentRunId,
+    expectedReservationVersion: reservation.version,
+    consumed: input.cumulativeUsage,
+    usageReceiptId: `failed-usage:${input.command.commandId}:${input.command.generation}`,
+    authorizationSnapshotDigest: reservation.authorizationSnapshotDigest,
+    runPolicyDigest: reservation.runPolicyDigest,
+    settledAt: input.at,
+  });
+  if (
+    settled.outcome === "settled" ||
+    settled.outcome === "replayed" ||
+    settled.outcome === "usage-unknown" ||
+    settled.outcome === "budget-exceeded"
+  ) {
+    return { settled: true };
+  }
+  return { settled: false };
+}