create-windy 0.6.12 → 0.6.14

package/template/apps/server/src/agent-workflow/scheduler-recovery-effect.ts

@@ -0,0 +1,331 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+  AgentWorkflowSideEffectCommand,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { AgentWorkflowAdapterDispatchResult } from "./adapter-dispatch.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { convergeTimeout } from "./scheduler-execution-convergence.js";
+import {
+  dispatchResultAt,
+  failScheduledEffect,
+  waitScheduledNode,
+} from "./scheduler-effect-result.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  awaitScheduledNode,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export type SchedulerRecoveryStep =
+  | { readonly outcome: "completed" }
+  | { readonly outcome: "waiting" }
+  | {
+      readonly outcome: "failed";
+      readonly safeCode: string;
+      readonly policyEligible: boolean;
+    };
+
+export async function recoverPersistedEffect(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly release: AgentWorkflowRelease;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly command: AgentWorkflowSideEffectCommand;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+): Promise<SchedulerRecoveryStep> {
+  if (input.command.status === "failed") {
+    if (input.command.safeErrorCode === "NODE_DEADLINE_EXCEEDED") {
+      await dependencies.serializeMutation(() =>
+        convergeTimeout(
+          dependencies,
+          input.snapshot.run.runId,
+          input.nodeRun,
+          monotonicAt(input.at, input.command.updatedAt),
+        ),
+      );
+      return { outcome: "completed" };
+    }
+    return {
+      outcome: "failed",
+      safeCode: input.command.safeErrorCode ?? "ADAPTER_DISPATCH_FAILED",
+      policyEligible: true,
+    };
+  }
+  const adapter = dependencies.adapters.require(input.node.type);
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: input.snapshot.run,
+    node: input.node,
+    nodeRun: input.nodeRun,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot: input.snapshot,
+    release: input.release,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  try {
+    const receipt = await dependencies.effectFence.reconcile(
+      input.command.commandId,
+    );
+    if (
+      !receipt &&
+      input.command.status === "prepared" &&
+      (deadline.expired() || Date.parse(input.at) >= Date.parse(deadlineAt))
+    ) {
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        "NODE_DEADLINE_EXCEEDED",
+        monotonicAt(input.at, deadlineAt),
+      );
+      await dependencies.serializeMutation(() =>
+        convergeTimeout(
+          dependencies,
+          input.snapshot.run.runId,
+          input.nodeRun,
+          monotonicAt(input.at, deadlineAt),
+        ),
+      );
+      return { outcome: "completed" };
+    }
+    const claimed = receipt
+      ? { outcome: "replayed" as const, receipt }
+      : input.command.status === "prepared"
+        ? await dependencies.effectFence.claim({
+            command: input.command,
+            expectedCommandVersion: input.command.version,
+            authorizationFence: input.authorizationFence,
+            budgetReservationId: input.command.budgetReservationId,
+            deadlineAt,
+          })
+        : await dependencies.effectFence.claimReconciliation({
+            command: input.command,
+            expectedCommandVersion: input.command.version,
+          });
+    if (claimed.outcome === "conflict") {
+      return { outcome: "waiting" };
+    }
+    if (claimed.outcome === "authorization-stale") {
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        claimed.safeCode,
+        monotonicAt(input.at),
+      );
+      return {
+        outcome: "failed",
+        safeCode: claimed.safeCode,
+        policyEligible: true,
+      };
+    }
+    if (!("fence" in claimed) && !("receipt" in claimed)) {
+      throw new Error("EFFECT_CLAIM_OUTCOME_UNHANDLED");
+    }
+    const result =
+      input.command.status === "prepared" && claimed.outcome === "claimed"
+        ? adapter.dispatch
+          ? await awaitScheduledNode(
+              adapter.dispatch({
+                context,
+                command: input.command,
+                fence: claimed.fence,
+              }),
+              deadline,
+            )
+          : { outcome: "failed" as const, safeCode: "ADAPTER_DISPATCH_MISSING" }
+        : adapter.reconcile
+          ? await adapter.reconcile({
+              context: { ...context, signal: input.signal },
+              command: input.command,
+              authorizationFence: input.authorizationFence,
+              ...("receipt" in claimed
+                ? { committedReceipt: claimed.receipt }
+                : { fence: claimed.fence }),
+            })
+          : { outcome: "pending" as const };
+    const at = monotonicAt(input.at, dispatchResultAt(input.at, result));
+    if (result.outcome === "pending" || result.outcome === "waiting") {
+      if (
+        result.outcome === "waiting" &&
+        input.nodeRun.status === "waiting" &&
+        !sameAdapter(input.nodeRun.adapter, result.adapter)
+      ) {
+        return {
+          outcome: "failed",
+          safeCode: "ADAPTER_COORDINATE_DRIFT",
+          policyEligible: false,
+        };
+      }
+      if (input.nodeRun.status === "running") {
+        await waitScheduledNode(
+          dependencies,
+          input.snapshot,
+          input.nodeRun,
+          result.outcome === "waiting" ? result.adapter : input.command.adapter,
+          at,
+        );
+      }
+      return { outcome: "waiting" };
+    }
+    if (result.outcome === "failed") {
+      if (
+        receipt ||
+        input.command.status === "committed" ||
+        input.command.status === "reconciled"
+      ) {
+        return {
+          outcome: "failed",
+          safeCode: result.safeCode,
+          policyEligible: false,
+        };
+      }
+      await failScheduledEffect(
+        dependencies,
+        input.command.commandId,
+        result.safeCode,
+        at,
+      );
+      return {
+        outcome: "failed",
+        safeCode: result.safeCode,
+        policyEligible: true,
+      };
+    }
+    if (claimed.outcome === "claimed") {
+      const committed = await dependencies.effectFence.commit({
+        fence: claimed.fence,
+        receipt: result.receipt,
+      });
+      if (committed.outcome === "conflict") {
+        return { outcome: "waiting" };
+      }
+    }
+    return completeRecoveredEffect(dependencies, input, result, at);
+  } catch {
+    if (input.signal.aborted) return { outcome: "waiting" };
+    return { outcome: "waiting" };
+  }
+}
+
+async function completeRecoveredEffect(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof recoverPersistedEffect>[1],
+  result: Extract<AgentWorkflowAdapterDispatchResult, { outcome: "committed" }>,
+  at: string,
+): Promise<SchedulerRecoveryStep> {
+  let snapshot = await dependencies.requireSnapshot(input.snapshot.run.runId);
+  let nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun)
+    return {
+      outcome: "failed",
+      safeCode: "NODE_RUN_MISSING",
+      policyEligible: false,
+    };
+  let usage = result.cumulativeUsage;
+  if (usage && input.command.budgetReservationId) {
+    const ledger = await dependencies.service.readBudget(snapshot.run.runId);
+    const reservation = ledger?.activeReservations.find(
+      ({ reservationId }) =>
+        reservationId === input.command.budgetReservationId,
+    );
+    if (!reservation) {
+      return {
+        outcome: "failed",
+        safeCode: "BUDGET_RESERVATION_MISSING",
+        policyEligible: false,
+      };
+    }
+    const settled = await dependencies.service.settleBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: snapshot.run.runId,
+      expectedReservationVersion: reservation.version,
+      consumed: usage,
+      usageReceiptId: result.receipt.receiptId,
+      authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+      runPolicyDigest: snapshot.run.executionPolicyDigest,
+      settledAt: at,
+    });
+    if ("safeCode" in settled) {
+      return {
+        outcome: "failed",
+        safeCode: settled.safeCode,
+        policyEligible: false,
+      };
+    }
+    usage = undefined;
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+    );
+    if (!nodeRun)
+      return {
+        outcome: "failed",
+        safeCode: "NODE_RUN_MISSING",
+        policyEligible: false,
+      };
+  }
+  try {
+    await dependencies.service.transitionNode({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedRunVersion: usage ? snapshot.run.version : undefined,
+      expectedSequence: snapshot.headSequence,
+      command: {
+        type: "complete",
+        receipt: result.nodeResult,
+        outputArtifacts: result.outputArtifacts,
+      },
+      cumulativeUsage: usage,
+      effectCommandId: input.command.commandId,
+      idempotencyKey: `scheduler:recovery:complete:${input.command.commandId}`,
+      at,
+    });
+    return { outcome: "completed" };
+  } catch {
+    const current = await dependencies.requireSnapshot(snapshot.run.runId);
+    const latest = current.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+    );
+    return latest?.status === "completed"
+      ? { outcome: "completed" }
+      : {
+          outcome: "failed",
+          safeCode: "NODE_RESULT_COMMIT_FAILED",
+          policyEligible: false,
+        };
+  }
+}
+
+function sameAdapter(
+  left: AgentWorkflowNodeRun["adapter"],
+  right: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+) {
+  return left?.kind === right.kind && left.coordinate === right.coordinate;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-event-time.ts

@@ -0,0 +1,26 @@
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+
+export async function recoveryEventAt(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  at: string,
+) {
+  const snapshot = await dependencies.requireSnapshot(runId);
+  const tail = await dependencies.repository.readEvents({
+    runId,
+    afterSequence: Math.max(0, snapshot.headSequence - 1),
+    limit: 1,
+  });
+  return new Date(
+    Math.max(
+      Date.now(),
+      Date.parse(at) || 0,
+      Date.parse(snapshot.run.updatedAt) || 0,
+      ...snapshot.nodeRuns.map(({ updatedAt }) => Date.parse(updatedAt) || 0),
+      ...snapshot.sideEffects.map(
+        ({ updatedAt }) => Date.parse(updatedAt) || 0,
+      ),
+      ...tail.map(({ occurredAt }) => Date.parse(occurredAt) || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-failed-policy.ts

@@ -0,0 +1,129 @@
+import {
+  isAgentWorkflowNodeTerminal,
+  isAgentWorkflowTerminal,
+  type AgentWorkflowNode,
+  type AgentWorkflowNodeRun,
+} from "@southwind-ai/shared";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failureDecision } from "./scheduler-support.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export async function convergeFailedNodePolicy(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly at: string;
+  },
+) {
+  if (input.nodeRun.status !== "failed")
+    return { outcome: "converged" as const };
+  const decision = failureDecision(input.node, input.nodeRun);
+  const safeCode = input.nodeRun.safeErrorCode ?? "NODE_EXECUTION_FAILED";
+  if (decision.type === "retry") {
+    const existing = input.snapshot.nodeRuns.some(
+      ({ previousNodeRunId }) => previousNodeRunId === input.nodeRun.nodeRunId,
+    );
+    if (!existing) {
+      await dependencies.service.scheduleRetry({
+        runId: input.snapshot.run.runId,
+        nodeRunId: input.nodeRun.nodeRunId,
+        expectedNodeVersion: input.nodeRun.version,
+        expectedSequence: input.snapshot.headSequence,
+        safeCode,
+        idempotencyKey: `scheduler:recovery:retry:${input.nodeRun.nodeRunId}:${input.nodeRun.attempt + 1}`,
+        at: monotonicAt(input.at, input.nodeRun.updatedAt),
+      });
+    }
+    return { outcome: "converged" as const };
+  }
+  if (decision.type === "fallback") {
+    const release = await dependencies.requireRelease(
+      input.snapshot.run.releaseId,
+    );
+    const fallback = release.definition.nodes.find(
+      ({ id }) => id === decision.nodeId,
+    );
+    if (!fallback) {
+      return {
+        outcome: "fail-run" as const,
+        safeCode: "FALLBACK_NODE_MISSING",
+      };
+    }
+    if (!input.snapshot.nodeRuns.some(({ nodeId }) => nodeId === fallback.id)) {
+      await dependencies.insertReady(
+        input.snapshot,
+        [fallback],
+        input.nodeRun,
+        monotonicAt(input.at, input.nodeRun.updatedAt),
+      );
+    }
+    return { outcome: "converged" as const };
+  }
+  return {
+    outcome: "fail-run" as const,
+    safeCode:
+      decision.type === "skip" ? "FAILED_SKIP_POLICY_INCONSISTENT" : safeCode,
+  };
+}
+
+export async function failRunForRecovery(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly runId: string;
+    readonly safeCode: string;
+    readonly at: string;
+  },
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  return dependencies.service.transitionRun({
+    runId: input.runId,
+    expectedVersion: snapshot.run.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "fail", safeCode: input.safeCode },
+    idempotencyKey: `scheduler:recovery:fail-run:${input.runId}:${snapshot.recoveryCount}`,
+    at: monotonicAt(input.at, snapshot.run.updatedAt),
+  });
+}
+
+export async function failNodeAndRunForRecovery(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly runId: string;
+    readonly nodeRunId: string;
+    readonly safeCode: string;
+    readonly at: string;
+  },
+) {
+  let snapshot = await dependencies.requireSnapshot(input.runId);
+  const nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRunId,
+  );
+  if (nodeRun && !isAgentWorkflowNodeTerminal(nodeRun.status)) {
+    await dependencies.service.transitionNode({
+      runId: input.runId,
+      nodeRunId: input.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedSequence: snapshot.headSequence,
+      command: { type: "fail", safeCode: input.safeCode },
+      idempotencyKey: `scheduler:recovery:unsafe-fail:${input.nodeRunId}`,
+      at: monotonicAt(input.at, nodeRun.updatedAt),
+    });
+  }
+  snapshot = await dependencies.requireSnapshot(input.runId);
+  if (isAgentWorkflowTerminal(snapshot.run.status)) return;
+  await failRunForRecovery(dependencies, input);
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-node.ts

@@ -0,0 +1,208 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { AgentWorkflowNodeAdapter } from "./adapter-dispatch.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { waitScheduledNode } from "./scheduler-effect-result.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  recoverPersistedEffect,
+  type SchedulerRecoveryStep,
+} from "./scheduler-recovery-effect.js";
+import { prepareAgentWorkflowSideEffect } from "./side-effect-state.js";
+import { workflowId } from "./service-support.js";
+import {
+  awaitScheduledNode,
+  SchedulerDeadlineExceeded,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+import { convergeTimedOutPreparation } from "./scheduler-execution-convergence.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export async function recoverUncommandedNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly snapshot: Snapshot;
+    readonly release: AgentWorkflowRelease;
+    readonly node: AgentWorkflowNode;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+): Promise<SchedulerRecoveryStep> {
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: input.snapshot.run,
+    node: input.node,
+    nodeRun: input.nodeRun,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot: input.snapshot,
+    release: input.release,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  const adapter = dependencies.adapters.require(input.node.type);
+  let preparation: Awaited<ReturnType<AgentWorkflowNodeAdapter["prepare"]>>;
+  try {
+    preparation = await awaitScheduledNode(adapter.prepare(context), deadline);
+  } catch (error) {
+    if (input.signal.aborted) return { outcome: "waiting" };
+    if (error instanceof SchedulerDeadlineExceeded) {
+      await convergeTimedOutPreparation(
+        dependencies,
+        {
+          release: input.release,
+          nodeRun: input.nodeRun,
+          node: input.node,
+          runId: input.snapshot.run.runId,
+          authorizationFence: input.authorizationFence,
+          at: input.at,
+          signal: input.signal,
+        },
+        input.snapshot,
+        input.nodeRun,
+        adapter,
+        deadlineAt,
+      );
+      const current = await dependencies.requireSnapshot(
+        input.snapshot.run.runId,
+      );
+      return current.nodeRuns.some(
+        ({ nodeRunId, status }) =>
+          nodeRunId === input.nodeRun.nodeRunId && status === "timed-out",
+      )
+        ? { outcome: "completed" }
+        : { outcome: "waiting" };
+    }
+    return {
+      outcome: "failed",
+      safeCode: "ADAPTER_PREPARATION_FAILED",
+      policyEligible: true,
+    };
+  }
+  const at = monotonicAt(
+    input.at,
+    input.nodeRun.updatedAt,
+    preparation.outcome === "direct"
+      ? preparation.result.recordedAt
+      : undefined,
+  );
+  if (preparation.outcome === "failed") {
+    return {
+      outcome: "failed",
+      safeCode: preparation.safeCode,
+      policyEligible: true,
+    };
+  }
+  if (preparation.outcome === "direct") {
+    try {
+      await dependencies.service.transitionNode({
+        runId: input.snapshot.run.runId,
+        nodeRunId: input.nodeRun.nodeRunId,
+        expectedNodeVersion: input.nodeRun.version,
+        expectedRunVersion: preparation.cumulativeUsage
+          ? input.snapshot.run.version
+          : undefined,
+        expectedSequence: input.snapshot.headSequence,
+        command: {
+          type: "complete",
+          receipt: preparation.result,
+          outputArtifacts: preparation.outputArtifacts,
+        },
+        cumulativeUsage: preparation.cumulativeUsage,
+        idempotencyKey: `scheduler:recovery:complete:${input.nodeRun.nodeRunId}`,
+        at,
+      });
+      return { outcome: "completed" };
+    } catch {
+      const current = await dependencies.requireSnapshot(
+        input.snapshot.run.runId,
+      );
+      const latest = current.nodeRuns.find(
+        ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+      );
+      return latest?.status === "completed"
+        ? { outcome: "completed" }
+        : {
+            outcome: "failed",
+            safeCode: "NODE_RESULT_COMMIT_FAILED",
+            policyEligible: true,
+          };
+    }
+  }
+  if (preparation.outcome === "waiting") {
+    if (
+      input.nodeRun.status === "waiting" &&
+      !sameAdapter(input.nodeRun.adapter, preparation.adapter)
+    ) {
+      return {
+        outcome: "failed",
+        safeCode: "ADAPTER_COORDINATE_DRIFT",
+        policyEligible: true,
+      };
+    }
+    if (input.nodeRun.status === "running") {
+      await waitScheduledNode(
+        dependencies,
+        input.snapshot,
+        input.nodeRun,
+        preparation.adapter,
+        at,
+      );
+    }
+    return { outcome: "waiting" };
+  }
+  const command = prepareAgentWorkflowSideEffect({
+    commandId: workflowId(
+      "workflow_effect",
+      input.nodeRun.nodeRunId,
+      preparation.effectKey,
+    ),
+    runId: input.snapshot.run.runId,
+    nodeRunId: input.nodeRun.nodeRunId,
+    effectKind: preparation.effectKind,
+    effectKey: preparation.effectKey,
+    adapter: preparation.adapter,
+    idempotencyKey: workflowId(
+      "workflow_effect_idem",
+      input.nodeRun.idempotencyKey,
+      preparation.effectKey,
+    ),
+    requestDigest: preparation.requestDigest,
+    budgetReservationId: preparation.budgetReservationId,
+    delegationId: preparation.delegationId,
+    createdAt: at,
+  });
+  const persisted = await dependencies.repository.prepareSideEffect(command);
+  return recoverPersistedEffect(dependencies, {
+    ...input,
+    command: persisted.command,
+    at,
+  });
+}
+
+function sameAdapter(
+  left: AgentWorkflowNodeRun["adapter"],
+  right: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+) {
+  return left?.kind === right.kind && left.coordinate === right.coordinate;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}