create-windy 0.6.12 → 0.6.15

package/template/apps/server/src/agent-workflow/scheduler-execution.ts

@@ -0,0 +1,211 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type {
+  AgentWorkflowNodeAdapter,
+  AgentWorkflowNodeAdapterRegistry,
+} from "./adapter-dispatch.js";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import type { WorkflowEffectCommandFencePort } from "./effect-command-fence.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  awaitScheduledNode,
+  SchedulerDeadlineExceeded,
+  scheduledNodeDeadlineAt,
+  scheduledNodeSignal,
+} from "./scheduler-deadline.js";
+import {
+  convergeFailure,
+  convergeTimedOutPreparation,
+  convergeWait,
+  currentScheduledNode,
+} from "./scheduler-execution-convergence.js";
+import { executePreparedSideEffect } from "./scheduler-side-effect-execution.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export interface SchedulerExecutionDependencies {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly effectFence: WorkflowEffectCommandFencePort;
+  readonly adapters: AgentWorkflowNodeAdapterRegistry;
+  readonly requireSnapshot: (runId: string) => Promise<Snapshot>;
+  readonly requireRelease: (releaseId: string) => Promise<AgentWorkflowRelease>;
+  readonly insertReady: (
+    snapshot: Snapshot,
+    nodes: readonly AgentWorkflowNode[],
+    source: AgentWorkflowNodeRun,
+    at: string,
+  ) => Promise<unknown>;
+  readonly serializeMutation: <T>(mutation: () => Promise<T>) => Promise<T>;
+}
+
+export async function executeScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly node: AgentWorkflowNode;
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+) {
+  const running = await claimScheduledNode(dependencies, input);
+  return executeClaimedScheduledNode(dependencies, {
+    ...input,
+    nodeRun: running,
+  });
+}
+
+export async function claimScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  await dependencies.service.transitionNode({
+    runId: input.runId,
+    nodeRunId: input.nodeRun.nodeRunId,
+    expectedNodeVersion: input.nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "start" },
+    idempotencyKey: `scheduler:start:${input.nodeRun.nodeRunId}`,
+    at: input.at,
+  });
+  const claimed = await dependencies.requireSnapshot(input.runId);
+  return claimed.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  )!;
+}
+
+export async function executeClaimedScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+) {
+  const snapshot = await dependencies.requireSnapshot(input.runId);
+  const running = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  )!;
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: snapshot.run,
+    node: input.node,
+    nodeRun: running,
+  });
+  const deadline = scheduledNodeSignal(input.signal, deadlineAt);
+  const context = schedulerAdapterContext({
+    snapshot,
+    release: input.release,
+    nodeRun: running,
+    node: input.node,
+    signal: deadline.signal,
+  });
+  const adapter = dependencies.adapters.require(input.node.type);
+  let preparation: Awaited<ReturnType<AgentWorkflowNodeAdapter["prepare"]>>;
+  try {
+    preparation = await awaitScheduledNode(adapter.prepare(context), deadline);
+  } catch (error) {
+    if (input.signal.aborted) return { outcome: "aborted" as const };
+    if (error instanceof SchedulerDeadlineExceeded) {
+      return convergeTimedOutPreparation(
+        dependencies,
+        input,
+        snapshot,
+        running,
+        adapter,
+        deadlineAt,
+      );
+    }
+    return dependencies.serializeMutation(() =>
+      convergeFailure(
+        dependencies,
+        input,
+        running,
+        "ADAPTER_PREPARATION_FAILED",
+        monotonicAt(input.at, running.updatedAt),
+      ),
+    );
+  }
+  const preparedAt = monotonicAt(
+    input.at,
+    running.updatedAt,
+    preparation.outcome === "direct"
+      ? preparation.result.recordedAt
+      : undefined,
+  );
+  if (preparation.outcome === "direct") {
+    return dependencies.serializeMutation(async () => {
+      const latest = await currentScheduledNode(
+        dependencies,
+        input.runId,
+        running,
+      );
+      return dependencies.service.transitionNode({
+        runId: input.runId,
+        nodeRunId: latest.nodeRun.nodeRunId,
+        expectedNodeVersion: latest.nodeRun.version,
+        expectedRunVersion: preparation.cumulativeUsage
+          ? latest.snapshot.run.version
+          : undefined,
+        expectedSequence: latest.snapshot.headSequence,
+        command: {
+          type: "complete",
+          receipt: preparation.result,
+          outputArtifacts: preparation.outputArtifacts,
+        },
+        cumulativeUsage: preparation.cumulativeUsage,
+        idempotencyKey: `scheduler:complete:${latest.nodeRun.nodeRunId}`,
+        at: preparedAt,
+      });
+    });
+  }
+  if (preparation.outcome === "waiting") {
+    return dependencies.serializeMutation(() =>
+      convergeWait(
+        dependencies,
+        input.runId,
+        running,
+        preparation.adapter,
+        preparedAt,
+      ),
+    );
+  }
+  if (preparation.outcome === "failed") {
+    return dependencies.serializeMutation(() =>
+      convergeFailure(
+        dependencies,
+        input,
+        running,
+        preparation.safeCode,
+        preparedAt,
+        preparation.cumulativeUsage,
+      ),
+    );
+  }
+  return executePreparedSideEffect(dependencies, {
+    input,
+    snapshot,
+    running,
+    context,
+    adapter,
+    preparation,
+    preparedAt,
+    deadlineAt,
+    deadline,
+  });
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-failure.ts

@@ -0,0 +1,102 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failureDecision } from "./scheduler-support.js";
+import { releaseActiveNodeReservations } from "./scheduler-cancellation-support.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export async function handleScheduledNodeFailure(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  node: AgentWorkflowNode,
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const released = await releaseActiveNodeReservations({
+    service: dependencies.service,
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    at,
+    reasonCode: "NODE_EXECUTION_FAILED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  )!;
+  const decision = [
+    "MODEL_USAGE_UNMEASURED",
+    "USAGE_UNKNOWN_OR_EXCEEDED",
+  ].includes(safeCode)
+    ? ({ type: "fail-run" } as const)
+    : failureDecision(node, nodeRun);
+  if (decision.type === "skip") {
+    return dependencies.service.transitionNode({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version,
+      expectedSequence: snapshot.headSequence,
+      command: { type: "skip" },
+      expectedRunVersion: cumulativeUsage ? snapshot.run.version : undefined,
+      cumulativeUsage,
+      idempotencyKey: `scheduler:skip:${nodeRun.nodeRunId}`,
+      at,
+    });
+  }
+  const failed = await dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "fail", safeCode },
+    expectedRunVersion: cumulativeUsage ? snapshot.run.version : undefined,
+    cumulativeUsage,
+    idempotencyKey: `scheduler:fail:${nodeRun.nodeRunId}`,
+    at,
+  });
+  if (decision.type === "fallback") {
+    const latest = await dependencies.requireSnapshot(snapshot.run.runId);
+    const release = await dependencies.requireRelease(snapshot.run.releaseId);
+    const fallback = release.definition.nodes.find(
+      ({ id }) => id === decision.nodeId,
+    );
+    if (!fallback) throw new Error("Fallback Node 不存在");
+    return dependencies.insertReady(latest, [fallback], nodeRun, at);
+  }
+  if (decision.type === "retry") {
+    const scheduled = await dependencies.service.scheduleRetry({
+      runId: snapshot.run.runId,
+      nodeRunId: nodeRun.nodeRunId,
+      expectedNodeVersion: nodeRun.version + 1,
+      expectedSequence: failed.record.headSequence,
+      safeCode,
+      idempotencyKey: `scheduler:retry:${nodeRun.nodeRunId}:${nodeRun.attempt + 1}`,
+      at,
+    });
+    return {
+      outcome: "retry-after" as const,
+      scheduled,
+      retryAt: new Date(
+        Date.parse(at) + decision.backoffSeconds * 1_000,
+      ).toISOString(),
+    };
+  }
+  const latest = await dependencies.requireSnapshot(snapshot.run.runId);
+  return dependencies.service.transitionRun({
+    runId: snapshot.run.runId,
+    expectedVersion: latest.run.version,
+    expectedSequence: failed.record.headSequence,
+    command: { type: "fail", safeCode },
+    idempotencyKey: `scheduler:fail-run:${nodeRun.nodeRunId}`,
+    at,
+  });
+}

package/template/apps/server/src/agent-workflow/scheduler-join-cancellation.ts

@@ -0,0 +1,282 @@
+import type { AgentWorkflowNode } from "@southwind-ai/shared";
+import type { AgentWorkflowNodeAdapterRegistry } from "./adapter-dispatch.js";
+import type { WorkflowEffectCommandFencePort } from "./effect-command-fence.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  cancelPreparedEffect,
+  reconcileEffectDuringCancellation,
+} from "./scheduler-cancellation-reconcile.js";
+import {
+  monotonicCancellationAt,
+  releaseCancelledReservation,
+  releaseActiveNodeReservations,
+  requireCancellationSnapshot,
+  terminalizeCancelledNode,
+} from "./scheduler-cancellation-support.js";
+import type { AgentWorkflowService } from "./service.js";
+import { transitionAgentWorkflowSideEffect } from "./side-effect-state.js";
+import { joinAnyLoserNodeIds } from "./scheduler-support.js";
+import { convergeFailedReservation } from "./scheduler-reservation-convergence.js";
+
+export async function cancelJoinAnyLosers(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly effectFence: WorkflowEffectCommandFencePort;
+  readonly adapters: AgentWorkflowNodeAdapterRegistry;
+  readonly join: Extract<AgentWorkflowNode, { readonly type: "join" }>;
+  readonly runId: string;
+  readonly at: string;
+  readonly signal: AbortSignal;
+}) {
+  let snapshot = await requireCancellationSnapshot(
+    input.repository,
+    input.runId,
+  );
+  const release = await input.repository.getRelease(snapshot.run.releaseId);
+  if (!release) throw new Error("Workflow Release 不存在");
+  const loserNodeIds = joinAnyLoserNodeIds({
+    definition: release.definition,
+    joinId: input.join.id,
+    nodeRuns: snapshot.nodeRuns,
+  });
+  const losers = snapshot.nodeRuns.filter(
+    ({ nodeId, status }) => loserNodeIds.has(nodeId) && !isTerminal(status),
+  );
+  const unresolved: string[] = [];
+  for (const loser of losers) {
+    snapshot = await requireCancellationSnapshot(input.repository, input.runId);
+    const nodeRun = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === loser.nodeRunId,
+    );
+    if (!nodeRun || isTerminal(nodeRun.status)) continue;
+    const node = release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    if (!node) {
+      unresolved.push(nodeRun.nodeRunId);
+      continue;
+    }
+    const command = snapshot.sideEffects.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    );
+    if (!command) {
+      const cancellation = await cancelAdapterWithoutEffect({
+        input,
+        snapshot,
+        release,
+        nodeRun,
+        node,
+      });
+      if (!cancellation.cancelled) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      const released = await releaseActiveNodeReservations({
+        service: input.service,
+        runId: input.runId,
+        nodeRunId: nodeRun.nodeRunId,
+        at: input.at,
+        reasonCode: "JOIN_ANY_LOSER_CANCELLED",
+      });
+      if (!released) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      await terminalize(input, nodeRun.nodeRunId, cancellation.cumulativeUsage);
+      continue;
+    }
+    if (command?.status === "prepared") {
+      await cancelPreparedEffect({
+        repository: input.repository,
+        command,
+        at: input.at,
+      });
+      if (!(await releaseReservation(input, command))) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+    } else {
+      snapshot = await requireCancellationSnapshot(
+        input.repository,
+        input.runId,
+      );
+      const reconciliation = await reconcileEffectDuringCancellation({
+        repository: input.repository,
+        service: input.service,
+        effectFence: input.effectFence,
+        adapters: input.adapters,
+        snapshot,
+        nodeRunId: nodeRun.nodeRunId,
+        at: input.at,
+        signal: input.signal,
+      });
+      if (reconciliation.outcome === "completed") continue;
+      if (reconciliation.outcome === "failed") {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      snapshot = await requireCancellationSnapshot(
+        input.repository,
+        input.runId,
+      );
+      let latest = snapshot.sideEffects.find(
+        ({ commandId }) => commandId === command.commandId,
+      );
+      if (!latest || ["committed", "reconciled"].includes(latest.status)) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      if (!["failed", "cancelled"].includes(latest.status)) {
+        if (latest.status === "dispatched") {
+          latest = await input.repository.updateSideEffect(
+            transitionAgentWorkflowSideEffect({
+              command: latest,
+              expectedVersion: latest.version,
+              transition: { type: "request-cancel" },
+              at: monotonicCancellationAt(input.at, latest.updatedAt),
+            }),
+            latest.version,
+          );
+        }
+      }
+      let cancellation:
+        | Awaited<
+            ReturnType<
+              NonNullable<ReturnType<typeof input.adapters.require>["cancel"]>
+            >
+          >
+        | undefined;
+      try {
+        cancellation = await input.adapters.require(node.type).cancel?.({
+          context: schedulerAdapterContext({
+            snapshot,
+            release,
+            nodeRun,
+            node,
+            signal: input.signal,
+          }),
+          command: latest,
+        });
+      } catch {
+        cancellation = undefined;
+      }
+      if (!cancellation?.cancelled) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      if (latest.budgetReservationId) {
+        if (!cancellation.cumulativeUsage) {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+        const convergence = await convergeFailedReservation({
+          service: input.service,
+          command: latest,
+          cumulativeUsage: cancellation.cumulativeUsage,
+          safeCode: cancellation.safeCode ?? "JOIN_ANY_LOSER_CANCELLED",
+          at: input.at,
+        });
+        if (!convergence.settled) {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+      }
+      if (latest.status === "cancellation-requested") {
+        try {
+          latest = await input.repository.updateSideEffect(
+            transitionAgentWorkflowSideEffect({
+              command: latest,
+              expectedVersion: latest.version,
+              transition: { type: "cancel" },
+              at: monotonicCancellationAt(input.at, latest.updatedAt),
+            }),
+            latest.version,
+          );
+        } catch {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+      }
+      if (!(await releaseReservation(input, latest))) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      await terminalize(
+        input,
+        nodeRun.nodeRunId,
+        latest.budgetReservationId ? undefined : cancellation.cumulativeUsage,
+      );
+      continue;
+    }
+    await terminalize(input, nodeRun.nodeRunId);
+  }
+  return unresolved.length
+    ? { outcome: "cancellation-pending" as const, unresolved }
+    : { outcome: "cancelled" as const, count: losers.length };
+}
+
+async function cancelAdapterWithoutEffect(input: {
+  readonly input: Parameters<typeof cancelJoinAnyLosers>[0];
+  readonly snapshot: Awaited<ReturnType<typeof requireCancellationSnapshot>>;
+  readonly release: NonNullable<
+    Awaited<ReturnType<AgentWorkflowRepository["getRelease"]>>
+  >;
+  readonly nodeRun: Awaited<
+    ReturnType<typeof requireCancellationSnapshot>
+  >["nodeRuns"][number];
+  readonly node: AgentWorkflowNode;
+}) {
+  const adapter = input.input.adapters.require(input.node.type);
+  if (!adapter.cancel) return { cancelled: true };
+  try {
+    const result = await adapter.cancel({
+      context: schedulerAdapterContext({
+        snapshot: input.snapshot,
+        release: input.release,
+        nodeRun: input.nodeRun,
+        node: input.node,
+        signal: input.input.signal,
+      }),
+    });
+    return result.cancelled && result.cumulativeUsage
+      ? result
+      : { cancelled: false };
+  } catch {
+    return { cancelled: false };
+  }
+}
+
+function terminalize(
+  input: Parameters<typeof cancelJoinAnyLosers>[0],
+  nodeRunId: string,
+  cumulativeUsage?: import("@southwind-ai/shared").AgentWorkflowUsage,
+) {
+  return terminalizeCancelledNode({
+    repository: input.repository,
+    service: input.service,
+    runId: input.runId,
+    nodeRunId,
+    terminalCommand: "cancel",
+    idempotencyKey: `scheduler:join-any-cancel:${input.join.id}:${nodeRunId}`,
+    at: input.at,
+    cumulativeUsage,
+  });
+}
+
+function releaseReservation(
+  input: Parameters<typeof cancelJoinAnyLosers>[0],
+  command: Parameters<typeof releaseCancelledReservation>[0]["command"],
+) {
+  return releaseCancelledReservation({
+    service: input.service,
+    command,
+    at: input.at,
+  });
+}
+
+function isTerminal(status: string) {
+  return ["completed", "failed", "cancelled", "skipped", "timed-out"].includes(
+    status,
+  );
+}

package/template/apps/server/src/agent-workflow/scheduler-mutation-queue.ts

@@ -0,0 +1,13 @@
+/** Serializes local convergence; PostgreSQL CAS remains authoritative across workers. */
+export class SchedulerMutationQueue {
+  #tail: Promise<unknown> = Promise.resolve();
+
+  run<T>(mutation: () => Promise<T>): Promise<T> {
+    const result = this.#tail.then(mutation, mutation);
+    this.#tail = result.then(
+      () => undefined,
+      () => undefined,
+    );
+    return result;
+  }
+}