create-windy 0.6.11 → 0.6.14

package/template/apps/server/src/agent-workflow/scheduler-join-cancellation.ts

@@ -0,0 +1,282 @@
+import type { AgentWorkflowNode } from "@southwind-ai/shared";
+import type { AgentWorkflowNodeAdapterRegistry } from "./adapter-dispatch.js";
+import type { WorkflowEffectCommandFencePort } from "./effect-command-fence.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import {
+  cancelPreparedEffect,
+  reconcileEffectDuringCancellation,
+} from "./scheduler-cancellation-reconcile.js";
+import {
+  monotonicCancellationAt,
+  releaseCancelledReservation,
+  releaseActiveNodeReservations,
+  requireCancellationSnapshot,
+  terminalizeCancelledNode,
+} from "./scheduler-cancellation-support.js";
+import type { AgentWorkflowService } from "./service.js";
+import { transitionAgentWorkflowSideEffect } from "./side-effect-state.js";
+import { joinAnyLoserNodeIds } from "./scheduler-support.js";
+import { convergeFailedReservation } from "./scheduler-reservation-convergence.js";
+
+export async function cancelJoinAnyLosers(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly effectFence: WorkflowEffectCommandFencePort;
+  readonly adapters: AgentWorkflowNodeAdapterRegistry;
+  readonly join: Extract<AgentWorkflowNode, { readonly type: "join" }>;
+  readonly runId: string;
+  readonly at: string;
+  readonly signal: AbortSignal;
+}) {
+  let snapshot = await requireCancellationSnapshot(
+    input.repository,
+    input.runId,
+  );
+  const release = await input.repository.getRelease(snapshot.run.releaseId);
+  if (!release) throw new Error("Workflow Release 不存在");
+  const loserNodeIds = joinAnyLoserNodeIds({
+    definition: release.definition,
+    joinId: input.join.id,
+    nodeRuns: snapshot.nodeRuns,
+  });
+  const losers = snapshot.nodeRuns.filter(
+    ({ nodeId, status }) => loserNodeIds.has(nodeId) && !isTerminal(status),
+  );
+  const unresolved: string[] = [];
+  for (const loser of losers) {
+    snapshot = await requireCancellationSnapshot(input.repository, input.runId);
+    const nodeRun = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === loser.nodeRunId,
+    );
+    if (!nodeRun || isTerminal(nodeRun.status)) continue;
+    const node = release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    if (!node) {
+      unresolved.push(nodeRun.nodeRunId);
+      continue;
+    }
+    const command = snapshot.sideEffects.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    );
+    if (!command) {
+      const cancellation = await cancelAdapterWithoutEffect({
+        input,
+        snapshot,
+        release,
+        nodeRun,
+        node,
+      });
+      if (!cancellation.cancelled) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      const released = await releaseActiveNodeReservations({
+        service: input.service,
+        runId: input.runId,
+        nodeRunId: nodeRun.nodeRunId,
+        at: input.at,
+        reasonCode: "JOIN_ANY_LOSER_CANCELLED",
+      });
+      if (!released) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      await terminalize(input, nodeRun.nodeRunId, cancellation.cumulativeUsage);
+      continue;
+    }
+    if (command?.status === "prepared") {
+      await cancelPreparedEffect({
+        repository: input.repository,
+        command,
+        at: input.at,
+      });
+      if (!(await releaseReservation(input, command))) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+    } else {
+      snapshot = await requireCancellationSnapshot(
+        input.repository,
+        input.runId,
+      );
+      const reconciliation = await reconcileEffectDuringCancellation({
+        repository: input.repository,
+        service: input.service,
+        effectFence: input.effectFence,
+        adapters: input.adapters,
+        snapshot,
+        nodeRunId: nodeRun.nodeRunId,
+        at: input.at,
+        signal: input.signal,
+      });
+      if (reconciliation.outcome === "completed") continue;
+      if (reconciliation.outcome === "failed") {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      snapshot = await requireCancellationSnapshot(
+        input.repository,
+        input.runId,
+      );
+      let latest = snapshot.sideEffects.find(
+        ({ commandId }) => commandId === command.commandId,
+      );
+      if (!latest || ["committed", "reconciled"].includes(latest.status)) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      if (!["failed", "cancelled"].includes(latest.status)) {
+        if (latest.status === "dispatched") {
+          latest = await input.repository.updateSideEffect(
+            transitionAgentWorkflowSideEffect({
+              command: latest,
+              expectedVersion: latest.version,
+              transition: { type: "request-cancel" },
+              at: monotonicCancellationAt(input.at, latest.updatedAt),
+            }),
+            latest.version,
+          );
+        }
+      }
+      let cancellation:
+        | Awaited<
+            ReturnType<
+              NonNullable<ReturnType<typeof input.adapters.require>["cancel"]>
+            >
+          >
+        | undefined;
+      try {
+        cancellation = await input.adapters.require(node.type).cancel?.({
+          context: schedulerAdapterContext({
+            snapshot,
+            release,
+            nodeRun,
+            node,
+            signal: input.signal,
+          }),
+          command: latest,
+        });
+      } catch {
+        cancellation = undefined;
+      }
+      if (!cancellation?.cancelled) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      if (latest.budgetReservationId) {
+        if (!cancellation.cumulativeUsage) {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+        const convergence = await convergeFailedReservation({
+          service: input.service,
+          command: latest,
+          cumulativeUsage: cancellation.cumulativeUsage,
+          safeCode: cancellation.safeCode ?? "JOIN_ANY_LOSER_CANCELLED",
+          at: input.at,
+        });
+        if (!convergence.settled) {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+      }
+      if (latest.status === "cancellation-requested") {
+        try {
+          latest = await input.repository.updateSideEffect(
+            transitionAgentWorkflowSideEffect({
+              command: latest,
+              expectedVersion: latest.version,
+              transition: { type: "cancel" },
+              at: monotonicCancellationAt(input.at, latest.updatedAt),
+            }),
+            latest.version,
+          );
+        } catch {
+          unresolved.push(nodeRun.nodeRunId);
+          continue;
+        }
+      }
+      if (!(await releaseReservation(input, latest))) {
+        unresolved.push(nodeRun.nodeRunId);
+        continue;
+      }
+      await terminalize(
+        input,
+        nodeRun.nodeRunId,
+        latest.budgetReservationId ? undefined : cancellation.cumulativeUsage,
+      );
+      continue;
+    }
+    await terminalize(input, nodeRun.nodeRunId);
+  }
+  return unresolved.length
+    ? { outcome: "cancellation-pending" as const, unresolved }
+    : { outcome: "cancelled" as const, count: losers.length };
+}
+
+async function cancelAdapterWithoutEffect(input: {
+  readonly input: Parameters<typeof cancelJoinAnyLosers>[0];
+  readonly snapshot: Awaited<ReturnType<typeof requireCancellationSnapshot>>;
+  readonly release: NonNullable<
+    Awaited<ReturnType<AgentWorkflowRepository["getRelease"]>>
+  >;
+  readonly nodeRun: Awaited<
+    ReturnType<typeof requireCancellationSnapshot>
+  >["nodeRuns"][number];
+  readonly node: AgentWorkflowNode;
+}) {
+  const adapter = input.input.adapters.require(input.node.type);
+  if (!adapter.cancel) return { cancelled: true };
+  try {
+    const result = await adapter.cancel({
+      context: schedulerAdapterContext({
+        snapshot: input.snapshot,
+        release: input.release,
+        nodeRun: input.nodeRun,
+        node: input.node,
+        signal: input.input.signal,
+      }),
+    });
+    return result.cancelled && result.cumulativeUsage
+      ? result
+      : { cancelled: false };
+  } catch {
+    return { cancelled: false };
+  }
+}
+
+function terminalize(
+  input: Parameters<typeof cancelJoinAnyLosers>[0],
+  nodeRunId: string,
+  cumulativeUsage?: import("@southwind-ai/shared").AgentWorkflowUsage,
+) {
+  return terminalizeCancelledNode({
+    repository: input.repository,
+    service: input.service,
+    runId: input.runId,
+    nodeRunId,
+    terminalCommand: "cancel",
+    idempotencyKey: `scheduler:join-any-cancel:${input.join.id}:${nodeRunId}`,
+    at: input.at,
+    cumulativeUsage,
+  });
+}
+
+function releaseReservation(
+  input: Parameters<typeof cancelJoinAnyLosers>[0],
+  command: Parameters<typeof releaseCancelledReservation>[0]["command"],
+) {
+  return releaseCancelledReservation({
+    service: input.service,
+    command,
+    at: input.at,
+  });
+}
+
+function isTerminal(status: string) {
+  return ["completed", "failed", "cancelled", "skipped", "timed-out"].includes(
+    status,
+  );
+}

package/template/apps/server/src/agent-workflow/scheduler-mutation-queue.ts

@@ -0,0 +1,13 @@
+/** Serializes local convergence; PostgreSQL CAS remains authoritative across workers. */
+export class SchedulerMutationQueue {
+  #tail: Promise<unknown> = Promise.resolve();
+
+  run<T>(mutation: () => Promise<T>): Promise<T> {
+    const result = this.#tail.then(mutation, mutation);
+    this.#tail = result.then(
+      () => undefined,
+      () => undefined,
+    );
+    return result;
+  }
+}

package/template/apps/server/src/agent-workflow/scheduler-node-timeout.ts

@@ -0,0 +1,139 @@
+import type {
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import {
+  deadlineCompletionAt,
+  scheduledNodeDeadlineAt,
+} from "./scheduler-deadline.js";
+import { convergeTimeout } from "./scheduler-execution-convergence.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { failScheduledEffect } from "./scheduler-effect-result.js";
+import { recoverPersistedEffect } from "./scheduler-recovery-effect.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+
+export function firstOverdueNode(input: {
+  readonly release: AgentWorkflowRelease;
+  readonly snapshot: Awaited<
+    ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+  >;
+  readonly at: string;
+}) {
+  return input.snapshot.nodeRuns.find((nodeRun) => {
+    if (
+      !nodeRun.startedAt ||
+      !["running", "waiting"].includes(nodeRun.status)
+    ) {
+      return false;
+    }
+    const node = input.release.definition.nodes.find(
+      ({ id }) => id === nodeRun.nodeId,
+    );
+    return Boolean(
+      node &&
+      Date.parse(input.at) >=
+        Date.parse(
+          scheduledNodeDeadlineAt({
+            run: input.snapshot.run,
+            node,
+            nodeRun,
+          }),
+        ),
+    );
+  });
+}
+
+export async function expireScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+  },
+) {
+  let snapshot = await dependencies.requireSnapshot(input.runId);
+  let nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun || !["running", "waiting"].includes(nodeRun.status)) {
+    return { outcome: "already-converged" as const };
+  }
+  const node = input.release.definition.nodes.find(
+    ({ id }) => id === nodeRun!.nodeId,
+  );
+  if (!node) throw new Error("Node Run 不属于 immutable Release");
+  const deadlineAt = scheduledNodeDeadlineAt({
+    run: snapshot.run,
+    node,
+    nodeRun,
+  });
+  const at = deadlineCompletionAt(deadlineAt, nodeRun.updatedAt);
+  const command = snapshot.sideEffects.find(
+    (effect) => effect.nodeRunId === nodeRun!.nodeRunId,
+  );
+  if (command && !["failed", "cancelled"].includes(command.status)) {
+    const recovered = await recoverPersistedEffect(dependencies, {
+      snapshot,
+      release: input.release,
+      node,
+      nodeRun,
+      command,
+      authorizationFence: input.authorizationFence,
+      at,
+      signal: AbortSignal.timeout(5_000),
+    });
+    if (recovered.outcome === "completed") return recovered;
+    if (recovered.outcome === "failed" && !recovered.policyEligible) {
+      return recovered;
+    }
+  }
+  snapshot = await dependencies.requireSnapshot(input.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRun.nodeRunId,
+  );
+  if (!nodeRun || !["running", "waiting"].includes(nodeRun.status)) {
+    return { outcome: "already-converged" as const };
+  }
+  const latestCommand = snapshot.sideEffects.find(
+    (effect) => effect.nodeRunId === nodeRun!.nodeRunId,
+  );
+  const adapter = dependencies.adapters.require(node.type);
+  let cancelled = latestCommand?.status === "prepared";
+  let cumulativeUsage: AgentWorkflowUsage | undefined;
+  if (!latestCommand && !adapter.cancel) cancelled = true;
+  if (!cancelled) {
+    try {
+      const result = await adapter.cancel?.({
+        context: schedulerAdapterContext({
+          snapshot,
+          release: input.release,
+          nodeRun,
+          node,
+          signal: AbortSignal.timeout(5_000),
+        }),
+        command: latestCommand,
+      });
+      cancelled = Boolean(result?.cancelled);
+      cumulativeUsage = result?.cumulativeUsage;
+    } catch {
+      cancelled = false;
+    }
+  }
+  if (!cancelled) return { outcome: "timeout-reconciliation-pending" as const };
+  if (latestCommand) {
+    await failScheduledEffect(
+      dependencies,
+      latestCommand.commandId,
+      "NODE_DEADLINE_EXCEEDED",
+      at,
+    );
+  }
+  return dependencies.serializeMutation(() =>
+    convergeTimeout(dependencies, input.runId, nodeRun!, at, cumulativeUsage),
+  );
+}

package/template/apps/server/src/agent-workflow/scheduler-parallel-dispatch.ts

@@ -0,0 +1,99 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { WorkflowAuthorizationFence } from "./budget-validation.js";
+import {
+  claimScheduledNode,
+  executeClaimedScheduledNode,
+  type SchedulerExecutionDependencies,
+} from "./scheduler-execution.js";
+
+export async function dispatchReadyNodeBatch(
+  dependencies: SchedulerExecutionDependencies,
+  input: {
+    readonly release: AgentWorkflowRelease;
+    readonly ready: readonly AgentWorkflowNodeRun[];
+    readonly runId: string;
+    readonly authorizationFence: WorkflowAuthorizationFence;
+    readonly at: string;
+    readonly signal: AbortSignal;
+    readonly maxParallelism: number;
+  },
+) {
+  const initial = await dependencies.requireSnapshot(input.runId);
+  const available = Math.max(
+    0,
+    input.maxParallelism - activeExecutionCount(initial.nodeRuns),
+  );
+  if (!available) return { outcome: "parallelism-blocked" as const };
+  const candidates = [...input.ready]
+    .sort((left, right) =>
+      `${left.createdAt}:${left.nodeRunId}`.localeCompare(
+        `${right.createdAt}:${right.nodeRunId}`,
+      ),
+    )
+    .slice(0, available);
+  const claimed: Array<{
+    readonly nodeRun: AgentWorkflowNodeRun;
+    readonly node: AgentWorkflowNode;
+  }> = [];
+  for (const candidate of candidates) {
+    if (input.signal.aborted) break;
+    const snapshot = await dependencies.requireSnapshot(input.runId);
+    const current = snapshot.nodeRuns.find(
+      ({ nodeRunId }) => nodeRunId === candidate.nodeRunId,
+    );
+    if (!current || current.status !== "ready") continue;
+    if (activeExecutionCount(snapshot.nodeRuns) >= input.maxParallelism) break;
+    const node = input.release.definition.nodes.find(
+      ({ id }) => id === current.nodeId,
+    );
+    if (!node) throw new Error("Ready Node 不属于 immutable Release");
+    try {
+      const nodeRun = await claimScheduledNode(dependencies, {
+        ...input,
+        node,
+        nodeRun: current,
+      });
+      claimed.push({ nodeRun, node });
+    } catch (error) {
+      const latest = await dependencies.requireSnapshot(input.runId);
+      const latestNode = latest.nodeRuns.find(
+        ({ nodeRunId }) => nodeRunId === current.nodeRunId,
+      );
+      const stateChanged =
+        latest.headSequence !== snapshot.headSequence ||
+        latestNode?.version !== current.version ||
+        latestNode?.status !== "ready";
+      if (!stateChanged) throw error;
+    }
+  }
+  if (!claimed.length) return { outcome: "parallelism-blocked" as const };
+  const settled = await Promise.allSettled(
+    claimed.map(({ nodeRun, node }) =>
+      executeClaimedScheduledNode(dependencies, {
+        ...input,
+        nodeRun,
+        node,
+      }),
+    ),
+  );
+  return {
+    outcome: "batch-dispatched" as const,
+    claimed: claimed.map(({ nodeRun }) => nodeRun.nodeRunId),
+    rejected: settled.flatMap((item, index) =>
+      item.status === "rejected" ? [claimed[index]!.nodeRun.nodeRunId] : [],
+    ),
+    results: settled.flatMap((item) =>
+      item.status === "fulfilled" ? [item.value] : [],
+    ),
+  };
+}
+
+function activeExecutionCount(nodeRuns: readonly AgentWorkflowNodeRun[]) {
+  return nodeRuns.filter(({ status }) =>
+    ["running", "waiting"].includes(status),
+  ).length;
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-activation.ts

@@ -0,0 +1,63 @@
+import type { AgentWorkflowRelease } from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export async function activatePendingWorkflowNode(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly service: AgentWorkflowService;
+  readonly at: string;
+}) {
+  const entry = input.snapshot.nodeRuns.find(
+    ({ nodeId, status, previousNodeRunId }) =>
+      nodeId === input.release.definition.entryNodeId &&
+      status === "pending" &&
+      !previousNodeRunId,
+  );
+  if (entry) {
+    return input.service.transitionNode({
+      runId: input.snapshot.run.runId,
+      nodeRunId: entry.nodeRunId,
+      expectedNodeVersion: entry.version,
+      expectedSequence: input.snapshot.headSequence,
+      command: { type: "ready" },
+      idempotencyKey: `scheduler:entry-ready:${entry.nodeRunId}`,
+      at: input.at,
+    });
+  }
+  const retry = input.snapshot.nodeRuns.find(
+    ({ status, previousNodeRunId }) =>
+      status === "pending" && previousNodeRunId,
+  );
+  if (!retry) return undefined;
+  const previous = input.snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === retry.previousNodeRunId,
+  );
+  const node = input.release.definition.nodes.find(
+    ({ id }) => id === retry.nodeId,
+  );
+  if (!previous?.finishedAt || !node) {
+    throw new Error("Retry durable backoff coordinate 不完整");
+  }
+  const backoff =
+    node.failurePolicy.mode === "retry" ? node.failurePolicy.backoffSeconds : 0;
+  if (
+    Date.parse(input.at) <
+    Date.parse(previous.finishedAt) + backoff * 1_000
+  ) {
+    return { outcome: "backoff" as const };
+  }
+  return input.service.transitionNode({
+    runId: input.snapshot.run.runId,
+    nodeRunId: retry.nodeRunId,
+    expectedNodeVersion: retry.version,
+    expectedSequence: input.snapshot.headSequence,
+    command: { type: "ready" },
+    idempotencyKey: `scheduler:retry-ready:${retry.nodeRunId}`,
+    at: input.at,
+  });
+}

package/template/apps/server/src/agent-workflow/scheduler-recovery-budget.ts

@@ -0,0 +1,41 @@
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+
+export async function releaseRecoveryReservation(
+  dependencies: SchedulerExecutionDependencies,
+  request: { readonly runId: string; readonly at: string },
+  reservationId: string,
+) {
+  const snapshot = await dependencies.requireSnapshot(request.runId);
+  const ledger = await dependencies.service.readBudget(request.runId);
+  const reservation = ledger?.activeReservations.find(
+    (item) => item.reservationId === reservationId,
+  );
+  if (!reservation) return true;
+  const released = await dependencies.service.releaseBudget({
+    reservationId,
+    parentRunId: request.runId,
+    expectedReservationVersion: reservation.version,
+    reasonCode: "NODE_EXECUTION_FAILED",
+    authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+    runPolicyDigest: snapshot.run.executionPolicyDigest,
+    releasedAt: monotonicAt(request.at, snapshot.run.updatedAt),
+  });
+  return released.outcome === "released" || released.outcome === "replayed";
+}
+
+export async function activeRecoveryReservationCount(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+) {
+  const ledger = await dependencies.service.readBudget(runId);
+  return ledger?.activeReservations.length;
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(
+      Date.now(),
+      ...values.map((value) => Date.parse(value ?? "") || 0),
+    ),
+  ).toISOString();
+}