create-windy 0.6.12 → 0.6.14

package/template/apps/server/src/agent-workflow/scheduler-cancellation-reconcile.ts

@@ -0,0 +1,253 @@
+import { isAgentWorkflowNodeTerminal } from "@southwind-ai/shared";
+import type { AgentWorkflowSideEffectCommand } from "@southwind-ai/shared";
+import type {
+  AgentWorkflowAdapterDispatchResult,
+  AgentWorkflowNodeAdapterRegistry,
+} from "./adapter-dispatch.js";
+import type {
+  WorkflowEffectCommandFencePort,
+  WorkflowEffectReconciliationFence,
+} from "./effect-command-fence.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import type { AgentWorkflowService } from "./service.js";
+import { transitionAgentWorkflowSideEffect } from "./side-effect-state.js";
+import { convergeFailedReservation } from "./scheduler-reservation-convergence.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export async function cancelPreparedEffect(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly command: AgentWorkflowSideEffectCommand;
+  readonly at: string;
+}) {
+  const requested = await input.repository.updateSideEffect(
+    transitionAgentWorkflowSideEffect({
+      command: input.command,
+      expectedVersion: input.command.version,
+      transition: { type: "request-cancel" },
+      at: monotonicAt(input.at, input.command.updatedAt),
+    }),
+    input.command.version,
+  );
+  return input.repository.updateSideEffect(
+    transitionAgentWorkflowSideEffect({
+      command: requested,
+      expectedVersion: requested.version,
+      transition: { type: "cancel" },
+      at: monotonicAt(input.at, requested.updatedAt),
+    }),
+    requested.version,
+  );
+}
+
+export async function reconcileEffectDuringCancellation(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly effectFence: WorkflowEffectCommandFencePort;
+  readonly adapters: AgentWorkflowNodeAdapterRegistry;
+  readonly snapshot: Snapshot;
+  readonly nodeRunId: string;
+  readonly at: string;
+  readonly signal: AbortSignal;
+}): Promise<
+  | { readonly outcome: "completed" }
+  | { readonly outcome: "pending" | "failed" | "unsupported" }
+> {
+  const command = input.snapshot.sideEffects.find(
+    (item) => item.nodeRunId === input.nodeRunId,
+  );
+  const nodeRun = input.snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === input.nodeRunId,
+  );
+  if (!command || !nodeRun) return { outcome: "unsupported" };
+  if (["failed", "cancelled"].includes(command.status)) {
+    return { outcome: "unsupported" };
+  }
+  const release = await input.repository.getRelease(
+    input.snapshot.run.releaseId,
+  );
+  const node = release?.definition.nodes.find(
+    ({ id }) => id === nodeRun.nodeId,
+  );
+  if (!release || !node) return { outcome: "failed" };
+  const adapter = input.adapters.require(node.type);
+  let receipt = await input.effectFence.reconcile(command.commandId);
+  if (receipt) {
+    await terminalizeEffect(input, command.commandId, true);
+  }
+  if (!adapter.reconcile) {
+    return { outcome: receipt ? "failed" : "unsupported" };
+  }
+  let fence: WorkflowEffectReconciliationFence | undefined;
+  if (!receipt) {
+    const claim = await input.effectFence.claimReconciliation({
+      command,
+      expectedCommandVersion: command.version,
+    });
+    if (claim.outcome === "conflict") return { outcome: "pending" };
+    if (claim.outcome === "replayed") receipt = claim.receipt;
+    else fence = claim.fence;
+  }
+  let result:
+    | AgentWorkflowAdapterDispatchResult
+    | { readonly outcome: "pending" };
+  try {
+    result = await adapter.reconcile({
+      context: schedulerAdapterContext({
+        snapshot: input.snapshot,
+        release,
+        nodeRun,
+        node,
+        signal: input.signal,
+      }),
+      command,
+      authorizationFence: reconciliationAuthorizationFence(
+        input.snapshot,
+        input.at,
+      ),
+      ...(fence ? { fence } : { committedReceipt: receipt! }),
+    });
+  } catch {
+    await terminalizeEffect(input, command.commandId, Boolean(receipt));
+    return { outcome: "unsupported" };
+  }
+  if (result.outcome === "pending" || result.outcome === "waiting") {
+    return { outcome: "pending" };
+  }
+  if (result.outcome === "failed") {
+    if (command.budgetReservationId) {
+      if (!result.cumulativeUsage) return { outcome: "pending" };
+      const convergence = await convergeFailedReservation({
+        service: input.service,
+        command,
+        cumulativeUsage: result.cumulativeUsage,
+        safeCode: result.safeCode,
+        at: input.at,
+      });
+      if (!convergence.settled) return { outcome: "pending" };
+    }
+    await terminalizeEffect(input, command.commandId, Boolean(receipt));
+    return { outcome: "unsupported" };
+  }
+  if (fence) {
+    const committed = await input.effectFence.commit({
+      fence,
+      receipt: result.receipt,
+    });
+    if (committed.outcome === "conflict") return { outcome: "pending" };
+    if (committed.outcome === "replayed") {
+      await terminalizeEffect(input, command.commandId, true);
+    }
+  } else {
+    await terminalizeEffect(input, command.commandId, true);
+  }
+  let snapshot = await requireSnapshot(
+    input.repository,
+    input.snapshot.run.runId,
+  );
+  const latest = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  );
+  if (!latest || isAgentWorkflowNodeTerminal(latest.status)) {
+    return { outcome: latest?.status === "completed" ? "completed" : "failed" };
+  }
+  let cumulativeUsage = result.cumulativeUsage;
+  if (cumulativeUsage && command.budgetReservationId) {
+    const ledger = await input.service.readBudget(snapshot.run.runId);
+    const reservation = ledger?.activeReservations.find(
+      (item) => item.reservationId === command.budgetReservationId,
+    );
+    if (!reservation) return { outcome: "failed" };
+    const settled = await input.service.settleBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: snapshot.run.runId,
+      expectedReservationVersion: reservation.version,
+      consumed: cumulativeUsage,
+      usageReceiptId: result.receipt.receiptId,
+      authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+      runPolicyDigest: snapshot.run.executionPolicyDigest,
+      settledAt: result.receipt.committedAt,
+    });
+    if (settled.outcome !== "settled" && settled.outcome !== "replayed") {
+      return { outcome: "failed" };
+    }
+    cumulativeUsage = undefined;
+    snapshot = await requireSnapshot(input.repository, snapshot.run.runId);
+  }
+  const current = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  )!;
+  await input.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: current.nodeRunId,
+    expectedNodeVersion: current.version,
+    expectedRunVersion: cumulativeUsage ? snapshot.run.version : undefined,
+    expectedSequence: snapshot.headSequence,
+    command: {
+      type: "complete",
+      receipt: result.nodeResult,
+      outputArtifacts: result.outputArtifacts,
+    },
+    cumulativeUsage,
+    effectCommandId: command.commandId,
+    idempotencyKey: `scheduler:cancellation-reconcile:${command.commandId}`,
+    at: monotonicAt(input.at, result.receipt.committedAt, current.updatedAt),
+  });
+  return { outcome: "completed" };
+}
+
+async function terminalizeEffect(
+  input: Parameters<typeof reconcileEffectDuringCancellation>[0],
+  commandId: string,
+  hasReceipt: boolean,
+) {
+  const command = await input.repository.getSideEffect(commandId);
+  if (
+    !command ||
+    ["failed", "cancelled", "reconciled"].includes(command.status)
+  ) {
+    return;
+  }
+  const transition =
+    hasReceipt && command.status === "committed"
+      ? ({ type: "reconcile" } as const)
+      : ({ type: "fail", safeCode: "EFFECT_RECONCILIATION_FAILED" } as const);
+  await input.repository.updateSideEffect(
+    transitionAgentWorkflowSideEffect({
+      command,
+      expectedVersion: command.version,
+      transition,
+      at: monotonicAt(input.at, command.updatedAt),
+    }),
+    command.version,
+  );
+}
+
+function reconciliationAuthorizationFence(snapshot: Snapshot, at: string) {
+  return {
+    proofId: `reconciliation:${snapshot.run.runId}`,
+    actorId: snapshot.run.actorId,
+    authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+    runPolicyDigest: snapshot.run.executionPolicyDigest,
+    checkedAt: at,
+    expiresAt: new Date(Date.parse(at) + 300_000).toISOString(),
+  };
+}
+
+async function requireSnapshot(
+  repository: AgentWorkflowRepository,
+  runId: string,
+) {
+  const snapshot = await repository.getSnapshot(runId);
+  if (!snapshot) throw new Error("Workflow Run 不存在");
+  return snapshot;
+}
+
+function monotonicAt(...values: string[]) {
+  return new Date(
+    Math.max(...values.map((value) => Date.parse(value) || 0)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-cancellation-support.ts

@@ -0,0 +1,145 @@
+import type {
+  AgentWorkflowSideEffectCommand,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+import { isAgentWorkflowNodeTerminal } from "@southwind-ai/shared";
+
+export async function releaseCancelledReservation(input: {
+  readonly service: AgentWorkflowService;
+  readonly command: AgentWorkflowSideEffectCommand | undefined;
+  readonly at: string;
+}) {
+  return releaseNodeReservation({
+    ...input,
+    reasonCode: "DELEGATION_CANCELLED",
+  });
+}
+
+export async function releaseNodeReservation(input: {
+  readonly service: AgentWorkflowService;
+  readonly command: AgentWorkflowSideEffectCommand | undefined;
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  if (!input.command?.budgetReservationId) return true;
+  return releaseReservationIds({
+    service: input.service,
+    runId: input.command.runId,
+    reservationIds: [input.command.budgetReservationId],
+    at: input.at,
+    reasonCode: input.reasonCode,
+  });
+}
+
+export async function releaseActiveNodeReservations(input: {
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly nodeRunId: string;
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  const ledger = await input.service.readBudget(input.runId);
+  if (!ledger) return false;
+  return releaseReservationIds({
+    service: input.service,
+    runId: input.runId,
+    reservationIds:
+      ledger?.activeReservations
+        .filter(({ parentNodeRunId }) => parentNodeRunId === input.nodeRunId)
+        .map(({ reservationId }) => reservationId) ?? [],
+    at: input.at,
+    reasonCode: input.reasonCode,
+  });
+}
+
+async function releaseReservationIds(input: {
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly reservationIds: readonly string[];
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  const ledger = await input.service.readBudget(input.runId);
+  if (!ledger) return false;
+  const reservations = ledger.activeReservations.filter(({ reservationId }) =>
+    input.reservationIds.includes(reservationId),
+  );
+  for (const reservation of reservations) {
+    const released = await input.service.releaseBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: reservation.parentRunId,
+      expectedReservationVersion: reservation.version,
+      reasonCode: input.reasonCode,
+      authorizationSnapshotDigest: reservation.authorizationSnapshotDigest,
+      runPolicyDigest: reservation.runPolicyDigest,
+      releasedAt: input.at,
+    });
+    if (released.outcome !== "released" && released.outcome !== "replayed") {
+      return false;
+    }
+  }
+  return true;
+}
+
+export async function requireCancellationSnapshot(
+  repository: AgentWorkflowRepository,
+  runId: string,
+) {
+  const value = await repository.getSnapshot(runId);
+  if (!value) throw new Error("Workflow Run 不存在");
+  return value;
+}
+
+export async function terminalizeCancelledNode(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly nodeRunId: string;
+  readonly terminalCommand: "cancel" | "timeout";
+  readonly idempotencyKey: string;
+  readonly at: string;
+  readonly cumulativeUsage?: AgentWorkflowUsage;
+}) {
+  const snapshot = await requireCancellationSnapshot(
+    input.repository,
+    input.runId,
+  );
+  const node = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRunId,
+  );
+  if (!node || isAgentWorkflowNodeTerminal(node.status)) return true;
+  await input.service.transitionNode({
+    runId: input.runId,
+    nodeRunId: input.nodeRunId,
+    expectedNodeVersion: node.version,
+    expectedRunVersion: input.cumulativeUsage
+      ? snapshot.run.version
+      : undefined,
+    expectedSequence: snapshot.headSequence,
+    command: { type: input.terminalCommand },
+    cumulativeUsage: input.cumulativeUsage,
+    idempotencyKey: input.idempotencyKey,
+    at: monotonicCancellationAt(input.at, node.updatedAt),
+  });
+  return true;
+}
+
+export function cancellationTerminalAt(
+  snapshot: Awaited<ReturnType<typeof requireCancellationSnapshot>>,
+  requestedAt: string,
+) {
+  return monotonicCancellationAt(
+    requestedAt,
+    snapshot.run.updatedAt,
+    ...snapshot.nodeRuns.map(({ updatedAt }) => updatedAt),
+    ...snapshot.sideEffects.map(({ updatedAt }) => updatedAt),
+  );
+}
+
+export function monotonicCancellationAt(...values: string[]) {
+  return new Date(
+    Math.max(...values.map((value) => Date.parse(value) || 0)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-cancellation.ts

@@ -0,0 +1,2 @@
+export { cancelJoinAnyLosers } from "./scheduler-join-cancellation.js";
+export { cancelScheduledWorkflow } from "./scheduler-workflow-cancellation.js";

package/template/apps/server/src/agent-workflow/scheduler-context.ts

@@ -0,0 +1,85 @@
+import {
+  isAgentWorkflowNodeTerminal,
+  type AgentWorkflowArtifactRef,
+  type AgentWorkflowNode,
+  type AgentWorkflowNodeRun,
+  type AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowNodeAdapterContext } from "./adapter-dispatch.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export function schedulerAdapterContext(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly nodeRun: AgentWorkflowNodeRun;
+  readonly node: AgentWorkflowNode;
+  readonly signal: AbortSignal;
+}): AgentWorkflowNodeAdapterContext {
+  const predecessors = latestTerminalPredecessors(
+    input.snapshot,
+    input.release,
+    input.node.id,
+  );
+  return {
+    run: input.snapshot.run,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    inputArtifacts:
+      input.node.type === "join"
+        ? uniqueArtifacts(predecessors.flatMap((item) => item.outputArtifacts))
+        : input.nodeRun.inputArtifacts,
+    inputReceipts: predecessors.flatMap((item) =>
+      item.resultReceipt ? [item.resultReceipt] : [],
+    ),
+    signal: input.signal,
+  };
+}
+
+export function successorInputArtifacts(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly node: AgentWorkflowNode;
+  readonly source: AgentWorkflowNodeRun;
+}) {
+  if (input.node.type !== "join") return input.source.outputArtifacts;
+  return uniqueArtifacts(
+    latestTerminalPredecessors(
+      input.snapshot,
+      input.release,
+      input.node.id,
+    ).flatMap((item) => item.outputArtifacts),
+  );
+}
+
+function latestTerminalPredecessors(
+  snapshot: Snapshot,
+  release: AgentWorkflowRelease,
+  nodeId: string,
+) {
+  return release.definition.edges
+    .filter(({ to }) => to === nodeId)
+    .map(
+      ({ from }) =>
+        snapshot.nodeRuns
+          .filter(
+            (item) =>
+              item.nodeId === from && isAgentWorkflowNodeTerminal(item.status),
+          )
+          .sort((left, right) => right.attempt - left.attempt)[0],
+    )
+    .filter((item): item is AgentWorkflowNodeRun => Boolean(item));
+}
+
+function uniqueArtifacts(values: readonly AgentWorkflowArtifactRef[]) {
+  const seen = new Set<string>();
+  return values.filter((item) => {
+    const key = `${item.artifactId}:${item.sha256}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}

package/template/apps/server/src/agent-workflow/scheduler-deadline.ts

@@ -0,0 +1,65 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRun,
+} from "@southwind-ai/shared";
+
+export class SchedulerDeadlineExceeded extends Error {
+  constructor(readonly deadlineAt: string) {
+    super("NODE_DEADLINE_EXCEEDED");
+    this.name = "SchedulerDeadlineExceeded";
+  }
+}
+
+export function scheduledNodeDeadlineAt(input: {
+  readonly run: AgentWorkflowRun;
+  readonly node: AgentWorkflowNode;
+  readonly nodeRun: AgentWorkflowNodeRun;
+}) {
+  const runDeadline = Date.parse(input.run.deadlineAt);
+  const nodeDeadline =
+    input.nodeRun.startedAt && input.node.timeoutSeconds
+      ? Date.parse(input.nodeRun.startedAt) + input.node.timeoutSeconds * 1_000
+      : runDeadline;
+  return new Date(Math.min(runDeadline, nodeDeadline)).toISOString();
+}
+
+export function scheduledNodeSignal(parent: AbortSignal, deadlineAt: string) {
+  const remaining = Date.parse(deadlineAt) - Date.now();
+  const deadline = AbortSignal.timeout(Math.max(1, remaining));
+  return {
+    deadlineAt,
+    signal: AbortSignal.any([parent, deadline]),
+    expired: () => !parent.aborted && Date.now() >= Date.parse(deadlineAt),
+  };
+}
+
+export function awaitScheduledNode<T>(
+  promise: Promise<T>,
+  scope: ReturnType<typeof scheduledNodeSignal>,
+) {
+  if (scope.signal.aborted) {
+    return Promise.reject(new SchedulerDeadlineExceeded(scope.deadlineAt));
+  }
+  return new Promise<T>((resolve, reject) => {
+    const aborted = () =>
+      reject(new SchedulerDeadlineExceeded(scope.deadlineAt));
+    scope.signal.addEventListener("abort", aborted, { once: true });
+    promise.then(
+      (value) => {
+        scope.signal.removeEventListener("abort", aborted);
+        resolve(value);
+      },
+      (error) => {
+        scope.signal.removeEventListener("abort", aborted);
+        reject(error);
+      },
+    );
+  });
+}
+
+export function deadlineCompletionAt(deadlineAt: string, updatedAt: string) {
+  return new Date(
+    Math.max(Date.now(), Date.parse(deadlineAt), Date.parse(updatedAt)),
+  ).toISOString();
+}