create-windy 0.6.12 → 0.6.15

package/template/apps/server/src/agent-workflow/scheduler-cancellation-support.ts

@@ -0,0 +1,145 @@
+import type {
+  AgentWorkflowSideEffectCommand,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowRepository } from "./repository.js";
+import type { AgentWorkflowService } from "./service.js";
+import { isAgentWorkflowNodeTerminal } from "@southwind-ai/shared";
+
+export async function releaseCancelledReservation(input: {
+  readonly service: AgentWorkflowService;
+  readonly command: AgentWorkflowSideEffectCommand | undefined;
+  readonly at: string;
+}) {
+  return releaseNodeReservation({
+    ...input,
+    reasonCode: "DELEGATION_CANCELLED",
+  });
+}
+
+export async function releaseNodeReservation(input: {
+  readonly service: AgentWorkflowService;
+  readonly command: AgentWorkflowSideEffectCommand | undefined;
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  if (!input.command?.budgetReservationId) return true;
+  return releaseReservationIds({
+    service: input.service,
+    runId: input.command.runId,
+    reservationIds: [input.command.budgetReservationId],
+    at: input.at,
+    reasonCode: input.reasonCode,
+  });
+}
+
+export async function releaseActiveNodeReservations(input: {
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly nodeRunId: string;
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  const ledger = await input.service.readBudget(input.runId);
+  if (!ledger) return false;
+  return releaseReservationIds({
+    service: input.service,
+    runId: input.runId,
+    reservationIds:
+      ledger?.activeReservations
+        .filter(({ parentNodeRunId }) => parentNodeRunId === input.nodeRunId)
+        .map(({ reservationId }) => reservationId) ?? [],
+    at: input.at,
+    reasonCode: input.reasonCode,
+  });
+}
+
+async function releaseReservationIds(input: {
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly reservationIds: readonly string[];
+  readonly at: string;
+  readonly reasonCode: string;
+}) {
+  const ledger = await input.service.readBudget(input.runId);
+  if (!ledger) return false;
+  const reservations = ledger.activeReservations.filter(({ reservationId }) =>
+    input.reservationIds.includes(reservationId),
+  );
+  for (const reservation of reservations) {
+    const released = await input.service.releaseBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: reservation.parentRunId,
+      expectedReservationVersion: reservation.version,
+      reasonCode: input.reasonCode,
+      authorizationSnapshotDigest: reservation.authorizationSnapshotDigest,
+      runPolicyDigest: reservation.runPolicyDigest,
+      releasedAt: input.at,
+    });
+    if (released.outcome !== "released" && released.outcome !== "replayed") {
+      return false;
+    }
+  }
+  return true;
+}
+
+export async function requireCancellationSnapshot(
+  repository: AgentWorkflowRepository,
+  runId: string,
+) {
+  const value = await repository.getSnapshot(runId);
+  if (!value) throw new Error("Workflow Run 不存在");
+  return value;
+}
+
+export async function terminalizeCancelledNode(input: {
+  readonly repository: AgentWorkflowRepository;
+  readonly service: AgentWorkflowService;
+  readonly runId: string;
+  readonly nodeRunId: string;
+  readonly terminalCommand: "cancel" | "timeout";
+  readonly idempotencyKey: string;
+  readonly at: string;
+  readonly cumulativeUsage?: AgentWorkflowUsage;
+}) {
+  const snapshot = await requireCancellationSnapshot(
+    input.repository,
+    input.runId,
+  );
+  const node = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === input.nodeRunId,
+  );
+  if (!node || isAgentWorkflowNodeTerminal(node.status)) return true;
+  await input.service.transitionNode({
+    runId: input.runId,
+    nodeRunId: input.nodeRunId,
+    expectedNodeVersion: node.version,
+    expectedRunVersion: input.cumulativeUsage
+      ? snapshot.run.version
+      : undefined,
+    expectedSequence: snapshot.headSequence,
+    command: { type: input.terminalCommand },
+    cumulativeUsage: input.cumulativeUsage,
+    idempotencyKey: input.idempotencyKey,
+    at: monotonicCancellationAt(input.at, node.updatedAt),
+  });
+  return true;
+}
+
+export function cancellationTerminalAt(
+  snapshot: Awaited<ReturnType<typeof requireCancellationSnapshot>>,
+  requestedAt: string,
+) {
+  return monotonicCancellationAt(
+    requestedAt,
+    snapshot.run.updatedAt,
+    ...snapshot.nodeRuns.map(({ updatedAt }) => updatedAt),
+    ...snapshot.sideEffects.map(({ updatedAt }) => updatedAt),
+  );
+}
+
+export function monotonicCancellationAt(...values: string[]) {
+  return new Date(
+    Math.max(...values.map((value) => Date.parse(value) || 0)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-cancellation.ts

@@ -0,0 +1,2 @@
+export { cancelJoinAnyLosers } from "./scheduler-join-cancellation.js";
+export { cancelScheduledWorkflow } from "./scheduler-workflow-cancellation.js";

package/template/apps/server/src/agent-workflow/scheduler-context.ts

@@ -0,0 +1,85 @@
+import {
+  isAgentWorkflowNodeTerminal,
+  type AgentWorkflowArtifactRef,
+  type AgentWorkflowNode,
+  type AgentWorkflowNodeRun,
+  type AgentWorkflowRelease,
+} from "@southwind-ai/shared";
+import type { AgentWorkflowNodeAdapterContext } from "./adapter-dispatch.js";
+import type { AgentWorkflowRepository } from "./repository.js";
+
+type Snapshot = NonNullable<
+  Awaited<ReturnType<AgentWorkflowRepository["getSnapshot"]>>
+>;
+
+export function schedulerAdapterContext(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly nodeRun: AgentWorkflowNodeRun;
+  readonly node: AgentWorkflowNode;
+  readonly signal: AbortSignal;
+}): AgentWorkflowNodeAdapterContext {
+  const predecessors = latestTerminalPredecessors(
+    input.snapshot,
+    input.release,
+    input.node.id,
+  );
+  return {
+    run: input.snapshot.run,
+    nodeRun: input.nodeRun,
+    node: input.node,
+    inputArtifacts:
+      input.node.type === "join"
+        ? uniqueArtifacts(predecessors.flatMap((item) => item.outputArtifacts))
+        : input.nodeRun.inputArtifacts,
+    inputReceipts: predecessors.flatMap((item) =>
+      item.resultReceipt ? [item.resultReceipt] : [],
+    ),
+    signal: input.signal,
+  };
+}
+
+export function successorInputArtifacts(input: {
+  readonly snapshot: Snapshot;
+  readonly release: AgentWorkflowRelease;
+  readonly node: AgentWorkflowNode;
+  readonly source: AgentWorkflowNodeRun;
+}) {
+  if (input.node.type !== "join") return input.source.outputArtifacts;
+  return uniqueArtifacts(
+    latestTerminalPredecessors(
+      input.snapshot,
+      input.release,
+      input.node.id,
+    ).flatMap((item) => item.outputArtifacts),
+  );
+}
+
+function latestTerminalPredecessors(
+  snapshot: Snapshot,
+  release: AgentWorkflowRelease,
+  nodeId: string,
+) {
+  return release.definition.edges
+    .filter(({ to }) => to === nodeId)
+    .map(
+      ({ from }) =>
+        snapshot.nodeRuns
+          .filter(
+            (item) =>
+              item.nodeId === from && isAgentWorkflowNodeTerminal(item.status),
+          )
+          .sort((left, right) => right.attempt - left.attempt)[0],
+    )
+    .filter((item): item is AgentWorkflowNodeRun => Boolean(item));
+}
+
+function uniqueArtifacts(values: readonly AgentWorkflowArtifactRef[]) {
+  const seen = new Set<string>();
+  return values.filter((item) => {
+    const key = `${item.artifactId}:${item.sha256}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}

package/template/apps/server/src/agent-workflow/scheduler-deadline.ts

@@ -0,0 +1,65 @@
+import type {
+  AgentWorkflowNode,
+  AgentWorkflowNodeRun,
+  AgentWorkflowRun,
+} from "@southwind-ai/shared";
+
+export class SchedulerDeadlineExceeded extends Error {
+  constructor(readonly deadlineAt: string) {
+    super("NODE_DEADLINE_EXCEEDED");
+    this.name = "SchedulerDeadlineExceeded";
+  }
+}
+
+export function scheduledNodeDeadlineAt(input: {
+  readonly run: AgentWorkflowRun;
+  readonly node: AgentWorkflowNode;
+  readonly nodeRun: AgentWorkflowNodeRun;
+}) {
+  const runDeadline = Date.parse(input.run.deadlineAt);
+  const nodeDeadline =
+    input.nodeRun.startedAt && input.node.timeoutSeconds
+      ? Date.parse(input.nodeRun.startedAt) + input.node.timeoutSeconds * 1_000
+      : runDeadline;
+  return new Date(Math.min(runDeadline, nodeDeadline)).toISOString();
+}
+
+export function scheduledNodeSignal(parent: AbortSignal, deadlineAt: string) {
+  const remaining = Date.parse(deadlineAt) - Date.now();
+  const deadline = AbortSignal.timeout(Math.max(1, remaining));
+  return {
+    deadlineAt,
+    signal: AbortSignal.any([parent, deadline]),
+    expired: () => !parent.aborted && Date.now() >= Date.parse(deadlineAt),
+  };
+}
+
+export function awaitScheduledNode<T>(
+  promise: Promise<T>,
+  scope: ReturnType<typeof scheduledNodeSignal>,
+) {
+  if (scope.signal.aborted) {
+    return Promise.reject(new SchedulerDeadlineExceeded(scope.deadlineAt));
+  }
+  return new Promise<T>((resolve, reject) => {
+    const aborted = () =>
+      reject(new SchedulerDeadlineExceeded(scope.deadlineAt));
+    scope.signal.addEventListener("abort", aborted, { once: true });
+    promise.then(
+      (value) => {
+        scope.signal.removeEventListener("abort", aborted);
+        resolve(value);
+      },
+      (error) => {
+        scope.signal.removeEventListener("abort", aborted);
+        reject(error);
+      },
+    );
+  });
+}
+
+export function deadlineCompletionAt(deadlineAt: string, updatedAt: string) {
+  return new Date(
+    Math.max(Date.now(), Date.parse(deadlineAt), Date.parse(updatedAt)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-effect-result.ts

@@ -0,0 +1,205 @@
+import type { AgentWorkflowNodeRun } from "@southwind-ai/shared";
+import type { AgentWorkflowAdapterDispatchResult } from "./adapter-dispatch.js";
+import { handleScheduledNodeFailure } from "./scheduler-failure.js";
+import type { SchedulerExecutionDependencies } from "./scheduler-execution.js";
+import { releaseNodeReservation } from "./scheduler-cancellation-support.js";
+import { convergeFailedReservation } from "./scheduler-reservation-convergence.js";
+import { transitionAgentWorkflowSideEffect } from "./side-effect-state.js";
+
+type Snapshot = Awaited<
+  ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+>;
+
+export function dispatchResultAt(
+  baseline: string,
+  result: AgentWorkflowAdapterDispatchResult | { readonly outcome: "pending" },
+) {
+  return monotonicAt(
+    baseline,
+    result.outcome === "committed" ? result.receipt.committedAt : undefined,
+    result.outcome === "committed" ? result.nodeResult.recordedAt : undefined,
+  );
+}
+
+export async function finishScheduledDispatch(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  commandId: string,
+  result: AgentWorkflowAdapterDispatchResult,
+  at: string,
+) {
+  snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+  nodeRun = snapshot.nodeRuns.find(
+    (item) => item.nodeRunId === nodeRun.nodeRunId,
+  )!;
+  if (result.outcome === "waiting") {
+    return waitScheduledNode(
+      dependencies,
+      snapshot,
+      nodeRun,
+      result.adapter,
+      at,
+    );
+  }
+  if (result.outcome === "failed") {
+    const command = snapshot.sideEffects.find(
+      (item) => item.commandId === commandId,
+    );
+    const budget = await convergeFailedReservation({
+      service: dependencies.service,
+      command,
+      cumulativeUsage: result.cumulativeUsage,
+      safeCode: result.safeCode,
+      at,
+    });
+    if (command?.budgetReservationId && !budget.settled) {
+      throw new Error("BUDGET_RESERVATION_CONVERGENCE_PENDING");
+    }
+    await failScheduledEffect(dependencies, commandId, result.safeCode, at);
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    )!;
+    return failNode(
+      dependencies,
+      snapshot,
+      nodeRun,
+      result.safeCode,
+      at,
+      budget.settled ? undefined : result.cumulativeUsage,
+    );
+  }
+  const command = snapshot.sideEffects.find(
+    (item) => item.commandId === commandId,
+  );
+  if (result.cumulativeUsage && command?.budgetReservationId) {
+    const ledger = await dependencies.service.readBudget(snapshot.run.runId);
+    const reservation = ledger?.activeReservations.find(
+      (item) => item.reservationId === command.budgetReservationId,
+    );
+    if (!reservation) {
+      return failNode(
+        dependencies,
+        snapshot,
+        nodeRun,
+        "BUDGET_RESERVATION_MISSING",
+        at,
+      );
+    }
+    const settled = await dependencies.service.settleBudget({
+      reservationId: reservation.reservationId,
+      parentRunId: snapshot.run.runId,
+      expectedReservationVersion: reservation.version,
+      consumed: result.cumulativeUsage,
+      usageReceiptId: result.receipt.receiptId,
+      authorizationSnapshotDigest: snapshot.run.authorizationSnapshotDigest,
+      runPolicyDigest: snapshot.run.executionPolicyDigest,
+      settledAt: at,
+    });
+    if ("safeCode" in settled) {
+      return failNode(dependencies, snapshot, nodeRun, settled.safeCode, at);
+    }
+    snapshot = await dependencies.requireSnapshot(snapshot.run.runId);
+    nodeRun = snapshot.nodeRuns.find(
+      (item) => item.nodeRunId === nodeRun.nodeRunId,
+    )!;
+  }
+  const settledByLedger = Boolean(
+    result.cumulativeUsage && command?.budgetReservationId,
+  );
+  return dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedRunVersion:
+      result.cumulativeUsage && !settledByLedger
+        ? snapshot.run.version
+        : undefined,
+    expectedSequence: snapshot.headSequence,
+    command: {
+      type: "complete",
+      receipt: result.nodeResult,
+      outputArtifacts: result.outputArtifacts,
+    },
+    cumulativeUsage: settledByLedger ? undefined : result.cumulativeUsage,
+    effectCommandId: commandId,
+    idempotencyKey: `scheduler:effect-complete:${nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+export async function failScheduledEffect(
+  dependencies: SchedulerExecutionDependencies,
+  commandId: string,
+  safeCode: string,
+  at: string,
+) {
+  const command = await dependencies.repository.getSideEffect(commandId);
+  if (
+    !command ||
+    ["failed", "cancelled", "reconciled"].includes(command.status)
+  ) {
+    return;
+  }
+  const released = await releaseNodeReservation({
+    service: dependencies.service,
+    command,
+    at,
+    reasonCode: "DELEGATION_EFFECT_FAILED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  await dependencies.repository.updateSideEffect(
+    transitionAgentWorkflowSideEffect({
+      command,
+      expectedVersion: command.version,
+      transition: { type: "fail", safeCode },
+      at: monotonicAt(at, command.updatedAt),
+    }),
+    command.version,
+  );
+}
+
+export function waitScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  adapter: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+  at: string,
+) {
+  return dependencies.service.transitionNode({
+    runId: snapshot.run.runId,
+    nodeRunId: nodeRun.nodeRunId,
+    expectedNodeVersion: nodeRun.version,
+    expectedSequence: snapshot.headSequence,
+    command: { type: "wait", adapter },
+    idempotencyKey: `scheduler:wait:${nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+async function failNode(
+  dependencies: SchedulerExecutionDependencies,
+  snapshot: Snapshot,
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: import("@southwind-ai/shared").AgentWorkflowUsage,
+) {
+  const release = await dependencies.requireRelease(snapshot.run.releaseId);
+  return handleScheduledNodeFailure(
+    dependencies,
+    snapshot,
+    release.definition.nodes.find(({ id }) => id === nodeRun.nodeId)!,
+    nodeRun,
+    safeCode,
+    at,
+    cumulativeUsage,
+  );
+}
+
+function monotonicAt(...values: Array<string | undefined>) {
+  return new Date(
+    Math.max(...values.map((value) => Date.parse(value ?? "") || 0)),
+  ).toISOString();
+}

package/template/apps/server/src/agent-workflow/scheduler-execution-convergence.ts

@@ -0,0 +1,174 @@
+import type {
+  AgentWorkflowNodeRun,
+  AgentWorkflowUsage,
+} from "@southwind-ai/shared";
+import type {
+  AgentWorkflowAdapterDispatchResult,
+  AgentWorkflowNodeAdapter,
+} from "./adapter-dispatch.js";
+import { deadlineCompletionAt } from "./scheduler-deadline.js";
+import {
+  finishScheduledDispatch,
+  waitScheduledNode,
+} from "./scheduler-effect-result.js";
+import type {
+  executeScheduledNode,
+  SchedulerExecutionDependencies,
+} from "./scheduler-execution.js";
+import { handleScheduledNodeFailure } from "./scheduler-failure.js";
+import { schedulerAdapterContext } from "./scheduler-context.js";
+import { releaseActiveNodeReservations } from "./scheduler-cancellation-support.js";
+
+export async function currentScheduledNode(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+) {
+  const snapshot = await dependencies.requireSnapshot(runId);
+  const current = snapshot.nodeRuns.find(
+    ({ nodeRunId }) => nodeRunId === nodeRun.nodeRunId,
+  );
+  if (!current) throw new Error("Node Run 不存在");
+  return { snapshot, nodeRun: current };
+}
+
+export async function convergeWait(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  adapter: NonNullable<AgentWorkflowNodeRun["adapter"]>,
+  at: string,
+) {
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return waitScheduledNode(
+    dependencies,
+    latest.snapshot,
+    latest.nodeRun,
+    adapter,
+    at,
+  );
+}
+
+export async function convergeFailure(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+  nodeRun: AgentWorkflowNodeRun,
+  safeCode: string,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const latest = await currentScheduledNode(dependencies, input.runId, nodeRun);
+  return handleScheduledNodeFailure(
+    dependencies,
+    latest.snapshot,
+    input.node,
+    latest.nodeRun,
+    safeCode,
+    at,
+    cumulativeUsage,
+  );
+}
+
+export async function convergeTimeout(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  at: string,
+  cumulativeUsage?: AgentWorkflowUsage,
+) {
+  const released = await releaseActiveNodeReservations({
+    service: dependencies.service,
+    runId,
+    nodeRunId: nodeRun.nodeRunId,
+    at,
+    reasonCode: "NODE_DEADLINE_EXCEEDED",
+  });
+  if (!released) throw new Error("BUDGET_RESERVATION_RELEASE_PENDING");
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return dependencies.service.transitionNode({
+    runId,
+    nodeRunId: latest.nodeRun.nodeRunId,
+    expectedNodeVersion: latest.nodeRun.version,
+    expectedRunVersion: cumulativeUsage
+      ? latest.snapshot.run.version
+      : undefined,
+    expectedSequence: latest.snapshot.headSequence,
+    command: { type: "timeout" },
+    cumulativeUsage,
+    idempotencyKey: `scheduler:timeout:${latest.nodeRun.nodeRunId}`,
+    at,
+  });
+}
+
+export async function convergeDispatch(
+  dependencies: SchedulerExecutionDependencies,
+  runId: string,
+  nodeRun: AgentWorkflowNodeRun,
+  commandId: string,
+  result: AgentWorkflowAdapterDispatchResult,
+  at: string,
+) {
+  const latest = await currentScheduledNode(dependencies, runId, nodeRun);
+  return finishScheduledDispatch(
+    dependencies,
+    latest.snapshot,
+    latest.nodeRun,
+    commandId,
+    result,
+    at,
+  );
+}
+
+export async function convergeTimedOutPreparation(
+  dependencies: SchedulerExecutionDependencies,
+  input: Parameters<typeof executeScheduledNode>[1],
+  snapshot: Awaited<
+    ReturnType<SchedulerExecutionDependencies["requireSnapshot"]>
+  >,
+  running: AgentWorkflowNodeRun,
+  adapter: AgentWorkflowNodeAdapter,
+  deadlineAt: string,
+) {
+  let cancellation:
+    | Awaited<ReturnType<NonNullable<AgentWorkflowNodeAdapter["cancel"]>>>
+    | undefined;
+  if (!adapter.cancel) cancellation = { cancelled: true };
+  try {
+    if (adapter.cancel) {
+      cancellation = await adapter.cancel({
+        context: schedulerAdapterContext({
+          snapshot,
+          release: input.release,
+          nodeRun: running,
+          node: input.node,
+          signal: AbortSignal.timeout(5_000),
+        }),
+      });
+    }
+  } catch {
+    cancellation = undefined;
+  }
+  const at = deadlineCompletionAt(deadlineAt, running.updatedAt);
+  return dependencies.serializeMutation(() =>
+    cancellation?.cancelled
+      ? convergeTimeout(
+          dependencies,
+          input.runId,
+          running,
+          at,
+          cancellation.cumulativeUsage,
+        )
+      : convergeWait(
+          dependencies,
+          input.runId,
+          running,
+          requiredAdapter(running),
+          at,
+        ),
+  );
+}
+
+function requiredAdapter(nodeRun: AgentWorkflowNodeRun) {
+  if (!nodeRun.adapter) throw new Error("Node Run Adapter coordinate 缺失");
+  return nodeRun.adapter;
+}