@abelxiaoxing/cadence 1.4.0 → 1.5.0

package/src/worker-broker.ts

@@ -5,11 +5,9 @@ import {
   type WorkerRole,
   type WorkerRoutePolicy,
 } from "./route-policy.ts";
+import { TransportTimeout } from "./transport-budget.ts";
 
 export const ROUTE_ATTEMPT_BOUNDS = Object.freeze({
-  connectMs: 10_000,
-  firstResponseMs: 90_000,
-  idleMs: 3 * 60_000,
   totalMs: 20 * 60_000,
   cooldownMs: 30_000,
   attemptsPerRoute: 2,
@@ -71,6 +69,7 @@ export interface RunWorkerAttemptInput<T> {
 export interface RouteAttemptContext {
   route: WorkerRoutePolicy;
   signal: AbortSignal;
+  onRequestStart(): void;
   onHeaders(): void;
   onProgress(): void;
 }
@@ -81,7 +80,7 @@ export interface BrokerAttemptEvidence {
 }
 
 export interface BrokerActivityUpdate {
-  state: "connecting" | "waiting-first-response" | "running" | "retrying";
+  state: "preparing" | "waiting-first-response" | "running" | "retrying";
   attempt: number;
   maxAttempts: number;
   code?: string;
@@ -100,12 +99,7 @@ function emitBrokerActivity(
 }
 
 class RouteAttemptStop extends Error {
-  readonly code:
-    | "connect-timeout"
-    | "first-response-timeout"
-    | "idle-timeout"
-    | "phase-timeout"
-    | "cancelled";
+  readonly code: "cancelled";
 
   constructor(code: RouteAttemptStop["code"]) {
     super(code);
@@ -182,7 +176,10 @@ function capable(
 }
 
 function safeFailureCode(error: unknown): string {
-  if (error instanceof RouteAttemptStop) return error.code;
+  if (error instanceof RouteAttemptStop || error instanceof TransportTimeout)
+    return error.code;
+  if (error instanceof Error && error.message === "child-provider-rate-limited")
+    return error.message;
   return "transport-failure";
 }
 
@@ -331,59 +328,39 @@ export class WorkerBroker {
     onActivity?: (state: "waiting-first-response" | "running") => void,
   ): Promise<T> {
     const controller = new AbortController();
-    const timers = new Set<ReturnType<typeof setTimeout>>();
-    let headersSeen = false;
-    let connectTimer: ReturnType<typeof setTimeout> | undefined;
-    let firstResponseTimer: ReturnType<typeof setTimeout> | undefined;
-    let idleTimer: ReturnType<typeof setTimeout> | undefined;
-
-    const stop = (code: RouteAttemptStop["code"]) => {
-      if (!controller.signal.aborted)
-        controller.abort(new RouteAttemptStop(code));
-    };
-    const schedule = (code: RouteAttemptStop["code"], milliseconds: number) => {
-      const timer = setTimeout(() => stop(code), milliseconds);
-      timers.add(timer);
-      return timer;
-    };
-    const clear = (timer: ReturnType<typeof setTimeout> | undefined) => {
-      if (timer !== undefined) {
-        clearTimeout(timer);
-        timers.delete(timer);
-      }
-    };
-    const onParentAbort = () => stop("cancelled");
-    if (parentSignal?.aborted) stop("cancelled");
+    let accepting = true;
+    let running = false;
+    const onParentAbort = () =>
+      controller.abort(new RouteAttemptStop("cancelled"));
+    if (parentSignal?.aborted) onParentAbort();
     else parentSignal?.addEventListener("abort", onParentAbort, { once: true });
-
-    connectTimer = schedule("connect-timeout", ROUTE_ATTEMPT_BOUNDS.connectMs);
-    schedule("phase-timeout", ROUTE_ATTEMPT_BOUNDS.totalMs);
+    const timer = setTimeout(
+      () => controller.abort(new TransportTimeout("attempt-timeout")),
+      ROUTE_ATTEMPT_BOUNDS.totalMs,
+    );
     let removeAbortRace: () => void = () => {};
     try {
-      const execution = Promise.resolve().then(() =>
-        execute({
+      const execution = Promise.resolve().then(() => {
+        controller.signal.throwIfAborted();
+        return execute({
           route: structuredClone(route),
           signal: controller.signal,
+          onRequestStart: () => {
+            if (!accepting || controller.signal.aborted) return;
+            running = false;
+            onActivity?.("waiting-first-response");
+          },
           onHeaders: () => {
-            if (controller.signal.aborted || headersSeen) return;
-            headersSeen = true;
-            clear(connectTimer);
+            if (!accepting || controller.signal.aborted || running) return;
             onActivity?.("waiting-first-response");
-            firstResponseTimer = schedule(
-              "first-response-timeout",
-              ROUTE_ATTEMPT_BOUNDS.firstResponseMs,
-            );
           },
           onProgress: () => {
-            if (controller.signal.aborted) return;
-            clear(connectTimer);
-            clear(firstResponseTimer);
-            clear(idleTimer);
-            idleTimer = schedule("idle-timeout", ROUTE_ATTEMPT_BOUNDS.idleMs);
+            if (!accepting || controller.signal.aborted) return;
+            running = true;
             onActivity?.("running");
           },
-        }),
-      );
+        });
+      });
       // A timed-out Provider may ignore its AbortSignal and settle later. The
       // broker owns the bounded result, so absorb that late settlement.
       void execution.catch(() => undefined);
@@ -403,7 +380,8 @@ export class WorkerBroker {
       return await Promise.race([execution, aborted]);
     } finally {
       removeAbortRace();
-      for (const timer of timers) clearTimeout(timer);
+      accepting = false;
+      clearTimeout(timer);
       parentSignal?.removeEventListener("abort", onParentAbort);
     }
   }
@@ -483,10 +461,10 @@ export class WorkerBroker {
       ).length;
       const attempt = evidence.length + 1;
       emitBrokerActivity(input.onActivity, {
-        state: "connecting",
+        state: "preparing",
         attempt,
         maxAttempts,
-        wait: "connection",
+        wait: "worker-setup",
       });
       try {
         const value = await this.#attempt(
@@ -576,6 +554,7 @@ export class WorkerBroker {
         code:
           | "approval-boundary"
           | "route-not-declared"
+          | "endpoint-unavailable"
           | "route-capability-insufficient";
       } {
     const allowed = new Set(["runId", "role", "routeId", "requirements"]);
@@ -597,12 +576,15 @@ export class WorkerBroker {
       routeId: input.routeId,
     });
     if (!selection.ok) {
+      const route = this.#policy.routes[input.routeId];
       return {
         ok: false,
         code:
           selection.code === "route-not-declared"
             ? "route-not-declared"
-            : "route-capability-insufficient",
+            : route && capable(route, input.role, input.requirements ?? {})
+              ? "endpoint-unavailable"
+              : "route-capability-insufficient",
       };
     }
     return {

package/src/workflow-policy.ts

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
 import path from "node:path";
+import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
 import { compareCanonicalStrings } from "./canonical.ts";
 import {
   type ApprovalBoundaryCode,
@@ -18,7 +19,6 @@ import type { RoutePolicy } from "./route-policy.ts";
 import { canonicalJson } from "./run-state.ts";
 import type { OperationLease } from "./run-store.ts";
 import type { ResolvedStateRoot } from "./state-root.ts";
-import type { WorkflowActivityUpdate } from "./subagent-activity.ts";
 import {
   type CandidateContextBoundary,
   type CandidateContextRef,
@@ -185,59 +185,20 @@ export interface RedArtifactCorrection {
   contextRequest?: WorkflowContextRequest;
 }
 
-export type RecoveryActionRequest = {
-  kind: "affected-repair" | "cumulative-repair" | "red-correction";
-  attempt: number;
-};
-export type RecoveryActionDecision =
-  | { allowed: true }
-  | {
-      allowed: false;
-      code: "repair-attempts-exhausted" | "artifact-attempts-exhausted";
-    };
+export type {
+  RecoveryActionDecision,
+  RecoveryActionRequest,
+  WorkflowRecoveryFact,
+  WorkflowRecoveryFeedback,
+} from "./workflow-recovery-policy.ts";
+export { decideRecoveryAction } from "./workflow-recovery-policy.ts";
 
-/** One policy for every nested recovery action; execution services supply facts only. */
-export function decideRecoveryAction(
-  plan: ImplementPlan,
-  request: RecoveryActionRequest,
-  authority: { additionalAttempt?: boolean; verificationOnly?: boolean } = {},
-): RecoveryActionDecision {
-  const correction = request.kind === "red-correction";
-  const limit =
-    authority.additionalAttempt || authority.verificationOnly
-      ? 1
-      : correction
-        ? plan.verification.artifactCorrection.maxAttempts
-        : plan.verification.repair.maxAttempts;
-  return Number.isSafeInteger(request.attempt) &&
-    request.attempt > 0 &&
-    request.attempt <= limit
-    ? { allowed: true }
-    : {
-        allowed: false,
-        code: correction
-          ? "artifact-attempts-exhausted"
-          : "repair-attempts-exhausted",
-      };
-}
-
-export interface WorkflowRecoveryFeedback {
-  code: string;
-  attempt: number;
-  maxAttempts: number;
-  strategy:
-    | "revise-candidate"
-    | "refresh-candidate"
-    | "repair-verification"
-    | "compact-patch";
-  failureIdentities?: string[];
-}
-
-export interface WorkflowRecoveryFact {
-  key: string;
-  failures: number;
-  feedback: WorkflowRecoveryFeedback;
-}
+import type {
+  RecoveryActionDecision,
+  RecoveryActionRequest,
+  WorkflowRecoveryFact,
+  WorkflowRecoveryFeedback,
+} from "./workflow-recovery-policy.ts";
 
 export const APPROVAL_AUTHORITY_CATEGORIES = [
   "observable-behavior",

package/src/workflow-recovery-policy.ts

@@ -0,0 +1,95 @@
+import type { ImplementPlan } from "./implement-plan.ts";
+
+export type RecoveryActionRequest = {
+  kind: "affected-repair" | "cumulative-repair" | "red-correction";
+  attempt: number;
+};
+export type RecoveryActionDecision =
+  | { allowed: true }
+  | {
+      allowed: false;
+      code: "repair-attempts-exhausted" | "artifact-attempts-exhausted";
+    };
+
+/** One policy for every nested recovery action; execution services supply facts only. */
+export function decideRecoveryAction(
+  plan: ImplementPlan,
+  request: RecoveryActionRequest,
+  authority: { additionalAttempt?: boolean; verificationOnly?: boolean } = {},
+): RecoveryActionDecision {
+  const correction = request.kind === "red-correction";
+  const limit =
+    authority.additionalAttempt || authority.verificationOnly
+      ? 1
+      : correction
+        ? plan.verification.artifactCorrection.maxAttempts
+        : plan.verification.repair.maxAttempts;
+  return Number.isSafeInteger(request.attempt) &&
+    request.attempt > 0 &&
+    request.attempt <= limit
+    ? { allowed: true }
+    : {
+        allowed: false,
+        code: correction
+          ? "artifact-attempts-exhausted"
+          : "repair-attempts-exhausted",
+      };
+}
+
+export interface WorkflowRecoveryFeedback {
+  code: string;
+  attempt: number;
+  maxAttempts: number;
+  strategy:
+    | "revise-candidate"
+    | "refresh-candidate"
+    | "repair-verification"
+    | "compact-patch";
+  failureIdentities?: string[];
+}
+
+export interface WorkflowRecoveryFact {
+  key: string;
+  failures: number;
+  feedback: WorkflowRecoveryFeedback;
+}
+
+export function recoveryExhausted(
+  incident: WorkflowRecoveryFact | undefined,
+  maximum: number,
+  granted: boolean,
+): boolean {
+  return !granted && incident !== undefined && incident.failures >= maximum;
+}
+
+/** A grant is a recommendation until the state machine validates and records it in its lease transaction. */
+export function assessRecoveryGrant(input: {
+  current: boolean;
+  incident?: WorkflowRecoveryFact;
+  failureSequence: number;
+  requestedSequence: number;
+  reason: string;
+  previous: { route?: string | null; context?: string | null };
+  observed: { route?: string | null; context?: string | null };
+  budget?: { used: number; max_work: number };
+}): string | undefined {
+  if (
+    !input.current ||
+    !input.incident ||
+    input.incident.failures < input.incident.feedback.maxAttempts ||
+    input.failureSequence !== input.requestedSequence
+  )
+    return "recovery-request-stale";
+  if (
+    (input.reason === "route-changed" &&
+      (!input.previous.route ||
+        input.previous.route === input.observed.route)) ||
+    (input.reason === "context-extended" &&
+      (!input.previous.context ||
+        input.previous.context === input.observed.context))
+  )
+    return "recovery-evidence-unavailable";
+  if (!input.budget || input.budget.used >= input.budget.max_work)
+    return "change-work-budget-exhausted";
+  return undefined;
+}

package/src/workflow-scheduling.ts

@@ -0,0 +1,105 @@
+import type { PlanTaskDraft } from "./implement-plan.ts";
+import { type EngineTaskRow, taskConflicts } from "./workflow-policy.ts";
+
+function taskPlan(row: EngineTaskRow): PlanTaskDraft {
+  return JSON.parse(row.plan_json) as PlanTaskDraft;
+}
+export function hasTaskConflict(
+  row: EngineTaskRow,
+  rows: EngineTaskRow[],
+  selectedTaskIds: ReadonlySet<string>,
+): boolean {
+  const candidate = taskPlan(row);
+  return rows.some((other) => {
+    if (other.task_id === row.task_id || other.state === "verified") {
+      return false;
+    }
+    if (other.state === "queued") {
+      if (row.state !== "queued") return false;
+      const candidatePosition = row.queue_position ?? Number.MAX_SAFE_INTEGER;
+      const otherPosition = other.queue_position ?? Number.MAX_SAFE_INTEGER;
+      if (
+        otherPosition > candidatePosition ||
+        (otherPosition === candidatePosition &&
+          other.task_order > row.task_order)
+      ) {
+        return false;
+      }
+    }
+    if (other.task_order > row.task_order && other.state === "pending") {
+      return false;
+    }
+    if (other.state === "pending" && !dependenciesVerified(other, rows)) {
+      return false;
+    }
+    const conflict = taskConflicts(candidate, taskPlan(other));
+    return (
+      conflict.taskLifetime ||
+      (conflict.verification &&
+        ([
+          "pending",
+          "phase-ready",
+          "phase-running",
+          "validating",
+          "queued",
+        ].includes(other.state) ||
+          selectedTaskIds.has(other.task_id)))
+    );
+  });
+}
+
+export function dependenciesVerified(
+  row: EngineTaskRow,
+  rows: EngineTaskRow[],
+): boolean {
+  const byId = new Map(rows.map((candidate) => [candidate.task_id, candidate]));
+  return taskPlan(row).dependsOn.every(
+    (dependency) => byId.get(dependency)?.state === "verified",
+  );
+}
+
+/** Simulate ordered queue decisions without storage access or launch authority. */
+export function selectRunnableTasks(input: {
+  rows: readonly EngineTaskRow[];
+  attempted: ReadonlySet<string>;
+  activeTasks: number;
+  capacity: number;
+  nextQueuePosition: number;
+}) {
+  const rows = input.rows.map((row) => ({ ...row }));
+  const runnable: string[] = [];
+  const selected = new Set<string>();
+  const updates: Array<
+    | { kind: "pending"; taskId: string }
+    | { kind: "queue"; taskId: string; reason: "conflict" | "capacity" }
+  > = [];
+  let position = input.nextQueuePosition;
+  let capacityBlocked = false;
+  for (const row of rows) {
+    if (row.state === "verified" || input.attempted.has(row.task_id)) continue;
+    if (!dependenciesVerified(row, rows)) {
+      if (row.state !== "pending") {
+        updates.push({ kind: "pending", taskId: row.task_id });
+        row.state = "pending";
+        row.pause_code = null;
+        row.queue_position = null;
+      }
+      continue;
+    }
+    const conflict = hasTaskConflict(row, rows, selected);
+    const capacity = input.activeTasks + runnable.length >= input.capacity;
+    if (conflict || capacity) {
+      const reason = conflict ? "conflict" : "capacity";
+      updates.push({ kind: "queue", taskId: row.task_id, reason });
+      if (row.state !== "queued" || row.queue_position === null)
+        row.queue_position = position++;
+      row.state = "queued";
+      row.pause_code = reason === "capacity" ? "worker-capacity" : null;
+      if (!conflict) capacityBlocked = true;
+      continue;
+    }
+    runnable.push(row.task_id);
+    selected.add(row.task_id);
+  }
+  return { runnable, updates, capacityBlocked };
+}