@abelxiaoxing/cadence 1.3.0 → 1.4.1

package/src/worker-broker.ts

@@ -5,11 +5,9 @@ import {
   type WorkerRole,
   type WorkerRoutePolicy,
 } from "./route-policy.ts";
+import { TransportTimeout } from "./transport-budget.ts";
 
 export const ROUTE_ATTEMPT_BOUNDS = Object.freeze({
-  connectMs: 10_000,
-  firstResponseMs: 90_000,
-  idleMs: 3 * 60_000,
   totalMs: 20 * 60_000,
   cooldownMs: 30_000,
   attemptsPerRoute: 2,
@@ -21,6 +19,8 @@ export interface RouteRequirements {
   dialects?: RouteDialect[];
   minContextWindow?: number;
   minOutputTokens?: number;
+  preferredContextWindow?: number;
+  preferredOutputTokens?: number;
 }
 
 interface RouteHealth {
@@ -56,6 +56,7 @@ export interface WorkerBrokerOptions {
 
 export interface RunWorkerAttemptInput<T> {
   runId: string;
+  taskId?: string;
   operationId: string;
   role: WorkerRole;
   requirements?: RouteRequirements;
@@ -68,6 +69,7 @@ export interface RunWorkerAttemptInput<T> {
 export interface RouteAttemptContext {
   route: WorkerRoutePolicy;
   signal: AbortSignal;
+  onRequestStart(): void;
   onHeaders(): void;
   onProgress(): void;
 }
@@ -78,7 +80,7 @@ export interface BrokerAttemptEvidence {
 }
 
 export interface BrokerActivityUpdate {
-  state: "connecting" | "waiting-first-response" | "running" | "retrying";
+  state: "preparing" | "waiting-first-response" | "running" | "retrying";
   attempt: number;
   maxAttempts: number;
   code?: string;
@@ -97,12 +99,7 @@ function emitBrokerActivity(
 }
 
 class RouteAttemptStop extends Error {
-  readonly code:
-    | "connect-timeout"
-    | "first-response-timeout"
-    | "idle-timeout"
-    | "phase-timeout"
-    | "cancelled";
+  readonly code: "cancelled";
 
   constructor(code: RouteAttemptStop["code"]) {
     super(code);
@@ -126,7 +123,13 @@ function validRequirements(value: unknown): value is RouteRequirements {
   if (
     Object.keys(candidate).some(
       (key) =>
-        !["dialects", "minContextWindow", "minOutputTokens"].includes(key),
+        ![
+          "dialects",
+          "minContextWindow",
+          "minOutputTokens",
+          "preferredContextWindow",
+          "preferredOutputTokens",
+        ].includes(key),
     )
   ) {
     return false;
@@ -147,7 +150,12 @@ function validRequirements(value: unknown): value is RouteRequirements {
         (candidate.minContextWindow as number) > 0)) &&
     (candidate.minOutputTokens === undefined ||
       (Number.isSafeInteger(candidate.minOutputTokens) &&
-        (candidate.minOutputTokens as number) > 0))
+        (candidate.minOutputTokens as number) > 0)) &&
+    [candidate.preferredContextWindow, candidate.preferredOutputTokens].every(
+      (value) =>
+        value === undefined ||
+        (Number.isSafeInteger(value) && (value as number) > 0),
+    )
   );
 }
 
@@ -168,7 +176,8 @@ function capable(
 }
 
 function safeFailureCode(error: unknown): string {
-  if (error instanceof RouteAttemptStop) return error.code;
+  if (error instanceof RouteAttemptStop || error instanceof TransportTimeout)
+    return error.code;
   return "transport-failure";
 }
 
@@ -176,7 +185,6 @@ export class WorkerBroker {
   readonly #policy: RoutePolicy;
   readonly #now: () => number;
   readonly #healthStore: RouteHealthStore;
-  readonly #health = new Map<string, RouteHealth>();
 
   constructor(policy: RoutePolicy, options: WorkerBrokerOptions = {}) {
     this.#policy = structuredClone(policy);
@@ -187,7 +195,6 @@ export class WorkerBroker {
   #routeHealth(routeId: string): RouteHealth {
     const route = this.#policy.routes[routeId];
     return (
-      this.#health.get(routeId) ??
       (route ? this.#healthStore.get(route.fingerprint) : undefined) ?? {
         state: "healthy",
       }
@@ -197,7 +204,6 @@ export class WorkerBroker {
   #setRouteHealth(routeId: string, health: RouteHealth): void {
     const route = this.#policy.routes[routeId];
     if (!route) throw new Error("route-not-declared");
-    this.#health.set(routeId, structuredClone(health));
     this.#healthStore.set(route.fingerprint, health);
   }
 
@@ -229,14 +235,13 @@ export class WorkerBroker {
     });
   }
 
-  select(input: {
-    role: WorkerRole;
-    dialects?: RouteDialect[];
-    minContextWindow?: number;
-    minOutputTokens?: number;
-    exclude?: readonly string[];
-    routeId?: string;
-  }):
+  select(
+    input: RouteRequirements & {
+      role: WorkerRole;
+      exclude?: readonly string[];
+      routeId?: string;
+    },
+  ):
     | { ok: true; route: WorkerRoutePolicy; health: RouteHealthState }
     | { ok: false; code: "endpoint-unavailable" | "route-not-declared" } {
     const requirements: RouteRequirements = {
@@ -247,6 +252,12 @@ export class WorkerBroker {
       ...(input.minOutputTokens === undefined
         ? {}
         : { minOutputTokens: input.minOutputTokens }),
+      ...(input.preferredContextWindow === undefined
+        ? {}
+        : { preferredContextWindow: input.preferredContextWindow }),
+      ...(input.preferredOutputTokens === undefined
+        ? {}
+        : { preferredOutputTokens: input.preferredOutputTokens }),
     };
     if (!isRole(input.role) || !validRequirements(requirements)) {
       return { ok: false, code: "endpoint-unavailable" };
@@ -261,7 +272,22 @@ export class WorkerBroker {
       return { ok: false, code: "route-not-declared" };
     }
     const excluded = new Set(input.exclude ?? []);
-    for (const routeId of candidates) {
+    // Stable preference tiers preserve declared order within each tier. Health,
+    // hard minima and explicit pins still determine eligibility.
+    const preferred = (routeId: string) => {
+      const route = this.#policy.routes[routeId];
+      return route &&
+        route.capabilities.contextWindow >=
+          (requirements.preferredContextWindow ?? 0) &&
+        route.capabilities.maxTokens >=
+          (requirements.preferredOutputTokens ?? 0)
+        ? 1
+        : 0;
+    };
+    const ordered = [...candidates].sort(
+      (left, right) => preferred(right) - preferred(left),
+    );
+    for (const routeId of ordered) {
       if (excluded.has(routeId)) continue;
       const route = this.#policy.routes[routeId];
       if (!route || !capable(route, input.role, requirements)) continue;
@@ -300,59 +326,39 @@ export class WorkerBroker {
     onActivity?: (state: "waiting-first-response" | "running") => void,
   ): Promise<T> {
     const controller = new AbortController();
-    const timers = new Set<ReturnType<typeof setTimeout>>();
-    let headersSeen = false;
-    let connectTimer: ReturnType<typeof setTimeout> | undefined;
-    let firstResponseTimer: ReturnType<typeof setTimeout> | undefined;
-    let idleTimer: ReturnType<typeof setTimeout> | undefined;
-
-    const stop = (code: RouteAttemptStop["code"]) => {
-      if (!controller.signal.aborted)
-        controller.abort(new RouteAttemptStop(code));
-    };
-    const schedule = (code: RouteAttemptStop["code"], milliseconds: number) => {
-      const timer = setTimeout(() => stop(code), milliseconds);
-      timers.add(timer);
-      return timer;
-    };
-    const clear = (timer: ReturnType<typeof setTimeout> | undefined) => {
-      if (timer !== undefined) {
-        clearTimeout(timer);
-        timers.delete(timer);
-      }
-    };
-    const onParentAbort = () => stop("cancelled");
-    if (parentSignal?.aborted) stop("cancelled");
+    let accepting = true;
+    let running = false;
+    const onParentAbort = () =>
+      controller.abort(new RouteAttemptStop("cancelled"));
+    if (parentSignal?.aborted) onParentAbort();
     else parentSignal?.addEventListener("abort", onParentAbort, { once: true });
-
-    connectTimer = schedule("connect-timeout", ROUTE_ATTEMPT_BOUNDS.connectMs);
-    schedule("phase-timeout", ROUTE_ATTEMPT_BOUNDS.totalMs);
+    const timer = setTimeout(
+      () => controller.abort(new TransportTimeout("attempt-timeout")),
+      ROUTE_ATTEMPT_BOUNDS.totalMs,
+    );
     let removeAbortRace: () => void = () => {};
     try {
-      const execution = Promise.resolve().then(() =>
-        execute({
+      const execution = Promise.resolve().then(() => {
+        controller.signal.throwIfAborted();
+        return execute({
           route: structuredClone(route),
           signal: controller.signal,
+          onRequestStart: () => {
+            if (!accepting || controller.signal.aborted) return;
+            running = false;
+            onActivity?.("waiting-first-response");
+          },
           onHeaders: () => {
-            if (controller.signal.aborted || headersSeen) return;
-            headersSeen = true;
-            clear(connectTimer);
+            if (!accepting || controller.signal.aborted || running) return;
             onActivity?.("waiting-first-response");
-            firstResponseTimer = schedule(
-              "first-response-timeout",
-              ROUTE_ATTEMPT_BOUNDS.firstResponseMs,
-            );
           },
           onProgress: () => {
-            if (controller.signal.aborted) return;
-            clear(connectTimer);
-            clear(firstResponseTimer);
-            clear(idleTimer);
-            idleTimer = schedule("idle-timeout", ROUTE_ATTEMPT_BOUNDS.idleMs);
+            if (!accepting || controller.signal.aborted) return;
+            running = true;
             onActivity?.("running");
           },
-        }),
-      );
+        });
+      });
       // A timed-out Provider may ignore its AbortSignal and settle later. The
       // broker owns the bounded result, so absorb that late settlement.
       void execution.catch(() => undefined);
@@ -372,7 +378,8 @@ export class WorkerBroker {
       return await Promise.race([execution, aborted]);
     } finally {
       removeAbortRace();
-      for (const timer of timers) clearTimeout(timer);
+      accepting = false;
+      clearTimeout(timer);
       parentSignal?.removeEventListener("abort", onParentAbort);
     }
   }
@@ -452,10 +459,10 @@ export class WorkerBroker {
       ).length;
       const attempt = evidence.length + 1;
       emitBrokerActivity(input.onActivity, {
-        state: "connecting",
+        state: "preparing",
         attempt,
         maxAttempts,
-        wait: "connection",
+        wait: "worker-setup",
       });
       try {
         const value = await this.#attempt(
@@ -545,6 +552,7 @@ export class WorkerBroker {
         code:
           | "approval-boundary"
           | "route-not-declared"
+          | "endpoint-unavailable"
           | "route-capability-insufficient";
       } {
     const allowed = new Set(["runId", "role", "routeId", "requirements"]);
@@ -566,12 +574,15 @@ export class WorkerBroker {
       routeId: input.routeId,
     });
     if (!selection.ok) {
+      const route = this.#policy.routes[input.routeId];
       return {
         ok: false,
         code:
           selection.code === "route-not-declared"
             ? "route-not-declared"
-            : "route-capability-insufficient",
+            : route && capable(route, input.role, input.requirements ?? {})
+              ? "endpoint-unavailable"
+              : "route-capability-insufficient",
       };
     }
     return {
@@ -606,7 +617,8 @@ export class WorkerBroker {
 }
 
 /**
- * Adds stable run/role route bindings to the stateless route-attempt broker.
+ * Adds run/role defaults and independent task bindings to route attempts.
+ * All bound brokers consult the same health store.
  * Attempt credentials and provider objects remain disposable and are never
  * retained by this facade.
  */
@@ -622,16 +634,21 @@ export class RunWorkerBroker {
 
   constructor(policy: RoutePolicy, options: WorkerBrokerOptions = {}) {
     this.#policy = structuredClone(policy);
-    this.#options = options;
-    this.#defaultBroker = new WorkerBroker(this.#policy, options);
+    this.#options = {
+      ...options,
+      healthStore: options.healthStore ?? new MemoryRouteHealthStore(),
+    };
+    this.#defaultBroker = new WorkerBroker(this.#policy, this.#options);
   }
 
-  #bindingKey(runId: string, role: WorkerRole): string {
-    return `${runId}\0${role}`;
+  #bindingKey(runId: string, role: WorkerRole, taskId?: string): string {
+    return JSON.stringify([runId, role, taskId ?? null]);
   }
 
-  #brokerFor(runId: string, role: WorkerRole): WorkerBroker {
-    const binding = this.#bindings.get(this.#bindingKey(runId, role));
+  #brokerFor(runId: string, role: WorkerRole, taskId?: string): WorkerBroker {
+    const binding =
+      this.#bindings.get(this.#bindingKey(runId, role, taskId)) ??
+      this.#bindings.get(this.#bindingKey(runId, role));
     if (!binding) return this.#defaultBroker;
     const routeId = binding.routeId;
     const key = `${role}\0${routeId}`;
@@ -657,6 +674,8 @@ export class RunWorkerBroker {
     if (
       typeof input.runId !== "string" ||
       input.runId.length === 0 ||
+      (input.taskId !== undefined &&
+        (typeof input.taskId !== "string" || input.taskId.length === 0)) ||
       typeof input.operationId !== "string" ||
       input.operationId.length === 0
     ) {
@@ -667,7 +686,11 @@ export class RunWorkerBroker {
         attempts: [],
       };
     }
-    const result = await this.#brokerFor(input.runId, input.role).run({
+    const result = await this.#brokerFor(
+      input.runId,
+      input.role,
+      input.taskId,
+    ).run({
       operationId: input.operationId,
       role: input.role,
       ...(input.requirements ? { requirements: input.requirements } : {}),
@@ -679,39 +702,54 @@ export class RunWorkerBroker {
     if (result.ok) {
       const route = this.#policy.routes[result.routeId];
       if (!route) throw new Error("broker-route-result-invalid");
-      this.#bindings.set(this.#bindingKey(input.runId, input.role), {
-        routeId: route.id,
-        fingerprint: route.fingerprint,
-      });
+      this.#bindings.set(
+        this.#bindingKey(input.runId, input.role, input.taskId),
+        {
+          routeId: route.id,
+          fingerprint: route.fingerprint,
+        },
+      );
     }
     return result;
   }
 
   rebind(input: {
     runId: string;
+    taskId?: string;
     role: WorkerRole;
     routeId: string;
     requirements?: RouteRequirements;
   }) {
-    const result = this.#defaultBroker.rebind(input);
+    const { taskId, ...routeInput } = input;
+    if (
+      taskId !== undefined &&
+      (typeof taskId !== "string" || taskId.length === 0)
+    ) {
+      return { ok: false as const, code: "approval-boundary" as const };
+    }
+    const result = this.#defaultBroker.rebind(routeInput);
     if (result.ok) {
-      this.#bindings.set(this.#bindingKey(input.runId, input.role), {
-        routeId: result.route.id,
-        fingerprint: result.route.fingerprint,
-      });
+      this.#bindings.set(
+        this.#bindingKey(input.runId, input.role, input.taskId),
+        {
+          routeId: result.route.id,
+          fingerprint: result.route.fingerprint,
+        },
+      );
     }
     return result;
   }
 
   resumeBinding(input: {
     runId: string;
+    taskId?: string;
     role: WorkerRole;
     routeId: string;
     expectedFingerprint?: string;
     requirements?: RouteRequirements;
   }) {
     const binding = this.#bindings.get(
-      this.#bindingKey(input.runId, input.role),
+      this.#bindingKey(input.runId, input.role, input.taskId),
     );
     const route = this.#policy.routes[input.routeId];
     if (!binding) {
@@ -724,6 +762,7 @@ export class RunWorkerBroker {
       }
       return this.rebind({
         runId: input.runId,
+        ...(input.taskId === undefined ? {} : { taskId: input.taskId }),
         role: input.role,
         routeId: input.routeId,
         ...(input.requirements ? { requirements: input.requirements } : {}),
@@ -751,8 +790,12 @@ export class RunWorkerBroker {
     this.#boundBrokers.clear();
   }
 
-  binding(runId: string, role: WorkerRole): string | undefined {
-    return this.#bindings.get(this.#bindingKey(runId, role))?.routeId;
+  binding(
+    runId: string,
+    role: WorkerRole,
+    taskId?: string,
+  ): string | undefined {
+    return this.#bindings.get(this.#bindingKey(runId, role, taskId))?.routeId;
   }
 
   status(): Record<string, unknown> {