@runuai/host 0.9.162 → 0.9.164

package/lib/agents/rate-limit.ts

@@ -4,14 +4,17 @@
  * When an engine reports an error that means "this account is temporarily out
  * of capacity" (usage cap, 429, quota, overloaded), the orchestrator cools the
  * current account and fails over to the next one. Each adapter's pure mapper
- * runs the error text through {@link isRateLimitMessage} and tags the resulting
- * `error` event with `retryable: "rate_limit"` so rotation can act on it
- * without re-parsing engine-specific strings downstream.
+ * runs the error text through {@link isRateLimitMessage}, emits a secret-free
+ * account usage observation, and tags the resulting `error` event with
+ * `retryable: "rate_limit"` so rotation can act on it without re-parsing
+ * engine-specific strings downstream.
  *
  * Deliberately broad but anchored on unambiguous tokens — a false positive
  * only rotates to another account (cheap, self-correcting), a false negative
  * just surfaces the error as today.
  */
+import type { AgentAccountUsage } from "./types";
+
 const RATE_LIMIT_PATTERNS: RegExp[] = [
   /rate[\s_-]?limit/i,
   /\b429\b/,
@@ -39,3 +42,296 @@ export function isRateLimitMessage(message: string | undefined | null): boolean
   if (!message) return false;
   return RATE_LIMIT_PATTERNS.some((re) => re.test(message));
 }
+
+/** Build the account observation shared by every adapter's error-text path. */
+export function rateLimitErrorUsage(
+  message: string,
+  now: number = Date.now(),
+  timeZone?: string,
+): AgentAccountUsage | null {
+  if (!isRateLimitMessage(message)) return null;
+  const resetsAt = parseResetHint(message, now, timeZone);
+  return {
+    status: "limited",
+    ...(resetsAt === null ? {} : { resetsAt }),
+    windows: [],
+    source: "error-text",
+    observedAt: now,
+  };
+}
+
+const MONTHS = new Map(
+  ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"].map(
+    (month, index) => [month, index + 1] as const,
+  ),
+);
+const MAX_RESET_SECONDS = 100_000_000_000;
+const MAX_RESET_AHEAD_MS = 365 * 24 * 60 * 60_000;
+
+/** Convert a provider Unix-seconds reset without accepting milliseconds or a
+ * corrupt timestamp that is more than a year beyond this observation. */
+export function resetSecondsToEpochMs(
+  value: unknown,
+  observedAt: number,
+): number | undefined {
+  if (
+    typeof value !== "number" ||
+    !Number.isSafeInteger(value) ||
+    value <= 0 ||
+    value > MAX_RESET_SECONDS ||
+    !Number.isFinite(observedAt)
+  ) {
+    return undefined;
+  }
+  const candidate = value * 1_000;
+  return candidate <= observedAt + MAX_RESET_AHEAD_MS
+    ? candidate
+    : undefined;
+}
+
+interface CivilTime {
+  year: number;
+  month: number;
+  day: number;
+  hour: number;
+  minute: number;
+  second: number;
+}
+
+function formatter(timeZone: string): Intl.DateTimeFormat | null {
+  try {
+    return new Intl.DateTimeFormat("en-US-u-ca-gregory-nu-latn", {
+      timeZone,
+      year: "numeric",
+      month: "2-digit",
+      day: "2-digit",
+      hour: "2-digit",
+      minute: "2-digit",
+      second: "2-digit",
+      hourCycle: "h23",
+    });
+  } catch {
+    return null;
+  }
+}
+
+function civilAt(epochMs: number, fmt: Intl.DateTimeFormat): CivilTime | null {
+  const values = new Map(
+    fmt
+      .formatToParts(new Date(epochMs))
+      .filter((part) => part.type !== "literal")
+      .map((part) => [part.type, Number(part.value)]),
+  );
+  const civil = {
+    year: values.get("year") ?? NaN,
+    month: values.get("month") ?? NaN,
+    day: values.get("day") ?? NaN,
+    hour: values.get("hour") ?? NaN,
+    minute: values.get("minute") ?? NaN,
+    second: values.get("second") ?? NaN,
+  };
+  return Object.values(civil).every(Number.isFinite) ? civil : null;
+}
+
+function sameCivil(left: CivilTime | null, right: CivilTime): boolean {
+  return (
+    left !== null &&
+    left.year === right.year &&
+    left.month === right.month &&
+    left.day === right.day &&
+    left.hour === right.hour &&
+    left.minute === right.minute &&
+    left.second === right.second
+  );
+}
+
+/** Convert a civil time in an IANA zone to an instant. Sampling the offsets
+ * on both sides of the date makes DST folds visible: zero or multiple matching
+ * instants are refused instead of guessing. */
+function epochForCivil(civil: CivilTime, timeZone: string): number | null {
+  const fmt = formatter(timeZone);
+  if (!fmt) return null;
+  const target = Date.UTC(
+    civil.year,
+    civil.month - 1,
+    civil.day,
+    civil.hour,
+    civil.minute,
+    civil.second,
+  );
+  const normalized = new Date(target);
+  if (
+    normalized.getUTCFullYear() !== civil.year ||
+    normalized.getUTCMonth() + 1 !== civil.month ||
+    normalized.getUTCDate() !== civil.day ||
+    normalized.getUTCHours() !== civil.hour ||
+    normalized.getUTCMinutes() !== civil.minute
+  ) {
+    return null;
+  }
+
+  const offsets = new Set<number>();
+  for (let hours = -36; hours <= 36; hours += 6) {
+    const probe = target + hours * 60 * 60_000;
+    const parts = civilAt(probe, fmt);
+    if (!parts) continue;
+    offsets.add(
+      Date.UTC(
+        parts.year,
+        parts.month - 1,
+        parts.day,
+        parts.hour,
+        parts.minute,
+        parts.second,
+      ) - probe,
+    );
+  }
+  const candidates = [...offsets]
+    .map((offset) => target - offset)
+    .filter((candidate) => sameCivil(civilAt(candidate, fmt), civil));
+  const unique = [...new Set(candidates)];
+  return unique.length === 1 ? unique[0]! : null;
+}
+
+function localDate(now: number, timeZone: string): CivilTime | null {
+  const fmt = formatter(timeZone);
+  return fmt ? civilAt(now, fmt) : null;
+}
+
+function parsedZone(raw: string | undefined, fallback: string): string | null {
+  if (!raw) return fallback;
+  const zone = raw.trim();
+  if (/^(?:UTC|GMT)$/i.test(zone)) return "UTC";
+  // Abbreviations such as CST name several zones and are therefore unsafe.
+  if (!zone.includes("/")) return null;
+  return formatter(zone) ? zone : null;
+}
+
+function clockHour(rawHour: string, meridiem: string): number | null {
+  const hour = Number(rawHour);
+  if (!Number.isInteger(hour) || hour < 1 || hour > 12) return null;
+  return (hour % 12) + (meridiem.toLowerCase() === "pm" ? 12 : 0);
+}
+
+/**
+ * Parse the deliberately small set of reset hints emitted by engine CLIs.
+ * Returns epoch milliseconds, or null when the text is invalid, names an
+ * ambiguous civil instant, contains multiple hints, or needs assumptions the
+ * message did not supply.
+ */
+export function parseResetHint(
+  message: string,
+  now: number,
+  timeZone: string = Intl.DateTimeFormat().resolvedOptions().timeZone,
+): number | null {
+  if (!message || !Number.isFinite(now) || !formatter(timeZone)) return null;
+  const found: number[] = [];
+
+  // Claude also emits a machine-readable suffix in some terminal errors.
+  // It is Unix seconds; reject millisecond-scale or implausibly distant input
+  // rather than turning malformed text into a long account bench.
+  const claudeEpoch = /\bClaude AI usage limit reached\|(\d+)\b/gi;
+  for (const match of message.matchAll(claudeEpoch)) {
+    const seconds = Number(match[1]);
+    const candidate = resetSecondsToEpochMs(seconds, now);
+    if (candidate !== undefined && candidate > now) found.push(candidate);
+  }
+
+  const relative = /\b(?:try\s+again|resets?)\s+in\s+((?:\d+\s*(?:days?|d|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)\s*){1,4})(?=$|[.,;!?])/gi;
+  for (const match of message.matchAll(relative)) {
+    const units = new Set<string>();
+    let duration = 0;
+    const token = /(\d+)\s*(days?|d|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)/gi;
+    let consumed = "";
+    for (const part of match[1]!.matchAll(token)) {
+      const value = Number(part[1]);
+      const rawUnit = part[2]!.toLowerCase();
+      const unit = rawUnit.startsWith("d")
+        ? "d"
+        : rawUnit.startsWith("h")
+          ? "h"
+          : rawUnit.startsWith("m")
+            ? "m"
+            : "s";
+      if (!Number.isSafeInteger(value) || value <= 0 || units.has(unit)) {
+        duration = NaN;
+        break;
+      }
+      units.add(unit);
+      consumed += part[0].replace(/\s+/g, "");
+      duration +=
+        value *
+        (unit === "d"
+          ? 24 * 60 * 60_000
+          : unit === "h"
+            ? 60 * 60_000
+            : unit === "m"
+              ? 60_000
+              : 1_000);
+    }
+    if (
+      Number.isSafeInteger(duration) &&
+      duration > 0 &&
+      consumed.toLowerCase() === match[1]!.replace(/\s+/g, "").toLowerCase()
+    ) {
+      found.push(now + duration);
+    }
+  }
+
+  const monthNames = "Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?";
+  const dated = new RegExp(
+    `\\bresets?\\s+(${monthNames})\\s+(\\d{1,2})(?:,?\\s+(\\d{4}))?(?:,?\\s+(\\d{1,2})(?::(\\d{2}))?\\s*(am|pm))?(?:\\s*\\(([^)]+)\\))?(?=$|[.,;!?])`,
+    "gi",
+  );
+  for (const match of message.matchAll(dated)) {
+    const month = MONTHS.get(match[1]!.slice(0, 3).toLowerCase());
+    const day = Number(match[2]);
+    const explicitYear = match[3] === undefined ? null : Number(match[3]);
+    const hour = match[4] === undefined ? 0 : clockHour(match[4], match[6]!);
+    const minute = match[5] === undefined ? 0 : Number(match[5]);
+    // The CLI runs in a container whose local zone need not match this host
+    // process. A calendar date is zone-sensitive even when it names no clock,
+    // so every absolute hint must name its zone.
+    const zone =
+      match[7] === undefined ? null : parsedZone(match[7], timeZone);
+    const base = zone ? localDate(now, zone) : null;
+    if (!base || !month || !zone || hour === null || minute < 0 || minute > 59) continue;
+    let year = explicitYear ?? base.year;
+    let candidate = epochForCivil({ year, month, day, hour, minute, second: 0 }, zone);
+    if (explicitYear === null && candidate !== null && candidate <= now) {
+      year += 1;
+      candidate = epochForCivil({ year, month, day, hour, minute, second: 0 }, zone);
+    }
+    if (candidate !== null && candidate > now) found.push(candidate);
+  }
+
+  const clock = /\bresets?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\(([^)]+)\))?(?=$|[.,;!?])/gi;
+  for (const match of message.matchAll(clock)) {
+    const hour = clockHour(match[1]!, match[3]!);
+    const minute = match[2] === undefined ? 0 : Number(match[2]);
+    const zone = match[4] === undefined ? null : parsedZone(match[4], timeZone);
+    const base = zone ? localDate(now, zone) : null;
+    if (!base || hour === null || minute < 0 || minute > 59 || !zone) continue;
+    let candidate = epochForCivil(
+      { year: base.year, month: base.month, day: base.day, hour, minute, second: 0 },
+      zone,
+    );
+    if (candidate !== null && candidate <= now) {
+      const tomorrow = new Date(Date.UTC(base.year, base.month - 1, base.day + 1));
+      candidate = epochForCivil(
+        {
+          year: tomorrow.getUTCFullYear(),
+          month: tomorrow.getUTCMonth() + 1,
+          day: tomorrow.getUTCDate(),
+          hour,
+          minute,
+          second: 0,
+        },
+        zone,
+      );
+    }
+    if (candidate !== null && candidate > now) found.push(candidate);
+  }
+
+  return found.length === 1 ? found[0]! : null;
+}

package/lib/agents/types.ts

@@ -13,6 +13,7 @@
 
 import { z } from "zod";
 
+import type { EngineUsageWindow } from "../../src/protocol";
 import type { TaskEnvironmentAgentSessionSurface } from "../task-environment/types";
 
 /** The host-advertised adapter kind ("claude" | "codex" | "gemini" | …). */
@@ -50,6 +51,13 @@ export const RosterAgent = z.object({
   kind: AgentKind,
   model: z.string().min(1).optional(),
   effort: z.string().min(1).optional(), // CLI reasoning level, passed through
+  // ADR-157 addendum §7: preferred cloud-leased AI account.
+  accountId: z
+    .string()
+    .min(1)
+    .max(128)
+    .regex(/^[A-Za-z0-9_.:-]+$/)
+    .optional(),
   defaultPrompt: z.string().optional(), // persona
   initialPrompt: z.string().optional(), // first-turn instructions, this task only
   skills: z.array(RosterSkill).optional(), // ADR-046 reference material
@@ -108,6 +116,24 @@ export interface AgentUsage {
   costUsd?: number;
 }
 
+/** Secret-free account usage exactly as an engine CLI reported it. The
+ * orchestrator adds task/account identity only after proving the session is
+ * running on a cloud-leased account. */
+export interface AgentAccountUsage {
+  status: "allowed" | "warning" | "limited";
+  resetsAt?: number;
+  usingOverage?: boolean;
+  windows: EngineUsageWindow[];
+  source: "claude-rate-limit-event" | "codex-rate-limits" | "error-text";
+  observedAt: number;
+  /** Which provider capacity this reading can prove usable again. Claude emits
+   * one event per named window; Codex snapshots cover every window. Internal
+   * runner metadata only: it is deliberately not part of engine.usage. */
+  coverage?:
+    | { kind: "window"; window: string }
+    | { kind: "all-windows" };
+}
+
 export type AgentEvent =
   /** A chunk of streaming assistant text. Appended to the in-progress message. */
   | { type: "message_delta"; text: string }
@@ -129,10 +155,19 @@ export type AgentEvent =
   /** The turn (one request → response cycle) is done; agent is idle.
    *  `usage` carries this turn's token/cost accounting when the CLI reports it. */
   | { type: "turn_complete"; usage?: AgentUsage }
+  /** Account-level capacity reported by the engine CLI. This never contains
+   * credentials or provider account identifiers. */
+  | { type: "account_usage"; usage: AgentAccountUsage }
   /** A recoverable error surfaced by the agent. `retryable` marks errors the
    *  orchestrator can fail over from — "rate_limit" triggers account rotation
    *  (ADR-076). */
-  | { type: "error"; message: string; retryable?: "rate_limit" }
+  | {
+      type: "error";
+      message: string;
+      retryable?: "rate_limit";
+      /** Parsed from the same rate-limit error text, when unambiguous. */
+      resetsAt?: number;
+    }
   /** The underlying process exited. */
   | { type: "exit"; code: number };
 

package/lib/engine-accounts.ts

@@ -440,7 +440,11 @@ export function engineAccountCount(
 // ---------------------------------------------------------------------------
 
 interface RotationState {
-  cooldownUntil: number;
+  /** One account may be capped by several independent provider windows. Null
+   * is a rate-limit error that named no window and needs a full snapshot to
+   * disprove. Each cause keeps its own deadline so clearing one cannot clear
+   * or extend another. */
+  cooldowns: Map<string | null, number>;
   lastUsedAt: number;
 }
 
@@ -448,23 +452,97 @@ const rotationState = new Map<string, RotationState>();
 
 /** Default cooldown when the engine gives no reset hint. */
 export const DEFAULT_COOLDOWN_MS = 15 * 60_000;
+/** Provider resets too close to the current instant still need a short bench
+ * so repeated terminal events cannot hot-loop the same account. */
+export const MIN_ENGINE_ACCOUNT_COOLDOWN_MS = 60_000;
+/** Refuse a corrupt/far-future provider timestamp from benching an account
+ * indefinitely. */
+export const MAX_ENGINE_ACCOUNT_COOLDOWN_MS = 7 * 24 * 60 * 60_000;
 
 function state(id: string): RotationState {
   let st = rotationState.get(id);
   if (!st) {
-    st = { cooldownUntil: 0, lastUsedAt: 0 };
+    st = { cooldowns: new Map(), lastUsedAt: 0 };
     rotationState.set(id, st);
   }
   return st;
 }
 
+function cooldown(
+  id: string,
+  window: string | null,
+  ms: number,
+  now: number,
+): void {
+  const account = state(id);
+  const previous = account.cooldowns.get(window) ?? 0;
+  account.cooldowns.set(window, Math.max(previous, now + ms));
+}
+
+function cooldownDuration(resetsAt: number, now: number): number {
+  return Number.isFinite(resetsAt)
+    ? Math.min(
+        MAX_ENGINE_ACCOUNT_COOLDOWN_MS,
+        Math.max(MIN_ENGINE_ACCOUNT_COOLDOWN_MS, resetsAt - now),
+      )
+    : DEFAULT_COOLDOWN_MS;
+}
+
+function accountCooldownUntil(id: string): number {
+  return Math.max(0, ...state(id).cooldowns.values());
+}
+
 /** Put an account on cooldown (rate-limited). */
 export function cooldownEngineAccount(
   id: string,
   ms: number = DEFAULT_COOLDOWN_MS,
   now: number = Date.now(),
 ): void {
-  state(id).cooldownUntil = now + ms;
+  cooldown(id, null, ms, now);
+}
+
+/** Put an account on cooldown through the engine's real reset, bounded against
+ * stale or corrupt timestamps. */
+export function cooldownEngineAccountUntil(
+  id: string,
+  resetsAt: number,
+  now: number = Date.now(),
+): void {
+  cooldown(id, null, cooldownDuration(resetsAt, now), now);
+}
+
+/** Bench one named provider window without conflating it with another. */
+export function cooldownEngineAccountWindow(
+  id: string,
+  window: string,
+  ms: number = DEFAULT_COOLDOWN_MS,
+  now: number = Date.now(),
+): void {
+  cooldown(id, window, ms, now);
+}
+
+/** Bench one named provider window through its bounded real reset. */
+export function cooldownEngineAccountWindowUntil(
+  id: string,
+  window: string,
+  resetsAt: number,
+  now: number = Date.now(),
+): void {
+  cooldown(id, window, cooldownDuration(resetsAt, now), now);
+}
+
+/** A per-window reading proves only that same provider window usable. */
+export function clearEngineAccountCooldownWindow(
+  id: string,
+  window: string,
+): void {
+  rotationState.get(id)?.cooldowns.delete(window);
+}
+
+/** A complete provider snapshot with no limited window disproves every bench,
+ * including an error that could not name which window was exhausted. */
+export function clearEngineAccountCooldowns(id: string): void {
+  rotationState.get(id)?.cooldowns.clear();
 }
 
 /** Record that an account was just used (least-recently-used ordering). */
@@ -480,14 +558,18 @@ export function noteEngineAccountUsed(id: string, now: number = Date.now()): voi
  */
 export function selectAccount(
   accounts: ResolvedEngineAccount[],
-  opts: { exclude?: Set<string>; now?: number } = {},
+  opts: { exclude?: Set<string>; now?: number; preferredId?: string } = {},
 ): ResolvedEngineAccount | null {
   const now = opts.now ?? Date.now();
   const exclude = opts.exclude ?? new Set<string>();
   const candidates = accounts.filter((a) => !exclude.has(a.id));
   if (candidates.length === 0) return null;
   const lru = (a: ResolvedEngineAccount): number => state(a.id).lastUsedAt;
-  const healthy = candidates.filter((a) => state(a.id).cooldownUntil <= now);
+  const healthy = candidates.filter((a) => accountCooldownUntil(a.id) <= now);
+  const preferred = opts.preferredId
+    ? healthy.find((account) => account.id === opts.preferredId)
+    : undefined;
+  if (preferred) return preferred;
   // Manage-v1: the list arrives in PRIORITY order (default slot, then extras
   // by position) — the first healthy account wins, so the operator's
   // ordering decides which account a task uses and which are fallbacks.
@@ -507,6 +589,8 @@ export function pickEngineAccount(
     exclude?: Set<string>;
     seams?: Partial<AccountSeams>;
     now?: number;
+    /** One agent's durable preference; cooling or absent falls through. */
+    preferredId?: string;
     /** ADR-131 D3: the task's scope; default = the host's own accounts. */
     scope?: string;
   } = {},
@@ -516,7 +600,11 @@ export function pickEngineAccount(
     opts.seams ?? {},
     opts.scope ?? HOST_ACCOUNT_SCOPE,
   );
-  return selectAccount(accounts, { exclude: opts.exclude, now: opts.now });
+  return selectAccount(accounts, {
+    exclude: opts.exclude,
+    now: opts.now,
+    preferredId: opts.preferredId,
+  });
 }
 
 /**