@moondeck/station 0.0.1 → 1.0.0

package/lib/agents/output-source.ts

@@ -0,0 +1,3 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+/** Stable outbox position follows adapter async work, never a timestamp. */
+export const agentOutputSource = new AsyncLocalStorage<string>();

package/lib/agents/proc.ts

@@ -0,0 +1,221 @@
+/** Provider-backed process adapters shared by the agent protocols. */
+
+import { EventEmitter } from "node:events";
+
+import type {
+  TaskEnvironmentAgentSessionSurface,
+  TaskEnvironmentProcess,
+  TaskEnvironmentSessionRequest,
+} from "../task-environment/types";
+import { agentCliArgv } from "../agent-cli";
+
+export type LineHandler = (line: string) => void;
+export type ExitHandler = (code: number | null) => void;
+
+/** How much trailing stderr to retain for error reporting. */
+const STDERR_CAP = 8 * 1024;
+/** Maximum unread stdout+stderr retained by a provider for an agent process. */
+export const AGENT_SESSION_OUTPUT_BUFFER_BYTES = 1024 * 1024;
+
+/**
+ * Line-oriented adapter over the provider-neutral environment
+ * stream. Startup may be asynchronous, so writes are serialized behind the
+ * provider's spawn promise without exposing a container command to callers.
+ */
+export class EnvironmentLineProcess {
+  private readonly processPromise: Promise<TaskEnvironmentProcess>;
+  private process: TaskEnvironmentProcess | null = null;
+  private stdoutBuf = "";
+  private stderrBuf = "";
+  private readonly lineHandlers = new Set<LineHandler>();
+  private readonly exitHandlers = new Set<ExitHandler>();
+  private inputChain: Promise<void> = Promise.resolve();
+  private closed = false;
+  private finished = false;
+  private exitCode: number | null = null;
+  private readonly debug: string | null;
+
+  constructor(opts: {
+    environment: TaskEnvironmentAgentSessionSurface;
+    request: TaskEnvironmentSessionRequest;
+    debugLabel?: string;
+  }) {
+    this.debug =
+      opts.debugLabel && process.env.UAI_DEBUG_AGENTS ? opts.debugLabel : null;
+    if (this.debug) this.log(`spawn: ${opts.request.argv.join(" ")}`);
+    this.processPromise = opts.environment.spawnSession(opts.request);
+    void this.consume();
+  }
+
+  private log(message: string): void {
+    console.error(`[uai-agent ${this.debug}] ${message}`);
+  }
+
+  private async consume(): Promise<void> {
+    try {
+      const process = await this.processPromise;
+      this.process = process;
+      const stdout = this.consumeStdout(process.stdout);
+      const stderr = this.consumeStderr(process.stderr);
+      const exit = await process.completion;
+      await Promise.allSettled([stdout, stderr]);
+      this.finish(exit.exitCode);
+    } catch (error) {
+      this.stderrBuf = (
+        this.stderrBuf +
+        (error instanceof Error ? error.message : String(error))
+      ).slice(-STDERR_CAP);
+      this.finish(null);
+    }
+  }
+
+  private async consumeStdout(stream: AsyncIterable<Uint8Array>): Promise<void> {
+    for await (const chunk of stream) this.onStdout(Buffer.from(chunk).toString("utf8"));
+  }
+
+  private async consumeStderr(stream: AsyncIterable<Uint8Array>): Promise<void> {
+    for await (const chunk of stream) {
+      const text = Buffer.from(chunk).toString("utf8");
+      this.stderrBuf = (this.stderrBuf + text).slice(-STDERR_CAP);
+      if (this.debug) this.log(`stderr: ${text.trimEnd()}`);
+    }
+  }
+
+  private onStdout(chunk: string): void {
+    this.stdoutBuf += chunk;
+    let newline: number;
+    while ((newline = this.stdoutBuf.indexOf("\n")) >= 0) {
+      const line = this.stdoutBuf.slice(0, newline).trim();
+      this.stdoutBuf = this.stdoutBuf.slice(newline + 1);
+      if (!line) continue;
+      if (this.debug) this.log(`<- ${line.slice(0, 1000)}`);
+      for (const handler of this.lineHandlers) {
+        try {
+          handler(line);
+        } catch {
+          // A broken protocol handler must not wedge stream consumption.
+        }
+      }
+    }
+  }
+
+  private finish(code: number | null): void {
+    if (this.finished) return;
+    this.finished = true;
+    this.closed = true;
+    this.exitCode = code;
+    if (this.debug) this.log(`exit: code ${code}`);
+    for (const handler of this.exitHandlers) handler(code);
+  }
+
+  onLine(handler: LineHandler): void {
+    this.lineHandlers.add(handler);
+  }
+
+  onExit(handler: ExitHandler): void {
+    this.exitHandlers.add(handler);
+    if (this.finished) queueMicrotask(() => handler(this.exitCode));
+  }
+
+  writeLine(value: unknown): void {
+    if (this.closed) return;
+    const json = JSON.stringify(value);
+    if (this.debug) this.log(`-> ${json.slice(0, 1000)}`);
+    this.inputChain = this.inputChain
+      .then(async () => {
+        const process = await this.processPromise;
+        await process.write(Buffer.from(`${json}\n`));
+      })
+      .catch((error: unknown) => {
+        this.stderrBuf = (
+          this.stderrBuf +
+          (error instanceof Error ? error.message : String(error))
+        ).slice(-STDERR_CAP);
+        this.finish(null);
+      });
+  }
+
+  get stderrTail(): string {
+    return this.stderrBuf;
+  }
+
+  get isClosed(): boolean {
+    return this.closed;
+  }
+
+  async close(): Promise<void> {
+    if (this.closed) return;
+    try {
+      const process = this.process ?? (await this.processPromise);
+      await process.closeInput().catch(() => {});
+      await process.terminate("SIGTERM").catch(() => {});
+    } catch {
+      // Startup already failed; consume() owns the terminal notification.
+    }
+  }
+}
+
+/** Minimal ChildProcess-shaped view retained by the one-shot adapters. */
+export interface AgentEnvironmentProcess {
+  readonly stdout: EventEmitter;
+  readonly stderr: EventEmitter;
+  on(event: "exit", listener: (code: number | null) => void): this;
+  on(event: "error", listener: (error: Error) => void): this;
+  kill(signal?: "SIGTERM" | "SIGKILL"): boolean;
+}
+
+/** Bridge a provider stream into the event shape used by one-shot adapters. */
+export function spawnEnvironmentProcess(
+  environment: TaskEnvironmentAgentSessionSurface,
+  request: TaskEnvironmentSessionRequest,
+): AgentEnvironmentProcess {
+  const launchRequest: TaskEnvironmentSessionRequest = {
+    ...request,
+    argv: agentCliArgv(environment, request.argv),
+  };
+  class EnvironmentProcess extends EventEmitter implements AgentEnvironmentProcess {
+    readonly stdout = new EventEmitter();
+    readonly stderr = new EventEmitter();
+    private process: TaskEnvironmentProcess | null = null;
+    private readonly processPromise: Promise<TaskEnvironmentProcess>;
+
+    constructor() {
+      super();
+      this.processPromise = environment.spawnSession(launchRequest);
+      void this.consume();
+    }
+
+    private async consume(): Promise<void> {
+      try {
+        const process = await this.processPromise;
+        this.process = process;
+        const stdout = pumpEnvironmentStream(process.stdout, this.stdout);
+        const stderr = pumpEnvironmentStream(process.stderr, this.stderr);
+        const exit = await process.completion;
+        await Promise.allSettled([stdout, stderr]);
+        this.emit("exit", exit.exitCode);
+      } catch (error) {
+        this.emit(
+          "error",
+          error instanceof Error ? error : new Error(String(error)),
+        );
+      }
+    }
+
+    kill(signal: "SIGTERM" | "SIGKILL" = "SIGTERM"): boolean {
+      const process = this.process;
+      if (process) void process.terminate(signal).catch(() => {});
+      else void this.processPromise.then((pending) => pending.terminate(signal)).catch(() => {});
+      return true;
+    }
+  }
+
+  return new EnvironmentProcess();
+}
+
+async function pumpEnvironmentStream(
+  stream: AsyncIterable<Uint8Array>,
+  target: EventEmitter,
+): Promise<void> {
+  for await (const chunk of stream) target.emit("data", Buffer.from(chunk));
+}

package/lib/agents/rate-limit.ts

@@ -0,0 +1,337 @@
+/**
+ * Rate-limit / quota classification for agent errors (ADR-076).
+ *
+ * When an engine reports an error that means "this account is temporarily out
+ * of capacity" (usage cap, 429, quota, overloaded), the orchestrator cools the
+ * current account and fails over to the next one. Each adapter's pure mapper
+ * runs the error text through {@link isRateLimitMessage}, emits a secret-free
+ * account usage observation, and tags the resulting `error` event with
+ * `retryable: "rate_limit"` so rotation can act on it without re-parsing
+ * engine-specific strings downstream.
+ *
+ * Deliberately broad but anchored on unambiguous tokens — a false positive
+ * only rotates to another account (cheap, self-correcting), a false negative
+ * just surfaces the error as today.
+ */
+import type { AgentAccountUsage } from "./types";
+
+const RATE_LIMIT_PATTERNS: RegExp[] = [
+  /rate[\s_-]?limit/i,
+  /\b429\b/,
+  /too many requests/i,
+  /quota/i,
+  /usage limit/i,
+  /usage cap/i,
+  // Claude subscription caps: "You've hit your weekly limit · resets …",
+  // "reached your 5-hour limit", "exceeded your limit". Anchored on
+  // hit/reached/exceeded so an "approaching your … limit" WARNING won't match
+  // (a warning isn't exhaustion — rotating on it would waste the account).
+  /(?:hit|reached|exceeded)\s+your\s+.{0,40}?\blimit\b/i,
+  /\blimit reached\b/i,
+  // A reset hint sitting next to "limit" ("limit · resets Jul 27") is a cap.
+  /\blimit\b[^.\n]{0,40}\bresets?\b/i,
+  /overloaded/i,
+  /capacity/i,
+  /resource[\s_-]?exhausted/i,
+  /insufficient_quota/i,
+  /billing (?:hard )?limit/i,
+];
+
+/** Whether an engine error message looks like a rate-limit / quota exhaustion. */
+export function isRateLimitMessage(message: string | undefined | null): boolean {
+  if (!message) return false;
+  return RATE_LIMIT_PATTERNS.some((re) => re.test(message));
+}
+
+/** Build the account observation shared by every adapter's error-text path. */
+export function rateLimitErrorUsage(
+  message: string,
+  now: number = Date.now(),
+  timeZone?: string,
+): AgentAccountUsage | null {
+  if (!isRateLimitMessage(message)) return null;
+  const resetsAt = parseResetHint(message, now, timeZone);
+  return {
+    status: "limited",
+    ...(resetsAt === null ? {} : { resetsAt }),
+    windows: [],
+    source: "error-text",
+    observedAt: now,
+  };
+}
+
+const MONTHS = new Map(
+  ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"].map(
+    (month, index) => [month, index + 1] as const,
+  ),
+);
+const MAX_RESET_SECONDS = 100_000_000_000;
+const MAX_RESET_AHEAD_MS = 365 * 24 * 60 * 60_000;
+
+/** Convert a provider Unix-seconds reset without accepting milliseconds or a
+ * corrupt timestamp that is more than a year beyond this observation. */
+export function resetSecondsToEpochMs(
+  value: unknown,
+  observedAt: number,
+): number | undefined {
+  if (
+    typeof value !== "number" ||
+    !Number.isSafeInteger(value) ||
+    value <= 0 ||
+    value > MAX_RESET_SECONDS ||
+    !Number.isFinite(observedAt)
+  ) {
+    return undefined;
+  }
+  const candidate = value * 1_000;
+  return candidate <= observedAt + MAX_RESET_AHEAD_MS
+    ? candidate
+    : undefined;
+}
+
+interface CivilTime {
+  year: number;
+  month: number;
+  day: number;
+  hour: number;
+  minute: number;
+  second: number;
+}
+
+function formatter(timeZone: string): Intl.DateTimeFormat | null {
+  try {
+    return new Intl.DateTimeFormat("en-US-u-ca-gregory-nu-latn", {
+      timeZone,
+      year: "numeric",
+      month: "2-digit",
+      day: "2-digit",
+      hour: "2-digit",
+      minute: "2-digit",
+      second: "2-digit",
+      hourCycle: "h23",
+    });
+  } catch {
+    return null;
+  }
+}
+
+function civilAt(epochMs: number, fmt: Intl.DateTimeFormat): CivilTime | null {
+  const values = new Map(
+    fmt
+      .formatToParts(new Date(epochMs))
+      .filter((part) => part.type !== "literal")
+      .map((part) => [part.type, Number(part.value)]),
+  );
+  const civil = {
+    year: values.get("year") ?? NaN,
+    month: values.get("month") ?? NaN,
+    day: values.get("day") ?? NaN,
+    hour: values.get("hour") ?? NaN,
+    minute: values.get("minute") ?? NaN,
+    second: values.get("second") ?? NaN,
+  };
+  return Object.values(civil).every(Number.isFinite) ? civil : null;
+}
+
+function sameCivil(left: CivilTime | null, right: CivilTime): boolean {
+  return (
+    left !== null &&
+    left.year === right.year &&
+    left.month === right.month &&
+    left.day === right.day &&
+    left.hour === right.hour &&
+    left.minute === right.minute &&
+    left.second === right.second
+  );
+}
+
+/** Convert a civil time in an IANA zone to an instant. Sampling the offsets
+ * on both sides of the date makes DST folds visible: zero or multiple matching
+ * instants are refused instead of guessing. */
+function epochForCivil(civil: CivilTime, timeZone: string): number | null {
+  const fmt = formatter(timeZone);
+  if (!fmt) return null;
+  const target = Date.UTC(
+    civil.year,
+    civil.month - 1,
+    civil.day,
+    civil.hour,
+    civil.minute,
+    civil.second,
+  );
+  const normalized = new Date(target);
+  if (
+    normalized.getUTCFullYear() !== civil.year ||
+    normalized.getUTCMonth() + 1 !== civil.month ||
+    normalized.getUTCDate() !== civil.day ||
+    normalized.getUTCHours() !== civil.hour ||
+    normalized.getUTCMinutes() !== civil.minute
+  ) {
+    return null;
+  }
+
+  const offsets = new Set<number>();
+  for (let hours = -36; hours <= 36; hours += 6) {
+    const probe = target + hours * 60 * 60_000;
+    const parts = civilAt(probe, fmt);
+    if (!parts) continue;
+    offsets.add(
+      Date.UTC(
+        parts.year,
+        parts.month - 1,
+        parts.day,
+        parts.hour,
+        parts.minute,
+        parts.second,
+      ) - probe,
+    );
+  }
+  const candidates = [...offsets]
+    .map((offset) => target - offset)
+    .filter((candidate) => sameCivil(civilAt(candidate, fmt), civil));
+  const unique = [...new Set(candidates)];
+  return unique.length === 1 ? unique[0]! : null;
+}
+
+function localDate(now: number, timeZone: string): CivilTime | null {
+  const fmt = formatter(timeZone);
+  return fmt ? civilAt(now, fmt) : null;
+}
+
+function parsedZone(raw: string | undefined, fallback: string): string | null {
+  if (!raw) return fallback;
+  const zone = raw.trim();
+  if (/^(?:UTC|GMT)$/i.test(zone)) return "UTC";
+  // Abbreviations such as CST name several zones and are therefore unsafe.
+  if (!zone.includes("/")) return null;
+  return formatter(zone) ? zone : null;
+}
+
+function clockHour(rawHour: string, meridiem: string): number | null {
+  const hour = Number(rawHour);
+  if (!Number.isInteger(hour) || hour < 1 || hour > 12) return null;
+  return (hour % 12) + (meridiem.toLowerCase() === "pm" ? 12 : 0);
+}
+
+/**
+ * Parse the deliberately small set of reset hints emitted by engine CLIs.
+ * Returns epoch milliseconds, or null when the text is invalid, names an
+ * ambiguous civil instant, contains multiple hints, or needs assumptions the
+ * message did not supply.
+ */
+export function parseResetHint(
+  message: string,
+  now: number,
+  timeZone: string = Intl.DateTimeFormat().resolvedOptions().timeZone,
+): number | null {
+  if (!message || !Number.isFinite(now) || !formatter(timeZone)) return null;
+  const found: number[] = [];
+
+  // Claude also emits a machine-readable suffix in some terminal errors.
+  // It is Unix seconds; reject millisecond-scale or implausibly distant input
+  // rather than turning malformed text into a long account bench.
+  const claudeEpoch = /\bClaude AI usage limit reached\|(\d+)\b/gi;
+  for (const match of message.matchAll(claudeEpoch)) {
+    const seconds = Number(match[1]);
+    const candidate = resetSecondsToEpochMs(seconds, now);
+    if (candidate !== undefined && candidate > now) found.push(candidate);
+  }
+
+  const relative = /\b(?:try\s+again|resets?)\s+in\s+((?:\d+\s*(?:days?|d|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)\s*){1,4})(?=$|[.,;!?])/gi;
+  for (const match of message.matchAll(relative)) {
+    const units = new Set<string>();
+    let duration = 0;
+    const token = /(\d+)\s*(days?|d|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)/gi;
+    let consumed = "";
+    for (const part of match[1]!.matchAll(token)) {
+      const value = Number(part[1]);
+      const rawUnit = part[2]!.toLowerCase();
+      const unit = rawUnit.startsWith("d")
+        ? "d"
+        : rawUnit.startsWith("h")
+          ? "h"
+          : rawUnit.startsWith("m")
+            ? "m"
+            : "s";
+      if (!Number.isSafeInteger(value) || value <= 0 || units.has(unit)) {
+        duration = NaN;
+        break;
+      }
+      units.add(unit);
+      consumed += part[0].replace(/\s+/g, "");
+      duration +=
+        value *
+        (unit === "d"
+          ? 24 * 60 * 60_000
+          : unit === "h"
+            ? 60 * 60_000
+            : unit === "m"
+              ? 60_000
+              : 1_000);
+    }
+    if (
+      Number.isSafeInteger(duration) &&
+      duration > 0 &&
+      consumed.toLowerCase() === match[1]!.replace(/\s+/g, "").toLowerCase()
+    ) {
+      found.push(now + duration);
+    }
+  }
+
+  const monthNames = "Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?";
+  const dated = new RegExp(
+    `\\bresets?\\s+(${monthNames})\\s+(\\d{1,2})(?:,?\\s+(\\d{4}))?(?:,?\\s+(\\d{1,2})(?::(\\d{2}))?\\s*(am|pm))?(?:\\s*\\(([^)]+)\\))?(?=$|[.,;!?])`,
+    "gi",
+  );
+  for (const match of message.matchAll(dated)) {
+    const month = MONTHS.get(match[1]!.slice(0, 3).toLowerCase());
+    const day = Number(match[2]);
+    const explicitYear = match[3] === undefined ? null : Number(match[3]);
+    const hour = match[4] === undefined ? 0 : clockHour(match[4], match[6]!);
+    const minute = match[5] === undefined ? 0 : Number(match[5]);
+    // The CLI runs in a container whose local zone need not match this host
+    // process. A calendar date is zone-sensitive even when it names no clock,
+    // so every absolute hint must name its zone.
+    const zone =
+      match[7] === undefined ? null : parsedZone(match[7], timeZone);
+    const base = zone ? localDate(now, zone) : null;
+    if (!base || !month || !zone || hour === null || minute < 0 || minute > 59) continue;
+    let year = explicitYear ?? base.year;
+    let candidate = epochForCivil({ year, month, day, hour, minute, second: 0 }, zone);
+    if (explicitYear === null && candidate !== null && candidate <= now) {
+      year += 1;
+      candidate = epochForCivil({ year, month, day, hour, minute, second: 0 }, zone);
+    }
+    if (candidate !== null && candidate > now) found.push(candidate);
+  }
+
+  const clock = /\bresets?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\(([^)]+)\))?(?=$|[.,;!?])/gi;
+  for (const match of message.matchAll(clock)) {
+    const hour = clockHour(match[1]!, match[3]!);
+    const minute = match[2] === undefined ? 0 : Number(match[2]);
+    const zone = match[4] === undefined ? null : parsedZone(match[4], timeZone);
+    const base = zone ? localDate(now, zone) : null;
+    if (!base || hour === null || minute < 0 || minute > 59 || !zone) continue;
+    let candidate = epochForCivil(
+      { year: base.year, month: base.month, day: base.day, hour, minute, second: 0 },
+      zone,
+    );
+    if (candidate !== null && candidate <= now) {
+      const tomorrow = new Date(Date.UTC(base.year, base.month - 1, base.day + 1));
+      candidate = epochForCivil(
+        {
+          year: tomorrow.getUTCFullYear(),
+          month: tomorrow.getUTCMonth() + 1,
+          day: tomorrow.getUTCDate(),
+          hour,
+          minute,
+          second: 0,
+        },
+        zone,
+      );
+    }
+    if (candidate !== null && candidate > now) found.push(candidate);
+  }
+
+  return found.length === 1 ? found[0]! : null;
+}