@saadnvd1/agent-os 0.3.738 → 0.3.742

package/lib/chat/drivers/transient-retry.ts

@@ -0,0 +1,87 @@
+// Retrying a turn the model's server dropped. OpenCode and Pi retry some
+// failures themselves (and say so); this covers the rest, with a hard limit
+// so nothing loops: a few tries with growing waits, the last one on a
+// smaller context, then the turn ends with a report.
+
+export type ErrorKind = "timeout" | "transient";
+
+export const MAX_RETRIES = 3;
+const BASE_MS = 2_000;
+// A context this full (percent of the window) is compacted before the last
+// try: a request that took too long to start is the one that shrinks.
+export const COMPACT_AT_PERCENT = 50;
+
+const NEVER =
+  /context (length|window)|maximum context|too many tokens|prompt is too long|\b40[0-4]\b|unauthori[sz]ed|invalid api key|authentication|forbidden|abort/i;
+const TIMEOUT =
+  /headers timed out|timed out|timeout|UND_ERR_(HEADERS|BODY|CONNECT)_TIMEOUT|ETIMEDOUT|ResponseStreamError/i;
+const TRANSIENT =
+  /fetch failed|socket hang up|ECONNRESET|ECONNREFUSED|EPIPE|terminated|connection (lost|closed|reset)|ended without|overloaded|\b(429|500|502|503|504|524)\b|unavailable|rate limit/i;
+
+// Whether a provider error is worth another try, and which kind it is.
+export function classifyProviderError(message: string): ErrorKind | undefined {
+  if (!message || NEVER.test(message)) return undefined;
+  if (TIMEOUT.test(message)) return "timeout";
+  if (TRANSIENT.test(message)) return "transient";
+  return undefined;
+}
+
+export function backoffMs(attempt: number, jitter = true): number {
+  const base = BASE_MS * 2 ** (attempt - 1);
+  return jitter ? Math.round(base * (0.8 + Math.random() * 0.4)) : base;
+}
+
+export type RetryDecision =
+  | { retry: true; attempt: number; delayMs: number; compactFirst: boolean }
+  | { retry: false; attempts: number; kind?: ErrorKind };
+
+// One per conversation. A turn that sends the user's message starts it over;
+// a retry never does.
+export class TurnRetry {
+  private attempts = 0;
+
+  constructor(private jitter = true) {}
+
+  reset(): void {
+    this.attempts = 0;
+  }
+
+  // Retries made in this turn.
+  get tried(): number {
+    return this.attempts;
+  }
+
+  next(o: {
+    error: string;
+    // The agent already retried this failure itself.
+    harnessRetried?: boolean;
+    // The context is big enough that compacting it is worth a try.
+    contextHeavy?: boolean;
+  }): RetryDecision {
+    const kind = classifyProviderError(o.error);
+    if (!kind || o.harnessRetried || this.attempts >= MAX_RETRIES)
+      return { retry: false, attempts: this.attempts, kind };
+    this.attempts++;
+    return {
+      retry: true,
+      attempt: this.attempts,
+      delayMs: backoffMs(this.attempts, this.jitter),
+      compactFirst: this.attempts === MAX_RETRIES && !!o.contextHeavy,
+    };
+  }
+}
+
+// The message that resumes a turn after a failure: the steps that ran are
+// already in the conversation.
+export function continuePrompt(error: string): string {
+  const why = error.replace(/\s+/g, " ").trim().slice(0, 200);
+  return `The previous request to the model failed (${why}). Continue the task from where it stopped. The tool steps above already ran: don't redo them.`;
+}
+
+export function retryNotice(
+  d: Extract<RetryDecision, { retry: true }>,
+  error: string
+): string {
+  const why = error.replace(/\s+/g, " ").trim().slice(0, 160);
+  return `The model's server didn't answer (${why}). Trying again in ${Math.round(d.delayMs / 1000)}s (${d.attempt} of ${MAX_RETRIES})${d.compactFirst ? ", after compacting the conversation" : ""}.`;
+}

package/lib/chat/drivers/turn-items.ts

@@ -6,12 +6,14 @@ import type {
   ToolStatus,
   TurnEndReason,
 } from "../events";
-import { clipOutput } from "../tools";
+import { clipOutput, tailOutput } from "../tools";
 
 type Tool = Extract<ChatItem, { kind: "tool" }>;
 type Text = Extract<ChatItem, { kind: "assistant" | "reasoning" }>;
 type Todo = Extract<ChatItem, { kind: "todos" }>["todos"][number];
 
+const PROGRESS_MS = 250;
+
 export interface ToolStart {
   name: string;
   title: string;
@@ -19,6 +21,18 @@ export interface ToolStart {
   diff?: FileDiff;
 }
 
+export interface TurnSummary {
+  steps: number;
+  failed: number;
+  files: { path: string; ok: number; failed: number }[];
+}
+
+const newStats = () => ({
+  steps: 0,
+  failed: 0,
+  files: new Map<string, { ok: number; failed: number }>(),
+});
+
 // An API's error as its message: providers wrap JSON in JSON.
 export function readableError(text: string): string {
   let message = text.trim();
@@ -50,6 +64,9 @@ export class TurnItems {
   // Texts that finished in this turn, so one can be corrected afterwards.
   private done = new Map<string, Text>();
   private startedAt = 0;
+  // What the turn has done so far, for the report when it stops short.
+  private stats = newStats();
+  private progressAt = new Map<string, number>();
   inTurn = false;
 
   constructor(private emit: (e: DriverEvent) => void) {}
@@ -58,6 +75,7 @@ export class TurnItems {
     if (this.inTurn) return;
     this.inTurn = true;
     this.startedAt = Date.now();
+    this.stats = newStats();
     this.emit({ type: "turn_start" });
   }
 
@@ -121,6 +139,20 @@ export class TurnItems {
     this.emit({ type: "item", item });
   }
 
+  // A running step's output so far (the whole of it each time, not a
+  // delta). Sent at most every PROGRESS_MS; the end replaces it.
+  progress(id: string, output: string): void {
+    const tool = this.tools.get(id);
+    if (!tool || !output) return;
+    const tail = tailOutput(output);
+    if (tail === tool.output) return;
+    const now = Date.now();
+    if (now - (this.progressAt.get(id) ?? 0) < PROGRESS_MS) return;
+    this.progressAt.set(id, now);
+    tool.output = tail;
+    this.emit({ type: "item", item: { ...tool } });
+  }
+
   hasText(key: string): boolean {
     return this.texts.has(key);
   }
@@ -138,6 +170,8 @@ export class TurnItems {
     const tool = this.tools.get(id);
     if (!tool) return;
     this.tools.delete(id);
+    this.count(tool, update, status);
+    this.progressAt.delete(id);
     this.emit({
       type: "item",
       item: {
@@ -153,6 +187,31 @@ export class TurnItems {
     });
   }
 
+  private count(
+    tool: Tool,
+    update: Partial<ToolStart> | undefined,
+    status: ToolStatus
+  ): void {
+    if (status === "stopped") return;
+    this.stats.steps++;
+    if (status === "error") this.stats.failed++;
+    const input = (update?.input ?? tool.input) as { file_path?: unknown };
+    if (!/^(Edit|Write|MultiEdit)$/.test(tool.name)) return;
+    if (typeof input?.file_path !== "string") return;
+    const file = this.stats.files.get(input.file_path) ?? { ok: 0, failed: 0 };
+    file[status === "error" ? "failed" : "ok"]++;
+    this.stats.files.set(input.file_path, file);
+  }
+
+  // The turn so far: tool steps run, how many failed, files edited.
+  summary(): TurnSummary {
+    return {
+      steps: this.stats.steps,
+      failed: this.stats.failed,
+      files: [...this.stats.files].map(([path, n]) => ({ path, ...n })),
+    };
+  }
+
   todos(todos: Todo[]): void {
     this.emit({
       type: "item",
@@ -160,6 +219,20 @@ export class TurnItems {
     });
   }
 
+  // A line in the conversation that isn't the agent's: a retry under way.
+  note(text: string): void {
+    this.emit({
+      type: "item",
+      item: {
+        id: randomUUID(),
+        kind: "note",
+        text,
+        tone: "note",
+        createdAt: Date.now(),
+      },
+    });
+  }
+
   error(message: string): void {
     this.emit({
       type: "item",

package/lib/chat/drivers/turn-outcome.ts

@@ -1,5 +1,7 @@
 import { redact } from "../../orchestrator/untrusted";
 import type { TurnEndReason } from "../events";
+import type { TurnSummary } from "./turn-items";
+import { classifyProviderError } from "./transient-retry";
 
 export interface TurnEnded {
   reason: TurnEndReason;
@@ -43,11 +45,49 @@ export function hasToolMarkup(text: string): boolean {
   );
 }
 
+const plural = (n: number, one: string, many = `${one}s`) =>
+  `${n} ${n === 1 ? one : many}`;
+
+// Where a turn stopped when the model's server kept failing: what the model
+// got done, what it didn't, and what to do next. The error text is scrubbed.
+export function failureReport(o: {
+  error: string;
+  retries: number;
+  // The agent retried it before AgentOS saw the error.
+  agentRetried?: boolean;
+  summary?: TurnSummary;
+}): string {
+  const kind = classifyProviderError(o.error);
+  const why = redact(o.error.replace(/\s+/g, " ").trim()).slice(0, 240);
+  const tries = o.retries
+    ? ` and ${plural(o.retries, "retry", "retries")} didn't get through`
+    : o.agentRetried
+      ? " even after the agent retried it"
+      : "";
+  const s = o.summary;
+  const done = s?.steps
+    ? `Finished before it stopped: ${plural(s.steps, "tool step")}${s.failed ? ` (${s.failed} failed)` : ""}${
+        s.files.length
+          ? `; edits to ${s.files
+              .map(
+                (f) =>
+                  `${f.path}${f.failed ? ` (${f.failed} failed${f.ok ? `, ${f.ok} applied` : ""})` : ""}`
+              )
+              .join(", ")}`
+          : ""
+      }.`
+    : "Nothing had finished yet.";
+  return `The model's server ${kind === "timeout" ? "timed out" : "kept failing"} (${why})${tries}. ${done} Not finished: the model's next reply, so the task is incomplete. Send "continue" to resume; the finished steps are kept.`;
+}
+
 export function turnOutcome(r: {
   finishReason?: string;
   text: string;
   toolCalls: number;
   error?: string;
+  retries?: number;
+  agentRetried?: boolean;
+  summary?: TurnSummary;
 }): TurnEnded | undefined {
   if (r.finishReason === "length")
     return {
@@ -58,7 +98,14 @@ export function turnOutcome(r: {
   if (r.error)
     return {
       reason: "error",
-      notice: "The model's server returned an error, shown above.",
+      notice: classifyProviderError(r.error)
+        ? failureReport({
+            error: r.error,
+            retries: r.retries ?? 0,
+            agentRetried: r.agentRetried,
+            summary: r.summary,
+          })
+        : "The model's server returned an error, shown above.",
     };
   if (r.toolCalls > 0) return undefined;
   if (hasToolMarkup(r.text))

package/lib/chat/events.ts

@@ -322,6 +322,14 @@ export interface Compacting {
   before?: number;
 }
 
+// A model call the turn is waiting on: since when, the last measured prompt
+// size (a floor: the next prompt is at least that big), and a retry under way.
+export interface ModelWait {
+  since: number;
+  promptTokens?: number;
+  retry?: { attempt: number; message?: string };
+}
+
 export interface UndoPreview {
   canUndo: boolean;
   error?: string;
@@ -347,7 +355,11 @@ export type DriverEvent =
   | { type: "at_rest" } // it has run every message it was given
   | { type: "state"; state: ChatState }
   // The agent started (true) or finished (false) compacting its context.
-  | { type: "compacting"; active: boolean };
+  | { type: "compacting"; active: boolean }
+  // One model response's measured prompt (input plus cache), in tokens.
+  | { type: "prompt_tokens"; tokens: number }
+  // The agent is retrying a failed model request.
+  | { type: "model_retry"; attempt: number; message?: string };
 
 // A tool call's output and diff, loaded when it's opened.
 export interface ToolBody {
@@ -368,6 +380,7 @@ export type ChatServerMessage =
       tasks: ChatItem[];
       state: ChatState;
       compacting?: Compacting | null;
+      modelWait?: ModelWait | null;
       queue: QueuedMessage[];
       suggestion: string | null;
     }
@@ -393,6 +406,7 @@ export type ChatServerMessage =
   | { type: "delta"; id: string; text: string }
   | { type: "state"; state: ChatState }
   | { type: "compacting"; compacting: Compacting | null }
+  | { type: "model_wait"; wait: ModelWait | null }
   | {
       type: "capabilities";
       commands: ChatCommand[];

package/lib/chat/model-wait-live.ts

@@ -0,0 +1,19 @@
+import { emit, type Live } from "./registry";
+import { ModelWaitTracker } from "./model-wait";
+import type { WorkerEvent } from "./worker/protocol";
+
+// Keeps the model call the turn waits on current; the events that only feed
+// it (prompt size, a retry) go no further.
+export function trackModelWait(
+  sessionId: string,
+  live: Live,
+  e: WorkerEvent
+): boolean {
+  const tracker = (live.waitTracker ??= new ModelWaitTracker());
+  const changed = tracker.on(e);
+  if (changed) {
+    live.modelWait = tracker.wait;
+    emit(sessionId, { type: "model_wait", wait: tracker.wait });
+  }
+  return e.type === "prompt_tokens" || e.type === "model_retry";
+}

package/lib/chat/model-wait.ts

@@ -0,0 +1,88 @@
+import type { ModelWait } from "./events";
+import type { WorkerEvent } from "./worker/protocol";
+
+// Follows a running turn's events to know when it is waiting on a model
+// call: no tool running, nothing streaming, not compacting. Only what a
+// driver measured is kept as the prompt size; nothing is guessed.
+export class ModelWaitTracker {
+  wait: ModelWait | null = null;
+  private lastPrompt: number | undefined;
+  private running = new Set<string>();
+  private active = false;
+
+  // Returns whether `wait` changed.
+  on(e: WorkerEvent, now = Date.now()): boolean {
+    const before = this.wait;
+    switch (e.type) {
+      case "state":
+        if (e.state === "running") {
+          if (!this.active) {
+            this.active = true;
+            this.begin(now);
+          }
+        } else this.end();
+        break;
+      case "item": {
+        const i = e.item;
+        if (i.kind === "turn_end") this.end();
+        else if (i.kind === "compacted") this.lastPrompt = undefined;
+        else if (i.kind === "tool") {
+          if (i.status === "running") {
+            this.running.add(i.id);
+            this.wait = null;
+          } else {
+            this.running.delete(i.id);
+            if (this.active && !this.running.size) this.begin(now);
+          }
+        } else if (i.kind === "assistant" || i.kind === "reasoning") {
+          if (i.streaming) this.wait = null;
+          else if (this.active && !this.running.size && !this.wait)
+            this.begin(now);
+        }
+        break;
+      }
+      case "delta":
+        this.wait = null;
+        break;
+      case "compacting":
+        if (e.compacting) {
+          this.wait = null;
+          this.lastPrompt = undefined;
+        } else if (this.active && !this.running.size) this.begin(now);
+        break;
+      case "prompt_tokens":
+        this.lastPrompt = e.tokens;
+        break;
+      case "context":
+        this.lastPrompt = e.context.usedTokens || this.lastPrompt;
+        break;
+      case "model_retry":
+        if (this.active)
+          this.wait = {
+            ...(this.wait ?? this.fresh(now)),
+            retry: { attempt: e.attempt, message: e.message },
+          };
+        break;
+    }
+    return (
+      before !== this.wait &&
+      JSON.stringify(before) !== JSON.stringify(this.wait)
+    );
+  }
+
+  private fresh(now: number): ModelWait {
+    return this.lastPrompt
+      ? { since: now, promptTokens: this.lastPrompt }
+      : { since: now };
+  }
+
+  private begin(now: number) {
+    this.wait = this.fresh(now);
+  }
+
+  private end() {
+    this.active = false;
+    this.wait = null;
+    this.running.clear();
+  }
+}

package/lib/chat/progress.ts

@@ -0,0 +1,169 @@
+import type { ChatItem, ChatState, Compacting, ModelWait } from "./events";
+import { formatTokens } from "./context";
+
+type Tool = Extract<ChatItem, { kind: "tool" }>;
+
+export type Phase = "planning" | "coding" | "testing" | "error";
+
+// A wait on one model call this long gets a hint that it may be normal.
+export const LONG_WAIT_MS = 2 * 60_000;
+export const LONG_WAIT_HINT =
+  "Still waiting on the model; large prompts on local models can take minutes";
+
+const TEST_RE =
+  /\b(test|tests|vitest|jest|pytest|mocha|playwright|tsc|typecheck|lint|eslint|build|check)\b|cargo (test|build|check)|go (test|build|vet)/;
+const EDIT_TOOLS = new Set([
+  "Edit",
+  "Write",
+  "MultiEdit",
+  "NotebookEdit",
+  "apply_patch",
+]);
+const VERBS: Record<string, string> = {
+  Read: "Reading",
+  Edit: "Editing",
+  Write: "Writing",
+  MultiEdit: "Editing",
+  NotebookEdit: "Editing",
+  Search: "Searching",
+  Find: "Finding",
+  Fetch: "Fetching",
+  List: "Listing",
+  Run: "Running",
+};
+
+const str = (v: unknown) => (typeof v === "string" ? v : "");
+const input = (t: Tool) => (t.input ?? {}) as Record<string, unknown>;
+const command = (t: Tool) => str(input(t).command);
+
+// "ollama/qwen3-coder:30b" -> "qwen3-coder:30b"; nothing known -> "the model".
+export function shortModel(label: string | undefined): string {
+  let m = (label ?? "").trim().split("/").at(-1) ?? "";
+  m = m.replace(/:latest$/, "").replace(/-\d{8}$/, "");
+  if (!m || m.toLowerCase() === "default") return "the model";
+  return m.length > 24 ? `${m.slice(0, 23)}…` : m;
+}
+
+const tail = (path: string) => path.split("/").slice(-3).join("/");
+const clip = (s: string, n = 60) =>
+  s.length > n ? `${s.slice(0, n - 1)}…` : s;
+
+// A tool call in plain words: "Running npm test", "Editing src/a/b.ts".
+export function stepLabel(t: Tool): string {
+  const cmd = command(t);
+  if (cmd) return `Running ${clip(cmd.replace(/\s+/g, " ").trim())}`;
+  const path = str(input(t).file_path);
+  if (path && EDIT_TOOLS.has(t.name)) {
+    const verb = VERBS[t.name] ?? "Editing";
+    return `${verb} ${tail(path)}`;
+  }
+  const [first, ...rest] = t.title.split(" ");
+  return VERBS[first] && rest.length
+    ? `${VERBS[first]} ${rest.join(" ")}`
+    : t.title;
+}
+
+export function turnTools(items: ChatItem[]): Tool[] {
+  let start = 0;
+  items.forEach((i, n) => {
+    if (i.kind === "user" && !i.midTurn) start = n + 1;
+  });
+  return items.slice(start).filter((i): i is Tool => i.kind === "tool");
+}
+
+const edited = (t: Tool) => EDIT_TOOLS.has(t.name) || !!t.diff;
+
+// Planning, coding, testing or error, from the turn's tools: the current
+// step (running, else the last one) decides error and testing; having
+// edited anything in the turn makes the rest coding.
+export function phaseOf(tools: Tool[]): Phase {
+  const step = tools.findLast((t) => t.status === "running") ?? tools.at(-1);
+  if (!step) return "planning";
+  if (step.status === "error") return "error";
+  if (TEST_RE.test(command(step))) return "testing";
+  return tools.some(edited) ? "coding" : "planning";
+}
+
+export interface Activity {
+  label: string;
+  since?: number;
+  compacting?: boolean;
+  kind: "answer" | "compacting" | "tool" | "writing" | "waiting" | "retry";
+  // The last measured prompt, when a driver reported one.
+  tokens?: number;
+  phase?: Phase;
+  steps: number;
+  // The last finished step, while no tool runs.
+  lastStep?: string;
+  turnSince?: number;
+  hint?: string;
+}
+
+// What the turn is doing now and for how long, from its items.
+export function currentActivity(
+  items: ChatItem[],
+  state: ChatState,
+  compacting: Compacting | null = null,
+  modelWait: ModelWait | null = null,
+  model?: string,
+  now: number = Date.now()
+): Activity | null {
+  if (state === "waiting")
+    return { label: "Waiting for your answer", kind: "answer", steps: 0 };
+  if (state !== "running") return null;
+  if (compacting)
+    return {
+      label: `Compacting context…${compacting.before ? ` ${formatTokens(compacting.before)} tokens` : ""}`,
+      since: compacting.since,
+      compacting: true,
+      kind: "compacting",
+      steps: 0,
+    };
+  const lastUser = items.findLast((i) => i.kind === "user" && !i.midTurn);
+  const tools = turnTools(items);
+  const base = {
+    steps: tools.length,
+    turnSince: lastUser?.createdAt,
+    phase: tools.length ? phaseOf(tools) : undefined,
+  };
+  const tool = tools.findLast((t) => t.status === "running");
+  if (tool)
+    return {
+      ...base,
+      label: stepLabel(tool),
+      since: tool.createdAt,
+      kind: "tool",
+    };
+  const last = items.at(-1);
+  if (
+    last &&
+    (last.kind === "assistant" || last.kind === "reasoning") &&
+    last.streaming
+  )
+    return {
+      ...base,
+      label: last.kind === "assistant" ? "Writing" : "Thinking",
+      since: lastUser?.createdAt,
+      kind: "writing",
+    };
+  const done = tools.findLast((t) => t.status !== "running");
+  const since =
+    modelWait?.since ?? done?.endedAt ?? done?.createdAt ?? lastUser?.createdAt;
+  const retry = modelWait?.retry;
+  return {
+    ...base,
+    label: retry
+      ? `Retrying model request (attempt ${retry.attempt})`
+      : `Waiting for ${shortModel(model)}`,
+    since,
+    kind: retry ? "retry" : "waiting",
+    tokens: modelWait?.promptTokens,
+    lastStep: done
+      ? `${stepLabel(done)}${done.status === "error" ? " failed" : ""}`
+      : undefined,
+    hint:
+      since !== undefined && now - since > LONG_WAIT_MS
+        ? LONG_WAIT_HINT
+        : undefined,
+  };
+}

package/lib/chat/registry.ts

@@ -10,9 +10,11 @@ import type {
   ChatServerMessage,
   ChatState,
   Compacting,
+  ModelWait,
 } from "./events";
 import { markStatusDirty } from "../status/dirty";
 import { FLUSH_MS } from "./store";
+import type { ModelWaitTracker } from "./model-wait";
 
 // Long enough for the worker's buffered write of an item to land.
 const EMIT_SETTLE_MS = 2 * FLUSH_MS;
@@ -23,6 +25,9 @@ export interface Live {
   state: ChatState;
   // Set while its agent compacts the context, over a "running" state.
   compacting?: Compacting | null;
+  // The model call the turn is waiting on, if any (lib/chat/model-wait).
+  modelWait?: ModelWait | null;
+  waitTracker?: ModelWaitTracker;
   // Streamed items carry their latest text here, ahead of SQLite.
   streaming: Map<string, ChatItem>;
   // What the turn is doing right now, for the session list.