openhermes 4.9.2 → 4.12.1

package/harness/lib/recovery/handler.ts

@@ -0,0 +1,243 @@
+// RecoveryHandler — singleton that classifies errors against patterns,
+// applies recovery actions, and tracks stats.
+
+import type {
+  ErrorCategory,
+  ErrorContext,
+  RecoveryAction,
+  RecoveryActionType,
+  RecoveryRecord,
+  RecoveryStats,
+} from "./interfaces.ts";
+import { PATTERNS, escalateAction } from "./patterns.ts";
+
+/**
+ * Represents the number of times we have *attempted recovery* for a
+ * given (sessionId, category) pair. Separate from the sub-agent's own
+ * attempt counter which is passed in ErrorContext.attempt.
+ */
+interface CategoryAttemptState {
+  count: number;
+}
+
+export class RecoveryHandler {
+  private static instance: RecoveryHandler;
+
+  /** All recovery records keyed by sessionId then timestamp. */
+  private history: RecoveryRecord[] = [];
+
+  /** Track attempts per (sessionId + category) to enforce maxAttempts. */
+  private attemptTracker = new Map<string, CategoryAttemptState>();
+
+  /** Total number of recoveries that succeeded (fn completed without throwing). */
+  private successCount = 0;
+  private failureCount = 0;
+
+  private constructor() {}
+
+  /** Get the singleton instance. */
+  static getInstance(): RecoveryHandler {
+    if (!RecoveryHandler.instance) {
+      RecoveryHandler.instance = new RecoveryHandler();
+    }
+    return RecoveryHandler.instance;
+  }
+
+  /**
+   * Classify an error and return the appropriate recovery action.
+   * Returns the first matching pattern's action, or escalates if no match.
+   */
+  handleError(context: ErrorContext): RecoveryAction {
+    const message = context.error.message ?? String(context.error);
+
+    for (const entry of PATTERNS) {
+      if (entry.pattern.test(message)) {
+        const action = entry.getAction(context);
+        this.record(context, action);
+        return action;
+      }
+    }
+
+    // No pattern matched — escalate
+    const action = escalateAction(context);
+    this.record(context, action);
+    return action;
+  }
+
+  /**
+   * Wraps an async function with auto-recovery.
+   *
+   * On each throw:
+   *  1. Classify the error via handleError()
+   *  2. If action is "abort" | "escalate" | "skip" — rethrow immediately
+   *  3. If action is "compact" | "retry" — check maxAttempts, delay, retry
+   *
+   * If the function succeeds, increments successCount.
+   */
+  async withRecovery<T>(
+    sessionId: string,
+    fn: () => Promise<T>,
+    options?: { maxAttempts?: number },
+  ): Promise<T> {
+    const globalMax = options?.maxAttempts ?? 5;
+    let attempt = 0;
+
+    while (attempt < globalMax) {
+      try {
+        const result = await fn();
+        this.successCount++;
+        return result;
+      } catch (err: unknown) {
+        const error = err instanceof Error ? err : new Error(String(err));
+        const context: ErrorContext = {
+          sessionId,
+          error,
+          attempt,
+          timestamp: Date.now(),
+        };
+
+        const action = this.handleError(context);
+
+        // Non-recoverable actions — rethrow immediately
+        if (action.type === "abort" || action.type === "escalate" || action.type === "skip") {
+          this.failureCount++;
+          throw error;
+        }
+
+        // Enforce category-specific maxAttempts
+        if (action.maxAttempts !== undefined) {
+          const category = this.findCategory(action.reason);
+          const key = `${sessionId}::${category}`;
+          const tracker = this.attemptTracker.get(key) ?? { count: 0 };
+          tracker.count++;
+          this.attemptTracker.set(key, tracker);
+
+          if (tracker.count >= action.maxAttempts) {
+            this.failureCount++;
+            throw error;
+          }
+        }
+
+        // Apply delay if specified
+        if (action.delay && action.delay > 0) {
+          await this.sleep(action.delay);
+        }
+
+        attempt++;
+      }
+    }
+
+    this.failureCount++;
+    // Exhausted global maxAttempts
+    throw new Error(
+      `[RecoveryHandler] Exhausted ${globalMax} attempts for session "${sessionId}"`,
+    );
+  }
+
+  /** Convenience: sleep for ms. */
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Derive a category from an action's reason string.
+   * Falls back to "timeout" if no pattern matches.
+   */
+  private findCategory(reason: string): ErrorCategory {
+    for (const entry of PATTERNS) {
+      if (entry.pattern.test(reason)) {
+        return entry.category;
+      }
+    }
+    // Attempt to extract from the reason string heuristically
+    const lower = reason.toLowerCase();
+    if (lower.includes("rate")) return "rate_limit";
+    if (lower.includes("context") || lower.includes("token")) return "context_overflow";
+    if (lower.includes("network") || lower.includes("econnrefused")) return "network";
+    if (lower.includes("session")) return "session";
+    if (lower.includes("tool")) return "tool_error";
+    if (lower.includes("parse") || lower.includes("json")) return "parse_error";
+    if (lower.includes("gibberish")) return "gibberish";
+    if (lower.includes("lsp") || lower.includes("tsc") || lower.includes("eslint")) {
+      return "lsp_diagnostic";
+    }
+    if (lower.includes("timeout") || lower.includes("timed")) return "timeout";
+    return "timeout";
+  }
+
+  /** Accumulated statistics. */
+  getStats(): RecoveryStats {
+    const byCategory: Record<ErrorCategory, number> = {
+      rate_limit: 0,
+      context_overflow: 0,
+      network: 0,
+      session: 0,
+      tool_error: 0,
+      parse_error: 0,
+      gibberish: 0,
+      lsp_diagnostic: 0,
+      timeout: 0,
+    };
+
+    const byAction: Record<RecoveryActionType, number> = {
+      retry: 0,
+      abort: 0,
+      skip: 0,
+      escalate: 0,
+      compact: 0,
+    };
+
+    for (const record of this.history) {
+      const cat = this.findCategory(record.action.reason);
+      if (byCategory[cat] !== undefined) byCategory[cat]++;
+      if (byAction[record.action.type] !== undefined) byAction[record.action.type]++;
+    }
+
+    const totalRecoveries = this.history.length;
+    const totalAttempts = this.successCount + this.failureCount;
+    const successRate = totalAttempts > 0 ? this.successCount / totalAttempts : 0;
+
+    return { totalRecoveries, byCategory, byAction, successRate };
+  }
+
+  /** Recent recovery records, most recent first. */
+  getHistory(limit?: number): RecoveryRecord[] {
+    const sorted = [...this.history].sort((a, b) => b.timestamp - a.timestamp);
+    return limit ? sorted.slice(0, limit) : sorted;
+  }
+
+  /** Clear all records and attempt state for a given session. */
+  clearSession(sessionId: string): void {
+    this.history = this.history.filter((r) => r.context.sessionId !== sessionId);
+    for (const key of this.attemptTracker.keys()) {
+      if (key.startsWith(`${sessionId}::`)) {
+        this.attemptTracker.delete(key);
+      }
+    }
+  }
+
+  /** Reset all state (useful in tests). */
+  reset(): void {
+    this.history = [];
+    this.attemptTracker.clear();
+    this.successCount = 0;
+    this.failureCount = 0;
+  }
+
+  // ── private helpers ──
+
+  private record(context: ErrorContext, action: RecoveryAction): void {
+    // Update attempt tracker using the derived category from the action's reason
+    const category = this.findCategory(action.reason);
+    const key = `${context.sessionId}::${category}`;
+    if (!this.attemptTracker.has(key)) {
+      this.attemptTracker.set(key, { count: 0 });
+    }
+
+    this.history.push({
+      context,
+      action,
+      timestamp: Date.now(),
+    });
+  }
+}

package/harness/lib/recovery/index.ts

@@ -0,0 +1,14 @@
+// Auto-Recovery module — barrel export.
+
+export type {
+  ErrorCategory,
+  RecoveryActionType,
+  RecoveryAction,
+  ErrorContext,
+  RecoveryRecord,
+  RecoveryStats,
+} from "./interfaces.ts";
+
+export { RecoveryHandler } from "./handler.ts";
+export { PATTERNS, escalateAction } from "./patterns.ts";
+export type { ErrorPattern } from "./patterns.ts";

package/harness/lib/recovery/interfaces.ts

@@ -0,0 +1,48 @@
+// Auto-Recovery type definitions for sub-agent error handling.
+
+export type ErrorCategory =
+  | "rate_limit"
+  | "context_overflow"
+  | "network"
+  | "session"
+  | "tool_error"
+  | "parse_error"
+  | "gibberish"
+  | "lsp_diagnostic"
+  | "timeout";
+
+export type RecoveryActionType =
+  | "retry"
+  | "abort"
+  | "skip"
+  | "escalate"
+  | "compact";
+
+export interface RecoveryAction {
+  type: RecoveryActionType;
+  delay?: number; // ms delay before retry
+  maxAttempts?: number; // max retry attempts
+  reason: string;
+  modifyPrompt?: string; // instruction to prepend to retry prompt
+}
+
+export interface ErrorContext {
+  sessionId: string;
+  error: Error;
+  attempt: number;
+  timestamp: number;
+  agent?: string;
+}
+
+export interface RecoveryRecord {
+  context: ErrorContext;
+  action: RecoveryAction;
+  timestamp: number;
+}
+
+export interface RecoveryStats {
+  totalRecoveries: number;
+  byCategory: Record<ErrorCategory, number>;
+  byAction: Record<RecoveryActionType, number>;
+  successRate: number;
+}

package/harness/lib/recovery/patterns.ts

@@ -0,0 +1,149 @@
+// Categorized error patterns with recovery action generators.
+// Each pattern is a RegExp matched against error messages.
+// The first match wins. If none match, the handler escalates.
+
+import type { ErrorCategory, ErrorContext, RecoveryAction } from "./interfaces.ts";
+
+export interface ErrorPattern {
+  pattern: RegExp;
+  category: ErrorCategory;
+  getAction: (ctx: ErrorContext) => RecoveryAction;
+}
+
+/**
+ * Default backoff delays (ms) for retries: [initial, second, third]
+ */
+const RATE_LIMIT_BACKOFF = [1_000, 2_000, 4_000] as const;
+const NETWORK_BACKOFF = [500, 1_000, 2_000] as const;
+
+export const PATTERNS: ErrorPattern[] = [
+  // ── rate_limit ─────────────────────────────────────────────────────
+  {
+    pattern: /rate.?limit|too.?many.?requests|429/i,
+    category: "rate_limit",
+    getAction: (ctx: ErrorContext): RecoveryAction => {
+      const delay = RATE_LIMIT_BACKOFF[Math.min(ctx.attempt, RATE_LIMIT_BACKOFF.length - 1)];
+      return {
+        type: "retry",
+        delay,
+        maxAttempts: 3,
+        reason: `Rate limited (attempt ${ctx.attempt + 1}): backing off ${delay}ms`,
+        modifyPrompt: undefined,
+      };
+    },
+  },
+
+  // ── context_overflow ──────────────────────────────────────────────
+  {
+    pattern: /context.?length|token.?limit|maximum.?context/i,
+    category: "context_overflow",
+    getAction: (): RecoveryAction => ({
+      type: "compact",
+      maxAttempts: 2,
+      reason: "Context overflow detected: compacting before retry",
+      modifyPrompt:
+        "--- COMPACTED: Previous context exceeded token limits. Continue from here, preserving only essential state. ---",
+    }),
+  },
+
+  // ── network ───────────────────────────────────────────────────────
+  {
+    pattern: /ECONNREFUSED|ETIMEDOUT|network|fetch.?failed/i,
+    category: "network",
+    getAction: (ctx: ErrorContext): RecoveryAction => {
+      const delay = NETWORK_BACKOFF[Math.min(ctx.attempt, NETWORK_BACKOFF.length - 1)];
+      return {
+        type: "retry",
+        delay,
+        maxAttempts: 3,
+        reason: `Network error (attempt ${ctx.attempt + 1}): retrying in ${delay}ms`,
+      };
+    },
+  },
+
+  // ── session ───────────────────────────────────────────────────────
+  {
+    pattern: /session.?not.?found|session.?expired/i,
+    category: "session",
+    getAction: (): RecoveryAction => ({
+      type: "abort",
+      reason: "Session not found or expired — cannot recover",
+    }),
+  },
+
+  // ── tool_error ────────────────────────────────────────────────────
+  {
+    pattern: /tool.?not.?found|unknown.?tool/i,
+    category: "tool_error",
+    getAction: (): RecoveryAction => ({
+      type: "escalate",
+      reason: "Unknown or missing tool — orchestrator must decide",
+    }),
+  },
+
+  // ── parse_error ───────────────────────────────────────────────────
+  {
+    pattern: /parse.?error|invalid.?json|syntax.?error/i,
+    category: "parse_error",
+    getAction: (ctx: ErrorContext): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: `Parse error on attempt ${ctx.attempt + 1}: retrying without modification`,
+    }),
+  },
+
+  // ── gibberish ─────────────────────────────────────────────────────
+  // Broad heuristic on output quality — catches gibberish, keyboard mash,
+  // repeated single characters, and long non-alphabetic sequences.
+  {
+    pattern: /gibberish|nonsens|unintelligible|asdfgh|qwerty|xxxxx|sdfsdf|[^a-z\s]{10,}|(.)\1{4,}/i,
+    category: "gibberish",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "Gibberish output detected — retrying with clean context",
+      modifyPrompt:
+        "--- RECOVERY: Previous response was incoherent. Restart with only essential instructions. ---",
+    }),
+  },
+
+  // ── lsp_diagnostic ────────────────────────────────────────────────
+  {
+    pattern: /lsp.?diagnostic|tsc.?error|eslint.?error/i,
+    category: "lsp_diagnostic",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "LSP diagnostic error — retrying with diagnostics in context",
+      modifyPrompt:
+        "--- RECOVERY: The following LSP diagnostics were detected in the last attempt. Fix them before continuing. ---",
+    }),
+  },
+
+  // ── timeout ───────────────────────────────────────────────────────
+  {
+    pattern: /execution.?timed.?out|timeout/i,
+    category: "timeout",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "Execution timed out — retrying with task breakdown hint",
+      modifyPrompt:
+        "--- RECOVERY: Previous execution timed out. Break the task into smaller steps. ---",
+    }),
+  },
+];
+
+/**
+ * Default escalation action when no pattern matches.
+ */
+export function escalateAction(ctx: ErrorContext): RecoveryAction {
+  return {
+    type: "escalate",
+    reason: `Unrecognized error "${ctx.error.message}" — escalating to orchestrator`,
+  };
+}