openhermes 4.3.0 → 4.11.2

package/harness/lib/recovery/patterns.ts

@@ -0,0 +1,149 @@
+// Categorized error patterns with recovery action generators.
+// Each pattern is a RegExp matched against error messages.
+// The first match wins. If none match, the handler escalates.
+
+import type { ErrorCategory, ErrorContext, RecoveryAction } from "./interfaces.ts";
+
+export interface ErrorPattern {
+  pattern: RegExp;
+  category: ErrorCategory;
+  getAction: (ctx: ErrorContext) => RecoveryAction;
+}
+
+/**
+ * Default backoff delays (ms) for retries: [initial, second, third]
+ */
+const RATE_LIMIT_BACKOFF = [1_000, 2_000, 4_000] as const;
+const NETWORK_BACKOFF = [500, 1_000, 2_000] as const;
+
+export const PATTERNS: ErrorPattern[] = [
+  // ── rate_limit ─────────────────────────────────────────────────────
+  {
+    pattern: /rate.?limit|too.?many.?requests|429/i,
+    category: "rate_limit",
+    getAction: (ctx: ErrorContext): RecoveryAction => {
+      const delay = RATE_LIMIT_BACKOFF[Math.min(ctx.attempt, RATE_LIMIT_BACKOFF.length - 1)];
+      return {
+        type: "retry",
+        delay,
+        maxAttempts: 3,
+        reason: `Rate limited (attempt ${ctx.attempt + 1}): backing off ${delay}ms`,
+        modifyPrompt: undefined,
+      };
+    },
+  },
+
+  // ── context_overflow ──────────────────────────────────────────────
+  {
+    pattern: /context.?length|token.?limit|maximum.?context/i,
+    category: "context_overflow",
+    getAction: (): RecoveryAction => ({
+      type: "compact",
+      maxAttempts: 2,
+      reason: "Context overflow detected: compacting before retry",
+      modifyPrompt:
+        "--- COMPACTED: Previous context exceeded token limits. Continue from here, preserving only essential state. ---",
+    }),
+  },
+
+  // ── network ───────────────────────────────────────────────────────
+  {
+    pattern: /ECONNREFUSED|ETIMEDOUT|network|fetch.?failed/i,
+    category: "network",
+    getAction: (ctx: ErrorContext): RecoveryAction => {
+      const delay = NETWORK_BACKOFF[Math.min(ctx.attempt, NETWORK_BACKOFF.length - 1)];
+      return {
+        type: "retry",
+        delay,
+        maxAttempts: 3,
+        reason: `Network error (attempt ${ctx.attempt + 1}): retrying in ${delay}ms`,
+      };
+    },
+  },
+
+  // ── session ───────────────────────────────────────────────────────
+  {
+    pattern: /session.?not.?found|session.?expired/i,
+    category: "session",
+    getAction: (): RecoveryAction => ({
+      type: "abort",
+      reason: "Session not found or expired — cannot recover",
+    }),
+  },
+
+  // ── tool_error ────────────────────────────────────────────────────
+  {
+    pattern: /tool.?not.?found|unknown.?tool/i,
+    category: "tool_error",
+    getAction: (): RecoveryAction => ({
+      type: "escalate",
+      reason: "Unknown or missing tool — orchestrator must decide",
+    }),
+  },
+
+  // ── parse_error ───────────────────────────────────────────────────
+  {
+    pattern: /parse.?error|invalid.?json|syntax.?error/i,
+    category: "parse_error",
+    getAction: (ctx: ErrorContext): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: `Parse error on attempt ${ctx.attempt + 1}: retrying without modification`,
+    }),
+  },
+
+  // ── gibberish ─────────────────────────────────────────────────────
+  // Broad heuristic on output quality — catches gibberish, keyboard mash,
+  // repeated single characters, and long non-alphabetic sequences.
+  {
+    pattern: /gibberish|nonsens|unintelligible|asdfgh|qwerty|xxxxx|sdfsdf|[^a-z\s]{10,}|(.)\1{4,}/i,
+    category: "gibberish",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "Gibberish output detected — retrying with clean context",
+      modifyPrompt:
+        "--- RECOVERY: Previous response was incoherent. Restart with only essential instructions. ---",
+    }),
+  },
+
+  // ── lsp_diagnostic ────────────────────────────────────────────────
+  {
+    pattern: /lsp.?diagnostic|tsc.?error|eslint.?error/i,
+    category: "lsp_diagnostic",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "LSP diagnostic error — retrying with diagnostics in context",
+      modifyPrompt:
+        "--- RECOVERY: The following LSP diagnostics were detected in the last attempt. Fix them before continuing. ---",
+    }),
+  },
+
+  // ── timeout ───────────────────────────────────────────────────────
+  {
+    pattern: /execution.?timed.?out|timeout/i,
+    category: "timeout",
+    getAction: (): RecoveryAction => ({
+      type: "retry",
+      delay: 0,
+      maxAttempts: 2,
+      reason: "Execution timed out — retrying with task breakdown hint",
+      modifyPrompt:
+        "--- RECOVERY: Previous execution timed out. Break the task into smaller steps. ---",
+    }),
+  },
+];
+
+/**
+ * Default escalation action when no pattern matches.
+ */
+export function escalateAction(ctx: ErrorContext): RecoveryAction {
+  return {
+    type: "escalate",
+    reason: `Unrecognized error "${ctx.error.message}" — escalating to orchestrator`,
+  };
+}

package/harness/lib/recovery/recovery.test.ts

@@ -0,0 +1,312 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { RecoveryHandler } from "./handler.ts";
+import type { ErrorContext, RecoveryAction } from "./interfaces.ts";
+
+/**
+ * Helper to build an ErrorContext.
+ */
+function ctx(
+  sessionId: string,
+  message: string,
+  attempt = 0,
+  agent?: string,
+): ErrorContext {
+  return {
+    sessionId,
+    error: new Error(message),
+    attempt,
+    timestamp: Date.now(),
+    agent,
+  };
+}
+
+/**
+ * Check that an action has the expected type & reason substring.
+ */
+function assertAction(
+  action: RecoveryAction,
+  type: string,
+  reasonSubstring?: string,
+): void {
+  assert.equal(action.type, type, `expected type ${type}, got ${action.type}`);
+  if (reasonSubstring) {
+    assert.ok(
+      action.reason.toLowerCase().includes(reasonSubstring.toLowerCase()),
+      `expected reason to include "${reasonSubstring}", got "${action.reason}"`,
+    );
+  }
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+describe("RecoveryHandler — pattern classification", () => {
+  let handler: RecoveryHandler;
+
+  before(() => {
+    handler = RecoveryHandler.getInstance();
+    handler.reset();
+  });
+
+  after(() => {
+    handler.reset();
+  });
+
+  // ── 9 category tests ─────────────────────────────────────────────
+
+  it("classifies rate_limit errors → retry with backoff", () => {
+    const action = handler.handleError(ctx("s1", "rate limit exceeded", 0));
+    assertAction(action, "retry", "rate");
+    assert.ok(action.delay! >= 1_000, `expected delay >= 1000, got ${action.delay}`);
+
+    // Second attempt gets longer backoff
+    const action2 = handler.handleError(ctx("s1", "429 Too Many Requests", 1));
+    assertAction(action2, "retry", "rate");
+    assert.ok(action2.delay! >= 2_000, `expected delay >= 2000, got ${action2.delay}`);
+  });
+
+  it("classifies context_overflow errors → compact", () => {
+    const action = handler.handleError(ctx("s1", "context length exceeded", 0));
+    assertAction(action, "compact", "context");
+    assert.ok(action.modifyPrompt, "compact action should include modifyPrompt");
+  });
+
+  it("classifies network errors → retry with backoff", () => {
+    const action = handler.handleError(ctx("s1", "ECONNREFUSED", 0));
+    assertAction(action, "retry", "network");
+    assert.ok(action.delay! >= 500, `expected delay >= 500, got ${action.delay}`);
+
+    const action2 = handler.handleError(ctx("s1", "fetch failed", 1));
+    assertAction(action2, "retry", "network");
+  });
+
+  it("classifies session errors → abort", () => {
+    const action = handler.handleError(ctx("s1", "session not found", 0));
+    assertAction(action, "abort", "session");
+  });
+
+  it("classifies tool_error → escalate", () => {
+    const action = handler.handleError(ctx("s1", "unknown tool: foo", 0));
+    assertAction(action, "escalate", "tool");
+  });
+
+  it("classifies parse_error → retry (max 2)", () => {
+    const action = handler.handleError(ctx("s1", "parse error at line 42", 0));
+    assertAction(action, "retry", "parse");
+    assert.equal(action.maxAttempts, 2);
+  });
+
+  it("classifies gibberish (explicit) → retry with clean context", () => {
+    // Gibberish pattern now matches "gibberish", "nonsens", keyboard mash,
+    // and other low-quality output patterns in error messages.
+    const action = handler.handleError(ctx("s1", "gibberish output detected", 0));
+    assertAction(action, "retry", "gibberish");
+    assert.ok(action.modifyPrompt, "gibberish action should include modifyPrompt");
+  });
+
+  it("classifies lsp_diagnostic → retry with diagnostic prompt", () => {
+    const action = handler.handleError(ctx("s1", "tsc error: Type 'X' not assignable", 0));
+    assertAction(action, "retry", "lsp");
+    assert.ok(action.modifyPrompt, "lsp action should include modifyPrompt");
+  });
+
+  it("classifies timeout → retry with breakdown hint", () => {
+    const action = handler.handleError(ctx("s1", "execution timed out", 0));
+    assertAction(action, "retry", "timed out");
+    assert.ok(action.modifyPrompt, "timeout action should include modifyPrompt");
+    assert.equal(action.maxAttempts, 2);
+  });
+
+  // ── Unknown error → escalate ──────────────────────────────────────
+
+  it("unknown error pattern → escalate", () => {
+    const action = handler.handleError(ctx("s1", "some weird error nobody expected", 0));
+    assertAction(action, "escalate", "unrecognized");
+  });
+
+  // ── Stats tracking ────────────────────────────────────────────────
+
+  it("getStats() reflects classified errors", () => {
+    handler.reset();
+
+    handler.handleError(ctx("s1", "rate limit", 0));
+    handler.handleError(ctx("s1", "ECONNREFUSED", 0));
+    handler.handleError(ctx("s1", "context length", 0));
+    handler.handleError(ctx("s1", "session not found", 0));
+    handler.handleError(ctx("s1", "unknown tool", 0));
+    handler.handleError(ctx("s1", "parse error", 0));
+
+    const stats = handler.getStats();
+    assert.equal(stats.totalRecoveries, 6);
+    assert.equal(stats.byCategory.rate_limit, 1);
+    assert.equal(stats.byCategory.network, 1);
+    assert.equal(stats.byCategory.context_overflow, 1);
+    assert.equal(stats.byCategory.session, 1);
+    assert.equal(stats.byCategory.tool_error, 1);
+    assert.equal(stats.byCategory.parse_error, 1);
+    // Unclassified categories should be 0
+    assert.equal(stats.byCategory.gibberish, 0);
+    assert.equal(stats.byCategory.lsp_diagnostic, 0);
+    assert.equal(stats.byCategory.timeout, 0);
+  });
+
+  it("getStats() tracks action types", () => {
+    handler.reset();
+
+    handler.handleError(ctx("s1", "rate limit", 0));       // retry
+    handler.handleError(ctx("s1", "session expired", 0));  // abort
+    handler.handleError(ctx("s1", "unknown tool", 0));     // escalate
+    handler.handleError(ctx("s1", "context length", 0));   // compact
+
+    const stats = handler.getStats();
+    assert.equal(stats.byAction.retry, 1);
+    assert.equal(stats.byAction.abort, 1);
+    assert.equal(stats.byAction.escalate, 1);
+    assert.equal(stats.byAction.compact, 1);
+  });
+
+  // ── getHistory ──────────────────────────────────────────────────
+
+  it("getHistory() returns records most recent first", () => {
+    handler.reset();
+
+    handler.handleError(ctx("s2", "rate limit", 0));
+    handler.handleError(ctx("s2", "timeout", 0));
+
+    const history = handler.getHistory();
+    assert.equal(history.length, 2);
+    assert.ok(history[0].timestamp >= history[1].timestamp);
+  });
+
+  it("getHistory(limit) respects limit", () => {
+    handler.reset();
+
+    handler.handleError(ctx("s3", "rate limit", 0));
+    handler.handleError(ctx("s3", "timeout", 0));
+    handler.handleError(ctx("s3", "ECONNREFUSED", 0));
+
+    const limited = handler.getHistory(2);
+    assert.equal(limited.length, 2);
+  });
+
+  // ── clearSession ────────────────────────────────────────────────
+
+  it("clearSession() removes records for a session", () => {
+    handler.reset();
+
+    handler.handleError(ctx("s_a", "rate limit", 0));
+    handler.handleError(ctx("s_b", "timeout", 0));
+    handler.handleError(ctx("s_a", "ECONNREFUSED", 0));
+
+    assert.equal(handler.getHistory().length, 3);
+
+    handler.clearSession("s_a");
+    const remaining = handler.getHistory();
+    assert.equal(remaining.length, 1);
+    assert.equal(remaining[0].context.sessionId, "s_b");
+  });
+});
+
+// ── withRecovery integration ──────────────────────────────────────────
+
+describe("RecoveryHandler — withRecovery", () => {
+  let handler: RecoveryHandler;
+
+  before(() => {
+    handler = RecoveryHandler.getInstance();
+    handler.reset();
+  });
+
+  after(() => {
+    handler.reset();
+  });
+
+  it("succeeds on first attempt", async () => {
+    const result = await handler.withRecovery("wr1", async () => "hello");
+    assert.equal(result, "hello");
+  });
+
+  it("retries on retryable error then succeeds", async () => {
+    handler.reset();
+    let callCount = 0;
+
+    const result = await handler.withRecovery("wr2", async () => {
+      callCount++;
+      if (callCount === 1) throw new Error("parse error: invalid json");
+      return "ok-after-retry";
+    });
+
+    assert.equal(result, "ok-after-retry");
+    assert.equal(callCount, 2);
+  });
+
+  it("throws on abort action", async () => {
+    handler.reset();
+
+    await assert.rejects(
+      handler.withRecovery("wr3", async () => {
+        throw new Error("session not found");
+      }),
+      /session not found/,
+    );
+  });
+
+  it("throws on escalate action", async () => {
+    handler.reset();
+
+    await assert.rejects(
+      handler.withRecovery("wr4", async () => {
+        throw new Error("unknown tool: foo");
+      }),
+      /unknown tool/,
+    );
+  });
+
+  it("respects category maxAttempts and then throws", async () => {
+    handler.reset();
+
+    await assert.rejects(
+      handler.withRecovery("wr5", async () => {
+        throw new Error("parse error: bad json");
+      }),
+      /parse error/,
+    );
+  });
+
+  it("respects global maxAttempts option", async () => {
+    handler.reset();
+    let callCount = 0;
+
+    // Use an unrecognized error (will escalate on first try — so throw immediately).
+    // Instead let's use a timeout pattern which has maxAttempts=2, but set global maxAttempts=1
+    await assert.rejects(
+      handler.withRecovery(
+        "wr6",
+        async () => {
+          callCount++;
+          throw new Error("execution timed out");
+        },
+        { maxAttempts: 1 },
+      ),
+    );
+    // With maxAttempts=1, it should only be called once
+    assert.equal(callCount, 1);
+  });
+
+  it("tracks success and failure in stats", async () => {
+    handler.reset();
+
+    // One success
+    await handler.withRecovery("wr7", async () => "good");
+    // One abort (failure)
+    await assert.rejects(
+      handler.withRecovery("wr7", async () => {
+        throw new Error("session expired");
+      }),
+    );
+
+    const stats = handler.getStats();
+    assert.ok(stats.successRate > 0);
+    assert.ok(stats.successRate < 1);
+  });
+});

package/harness/lib/sanity/anomaly-tracker.ts

@@ -0,0 +1,127 @@
+// ---------------------------------------------------------------------------
+// AnomalyTracker — singleton that tracks consecutive anomaly records per session
+// ---------------------------------------------------------------------------
+
+import type { AnomalyRecord, AnomalyTrackerConfig, SanityResult } from "./interfaces.ts";
+
+const DEFAULT_CONFIG: AnomalyTrackerConfig = {
+  maxConsecutiveAnomalies: 2,
+  escalationMessage: "recovery: compact context",
+};
+
+export class AnomalyTracker {
+  private static instance: AnomalyTracker;
+
+  private records = new Map<string, AnomalyRecord>();
+  private config: AnomalyTrackerConfig;
+
+  private constructor(config?: Partial<AnomalyTrackerConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /** Get the singleton instance. */
+  static getInstance(config?: Partial<AnomalyTrackerConfig>): AnomalyTracker {
+    if (!AnomalyTracker.instance) {
+      AnomalyTracker.instance = new AnomalyTracker(config);
+    }
+    return AnomalyTracker.instance;
+  }
+
+  /**
+   * Record a sanity result for a session.
+   * If unhealthy: increments consecutive count, updates reason/timestamp.
+   * If healthy: resets consecutive count to 0.
+   * Returns tracking info including whether escalation is needed.
+   */
+  record(
+    sessionId: string,
+    result: SanityResult,
+  ): {
+    shouldEscalate: boolean;
+    consecutiveAnomalies: number;
+    recoveryMessage?: string;
+  } {
+    const existing = this.records.get(sessionId);
+
+    if (!result.isHealthy) {
+      const count = (existing?.count ?? 0) + 1;
+      this.records.set(sessionId, {
+        sessionId,
+        count,
+        lastReason: result.reason ?? "Unknown anomaly",
+        lastTimestamp: Date.now(),
+      });
+
+      const shouldEscalate = count >= this.config.maxConsecutiveAnomalies;
+
+      return {
+        shouldEscalate,
+        consecutiveAnomalies: count,
+        recoveryMessage: shouldEscalate ? this.config.escalationMessage : undefined,
+      };
+    }
+
+    // Healthy output — reset counter
+    if (existing) {
+      this.records.set(sessionId, {
+        ...existing,
+        count: 0,
+        lastReason: "reset on healthy output",
+        lastTimestamp: Date.now(),
+      });
+    }
+
+    return {
+      shouldEscalate: false,
+      consecutiveAnomalies: 0,
+    };
+  }
+
+  /** Get the current anomaly record for a session. */
+  getRecord(sessionId: string): AnomalyRecord | undefined {
+    return this.records.get(sessionId);
+  }
+
+  /** Clear anomaly record for a specific session. */
+  clearSession(sessionId: string): void {
+    this.records.delete(sessionId);
+  }
+
+  /** Reset all tracking state (useful in tests). */
+  resetAll(): void {
+    this.records.clear();
+  }
+
+  /** Get the current config. */
+  getConfig(): AnomalyTrackerConfig {
+    return { ...this.config };
+  }
+
+  /** Update config at runtime. */
+  setConfig(config: Partial<AnomalyTrackerConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+
+  // ── Cross-invocation identical output detection ────────────────
+
+  private lastOutput: string | null = null;
+  private identicalOutputCount = 0;
+  readonly MAX_IDENTICAL_OUTPUTS = 3;
+
+  /**
+   * Track output for repeated identical content.
+   * Returns true if output should be flagged as degenerate.
+   */
+  trackOutput(text: string): boolean {
+    if (text === this.lastOutput) {
+      this.identicalOutputCount++;
+      if (this.identicalOutputCount >= this.MAX_IDENTICAL_OUTPUTS) {
+        return true; // Flagged — repeated identical output
+      }
+    } else {
+      this.identicalOutputCount = 0;
+    }
+    this.lastOutput = text;
+    return false;
+  }
+}