llm-apology-loop-detector-mcp 1.0.0

package/README.md

@@ -0,0 +1,44 @@
+# llm-apology-loop-detector-mcp
+
+Detect **apology loops**, **retry deadlocks**, and **"I'll try again"** failure modes in LLM transcripts. One MCP call gives you a health score and a ranked escalation plan.
+
+## Why
+Production AI agents (Claude/GPT/Gemini) get stuck spiraling: "Sorry, you're right, let me try again" repeated 30 turns deep, burning $$$ in tokens and never converging. On-call engineers spot it eventually — this MCP server spots it in one call.
+
+## Tools
+
+| Tool | What it does |
+|------|--------------|
+| `detect_apology_loop` | Phrase scan + sliding-window hot spots (density, severity, samples) |
+| `detect_retry_loop` | Identical tool-call repeats + `A→B→A→B` sequence cycles |
+| `loop_health_score` | 0-100 score: `healthy / watch / degraded / deadlocked` |
+| `escalation_advisor` | Health + ranked actions: `human_handoff`, `model_swap`, `reset_context`, `raise_temperature`, `kill_session`, `memoize_tool_failure`, `break_sequence` |
+| `diagnose_transcript` | One-shot on-call triage: headline + postmortem + first failure + everything above |
+
+## Install
+
+```bash
+npm install -g llm-apology-loop-detector-mcp
+```
+
+```jsonc
+// claude_desktop_config.json
+{
+  "mcpServers": {
+    "apology-loop": {
+      "command": "npx",
+      "args": ["-y", "llm-apology-loop-detector-mcp"]
+    }
+  }
+}
+```
+
+## Input shape
+
+Any of:
+- JSON array of messages: `[{role, content, tool?, input?}, ...]`
+- JSONL string (one message per line)
+- `{messages: [...]}` object
+
+## License
+MIT

package/index.js

@@ -0,0 +1,311 @@
+#!/usr/bin/env node
+/**
+ * llm-apology-loop-detector-mcp
+ * ─────────────────────────────
+ * Detect apology loops, retry deadlocks, and "I'll try again" failure modes in LLM
+ * transcripts. Built for production agent ops teams who watch Claude/GPT spirals in
+ * dashboards and need a single MCP call to triage and escalate.
+ */
+
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+
+// ─── Phrase taxonomy ──────────────────────────────────────────────────────
+const APOLOGY_PATTERNS = [
+  { rx: /\b(I\s?(?:am|'m)\s+sorry|my\s+apologies|apologies|I\s+apologi[sz]e)\b/i, w: 25, why: "explicit_apology" },
+  { rx: /\b(you\s+(?:are|'re)\s+(?:right|correct)|you\s+make\s+a\s+good\s+point)\b/i, w: 18, why: "concede_to_user" },
+  { rx: /\b(let\s+me\s+(?:try|do)\s+(?:that|this)\s+again|I'?ll\s+try\s+again|let\s+me\s+retry)\b/i, w: 30, why: "retry_intent" },
+  { rx: /\b(I\s+(?:made|see)\s+(?:an?\s+)?(?:mistake|error)|I\s+(?:was|am)\s+(?:wrong|incorrect))\b/i, w: 28, why: "self_correction" },
+  { rx: /\b(I\s+misunderstood|I\s+misread|I\s+wasn'?t\s+clear|I\s+wasn'?t\s+careful)\b/i, w: 22, why: "misunderstanding" },
+  { rx: /\b(let\s+me\s+(?:rethink|reconsider|reconsider this|approach this differently))\b/i, w: 24, why: "rethink_attempt" },
+  { rx: /\b(I\s+see\s+the\s+(?:issue|problem|confusion)\s+now)\b/i, w: 20, why: "insight_claim" },
+  { rx: /\b(thank\s+you\s+for\s+(?:catching|pointing\s+out|the\s+correction))\b/i, w: 18, why: "thanks_for_correction" },
+];
+
+const FRUSTRATION_PATTERNS = [
+  { rx: /\b(actually|wait|hmm|hmmm|let\s+me\s+think)\b/i, w: 6, why: "hedge" },
+  { rx: /\b(unfortunately|however|but\s+actually|that\s+said)\b/i, w: 8, why: "soft_contradiction" },
+  { rx: /\b(I\s+cannot|I\s+can'?t|I'?m\s+unable\s+to)\b/i, w: 15, why: "capability_denial" },
+];
+
+const TOOL_FAIL_HINTS = [
+  /\b(error|exception|failed|failure|crash|panic|timeout|timed?\s?out)\b/i,
+  /\b(403|404|429|500|502|503|504)\b/,
+  /\b(connection\s+(?:refused|reset)|enotfound|econnrefused)\b/i,
+];
+
+const RETRY_TOOL_WHITELIST = new Set([
+  "Read","Write","Edit","Bash","Glob","Grep","fetch","http","curl","wrangler","node","npm","python",
+]);
+
+// ─── Helpers ─────────────────────────────────────────────────────────────
+function safeJson(s, fallback = null) { try { return JSON.parse(s); } catch { return fallback; } }
+
+function normalizeMessages(raw) {
+  if (Array.isArray(raw)) return raw;
+  if (raw && typeof raw === "object" && Array.isArray(raw.messages)) return raw.messages;
+  if (typeof raw !== "string") return [];
+  const t = raw.trim();
+  if (t.startsWith("[")) { const p = safeJson(t); if (Array.isArray(p)) return p; }
+  if (t.startsWith("{")) { const p = safeJson(t); if (p && Array.isArray(p.messages)) return p.messages; }
+  return t.split(/\r?\n/).map((line) => safeJson(line.trim())).filter(Boolean);
+}
+
+function roleOf(msg) {
+  return (msg?.role || msg?.author || msg?.from || "assistant").toLowerCase();
+}
+function textOf(msg) {
+  if (typeof msg === "string") return msg;
+  if (typeof msg?.content === "string") return msg.content;
+  if (Array.isArray(msg?.content)) {
+    return msg.content.map((c) => typeof c === "string" ? c : (c?.text || c?.input || c?.output || "")).filter(Boolean).join("\n");
+  }
+  return msg?.text || msg?.message || "";
+}
+function toolOf(msg) {
+  return msg?.tool || msg?.name || msg?.function?.name || null;
+}
+
+function ngram(arr, n) {
+  const out = [];
+  for (let i = 0; i + n <= arr.length; i++) out.push(arr.slice(i, i + n).join("|"));
+  return out;
+}
+
+// ─── Apology loop detection ──────────────────────────────────────────────
+function scoreApologyPhrases(text) {
+  let score = 0;
+  const reasons = [];
+  for (const p of APOLOGY_PATTERNS) {
+    if (p.rx.test(text)) { score += p.w; reasons.push(p.why); }
+  }
+  for (const p of FRUSTRATION_PATTERNS) {
+    if (p.rx.test(text)) { score += p.w; reasons.push(p.why); }
+  }
+  return { score, reasons };
+}
+
+function detectApologyLoop(raw, { window = 6, threshold = 2 } = {}) {
+  const messages = normalizeMessages(raw);
+  const assistantMessages = messages.map((m, i) => ({ i, role: roleOf(m), text: textOf(m), tool: toolOf(m) }))
+    .filter((m) => m.role === "assistant" && m.text);
+  const scored = assistantMessages.map((m) => {
+    const s = scoreApologyPhrases(m.text);
+    return { ...m, apology_score: s.score, reasons: s.reasons };
+  });
+  const apologyTurns = scored.filter((s) => s.apology_score >= 20);
+  // Sliding window: ≥threshold apologies inside any `window`-sized assistant span
+  const windows = [];
+  for (let i = 0; i + window <= scored.length; i++) {
+    const slice = scored.slice(i, i + window);
+    const hits = slice.filter((s) => s.apology_score >= 20);
+    if (hits.length >= threshold) {
+      windows.push({
+        start_msg_index: slice[0].i,
+        end_msg_index: slice[slice.length - 1].i,
+        apology_count: hits.length,
+        window_size: window,
+        severity: hits.length >= window * 0.6 ? "high" : hits.length >= window * 0.4 ? "medium" : "low",
+      });
+    }
+  }
+  return {
+    total_assistant_messages: scored.length,
+    total_apology_turns: apologyTurns.length,
+    apology_density: scored.length ? Math.round((apologyTurns.length / scored.length) * 1000) / 10 : 0,
+    apology_windows: windows.slice(0, 8),
+    sample_apology_turns: apologyTurns.slice(0, 5).map((a) => ({
+      msg_index: a.i,
+      reasons: a.reasons,
+      score: a.apology_score,
+      text_excerpt: a.text.slice(0, 200),
+    })),
+  };
+}
+
+// ─── Retry loop detection ────────────────────────────────────────────────
+function detectRetryLoop(raw, { min_repeats = 3 } = {}) {
+  const messages = normalizeMessages(raw);
+  // Extract tool calls with their (tool,input) signature
+  const calls = [];
+  for (const m of messages) {
+    const tool = toolOf(m);
+    if (!tool) continue;
+    const input = m?.input || m?.arguments || m?.params || m?.payload || null;
+    const sig = `${tool}:${stableHash(input)}`;
+    calls.push({ tool, sig, hadError: messages.some((mm) => mm?.parent_id === m?.id && isError(mm)) || isError(m) });
+  }
+  const sigCount = {};
+  for (const c of calls) sigCount[c.sig] = (sigCount[c.sig] || 0) + 1;
+  const repeats = Object.entries(sigCount)
+    .filter(([, n]) => n >= min_repeats)
+    .map(([sig, n]) => {
+      const [tool, hash] = sig.split(":");
+      return {
+        signature: sig,
+        tool,
+        identical_calls: n,
+        likely_retry: RETRY_TOOL_WHITELIST.has(tool) || calls.find((c) => c.sig === sig && c.hadError) ? true : false,
+      };
+    });
+  // Also detect tool-sequence cycles (A→B→A→B)
+  const sigs = calls.map((c) => c.sig);
+  const cycles = [];
+  for (let len = 2; len <= 4; len++) {
+    for (let i = 0; i + len * min_repeats <= sigs.length; i++) {
+      const pattern = sigs.slice(i, i + len).join("→");
+      let reps = 1;
+      let cursor = i + len;
+      while (cursor + len <= sigs.length && sigs.slice(cursor, cursor + len).join("→") === pattern) {
+        reps++; cursor += len;
+      }
+      if (reps >= min_repeats) {
+        cycles.push({ pattern, length: len, repeats: reps, starts_at: i, ends_at: cursor - 1 });
+        i = cursor - 1;
+      }
+    }
+  }
+  return {
+    total_tool_calls: calls.length,
+    identical_call_repeats: repeats,
+    sequence_cycles: cycles.slice(0, 6),
+    has_retry_loop: repeats.some((r) => r.likely_retry) || cycles.length > 0,
+  };
+}
+
+function isError(msg) {
+  if (!msg) return false;
+  if (msg.error || msg.is_error || msg.failed === true) return true;
+  const t = textOf(msg);
+  return TOOL_FAIL_HINTS.some((rx) => rx.test(t));
+}
+
+function stableHash(v) {
+  if (v == null) return "∅";
+  const s = typeof v === "string" ? v : JSON.stringify(v);
+  let h = 0;
+  for (let i = 0; i < s.length; i++) h = ((h << 5) - h + s.charCodeAt(i)) | 0;
+  return `h_${(h >>> 0).toString(36)}`;
+}
+
+// ─── Health score + advice ───────────────────────────────────────────────
+function loopHealth(raw) {
+  const apology = detectApologyLoop(raw);
+  const retry = detectRetryLoop(raw);
+  let health = 100;
+  if (apology.apology_density >= 60) health -= 50;
+  else if (apology.apology_density >= 30) health -= 30;
+  else if (apology.apology_density >= 15) health -= 15;
+  if (apology.apology_windows.some((w) => w.severity === "high")) health -= 20;
+  if (retry.identical_call_repeats.some((r) => r.identical_calls >= 5)) health -= 25;
+  if (retry.sequence_cycles.length > 0) health -= 20;
+  health = Math.max(0, Math.min(100, health));
+  return {
+    health_score: health,
+    band: health >= 80 ? "healthy" : health >= 60 ? "watch" : health >= 40 ? "degraded" : "deadlocked",
+    apology,
+    retry,
+  };
+}
+
+function escalation(raw, options = {}) {
+  const h = loopHealth(raw);
+  const recs = [];
+  if (h.band === "deadlocked") {
+    recs.push({ action: "human_handoff", reason: "Health < 40 — escalate the conversation to a human operator immediately." });
+    recs.push({ action: "kill_session", reason: "Stop further tool calls to avoid burning tokens on a known dead loop." });
+  }
+  if (h.band === "degraded") {
+    recs.push({ action: "model_swap", to: options.fallback_model || "gpt-5", reason: "Same model is stuck — switch family (Claude→GPT or vice versa)." });
+    recs.push({ action: "raise_temperature", to: 0.9, reason: "Determinism trap; jiggle the sampler." });
+  }
+  if (h.apology.apology_density >= 30) {
+    recs.push({ action: "reset_context", reason: "Apology streak ≥30% — context contains poisoned correction loop; reset to system prompt + last user message." });
+  }
+  if (h.retry.identical_call_repeats.some((r) => r.identical_calls >= 4)) {
+    recs.push({ action: "memoize_tool_failure", reason: "Identical tool calls repeated; remember the failure and stop re-attempting in this session." });
+  }
+  if (h.retry.sequence_cycles.length > 0) {
+    recs.push({ action: "break_sequence", reason: `Detected A→B→A→B cycle (${h.retry.sequence_cycles[0].repeats}× repeats) — inject a stop tool or hand off.` });
+  }
+  if (recs.length === 0) {
+    recs.push({ action: "continue", reason: "No loop pathology detected — continue current run." });
+  }
+  return { ...h, recommendations: recs };
+}
+
+// ─── MCP wiring ──────────────────────────────────────────────────────────
+const server = new McpServer({
+  name: "llm-apology-loop-detector-mcp",
+  version: "1.0.0",
+  description: "Detect apology loops, retry deadlocks, and 'I'll try again' agent failure modes — with escalation recommendations.",
+});
+
+const TranscriptInput = z.union([z.string(), z.array(z.any()), z.record(z.any())])
+  .describe("Transcript as JSON array of messages, JSONL string, or {messages:[...]} object. Each message: {role, content, tool?, input?}.");
+
+server.tool(
+  "detect_apology_loop",
+  "Scan the transcript for apology / 'try again' / self-correction phrases. Returns density, sliding-window hot spots, and sample turns.",
+  { transcript: TranscriptInput, window: z.number().int().min(2).default(6), threshold: z.number().int().min(2).default(2) },
+  async ({ transcript, window, threshold }) => {
+    const out = detectApologyLoop(transcript, { window, threshold });
+    return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
+  }
+);
+
+server.tool(
+  "detect_retry_loop",
+  "Find identical tool-call repeats and A→B→A→B sequence cycles — the classic sign of an agent stuck retrying a broken action.",
+  { transcript: TranscriptInput, min_repeats: z.number().int().min(2).default(3) },
+  async ({ transcript, min_repeats }) => {
+    const out = detectRetryLoop(transcript, { min_repeats });
+    return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
+  }
+);
+
+server.tool(
+  "loop_health_score",
+  "Combine apology + retry signals into a 0-100 health score (healthy / watch / degraded / deadlocked).",
+  { transcript: TranscriptInput },
+  async ({ transcript }) => {
+    const out = loopHealth(transcript);
+    return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
+  }
+);
+
+server.tool(
+  "escalation_advisor",
+  "Health score + a ranked list of recommended actions (human_handoff, model_swap, reset_context, raise_temperature, kill_session, etc.).",
+  { transcript: TranscriptInput, fallback_model: z.string().optional() },
+  async ({ transcript, fallback_model }) => {
+    const out = escalation(transcript, { fallback_model });
+    return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
+  }
+);
+
+server.tool(
+  "diagnose_transcript",
+  "One-shot triage for an on-call: apology + retry + health + recommendations + first failing tool + a one-paragraph postmortem.",
+  { transcript: TranscriptInput },
+  async ({ transcript }) => {
+    const adv = escalation(transcript);
+    const messages = normalizeMessages(transcript);
+    const firstFail = messages.find((m) => isError(m));
+    const postmortem = [
+      `Health: ${adv.health_score}/100 (${adv.band}).`,
+      `Apology density: ${adv.apology.apology_density}% across ${adv.apology.total_assistant_messages} assistant turns.`,
+      `Retry signals: ${adv.retry.identical_call_repeats.length} repeated-call patterns, ${adv.retry.sequence_cycles.length} sequence cycles.`,
+      adv.recommendations[0] ? `Primary action: ${adv.recommendations[0].action} — ${adv.recommendations[0].reason}` : "",
+    ].filter(Boolean).join(" ");
+    return { content: [{ type: "text", text: JSON.stringify({
+      headline: `${adv.band.toUpperCase()} — health ${adv.health_score}/100`,
+      postmortem,
+      first_failure: firstFail ? { role: roleOf(firstFail), tool: toolOf(firstFail), excerpt: textOf(firstFail).slice(0, 240) } : null,
+      ...adv,
+    }, null, 2) }] };
+  }
+);
+
+await server.connect(new StdioServerTransport());

package/mcpize.yaml

@@ -0,0 +1,12 @@
+name: llm-apology-loop-detector-mcp
+version: 1.0.0
+description: Detect apology loops, retry deadlocks, and "I'll try again" agent failure modes in LLM transcripts. Surfaces the loop, scores severity, and prescribes an escalation action.
+license: MIT
+runtime: node
+entry: index.js
+tools:
+  - detect_apology_loop
+  - detect_retry_loop
+  - loop_health_score
+  - escalation_advisor
+  - diagnose_transcript

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "llm-apology-loop-detector-mcp",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Detect apology loops, retry deadlocks, and 'I'll try again' agent failure modes in LLM transcripts. Surfaces the loop, scores severity, and prescribes an escalation action (temperature bump, model swap, human handoff).",
+  "bin": { "llm-apology-loop-detector-mcp": "index.js" },
+  "main": "index.js",
+  "mcp": {
+    "tools": [
+      "detect_apology_loop",
+      "detect_retry_loop",
+      "loop_health_score",
+      "escalation_advisor",
+      "diagnose_transcript"
+    ]
+  },
+  "keywords": [
+    "mcp",
+    "llm",
+    "agent",
+    "observability",
+    "debugging",
+    "apology",
+    "loop",
+    "deadlock"
+  ],
+  "license": "MIT",
+  "author": "lazymac2x",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "latest",
+    "zod": "^3.23.0"
+  }
+}