llm-output-guard 1.10.0 → 1.11.1

package/README.md

@@ -142,6 +142,10 @@ A turn carrying tool calls is judged by its **arguments**, never its prose —
 that is the difference between an agent working through a list and an agent
 stuck on one item.
 
+**[Try it on a run →](https://edwinsatya.github.io/llm-output-guard/)** — the
+playground has an *agent run* mode: pick a trace, see which turns form the
+cycle. The healthy traps are the half worth looking at.
+
 **[docs/agent-loops.md](docs/agent-loops.md)**
 
 ## The hard part is not catching loops
@@ -267,7 +271,7 @@ because only one of those is evidence.
 - **Scores, not booleans.** Detectors report 0–1 and leave the threshold decision to you.
 - **Abstains rather than guesses.** Samples too short to judge score 0.
 - **Hand-rolled LZ77** rather than `node:zlib`, so the package stays runtime-agnostic.
-- **Sub-millisecond**, 0.383 ms at 500 B and 1.014 ms at 32 KB, with one detector accounting for most of it. `npm run bench` reproduces it — **[docs/performance.md](docs/performance.md)**
+- **Sub-millisecond**, 0.383 ms at 500 B and 1.014 ms at 32 KB, with one detector accounting for most of it. A 12-turn agent run costs 0.075 ms. `npm run bench` reproduces both — **[docs/performance.md](docs/performance.md)**
 - **Chinese, Japanese and Thai** are handled where they differ: `TAIL_LOOP` switches to character mode, `REPETITION` is blind and says so — **[docs/script-coverage.md](docs/script-coverage.md)**
 
 ## Stability

package/dist/bin.cjs

@@ -1033,6 +1033,16 @@ function extractTurn(value) {
     return typeof value === "string" && value.length > 0 ? { text: value } : null;
   }
   const record = value;
+  if (typeof record.role === "string" && record.role !== "assistant" && record.role !== "model") {
+    return null;
+  }
+  const messageCalls = Array.isArray(record.tool_calls) ? record.tool_calls : Array.isArray(record.toolCalls) && typeof record.content === "string" ? record.toolCalls : null;
+  if (messageCalls && (typeof record.content === "string" || record.content == null)) {
+    return toTurn({ choices: [{ message: { ...record, tool_calls: messageCalls } }] });
+  }
+  if (typeof record.content === "string" && record.role != null) {
+    return record.content.length > 0 ? { text: record.content } : null;
+  }
   if (Array.isArray(record.toolCalls)) {
     const calls = record.toolCalls;
     const native = calls.every(
@@ -1064,11 +1074,32 @@ function extractTurn(value) {
   if (typeof record.text === "string") return { text: record.text };
   return null;
 }
+var TURN_KEYS = ["turns", "messages", "history", "conversation", "steps"];
 function extractTurns(value) {
-  const list = Array.isArray(value) ? value : value && typeof value === "object" && Array.isArray(value.turns) ? value.turns : null;
-  if (!list) return null;
-  const turns = list.map(extractTurn).filter((turn) => turn !== null);
-  return turns.length > 0 ? turns : null;
+  const read = (list) => {
+    if (!Array.isArray(list)) return null;
+    const turns = list.map(extractTurn).filter((turn) => turn !== null);
+    return turns.length > 0 ? turns : null;
+  };
+  if (Array.isArray(value)) return read(value);
+  if (!value || typeof value !== "object") return null;
+  const record = value;
+  const named = (from) => {
+    for (const key of TURN_KEYS) {
+      const found = read(from[key]);
+      if (found) return found;
+    }
+    return null;
+  };
+  const here = named(record);
+  if (here) return here;
+  for (const nested of Object.values(record)) {
+    if (nested && typeof nested === "object" && !Array.isArray(nested)) {
+      const found = named(nested);
+      if (found) return found;
+    }
+  }
+  return null;
 }
 function describe2(item) {
   const { label, verdict } = item;
@@ -1086,6 +1117,12 @@ function runCheck(args) {
   let presetName = "chat";
   const presetAt = args.indexOf("--preset");
   if (presetAt !== -1) {
+    if (asTrace) {
+      process.stderr.write(
+        "--preset applies to responses, not runs. --trace scores AGENT_LOOP, which has one threshold: use --max-agent-loop.\n"
+      );
+      return 2;
+    }
     const name = args[presetAt + 1];
     if (!name || !Object.hasOwn(presets, name)) {
       process.stderr.write(
@@ -1097,8 +1134,41 @@ function runCheck(args) {
     preset = presets[name];
     presetName = name;
   }
+  const traceOptions = {};
+  const numeric = [
+    ["--window", "window"],
+    ["--min-turns", "minTurns"],
+    ["--max-agent-loop", "maxAgentLoop"]
+  ];
+  const VALUED = /* @__PURE__ */ new Set(["--preset", "--ignore-tools", ...numeric.map(([flag]) => flag)]);
+  for (const [flag, key] of numeric) {
+    const at = args.indexOf(flag);
+    if (at === -1) continue;
+    const value = Number(args[at + 1]);
+    if (!Number.isFinite(value) || value < 0) {
+      process.stderr.write(`${flag} expects a number, got: ${args[at + 1] ?? "(nothing)"}
+`);
+      return 2;
+    }
+    traceOptions[key] = value;
+  }
+  const ignoreAt = args.indexOf("--ignore-tools");
+  if (ignoreAt !== -1) {
+    const names = (args[ignoreAt + 1] ?? "").split(",").map((n) => n.trim()).filter(Boolean);
+    if (names.length === 0) {
+      process.stderr.write("--ignore-tools expects a comma-separated list of tool names\n");
+      return 2;
+    }
+    traceOptions.ignoreTools = names;
+  }
+  const usedTraceFlag = [...VALUED].find((f) => f !== "--preset" && args.includes(f));
+  if (!asTrace && usedTraceFlag) {
+    process.stderr.write(`${usedTraceFlag} only applies with --trace
+`);
+    return 2;
+  }
   const files = args.filter(
-    (arg, i) => !arg.startsWith("--") && args[i - 1] !== "--preset"
+    (arg, i) => !arg.startsWith("--") && !VALUED.has(args[i - 1] ?? "")
   );
   return (async () => {
     const items = [];
@@ -1173,7 +1243,7 @@ function runCheck(args) {
     }
     const checked = asTrace ? traces.map(({ label, turns }) => ({
       label: `${label} (${turns.length} turns)`,
-      verdict: checkTrace(turns)
+      verdict: checkTrace(turns, traceOptions)
     })) : items.map(({ label, text }) => ({
       label,
       verdict: checkOutput(text, preset)
@@ -1209,6 +1279,13 @@ Options
   --preset <name>  chat | strictJson | longForm | lenient (default chat)
   --jsonl          read input as JSONL, one logged response per line
   --trace          read input as agent runs, one run per line
+
+With --trace, scoring the run instead of the response:
+  --ignore-tools <a,b>   tools whose calls are not compared \u2014 a job poller,
+                         a sleep, a clock. By shape those are a loop
+  --max-agent-loop <n>   cycle-coverage threshold (default 0.4)
+  --window <n>           trailing turns inspected (default 12)
+  --min-turns <n>        turns required before judging at all (default 4)
   --json           emit a verdict per response as JSONL, for calibrate
   --quiet          no per-response output; the exit code is the answer
 
@@ -1231,6 +1308,14 @@ a native turn, or the raw envelope from any adapter this package ships:
   [{"text":"Reading.","toolCalls":[{"name":"read_file","arguments":{"p":"a.ts"}}]}]
   {"run":"job-14","turns":[{"choices":[{"message":{"content":"..."}}]}]}
 
+The turns are found under "turns", "messages", "history", "conversation" or
+"steps" -- at the top level or one object down -- so a request body or a logged
+run needs no reshaping. Only the model's own messages are read: system, user
+and tool messages are skipped rather than counted as turns.
+
+  {"turns":[{"role":"assistant","content":"\u2026","tool_calls":[\u2026]},
+            {"role":"tool","content":"\u2026"}]}
+
 A file that parses whole as one array is read as a single run, so a run logged
 as one pretty-printed document works without reshaping.