openhermes 4.9.2 → 4.12.1

package/harness/lib/routing/routing.test.ts

@@ -0,0 +1,195 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { consumeRouteGuidance, parseSkillFrontmatter, resolveRoute } from "./index.ts";
+
+describe("routing frontmatter", () => {
+  it("parses scalar route values", () => {
+    const parsed = parseSkillFrontmatter(`---
+name: oh-test
+route:
+  pass: oh-builder
+  fail: oh-review
+  blocker: surface
+---`);
+
+    assert.ok(parsed);
+    assert.deepEqual(parsed.route, {
+      pass: ["oh-builder"],
+      fail: ["oh-review"],
+      blocker: ["surface"],
+    });
+  });
+
+  it("parses yaml route lists", () => {
+    const parsed = parseSkillFrontmatter(`---
+name: oh-test
+route:
+  pass:
+    - oh-gauntlet
+    - oh-ship
+  fail:
+    - oh-builder
+  blocker: surface
+---`);
+
+    assert.ok(parsed);
+    assert.deepEqual(parsed.route, {
+      pass: ["oh-gauntlet", "oh-ship"],
+      fail: ["oh-builder"],
+      blocker: ["surface"],
+    });
+  });
+
+  it("parses inline route arrays", () => {
+    const parsed = parseSkillFrontmatter(`---
+name: oh-test
+route:
+  pass: [oh-gauntlet, oh-ship]
+  fail: [surface, oh-expert]
+  blocker: surface
+---`);
+
+    assert.ok(parsed);
+    assert.deepEqual(parsed.route, {
+      pass: ["oh-gauntlet", "oh-ship"],
+      fail: ["surface", "oh-expert"],
+      blocker: ["surface"],
+    });
+  });
+});
+
+describe("resolveRoute", () => {
+  const route = {
+    pass: ["oh-gauntlet", "oh-ship"],
+    fail: ["oh-builder"],
+    blocker: ["surface"],
+  };
+
+  it("defaults to the first candidate for an outcome", () => {
+    const resolved = resolveRoute(route, { outcome: "pass" });
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      candidates: ["oh-gauntlet", "oh-ship"],
+      selected: "oh-gauntlet",
+      reason: 'Selected first declared route for outcome "pass".',
+    });
+  });
+
+  it("prefers an evidence target when it matches a candidate", () => {
+    const resolved = resolveRoute(route, { outcome: "pass", target: "oh-ship" });
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      candidates: ["oh-gauntlet", "oh-ship"],
+      selected: "oh-ship",
+      reason: 'Selected "oh-ship" from output evidence.',
+    });
+  });
+
+  it("prefers oh-ship for verified done ship work", () => {
+    const resolved = resolveRoute(route, {
+      outcome: "pass",
+      verification: "verified",
+      action: "done",
+      work: "ship",
+    });
+
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      verification: "verified",
+      action: "done",
+      work: "ship",
+      candidates: ["oh-gauntlet", "oh-ship"],
+      selected: "oh-ship",
+      reason: 'Selected "oh-ship" for verified ship-ready work.',
+    });
+  });
+
+  it("prefers oh-gauntlet for unverified work", () => {
+    const resolved = resolveRoute(route, {
+      outcome: "pass",
+      verification: "unverified",
+      action: "done",
+      work: "ship",
+    });
+
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      verification: "unverified",
+      action: "done",
+      work: "ship",
+      candidates: ["oh-gauntlet", "oh-ship"],
+      selected: "oh-gauntlet",
+      reason: 'Selected "oh-gauntlet" because work is still unverified.',
+    });
+  });
+
+  it("prefers oh-gauntlet for verify work", () => {
+    const resolved = resolveRoute(route, {
+      outcome: "pass",
+      verification: "verified",
+      action: "done",
+      work: "verify",
+    });
+
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      verification: "verified",
+      action: "done",
+      work: "verify",
+      candidates: ["oh-gauntlet", "oh-ship"],
+      selected: "oh-gauntlet",
+      reason: 'Selected "oh-gauntlet" for verification work.',
+    });
+  });
+
+  it("prefers oh-builder for fixable implementation work", () => {
+    const resolved = resolveRoute({
+      pass: ["oh-gauntlet", "oh-ship", "oh-builder"],
+      fail: ["oh-builder"],
+      blocker: ["surface"],
+    }, {
+      outcome: "pass",
+      verification: "verified",
+      action: "fixable",
+      work: "implement",
+    });
+
+    assert.deepEqual(resolved, {
+      outcome: "pass",
+      verification: "verified",
+      action: "fixable",
+      work: "implement",
+      candidates: ["oh-gauntlet", "oh-ship", "oh-builder"],
+      selected: "oh-builder",
+      reason: 'Selected "oh-builder" for fixable implementation work.',
+    });
+  });
+});
+
+describe("consumeRouteGuidance", () => {
+  it("promotes a selected route into an explicit next-route instruction", () => {
+    const consumed = consumeRouteGuidance([
+      "Review complete",
+      'ROUTE_GUIDANCE: {"outcome":"pass","candidates":["oh-gauntlet","oh-ship"],"selected":"oh-ship","reason":"Selected \\\"oh-ship\\\" from output evidence."}',
+    ].join("\n"));
+
+    assert.equal(consumed.selected, "oh-ship");
+    assert.match(consumed.output, /NEXT_ROUTE: oh-ship/);
+  });
+
+  it("leaves output unchanged when no route guidance is present", () => {
+    const output = "plain output";
+    const consumed = consumeRouteGuidance(output);
+
+    assert.equal(consumed.selected, null);
+    assert.equal(consumed.output, output);
+  });
+
+  it("ignores malformed route guidance safely", () => {
+    const output = 'Review complete\nROUTE_GUIDANCE: {"selected":42}';
+    const consumed = consumeRouteGuidance(output);
+
+    assert.equal(consumed.selected, null);
+    assert.equal(consumed.output, output);
+  });
+});

package/harness/lib/routing/skill-frontmatter.ts

@@ -0,0 +1,125 @@
+import fs from "node:fs";
+import type { RouteOutcome, SkillRouteMap, SkillRoutingFrontmatter } from "./types.ts";
+
+const EMPTY_ROUTES: SkillRouteMap = {
+  pass: [],
+  fail: [],
+  blocker: [],
+};
+
+function stripQuotes(value: string): string {
+  return value.trim().replace(/^['"]|['"]$/g, "");
+}
+
+function parseRouteValue(value: string): string[] {
+  const trimmed = value.trim();
+  if (!trimmed) return [];
+
+  if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
+    return trimmed
+      .slice(1, -1)
+      .split(",")
+      .map((entry) => stripQuotes(entry))
+      .filter(Boolean);
+  }
+
+  return [stripQuotes(trimmed)].filter(Boolean);
+}
+
+function isRouteOutcome(value: string): value is RouteOutcome {
+  return value === "pass" || value === "fail" || value === "blocker";
+}
+
+export function extractFrontmatter(source: string): string | null {
+  const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
+  return match?.[1] ?? null;
+}
+
+export function parseSkillFrontmatter(source: string): SkillRoutingFrontmatter | null {
+  const rawFrontmatter = extractFrontmatter(source);
+  if (!rawFrontmatter) return null;
+
+  const route: SkillRouteMap = {
+    pass: [],
+    fail: [],
+    blocker: [],
+  };
+
+  let name: string | undefined;
+  let description: string | undefined;
+  let tier: string | undefined;
+  let inRouteBlock = false;
+  let activeRouteKey: RouteOutcome | null = null;
+
+  for (const rawLine of rawFrontmatter.split(/\r?\n/)) {
+    const line = rawLine.trimEnd();
+    const trimmed = line.trim();
+
+    if (!trimmed) continue;
+
+    if (/^route:\s*$/.test(trimmed)) {
+      inRouteBlock = true;
+      activeRouteKey = null;
+      continue;
+    }
+
+    if (inRouteBlock) {
+      const routeMatch = line.match(/^\s{2,}(pass|fail|blocker):\s*(.*)$/);
+      if (routeMatch && isRouteOutcome(routeMatch[1])) {
+        activeRouteKey = routeMatch[1];
+        route[activeRouteKey].push(...parseRouteValue(routeMatch[2]));
+        continue;
+      }
+
+      const listMatch = line.match(/^\s{4,}-\s+(.+)$/);
+      if (listMatch && activeRouteKey) {
+        route[activeRouteKey].push(stripQuotes(listMatch[1]));
+        continue;
+      }
+
+      if (!/^\s/.test(line)) {
+        inRouteBlock = false;
+        activeRouteKey = null;
+      } else {
+        activeRouteKey = null;
+        continue;
+      }
+    }
+
+    const fieldMatch = line.match(/^(name|description|tier):\s*(.+)$/);
+    if (!fieldMatch) continue;
+
+    const value = stripQuotes(fieldMatch[2]);
+    switch (fieldMatch[1]) {
+      case "name":
+        name = value;
+        break;
+      case "description":
+        description = value;
+        break;
+      case "tier":
+        tier = value;
+        break;
+    }
+  }
+
+  return {
+    name,
+    description,
+    tier,
+    route,
+  };
+}
+
+export function readSkillFrontmatter(skillFilePath: string): SkillRoutingFrontmatter | null {
+  if (!fs.existsSync(skillFilePath)) return null;
+  return parseSkillFrontmatter(fs.readFileSync(skillFilePath, "utf8"));
+}
+
+export function emptySkillRoutes(): SkillRouteMap {
+  return {
+    pass: [...EMPTY_ROUTES.pass],
+    fail: [...EMPTY_ROUTES.fail],
+    blocker: [...EMPTY_ROUTES.blocker],
+  };
+}

package/harness/lib/routing/types.ts

@@ -0,0 +1,52 @@
+export const ROUTE_OUTCOMES = ["pass", "fail", "blocker"] as const;
+export const ROUTE_VERIFICATIONS = ["verified", "unverified"] as const;
+export const ROUTE_ACTIONS = ["done", "fixable", "needs-context", "blocked"] as const;
+export const ROUTE_WORK_TYPES = ["implement", "verify", "ship", "diagnose", "surface"] as const;
+
+export type RouteOutcome = (typeof ROUTE_OUTCOMES)[number];
+export type RouteVerification = (typeof ROUTE_VERIFICATIONS)[number];
+export type RouteAction = (typeof ROUTE_ACTIONS)[number];
+export type RouteWork = (typeof ROUTE_WORK_TYPES)[number];
+
+export interface SkillRouteMap {
+  pass: string[];
+  fail: string[];
+  blocker: string[];
+}
+
+export interface SkillRoutingFrontmatter {
+  name?: string;
+  description?: string;
+  tier?: string;
+  route: SkillRouteMap;
+}
+
+export interface RouteEvidence {
+  outcome: RouteOutcome;
+  verification?: RouteVerification;
+  action?: RouteAction;
+  work?: RouteWork;
+  target?: string;
+  reason?: string;
+}
+
+export interface RouteResolution {
+  outcome: RouteOutcome;
+  verification?: RouteVerification;
+  action?: RouteAction;
+  work?: RouteWork;
+  candidates: string[];
+  selected: string | null;
+  reason: string;
+}
+
+export interface RuntimeRouteDecision {
+  selected: string;
+  source: "next_route" | "route_guidance";
+  outcome?: RouteOutcome;
+  verification?: RouteVerification;
+  action?: RouteAction;
+  work?: RouteWork;
+  candidates?: string[];
+  reason?: string;
+}

package/harness/lib/sanity/anomaly-tracker.ts

@@ -0,0 +1,127 @@
+// ---------------------------------------------------------------------------
+// AnomalyTracker — singleton that tracks consecutive anomaly records per session
+// ---------------------------------------------------------------------------
+
+import type { AnomalyRecord, AnomalyTrackerConfig, SanityResult } from "./interfaces.ts";
+
+const DEFAULT_CONFIG: AnomalyTrackerConfig = {
+  maxConsecutiveAnomalies: 2,
+  escalationMessage: "recovery: compact context",
+};
+
+export class AnomalyTracker {
+  private static instance: AnomalyTracker;
+
+  private records = new Map<string, AnomalyRecord>();
+  private config: AnomalyTrackerConfig;
+
+  private constructor(config?: Partial<AnomalyTrackerConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /** Get the singleton instance. */
+  static getInstance(config?: Partial<AnomalyTrackerConfig>): AnomalyTracker {
+    if (!AnomalyTracker.instance) {
+      AnomalyTracker.instance = new AnomalyTracker(config);
+    }
+    return AnomalyTracker.instance;
+  }
+
+  /**
+   * Record a sanity result for a session.
+   * If unhealthy: increments consecutive count, updates reason/timestamp.
+   * If healthy: resets consecutive count to 0.
+   * Returns tracking info including whether escalation is needed.
+   */
+  record(
+    sessionId: string,
+    result: SanityResult,
+  ): {
+    shouldEscalate: boolean;
+    consecutiveAnomalies: number;
+    recoveryMessage?: string;
+  } {
+    const existing = this.records.get(sessionId);
+
+    if (!result.isHealthy) {
+      const count = (existing?.count ?? 0) + 1;
+      this.records.set(sessionId, {
+        sessionId,
+        count,
+        lastReason: result.reason ?? "Unknown anomaly",
+        lastTimestamp: Date.now(),
+      });
+
+      const shouldEscalate = count >= this.config.maxConsecutiveAnomalies;
+
+      return {
+        shouldEscalate,
+        consecutiveAnomalies: count,
+        recoveryMessage: shouldEscalate ? this.config.escalationMessage : undefined,
+      };
+    }
+
+    // Healthy output — reset counter
+    if (existing) {
+      this.records.set(sessionId, {
+        ...existing,
+        count: 0,
+        lastReason: "reset on healthy output",
+        lastTimestamp: Date.now(),
+      });
+    }
+
+    return {
+      shouldEscalate: false,
+      consecutiveAnomalies: 0,
+    };
+  }
+
+  /** Get the current anomaly record for a session. */
+  getRecord(sessionId: string): AnomalyRecord | undefined {
+    return this.records.get(sessionId);
+  }
+
+  /** Clear anomaly record for a specific session. */
+  clearSession(sessionId: string): void {
+    this.records.delete(sessionId);
+  }
+
+  /** Reset all tracking state (useful in tests). */
+  resetAll(): void {
+    this.records.clear();
+  }
+
+  /** Get the current config. */
+  getConfig(): AnomalyTrackerConfig {
+    return { ...this.config };
+  }
+
+  /** Update config at runtime. */
+  setConfig(config: Partial<AnomalyTrackerConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+
+  // ── Cross-invocation identical output detection ────────────────
+
+  private lastOutput: string | null = null;
+  private identicalOutputCount = 0;
+  readonly MAX_IDENTICAL_OUTPUTS = 3;
+
+  /**
+   * Track output for repeated identical content.
+   * Returns true if output should be flagged as degenerate.
+   */
+  trackOutput(text: string): boolean {
+    if (text === this.lastOutput) {
+      this.identicalOutputCount++;
+      if (this.identicalOutputCount >= this.MAX_IDENTICAL_OUTPUTS) {
+        return true; // Flagged — repeated identical output
+      }
+    } else {
+      this.identicalOutputCount = 0;
+    }
+    this.lastOutput = text;
+    return false;
+  }
+}

package/harness/lib/sanity/checker.ts

@@ -0,0 +1,189 @@
+// ---------------------------------------------------------------------------
+// Output Sanity Checker — detect LLM output degeneration patterns
+// ---------------------------------------------------------------------------
+
+import type { SanityResult } from "./interfaces.ts";
+import { AnomalyTracker } from "./anomaly-tracker.ts";
+
+/**
+ * Check a text string for output degeneration patterns.
+ * Returns unhealthy with severity + reason on first matching pattern.
+ * Returns healthy if no pattern matches.
+ *
+ * Check ordering: all critical-severity checks first (most specific first),
+ * then warning-severity checks. This ensures the most actionable, severe
+ * issues are reported before mild ones.
+ *
+ * Accepts an optional AnomalyTracker for cross-invocation dedup detection.
+ */
+export function checkOutputSanity(
+  text: unknown,
+  anomalyTracker?: AnomalyTracker,
+): SanityResult {
+  if (typeof text !== "string") {
+    return {
+      isHealthy: false,
+      severity: "critical",
+      reason: "Output is not a string (possibly undefined/null)",
+      patternName: "empty_output",
+    };
+  }
+  if (text.length === 0) {
+    return {
+      isHealthy: false,
+      severity: "warning",
+      reason: "Output is an empty string",
+      patternName: "empty_output",
+    };
+  }
+
+  // ═══════════════════════════════════════════════════════════════════
+  // CRITICAL checks — severe degeneration
+  // ═══════════════════════════════════════════════════════════════════
+
+  // ── 1. Single character repetition ──────────────────────────────
+  // 16+ consecutive identical characters
+  const singleCharMatch = text.match(/(.)\1{15,}/);
+  if (singleCharMatch) {
+    return {
+      isHealthy: false,
+      severity: "critical",
+      reason: `Single character repetition detected: "${singleCharMatch[0].slice(0, 20)}..."`,
+      patternName: "single_char_repetition",
+    };
+  }
+
+  // ── 2. Short pattern loop ───────────────────────────────────────
+  // 9+ repetitions of a 2-6 character sequence
+  const patternLoopMatch = text.match(/(.{2,6})\1{8,}/);
+  if (patternLoopMatch) {
+    return {
+      isHealthy: false,
+      severity: "critical",
+      reason: `Pattern loop detected: "${patternLoopMatch[0].slice(0, 30)}..."`,
+      patternName: "pattern_loop",
+    };
+  }
+
+  // ── 3. Excessive box/block drawing characters ───────────────────
+  // Unicode box drawing, block elements, and Braille patterns
+  const boxDrawChars = text.match(/[\u2500-\u257f\u2580-\u259f\u2800-\u28ff]/g);
+  if (boxDrawChars && boxDrawChars.length > 100) {
+    const ratio = boxDrawChars.length / text.length;
+    if (ratio > 0.3) {
+      return {
+        isHealthy: false,
+        severity: "critical",
+        reason: `Visual gibberish detected: ${boxDrawChars.length} box/block chars (${(ratio * 100).toFixed(1)}% of output)`,
+        patternName: "visual_gibberish",
+      };
+    }
+  }
+
+  // ── 4. CJK character spam ─────────────────────────────────────
+  // Lots of CJK characters with very few unique ones
+  const cjkChars = text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g);
+  if (cjkChars && cjkChars.length > 200) {
+    const uniqueCjk = new Set(cjkChars).size;
+    if (uniqueCjk < 10 && cjkChars.length / uniqueCjk > 20) {
+      return {
+        isHealthy: false,
+        severity: "critical",
+        reason: `CJK character spam detected: ${cjkChars.length} chars, ${uniqueCjk} unique (ratio ${(cjkChars.length / uniqueCjk).toFixed(0)})`,
+        patternName: "cjk_spam",
+      };
+    }
+  }
+
+  // ── 5. Low character diversity ────────────────────────────────
+  // General catch-all for text with very few distinct characters
+  if (text.length > 200) {
+    const cleanText = text.replace(/\s/g, "");
+    if (cleanText.length > 0) {
+      const uniqueChars = new Set(cleanText).size;
+      const diversity = uniqueChars / cleanText.length;
+      if (diversity < 0.02) {
+        return {
+          isHealthy: false,
+          severity: "critical",
+          reason: `Low information density: ${uniqueChars} unique chars out of ${cleanText.length} (ratio ${diversity.toFixed(4)} < 0.02)`,
+          patternName: "low_diversity",
+        };
+      }
+    }
+  }
+
+  // ═══════════════════════════════════════════════════════════════════
+  // WARNING checks — mild or context-dependent issues
+  // ═══════════════════════════════════════════════════════════════════
+
+  // ── 6. Excessive JSON/error stack lines ─────────────────────────
+  const lines = text.split(/\r?\n/);
+  let errorStackLineCount = 0;
+  const repetitionLines: string[] = [];
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (line.includes("Error:") || trimmed.startsWith("at ") || line.includes("Exception:")) {
+      errorStackLineCount++;
+    }
+
+    if (trimmed.length > 10) {
+      repetitionLines.push(line);
+    }
+  }
+
+  if (errorStackLineCount > 5) {
+    return {
+      isHealthy: false,
+      severity: "warning",
+      reason: `Error stack bleed detected: ${errorStackLineCount} error/stack lines`,
+      patternName: "error_stack_bleed",
+    };
+  }
+
+  // ── 7. Line-by-line repetition ──────────────────────────────────
+  if (repetitionLines.length > 10) {
+    const uniqueLines = new Set(repetitionLines);
+    if (uniqueLines.size < repetitionLines.length * 0.2) {
+      return {
+        isHealthy: false,
+        severity: "warning",
+        reason: `Excessive line repetition: ${uniqueLines.size} unique lines out of ${repetitionLines.length} (${(uniqueLines.size / repetitionLines.length * 100).toFixed(0)}% unique)`,
+        patternName: "line_repetition",
+      };
+    }
+  }
+
+  // ── 8. Empty/tiny output ────────────────────────────────────────
+  // Only flag if the entire output is small enough to be suspicious
+  // (exclude common status messages like "ok", "done")
+  if (text.length < 50 && text.length > 0) {
+    const minimalWords = ["ok", "done", "yes", "no", "passed", "failed", "error", "null", "undefined", "true", "false"];
+    const trimmed = text.trim().toLowerCase();
+    if (!minimalWords.includes(trimmed) && !/^[\d.]+$/.test(trimmed)) {
+      return {
+        isHealthy: false,
+        severity: "warning",
+        reason: `Output too short: ${text.length} characters`,
+        patternName: "output_too_short",
+      };
+    }
+  }
+
+  // ── 9. Cross-invocation dedup check ────────────────────────────────
+  if (anomalyTracker) {
+    const isRepeated = anomalyTracker.trackOutput(text);
+    if (isRepeated) {
+      return {
+        isHealthy: false,
+        severity: "warning",
+        reason: `Output identical to previous ${anomalyTracker.MAX_IDENTICAL_OUTPUTS} invocations`,
+        patternName: "repeated_identical_output",
+      };
+    }
+  }
+
+  // No pattern matched — healthy
+  return { isHealthy: true, severity: "ok" };
+}

package/harness/lib/sanity/index.ts

@@ -0,0 +1,13 @@
+// ---------------------------------------------------------------------------
+// Sanity Checker module — barrel export
+// ---------------------------------------------------------------------------
+
+export type {
+  Severity,
+  SanityResult,
+  AnomalyRecord,
+  AnomalyTrackerConfig,
+} from "./interfaces.ts";
+
+export { checkOutputSanity } from "./checker.ts";
+export { AnomalyTracker } from "./anomaly-tracker.ts";