@hellcoder/companion 0.96.0

package/server/recording-hub/compat-validator.ts

@@ -0,0 +1,284 @@
+/**
+ * Compatibility validator for recorded sessions.
+ *
+ * Compares a recording's browser output messages structurally to detect
+ * protocol drift. This catches changes when Claude Code or Codex update
+ * their message format.
+ */
+
+import type { Recording } from "../replay.js";
+import { filterEntries } from "../replay.js";
+
+// ─── Types ───────────────────────────────────────────────────────────────────
+
+export interface ProtocolDiff {
+  entryIndex: number;
+  expected: { type: string; [key: string]: unknown };
+  actual: { type: string; [key: string]: unknown } | null;
+  kind: "missing" | "extra" | "type_mismatch" | "field_mismatch";
+  details: string;
+}
+
+export interface ValidationResult {
+  compatible: boolean;
+  backendType: string;
+  totalMessages: number;
+  diffs: ProtocolDiff[];
+  messageTypeBreakdown: Record<string, { count: number; issues: number }>;
+}
+
+// Fields to ignore during comparison (they change between runs)
+const IGNORED_FIELDS = new Set([
+  "timestamp",
+  "ts",
+  "created_at",
+  "updated_at",
+  "session_id",
+  "uuid",
+  "id",
+  "request_id",
+  "duration_ms",
+  "duration_api_ms",
+  "cost_usd",
+  "total_cost_usd",
+  "api_tokens",
+  "input_tokens",
+  "output_tokens",
+  "cache_read_input_tokens",
+  "cache_creation_input_tokens",
+]);
+
+// ─── Validation ──────────────────────────────────────────────────────────────
+
+/**
+ * Validate a recording's structural consistency.
+ *
+ * Checks that browser messages have expected types, required fields are present,
+ * and message type distribution is reasonable.
+ */
+export function validateRecording(recording: Recording): ValidationResult {
+  const browserMessages = filterEntries(recording.entries, "out", "browser");
+  const diffs: ProtocolDiff[] = [];
+  const typeBreakdown: Record<string, { count: number; issues: number }> = {};
+
+  for (let i = 0; i < browserMessages.length; i++) {
+    const entry = browserMessages[i];
+    let parsed: Record<string, unknown>;
+
+    try {
+      parsed = JSON.parse(entry.raw);
+    } catch {
+      diffs.push({
+        entryIndex: i,
+        expected: { type: "valid_json" },
+        actual: null,
+        kind: "missing",
+        details: `Entry ${i}: unparseable JSON`,
+      });
+      continue;
+    }
+
+    const msgType = String(parsed.type || "unknown");
+
+    if (!typeBreakdown[msgType]) {
+      typeBreakdown[msgType] = { count: 0, issues: 0 };
+    }
+    typeBreakdown[msgType].count++;
+
+    // Validate required fields per message type
+    const issues = validateMessageStructure(msgType, parsed, i);
+    for (const issue of issues) {
+      diffs.push(issue);
+      typeBreakdown[msgType].issues++;
+    }
+  }
+
+  return {
+    compatible: diffs.length === 0,
+    backendType: recording.header.backend_type,
+    totalMessages: browserMessages.length,
+    diffs,
+    messageTypeBreakdown: typeBreakdown,
+  };
+}
+
+/**
+ * Compare two recordings structurally.
+ *
+ * Useful for verifying that replaying CLI input through the adapter produces
+ * the same browser output. Returns diffs where messages diverge.
+ */
+export function compareRecordings(
+  expected: Recording,
+  actual: { type: string; [key: string]: unknown }[],
+): ProtocolDiff[] {
+  const expectedMsgs = filterEntries(expected.entries, "out", "browser");
+  const diffs: ProtocolDiff[] = [];
+
+  const maxLen = Math.max(expectedMsgs.length, actual.length);
+
+  for (let i = 0; i < maxLen; i++) {
+    if (i >= expectedMsgs.length) {
+      diffs.push({
+        entryIndex: i,
+        expected: { type: "none" },
+        actual: actual[i],
+        kind: "extra",
+        details: `Extra message at index ${i}: type=${actual[i].type}`,
+      });
+      continue;
+    }
+
+    if (i >= actual.length) {
+      let expectedParsed: Record<string, unknown>;
+      try {
+        expectedParsed = JSON.parse(expectedMsgs[i].raw);
+      } catch {
+        expectedParsed = { type: "unparseable" };
+      }
+      diffs.push({
+        entryIndex: i,
+        expected: expectedParsed as { type: string },
+        actual: null,
+        kind: "missing",
+        details: `Missing message at index ${i}: expected type=${expectedParsed.type}`,
+      });
+      continue;
+    }
+
+    let expectedParsed: Record<string, unknown>;
+    try {
+      expectedParsed = JSON.parse(expectedMsgs[i].raw);
+    } catch {
+      diffs.push({
+        entryIndex: i,
+        expected: { type: "unparseable" },
+        actual: actual[i],
+        kind: "field_mismatch",
+        details: `Entry ${i}: expected message has unparseable JSON`,
+      });
+      continue;
+    }
+
+    const actualMsg = actual[i];
+
+    // Type must match
+    if (expectedParsed.type !== actualMsg.type) {
+      diffs.push({
+        entryIndex: i,
+        expected: expectedParsed as { type: string },
+        actual: actualMsg as { type: string },
+        kind: "type_mismatch",
+        details: `Type mismatch at index ${i}: expected=${expectedParsed.type}, actual=${actualMsg.type}`,
+      });
+      continue;
+    }
+
+    // Check for missing/extra top-level fields (excluding ignored fields)
+    const fieldDiffs = compareFields(expectedParsed, actualMsg, i);
+    diffs.push(...fieldDiffs);
+  }
+
+  return diffs;
+}
+
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+function validateMessageStructure(
+  type: string,
+  msg: Record<string, unknown>,
+  index: number,
+): ProtocolDiff[] {
+  const issues: ProtocolDiff[] = [];
+
+  // All messages must have a type field
+  if (!msg.type) {
+    issues.push({
+      entryIndex: index,
+      expected: { type: "any" },
+      actual: msg as { type: string },
+      kind: "field_mismatch",
+      details: `Entry ${index}: missing 'type' field`,
+    });
+  }
+
+  // Type-specific validation
+  switch (type) {
+    case "session_init":
+      if (!msg.session || typeof msg.session !== "object") {
+        issues.push({
+          entryIndex: index,
+          expected: { type, session: "object" },
+          actual: msg as { type: string },
+          kind: "field_mismatch",
+          details: `Entry ${index}: session_init missing 'session' object`,
+        });
+      }
+      break;
+
+    case "permission_request":
+      if (!msg.tool_name) {
+        issues.push({
+          entryIndex: index,
+          expected: { type, tool_name: "string" },
+          actual: msg as { type: string },
+          kind: "field_mismatch",
+          details: `Entry ${index}: permission_request missing 'tool_name'`,
+        });
+      }
+      break;
+
+    case "result":
+      if (!msg.subtype) {
+        issues.push({
+          entryIndex: index,
+          expected: { type, subtype: "string" },
+          actual: msg as { type: string },
+          kind: "field_mismatch",
+          details: `Entry ${index}: result missing 'subtype'`,
+        });
+      }
+      break;
+  }
+
+  return issues;
+}
+
+function compareFields(
+  expected: Record<string, unknown>,
+  actual: Record<string, unknown>,
+  index: number,
+): ProtocolDiff[] {
+  const diffs: ProtocolDiff[] = [];
+
+  const expectedKeys = Object.keys(expected).filter((k) => !IGNORED_FIELDS.has(k));
+  const actualKeys = Object.keys(actual).filter((k) => !IGNORED_FIELDS.has(k));
+
+  // Check for missing fields in actual
+  for (const key of expectedKeys) {
+    if (!(key in actual)) {
+      diffs.push({
+        entryIndex: index,
+        expected: expected as { type: string },
+        actual: actual as { type: string },
+        kind: "field_mismatch",
+        details: `Entry ${index}: missing field '${key}' in actual (type=${expected.type})`,
+      });
+    }
+  }
+
+  // Check for unexpected new fields in actual (informational, not necessarily a break)
+  for (const key of actualKeys) {
+    if (!(key in expected)) {
+      diffs.push({
+        entryIndex: index,
+        expected: expected as { type: string },
+        actual: actual as { type: string },
+        kind: "field_mismatch",
+        details: `Entry ${index}: new field '${key}' in actual (type=${actual.type})`,
+      });
+    }
+  }
+
+  return diffs;
+}

package/server/recording-hub/diagnostics.test.ts

@@ -0,0 +1,140 @@
+import { describe, it, expect } from "vitest";
+import { analyzeDisconnections, buildTimeline } from "./diagnostics.js";
+import type { Recording } from "../replay.js";
+import type { RecordingEntry } from "../recorder.js";
+
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+function makeRecording(entries: Partial<RecordingEntry>[]): Recording {
+  return {
+    header: {
+      _header: true as const,
+      version: 1 as const,
+      session_id: "test",
+      backend_type: "claude" as const,
+      started_at: 0,
+      cwd: "/",
+    },
+    entries: entries.map((e) => ({
+      ts: e.ts ?? 0,
+      dir: e.dir ?? "in",
+      raw: e.raw ?? "",
+      ch: e.ch ?? "cli",
+      ...(e.event ? { event: e.event } : {}),
+      ...(e.meta ? { meta: e.meta } : {}),
+    })) as RecordingEntry[],
+  };
+}
+
+// ─── Tests ───────────────────────────────────────────────────────────────────
+
+describe("diagnostics", () => {
+  describe("buildTimeline", () => {
+    it("extracts explicit lifecycle events", () => {
+      const recording = makeRecording([
+        { ts: 100, event: "ws_open", ch: "cli" },
+        { ts: 200, dir: "in", raw: '{"type":"system"}', ch: "cli" },
+        { ts: 500, event: "ws_close", ch: "cli", meta: { code: 1000 } },
+      ]);
+
+      const timeline = buildTimeline(recording);
+      expect(timeline).toHaveLength(2); // ws_open and ws_close (data messages aren't timeline events)
+      expect(timeline[0].event).toBe("ws_open");
+      expect(timeline[1].event).toBe("ws_close");
+      expect(timeline[1].detail).toContain("1000");
+    });
+
+    it("extracts cli_connected/disconnected from data messages", () => {
+      const recording = makeRecording([
+        { ts: 100, dir: "out", raw: JSON.stringify({ type: "cli_connected" }), ch: "browser" },
+        { ts: 500, dir: "out", raw: JSON.stringify({ type: "cli_disconnected" }), ch: "browser" },
+      ]);
+
+      const timeline = buildTimeline(recording);
+      expect(timeline).toHaveLength(2);
+      expect(timeline[0].event).toBe("cli_connected");
+      expect(timeline[1].event).toBe("cli_disconnected");
+    });
+
+    it("returns sorted timeline", () => {
+      const recording = makeRecording([
+        { ts: 500, event: "ws_close", ch: "cli" },
+        { ts: 100, event: "ws_open", ch: "cli" },
+      ]);
+
+      const timeline = buildTimeline(recording);
+      expect(timeline[0].ts).toBeLessThan(timeline[1].ts);
+    });
+  });
+
+  describe("analyzeDisconnections", () => {
+    it("reports no disconnections for clean session", () => {
+      const recording = makeRecording([
+        { ts: 100, dir: "in", raw: '{"type":"system"}', ch: "cli" },
+        { ts: 200, dir: "in", raw: '{"type":"assistant"}', ch: "cli" },
+        { ts: 300, dir: "in", raw: '{"type":"result"}', ch: "cli" },
+      ]);
+
+      const report = analyzeDisconnections(recording);
+      expect(report.totalDisconnections).toBe(0);
+      expect(report.dataGaps).toHaveLength(0);
+      expect(report.patterns).toContain("No disconnection issues detected in this recording.");
+    });
+
+    it("detects disconnection from lifecycle events", () => {
+      const recording = makeRecording([
+        { ts: 100, event: "ws_open", ch: "cli" },
+        { ts: 1000, event: "ws_close", ch: "cli" },
+        { ts: 5000, event: "ws_open", ch: "cli" },
+      ]);
+
+      const report = analyzeDisconnections(recording);
+      expect(report.totalDisconnections).toBe(1);
+      expect(report.disconnections[0].gapMs).toBe(4000);
+    });
+
+    it("detects data gaps in CLI messages", () => {
+      // Gap of 60s between two CLI messages (exceeds 30s threshold)
+      const recording = makeRecording([
+        { ts: 1000, dir: "in", raw: '{"type":"a"}', ch: "cli" },
+        { ts: 61000, dir: "in", raw: '{"type":"b"}', ch: "cli" },
+      ]);
+
+      const report = analyzeDisconnections(recording);
+      expect(report.dataGaps).toHaveLength(1);
+      expect(report.dataGaps[0].gapMs).toBe(60000);
+    });
+
+    it("reports asymmetric disconnection pattern (CLI-only)", () => {
+      const recording = makeRecording([
+        { ts: 100, dir: "out", raw: JSON.stringify({ type: "cli_disconnected" }), ch: "browser" },
+        { ts: 5000, dir: "out", raw: JSON.stringify({ type: "cli_connected" }), ch: "browser" },
+      ]);
+
+      const report = analyzeDisconnections(recording);
+      expect(report.patterns.some((p) => p.includes("CLI-side"))).toBe(true);
+    });
+
+    it("detects rapid reconnect cycling", () => {
+      // 3 rapid disconnect/reconnect cycles (< 5s gaps), spaced >20s apart so dedup doesn't merge them
+      const recording = makeRecording([
+        { ts: 1000, dir: "out", raw: JSON.stringify({ type: "cli_disconnected" }), ch: "browser" },
+        { ts: 2000, dir: "out", raw: JSON.stringify({ type: "cli_connected" }), ch: "browser" },
+        { ts: 25000, dir: "out", raw: JSON.stringify({ type: "cli_disconnected" }), ch: "browser" },
+        { ts: 26000, dir: "out", raw: JSON.stringify({ type: "cli_connected" }), ch: "browser" },
+        { ts: 50000, dir: "out", raw: JSON.stringify({ type: "cli_disconnected" }), ch: "browser" },
+        { ts: 51000, dir: "out", raw: JSON.stringify({ type: "cli_connected" }), ch: "browser" },
+      ]);
+
+      const report = analyzeDisconnections(recording);
+      expect(report.patterns.some((p) => p.includes("rapid reconnections"))).toBe(true);
+    });
+
+    it("includes session metadata in report", () => {
+      const recording = makeRecording([{ ts: 100, dir: "in", raw: "{}", ch: "cli" }]);
+      const report = analyzeDisconnections(recording);
+      expect(report.sessionId).toBe("test");
+      expect(report.backendType).toBe("claude");
+    });
+  });
+});