@monster-orange/den 0.0.0-stage → 0.1.0

package/src/pillars/journeys/report/friction.ts

@@ -0,0 +1,304 @@
+/**
+ * Journeys friction extraction — deterministic, harness-side, zero LLM
+ * involvement (plan: jrn-reporting packet, Approach §Reporting).
+ *
+ * Extracts friction_points[] from the persisted transcript (parsed JSONL
+ * lines), optional tool telemetry, and the SnapshotIndex. Friction is
+ * OBSERVABILITY ONLY: nothing in this module touches the verdict, and the
+ * report row's status is computed exclusively from the deterministic
+ * verifier's Verdict (pinned by tests in reporting.test.ts).
+ *
+ * Categories:
+ * - tool_error            — a tool call finished with status "error";
+ * - tool_retry            — a same-name call immediately after a failed one;
+ * - http_error            — a structured tool result carrying HTTP 4xx/5xx
+ *                           (both surfaces: scoped HTTP tool and browser);
+ * - error_page            — error-page markers in a browser page summary;
+ * - dead_end_navigation   — navigation back to an already-visited URL;
+ * - latency_outlier       — a tool call far slower than the per-step budget.
+ *
+ * Every point carries evidence {transcriptRef, snapshotPath?}: the 1-based
+ * transcript line and, for browser actions, the nearest SnapshotIndex frame
+ * (falling back to the closest earlier non-suppressed frame).
+ */
+
+import type { JourneyBudgets } from "../spec/schema.js";
+import type { SnapshotIndexEntry } from "../tools/snapshot-store.js";
+
+/**
+ * Markers that identify an error page in a browser tool's text page summary.
+ * Deliberately conservative: generic web-app failure copy only, so ordinary
+ * content ("error handling docs") does not false-positive.
+ */
+const ERROR_PAGE_MARKERS =
+  /something went wrong|an error occurred|page not found|internal server error|service unavailable/i;
+
+/**
+ * A step is a latency outlier when it exceeds this multiple of the per-step
+ * budget (timeout_s * 1000 / max_turns). Factor 2 because the per-step
+ * budget is an average — individual steps legitimately vary around it, and
+ * friction is advisory, so we only flag clear outliers.
+ */
+const LATENCY_OUTLIER_FACTOR = 2;
+
+/** Tool-name prefix that assigns an event to the browser surface. */
+const BROWSER_TOOL_PREFIX = "browser_";
+
+export type FrictionSurface = "browser" | "api";
+export type FrictionSeverity = "low" | "medium" | "high";
+export type FrictionCategory =
+  | "tool_error"
+  | "tool_retry"
+  | "http_error"
+  | "error_page"
+  | "dead_end_navigation"
+  | "latency_outlier";
+
+export interface FrictionEvidence {
+  /** Transcript position, e.g. "transcript.jsonl:12" (1-based line). */
+  transcriptRef: string;
+  /** Nearest snapshot frame, when one exists for the action. */
+  snapshotPath?: string;
+}
+
+export interface FrictionPoint {
+  /** 1-based transcript line of the triggering event. */
+  step: number;
+  surface: FrictionSurface;
+  severity: FrictionSeverity;
+  category: FrictionCategory;
+  evidence: FrictionEvidence;
+  description: string;
+}
+
+/** Per-call duration telemetry, keyed by the transcript's call_id. */
+export interface ToolCallTelemetry {
+  callId: string;
+  durationMs: number;
+}
+
+export interface FrictionInputs {
+  /** Parsed transcript JSONL lines, in persisted order. */
+  transcript: readonly unknown[];
+  snapshotIndex: readonly SnapshotIndexEntry[];
+  /** Spec budgets — the per-step latency baseline derives from these. */
+  budgets: JourneyBudgets;
+  toolTelemetry?: readonly ToolCallTelemetry[];
+}
+
+/** Structural shape of a terminal tool_call transcript line. */
+interface TerminalToolCall {
+  line: number;
+  callId?: string;
+  name: string;
+  status: "completed" | "error";
+  result?: unknown;
+}
+
+function asTerminalToolCall(
+  message: unknown,
+  line: number,
+): TerminalToolCall | undefined {
+  if (typeof message !== "object" || message === null) return undefined;
+  const candidate = message as {
+    type?: unknown;
+    call_id?: unknown;
+    name?: unknown;
+    status?: unknown;
+    result?: unknown;
+  };
+  if (candidate.type !== "tool_call") return undefined;
+  if (typeof candidate.name !== "string") return undefined;
+  if (candidate.status !== "completed" && candidate.status !== "error") {
+    return undefined;
+  }
+  return {
+    line,
+    callId:
+      typeof candidate.call_id === "string" ? candidate.call_id : undefined,
+    name: candidate.name,
+    status: candidate.status,
+    result: candidate.result,
+  };
+}
+
+function surfaceOf(call: TerminalToolCall): FrictionSurface {
+  return call.name.startsWith(BROWSER_TOOL_PREFIX) ? "browser" : "api";
+}
+
+function resultField(call: TerminalToolCall, field: string): unknown {
+  if (typeof call.result !== "object" || call.result === null) return undefined;
+  return (call.result as Record<string, unknown>)[field];
+}
+
+function httpStatusOf(call: TerminalToolCall): number | undefined {
+  const status = resultField(call, "status");
+  return typeof status === "number" ? status : undefined;
+}
+
+function pageSummaryOf(call: TerminalToolCall): string | undefined {
+  const summary = resultField(call, "summary");
+  if (typeof summary === "string") return summary;
+  return typeof call.result === "string" ? call.result : undefined;
+}
+
+/** Did this terminal call fail (tool error or HTTP >= 400 result)? */
+function callFailed(call: TerminalToolCall): boolean {
+  if (call.status === "error") return true;
+  const status = httpStatusOf(call);
+  return status !== undefined && status >= 400;
+}
+
+/**
+ * Nearest snapshot for the browser action at `ordinal` (1-based count of
+ * terminal browser_* calls so far): the entry with the matching seq when it
+ * has a persisted frame, else the closest earlier non-suppressed frame.
+ */
+function nearestSnapshotPath(
+  snapshotIndex: readonly SnapshotIndexEntry[],
+  ordinal: number,
+): string | undefined {
+  for (let seq = ordinal; seq >= 1; seq -= 1) {
+    const entry = snapshotIndex.find((candidate) => candidate.seq === seq);
+    if (entry?.path != null) return entry.path;
+  }
+  return undefined;
+}
+
+/**
+ * Extract deterministic friction points from one run's persisted artifacts.
+ * Output order follows transcript order; identical inputs always produce
+ * identical output.
+ */
+export function extractFrictionPoints(inputs: FrictionInputs): FrictionPoint[] {
+  const points: FrictionPoint[] = [];
+  const telemetryByCallId = new Map<string, number>(
+    (inputs.toolTelemetry ?? []).map((t) => [t.callId, t.durationMs]),
+  );
+  // Per-step baseline: the budgets guarantee max_turns > 0 (schema-enforced).
+  const stepBudgetMs =
+    (inputs.budgets.timeout_s * 1000) / inputs.budgets.max_turns;
+  const latencyThresholdMs = stepBudgetMs * LATENCY_OUTLIER_FACTOR;
+
+  const visitedUrls = new Set<string>();
+  let previousCall: TerminalToolCall | undefined;
+  let browserOrdinal = 0;
+
+  const push = (
+    call: TerminalToolCall,
+    category: FrictionCategory,
+    severity: FrictionSeverity,
+    description: string,
+  ): void => {
+    const evidence: FrictionEvidence = {
+      transcriptRef: `transcript.jsonl:${call.line}`,
+    };
+    if (surfaceOf(call) === "browser") {
+      const snapshotPath = nearestSnapshotPath(
+        inputs.snapshotIndex,
+        browserOrdinal,
+      );
+      if (snapshotPath !== undefined) evidence.snapshotPath = snapshotPath;
+    }
+    points.push({
+      step: call.line,
+      surface: surfaceOf(call),
+      severity,
+      category,
+      evidence,
+      description,
+    });
+  };
+
+  inputs.transcript.forEach((message, index) => {
+    const call = asTerminalToolCall(message, index + 1);
+    if (call === undefined) return;
+
+    if (surfaceOf(call) === "browser") browserOrdinal += 1;
+
+    if (call.status === "error") {
+      push(
+        call,
+        "tool_error",
+        "high",
+        `${call.name} failed: ${JSON.stringify(call.result ?? null)}`,
+      );
+    }
+
+    const httpStatus = httpStatusOf(call);
+    if (call.status === "completed" && httpStatus !== undefined) {
+      if (httpStatus >= 500) {
+        push(
+          call,
+          "http_error",
+          "high",
+          `${call.name} returned HTTP ${httpStatus}`,
+        );
+      } else if (httpStatus >= 400) {
+        push(
+          call,
+          "http_error",
+          "medium",
+          `${call.name} returned HTTP ${httpStatus}`,
+        );
+      }
+    }
+
+    if (call.status === "completed" && surfaceOf(call) === "browser") {
+      const summary = pageSummaryOf(call);
+      if (summary !== undefined && ERROR_PAGE_MARKERS.test(summary)) {
+        push(
+          call,
+          "error_page",
+          "medium",
+          `${call.name} landed on an error page: ${summary.slice(0, 200)}`,
+        );
+      }
+    }
+
+    if (
+      previousCall !== undefined &&
+      previousCall.name === call.name &&
+      callFailed(previousCall)
+    ) {
+      push(
+        call,
+        "tool_retry",
+        "low",
+        `${call.name} retried after a failed attempt`,
+      );
+    }
+
+    if (call.name === "browser_navigate" && call.status === "completed") {
+      const url = resultField(call, "url");
+      if (typeof url === "string") {
+        if (visitedUrls.has(url)) {
+          push(
+            call,
+            "dead_end_navigation",
+            "low",
+            `navigated back to already-visited ${url} (backtracking)`,
+          );
+        }
+        visitedUrls.add(url);
+      }
+    }
+
+    if (call.callId !== undefined) {
+      const durationMs = telemetryByCallId.get(call.callId);
+      if (durationMs !== undefined && durationMs > latencyThresholdMs) {
+        push(
+          call,
+          "latency_outlier",
+          "medium",
+          `${call.name} took ${durationMs}ms (>${Math.round(latencyThresholdMs)}ms, ` +
+            `${LATENCY_OUTLIER_FACTOR}x the per-step budget)`,
+        );
+      }
+    }
+
+    previousCall = call;
+  });
+
+  return points;
+}

package/src/pillars/journeys/report/junit.ts

@@ -0,0 +1,105 @@
+/**
+ * Journeys JUnit XML emitter (plan: jrn-reporting packet).
+ *
+ * Journeys are not Playwright-run, so no reporter emits JUnit for them —
+ * this is a small hand-rolled writer producing one <testcase> per journey
+ * under a single "journeys" <testsuite>. Failure detail (failure kind +
+ * assertion diagnostics) goes into <failure>; teardown warnings are
+ * NON-GATING and surface in <system-out>.
+ */
+
+import { mkdir, writeFile } from "fs/promises";
+import * as path from "path";
+
+/** Conventional output path (repo-relative), mirroring the smoke/api junit files under reports/. */
+export const JUNIT_OUTPUT_PATH = "reports/junit-journeys-results.xml";
+
+/** One journey's JUnit row, produced by finalizeJourneyReport. */
+export interface JourneysJUnitCase {
+  name: string;
+  environment: string;
+  durationMs: number;
+  passed: boolean;
+  /** RunOutcome failure kind, when the run itself failed. */
+  failureKind?: string;
+  /** Structured assertion diagnostics for the <failure> body. */
+  failureDetails: string[];
+  /** Teardown failures + unmanaged-mutation warnings (non-gating). */
+  teardownWarnings: string[];
+}
+
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+function renderTestCase(testCase: JourneysJUnitCase): string {
+  const timeS = testCase.durationMs / 1000;
+  const lines: string[] = [
+    `    <testcase name="${escapeXml(testCase.name)}" ` +
+      `classname="journeys.${escapeXml(testCase.environment)}" ` +
+      `time="${timeS}">`,
+  ];
+
+  if (!testCase.passed) {
+    const message =
+      testCase.failureKind !== undefined
+        ? `failure_kind=${testCase.failureKind}`
+        : "journey assertions failed";
+    const body = [message, ...testCase.failureDetails]
+      .map(escapeXml)
+      .join("\n");
+    lines.push(
+      `      <failure message="${escapeXml(message)}">${body}</failure>`,
+    );
+  }
+
+  if (testCase.teardownWarnings.length > 0) {
+    lines.push(
+      `      <system-out>${testCase.teardownWarnings
+        .map(escapeXml)
+        .join("\n")}</system-out>`,
+    );
+  }
+
+  lines.push("    </testcase>");
+  return lines.join("\n");
+}
+
+/** Render the full JUnit document for one run's journeys. */
+export function renderJourneysJUnitXml(
+  cases: readonly JourneysJUnitCase[],
+): string {
+  const failures = cases.filter((testCase) => !testCase.passed).length;
+  const totalTimeS =
+    cases.reduce((sum, testCase) => sum + testCase.durationMs, 0) / 1000;
+
+  return [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    `<testsuites tests="${cases.length}" failures="${failures}" time="${totalTimeS}">`,
+    `  <testsuite name="journeys" tests="${cases.length}" ` +
+      `failures="${failures}" time="${totalTimeS}">`,
+    ...cases.map(renderTestCase),
+    "  </testsuite>",
+    "</testsuites>",
+    "",
+  ].join("\n");
+}
+
+/**
+ * Write the JUnit document to disk (default: reports/junit-journeys-
+ * results.xml). Returns the resolved file path.
+ */
+export async function writeJourneysJUnitXml(
+  cases: readonly JourneysJUnitCase[],
+  filePath: string = JUNIT_OUTPUT_PATH,
+): Promise<string> {
+  const resolved = path.resolve(filePath);
+  await mkdir(path.dirname(resolved), { recursive: true });
+  await writeFile(resolved, renderJourneysJUnitXml(cases));
+  return resolved;
+}

package/src/pillars/journeys/report/render.ts

@@ -0,0 +1,273 @@
+/**
+ * Journeys per-run markdown report (plan: jrn-reporting packet, Approach
+ * §Browser snapshots in every report).
+ *
+ * Renders the human-facing run report: verdict summary, budget/usage
+ * numbers, the embedded snapshot gallery (image links RELATIVE to the
+ * report dir so the report stays portable), the friction section with
+ * evidence references, assertion diagnostics, and teardown warnings.
+ * Friction and teardown sections are informational — the verdict line
+ * reflects the deterministic verifier only.
+ */
+
+import * as path from "path";
+import type { JourneyEvent } from "../observability/events.js";
+import type { TeardownReport } from "../teardown.js";
+import type { SnapshotIndexEntry } from "../tools/snapshot-store.js";
+import type { CollectedJourneyReport } from "./results.js";
+
+function snapshotAlt(entry: SnapshotIndexEntry): string {
+  return `${String(entry.seq).padStart(3, "0")}-${entry.action}`;
+}
+
+function renderGallery(collected: CollectedJourneyReport): string[] {
+  if (collected.snapshotIndex.length === 0) {
+    return ["_No snapshots captured (api-surface journey)._", ""];
+  }
+  const lines: string[] = [];
+  for (const entry of collected.snapshotIndex) {
+    if (entry.captureError !== undefined) {
+      lines.push(
+        `⚠️ _Snapshot ${String(entry.seq).padStart(3, "0")} ` +
+          `(${entry.action}) capture FAILED — evidence missing: ` +
+          `${entry.captureError}_`,
+      );
+    } else if (entry.path === null) {
+      lines.push(
+        `_Snapshot ${String(entry.seq).padStart(3, "0")} (${entry.action}) ` +
+          `suppressed${entry.masked ? " — sensitive input masked" : ""}._`,
+      );
+    } else {
+      const relative = path.relative(collected.reportDir, entry.path);
+      lines.push(`![${snapshotAlt(entry)}](${relative})`);
+    }
+    lines.push("");
+  }
+  return lines;
+}
+
+function renderFriction(collected: CollectedJourneyReport): string[] {
+  if (collected.frictionPoints.length === 0) {
+    return ["_No friction points detected._", ""];
+  }
+  const lines: string[] = [];
+  for (const point of collected.frictionPoints) {
+    const snapshotSuffix =
+      point.evidence.snapshotPath !== undefined
+        ? ` (snapshot: ${path.relative(collected.reportDir, point.evidence.snapshotPath)})`
+        : "";
+    lines.push(
+      `- **${point.severity}** \`${point.category}\` @ ` +
+        `${point.evidence.transcriptRef} — ${point.description}${snapshotSuffix}`,
+    );
+  }
+  lines.push("");
+  return lines;
+}
+
+function renderAssertions(collected: CollectedJourneyReport): string[] {
+  if (collected.assertionResults.length === 0) {
+    return ["_No assertion diagnostics recorded._", ""];
+  }
+  const lines: string[] = [];
+  for (const result of collected.assertionResults) {
+    const marker = result.passed ? "✅" : "❌";
+    const detail = result.passed
+      ? ""
+      : ` — ${result.message ?? `expected ${JSON.stringify(result.expected)}, actual ${JSON.stringify(result.actual)}`}`;
+    lines.push(`- ${marker} \`${result.assertion.kind}\`${detail}`);
+  }
+  lines.push("");
+  return lines;
+}
+
+function renderTeardown(teardown: TeardownReport | undefined): string[] {
+  if (teardown === undefined) {
+    return ["_No teardown report (teardown did not run)._", ""];
+  }
+  const lines: string[] = [
+    `Attempted deletions: ${teardown.results.length}, ` +
+      `failed: ${teardown.failures.length}.`,
+    "",
+  ];
+  for (const failure of teardown.failures) {
+    lines.push(
+      `- ⚠️ delete failed for ${failure.entry.resourceType} ` +
+        `${failure.entry.id}: ${failure.entry.cleanup.method} ` +
+        `${failure.entry.cleanup.path}` +
+        (failure.status !== undefined ? ` (${failure.status})` : "") +
+        (failure.error !== undefined ? ` — ${failure.error}` : ""),
+    );
+  }
+  for (const warning of teardown.unmanagedWarnings) {
+    lines.push(
+      `- ⚠️ unmanaged mutation (never guessed-deleted): ` +
+        `${warning.method} ${warning.path} (${warning.status}, ` +
+        `source: ${warning.source})`,
+    );
+  }
+  for (const message of teardown.interceptionErrors) {
+    lines.push(
+      `- ⚠️ browser interception error (a creation may be untracked): ` +
+        message,
+    );
+  }
+  if (
+    teardown.failures.length > 0 ||
+    teardown.unmanagedWarnings.length > 0 ||
+    teardown.interceptionErrors.length > 0
+  ) {
+    lines.push("");
+  }
+  return lines;
+}
+
+function ts(event: JourneyEvent): string {
+  return `${(event.tMs / 1000).toFixed(1)}s`;
+}
+
+/**
+ * Condensed chronological timeline of what the agent actually did —
+ * tool calls (with results), bindings, denials, thinking/assistant text.
+ * This is the section that turns a black-box run into a readable story.
+ */
+function renderTimeline(collected: CollectedJourneyReport): string[] {
+  if (collected.events.length === 0) {
+    return ["_No events recorded._", ""];
+  }
+  const lines: string[] = [];
+  for (const event of collected.events) {
+    switch (event.kind) {
+      case "turn_started":
+        lines.push(`- \`${ts(event)}\` **turn ${event.turn}**`);
+        break;
+      case "tool_call": {
+        const icon =
+          event.status === "completed"
+            ? "✓"
+            : event.status === "error"
+              ? "✗"
+              : "…";
+        const summary = event.summary ? ` \`${event.summary}\`` : "";
+        const result = event.result ? ` → ${event.result}` : "";
+        lines.push(
+          `- \`${ts(event)}\` ${icon} ${event.tool}${summary}${result}`,
+        );
+        break;
+      }
+      case "ledger_binding":
+        lines.push(
+          `- \`${ts(event)}\` 🔗 bound **${event.resourceType}** = ` +
+            `\`${event.id}\` (${event.method} ${event.pathValue} → ${event.status})`,
+        );
+        break;
+      case "unmanaged_mutation":
+        lines.push(
+          `- \`${ts(event)}\` ⚠️ unmanaged ${event.method} ` +
+            `${event.pathValue} → ${event.status}`,
+        );
+        break;
+      case "interception_error":
+        lines.push(
+          `- \`${ts(event)}\` ⚠️ interception error: ${event.message}`,
+        );
+        break;
+      case "agent_error":
+        lines.push(
+          `- \`${ts(event)}\` ✗ **agent error**` +
+            (event.code ? ` [${event.code}]` : "") +
+            `: ${event.message}`,
+        );
+        break;
+      case "assistant_text":
+        lines.push(`- \`${ts(event)}\` 🗣 ${truncate(event.text, 160)}`);
+        break;
+      default:
+        // thinking / turn_ended / run_started / run_ended are omitted from
+        // the human timeline (they live in events.jsonl); keeps it readable.
+        break;
+    }
+  }
+  lines.push("");
+  return lines;
+}
+
+/**
+ * Resource bindings — the direct answer to "did {<type>_id} bind, and to
+ * what?". Empty for a journey that created nothing (or failed before it).
+ */
+function renderBindings(collected: CollectedJourneyReport): string[] {
+  const bindings = collected.events.filter(
+    (e): e is Extract<JourneyEvent, { kind: "ledger_binding" }> =>
+      e.kind === "ledger_binding",
+  );
+  if (bindings.length === 0) {
+    return [
+      "_No resources bound (nothing created, or created pre-binding)._",
+      "",
+    ];
+  }
+  const lines = bindings.map(
+    (b) =>
+      `- **${b.resourceType}** = \`${b.id}\` ` +
+      `(${b.method} ${b.pathValue} → ${b.status}, via ${b.source})`,
+  );
+  lines.push("");
+  return lines;
+}
+
+function truncate(text: string, max: number): string {
+  const flat = text.replace(/\s+/g, " ").trim();
+  return flat.length <= max ? flat : `${flat.slice(0, max - 1)}…`;
+}
+
+/** Render the full per-run markdown report. */
+export function renderJourneyReportMarkdown(
+  collected: CollectedJourneyReport,
+  teardown?: TeardownReport,
+): string {
+  const verdictLine = collected.status === "passed" ? "✅ passed" : "❌ failed";
+
+  return [
+    `# Journey report — ${collected.journeyName}`,
+    "",
+    `- **Run id**: ${collected.runId}`,
+    `- **Environment**: ${collected.environment}`,
+    `- **Surface**: ${collected.surface}`,
+    `- **Verdict**: ${verdictLine}` +
+      (collected.failureKind !== undefined
+        ? ` (failure_kind: ${collected.failureKind})`
+        : ""),
+    `- **Duration**: ${collected.durationMs}ms`,
+    `- **Tokens used**: ${collected.usageTotalTokens ?? "unreported"} / ` +
+      `${collected.budgetConsumption.maxTotalTokens} budgeted`,
+    `- **Budgets**: max_turns=${collected.budgetConsumption.maxTurns}, ` +
+      `timeout_s=${collected.budgetConsumption.timeoutS}`,
+    `- **Transcript**: ${path.basename(collected.transcriptPath)}`,
+    `- **Events**: ${collected.eventsFile}`,
+    `- **Snapshots**: ${collected.snapshotCount} in ` +
+      `${path.relative(collected.reportDir, collected.snapshotsDir) || "."}/`,
+    ...(collected.agentError !== undefined
+      ? [`- **Agent error**: ${collected.agentError}`]
+      : []),
+    "",
+    "## Timeline",
+    "",
+    ...renderTimeline(collected),
+    "## Bindings",
+    "",
+    ...renderBindings(collected),
+    "## Assertions",
+    "",
+    ...renderAssertions(collected),
+    "## Snapshots",
+    "",
+    ...renderGallery(collected),
+    "## Friction",
+    "",
+    ...renderFriction(collected),
+    "## Teardown",
+    "",
+    ...renderTeardown(teardown),
+  ].join("\n");
+}