@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/tui/transcript.tsx

@@ -0,0 +1,344 @@
+import path from "node:path"
+import { SyntaxStyle } from "@opentui/core"
+
+import type { CodeResult, RunStatus } from "../types.ts"
+import { CARD_MARK, type Card } from "../card.ts"
+import { counts } from "../suite/report.ts"
+import { suiteExitCode, type SuiteResult } from "../suite/run.ts"
+import { STATUS_MARK, shortPath, type RunSummary } from "../summary.ts"
+import { BAR } from "./components.tsx"
+import { formatDuration, formatTokens, theme } from "./theme.ts"
+import { clip } from "./text.ts"
+
+/** The transcript: its lines, and the cards for a run, a suite, a tool result, and what a request produced. */
+
+export type Line = {
+  id: number
+  kind: "sys" | "user" | "thought" | "timing" | "memory" | "tool" | "file" | "code" | "ok" | "err" | "summary" | "suitesummary" | "reply" | "card"
+  text: string
+  summary?: RunSummary
+  suite?: SuiteResult
+  card?: Card
+  outcome?: "done" | "failed" | "needs_input"
+}
+
+/** What one request produced: browser runs with their files, suites, and the tests the coding agent wrote. */
+export type Produced = {
+  runs: { runDir: string; status: RunStatus; files: string[] }[]
+  suites: { html: string; junit: string }[]
+  tests: { runDir: string; result?: CodeResult; error?: string }[]
+}
+
+export const emptyProduced = (): Produced => ({ runs: [], suites: [], tests: [] })
+
+/** Whether passed runs become tests this session, and why not when they don't. */
+export type TestsSetting = { on: boolean; why: string }
+
+export const TEST_VERDICT: Record<CodeResult["status"], [string, string]> = {
+  passed: ["✓", "passed when ForgeQE ran it"],
+  failed: ["✗", "still fails when ForgeQE runs it"],
+  stopped: ["■", "stopped before it passed"],
+  blocked: ["✗", "blocked: the app stopped the test browser"],
+}
+
+/** The closing card of a request: the report of each run, and the test written for it, or why none was. */
+export function producedCard(p: Produced, tests: TestsSetting): Card | null {
+  if (!p.runs.length && !p.suites.length && !p.tests.length) return null
+  const show = shortPath
+  const row = (mark: string, label: string, value: string) => `${mark.padEnd(2)}${label.padEnd(10)}${value}`
+  const body: string[] = []
+  const testRows = (t: Produced["tests"][number]) => {
+    if (!t.result) return [row("✗", "Test", `not written: ${t.error || "the coding agent failed"}`)]
+    const r = t.result
+    const [mark, verdict] = TEST_VERDICT[r.status]
+    return [
+      row(mark, "Test", r.testFile ? `${show(path.resolve(r.repo, r.testFile))}  ${verdict}` : "no test file reported by the coding agent"),
+      ...(r.command ? [row("", "Command", r.command)] : []),
+      row("", "Handoff", show(r.handoff)),
+    ]
+  }
+  for (const run of p.runs) {
+    const find = (name: string) => run.files.find((f) => path.basename(f) === name)
+    const report = find("report.html")
+    if (body.length) body.push("")
+    if (p.runs.length > 1) body.push(row("", "Run", `${show(run.runDir)}/  ${run.status === "complete" ? "passed" : run.status === "stopped" ? "stopped" : "failed"}`))
+    body.push(row("", "Report", report ? show(report) : "not written"))
+    const written = p.tests.filter((t) => t.runDir === run.runDir)
+    if (written.length) body.push(...written.flatMap(testRows))
+    else if (run.status !== "complete") body.push(row("", "Test", "not written: the browser run did not pass"))
+    else if (!tests.on) body.push(row("", "Test", `not written: ${tests.why}`))
+    else body.push(row("?", "Test", "not written: the run was not handed to the coding agent"))
+  }
+  const other = p.tests.filter((t) => !p.runs.some((r) => r.runDir === t.runDir))
+  if (other.length) {
+    if (body.length) body.push("")
+    body.push(...other.flatMap(testRows))
+  }
+  for (const s of p.suites) {
+    if (body.length) body.push("")
+    body.push(row("", "Report", show(s.html)), row("", "JUnit", show(s.junit)))
+  }
+  const results = p.tests.map((t) => t.result?.status)
+  const status = results.includes("failed") || p.tests.some((t) => t.error) ? "fail" : results.includes("passed") ? "ok" : "info"
+  return { status, title: "What this produced", body }
+}
+
+export function SummaryCard({ summary }: { summary: RunSummary }) {
+  const color = summary.status === "complete" ? theme.ok : summary.status === "stopped" ? theme.warn : theme.err
+  return (
+    <box
+      border={["left"]}
+      borderColor={color}
+      customBorderChars={BAR}
+      style={{
+        flexDirection: "column",
+        backgroundColor: theme.panel,
+        paddingLeft: 2,
+        paddingRight: 2,
+        paddingTop: 1,
+        paddingBottom: 1,
+        marginTop: 1,
+        marginBottom: 1,
+        flexShrink: 0,
+      }}
+    >
+      <box style={{ flexDirection: "row", justifyContent: "space-between" }}>
+        <text>
+          <span fg={color}>{`${STATUS_MARK[summary.status]} `}</span>
+          <span fg={theme.text}>{summary.title}</span>
+        </text>
+        <text fg={theme.muted}>{summary.stats}</text>
+      </box>
+      {summary.failed.map((f, i) => (
+        <text key={i} fg={theme.err} style={i === 0 ? { marginTop: 1 } : undefined}>
+          {`✗ ${f}`}
+        </text>
+      ))}
+      <text fg={theme.text} style={{ marginTop: 1 }}>
+        {summary.text}
+      </text>
+      {summary.finalUrl || summary.savedDir ? <box style={{ height: 1 }} /> : null}
+      {summary.finalUrl ? (
+        <text>
+          <span fg={theme.faint}>{"Ended on  "}</span>
+          <span fg={theme.muted}>{summary.finalUrl}</span>
+        </text>
+      ) : null}
+      {summary.memory ? (
+        <text>
+          <span fg={theme.faint}>{"Memory    "}</span>
+          <span fg={theme.memory}>{summary.memory}</span>
+        </text>
+      ) : null}
+      {summary.quality ? (
+        <text>
+          <span fg={theme.faint}>{"Quality   "}</span>
+          <span fg={theme.muted}>{summary.quality}</span>
+        </text>
+      ) : null}
+      {summary.usage ? (
+        <text>
+          <span fg={theme.faint}>{"Usage     "}</span>
+          <span fg={theme.muted}>{summary.usage}</span>
+        </text>
+      ) : null}
+      {summary.savedDir ? (
+        <text>
+          <span fg={theme.faint}>{"Saved     "}</span>
+          <span fg={theme.muted}>{`${summary.savedDir}/  `}</span>
+          <span fg={theme.faint}>{summary.savedFiles.join(" · ")}</span>
+        </text>
+      ) : null}
+    </box>
+  )
+}
+
+export function Row({ label, value, color = theme.muted }: { label: string; value: string; color?: string }) {
+  if (!value) return null
+  return (
+    <text>
+      <span fg={theme.faint}>{label.padEnd(10)}</span>
+      <span fg={color}>{value}</span>
+    </text>
+  )
+}
+
+export const CASE_MARK = { passed: "✓", failed: "✗", error: "!", stopped: "■" } as const
+
+export function SuiteCard({ suite }: { suite: SuiteResult }) {
+  const c = counts(suite.results)
+  const ok = !suite.stopped && suiteExitCode(suite) === 0
+  const color = suite.stopped ? theme.warn : ok ? theme.ok : theme.err
+  const tokens = suite.results.reduce((n, r) => n + r.tokens, 0)
+  const show = (f: string) => path.relative(process.cwd(), f) || f
+  const caseColor = (s: keyof typeof CASE_MARK) => (s === "passed" ? theme.ok : s === "stopped" ? theme.warn : theme.err)
+  return (
+    <box
+      border={["left"]}
+      borderColor={color}
+      customBorderChars={BAR}
+      style={{
+        flexDirection: "column",
+        backgroundColor: theme.panel,
+        paddingLeft: 2,
+        paddingRight: 2,
+        paddingTop: 1,
+        paddingBottom: 1,
+        marginTop: 1,
+        marginBottom: 1,
+        flexShrink: 0,
+      }}
+    >
+      <box style={{ flexDirection: "row", justifyContent: "space-between" }}>
+        <text>
+          <span fg={color}>{suite.stopped ? "■ " : ok ? "✓ " : "✗ "}</span>
+          <span fg={theme.text}>{suite.stopped ? "Suite stopped" : ok ? "Suite passed" : "Suite failed"}</span>
+        </text>
+        <text fg={theme.muted}>{`${c.passed}/${c.total} passed · ${formatDuration(suite.ms)} · ${formatTokens(tokens)} tokens`}</text>
+      </box>
+      <box style={{ height: 1 }} />
+      {suite.results.map((r) => (
+        <box key={r.case.id} style={{ flexDirection: "column", flexShrink: 0 }}>
+          <text>
+            <span fg={caseColor(r.status)}>{`${CASE_MARK[r.status]} `}</span>
+            <span fg={theme.text}>{`${r.case.id}  ${clip(r.case.title, 60)}`}</span>
+            <span fg={theme.faint}>{`  ${formatDuration(r.ms)}`}</span>
+            {r.code ? (
+              <span fg={r.code.status === "passed" ? theme.ok : theme.err}>{`  test ${r.code.status}${r.code.testFile ? `: ${r.code.testFile}` : ""}`}</span>
+            ) : null}
+          </text>
+          {r.status !== "passed" && r.summary ? <text fg={theme.muted}>{`  ${clip(r.summary.split("\n")[0], 110)}`}</text> : null}
+        </box>
+      ))}
+      {suite.writeBack.length ? <box style={{ height: 1 }} /> : null}
+      {suite.writeBack.map((w, i) => (
+        <text key={i}>
+          <span fg={w.error ? theme.err : theme.ok}>{w.error ? "✗ " : "↑ "}</span>
+          <span fg={theme.muted}>{`${w.text}${w.url ? `  ${w.url}` : ""}${w.error ? `: ${w.error}` : ""}`}</span>
+        </text>
+      ))}
+      <box style={{ height: 1 }} />
+      <Row label="Report" value={show(suite.files.html)} color={theme.text} />
+      <Row label="JUnit" value={show(suite.files.junit)} />
+    </box>
+  )
+}
+
+export const CARD_COLOR: Record<Card["status"], string> = { ok: theme.ok, fail: theme.err, stopped: theme.warn, info: theme.accent }
+
+export function ResultCard({ card }: { card: Card }) {
+  const color = CARD_COLOR[card.status]
+  const lineColor = (l: string) => (/^\s*✓/.test(l) ? theme.ok : /^\s*[✗!]/.test(l) ? theme.err : /^\s*\?/.test(l) ? theme.warn : theme.muted)
+  return (
+    <box
+      border={["left"]}
+      borderColor={color}
+      customBorderChars={BAR}
+      style={{
+        flexDirection: "column",
+        backgroundColor: theme.panel,
+        paddingLeft: 2,
+        paddingRight: 2,
+        paddingTop: 1,
+        paddingBottom: 1,
+        marginTop: 1,
+        marginBottom: 1,
+        flexShrink: 0,
+      }}
+    >
+      <box style={{ flexDirection: "row", justifyContent: "space-between" }}>
+        <text>
+          <span fg={color}>{`${CARD_MARK[card.status]} `}</span>
+          <span fg={theme.text}>{card.title}</span>
+        </text>
+        {card.stats ? <text fg={theme.muted}>{card.stats}</text> : null}
+      </box>
+      {card.body.length ? <box style={{ height: 1 }} /> : null}
+      {card.body.map((l, i) => (
+        <text key={i} fg={lineColor(l)}>
+          {l || " "}
+        </text>
+      ))}
+    </box>
+  )
+}
+
+let style: SyntaxStyle | undefined
+/** How the agent's markdown replies look: headings, bold, code, and links in the shell's colours, markers hidden. */
+function replyStyle(): SyntaxStyle {
+  style ||= SyntaxStyle.fromStyles({
+    default: { fg: theme.text },
+    "markup.heading": { fg: theme.accent, bold: true },
+    "markup.strong": { bold: true },
+    "markup.italic": { italic: true },
+    "markup.raw": { fg: theme.memory },
+    "markup.quote": { fg: theme.muted, italic: true },
+    "markup.list": { fg: theme.accent },
+    "markup.link": { fg: theme.accent, underline: true },
+    "markup.link.label": { fg: theme.accent, underline: true },
+    "markup.link.url": { fg: theme.muted },
+  })
+  return style
+}
+
+export function TranscriptLine({ line }: { line: Line }) {
+  if (line.kind === "card" && line.card) return <ResultCard card={line.card} />
+  if (line.kind === "summary" && line.summary) return <SummaryCard summary={line.summary} />
+  if (line.kind === "suitesummary" && line.suite) return <SuiteCard suite={line.suite} />
+  if (line.kind === "reply") {
+    const color = line.outcome === "failed" ? theme.err : line.outcome === "needs_input" ? theme.warn : theme.ok
+    return (
+      <box
+        border={["left"]}
+        borderColor={color}
+        customBorderChars={BAR}
+        style={{ backgroundColor: theme.panel, paddingLeft: 2, paddingRight: 2, paddingTop: 1, paddingBottom: 1, marginTop: 1, marginBottom: 1, flexShrink: 0 }}
+      >
+        <markdown content={line.text} syntaxStyle={replyStyle()} fg={theme.text} conceal tableOptions={{ widthMode: "content", wrapMode: "word" }} />
+      </box>
+    )
+  }
+  if (line.kind === "tool") return <text fg={theme.faint}>{`    ⚙ ${line.text}`}</text>
+  if (line.kind === "file") return <text fg={theme.accent}>{`    ✎ ${line.text}`}</text>
+  if (line.kind === "code") return <text fg={theme.memory}>{`  ▸ ${line.text}`}</text>
+  if (line.kind === "user") {
+    return (
+      <box
+        border={["left"]}
+        borderColor={theme.accent}
+        customBorderChars={BAR}
+        style={{
+          backgroundColor: theme.panel,
+          paddingLeft: 2,
+          paddingTop: 1,
+          paddingBottom: 1,
+          marginTop: 1,
+          marginBottom: 1,
+          flexShrink: 0,
+        }}
+      >
+        <text fg={theme.text}>{line.text}</text>
+      </box>
+    )
+  }
+  if (line.kind === "timing") return <text fg={theme.warn}>{`  + ${line.text}`}</text>
+  if (line.kind === "memory") return <text fg={theme.memory}>{`  ↻ ${line.text}`}</text>
+  if (line.kind === "thought") return <text fg={theme.thought}>{`    ${line.text}`}</text>
+  if (line.kind === "ok") {
+    return (
+      <text>
+        <span fg={theme.ok}>{"  ✓ "}</span>
+        <span fg={theme.text}>{line.text}</span>
+      </text>
+    )
+  }
+  if (line.kind === "err") {
+    return (
+      <text>
+        <span fg={theme.err}>{"  ✗ "}</span>
+        <span fg={theme.text}>{line.text}</span>
+      </text>
+    )
+  }
+  return <text fg={theme.muted}>{`  ${line.text || " "}`}</text>
+}

package/src/types.ts

@@ -0,0 +1,136 @@
+import type { QualityReport } from "./quality/budgets.ts"
+import type { Cost } from "./usage.ts"
+
+export type Usage = {
+  /** Every prompt token, cached ones included. */
+  input: number
+  output: number
+  /** The part of `input` the provider served from its prompt cache (billed at a lower rate). */
+  cached?: number
+  /** Prompt tokens written to the provider's cache (Anthropic bills these above the input price). */
+  cacheWrite?: number
+}
+
+export type RunStatus = "complete" | "fail" | "stopped"
+
+export type ReplayMode = "verify" | "trust" | "off"
+
+/** What page memory did during a live run. */
+export type MemoryInfo = {
+  dir: string
+  mode: ReplayMode
+  /** Steps replayed from memory without the LLM, out of the recorded flow's replayable steps. */
+  replayed: number
+  replayTotal: number
+  replayStopped?: string
+  llmSteps: number
+  /** Pages and elements observed this run; newElements were not in memory before. */
+  pages: number
+  elements: number
+  newElements: number
+  flow: "saved" | "updated" | null
+  /** After a failed run: the steps that worked before it went wrong, saved in resume.json for a retry to replay. */
+  resume?: { steps: number; failedAt: number; reason: string; lines: string[]; file?: string }
+}
+
+export type RunResult = {
+  status: RunStatus
+  goal: string
+  startUrl: string
+  finalUrl: string
+  summary: string
+  usage: Usage
+  /** What `usage` cost at the model's price in llms.json; unset without a price. */
+  cost?: Cost
+  driver: "synthetic" | "engine"
+  memory?: MemoryInfo
+  /**
+   * passed; assertion_failed: the steps were done and an expected result does not hold (the test failed, which is a
+   * valid result); error: the agent could not do the steps. Unset for runs that did not say.
+   */
+  outcome?: "passed" | "assertion_failed" | "error"
+  /** The expected results that did not hold, and what the page showed instead. */
+  failedChecks?: { expected: string; actual: string }[]
+  /** For a test case: the agent's verdict on each numbered test step. */
+  caseReport?: CaseReport
+  /** Skills added to the browser agent's prompt. */
+  skills?: string[]
+  /** Accessibility (WCAG rules) and Core Web Vitals of the pages visited, judged against the budgets. */
+  quality?: QualityReport
+  /** Budgets this run exceeded: the run counts as failed, though its steps passed. */
+  budgetExceeded?: string[]
+}
+
+/** Why a failed step failed, in the browser agent's judgement: see src/cases/failure.ts. */
+export type StepCause = "app" | "test_case" | "environment" | "automation"
+
+export type StepVerdict = { step: number; status: "passed" | "failed" | "not_run"; actual?: string; cause?: StepCause }
+
+export type CaseReport = { summary: string; steps: StepVerdict[] }
+
+/** blocked: the app could not be reached or blocked automation, so no honest test could pass. */
+export type CodeStatus = "passed" | "failed" | "stopped" | "blocked"
+
+/** A read-only reviewer's verdict on a passing test: does it test the goal, the way the test rules say. */
+export type CodeReview = {
+  approved: boolean
+  summary: string
+  /** correctness: the test does not test what the goal asks, or not reliably. standards: it works, but not in the repository's way (data in a page object, naming, leftovers). */
+  issues: { file?: string; problem: string; fix?: string; kind?: "correctness" | "standards" }[]
+  /** The coding-standard issues the tidy-up round could not fix without breaking the test: the test was kept as it passed. */
+  standardsLeft?: boolean
+}
+
+/** The coding agent's job: a test in the target repo that ForgeQE itself ran until it passed. */
+export type CodeResult = {
+  status: CodeStatus
+  repo: string
+  attempts: number
+  maxAttempts: number
+  testFile?: string
+  command?: string
+  /** Existing helpers, page objects, and fixtures the agent reused; files it created or changed. */
+  reused: string[]
+  changed: string[]
+  notes: string
+  /** Tail of the last test run's output. */
+  output: string
+  /** Browsers ForgeQE saw the last test run launch. */
+  browsers?: string[]
+  review?: CodeReview
+  /** The replay of the run's steps in Playwright before Scribe started: the steps that failed in each launch setup. */
+  replay?: { dir: string; error?: string; passes: { setup: string; failed: number[]; reached: boolean }[] }
+  usage: Usage
+  /** What `usage` cost at the model's price in llms.json; unset without a price. */
+  cost?: Cost
+  /** Skills Scribe loaded during the job. */
+  skills?: string[]
+  sessionId?: string
+  /** Scribe reopened the session of an earlier job on this run that stopped before it finished. */
+  resumed?: boolean
+  model: string
+  handoff: string
+  /** The test's full name as the framework reports it, for mapping it to its test case. */
+  testName?: string
+  /** The test asserts the correct behaviour of a known defect and is marked as an expected failure. */
+  knownDefect?: boolean
+  /** The job did not finish: what it wrote was moved to the needs-attention folder (see src/code/attention.ts). */
+  attention?: { dir: string; files: string[]; patch?: string; readme: string }
+}
+
+export type CodeEvent =
+  | { type: "code"; kind: "info"; text: string }
+  | { type: "code"; kind: "tool"; text: string; ok: boolean }
+  | { type: "code"; kind: "say"; text: string }
+  | { type: "code"; kind: "test"; text: string; ok: boolean; attempt: number }
+  | { type: "code"; kind: "usage"; usage: Usage }
+
+export type AgentEvent =
+  | { type: "log"; level: "info" | "warn" | "error"; text: string }
+  | CodeEvent
+  | { type: "step"; step: number; url: string; title: string; category: string; source?: "memory" }
+  | { type: "memory"; text: string }
+  | { type: "thought"; step: number; text: string }
+  | { type: "outcome"; step: number; ok: boolean; text: string }
+  | { type: "usage"; usage: Usage }
+  | { type: "done"; result: RunResult }