@capagents/forgeqe 0.0.0-stage → 1.0.1

package/src/explore/run.ts

@@ -0,0 +1,280 @@
+import { mkdirSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { AppConfig, LoginConfig } from "../config.ts"
+import type { Card } from "../card.ts"
+import { fileBugs } from "../cases/bugs.ts"
+import { classifyFailure, FAILURE_LABEL, type Failure } from "../cases/failure.ts"
+import { goalFor } from "../cases/goal.ts"
+import type { CaseResult, TestCase, WriteBack } from "../cases/types.ts"
+import { browserErrors, problemLine } from "../report/data.ts"
+import { executeRun } from "../run.ts"
+import type { AgentEvent, RunResult, Usage } from "../types.ts"
+import { addUsage, costOf } from "../usage.ts"
+import type { Charter } from "./charters.ts"
+import type { Story } from "./story.ts"
+
+/**
+ * Exploratory testing of a story: each charter runs in a real browser (as its role when it names one), and each
+ * expected result is judged on the live page. A charter that finds a functional defect runs again: a defect is
+ * confirmed only when the second run fails the same way. What does not reproduce, the browser errors seen on the way,
+ * and what the browser agent noticed beyond the steps are observations, not defects. Charters that ran clean or
+ * found a confirmed defect become proposed test cases; bugs are filed for confirmed defects only.
+ */
+
+export type CharterStatus = "passed" | "defect" | "not_reproduced" | "could_not_explore" | "stopped"
+
+export type CharterResult = {
+  charter: Charter
+  status: CharterStatus
+  summary: string
+  runDir: string
+  reproRunDir?: string
+  failure?: Failure
+  /** What the browser agent noticed beyond the steps, and the browser errors on the way. */
+  observations: string[]
+  finalUrl?: string
+  usage: Usage
+  /** The run as a test case result, for bugs and reports. */
+  caseResult: CaseResult
+}
+
+export type ProposedCase = { title: string; steps: { action: string; expected: string }[]; preconditions?: string[]; description?: string; from: string; kind: "regression" | "defect" }
+
+export type Exploration = {
+  dir: string
+  story: Story
+  results: CharterResult[]
+  proposed: ProposedCase[]
+  bugs: WriteBack[]
+  usage: { plan: Usage; browser: Usage }
+  report: string
+}
+
+export const NOTICED = "Also noticed:"
+
+/** The goal for one charter: the charter as a test case, plus the request to report what else looks wrong. */
+export function charterCase(story: Story, c: Charter): TestCase {
+  return {
+    id: `EXP-${story.id === "story" ? "" : `${story.id}-`}${c.id}`,
+    title: `${story.id === "story" ? story.title : story.id} exploratory: ${c.title}`,
+    source: "text",
+    description: [`Exploratory charter (${c.category.replace("_", " ")}): ${c.mission}`, c.criterion ? `Acceptance criterion: ${c.criterion}` : "", story.url ? `Story: ${story.url}` : ""].filter(Boolean).join("\n"),
+    ...(c.role ? { preconditions: [`Signed in as the ${c.role} login`] } : {}),
+    steps: c.steps,
+  }
+}
+
+export function charterGoal(story: Story, c: Charter): string {
+  return [
+    goalFor(charterCase(story, c)),
+    "",
+    `This is exploratory testing. Do the steps as written, with the data given, even when it is invalid: that is the point. While you do them, watch for anything else that looks wrong (an error message or page, a broken layout, wrong or lost data, a missing validation, a slow page) and list it at the end of your summary after "${NOTICED}", one item per line, or "${NOTICED} nothing".`,
+    c.role ? `Sign in as the ${c.role} login before the first step.` : "",
+  ]
+    .filter(Boolean)
+    .join("\n")
+}
+
+/** The browser agent's "Also noticed:" lines. */
+export function noticed(summary: string): string[] {
+  const at = summary.lastIndexOf(NOTICED)
+  if (at < 0) return []
+  return summary
+    .slice(at + NOTICED.length)
+    .split("\n")
+    .map((l) => l.replace(/^\s*[-*•\d.)]+\s*/, "").trim())
+    .filter((l) => l && !/^nothing\.?$/i.test(l) && !/^none\.?$/i.test(l))
+    .slice(0, 10)
+}
+
+/** Only the role's login on its sites: two logins on one site would leave the agent with both sessions. */
+export function loginsFor(logins: LoginConfig[], role?: string): LoginConfig[] {
+  if (!role) return logins
+  const mine = logins.find((l) => l.name === role)
+  if (!mine) return logins
+  const hosts = (l: LoginConfig) => l.domains.map((d) => d.replace(/^\w+:\/\//, "").split("/")[0].replace(/^\*\./, ""))
+  const taken = new Set(hosts(mine))
+  return logins.filter((l) => l === mine || !hosts(l).some((h) => taken.has(h)))
+}
+
+const NO_USAGE: Usage = { input: 0, output: 0 }
+const STATUS = { complete: "passed", fail: "failed", stopped: "stopped" } as const
+
+export type ExploreEvents = {
+  onRunStart?: (c: Charter, goal: string, repro: boolean) => void
+  onRunEvent?: (c: Charter, e: AgentEvent) => void
+  onRunDone?: (c: Charter, r: { result?: RunResult; error?: string; runDir: string; ms: number }) => void
+  onInfo?: (text: string) => void
+}
+
+export async function exploreCharters(opts: {
+  config: AppConfig
+  story: Story
+  charters: Charter[]
+  dir: string
+  url?: string
+  headed?: boolean
+  maxSteps?: number
+  signal?: AbortSignal
+  execute?: typeof executeRun
+  events?: ExploreEvents
+}): Promise<CharterResult[]> {
+  const execute = opts.execute || executeRun
+  const ev = opts.events || {}
+  const out: CharterResult[] = []
+  for (const [i, c] of opts.charters.entries()) {
+    if (opts.signal?.aborted) break
+    const goal = charterGoal(opts.story, c)
+    const config: AppConfig = c.role ? { ...opts.config, logins: loginsFor(opts.config.logins, c.role) } : opts.config
+    const go = async (runDir: string, repro: boolean): Promise<{ run?: RunResult; error?: string }> => {
+      mkdirSync(runDir, { recursive: true })
+      ev.onRunStart?.(c, goal, repro)
+      const t0 = Date.now()
+      try {
+        const { result } = await execute({ config, goal, url: opts.url, headed: opts.headed, maxSteps: opts.maxSteps, caseSteps: c.steps.length, outDir: runDir, signal: opts.signal, onEvent: (e) => ev.onRunEvent?.(c, e) })
+        ev.onRunDone?.(c, { result, runDir, ms: Date.now() - t0 })
+        return { run: result }
+      } catch (err) {
+        const error = (err as Error).message || String(err)
+        ev.onRunDone?.(c, { error, runDir, ms: Date.now() - t0 })
+        return { error }
+      }
+    }
+    const runDir = path.join(opts.dir, `${String(i + 1).padStart(2, "0")}-${c.id}`)
+    let { run, error } = await go(runDir, false)
+    let failure = opts.signal?.aborted ? undefined : classifyFailure(run, error)
+    let usage = run?.usage || NO_USAGE
+    if (failure?.type === "environment" && !opts.signal?.aborted) {
+      ev.onInfo?.(`${c.id}: ${FAILURE_LABEL.environment} (${failure.reason}); running it once more`)
+      ;({ run, error } = await go(runDir, false))
+      failure = opts.signal?.aborted ? undefined : classifyFailure(run, error)
+      usage = addUsage(usage, run?.usage)
+    }
+    let status: CharterStatus = opts.signal?.aborted ? "stopped" : !failure ? (run?.status === "complete" ? "passed" : "stopped") : failure.type === "functional_defect" ? "defect" : "could_not_explore"
+    let reproRunDir: string | undefined
+    const observations = [...noticed(run?.summary || ""), ...browserErrors(runDir).map(problemLine)]
+    if (status === "defect" && !opts.signal?.aborted) {
+      // Confirmed only when it happens again: a defect that does not reproduce is an observation.
+      ev.onInfo?.(`${c.id}: possible defect (${failure!.reason}); running it again to confirm`)
+      reproRunDir = `${runDir}-repro`
+      const again = await go(reproRunDir, true)
+      usage = addUsage(usage, again.run?.usage)
+      const second = classifyFailure(again.run, again.error)
+      const same = second?.type === "functional_defect" && (second.step === undefined || failure!.step === undefined || second.step === failure!.step)
+      if (!same) {
+        status = "not_reproduced"
+        observations.unshift(`Not reproduced on a second run: ${failure!.reason}`)
+      }
+    }
+    const tc = charterCase(opts.story, c)
+    const caseResult: CaseResult = {
+      case: tc,
+      status: run ? STATUS[run.status] : opts.signal?.aborted ? "stopped" : "error",
+      summary: run?.summary || error || "",
+      startedAt: new Date().toISOString(),
+      finishedAt: new Date().toISOString(),
+      ms: 0,
+      tokens: usage.input + usage.output,
+      usage,
+      cost: costOf(usage, opts.config.llm.price),
+      runDir,
+      finalUrl: run?.finalUrl || undefined,
+      ...(run?.caseReport?.steps.length ? { stepResults: run.caseReport.steps } : {}),
+      ...(failure ? { failure } : {}),
+    }
+    out.push({ charter: c, status, summary: run?.summary || error || "", runDir, ...(reproRunDir ? { reproRunDir } : {}), ...(failure ? { failure } : {}), observations: [...new Set(observations)].slice(0, 15), ...(run?.finalUrl ? { finalUrl: run.finalUrl } : {}), usage, caseResult })
+  }
+  return out
+}
+
+/** Charters that ran clean become regression cases; confirmed defects become cases that check the correct behaviour. */
+export function proposedCases(results: CharterResult[]): ProposedCase[] {
+  return results
+    .filter((r) => r.status === "passed" || r.status === "defect")
+    .map((r) => ({
+      title: r.charter.title,
+      steps: r.charter.steps.map((s) => ({ action: s.data ? `${s.action} (data: ${s.data})` : s.action, expected: s.expected })),
+      ...(r.charter.role ? { preconditions: [`Signed in as ${r.charter.role}`] } : {}),
+      description: `${r.charter.mission}${r.status === "defect" ? " Found a defect when explored: this case fails until it is fixed." : ""}`,
+      from: r.charter.id,
+      kind: r.status === "defect" ? "defect" : "regression",
+    }))
+}
+
+/** File bugs for confirmed defects only, through the configured tracker (one per charter, commented on when it is still open). */
+export async function fileConfirmed(config: AppConfig, story: Story, results: CharterResult[]): Promise<WriteBack[]> {
+  const confirmed = results.filter((r) => r.status === "defect")
+  if (!confirmed.length || !config.sources.bugs) return []
+  return fileBugs(
+    confirmed.map((r) => r.caseResult),
+    config.sources,
+    `exploratory testing of ${story.id === "story" ? story.title : `${story.id} ${story.title}`}`,
+  )
+}
+
+const LABEL: Record<CharterStatus, string> = { passed: "✓ as expected", defect: "✗ defect (confirmed)", not_reproduced: "? not reproduced", could_not_explore: "– could not explore", stopped: "■ stopped" }
+
+export function explorationMarkdown(x: Omit<Exploration, "report">): string {
+  const s = x.story
+  const defects = x.results.filter((r) => r.status === "defect")
+  const lines = [
+    `# Exploratory testing: ${s.id === "story" ? "" : `${s.id} `}${s.title}`,
+    "",
+    s.url ? `Story: ${s.url}` : "",
+    `${x.results.length} charter${x.results.length === 1 ? "" : "s"} · ${defects.length} confirmed defect${defects.length === 1 ? "" : "s"} · ${x.results.filter((r) => r.status === "not_reproduced").length} not reproduced · ${x.proposed.length} proposed test cases`,
+    "",
+    "## Charters",
+    "",
+    "| | Charter | Kind | Role | Result |",
+    "|---|---|---|---|---|",
+    ...x.results.map((r) => `| ${r.charter.id} | ${r.charter.title} | ${r.charter.category.replace("_", " ")} | ${r.charter.role || ""} | ${LABEL[r.status]}${r.failure && r.status !== "defect" ? `: ${FAILURE_LABEL[r.failure.type]}` : ""} |`),
+  ]
+  if (defects.length) {
+    lines.push("", "## Confirmed defects", "")
+    for (const r of defects) {
+      lines.push(`### ${r.charter.id}: ${r.charter.title}`, "", r.failure?.reason || r.summary, "", ...r.charter.steps.map((st, i) => `${i + 1}. ${st.action}${st.data ? ` (data: ${st.data})` : ""} → expected: ${st.expected}`), "", `Evidence: ${r.runDir}${r.reproRunDir ? ` and ${r.reproRunDir}` : ""}`)
+      const bug = x.bugs.find((b) => b.caseId === r.caseResult.case.id)
+      if (bug) lines.push(`Bug: ${bug.text}${bug.url ? ` ${bug.url}` : ""}${bug.error ? ` (${bug.error})` : ""}`)
+      lines.push("")
+    }
+  }
+  const obs = x.results.filter((r) => r.observations.length)
+  if (obs.length) {
+    lines.push("", "## Observations (not confirmed defects)", "")
+    for (const r of obs) lines.push(`- ${r.charter.id} ${r.charter.title}:`, ...r.observations.map((o) => `  - ${o}`))
+  }
+  const blocked = x.results.filter((r) => r.status === "could_not_explore")
+  if (blocked.length) lines.push("", "## Could not explore", "", ...blocked.map((r) => `- ${r.charter.id} ${r.charter.title}: ${r.failure ? `${FAILURE_LABEL[r.failure.type]}: ${r.failure.reason}` : r.summary}`))
+  if (x.proposed.length) {
+    lines.push("", "## Proposed test cases", "")
+    for (const p of x.proposed) lines.push(`### ${p.title}${p.kind === "defect" ? " (fails until the defect is fixed)" : ""}`, "", ...(p.preconditions || []).map((c) => `Precondition: ${c}`), ...p.steps.map((st, i) => `${i + 1}. ${st.action} → ${st.expected}`), "")
+  }
+  return `${lines.filter((l, i, all) => l !== "" || all[i - 1] !== "").join("\n").trim()}\n`
+}
+
+export function explorationCard(x: Exploration, cwd: string): Card {
+  const defects = x.results.filter((r) => r.status === "defect")
+  const body: string[] = x.results.map((r) => `${LABEL[r.status].split(" ")[0]} ${r.charter.id} ${r.charter.title}${r.charter.role ? ` [${r.charter.role}]` : ""}${r.status === "defect" || r.status === "could_not_explore" ? `\n    ${r.failure?.reason || r.summary}` : ""}`)
+  const observations = x.results.reduce((n, r) => n + r.observations.length, 0)
+  if (observations) body.push("", `${observations} observation${observations === 1 ? "" : "s"} (not confirmed): see the report`)
+  if (x.bugs.length) body.push("", ...x.bugs.map((b) => `${b.error ? "✗" : "✓"} ${b.text}${b.url ? ` ${b.url}` : ""}${b.error ? `: ${b.error}` : ""}`))
+  if (x.proposed.length) body.push("", `${x.proposed.length} proposed test case${x.proposed.length === 1 ? "" : "s"}: ${x.proposed.map((p) => p.title).slice(0, 6).join("; ")}${x.proposed.length > 6 ? "; …" : ""}`)
+  const rel = path.relative(cwd, x.report)
+  body.push("", `Report    ${rel && !rel.startsWith("..") ? rel : x.report}`)
+  return {
+    status: defects.length ? "fail" : x.results.some((r) => r.status === "passed") ? "ok" : "stopped",
+    title: `Explored ${x.story.id === "story" ? x.story.title : x.story.id}: ${defects.length ? `${defects.length} confirmed defect${defects.length === 1 ? "" : "s"}` : "no confirmed defects"}`,
+    stats: `${x.results.length} charter${x.results.length === 1 ? "" : "s"}`,
+    body,
+  }
+}
+
+/** Save the exploration: exploration.md and exploration.json in its folder. */
+export function saveExploration(x: Omit<Exploration, "report">): string {
+  mkdirSync(x.dir, { recursive: true })
+  const report = path.join(x.dir, "exploration.md")
+  writeFileSync(report, explorationMarkdown(x))
+  writeFileSync(path.join(x.dir, "exploration.json"), JSON.stringify({ story: { ...x.story, scripted: x.story.scripted.map((c) => c.id) }, results: x.results.map(({ caseResult, ...r }) => r), proposed: x.proposed, bugs: x.bugs, usage: x.usage }, null, 2))
+  return report
+}

package/src/explore/story.ts

@@ -0,0 +1,106 @@
+import type { AppConfig } from "../config.ts"
+import { adoRequirements, type Requirement } from "../cases/context.ts"
+import { adfToText, http } from "../cases/http.ts"
+import { jiraApi } from "../cases/jira.ts"
+import { loadCases } from "../cases/index.ts"
+import { searchRequirements } from "../cases/rag.ts"
+import type { TestCase } from "../cases/types.ts"
+
+/**
+ * The story to explore, with what it needs to be explored well: its description and acceptance criteria, the feature
+ * and epic above it, the test cases that already script it (exploration goes beyond them), and passages from the
+ * requirements search when one is set up.
+ */
+
+export type Story = {
+  /** "ADO-123", "SHOP-12", or "story" for text. */
+  id: string
+  system: "ado" | "jira" | "text"
+  title: string
+  url?: string
+  description: string
+  acceptance: string
+  /** The feature and epic above it, nearest first. */
+  parents: Requirement[]
+  /** Test cases that already test it: their steps are the scripted coverage. */
+  scripted: TestCase[]
+  passages: string[]
+  errors: string[]
+}
+
+const MAX_SCRIPTED = 10
+
+/** ado:123, #123, an Azure DevOps work item link; jira:SHOP-12, SHOP-12, a Jira browse link; anything else is the story's text. */
+export function storyRef(ref: string): { system: "ado" | "jira" | "text"; key: string } {
+  const r = ref.trim()
+  const ado = r.match(/^(?:ado|azure):#?(\d+)$/i) || r.match(/^#(\d+)$/) || r.match(/_workitems\/edit\/(\d+)/) || (/^\d+$/.test(r) ? [r, r] : null)
+  if (ado) return { system: "ado", key: ado[1] }
+  const jira = r.match(/^jira:([A-Z][A-Z0-9_]+-\d+)$/i) || r.match(/^([A-Z][A-Z0-9_]+-\d+)$/) || r.match(/\/browse\/([A-Z][A-Z0-9_]+-\d+)/)
+  if (jira) return { system: "jira", key: jira[1].toUpperCase() }
+  return { system: "text", key: r }
+}
+
+async function adoStory(cfg: AppConfig, id: number, story: Story): Promise<void> {
+  const reqs = new Map<string, Requirement>()
+  const { chain } = await adoRequirements(cfg, [id], { requirements: reqs })
+  const own = reqs.get(`ADO-${id}`)
+  if (!own) throw new Error(`Azure DevOps work item ${id} was not found.`)
+  Object.assign(story, { id: own.id, title: own.title, url: own.url, description: own.description || "", acceptance: own.acceptance || "" })
+  story.parents = chain.slice(1).map((k) => reqs.get(k)!).filter(Boolean)
+  const tested = (own.testedBy || []).slice(0, MAX_SCRIPTED).map((k) => k.slice(4))
+  if (tested.length) {
+    try {
+      story.scripted = await loadCases([`ado:${tested.join(",")}`], cfg.sources)
+    } catch (err) {
+      story.errors.push(`scripted test cases: ${(err as Error).message.slice(0, 200)}`)
+    }
+  }
+}
+
+async function jiraStory(cfg: AppConfig, key: string, story: Story): Promise<void> {
+  const j = jiraApi(cfg.sources, "Reading the story from Jira")
+  type Issue = { key: string; fields: Record<string, any> }
+  const read = async (k: string) => (await http<Issue>(`${j.api}/issue/${encodeURIComponent(k)}?fields=summary,description,issuetype,parent,issuelinks`, { auth: j.auth })).data
+  const issue = await read(key)
+  const text = (d: unknown) => (typeof d === "string" ? d : adfToText(d))
+  const description = text(issue.fields.description)
+  // Acceptance criteria usually sit in the description under a heading; keep them apart when they do.
+  const ac = description.split(/\n\s*(?:#+\s*)?acceptance criteria:?\s*\n/i)
+  Object.assign(story, { id: key, title: String(issue.fields.summary || ""), url: `${j.url}/browse/${key}`, description: ac[0].trim(), acceptance: (ac[1] || "").trim() })
+  let parent = issue.fields.parent?.key as string | undefined
+  for (let level = 0; parent && level < 2; level++) {
+    const p = await read(parent)
+    story.parents.push({ id: p.key, type: String(p.fields.issuetype?.name || "Issue"), title: String(p.fields.summary || ""), url: `${j.url}/browse/${p.key}`, description: text(p.fields.description).slice(0, 2000) })
+    parent = p.fields.parent?.key
+  }
+  const links = (issue.fields.issuelinks || []) as { type: { name: string; inward: string; outward: string }; inwardIssue?: { key: string }; outwardIssue?: { key: string } }[]
+  const tests = links.filter((l) => /test/i.test(`${l.type.name} ${l.type.inward} ${l.type.outward}`)).map((l) => (l.inwardIssue || l.outwardIssue)?.key).filter((k): k is string => Boolean(k))
+  if (tests.length) {
+    try {
+      story.scripted = await loadCases(tests.slice(0, MAX_SCRIPTED).map((k) => `jira:${k}`), cfg.sources)
+    } catch (err) {
+      story.errors.push(`scripted test cases: ${(err as Error).message.slice(0, 200)}`)
+    }
+  }
+}
+
+export async function readStory(cfg: AppConfig, ref: string): Promise<Story> {
+  const { system, key } = storyRef(ref)
+  const story: Story = { id: "story", system, title: "", description: "", acceptance: "", parents: [], scripted: [], passages: [], errors: [] }
+  if (system === "ado") await adoStory(cfg, Number(key), story)
+  else if (system === "jira") await jiraStory(cfg, key, story)
+  else {
+    const [first, ...rest] = key.split("\n")
+    const body = rest.join("\n")
+    const ac = body.split(/\n?\s*acceptance criteria:?\s*\n/i)
+    Object.assign(story, { title: first.trim().slice(0, 200), description: ac[0].trim(), acceptance: (ac[1] || "").trim() })
+  }
+  if (cfg.rag) {
+    try {
+      story.passages = (await searchRequirements(cfg.rag, `${story.title}\n${story.acceptance || story.description}`.slice(0, 1000))).map((p) => `${p.source ? `${p.source}: ` : ""}${p.text}`.slice(0, 800)).slice(0, 5)
+    } catch (err) {
+      story.errors.push(`RAG: ${(err as Error).message.slice(0, 200)}`)
+    }
+  }
+  return story
+}

package/src/guard.ts

@@ -0,0 +1,51 @@
+export function normalizeHttpUrl(raw: string | undefined | null): string | null {
+  if (raw == null) return null
+  let u = String(raw).trim()
+  if (!u) return null
+  if (!/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(u)) u = `https://${u.replace(/^\/+/, "")}`
+  try {
+    const parsed = new URL(u)
+    if (!parsed.hostname) return null
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null
+    return parsed.toString()
+  } catch {
+    return null
+  }
+}
+
+export function extractUrls(text: string): string[] {
+  return text.match(/https?:\/\/[^\s<>"']+/g) ?? []
+}
+
+function hostMatches(hostname: string, pattern: string): boolean {
+  const h = hostname.toLowerCase().replace(/\.$/, "")
+  const p = pattern.trim().toLowerCase()
+  if (!p) return false
+  if (p.startsWith("*.")) {
+    const suffix = p.slice(1)
+    return h.endsWith(suffix) || h === p.slice(2)
+  }
+  return h === p || h.endsWith(`.${p}`)
+}
+
+/** Empty allow-list means every host is allowed. Returns an error string when blocked. */
+export function hostAllowed(url: string, allowed: string[]): string | null {
+  if (!allowed.length) return null
+  let host = ""
+  try {
+    host = new URL(url).hostname
+  } catch {
+    return `Invalid URL: ${url}`
+  }
+  if (!host) return null
+  if (allowed.some((pattern) => hostMatches(host, pattern))) return null
+  return `Host not allowed: ${host}`
+}
+
+export function assertGoalUrlsAllowed(goal: string, allowed: string[]): string | null {
+  for (const url of extractUrls(goal)) {
+    const err = hostAllowed(url, allowed)
+    if (err) return err
+  }
+  return null
+}