@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/cases/confluence.ts

@@ -0,0 +1,111 @@
+import type { SourcesConfig } from "../config.ts"
+import { basic, envSecret, htmlToText, http } from "./http.ts"
+
+export type ConfluencePage = { id: string; title: string; url: string; space?: string; text: string }
+
+const PAGE_CHARS = 20_000
+const TOTAL_CHARS = 60_000
+
+export function confluenceApi(cfg: SourcesConfig) {
+  const c = cfg.confluence
+  if (!c) throw new Error("Confluence is not set up. Add sources.confluence to forgeqe.yaml (confluence: true on a Jira Cloud site), or set CONFLUENCE_URL.")
+  const token = envSecret(c.tokenEnv, "Confluence")
+  return { ...c, auth: c.cloud ? basic(envSecret(c.emailEnv, "Confluence Cloud"), token) : `Bearer ${token}` }
+}
+
+/** Page storage format as text; table cells are kept apart with " | " so requirement tables stay readable. */
+export function storageToText(html: string): string {
+  return htmlToText(
+    html
+      .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+      .replace(/<\/(td|th)\s*>/gi, " | ")
+      .replace(/<ac:parameter[^>]*>[\s\S]*?<\/ac:parameter>/gi, ""),
+  )
+    .split("\n")
+    .map((l) => l.replace(/\s*\|\s*$/, ""))
+    .join("\n")
+}
+
+type Ref = { id?: string; space?: string; title?: string }
+
+/** A page id, a page URL (…/pages/<id>/…, ?pageId=<id>), or a Server/Data Center /display/<SPACE>/<Title> URL. */
+export function pageRef(ref: string): Ref {
+  const r = ref.trim()
+  if (/^\d+$/.test(r)) return { id: r }
+  let u: URL
+  try {
+    u = new URL(r)
+  } catch {
+    throw new Error(`"${ref}" is not a Confluence page id or URL.`)
+  }
+  const pageId = u.searchParams.get("pageId")
+  if (pageId) return { id: pageId }
+  const parts = u.pathname.split("/").filter(Boolean).map(decodeURIComponent)
+  const at = parts.indexOf("pages")
+  if (at >= 0 && /^\d+$/.test(parts[at + 1] || "")) return { id: parts[at + 1] }
+  const display = parts.indexOf("display")
+  if (display >= 0 && parts[display + 2]) return { space: parts[display + 1], title: parts[display + 2].replace(/\+/g, " ") }
+  throw new Error(`Could not find the page in ${ref}; give the page id or a link to the page.`)
+}
+
+type V1Content = { id: string; title: string; space?: { key?: string }; body?: { storage?: { value?: string } }; _links?: { webui?: string; base?: string } }
+
+/** Pages by id or link, or by a CQL search, as text for the agent. Cloud uses the v2 pages API; Server/Data Center v1. */
+export async function readConfluence(cfg: SourcesConfig, q: { pages?: string[]; cql?: string; limit?: number }): Promise<{ pages: ConfluencePage[]; note?: string }> {
+  const c = confluenceApi(cfg)
+  const limit = Math.min(Math.max(1, q.limit || 10), 25)
+  const web = (path?: string) => (path ? `${c.url}${path}` : c.url)
+  const v1 = async (id: string): Promise<ConfluencePage> => {
+    const { data } = await http<V1Content>(`${c.url}/rest/api/content/${id}?expand=body.storage,space`, { auth: c.auth })
+    return { id: data.id, title: data.title, url: web(data._links?.webui), space: data.space?.key, text: storageToText(data.body?.storage?.value || "") }
+  }
+  const byId = async (id: string): Promise<ConfluencePage> => {
+    if (!c.cloud) return v1(id)
+    const { data } = await http<{ id: string; title: string; spaceId?: string; body?: { storage?: { value?: string } }; _links?: { webui?: string } }>(
+      `${c.url}/api/v2/pages/${id}?body-format=storage`,
+      { auth: c.auth },
+    )
+    return { id: data.id, title: data.title, url: web(data._links?.webui), text: storageToText(data.body?.storage?.value || "") }
+  }
+  const ids: string[] = []
+  for (const p of q.pages || []) {
+    const ref = pageRef(p)
+    if (ref.id) ids.push(ref.id)
+    else {
+      const params = new URLSearchParams({ spaceKey: ref.space!, title: ref.title!, limit: "1" })
+      const { data } = await http<{ results?: { id: string }[] }>(`${c.url}/rest/api/content?${params}`, { auth: c.auth })
+      const hit = data?.results?.[0]
+      if (!hit) throw new Error(`No Confluence page "${ref.title}" in space ${ref.space}.`)
+      ids.push(hit.id)
+    }
+  }
+  if (q.cql) {
+    const cql = c.space && !/\bspace\b/i.test(q.cql) ? `space = "${c.space}" AND (${q.cql})` : q.cql
+    const params = new URLSearchParams({ cql, limit: String(limit) })
+    const path = c.cloud ? `/rest/api/search?${params}` : `/rest/api/content/search?${params}`
+    const { data } = await http<{ results?: ({ content?: { id: string; type?: string } } & { id?: string; type?: string })[] }>(`${c.url}${path}`, { auth: c.auth })
+    for (const r of data?.results || []) {
+      const hit = r.content || r
+      if (hit?.id && (hit.type || "page") === "page") ids.push(hit.id)
+    }
+  }
+  if (!ids.length) throw new Error(q.cql ? `No Confluence pages match: ${q.cql}` : "Give page ids or links, or a CQL query.")
+  const pages: ConfluencePage[] = []
+  let total = 0
+  let cut = false
+  for (const id of [...new Set(ids)].slice(0, limit)) {
+    const page = await byId(id)
+    const room = Math.min(PAGE_CHARS, TOTAL_CHARS - total)
+    if (room <= 0) {
+      cut = true
+      break
+    }
+    if (page.text.length > room) {
+      page.text = `${page.text.slice(0, room)}… (cut)`
+      cut = true
+    }
+    total += page.text.length
+    pages.push(page)
+  }
+  return { pages, ...(cut ? { note: "Some text was cut; read fewer pages at a time for the rest." } : {}) }
+}

package/src/cases/context.ts

@@ -0,0 +1,223 @@
+import type { AppConfig } from "../config.ts"
+import { adoSettings } from "./ado.ts"
+import { readConfluence } from "./confluence.ts"
+import { adfToText, htmlToText, http } from "./http.ts"
+import { jiraApi } from "./jira.ts"
+import { searchRequirements, type Passage } from "./rag.ts"
+import type { TestCase } from "./types.ts"
+
+/**
+ * What a test case is for, gathered for the quality check: the requirements above it (the story or PBI it tests,
+ * then its feature and epic, with descriptions and acceptance criteria), the Confluence pages they link to, and, when
+ * a RAG service is configured, the passages it finds for the case. Requirements and pages are shared between the
+ * cases of one check, so each is read once.
+ */
+
+export type Requirement = {
+  id: string
+  type: string
+  title: string
+  url?: string
+  description?: string
+  acceptance?: string
+  parent?: string
+  /** Azure DevOps Area Path: where the team files it in the product. */
+  area?: string
+  /** Azure DevOps test cases that test it ("Tested By"), as ADO-<id>. */
+  testedBy?: string[]
+}
+export type Doc = { title: string; url: string; text: string }
+
+export type CaseContext = {
+  /** Requirement ids, nearest first: the story, then its feature, then its epic. */
+  chain: string[]
+  docs: string[]
+  passages: Passage[]
+  /** Acceptance criteria no step seems to cover: hints for the agent, by shared words. */
+  uncovered: string[]
+  errors: string[]
+}
+
+export type Context = { requirements: Map<string, Requirement>; docs: Map<string, Doc>; cases: Map<string, CaseContext> }
+
+const MAX_TEXT = 2500
+const MAX_LEVELS = 3
+const clip = (s: string | undefined, n = MAX_TEXT) => {
+  const t = (s || "").trim()
+  return t.length > n ? `${t.slice(0, n - 1)}…` : t
+}
+
+const STOP = new Set("the a an and or of to in on for with is are be should must can will user users when then that this from into able page shown show shows display displayed form forms field fields button buttons click enter valid value values after before".split(" "))
+const keywords = (s: string) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4 && !STOP.has(w)))
+
+/**
+ * Acceptance criteria (one per line, bullet, or numbered item) that share few of their meaningful words with the
+ * case's steps: a cheap hint of a gap. It cannot tell "validates the email format" from "enters an email"; the agent
+ * judges coverage from the full criteria.
+ */
+export function uncoveredCriteria(acceptance: string | undefined, tc: TestCase): string[] {
+  if (!acceptance?.trim()) return []
+  const steps = keywords(tc.steps.map((s) => `${s.action} ${s.data || ""} ${s.expected || ""}`).join(" ") + ` ${tc.title}`)
+  return acceptance
+    .split(/\n|\s(?=(?:\d+[.)]|[-•*])\s)/)
+    .map((c) => c.replace(/^\s*(?:\d+[.)]|[-•*])\s+/, "").trim())
+    .filter((c) => c.length >= 12)
+    .filter((c) => {
+      const words = [...keywords(c)]
+      if (!words.length) return false
+      return words.filter((w) => steps.has(w)).length / words.length < 0.5
+    })
+    .slice(0, 10)
+}
+
+/** Confluence links in a text: the configured site, or any *.atlassian.net/wiki page. */
+function confluenceLinks(text: string, cfg: AppConfig): string[] {
+  const host = cfg.sources.confluence ? new URL(cfg.sources.confluence.url).host : ""
+  return [...new Set([...(text || "").matchAll(/https?:\/\/[^\s"'<>)]+/g)].map((m) => m[0].replace(/[.,;]+$/, "")))].filter(
+    (u) => (host && u.includes(host)) || /atlassian\.net\/wiki\//.test(u) || /\/(display|pages\/viewpage\.action)/.test(u),
+  )
+}
+
+type AdoItem = { id: number; fields: Record<string, unknown>; relations?: { rel: string; url: string }[] }
+
+const idOf = (url: string) => Number(url.split("/").pop())
+
+async function readAdo(cfg: AppConfig, id: number): Promise<AdoItem> {
+  const { org, auth } = adoSettings(cfg.sources)
+  const { data } = await http<AdoItem>(`${org}/_apis/wit/workitems/${id}?$expand=all&api-version=7.1`, { auth })
+  return data
+}
+
+/**
+ * Azure DevOps requirements from the given work items up through their parents (story, feature, epic), each read
+ * once into ctx.requirements. Returns the ids in order, nearest first, and the links (Hyperlink relations and
+ * Confluence links in their text) they carry.
+ */
+export async function adoRequirements(cfg: AppConfig, ids: number[], ctx: Pick<Context, "requirements">): Promise<{ chain: string[]; links: string[] }> {
+  const { org } = adoSettings(cfg.sources)
+  const chain: string[] = []
+  const links: string[] = []
+  let next = [...new Set(ids)]
+  for (let level = 0; level < MAX_LEVELS && next.length; level++) {
+    const parents: number[] = []
+    for (const id of next.slice(0, 5)) {
+      const key = `ADO-${id}`
+      let req = ctx.requirements.get(key)
+      if (!req) {
+        const item = await readAdo(cfg, id)
+        const f = item.fields
+        const rel = item.relations || []
+        const parent = rel.find((r) => r.rel === "System.LinkTypes.Hierarchy-Reverse")
+        const testedBy = rel.filter((r) => r.rel === "Microsoft.VSTS.Common.TestedBy-Forward").map((r) => `ADO-${idOf(r.url)}`)
+        req = {
+          id: key,
+          type: String(f["System.WorkItemType"] || "Work item"),
+          title: String(f["System.Title"] || ""),
+          url: `${org}/${encodeURIComponent(String(f["System.TeamProject"] || ""))}/_workitems/edit/${id}`,
+          description: clip(htmlToText(String(f["System.Description"] || "") || String(f["Microsoft.VSTS.TCM.ReproSteps"] || ""))),
+          acceptance: clip(htmlToText(String(f["Microsoft.VSTS.Common.AcceptanceCriteria"] || ""))),
+          ...(parent ? { parent: `ADO-${idOf(parent.url)}` } : {}),
+          ...(f["System.AreaPath"] ? { area: String(f["System.AreaPath"]) } : {}),
+          ...(testedBy.length ? { testedBy } : {}),
+        }
+        ctx.requirements.set(key, req)
+        links.push(...rel.filter((r) => r.rel === "Hyperlink").map((r) => r.url))
+        links.push(...confluenceLinks(`${f["System.Description"] || ""} ${f["Microsoft.VSTS.Common.AcceptanceCriteria"] || ""}`, cfg))
+      }
+      if (!chain.includes(key)) chain.push(key)
+      if (req.parent) parents.push(Number(req.parent.slice(4)))
+    }
+    next = [...new Set(parents)].filter((p) => !chain.includes(`ADO-${p}`))
+  }
+  return { chain, links }
+}
+
+async function adoChain(tc: TestCase, cfg: AppConfig, ctx: Context): Promise<{ chain: string[]; links: string[] }> {
+  const testCase = await readAdo(cfg, Number(tc.ref?.workItem))
+  const links: string[] = (testCase.relations || []).filter((r) => r.rel === "Hyperlink").map((r) => r.url)
+  // "Tests": the requirement this test case tests (a story, PBI, bug, or feature), then up through its parents.
+  const tested = (testCase.relations || []).filter((r) => r.rel === "Microsoft.VSTS.Common.TestedBy-Reverse").map((r) => idOf(r.url))
+  const up = await adoRequirements(cfg, tested.slice(0, 3), ctx)
+  return { chain: up.chain, links: [...links, ...up.links] }
+}
+
+async function jiraChain(tc: TestCase, cfg: AppConfig, ctx: Context): Promise<{ chain: string[]; links: string[] }> {
+  const j = jiraApi(cfg.sources, "Reading requirements from Jira")
+  const read = async (key: string) => {
+    const { data } = await http<{ key: string; fields: Record<string, any> }>(`${j.api}/issue/${encodeURIComponent(key)}?fields=summary,description,issuetype,parent,issuelinks`, { auth: j.auth })
+    return data
+  }
+  const key = String(tc.ref?.key || tc.id)
+  const test = await read(key)
+  const text = (d: unknown) => (typeof d === "string" ? d : adfToText(d))
+  const links: string[] = confluenceLinks(text(test.fields.description), cfg)
+  // The issues the test is linked to as tests / is tested by (or relates to, when nothing says tests), then parents.
+  const linked = (test.fields.issuelinks || []) as { type: { name: string; inward: string; outward: string }; inwardIssue?: { key: string }; outwardIssue?: { key: string } }[]
+  const testLinks = linked.filter((l) => /test/i.test(`${l.type.name} ${l.type.inward} ${l.type.outward}`))
+  let next = (testLinks.length ? testLinks : linked).map((l) => (l.inwardIssue || l.outwardIssue)?.key).filter((k): k is string => Boolean(k))
+  if (!next.length && test.fields.parent?.key) next = [test.fields.parent.key]
+  const chain: string[] = []
+  for (let level = 0; level < MAX_LEVELS && next.length; level++) {
+    const parents: string[] = []
+    for (const k of next.slice(0, 3)) {
+      let req = ctx.requirements.get(k)
+      if (!req) {
+        const issue = await read(k)
+        const description = text(issue.fields.description)
+        req = {
+          id: k,
+          type: String(issue.fields.issuetype?.name || "Issue"),
+          title: String(issue.fields.summary || ""),
+          url: `${j.url}/browse/${k}`,
+          description: clip(description),
+          ...(issue.fields.parent?.key ? { parent: String(issue.fields.parent.key) } : {}),
+        }
+        ctx.requirements.set(k, req)
+        links.push(...confluenceLinks(description, cfg))
+      }
+      chain.push(k)
+      if (req.parent) parents.push(req.parent)
+    }
+    next = [...new Set(parents)].filter((p) => !chain.includes(p))
+  }
+  return { chain, links }
+}
+
+/**
+ * Gather each case's context. Errors (no access to a work item, Confluence not set up, the RAG service down) are kept
+ * per case and the rest goes on: a check with less context still runs.
+ */
+export async function gatherContext(cases: TestCase[], cfg: AppConfig): Promise<Context> {
+  const ctx: Context = { requirements: new Map(), docs: new Map(), cases: new Map() }
+  for (const tc of cases) {
+    const c: CaseContext = { chain: [], docs: [], passages: [], uncovered: [], errors: [] }
+    ctx.cases.set(tc.id, c)
+    let links: string[] = []
+    try {
+      if (tc.source === "ado" && tc.ref?.workItem && cfg.sources.ado) ({ chain: c.chain, links } = await adoChain(tc, cfg, ctx))
+      else if ((tc.source === "jira" || tc.source === "xray") && cfg.sources.jira) ({ chain: c.chain, links } = await jiraChain(tc, cfg, ctx))
+    } catch (err) {
+      c.errors.push(`requirements: ${(err as Error).message.slice(0, 200)}`)
+    }
+    const pages = [...new Set(links)].filter((u) => !ctx.docs.has(u)).slice(0, 3)
+    if (pages.length && cfg.sources.confluence) {
+      try {
+        const { pages: read } = await readConfluence(cfg.sources, { pages })
+        read.forEach((p, i) => ctx.docs.set(pages[i] || p.url, { title: p.title, url: p.url, text: clip(p.text, 4000) }))
+      } catch (err) {
+        c.errors.push(`Confluence: ${(err as Error).message.slice(0, 200)}`)
+      }
+    }
+    c.docs = [...new Set(links)].filter((u) => ctx.docs.has(u))
+    if (cfg.rag) {
+      try {
+        c.passages = await searchRequirements(cfg.rag, [tc.title, ...tc.steps.map((s) => s.action)].join("\n").slice(0, 1000))
+      } catch (err) {
+        c.errors.push(`RAG: ${(err as Error).message.slice(0, 200)}`)
+      }
+    }
+    const story = c.chain[0] ? ctx.requirements.get(c.chain[0]) : undefined
+    c.uncovered = uncoveredCriteria(story?.acceptance, tc)
+  }
+  return ctx
+}

package/src/cases/dashboards.ts

@@ -0,0 +1,137 @@
+import type { SourcesConfig } from "../config.ts"
+import { adoSettings } from "./ado.ts"
+import { http } from "./http.ts"
+
+const API = "api-version=7.1"
+const DASHBOARD_API = "api-version=7.1-preview.3"
+/** Shared queries the dashboard's widgets read, kept in one folder so ForgeQE can find and update them. */
+const FOLDER = "ForgeQE"
+const ROOT = "Shared Queries"
+
+export type DashboardWidget =
+  /** `list`: the query's work items with its columns. `count`: a tile with how many items it returns. */
+  | { kind?: "list" | "count"; title: string; wiql: string }
+  | { kind: "markdown"; title?: string; text: string }
+
+export type DashboardRequest = {
+  /** Project the queries and dashboard go in; sources.azure_devops.project when left out. */
+  project?: string
+  /** A team dashboard when set; otherwise a project dashboard. */
+  team?: string
+  name: string
+  description?: string
+  widgets: DashboardWidget[]
+}
+
+export type DashboardResult = {
+  id: string
+  name: string
+  url: string
+  project: string
+  team?: string
+  queries: { name: string; path: string; id: string; created: boolean; url: string }[]
+  widgets: string[]
+}
+
+type QueryItem = { id: string; name: string; path: string; isFolder?: boolean }
+
+const CONTRIBUTIONS = {
+  list: "ms.vss-mywork-web.Microsoft.VisualStudioOnline.MyWork.WitViewWidget",
+  count: "ms.vss-dashboards-web.Microsoft.VisualStudioOnline.Dashboards.QueryScalarWidget",
+  markdown: "ms.vss-dashboards-web.Microsoft.VisualStudioOnline.Dashboards.MarkdownWidget",
+} as const
+
+/**
+ * Saves each widget's WIQL as a shared query under Shared Queries/ForgeQE (updating one of the same name), then
+ * creates the dashboard with a widget per query: a work item list or a count tile, plus any markdown text.
+ * Every query is run once first, so a bad WIQL fails before anything is saved.
+ */
+export async function createAdoDashboard(cfg: SourcesConfig, req: DashboardRequest): Promise<DashboardResult> {
+  const { auth, org, project: configured } = adoSettings(cfg)
+  const project = (req.project || configured || "").trim()
+  if (!project) throw new Error("Name the Azure DevOps project for the dashboard, or set sources.azure_devops.project.")
+  const name = req.name.trim()
+  if (!name) throw new Error("The dashboard needs a name.")
+  if (!req.widgets.length) throw new Error("The dashboard needs at least one widget.")
+  const base = `${org}/${encodeURIComponent(project)}`
+
+  const queryWidgets = req.widgets.filter((w): w is Extract<DashboardWidget, { wiql: string }> => w.kind !== "markdown")
+  for (const w of queryWidgets) {
+    if (!w.title?.trim()) throw new Error("Each list or count widget needs a title; it also names its saved query.")
+    if (!w.wiql?.trim()) throw new Error(`Widget "${w.title}" needs a WIQL query.`)
+    try {
+      await http(`${base}/_apis/wit/wiql?${API}&$top=1`, { auth, body: { query: w.wiql } })
+    } catch (err) {
+      throw new Error(`The query for "${w.title}" was rejected by Azure DevOps, so nothing was saved: ${(err as Error).message}`)
+    }
+  }
+
+  if (queryWidgets.length) await ensureFolder(base, auth)
+  const saved = new Map<string, DashboardResult["queries"][number]>()
+  for (const w of queryWidgets) {
+    const title = w.title.trim()
+    if (!saved.has(title)) saved.set(title, await saveQuery(base, auth, title, w.wiql))
+  }
+
+  const tiles = req.widgets.filter((w) => w.kind === "count")
+  const panels = req.widgets.filter((w) => w.kind !== "count")
+  const tileRows = Math.ceil(tiles.length / 8)
+  const widgets = [
+    ...tiles.map((w, i) => {
+      const q = saved.get((w as { title: string }).title.trim())!
+      const at = { row: 1 + Math.floor(i / 8), column: 1 + (i % 8) }
+      return widget(q.name, CONTRIBUTIONS.count, JSON.stringify({ queryId: q.id, queryName: q.name }), at, { rowSpan: 1, columnSpan: 1 })
+    }),
+    ...panels.map((w, i) => {
+      const at = { row: 1 + tileRows + Math.floor(i / 2) * 3, column: i % 2 ? 5 : 1 }
+      const size = { rowSpan: 3, columnSpan: 4 }
+      if (w.kind === "markdown") return widget(w.title?.trim() || "Notes", CONTRIBUTIONS.markdown, w.text, at, size)
+      const q = saved.get(w.title.trim())!
+      return widget(q.name, CONTRIBUTIONS.list, JSON.stringify({ query: { queryId: q.id, queryName: q.name } }), at, size)
+    }),
+  ]
+
+  const scope = req.team?.trim() ? `${base}/${encodeURIComponent(req.team.trim())}` : base
+  const { data } = await http<{ id: string; name: string }>(`${scope}/_apis/dashboard/dashboards?${DASHBOARD_API}`, {
+    auth,
+    body: { name, ...(req.description ? { description: req.description } : {}), widgets },
+  })
+  return {
+    id: data.id,
+    name: data.name || name,
+    url: `${base}/_dashboards/dashboard/${data.id}`,
+    project,
+    ...(req.team?.trim() ? { team: req.team.trim() } : {}),
+    queries: [...saved.values()],
+    widgets: widgets.map((w) => w.name),
+  }
+}
+
+function widget(name: string, contributionId: string, settings: string, position: { row: number; column: number }, size: { rowSpan: number; columnSpan: number }) {
+  return { name, position, size, settings, settingsVersion: { major: 1, minor: 0, patch: 0 }, contributionId }
+}
+
+const queryPath = (...parts: string[]) => parts.map(encodeURIComponent).join("/")
+
+async function findQuery(base: string, auth: string, ...parts: string[]): Promise<QueryItem | null> {
+  try {
+    return (await http<QueryItem>(`${base}/_apis/wit/queries/${queryPath(...parts)}?${API}`, { auth })).data
+  } catch (err) {
+    if (/failed: 404\b/.test((err as Error).message)) return null
+    throw err
+  }
+}
+
+async function ensureFolder(base: string, auth: string): Promise<void> {
+  if (await findQuery(base, auth, ROOT, FOLDER)) return
+  await http(`${base}/_apis/wit/queries/${queryPath(ROOT)}?${API}`, { auth, body: { name: FOLDER, isFolder: true } })
+}
+
+async function saveQuery(base: string, auth: string, name: string, wiql: string): Promise<DashboardResult["queries"][number]> {
+  const existing = await findQuery(base, auth, ROOT, FOLDER, name)
+  const { data } = existing
+    ? await http<QueryItem>(`${base}/_apis/wit/queries/${existing.id}?${API}`, { auth, method: "PATCH", body: { wiql } })
+    : await http<QueryItem>(`${base}/_apis/wit/queries/${queryPath(ROOT, FOLDER)}?${API}`, { auth, body: { name, wiql } })
+  const id = data?.id || existing?.id || ""
+  return { name, path: data?.path || `${ROOT}/${FOLDER}/${name}`, id, created: !existing, url: `${base}/_queries/query/${id}/` }
+}

package/src/cases/failure.ts

@@ -0,0 +1,97 @@
+import type { RunResult, StepCause } from "../types.ts"
+
+/**
+ * When a test case's browser run does not pass, what kind of failure it is decides what happens next:
+ *
+ * - functional_defect: the steps were done and the application behaves wrong. A bug is filed, and a test is still
+ *   written: it asserts the correct behaviour and is marked as an expected failure linked to the bug.
+ * - test_case_issue: the test case does not fit the application (a field or page it names is not there). No bug and no
+ *   test: the case needs updating, and the report says what the application shows instead.
+ * - environment: the application or something it needs was unavailable (an error page, a 5xx, a network failure,
+ *   sign-in or test data missing, a bot check). The run is repeated once; no bug and no test.
+ * - automation: a control that is there could not be operated, or the run ran out of steps. No bug and no test yet:
+ *   page memory keeps the steps that worked and the next run resumes from them. (A coding job that does not finish,
+ *   whatever the failure, moves its unfinished test to the needs-attention folder: src/code/attention.ts.)
+ */
+export type FailureType = "functional_defect" | "test_case_issue" | "environment" | "automation"
+
+export type Failure = {
+  type: FailureType
+  /** One sentence: why this type, from the run. */
+  reason: string
+  /** The first failing test step, when the case has numbered steps. */
+  step?: number
+}
+
+export const FAILURE_LABEL: Record<FailureType, string> = {
+  functional_defect: "functional defect",
+  test_case_issue: "test case needs updating",
+  environment: "environment problem",
+  automation: "automation problem",
+}
+
+/** What ForgeQE does about each type, for the agent's reply and the report. */
+export const FAILURE_ACTION: Record<FailureType, string> = {
+  functional_defect: "a bug is filed and a test is written that asserts the correct behaviour, marked as an expected failure linked to the bug",
+  test_case_issue: "no bug and no test: update the test case to match the application",
+  environment: "repeated once; no bug and no test until the environment works",
+  automation: "no bug and no test yet: the steps that worked are kept, and the next run resumes from them",
+}
+
+const CAUSE: Record<StepCause, FailureType> = { app: "functional_defect", test_case: "test_case_issue", environment: "environment", automation: "automation" }
+
+/** Pages and errors that mean the application or its network was not there to test, whatever the agent concluded. */
+const ENVIRONMENT = [
+  /ERR_(NAME_NOT_RESOLVED|CONNECTION_(REFUSED|RESET|TIMED_OUT|CLOSED)|INTERNET_DISCONNECTED|NETWORK_CHANGED|SSL_PROTOCOL_ERROR|CERT_)/i,
+  /\b(502 Bad Gateway|503 Service Unavailable|504 Gateway Time-?out|Service Unavailable|site can'?t be reached)\b/i,
+  /\b(captcha|verify you are (a )?human|access denied|request blocked|unusual traffic)\b/i,
+]
+
+/** The engine gave up for its own reasons: the step budget, a loop, or the agent's model. */
+const AUTOMATION = [/\b(max(imum)? steps|step limit|ran out of steps)\b/i, /Stopped after repeated/i, /\b(rate limit|ProviderAuthError|APIError|context length)\b/i]
+
+/**
+ * The kind of failure of a case's browser run, or undefined when it passed or was stopped. The browser agent's own
+ * cause for the first failing step decides, since it saw the page. Without one, or when it blamed its own automation,
+ * clear evidence in the run (an unreachable site, an error page) makes it an environment problem.
+ */
+export function classifyFailure(run: RunResult | undefined, error?: string): Failure | undefined {
+  if (!run) {
+    const text = error || ""
+    if (ENVIRONMENT.some((r) => r.test(text))) return { type: "environment", reason: `The run could not start: ${clip(text)}` }
+    return { type: "automation", reason: `The browser run could not finish: ${clip(text) || "the engine stopped"}` }
+  }
+  if (run.status === "complete" || run.status === "stopped") return undefined
+  const text = [run.summary, ...(run.caseReport?.steps || []).map((s) => s.actual || "")].join("\n")
+  const failed = (run.caseReport?.steps || []).find((s) => s.status === "failed")
+  // The agent's judgement wins where it named the application or the test case: a test about an "Access denied"
+  // message must not read as a blocked site. Error pages and network failures override "could not operate it".
+  if (failed?.cause === "app" || failed?.cause === "test_case" || failed?.cause === "environment") {
+    return { type: CAUSE[failed.cause], reason: `Step ${failed.step}: ${clip(failed.actual || run.summary)}`, step: failed.step }
+  }
+  if (ENVIRONMENT.some((r) => r.test(text))) return { type: "environment", reason: `The application was not available to test: ${clip(firstMatch(text, ENVIRONMENT))}`, ...(failed ? { step: failed.step } : {}) }
+  if (failed?.cause) return { type: CAUSE[failed.cause], reason: `Step ${failed.step}: ${clip(failed.actual || run.summary)}`, step: failed.step }
+  if (AUTOMATION.some((r) => r.test(run.summary))) return { type: "automation", reason: clip(run.summary), ...(failed ? { step: failed.step } : {}) }
+  // The steps were done and an expected result does not hold: the application, unless the agent said otherwise above.
+  if (run.outcome === "assertion_failed") {
+    const what = (run.failedChecks || []).map((c) => `expected ${c.expected}, the page shows ${c.actual}`).join("; ")
+    return { type: "functional_defect", reason: clip(what || run.summary), ...(failed ? { step: failed.step } : {}) }
+  }
+  return { type: "automation", reason: clip(run.summary) || "the browser agent could not finish the steps", ...(failed ? { step: failed.step } : {}) }
+}
+
+function firstMatch(text: string, patterns: RegExp[]): string {
+  for (const r of patterns) {
+    const m = r.exec(text)
+    if (m) {
+      const line = text.split("\n").find((l) => l.includes(m[0])) || m[0]
+      return line
+    }
+  }
+  return text
+}
+
+const clip = (s: string, n = 300) => {
+  const t = (s || "").replace(/\s+/g, " ").trim()
+  return t.length > n ? `${t.slice(0, n - 1)}…` : t
+}