@capagents/forgeqe 1.2.1 → 1.3.0

package/src/cases/save.ts

@@ -1,7 +1,8 @@
 import type { SourcesConfig } from "../config.ts"
-import { adoSettings } from "./ado.ts"
-import { http, textToAdf } from "./http.ts"
+import { adoSettings, parseSteps } from "./ado.ts"
+import { adfToText, http, textToAdf } from "./http.ts"
 import { jiraApi } from "./jira.ts"
+import { stepText } from "./types.ts"
 
 /** A test case the agent wrote from requirements, to save in Azure DevOps or Jira. */
 export type NewCase = {
@@ -11,13 +12,28 @@ export type NewCase = {
   description?: string
 }
 
-/** One saved case: created now, or found already linked to the story under the same title (never duplicated). */
-export type SavedCase = { title: string; id: string; url: string; created: boolean; error?: string }
+/** A test case already linked to the story, with its steps as its system holds them. */
+export type ExistingCase = { id: string; title: string; url: string; steps: NewCase["steps"] }
 
-export type SaveResult = { system: "azure_devops" | "jira"; story: { id: string; title: string; url: string }; cases: SavedCase[] }
+/** The story and the test cases already linked to it. */
+export type StoryCases = { system: "azure_devops" | "jira"; story: { id: string; title: string; url: string }; cases: ExistingCase[] }
+
+/**
+ * What saving did with one case, never duplicating one already linked to the story:
+ * - created: a new test case.
+ * - unchanged: a linked case has the same steps (under this title or another): nothing was written.
+ * - differs: a linked case has this title but other steps: nothing was written; update it with update_test_case.
+ * - similar: a linked case has a close title or most of the same steps: nothing was written; update that case, or
+ *   save again with this title in force_new when it really is a different case.
+ * - failed: the system refused it (error).
+ */
+export type SaveStatus = "created" | "unchanged" | "differs" | "similar" | "failed"
+
+export type SavedCase = { title: string; id: string; url: string; created: boolean; status: SaveStatus; existing?: ExistingCase; error?: string }
+
+export type SaveResult = Omit<StoryCases, "cases"> & { cases: SavedCase[]; existing: ExistingCase[] }
 
 const API = "api-version=7.1"
-const sameTitle = (a: string, b: string) => a.trim().toLowerCase() === b.trim().toLowerCase()
 
 const escXml = (s: string) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;")
 /** One step string as Azure DevOps stores it: HTML, escaped again inside the steps XML. */
@@ -43,11 +59,87 @@ function describe(c: NewCase): string[] {
   return out
 }
 
+/** Steps as ForgeQE writes them in a Jira description, after "Steps:". */
+const stepLines = (steps: NewCase["steps"]) => steps.map((s, i) => `${i + 1}. ${s.action}${s.expected ? `\n   Expected: ${s.expected}` : ""}`)
+
+/** Numbered steps (`1. …`, with `Expected: …` lines) from a description, as ForgeQE writes them in Jira. */
+export function stepsFromText(text: string): NewCase["steps"] {
+  const lines = text.split(/\r?\n/)
+  const start = lines.findIndex((l) => /^\s*steps:\s*$/i.test(l))
+  const steps: NewCase["steps"] = []
+  for (const line of lines.slice(start + 1)) {
+    const step = /^\s*\d+[.)]\s+(.+)$/.exec(line)
+    const expected = /^\s*expected(?: result)?:\s*(.+)$/i.exec(line)
+    if (step) steps.push({ action: step[1].trim() })
+    else if (expected && steps.length) steps[steps.length - 1].expected = expected[1].trim()
+  }
+  return steps
+}
+
+// ---------------------------------------------------------------------------------------------
+// Matching a new case to the cases already linked to the story
+// ---------------------------------------------------------------------------------------------
+
+/** Words that change how a title reads, not what it tests: "Verify that the user can sign in" ≈ "Sign in". */
+const FILLER = new Set("a an and are as be can check checks correct correctly ensure for from if in is it of on should successfully test tests that the to user users validate verify verifies when with".split(" "))
+/** A numbering prefix teams put on titles ("TC-10 ", "AC02 - ", "PBI 2237 - ", "E2E: "), which says nothing about what is tested. */
+const NUMBERING = /^\s*(?:(?:tc|ac|ado|pbi|us|story|test|case|e2e)[-_ ]?\d*[a-z]?|\d+)\s*[-:.)]?\s+/i
+/** Tokens of ids anywhere in a title ("TC_2211_REQ_002", "for PBI 2211"): numbers, and the kinds of id they belong to. */
+const ID_WORD = /\d|^(?:tc|ac|req|pbi|ado)$/
+const titleWords = (t: string) => new Set(stepText(t.replace(NUMBERING, "")).split(" ").filter((w) => w && !FILLER.has(w) && !ID_WORD.test(w)))
+const stepWords = (s: NewCase["steps"][number]) => `${stepText(s.action)}|${stepText(s.expected)}`
+const sameSteps = (a: NewCase["steps"], b: NewCase["steps"]) => a.length === b.length && a.every((s, i) => stepWords(s) === stepWords(b[i]))
+
 /**
- * Create each case as an Azure DevOps Test Case linked to the story (Tests / Tested By), in the story's project, area,
- * and iteration. A case whose title matches a test case already linked to the story is returned instead of created.
+ * How close two titles are: the share of words they have in common (Jaccard), or, when the shorter title has at least
+ * three words, the share of the shorter title's words the longer one has ("Start quote journey from landing page" is in
+ * "TC-10 Launch quote journey from landing page to step 1").
  */
-export async function saveAdoCases(cfg: SourcesConfig, storyId: number, cases: NewCase[]): Promise<SaveResult> {
+function titleScore(a: Set<string>, b: Set<string>): number {
+  if (!a.size || !b.size) return 0
+  let both = 0
+  for (const w of a) if (b.has(w)) both++
+  const jaccard = both / (a.size + b.size - both)
+  const shorter = Math.min(a.size, b.size)
+  return shorter >= 3 ? Math.max(jaccard, both / shorter) : jaccard
+}
+
+/** Share of `a`'s steps that `b` has too, in any order. */
+function stepOverlap(a: NewCase["steps"], b: NewCase["steps"]): number {
+  if (!a.length || !b.length) return 0
+  const have = new Set(b.map(stepWords))
+  return a.filter((s) => have.has(stepWords(s))).length / a.length
+}
+
+/** Titles this close (titleScore, filler words and numbering left out), or this share of the same steps, are alike. */
+const SIMILAR_TITLE = 0.8
+const SIMILAR_STEPS = 0.8
+
+/** How a new case stands against the cases already linked to the story; see SaveStatus. */
+export function matchCase(c: NewCase, existing: ExistingCase[], forceNew = false): { status: Exclude<SaveStatus, "created" | "failed">; found: ExistingCase } | undefined {
+  const sameTitle = existing.find((e) => stepText(e.title) === stepText(c.title))
+  if (sameTitle) return { status: sameSteps(sameTitle.steps, c.steps) ? "unchanged" : "differs", found: sameTitle }
+  const reworded = c.steps.length ? existing.find((e) => sameSteps(e.steps, c.steps)) : undefined
+  if (reworded) return { status: "unchanged", found: reworded }
+  if (forceNew) return undefined
+  const words = titleWords(c.title)
+  let best: { score: number; found: ExistingCase } | undefined
+  for (const e of existing) {
+    const t = titleScore(words, titleWords(e.title))
+    const s = c.steps.length >= 2 && e.steps.length >= 2 ? Math.min(stepOverlap(c.steps, e.steps), stepOverlap(e.steps, c.steps)) : 0
+    const score = t >= SIMILAR_TITLE || s >= SIMILAR_STEPS ? Math.max(t, s) : 0
+    if (score && (!best || score > best.score)) best = { score, found: e }
+  }
+  return best ? { status: "similar", found: best.found } : undefined
+}
+
+// ---------------------------------------------------------------------------------------------
+// Azure DevOps
+// ---------------------------------------------------------------------------------------------
+
+type AdoStory = StoryCases & { project: string; area: string; iteration: string; base: string; org: string; auth: string }
+
+async function adoStory(cfg: SourcesConfig, storyId: number): Promise<AdoStory> {
   const { org, auth } = adoSettings(cfg)
   const { data: story } = await http<{ id: number; fields: Record<string, string>; relations?: { rel: string; url: string }[] }>(
     `${org}/_apis/wit/workitems/${storyId}?$expand=relations&${API}`,
@@ -60,57 +152,141 @@ export async function saveAdoCases(cfg: SourcesConfig, storyId: number, cases: N
     .filter((r) => r.rel === "Microsoft.VSTS.Common.TestedBy-Forward")
     .map((r) => Number(r.url.split("/").pop()))
     .filter(Boolean)
-  const existing = linked.length
-    ? ((await http<{ value: { id: number; fields: Record<string, string> }[] }>(`${org}/_apis/wit/workitemsbatch?${API}`, { auth, body: { ids: linked, fields: ["System.Title"], errorPolicy: "omit" } })).data?.value || []).filter(Boolean)
+  const items = linked.length
+    ? ((await http<{ value: ({ id: number; fields: Record<string, string> } | null)[] }>(`${org}/_apis/wit/workitemsbatch?${API}`, { auth, body: { ids: linked, fields: ["System.Title", "Microsoft.VSTS.TCM.Steps"], errorPolicy: "omit" } })).data?.value || []).filter(
+        (w): w is { id: number; fields: Record<string, string> } => Boolean(w),
+      )
     : []
+  const cases: ExistingCase[] = []
+  for (const w of items) {
+    const steps = await parseSteps(String(w.fields["Microsoft.VSTS.TCM.Steps"] || ""), base, auth)
+    cases.push({ id: String(w.id), title: w.fields["System.Title"] || "", url: link(w.id), steps: steps.map((s) => ({ action: s.action, ...(s.expected ? { expected: s.expected } : {}) })) })
+  }
+  return {
+    system: "azure_devops",
+    story: { id: String(storyId), title: story.fields["System.Title"] || "", url: link(storyId) },
+    cases,
+    project,
+    area: story.fields["System.AreaPath"],
+    iteration: story.fields["System.IterationPath"],
+    base,
+    org,
+    auth,
+  }
+}
 
+/**
+ * Create each case as an Azure DevOps Test Case linked to the story (Tests / Tested By), in the story's project, area,
+ * and iteration. A case matching one already linked to the story (see matchCase) is returned instead of created.
+ */
+export async function saveAdoCases(cfg: SourcesConfig, storyId: number, cases: NewCase[], opts: { forceNew?: string[] } = {}): Promise<SaveResult> {
+  const s = await adoStory(cfg, storyId)
+  const link = (id: number | string) => `${s.base}/_workitems/edit/${id}`
+  const known = [...s.cases]
+  const force = new Set((opts.forceNew || []).map(stepText))
   const out: SavedCase[] = []
   for (const c of cases) {
-    const found = existing.find((w) => sameTitle(w.fields["System.Title"] || "", c.title))
-    if (found) {
-      out.push({ title: c.title, id: String(found.id), url: link(found.id), created: false })
+    const match = matchCase(c, known, force.has(stepText(c.title)))
+    if (match) {
+      out.push({ title: c.title, id: match.found.id, url: match.found.url, created: false, status: match.status, ...(match.status === "unchanged" ? {} : { existing: match.found }) })
       continue
     }
     const ops: Record<string, unknown>[] = [
       { op: "add", path: "/fields/System.Title", value: c.title.slice(0, 255) },
       { op: "add", path: "/fields/Microsoft.VSTS.TCM.Steps", value: adoStepsXml(c.steps) },
-      { op: "add", path: "/fields/System.AreaPath", value: story.fields["System.AreaPath"] },
-      { op: "add", path: "/fields/System.IterationPath", value: story.fields["System.IterationPath"] },
-      { op: "add", path: "/relations/-", value: { rel: "Microsoft.VSTS.Common.TestedBy-Reverse", url: `${org}/_apis/wit/workItems/${storyId}` } },
+      { op: "add", path: "/fields/System.AreaPath", value: s.area },
+      { op: "add", path: "/fields/System.IterationPath", value: s.iteration },
+      { op: "add", path: "/relations/-", value: { rel: "Microsoft.VSTS.Common.TestedBy-Reverse", url: `${s.org}/_apis/wit/workItems/${storyId}` } },
     ]
     const text = describe(c)
     if (text.length) ops.push({ op: "add", path: "/fields/System.Description", value: html(text) })
     try {
-      const { data } = await http<{ id: number }>(`${base}/_apis/wit/workitems/$Test%20Case?${API}`, { auth, body: ops, contentType: "application/json-patch+json" })
-      out.push({ title: c.title, id: String(data.id), url: link(data.id), created: true })
+      const { data } = await http<{ id: number }>(`${s.base}/_apis/wit/workitems/$Test%20Case?${API}`, { auth: s.auth, body: ops, contentType: "application/json-patch+json" })
+      out.push({ title: c.title, id: String(data.id), url: link(data.id), created: true, status: "created" })
+      known.push({ id: String(data.id), title: c.title, url: link(data.id), steps: c.steps })
     } catch (err) {
-      out.push({ title: c.title, id: "", url: "", created: false, error: (err as Error).message })
+      out.push({ title: c.title, id: "", url: "", created: false, status: "failed", error: (err as Error).message })
     }
   }
-  return { system: "azure_devops", story: { id: String(storyId), title: story.fields["System.Title"] || "", url: link(storyId) }, cases: out }
+  return { system: "azure_devops", story: s.story, cases: out, existing: s.cases }
 }
 
 /**
- * Create each case as a Jira issue (type `issueType`, default Test) in the story's project with its steps in the
- * description, linked to the story. A case whose summary matches an issue already linked to the story is reused.
+ * Replace an Azure DevOps Test Case's steps (and title, when given), and say why in its history. Returns its link.
+ * The previous steps stay in the work item's history, so the change can be undone there.
  */
-export async function saveJiraCases(cfg: SourcesConfig, storyKey: string, cases: NewCase[], opts: { issueType?: string; linkType?: string } = {}): Promise<SaveResult> {
+export async function updateAdoCase(cfg: SourcesConfig, id: number, change: { title?: string; steps: NewCase["steps"]; reason: string }): Promise<string> {
+  const { org, auth } = adoSettings(cfg)
+  const { data: item } = await http<{ fields: Record<string, string> }>(`${org}/_apis/wit/workitems/${id}?fields=System.TeamProject,System.WorkItemType&${API}`, { auth })
+  if (item.fields["System.WorkItemType"] && item.fields["System.WorkItemType"] !== "Test Case") throw new Error(`Work item ${id} is a ${item.fields["System.WorkItemType"]}, not a Test Case.`)
+  const base = `${org}/${encodeURIComponent(item.fields["System.TeamProject"])}`
+  const ops: Record<string, unknown>[] = [
+    { op: "add", path: "/fields/Microsoft.VSTS.TCM.Steps", value: adoStepsXml(change.steps) },
+    { op: "add", path: "/fields/System.History", value: html([`Updated by ForgeQE: ${change.reason}`]) },
+  ]
+  if (change.title?.trim()) ops.push({ op: "add", path: "/fields/System.Title", value: change.title.trim() })
+  await http(`${base}/_apis/wit/workitems/${id}?${API}`, { auth, method: "PATCH", body: ops, contentType: "application/json-patch+json" })
+  return `${base}/_workitems/edit/${id}`
+}
+
+// ---------------------------------------------------------------------------------------------
+// Jira
+// ---------------------------------------------------------------------------------------------
+
+/** Linked issues of these types are the story's work, not its test cases. */
+const NOT_TESTS = new Set(["bug", "defect", "epic", "story", "feature", "improvement", "sub-task", "subtask"])
+
+type JiraIssueFields = { summary?: string; description?: unknown; issuetype?: { name?: string } }
+
+/** A Jira issue's title, link, description as text, and the steps in it. */
+export async function jiraCase(cfg: SourcesConfig, key: string): Promise<ExistingCase & { text: string; type?: string }> {
+  const j = jiraApi(cfg)
+  const { data } = await http<{ key: string; fields: JiraIssueFields }>(`${j.api}/issue/${encodeURIComponent(key)}?fields=summary,description,issuetype`, { auth: j.auth })
+  const text = typeof data.fields.description === "string" ? data.fields.description : adfToText(data.fields.description)
+  return { id: data.key, title: data.fields.summary || data.key, url: `${j.url}/browse/${data.key}`, steps: stepsFromText(text), text, type: data.fields.issuetype?.name }
+}
+
+async function jiraStory(cfg: SourcesConfig, storyKey: string) {
   const j = jiraApi(cfg, "Saving test cases in Jira")
-  const { data: story } = await http<{ key: string; fields: { summary?: string; project?: { key: string }; issuelinks?: { inwardIssue?: { key: string; fields?: { summary?: string } }; outwardIssue?: { key: string; fields?: { summary?: string } } }[] } }>(
+  type Linked = { key: string; fields?: { summary?: string; issuetype?: { name?: string } } }
+  const { data: story } = await http<{ key: string; fields: { summary?: string; project?: { key: string }; issuelinks?: { inwardIssue?: Linked; outwardIssue?: Linked }[] } }>(
     `${j.api}/issue/${encodeURIComponent(storyKey)}?fields=summary,project,issuelinks`,
     { auth: j.auth },
   )
   const link = (key: string) => `${j.url}/browse/${key}`
-  const existing = (story.fields.issuelinks || []).flatMap((l) => [l.inwardIssue, l.outwardIssue]).filter((i): i is { key: string; fields?: { summary?: string } } => Boolean(i))
+  const linked = (story.fields.issuelinks || [])
+    .flatMap((l) => [l.inwardIssue, l.outwardIssue])
+    .filter((i): i is Linked => Boolean(i?.key))
+    .filter((i) => !NOT_TESTS.has((i.fields?.issuetype?.name || "").toLowerCase()))
+  const cases: ExistingCase[] = []
+  for (const i of linked) {
+    // The steps live in the description, which links do not carry; a case that cannot be read still counts by its title.
+    try {
+      const c = await jiraCase(cfg, i.key)
+      if (!NOT_TESTS.has((c.type || "").toLowerCase())) cases.push({ id: c.id, title: c.title, url: c.url, steps: c.steps })
+    } catch {
+      cases.push({ id: i.key, title: i.fields?.summary || i.key, url: link(i.key), steps: [] })
+    }
+  }
+  return { j, story, link, cases }
+}
+
+/**
+ * Create each case as a Jira issue (type `issueType`, default Test) in the story's project with its steps in the
+ * description, linked to the story. A case matching an issue already linked to the story (see matchCase) is reused.
+ */
+export async function saveJiraCases(cfg: SourcesConfig, storyKey: string, cases: NewCase[], opts: { issueType?: string; linkType?: string; forceNew?: string[] } = {}): Promise<SaveResult> {
+  const { j, story, link, cases: existing } = await jiraStory(cfg, storyKey)
+  const known = [...existing]
+  const force = new Set((opts.forceNew || []).map(stepText))
   const out: SavedCase[] = []
   for (const c of cases) {
-    const found = existing.find((i) => sameTitle(i.fields?.summary || "", c.title))
-    if (found) {
-      out.push({ title: c.title, id: found.key, url: link(found.key), created: false })
+    const match = matchCase(c, known, force.has(stepText(c.title)))
+    if (match) {
+      out.push({ title: c.title, id: match.found.id, url: match.found.url, created: false, status: match.status, ...(match.status === "unchanged" ? {} : { existing: match.found }) })
       continue
     }
-    const lines = [...describe(c), "Steps:", ...c.steps.map((s, i) => `${i + 1}. ${s.action}${s.expected ? `\n   Expected: ${s.expected}` : ""}`)]
-    const text = lines.join("\n")
+    const text = [...describe(c), "Steps:", ...stepLines(c.steps)].join("\n")
     try {
       const { data } = await http<{ key: string }>(`${j.api}/issue`, {
         auth: j.auth,
@@ -122,28 +298,45 @@ export async function saveJiraCases(cfg: SourcesConfig, storyKey: string, cases:
       } catch (err) {
         error = `created but not linked to ${story.key}: ${(err as Error).message}`
       }
-      out.push({ title: c.title, id: data.key, url: link(data.key), created: true, ...(error ? { error } : {}) })
+      out.push({ title: c.title, id: data.key, url: link(data.key), created: true, status: "created", ...(error ? { error } : {}) })
+      known.push({ id: data.key, title: c.title, url: link(data.key), steps: c.steps })
     } catch (err) {
-      out.push({ title: c.title, id: "", url: "", created: false, error: (err as Error).message })
+      out.push({ title: c.title, id: "", url: "", created: false, status: "failed", error: (err as Error).message })
     }
   }
-  return { system: "jira", story: { id: story.key, title: story.fields.summary || "", url: link(story.key) }, cases: out }
+  return { system: "jira", story: { id: story.key, title: story.fields.summary || "", url: link(story.key) }, cases: out, existing }
 }
 
 /**
- * Replace an Azure DevOps Test Case's steps (and title, when given), and say why in its history. Returns its link.
- * The previous steps stay in the work item's history, so the change can be undone there.
+ * Replace the steps in a Jira test issue's description (and its summary, when given), keeping the text above them,
+ * and say why in a comment. Jira keeps the old description in the issue's history. Returns its link.
  */
-export async function updateAdoCase(cfg: SourcesConfig, id: number, change: { title?: string; steps: NewCase["steps"]; reason: string }): Promise<string> {
-  const { org, auth } = adoSettings(cfg)
-  const { data: item } = await http<{ fields: Record<string, string> }>(`${org}/_apis/wit/workitems/${id}?fields=System.TeamProject,System.WorkItemType&${API}`, { auth })
-  if (item.fields["System.WorkItemType"] && item.fields["System.WorkItemType"] !== "Test Case") throw new Error(`Work item ${id} is a ${item.fields["System.WorkItemType"]}, not a Test Case.`)
-  const base = `${org}/${encodeURIComponent(item.fields["System.TeamProject"])}`
-  const ops: Record<string, unknown>[] = [
-    { op: "add", path: "/fields/Microsoft.VSTS.TCM.Steps", value: adoStepsXml(change.steps) },
-    { op: "add", path: "/fields/System.History", value: html([`Updated by ForgeQE: ${change.reason}`]) },
-  ]
-  if (change.title?.trim()) ops.push({ op: "add", path: "/fields/System.Title", value: change.title.trim() })
-  await http(`${base}/_apis/wit/workitems/${id}?${API}`, { auth, method: "PATCH", body: ops, contentType: "application/json-patch+json" })
-  return `${base}/_workitems/edit/${id}`
+export async function updateJiraCase(cfg: SourcesConfig, key: string, change: { title?: string; steps: NewCase["steps"]; reason: string }): Promise<string> {
+  const j = jiraApi(cfg, "Updating test cases in Jira")
+  const current = await jiraCase(cfg, key)
+  const lines = current.text.split(/\r?\n/)
+  let cut = lines.findIndex((l) => /^\s*steps:\s*$/i.test(l))
+  if (cut < 0) cut = lines.findIndex((l) => /^\s*1[.)]\s+/.test(l))
+  const above = (cut < 0 ? lines : lines.slice(0, cut)).join("\n").trimEnd()
+  const text = [...(above ? [above, ""] : []), "Steps:", ...stepLines(change.steps)].join("\n")
+  const fields: Record<string, unknown> = { description: j.cloud ? textToAdf(text) : text }
+  if (change.title?.trim()) fields.summary = change.title.trim().slice(0, 255)
+  await http(`${j.api}/issue/${encodeURIComponent(current.id)}`, { auth: j.auth, method: "PUT", body: { fields } })
+  const note = `Updated by ForgeQE: ${change.reason}`
+  await http(`${j.api}/issue/${encodeURIComponent(current.id)}/comment`, { auth: j.auth, body: { body: j.cloud ? textToAdf(note) : note } })
+  return current.url
+}
+
+// ---------------------------------------------------------------------------------------------
+// Either system
+// ---------------------------------------------------------------------------------------------
+
+/** The story and every test case already linked to it, with their steps: read before writing cases for it. */
+export async function storyCases(cfg: SourcesConfig, system: "azure_devops" | "jira", story: string): Promise<StoryCases> {
+  if (system === "jira") {
+    const { story: s, link, cases } = await jiraStory(cfg, story)
+    return { system, story: { id: s.key, title: s.fields.summary || "", url: link(s.key) }, cases }
+  }
+  const s = await adoStory(cfg, Number(story.replace(/^(#|ADO-)/i, "")))
+  return { system, story: s.story, cases: s.cases }
 }

package/src/cases/types.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto"
+
 import type { CodeStatus, StepVerdict, Usage } from "../types.ts"
 import type { Cost } from "../usage.ts"
 import type { CaseDefect, Reproduced } from "./defects.ts"
@@ -5,6 +7,15 @@ import type { Failure } from "./failure.ts"
 
 export type CaseStep = { action: string; data?: string; expected?: string }
 
+/** A step's words, ignoring case, spacing, and punctuation: two wordings of the same step compare equal. */
+export const stepText = (s: string | undefined) => (s || "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim()
+
+/** One step as a short hash of its action, data, and expected result; the engine compares these to replay unchanged steps. */
+export const stepKey = (s: CaseStep) => createHash("sha1").update(`${stepText(s.action)}|${stepText(s.data)}|${stepText(s.expected)}`).digest("hex").slice(0, 12)
+
+/** A case's steps as one hash: it changes when any step is added, removed, reordered, or reworded. */
+export const stepsHash = (steps: CaseStep[]) => createHash("sha1").update(steps.map(stepKey).join(",")).digest("hex").slice(0, 12)
+
 export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest" | "github"
 
 /** A test case from any source, in one shape. It becomes one browser goal and, once it passes, one test. */

package/src/cli/help.ts

@@ -47,6 +47,14 @@ Usage:
   forgeqe pr-tests [pr] [--base BRANCH] [--agent] [--run] [--post]
                                                the tests a pull request (or the commits since BRANCH) can break; --run runs
                                                them and exits 1 when one fails (a CI gate), --post puts the results on the PR
+  forgeqe pack build [--name N] [--sprint S|--iteration PATH|--area A|--parent ID|--work-items 1,2] [--from REF [--to REF] [--range-repo DIR]]
+                    [--prs 12,13|--merged-days N] [--pr-repo DIR] [--no-risk] [--no-smoke]
+                                               a regression pack: the tests a release's changes can break, each with why, and its gaps
+  forgeqe pack run <name> [--on local|testmu|pipeline] [--pipeline github|azure_devops] [--browsers local|testmu] [--yes]
+                                               run a pack (asks first), compare with its last run; exits 1 when a test fails
+  forgeqe pack list | show <name>             the saved packs, or one pack's tests and gaps
+  forgeqe pack pipeline init [--github] [--azure-devops] [--force]
+                                               write the GitHub Actions workflow and Azure DevOps pipeline that run a pack
   forgeqe quality [run-dir]                   a run's WCAG rules passed and failed, Core Web Vitals, and budgets
   forgeqe quality trends [page]               accessibility and timings per page over time: what got worse or better
   forgeqe defects [--folder DIR]              defects over the saved suites: still there, gone, came back, time to fix, by area

package/src/cli/index.ts

@@ -34,6 +34,7 @@ import { runSuite, suiteExitCode, suiteParallel } from "../suite/run.ts"
 import { packageVersion } from "../tui/theme.ts"
 import { update } from "../update.ts"
 import { meterSnapshot, requestLine } from "../usage.ts"
+import { watchBudgets } from "../tokenomics/budget.ts"
 import { boolFlag, configOpts, maxStepsFlag, memoryFlags, parseArgs, repoFlag, skillFlags, strFlag, type Flags } from "./args.ts"
 import { HELP_TEXT } from "./help.ts"
 import { suiteEventLine, terminalAsk, terminalInput, terminalWait, toolPrinter, writeJobSummary } from "./terminal.ts"
@@ -80,7 +81,11 @@ function debugRequested(flags: Flags): boolean {
 /** Every command that runs something writes a session log; a broken config is reported by the command itself. */
 function openSessionLog(command: string, flags: Flags, argv: string[]): void {
   try {
-    const file = startSessionLog(loadConfig(configOpts(flags)), { command, debug: debugRequested(flags), argv })
+    const cfg = loadConfig(configOpts(flags))
+    const file = startSessionLog(cfg, { command, debug: debugRequested(flags), argv })
+    // Budgets only warn, on stderr so a command's output stays clean. The shell (start) says them in the conversation;
+    // the MCP server's calls are in the ledger, where the shell or command that started it reads them.
+    if (cfg.budget && command !== "start" && command !== "mcp") watchBudgets(cfg.budget, (w) => console.error(w.text))
     if (file && debugRequested(flags) && command !== "start") console.error(`Session log: ${file}`)
   } catch {
     // no config yet
@@ -240,6 +245,64 @@ export async function main(argv: string[]): Promise<number> {
         await TOOLS.find((t) => t.name === "defect_trends")!.run({ folder: str("folder") }, ctx)
         return 0
       }
+      case "pack": {
+        // forgeqe pack build [--name N] [--sprint S|--work-items 1,2|--parent ID] [--from REF --to REF --range-repo DIR] [--prs 12,13|--merged-days N]
+        // forgeqe pack run <name> [--on local|testmu|pipeline] [--yes]; forgeqe pack list; forgeqe pack show <name>.
+        const cfg = loadConfig(configOpts(flags))
+        const color = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR
+        const str = (k: string) => (typeof flags.get(k) === "string" ? String(flags.get(k)) : undefined)
+        const csv = (k: string) => (str(k) || "").split(",").map((x) => x.trim()).filter(Boolean)
+        const sub = positionals[1] || "list"
+        const ask = sub === "run" && process.stdin.isTTY && !boolFlag(flags, "yes") ? terminalAsk(new AbortController().signal) : undefined
+        const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)), ...(ask ? { ask } : {}) }
+        const tool = (name: string) => TOOLS.find((t) => t.name === name)!
+        if (sub === "build") {
+          await tool("build_regression_pack").run(
+            {
+              name: str("name") || positionals[2],
+              repo: str("repo"),
+              sprint: str("sprint"),
+              iteration: str("iteration"),
+              area: str("area"),
+              parent: str("parent") ? Number(str("parent")) : undefined,
+              work_items: csv("work-items").map(Number).filter(Boolean),
+              project: str("project"),
+              from: str("from"),
+              to: str("to"),
+              range_repo: str("range-repo"),
+              prs: csv("prs"),
+              merged_days: str("merged-days") ? Number(str("merged-days")) : undefined,
+              pr_repo: str("pr-repo"),
+              risk: !boolFlag(flags, "no-risk"),
+              smoke: !boolFlag(flags, "no-smoke"),
+            },
+            ctx,
+          )
+          return 0
+        }
+        if (sub === "run") {
+          const out = (await tool("run_regression_pack").run({ name: positionals[2] || str("name"), on: str("on") || "local", pipeline: str("pipeline"), browsers: str("browsers"), repo: str("repo"), confirmed: boolFlag(flags, "yes") }, ctx)) as { ran?: boolean; status?: string }
+          return out.ran && out.status !== "passed" ? 1 : 0
+        }
+        if (sub === "pipeline") {
+          // forgeqe pack pipeline init [--github] [--azure-devops] [--force]: the pipeline files that run a pack.
+          const { writePipelineFiles } = await import("../pack/init.ts")
+          const repo = str("repo") ? path.resolve(str("repo")!) : process.cwd()
+          const both = !boolFlag(flags, "github") && !boolFlag(flags, "azure-devops")
+          for (const line of writePipelineFiles(cfg, repo, { github: both || boolFlag(flags, "github"), azureDevops: both || boolFlag(flags, "azure-devops"), force: boolFlag(flags, "force") })) console.log(line)
+          return 0
+        }
+        if (sub === "show") {
+          const { loadPack } = await import("../pack/pack.ts")
+          const { packCard } = await import("../pack/index.ts")
+          ctx.emit?.({ type: "card", card: packCard(loadPack(str("repo") ? path.resolve(str("repo")!) : process.cwd(), positionals[2] || str("name") || "")) })
+          return 0
+        }
+        const out = (await tool("list_regression_packs").run({ repo: str("repo") }, ctx)) as { packs: { name: string; built: string; tests: number; not_covered: number; last_run?: { at: string; on?: string; status: string } }[] }
+        if (!out.packs.length) console.log("No regression packs yet: forgeqe pack build --sprint current (or --from REF, --prs, --merged-days).")
+        for (const p of out.packs) console.log(`${p.name}  ${p.built}  ${p.tests} tests, ${p.not_covered} not covered${p.last_run ? `  · last run ${p.last_run.at} on ${p.last_run.on}: ${p.last_run.status}` : ""}`)
+        return 0
+      }
       case "logs":
         return cmdLogs(positionals.slice(1), flags)
       case "skills":

package/src/code/coverage.ts

@@ -19,9 +19,10 @@ import type { Check, Locator, Step } from "./handoff.ts"
 export type CoverageItem = { label: string; kind: "step" | "check"; members: string[] }
 
 /** An existing test and which of the run's items (by index) it reaches. */
-export type Candidate = { test: string; file: string; line: number; covered: number[]; missing: number[]; share: number }
+export type Candidate = { test: string; file: string; line: number; covered: number[]; missing: number[]; share: number; lines?: number }
 
-export type Mode = "new" | "extend" | "reuse" | "update"
+/** sibling: a new test method beside an existing one, in its class, reusing its setup and steps. */
+export type Mode = "new" | "extend" | "reuse" | "update" | "sibling"
 
 /** `ask`: the evidence fits more than one way, so the person chooses (the options, each with what it changes). */
 export type Decision = { mode: Mode; ask: boolean; target?: Candidate; why: string }
@@ -32,6 +33,10 @@ export type Coverage = { items: CoverageItem[]; candidates: Candidate[]; decisio
 export const EXTEND_FROM = 0.4
 /** Another test this close to the best one makes the choice the person's. */
 const RIVAL = 0.75
+/** A test this long is not made longer: the new steps go in a test beside it, which reuses its setup and steps. */
+export const MAX_EXTEND_LINES = 80
+/** With no one to ask, a test covering at least this share is extended; one covering less gets a test beside it. */
+export const EXTEND_ALONE_FROM = 0.6
 
 type RunJson = { final_url?: string; memory?: { dir?: string } }
 
@@ -84,7 +89,7 @@ export function measureCoverage(runDir: string, graph: CodeGraph, memoryDir?: st
   // test in its own way, so it is unknown, not missing.
   const measurable = items.flatMap((it, i) => (it.members.length ? [i] : []))
   const candidates = [...byTest.values()]
-    .map(({ at, covered }) => ({ test: qualified(at), file: at.file, line: at.member.line, covered: [...covered].sort((a, b) => a - b), missing: measurable.filter((i) => !covered.has(i)), share: measurable.length ? covered.size / measurable.length : 0 }))
+    .map(({ at, covered }) => ({ test: qualified(at), file: at.file, line: at.member.line, lines: at.member.endLine - at.member.line + 1, covered: [...covered].sort((a, b) => a - b), missing: measurable.filter((i) => !covered.has(i)), share: measurable.length ? covered.size / measurable.length : 0 }))
     .sort((a, b) => b.covered.length - a.covered.length || a.missing.length - b.missing.length)
   return { items, candidates, decision: decide(items, candidates) }
 }
@@ -104,6 +109,20 @@ export function decide(items: CoverageItem[], candidates: Candidate[]): Decision
   return { mode: "extend", ask: false, target: best, why: `${best.test} covers ${best.covered.length} of ${n}; only ${best.missing.length} ${best.missing.length === 1 ? "is" : "are"} missing` }
 }
 
+/**
+ * The decision when it is ForgeQE's to make, not the person's: nobody can be asked (a pipeline, a plain run) or the
+ * choice came from ForgeQE's agent. Tests stay short: a long test gets a test beside it instead of more steps. Unclear
+ * between close tests, the best is extended when it covers most of the flow, else a test is added beside it.
+ */
+export function settle(d: Decision): Decision {
+  const t = d.target
+  if (!t || d.mode === "new" || d.mode === "update") return { ...d, ask: false }
+  if (d.mode === "reuse") return { ...d, ask: false, why: d.ask ? `${d.why}; with no one to ask, the one covering the most is reused` : d.why }
+  if (t.lines && t.lines > MAX_EXTEND_LINES) return { ...d, mode: "sibling", ask: false, why: `${d.why}; ${t.test} is already ${t.lines} lines, so the new steps go in a test beside it` }
+  if (d.ask && t.share < EXTEND_ALONE_FROM) return { ...d, mode: "sibling", ask: false, why: `${d.why}; with no one to ask and ${t.test} covering ${Math.round(t.share * 100)}%, a test beside it rather than a longer one` }
+  return { ...d, ask: false, why: d.ask ? `${d.why}; with no one to ask, the one covering the most (${Math.round(t.share * 100)}%) is extended` : d.why }
+}
+
 /** The evidence as card lines: each close test, what it covers, and what it lacks. */
 export function coverageLines(c: Coverage, max = 3): string[] {
   const n = c.items.filter((i) => i.members.length).length

package/src/code/decisions.ts

@@ -0,0 +1,73 @@
+import { appendFileSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import { logEvent } from "../sessionLog.ts"
+import { projectDir } from "../tools.ts"
+
+/**
+ * Every decision about existing tests, kept so it can be read back: which tests were shown before a browser run, who
+ * chose what (ForgeQE's agent, the user, ForgeQE's policy, the coverage measure, or a test an earlier job wrote for the
+ * same flow), why, and how it ended. One JSON line each in .forgeqe/decisions.jsonl in the repository (commit it, or
+ * keep it between CI runs, like test-history.json), and in the session log. What a later job learns from comes from here.
+ */
+
+export const DECISIONS_FILE = "decisions.jsonl"
+/** Past this size the oldest half is dropped. */
+const MAX_BYTES = 2_000_000
+
+export type Shown = { test?: string; file: string; score?: number; covers?: string[]; lacks?: string[] }
+
+export type DecisionRecord = {
+  at: string
+  goal: string
+  /** shown: tests offered before the browser; chosen: what was picked; outcome: how it ended. */
+  stage: "shown" | "chosen" | "outcome"
+  /** Where it was decided: before the browser run, or in write_test after it. */
+  where: "browser_run" | "write_test"
+  by?: "agent" | "user" | "policy" | "coverage" | "history"
+  mode?: "reuse" | "extend" | "sibling" | "update" | "new" | "fresh"
+  test?: string
+  file?: string
+  why?: string
+  shown?: Shown[]
+  status?: string
+  existing?: string
+  run_dir?: string
+  /** Times Scribe changed the existing test it was asked to keep, and was sent back. */
+  breaches?: number
+}
+
+export const decisionsFile = (repo: string) => path.join(projectDir(repo), DECISIONS_FILE)
+
+export function recordDecision(repo: string, rec: Omit<DecisionRecord, "at">): void {
+  const full: DecisionRecord = { at: new Date().toISOString(), ...rec }
+  logEvent("decision", full)
+  const file = decisionsFile(repo)
+  try {
+    mkdirSync(path.dirname(file), { recursive: true })
+    appendFileSync(file, `${JSON.stringify(full)}\n`)
+    if (statSync(file).size > MAX_BYTES) {
+      const lines = readFileSync(file, "utf8").trim().split("\n")
+      writeFileSync(file, `${lines.slice(Math.floor(lines.length / 2)).join("\n")}\n`)
+    }
+  } catch {
+    // The record is for later jobs; a repository ForgeQE cannot write to still gets its test.
+  }
+}
+
+export function readDecisions(repo: string): DecisionRecord[] {
+  try {
+    return readFileSync(decisionsFile(repo), "utf8")
+      .split("\n")
+      .filter(Boolean)
+      .flatMap((l) => {
+        try {
+          return [JSON.parse(l) as DecisionRecord]
+        } catch {
+          return []
+        }
+      })
+  } catch {
+    return []
+  }
+}