@capagents/forgeqe 1.1.6 → 1.1.8

package/src/report/evidence.ts

@@ -0,0 +1,313 @@
+/**
+ * Evidence for reports: every tool result of the session, kept as it came back, so a report's tables are built from
+ * what the tools returned and not from what the model wrote. A dataset names the calls it comes from (or a read-only
+ * query ForgeQE runs itself); its rows are taken from their output, flattened into columns.
+ */
+
+export type EvidenceEntry = {
+  /** E1, E2, …: how a report cites it. */
+  id: string
+  tool: string
+  input: Record<string, unknown>
+  output: string
+  at: string
+}
+
+const MAX_ENTRIES = 300
+const MAX_OUTPUT = 4_000_000
+
+/** Tools whose results are never data for a report: files, the shell, the agent's own bookkeeping. */
+const NOT_DATA = new Set(["write_report", "read", "edit", "write", "patch", "multiedit", "bash", "glob", "grep", "list", "todowrite", "todoread", "skill", "task", "StructuredOutput"])
+
+export class EvidenceLog {
+  private entries: EvidenceEntry[] = []
+  private n = 0
+
+  /** Keeps a successful call's result; files, the shell, and empty results are not evidence. */
+  record(tool: string, input: Record<string, unknown>, output: string): EvidenceEntry | null {
+    if (NOT_DATA.has(tool) || NOT_DATA.has(bareTool(tool)) || !output.trim()) return null
+    const entry = { id: `E${++this.n}`, tool, input, output: output.length > MAX_OUTPUT ? output.slice(0, MAX_OUTPUT) : output, at: new Date().toISOString() }
+    this.entries.push(entry)
+    if (this.entries.length > MAX_ENTRIES) this.entries.shift()
+    return entry
+  }
+
+  all(): EvidenceEntry[] {
+    return [...this.entries]
+  }
+}
+
+/** A tool's name without its MCP server prefix: forgeqe_query_api → query_api, azure_devops_wit_my_work_items → wit_my_work_items. */
+export function bareTool(tool: string): string {
+  for (const prefix of ["forgeqe_", "azure_devops_", "atlassian_", "confluence_", "github_", "jira_"]) if (tool.startsWith(prefix)) return tool.slice(prefix.length)
+  return tool
+}
+
+export type Cell = string | number | boolean | null
+export type Row = Record<string, Cell>
+export type ColumnKind = "text" | "number" | "date" | "link"
+export type Column = { key: string; label: string; kind: ColumnKind }
+
+/** Field names as people read them; anything else keeps its own name, split into words. */
+const LABELS: Record<string, string> = {
+  id: "ID",
+  key: "Key",
+  teamproject: "Project",
+  project: "Project",
+  workitemtype: "Type",
+  issuetype: "Type",
+  type: "Type",
+  title: "Title",
+  summary: "Title",
+  name: "Name",
+  state: "State",
+  status: "Status",
+  assignedto: "Assigned To",
+  assignee: "Assigned To",
+  priority: "Priority",
+  severity: "Severity",
+  areapath: "Area",
+  area: "Area",
+  iterationpath: "Iteration",
+  iteration: "Iteration",
+  sprint: "Sprint",
+  reason: "Reason",
+  tags: "Tags",
+  labels: "Tags",
+  createddate: "Created",
+  created: "Created",
+  changeddate: "Changed",
+  changed: "Changed",
+  updated: "Changed",
+  resolveddate: "Resolved",
+  resolved: "Resolved",
+  closeddate: "Closed",
+  createdby: "Created By",
+  reporter: "Reported By",
+  resolution: "Resolution",
+  components: "Components",
+  fixversions: "Fix Versions",
+  result: "Result",
+  outcome: "Result",
+  buildnumber: "Build",
+  sourcebranch: "Branch",
+  definition: "Pipeline",
+  pipeline: "Pipeline",
+  requestedfor: "Requested For",
+  starttime: "Started",
+  finishtime: "Finished",
+  queuetime: "Queued",
+}
+
+/** Columns a reader filters by first, in this order, when the data has them. */
+const FILTER_FIRST = ["Project", "State", "Status", "Type", "Priority", "Severity", "Assigned To", "Area", "Iteration", "Sprint", "Result", "Pipeline", "Branch", "Reason", "Resolution", "Components"]
+/** Columns read first in the table. */
+const COLUMN_FIRST = ["ID", "Key", "Title", "Name", "Project", "Type", "State", "Status", "Priority", "Severity", "Assigned To", "Result", "Area", "Iteration", "Created", "Changed"]
+/** Fields that are plumbing, not data. */
+const DROP = new Set(["_links", "url", "self", "rev", "commentversionref", "relations", "expand", "avatarurls", "iconurl", "watermark", "boardcolumndone", "nodeid", "authorizedas", "authorizeddate", "reviseddate", "reviseby", "revisedby", "personid", "descriptor", "imageurl", "uniquename"])
+
+const isPlain = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v)
+
+/** The last part of a dotted field name: System.TeamProject → TeamProject, Microsoft.VSTS.Common.Priority → Priority. */
+function fieldName(k: string): string {
+  const last = k.includes(".") ? k.slice(k.lastIndexOf(".") + 1) : k
+  return last.replace(/^custom_?/i, "")
+}
+
+export function columnLabel(k: string): string {
+  const name = fieldName(k)
+  const known = LABELS[name.toLowerCase().replace(/[\s_-]/g, "")]
+  if (known) return known
+  const words = name.replace(/_/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").trim()
+  return words.charAt(0).toUpperCase() + words.slice(1)
+}
+
+/** A value as one cell: a person by their name, a status by its name, a list joined. */
+function cell(v: unknown): Cell | undefined {
+  if (v === null || v === undefined || v === "") return undefined
+  if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") return v
+  if (Array.isArray(v)) {
+    const parts = v.map((x) => cell(x)).filter((x) => x !== undefined && x !== null)
+    return parts.length ? parts.join(", ") : undefined
+  }
+  if (isPlain(v)) {
+    for (const k of ["displayName", "name", "value", "key", "title", "label"]) if (typeof v[k] === "string" || typeof v[k] === "number") return v[k] as string | number
+  }
+  return undefined
+}
+
+/** A web page for the item, when its record says where: Azure DevOps, Jira, GitHub, or a plain web URL. */
+function webLink(o: Record<string, unknown>, idValue: Cell | undefined): string | undefined {
+  const links = isPlain(o._links) ? o._links : undefined
+  const html = links && isPlain(links.html) ? links.html.href : undefined
+  if (typeof html === "string") return html
+  for (const k of ["html_url", "webUrl", "web_url", "webLink"]) if (typeof o[k] === "string") return o[k] as string
+  if (typeof o.url === "string") {
+    // https://dev.azure.com/acme/<project id>/_apis/wit/workItems/123 → https://dev.azure.com/acme/_workitems/edit/123
+    const ado = /^(https:\/\/[^/]+(?:\/[^/_][^/]*)?)\/(?:[^/]+\/)?_apis\/wit\/workItems\/(\d+)/i.exec(o.url)
+    if (ado) return `${ado[1]}/_workitems/edit/${ado[2]}`
+    if (!/\/_apis\/|\/rest\//.test(o.url)) return o.url
+  }
+  if (typeof o.self === "string" && typeof idValue === "string" && /^[A-Z][A-Z0-9_]+-\d+$/.test(idValue)) {
+    const m = /^(https?:\/\/[^/]+(?:\/[^/]+)*?)\/rest\/api\//.exec(o.self)
+    if (m) return `${m[1]}/browse/${idValue}`
+  }
+  return undefined
+}
+
+/** One record as a row: its own fields and those under `fields` (Azure DevOps, Jira), named for people. */
+export function flattenRow(o: Record<string, unknown>): Row {
+  const row: Row = {}
+  const put = (k: string, v: unknown) => {
+    if (DROP.has(k.toLowerCase()) || DROP.has(fieldName(k).toLowerCase())) return
+    const c = cell(v)
+    if (c === undefined) return
+    const label = columnLabel(k)
+    if (!(label in row)) row[label] = c
+  }
+  for (const [k, v] of Object.entries(o)) if (k !== "fields") put(k, v)
+  if (isPlain(o.fields)) for (const [k, v] of Object.entries(o.fields)) put(k, v)
+  // Jira's record id is numeric and its key is what people use.
+  if ("Key" in row && "ID" in row && typeof o.key === "string") delete row.ID
+  const link = webLink(o, row.Key ?? row.ID)
+  if (link) row.Link = link
+  return row
+}
+
+/** The JSON in a tool's output: the whole of it, or the first object or list in its text. */
+export function parseOutput(output: string): unknown {
+  const text = output.trim()
+  try {
+    return JSON.parse(text)
+  } catch {}
+  const start = text.search(/[[{]/)
+  if (start < 0) return undefined
+  try {
+    return JSON.parse(text.slice(start))
+  } catch {
+    return undefined
+  }
+}
+
+/** The records in a result: the largest list of objects in it, looked for a few levels down. */
+export function findRecords(value: unknown, depth = 0): Record<string, unknown>[] {
+  if (typeof value === "string" && depth === 0) return findRecords(parseOutput(value), 1)
+  if (Array.isArray(value)) {
+    const objs = value.filter(isPlain)
+    if (objs.length && objs.length >= value.length / 2) return objs
+    return []
+  }
+  if (!isPlain(value) || depth > 4) return []
+  // An MCP result wraps its JSON in content[].text.
+  if (Array.isArray(value.content) && value.content.every((c) => isPlain(c) && typeof c.text === "string")) {
+    return findRecords((value.content as { text: string }[]).map((c) => c.text).join("\n"), 0)
+  }
+  let best: Record<string, unknown>[] = []
+  for (const v of Object.values(value)) {
+    const found = findRecords(typeof v === "string" && /^\s*[[{]/.test(v) ? parseOutput(v) : v, depth + 1)
+    if (found.length > best.length) best = found
+  }
+  return best
+}
+
+/** The rows in a tool's output; null when it holds no data (text, an error). An empty list is data: nothing matched. */
+export function rowsOf(output: string): Row[] | null {
+  const value = parseOutput(output)
+  if (value === undefined || typeof value !== "object" || value === null) return null
+  const records = findRecords(value, 1)
+  if (records.length) return records.map(flattenRow)
+  if (Array.isArray(value)) return value.length ? null : []
+  // A list that came back empty ({ count: 0, items: [] }) or a single record (one pipeline run).
+  const lists = Object.values(value).filter(Array.isArray)
+  if (lists.length && lists.every((l) => !l.length)) return []
+  return [flattenRow(value as Record<string, unknown>)]
+}
+
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/
+
+function kindOf(rows: Row[], key: string): ColumnKind {
+  const vals = rows.map((r) => r[key]).filter((v) => v !== null && v !== undefined)
+  if (!vals.length) return "text"
+  if (vals.every((v) => typeof v === "number")) return "number"
+  if (vals.every((v) => typeof v === "string" && ISO_DATE.test(v))) return "date"
+  if (vals.every((v) => typeof v === "string" && /^https?:\/\//.test(v))) return "link"
+  return "text"
+}
+
+/** The columns to show: those asked for, or the ones people read first and then the rest, up to 12. */
+export function chooseColumns(rows: Row[], asked?: string[]): Column[] {
+  const present: string[] = []
+  for (const r of rows) for (const k of Object.keys(r)) if (!present.includes(k)) present.push(k)
+  const find = (name: string) => present.find((k) => k.toLowerCase() === name.toLowerCase()) || present.find((k) => k.toLowerCase() === columnLabel(name).toLowerCase())
+  let keys: string[]
+  if (asked?.length) {
+    keys = asked.map(find).filter((k): k is string => !!k)
+    if (!keys.length) throw new Error(`None of the columns ${asked.join(", ")} is in the data. Its columns: ${present.join(", ")}.`)
+  } else {
+    // A column with the same value in every row (Type: Bug) says nothing in a table; the filters and counts still have it.
+    const varies = (k: string) => rows.length < 2 || ["ID", "Key", "Title", "Name"].includes(k) || new Set(rows.map((r) => r[k] ?? null)).size > 1
+    const first = COLUMN_FIRST.filter((k) => present.includes(k) && varies(k))
+    const rest = present.filter((k) => k !== "Link" && !first.includes(k) && varies(k) && rows.some((r) => typeof r[k] !== "string" || String(r[k]).length <= 200))
+    keys = [...first, ...rest].slice(0, 10)
+  }
+  if (present.includes("Link") && !keys.includes("Link")) keys.push("Link")
+  return keys.map((key) => ({ key, label: key, kind: key === "Link" ? "link" : kindOf(rows, key) }))
+}
+
+/** Columns worth a filter: those asked for, or the usual ones (project, state, priority, …) and other short lists of values. */
+export function chooseFilters(rows: Row[], columns: Column[], asked?: string[]): string[] {
+  const distinct = (k: string) => new Set(rows.map((r) => r[k]).filter((v) => v !== null && v !== undefined && v !== "")).size
+  const text = columns.filter((c) => c.kind === "text").map((c) => c.key)
+  if (asked?.length) return asked.map((a) => columns.find((c) => c.key.toLowerCase() === a.toLowerCase() || c.key.toLowerCase() === columnLabel(a).toLowerCase())?.key).filter((k): k is string => !!k)
+  // Priority and severity are numbers in Azure DevOps, and still categories.
+  const known = FILTER_FIRST.filter((k) => columns.some((c) => c.key === k && c.kind !== "date" && c.kind !== "link") && distinct(k) >= 2 && distinct(k) <= 60)
+  const other = text.filter((k) => !known.includes(k) && !["ID", "Key", "Title", "Name"].includes(k)).filter((k) => {
+    const n = distinct(k)
+    return n >= 2 && n <= 15 && n <= rows.length / 2
+  })
+  return [...known, ...other].slice(0, 6)
+}
+
+/** Rows that are the same record (the same ID or key) from more than one call, kept once. */
+export function dedupe(rows: Row[]): Row[] {
+  const seen = new Set<string>()
+  return rows.filter((r) => {
+    const id = r.Key ?? r.ID
+    if (id === undefined || id === null) return true
+    const k = String(id)
+    if (seen.has(k)) return false
+    seen.add(k)
+    return true
+  })
+}
+
+/** What a call was, in one line, for the list of sources. */
+export function describeCall(e: Pick<EvidenceEntry, "tool" | "input">): string {
+  const i = e.input
+  const what = [i.wiql, i.jql, i.query, i.path, i.project, i.ids, i.searchText, i.title].find((v) => (typeof v === "string" && v.trim()) || Array.isArray(v))
+  // A long list of ids as its count and ends: "42 ids: 1000, 1001, … 1041".
+  const text = Array.isArray(what) ? (what.length > 6 ? `${what.length} ids: ${what.slice(0, 2).join(", ")}, … ${what.at(-1)}` : what.join(", ")) : String(what || "")
+  const flat = text.replace(/\s+/g, " ").trim()
+  return `${e.tool}${flat ? `: ${flat.length > 160 ? `${flat.slice(0, 159)}…` : flat}` : ""}`
+}
+
+/** The evidence a dataset names: an id (E3), or every successful call of a tool, optionally those whose input mentions `match`. */
+export function pickEvidence(log: EvidenceEntry[], from: string, match?: string): EvidenceEntry[] {
+  const ids = from.split(/[\s,]+/).filter(Boolean)
+  if (ids.length && ids.every((x) => /^E\d+$/i.test(x))) return log.filter((e) => ids.some((x) => x.toUpperCase() === e.id))
+  const want = bareTool(from)
+  const calls = log.filter((e) => e.tool === from || bareTool(e.tool) === want)
+  return match ? calls.filter((e) => JSON.stringify(e.input).toLowerCase().includes(match.toLowerCase())) : calls
+}
+
+/** The calls a report could use, for the error that says none was found. */
+export function evidenceList(log: EvidenceEntry[]): string {
+  if (!log.length) return "No tool has returned data in this session yet."
+  return log
+    .slice(-25)
+    .map((e) => {
+      const rows = rowsOf(e.output)
+      return `${e.id} ${describeCall(e)} (${rows ? `${rows.length} rows` : "no data"})`
+    })
+    .join("\n")
+}

package/src/report/html.ts

@@ -87,11 +87,15 @@ const TEMPLATE = new URL("./template.html", import.meta.url)
 
 /** A self-contained report page: the report-ui app with this run's data (and screenshots) embedded. */
 export function renderReport(p: ReportInput, opts?: DataOptions): string {
-  const data = buildReportData({ kind: p.kind || "suite", ...p }, opts)
+  return reportPage(p.title, buildReportData({ kind: p.kind || "suite", ...p }, opts))
+}
+
+/** The report-ui app with `data` embedded: a run, a suite, or a data report written by write_report. */
+export function reportPage(title: string, data: unknown): string {
   const json = JSON.stringify(data).replace(/</g, "\\u003c").replace(/>/g, "\\u003e").replace(/\u2028/g, "\\u2028").replace(/\u2029/g, "\\u2029")
-  const title = p.title.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
+  const safe = title.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
   return readFileSync(TEMPLATE, "utf8")
-    .replace(/<title>[^<]*<\/title>/, () => `<title>${title}</title>`)
+    .replace(/<title>[^<]*<\/title>/, () => `<title>${safe}</title>`)
     .replace("<head>", () => `<head>\n<script>window.__FORGEQE_REPORT__=${json}</script>`)
 }