@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/cases/jira.ts

@@ -0,0 +1,122 @@
+import type { SourcesConfig } from "../config.ts"
+import { adfToText, basic, envSecret, http, textToAdf } from "./http.ts"
+import { stepVerdictLines, type CaseResult, type TestCase, type WriteBack } from "./types.ts"
+
+export type JiraIssue = {
+  id: string
+  key: string
+  fields: {
+    summary?: string
+    description?: unknown
+    labels?: string[]
+    components?: { name?: string }[]
+    parent?: { key?: string; fields?: { summary?: string } }
+    issuelinks?: { type?: { outward?: string }; outwardIssue?: { key?: string } }[]
+  }
+}
+
+export type JiraApi = { url: string; cloud: boolean; auth: string; api: string }
+
+export function jiraApi(cfg: SourcesConfig, what = "Jira"): JiraApi {
+  const j = cfg.jira
+  if (!j) throw new Error(`${what} needs sources.jira.url in forgeqe.yaml, or JIRA_URL in the environment.`)
+  const token = envSecret(j.tokenEnv, what)
+  const auth = j.cloud ? basic(envSecret(j.emailEnv, `${what} (Jira Cloud)`), token) : `Bearer ${token}`
+  return { url: j.url, cloud: j.cloud, auth, api: `${j.url}/rest/api/${j.cloud ? 3 : 2}` }
+}
+
+const FIELDS = "summary,description,labels,components,parent,issuelinks"
+
+/** Issues by key (`SHOP-1,SHOP-2`) or by JQL (`jql=project = SHOP AND labels = e2e`). */
+export async function jiraIssues(ref: string, j: JiraApi): Promise<JiraIssue[]> {
+  const jql = ref.match(/^jql=(.+)$/s)?.[1]
+  if (!jql) {
+    const keys = ref.split(/[,\s]+/).map((k) => k.trim().toUpperCase()).filter(Boolean)
+    if (!keys.length) throw new Error(`Jira reference "${ref}": use jira:SHOP-1,SHOP-2 or jira:jql=<query>.`)
+    const out: JiraIssue[] = []
+    for (const key of keys) out.push((await http<JiraIssue>(`${j.api}/issue/${encodeURIComponent(key)}?fields=${FIELDS}`, { auth: j.auth })).data)
+    return out
+  }
+  const out: JiraIssue[] = []
+  if (j.cloud) {
+    let next: string | undefined
+    do {
+      const q = new URLSearchParams({ jql, fields: FIELDS, maxResults: "100", ...(next ? { nextPageToken: next } : {}) })
+      const { data } = await http<{ issues?: JiraIssue[]; nextPageToken?: string; isLast?: boolean }>(`${j.api}/search/jql?${q}`, { auth: j.auth })
+      out.push(...(data?.issues || []))
+      next = data?.isLast === false || data?.nextPageToken ? data?.nextPageToken : undefined
+    } while (next)
+  } else {
+    let startAt = 0
+    for (;;) {
+      const q = new URLSearchParams({ jql, fields: FIELDS, maxResults: "100", startAt: String(startAt) })
+      const { data } = await http<{ issues?: JiraIssue[]; total?: number }>(`${j.api}/search?${q}`, { auth: j.auth })
+      const page = data?.issues || []
+      out.push(...page)
+      startAt += page.length
+      if (!page.length || startAt >= Number(data?.total || 0)) break
+    }
+  }
+  if (!out.length) throw new Error(`No Jira issues match: ${jql}`)
+  return out
+}
+
+/** A Jira issue as a test case: summary is the title, the description holds the steps and expected outcome. */
+function issueGroup(issue: JiraIssue): string | undefined {
+  const components = (issue.fields.components || []).map((c) => c.name).filter(Boolean)
+  if (components.length) return components.join(", ")
+  const p = issue.fields.parent
+  return p?.key ? `${p.key}${p.fields?.summary ? ` ${p.fields.summary}` : ""}` : undefined
+}
+
+/** Issues this one "tests", through Jira's Test link type (outward "tests"). */
+function testedIssues(issue: JiraIssue): string[] | undefined {
+  const keys = (issue.fields.issuelinks || [])
+    .filter((l) => l.outwardIssue?.key && /^tests$/i.test((l.type?.outward || "").trim()))
+    .map((l) => l.outwardIssue!.key!)
+  return keys.length ? keys : undefined
+}
+
+export async function jiraCases(ref: string, cfg: SourcesConfig): Promise<TestCase[]> {
+  const j = jiraApi(cfg)
+  return (await jiraIssues(ref, j)).map((issue) => ({
+    id: issue.key,
+    title: issue.fields.summary || issue.key,
+    source: "jira" as const,
+    url: `${j.url}/browse/${issue.key}`,
+    description: adfToText(issue.fields.description) || undefined,
+    steps: [],
+    tags: issue.fields.labels?.length ? issue.fields.labels : undefined,
+    group: issueGroup(issue),
+    requirements: testedIssues(issue),
+    ref: { issue: issue.key },
+  }))
+}
+
+export async function jiraWriteBack(results: CaseResult[], cfg: SourcesConfig): Promise<WriteBack[]> {
+  const j = jiraApi(cfg)
+  const out: WriteBack[] = []
+  for (const r of results) {
+    const key = String(r.case.ref?.issue || r.case.id)
+    const text = resultText(r)
+    try {
+      await http(`${j.api}/issue/${encodeURIComponent(key)}/comment`, { auth: j.auth, body: { body: j.cloud ? textToAdf(text) : text } })
+      out.push({ source: "jira", caseId: r.case.id, text: `${key}: result added as a comment`, url: r.case.url })
+    } catch (err) {
+      out.push({ source: "jira", caseId: r.case.id, text: `${key}: comment not added`, error: (err as Error).message })
+    }
+  }
+  return out
+}
+
+export function resultText(r: CaseResult): string {
+  const lines = [`ForgeQE: ${r.status.toUpperCase()} in ${Math.round(r.ms / 1000)}s`, "", r.summary.trim()]
+  const steps = stepVerdictLines(r)
+  if (steps.length) lines.push("", ...steps)
+  if (r.finalUrl) lines.push("", `Ended on ${r.finalUrl}`)
+  if (r.code?.testFile) {
+    lines.push("", `Test ${r.code.status === "passed" ? "written and passing" : r.code.status}: ${r.code.testFile}`)
+    if (r.code.command) lines.push(`Run with: ${r.code.command}`)
+  }
+  return lines.join("\n").trim()
+}

package/src/cases/qtest.ts

@@ -0,0 +1,108 @@
+import type { SourcesConfig } from "../config.ts"
+import { envSecret, htmlToText, http } from "./http.ts"
+import { resultText } from "./jira.ts"
+import type { CaseResult, CaseStep, TestCase, WriteBack } from "./types.ts"
+
+type QCase = {
+  id: number
+  pid?: string
+  name: string
+  description?: string | null
+  precondition?: string | null
+  web_url?: string
+  test_steps?: { description?: string | null; expected?: string | null; order?: number }[]
+}
+type QRun = { id: number; pid?: string; name?: string; test_case?: { id: number } }
+
+export function qtestApi(cfg: SourcesConfig) {
+  const q = cfg.qtest
+  if (!q) throw new Error("qTest is not set up. Add sources.qtest.url and project_id to forgeqe.yaml, or set QTEST_URL and QTEST_PROJECT_ID.")
+  const token = envSecret(q.tokenEnv, "qTest")
+  return { base: q.url, api: `${q.url}/api/v3/projects/${q.projectId}`, auth: /^bearer\s/i.test(token) ? token : `Bearer ${token}`, statuses: q.statuses }
+}
+
+function toCase(c: QCase, run?: QRun): TestCase {
+  const steps: CaseStep[] = [...(c.test_steps || [])]
+    .sort((a, b) => (a.order || 0) - (b.order || 0))
+    .map((s) => ({ action: htmlToText(s.description || ""), expected: htmlToText(s.expected || "") || undefined }))
+    .filter((s) => s.action)
+  return {
+    id: c.pid || `TC-${c.id}`,
+    title: c.name,
+    source: "qtest",
+    url: c.web_url,
+    description: htmlToText(c.description || "") || undefined,
+    preconditions: htmlToText(c.precondition || "") ? [htmlToText(c.precondition || "")] : undefined,
+    steps,
+    ref: { testCase: c.id, ...(run ? { run: run.id } : {}) },
+  }
+}
+
+/** `qtest:12345,12346` (test case ids from the URL), `qtest:cycle=678` or `qtest:suite=910` (their test runs; results go to those runs). */
+export async function qtestCases(ref: string, cfg: SourcesConfig): Promise<TestCase[]> {
+  const q = qtestApi(cfg)
+  const getCase = async (id: number) => (await http<QCase>(`${q.api}/test-cases/${id}?expandSteps=true`, { auth: q.auth })).data
+  const parent = ref.match(/^(cycle|suite)=(\d+)$/i)
+  if (parent) {
+    const type = parent[1].toLowerCase() === "cycle" ? "test-cycle" : "test-suite"
+    const runs: QRun[] = []
+    for (let page = 1; ; page++) {
+      const { data } = await http<QRun[] | { items?: QRun[] }>(`${q.api}/test-runs?parentId=${parent[2]}&parentType=${type}&page=${page}&pageSize=100`, { auth: q.auth })
+      const items = Array.isArray(data) ? data : data?.items || []
+      runs.push(...items)
+      if (items.length < 100) break
+    }
+    const out: TestCase[] = []
+    for (const run of runs) if (run.test_case?.id) out.push(toCase(await getCase(run.test_case.id), run))
+    if (!out.length) throw new Error(`No test runs in qTest ${type} ${parent[2]}.`)
+    return out
+  }
+  const ids = ref.split(/[,\s]+/).map((s) => Number(s)).filter((n) => Number.isInteger(n) && n > 0)
+  if (!ids.length) throw new Error(`qTest reference "${ref}": use qtest:12345 (the test case id from its URL), qtest:cycle=678, or qtest:suite=910.`)
+  const out: TestCase[] = []
+  for (const id of ids) out.push(toCase(await getCase(id)))
+  return out
+}
+
+/** An automation test log per case, on the test run it came from; cases read by id get test runs in a new test cycle. */
+export async function qtestWriteBack(results: CaseResult[], cfg: SourcesConfig, runName: string): Promise<WriteBack[]> {
+  const q = qtestApi(cfg)
+  const status = { passed: q.statuses.passed, failed: q.statuses.failed, error: q.statuses.error } as const
+  const stepStatus = { passed: q.statuses.passed, failed: q.statuses.failed, not_run: q.statuses.error } as const
+  const out: WriteBack[] = []
+  let cycle: number | undefined
+  let logged = 0
+  for (const r of results) {
+    try {
+      let runId = Number(r.case.ref?.run || 0)
+      if (!runId) {
+        if (!cycle) cycle = (await http<{ id: number }>(`${q.api}/test-cycles?parentId=0&parentType=root`, { auth: q.auth, body: { name: runName } })).data.id
+        runId = (await http<{ id: number }>(`${q.api}/test-runs?parentId=${cycle}&parentType=test-cycle`, { auth: q.auth, body: { name: r.case.title, test_case: { id: Number(r.case.ref?.testCase) } } })).data.id
+      }
+      const verdicts = new Map((r.stepResults || []).map((v) => [v.step, v]))
+      const body: Record<string, unknown> = {
+        name: r.case.title,
+        status: status[r.status as keyof typeof status] ?? q.statuses.error,
+        exe_start_date: r.startedAt,
+        exe_end_date: r.finishedAt,
+        note: resultText(r).slice(0, 4000),
+      }
+      const withSteps = verdicts.size
+        ? {
+            ...body,
+            test_step_logs: r.case.steps.map((s, i) => {
+              const v = verdicts.get(i + 1)
+              return { description: s.action, expected_result: s.expected || "", actual_result: v?.actual || "", status: v ? stepStatus[v.status] : q.statuses.error, order: i + 1 }
+            }),
+          }
+        : body
+      const url = `${q.api}/test-runs/${runId}/auto-test-logs`
+      await http(url, { auth: q.auth, body: withSteps }).catch((err) => (withSteps === body ? Promise.reject(err) : http(url, { auth: q.auth, body })))
+      logged++
+    } catch (err) {
+      out.push({ source: "qtest", caseId: r.case.id, text: `${r.case.id}: result not recorded`, error: (err as Error).message })
+    }
+  }
+  if (logged) out.push({ source: "qtest", text: `qTest: ${logged} test log${logged === 1 ? "" : "s"}${cycle ? ` (new test cycle ${cycle})` : ""}` })
+  return out
+}

package/src/cases/quality.ts

@@ -0,0 +1,190 @@
+import { findPage, type MemoryPage } from "../memory.ts"
+import type { TestCase } from "./types.ts"
+
+/**
+ * Test case quality, checked before a case costs a browser run: steps a browser agent cannot follow, or expected
+ * results it cannot check, make a run fail for the test case's sake rather than the application's. These checks are
+ * rules (no model): what they cannot judge (whether a step is ambiguous for this application) is left to the agent.
+ */
+
+export type CheckSeverity = "blocker" | "warning" | "info"
+
+export type CheckCode =
+  | "no_steps"
+  | "no_expected_results"
+  | "step_without_expected"
+  | "vague_expected"
+  | "missing_test_data"
+  | "placeholder"
+  | "credential_in_case"
+  | "several_actions"
+  | "no_start"
+  | "duplicate"
+  | "long_case"
+  | "not_on_page"
+
+export type CaseFinding = { code: CheckCode; severity: CheckSeverity; step?: number; message: string; suggestion: string }
+
+/** ready: nothing to fix; review: worth fixing, it runs; not_runnable: a browser agent cannot test it as written. */
+export type CaseVerdict = "ready" | "review" | "not_runnable"
+
+export type CaseCheck = { id: string; title: string; url?: string; verdict: CaseVerdict; findings: CaseFinding[] }
+
+const VAGUE = /^(it |the (page|app|application|system|form) )?(works?|working|ok|okay|fine|success(ful(ly)?)?|correct(ly)?|proper(ly)?|as expected|no (error|issue)s?|should work|passes|is (displayed|shown|done))\.?$/i
+const VAGUE_WORDS = /\b(as expected|works? (fine|correctly|properly)|correctly|properly|appropriate(ly)?|successfully)\b/i
+const NEEDS_DATA = /\b(valid|invalid|some|any|appropriate|relevant|correct|required|mandatory|test)\s+(details|data|values?|information|info|inputs?|users?|credentials|e-?mail( address)?|password|name|address)\b/i
+const PLACEHOLDER = /\b(TBD|TODO|FIXME|xxx+|lorem ipsum)\b|<[^<>\s]{2,30}>|\{\{[^}]*\}\}|\[(insert|enter|add)[^\]]*\]/i
+const CREDENTIAL = /\b(password|passwd|pwd|secret|api[ -]?key|token)\s*[:=]\s*\S+/i
+const START = /\b(https?:\/\/|www\.|navigate|open|go to|launch|visit|browse to|log ?in|sign ?in)\b/i
+
+const words = (s: string) => s.trim().split(/\s+/).filter(Boolean).length
+const norm = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim()
+
+/** Every rule on one case; duplicates are found across cases by checkCases. */
+export function checkCase(tc: TestCase): CaseFinding[] {
+  const out: CaseFinding[] = []
+  const steps = tc.steps
+  if (!steps.length) {
+    out.push(
+      tc.description?.trim()
+        ? { code: "no_steps", severity: "warning", message: "No numbered steps: the browser agent works from the description alone.", suggestion: "Write the description as numbered steps, each with an expected result." }
+        : { code: "no_steps", severity: "blocker", message: "No steps and no description: there is nothing to test.", suggestion: "Add numbered steps, each with an expected result." },
+    )
+    return out
+  }
+  const withExpected = steps.filter((s) => s.expected?.trim())
+  if (!withExpected.length) {
+    out.push({ code: "no_expected_results", severity: "blocker", message: "No step has an expected result, so the run cannot pass or fail on anything.", suggestion: "Give at least the steps that matter an expected result the page shows (a message, a value, a page)." })
+  }
+  steps.forEach((s, i) => {
+    const n = i + 1
+    const text = `${s.action} ${s.data || ""} ${s.expected || ""}`
+    const last = i === steps.length - 1
+    if (withExpected.length && !s.expected?.trim() && last) {
+      out.push({ code: "step_without_expected", severity: "warning", step: n, message: "The last step has no expected result, so the case does not say how it ends.", suggestion: "Say what the page shows when the case has passed." })
+    }
+    if (s.expected?.trim() && (VAGUE.test(s.expected.trim()) || (VAGUE_WORDS.test(s.expected) && words(s.expected) <= 6))) {
+      out.push({
+        code: "vague_expected",
+        severity: "warning",
+        step: n,
+        message: `"${s.expected.trim()}" does not say what the page shows, so a run can only guess what passing means.`,
+        suggestion: "Name what is on the page when it works: a confirmation message, a value, a heading, a URL.",
+      })
+    }
+    if (NEEDS_DATA.test(s.action) && !s.data?.trim() && !/["'“][^"'”]+["'”]|\d/.test(s.action)) {
+      out.push({ code: "missing_test_data", severity: "warning", step: n, message: `"${clip(s.action, 80)}" does not give the values to use.`, suggestion: "Give the values (or a data table) the step should use, or say where they come from." })
+    }
+    if (PLACEHOLDER.test(text)) out.push({ code: "placeholder", severity: "blocker", step: n, message: `An unfilled placeholder: "${clip(PLACEHOLDER.exec(text)![0], 40)}".`, suggestion: "Replace the placeholder with the real value." })
+    if (CREDENTIAL.test(text)) {
+      out.push({ code: "credential_in_case", severity: "warning", step: n, message: "A password or key is written in the test case.", suggestion: "Keep credentials out of test cases: use a saved login (logins: in forgeqe.yaml) and say \"sign in as <role>\"." })
+    }
+    if ((s.action.match(/\b(and then|then|and)\b|;/gi) || []).length >= 3) {
+      out.push({ code: "several_actions", severity: "info", step: n, message: "Several actions in one step, so a failure cannot say which of them went wrong.", suggestion: "Split it into one action per step." })
+    }
+  })
+  if (!tc.startUrl && !START.test(steps[0].action) && !START.test(tc.description || "") && !(tc.preconditions || []).some((p) => START.test(p))) {
+    out.push({ code: "no_start", severity: "warning", step: 1, message: "The case does not say where to start (a URL or page).", suggestion: "Start with \"Open <URL>\", or set the start page in the case." })
+  }
+  if (steps.length > 25) out.push({ code: "long_case", severity: "info", message: `${steps.length} steps in one case: one failure stops the rest from being checked.`, suggestion: "Split it into cases that each check one behaviour." })
+  return out
+}
+
+/** Fields and controls a step can name without quoting them: "enter a valid email address". */
+const CONTROL_WORDS = [
+  "email", "e-mail", "password", "username", "user name", "first name", "last name", "full name", "phone", "mobile",
+  "address", "postcode", "zip", "city", "country", "date of birth", "date", "search", "message", "comment", "subject",
+  "company", "quantity", "coupon", "promo code", "captcha", "terms", "upload", "file",
+]
+const QUOTED = /["“'‘]([^"”'’]{2,40})["”'’]/g
+const BEFORE_CONTROL = /((?:[A-Za-z][\w-]*\s+){0,2}[A-Za-z][\w-]*)\s+(?:field|button|link|checkbox|check box|dropdown|drop-down|radio|tab|menu|textbox|text box|input|option)\b/gi
+const FILLER = new Set(["the", "a", "an", "all", "any", "valid", "invalid", "new", "mandatory", "required", "and", "or", "then", "from", "on", "in", "into", "to", "click", "tick", "untick", "select", "choose", "enter", "type", "press", "open", "fill", "check", "uncheck", "its", "this", "that", "of"])
+const URL = /https?:\/\/[^\s"'<>)]+/i
+
+/** What a step names on the page: quoted labels, "X button" / "X field", and common field words. */
+export function namedControls(action: string): string[] {
+  const out = new Set<string>()
+  const keep = (name: string) => {
+    const n = name.trim().toLowerCase()
+    if (n && !URL.test(n) && !/^\d+$/.test(n)) out.add(n)
+  }
+  for (const m of action.matchAll(QUOTED)) keep(m[1])
+  const unquoted = action.replace(QUOTED, " ")
+  for (const m of unquoted.matchAll(BEFORE_CONTROL)) {
+    // The control's name is the last word or two before "button", "field"…, without the verbs and articles around it.
+    const words = m[1].split(/\s+/)
+    while (words.length && FILLER.has(words[0].toLowerCase())) words.shift()
+    if (words.length) keep(words.join(" "))
+  }
+  const low = action.toLowerCase()
+  for (const w of CONTROL_WORDS) if (new RegExp(`\\b${w}\\b`).test(low)) out.add(w)
+  // "email address" is the email field, not an address field.
+  if (/\bemail address\b/.test(low) && !/\baddress (field|line)\b/.test(low)) out.delete("address")
+  return [...out]
+}
+
+const label = (e: MemoryPage["elements"][string]) =>
+  [e.name, e.text, e.attributes?.placeholder, e.attributes?.["aria-label"], e.attributes?.name, e.attributes?.id, e.attributes?.title, e.attributes?.type]
+    .filter(Boolean)
+    .join(" ")
+    .toLowerCase()
+
+/**
+ * Steps that name a field or control earlier runs never saw on that page: page memory has every element ForgeQE met
+ * on each page. Only pages it knows well (several visits, a fair number of elements) are judged, and a miss is a
+ * warning: the control may appear later (after a click), or the page may have changed.
+ */
+export function checkAgainstPages(tc: TestCase, pages: MemoryPage[]): CaseFinding[] {
+  const out: CaseFinding[] = []
+  let url = tc.startUrl || ""
+  tc.steps.forEach((s, i) => {
+    const found = URL.exec(`${s.action} ${s.data || ""}`)?.[0]
+    if (found) {
+      url = found.replace(/[.,;]+$/, "")
+      return
+    }
+    if (!url) return
+    const page = findPage(pages, url)
+    const elements = page ? Object.values(page.elements) : []
+    if (!page || elements.length < 8 || (page.visits || 0) < 1) return
+    const labels = elements.map(label)
+    const missing = namedControls(s.action).filter((name) => !labels.some((l) => l.includes(name.replace(/-/g, "")) || l.includes(name)))
+    if (!missing.length) return
+    const seen = [...new Set(elements.filter((e) => e.interactions || ["input", "select", "textarea", "button"].includes(e.tag)).map((e) => (e.name || e.attributes?.placeholder || e.text || "").trim()).filter((n) => n && n.length <= 30))].slice(0, 10)
+    out.push({
+      code: "not_on_page",
+      severity: "warning",
+      step: i + 1,
+      message: `${missing.map((m) => `"${m}"`).join(", ")} ${missing.length === 1 ? "was" : "were"} never seen on ${page.host}${page.path} in ${page.visits} earlier ${page.visits === 1 ? "visit" : "visits"}${seen.length ? `; it has ${seen.join(", ")}` : ""}.`,
+      suggestion: "Check the step against the page: the control may have another name, sit on another page, or not exist (then the test case needs updating).",
+    })
+  })
+  return out
+}
+
+function verdictOf(findings: CaseFinding[]): CaseVerdict {
+  if (findings.some((f) => f.severity === "blocker")) return "not_runnable"
+  if (findings.some((f) => f.severity === "warning")) return "review"
+  return "ready"
+}
+
+/** Check each case (against page memory too, when given), and cases whose steps are the same as an earlier one's. */
+export function checkCases(cases: TestCase[], pages: MemoryPage[] = []): CaseCheck[] {
+  const seen = new Map<string, TestCase>()
+  return cases.map((tc) => {
+    const findings = [...checkCase(tc), ...(pages.length ? checkAgainstPages(tc, pages) : [])]
+    const signature = tc.steps.map((s) => `${norm(s.action)}>${norm(s.expected || "")}`).join("|")
+    const same = signature && seen.get(signature)
+    if (same) findings.push({ code: "duplicate", severity: "warning", message: `Same steps and expected results as ${same.id} "${same.title}".`, suggestion: `Merge it with ${same.id}, or make the difference explicit.` })
+    else if (signature) seen.set(signature, tc)
+    return { id: tc.id, title: tc.title, ...(tc.url ? { url: tc.url } : {}), verdict: verdictOf(findings), findings }
+  })
+}
+
+/** "3 ready, 1 to review, 1 not runnable". */
+export function checkSummary(checks: CaseCheck[]): string {
+  const n = (v: CaseVerdict) => checks.filter((c) => c.verdict === v).length
+  return [`${n("ready")} ready`, n("review") ? `${n("review")} to review` : "", n("not_runnable") ? `${n("not_runnable")} not runnable` : ""].filter(Boolean).join(", ")
+}
+
+const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n - 1)}…` : s)

package/src/cases/rag.ts

@@ -0,0 +1,100 @@
+import { spawn } from "node:child_process"
+
+import { withoutTelemetry } from "../privacy.ts"
+
+/**
+ * An optional retrieval service for requirements (a RAG index over Confluence, a wiki, specs, tickets): ForgeQE asks
+ * it for the passages relevant to a test case and adds them to the quality check. Plug and play, either kind:
+ *
+ *   rag:
+ *     url: http://localhost:8080/search     # POST {"query": "...", "top_k": 5}
+ *     token_env: RAG_TOKEN                  # optional: sent as "Authorization: Bearer <value>"
+ *     top_k: 5
+ *
+ *   rag:
+ *     command: ["python", "rag/search.py"]  # gets {"query": "...", "top_k": 5} on stdin, prints the answer on stdout
+ *
+ * The answer is a JSON list of passages, or an object with one under results, passages, documents, or hits. A passage
+ * is { text, source, score } (also content / page_content for the text, url / title / metadata.source for the
+ * source). A RAG server reachable over MCP (under mcp: in forgeqe.yaml) works too: ForgeQE's agent calls its tools.
+ */
+
+export type RagConfig = { url?: string; command?: string[]; tokenEnv?: string; topK: number; timeoutMs: number }
+export type Passage = { text: string; source?: string; score?: number }
+
+const MAX_TEXT = 1500
+
+export function ragConfig(raw: unknown): RagConfig | undefined {
+  if (!raw || typeof raw !== "object") return undefined
+  const r = raw as Record<string, unknown>
+  const url = r.url ? String(r.url) : undefined
+  const command = Array.isArray(r.command) ? r.command.map(String) : typeof r.command === "string" ? r.command.split(/\s+/).filter(Boolean) : undefined
+  if (!url && !command?.length) return undefined
+  return {
+    ...(url ? { url } : {}),
+    ...(command?.length ? { command } : {}),
+    ...(r.token_env ? { tokenEnv: String(r.token_env) } : {}),
+    topK: Number(r.top_k) > 0 ? Math.min(20, Number(r.top_k)) : 5,
+    timeoutMs: Number(r.timeout_seconds) > 0 ? Number(r.timeout_seconds) * 1000 : 20_000,
+  }
+}
+
+/** Passages from any of the answer shapes above, best first, text shortened. */
+export function passagesFrom(answer: unknown): Passage[] {
+  const list = Array.isArray(answer)
+    ? answer
+    : answer && typeof answer === "object"
+      ? ((answer as Record<string, unknown>).results ?? (answer as Record<string, unknown>).passages ?? (answer as Record<string, unknown>).documents ?? (answer as Record<string, unknown>).hits)
+      : undefined
+  if (!Array.isArray(list)) return []
+  return list
+    .map((p): Passage | null => {
+      if (typeof p === "string") return { text: p.slice(0, MAX_TEXT) }
+      if (!p || typeof p !== "object") return null
+      const o = p as Record<string, any>
+      const text = String(o.text ?? o.content ?? o.page_content ?? o.chunk ?? "").trim()
+      if (!text) return null
+      const source = o.source ?? o.url ?? o.title ?? o.metadata?.source ?? o.metadata?.url ?? o.metadata?.title
+      const score = typeof o.score === "number" ? o.score : typeof o.similarity === "number" ? o.similarity : undefined
+      return { text: text.slice(0, MAX_TEXT), ...(source ? { source: String(source) } : {}), ...(score !== undefined ? { score } : {}) }
+    })
+    .filter((p): p is Passage => p !== null)
+}
+
+/** Ask the configured service; throws with what went wrong, so the check can say why there are no passages. */
+export async function searchRequirements(cfg: RagConfig, query: string): Promise<Passage[]> {
+  const body = JSON.stringify({ query, top_k: cfg.topK })
+  if (cfg.url) {
+    const token = cfg.tokenEnv ? process.env[cfg.tokenEnv] : undefined
+    if (cfg.tokenEnv && !token) throw new Error(`rag.token_env names ${cfg.tokenEnv}, which is not set`)
+    const res = await fetch(cfg.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Accept: "application/json", ...(token ? { Authorization: `Bearer ${token}` } : {}) },
+      body,
+      signal: AbortSignal.timeout(cfg.timeoutMs),
+    })
+    if (!res.ok) throw new Error(`the RAG service answered ${res.status}`)
+    return passagesFrom(await res.json()).slice(0, cfg.topK)
+  }
+  const [bin, ...args] = cfg.command!
+  const out = await new Promise<string>((resolve, reject) => {
+    const child = spawn(bin, args, { env: withoutTelemetry(process.env), stdio: ["pipe", "pipe", "pipe"], shell: process.platform === "win32" })
+    let stdout = ""
+    let stderr = ""
+    const timer = setTimeout(() => child.kill(), cfg.timeoutMs)
+    child.stdout.on("data", (d) => (stdout += d))
+    child.stderr.on("data", (d) => (stderr += d))
+    child.on("error", reject)
+    child.on("close", (code) => {
+      clearTimeout(timer)
+      if (code === 0) resolve(stdout)
+      else reject(new Error(`the RAG command exited ${code}: ${stderr.trim().slice(-300)}`))
+    })
+    child.stdin.end(body)
+  })
+  try {
+    return passagesFrom(JSON.parse(out)).slice(0, cfg.topK)
+  } catch {
+    throw new Error("the RAG command did not print JSON")
+  }
+}