@capagents/forgeqe 0.0.0-stage → 1.0.1

package/src/api/write.ts

@@ -0,0 +1,272 @@
+import { existsSync, realpathSync, statSync } from "node:fs"
+import path from "node:path"
+
+import type { AppConfig } from "../config.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import { addUsage } from "../usage.ts"
+import type { CodeEvent, Usage } from "../types.ts"
+import { loadSkills } from "../skills/registry.ts"
+import { buildProfile, profileSection } from "../code/profile.ts"
+import { RESULT_SCHEMA, type AgentReport } from "../code/prompt.ts"
+import { openSession, type AgentSpec, type OpencodeSession } from "../code/runtime.ts"
+import { headAndTail, runTestCommand } from "../code/testrun.ts"
+import { ownPaths, protectPatterns } from "../code/worktree.ts"
+import { associateAdoAutomation, recordMapping } from "../code/traceability.ts"
+import type { WriteBack } from "../cases/types.ts"
+import { briefMarkdown, scenarioMarkdown } from "./store.ts"
+import type { ContractSet, Scenario } from "./types.ts"
+import { prompt } from "../prompts.ts"
+import { domainNotes } from "../domain.ts"
+
+/**
+ * The coding agent writes API and contract tests for a contract set in the repository's own framework, and ForgeQE
+ * runs them until they pass: Playwright's request fixture in a Playwright repository, pytest with requests or httpx,
+ * REST Assured, RestSharp or HttpClient, supertest... whatever the repository already uses for HTTP, else what its
+ * language uses most. No browser: these tests call the API directly, so they are fast and do not break when the page
+ * changes.
+ */
+
+export const apiAgentPrompt = () => prompt("api-tests")
+
+export const API_AGENT: AgentSpec = {
+  name: "forgeqe-api",
+  description: "Writes API and contract tests in this repository from ForgeQE's contracts",
+  get prompt() {
+    return apiAgentPrompt()
+  },
+  permission: { "live_*": "deny" },
+  skills: "scribe",
+}
+
+/** The test result Scribe reports, plus the scenario tests' names and the regression command for changed helpers. */
+export const API_RESULT_SCHEMA = {
+  ...RESULT_SCHEMA,
+  properties: {
+    ...RESULT_SCHEMA.properties,
+    scenario_tests: {
+      type: "array",
+      items: { type: "object", properties: { scenario_id: { type: "string" }, test_name: { type: "string" } }, required: ["scenario_id", "test_name"] },
+      description: "Each scenario's test, by the scenario id ForgeQE gave and the test's full name as the framework reports it",
+    },
+    helpers_changed: { type: "array", items: { type: "string" }, description: "Existing helpers you changed that other tests use" },
+    regression_command: { type: "string", description: "When you changed helpers other tests use: the command that runs those tests" },
+  },
+} as const
+
+type ApiReport = AgentReport & { scenario_tests?: { scenario_id: string; test_name: string }[]; helpers_changed?: string[]; regression_command?: string }
+
+export type ApiTestsResult = {
+  status: "passed" | "failed" | "blocked" | "stopped"
+  repo: string
+  attempts: number
+  testFile?: string
+  command?: string
+  changed: string[]
+  reused: string[]
+  notes: string
+  output: string
+  usage: Usage
+  model: string
+  /** Scenarios with a test case: the test each got, and the write-back to the test case's system. */
+  mapped: { caseId: string; testName: string }[]
+  writeBack: WriteBack[]
+  helpersChanged: string[]
+}
+
+/** The names of environment variables api.headers reads (never their values), for the agent to use the same ones. */
+export function credentialNames(config: AppConfig): string[] {
+  const names = new Set<string>()
+  for (const v of Object.values(config.api?.headers || {})) for (const m of v.matchAll(/\$\{(\w+)\}|\$(\w+)/g)) names.add(m[1] || m[2])
+  return [...names]
+}
+
+export function apiTestsPrompt(set: ContractSet, opts: { baseUrl: string; profile?: string; credentials: string[]; headers: string[]; writes: boolean; scenarios?: Scenario[] }): string {
+  if (opts.scenarios?.length) return scenarioTestsPrompt(set, { ...opts, scenarios: opts.scenarios })
+  const runnable = set.contracts.filter((c) => opts.writes || !c.writes)
+  const leftOut = set.contracts.length - runnable.length
+  return [
+    `Write API and contract tests for these ${runnable.length} operation${runnable.length === 1 ? "" : "s"} of ${set.name}.`,
+    `Base URL: ${opts.baseUrl || "(not known: read it from API_BASE_URL)"}`,
+    opts.headers.length
+      ? `Every call sends these headers: ${opts.headers.join(", ")}. Read their values from the environment variables ${opts.credentials.join(", ") || "(none named)"}, as forgeqe.yaml's api.headers does.`
+      : set.contracts.some((c) => c.auth)
+        ? "Some operations need auth (see each one). No credential variables are configured: read a bearer token from API_TOKEN, and skip the test with a clear message when it is not set."
+        : "",
+    leftOut ? `${leftOut} operation${leftOut === 1 ? "" : "s"} that change data are left out: do not write tests for them.` : "",
+    opts.profile || "",
+    "",
+    "The contracts (contracts.json has the full schemas):",
+    "",
+    briefMarkdown({ ...set, contracts: runnable }),
+  ]
+    .filter((l, i, all) => l !== "" || all[i - 1] !== "")
+    .join("\n")
+}
+
+function credentialLine(set: ContractSet, opts: { credentials: string[]; headers: string[] }): string {
+  return opts.headers.length
+    ? `Every call sends these headers: ${opts.headers.join(", ")}. Read their values from the environment variables ${opts.credentials.join(", ") || "(none named)"}, as forgeqe.yaml's api.headers does.`
+    : set.contracts.some((c) => c.auth)
+      ? "Some operations need auth. No credential variables are configured: read a bearer token from API_TOKEN, and skip the test with a clear message when it is not set."
+      : ""
+}
+
+/** End-to-end scenarios to write as tests, on the repository's existing API helpers. */
+export function scenarioTestsPrompt(set: ContractSet, opts: { baseUrl: string; profile?: string; credentials: string[]; headers: string[]; scenarios: Scenario[] }): string {
+  const used = new Set(opts.scenarios.flatMap((s) => [...s.steps, ...(s.cleanup || [])].map((st) => st.operation).filter(Boolean)))
+  const ops = set.contracts.filter((c) => used.has(c.id))
+  return [
+    `Write ${opts.scenarios.length === 1 ? "an end-to-end API scenario test" : `${opts.scenarios.length} end-to-end API scenario tests`} for ${set.name}: one test per scenario, built on the repository's existing API helpers (map them first, as your instructions say).`,
+    `Base URL: ${opts.baseUrl || "(not known: read it from API_BASE_URL)"}`,
+    credentialLine(set, opts),
+    opts.profile || "",
+    "",
+    briefMarkdown({ ...set, contracts: ops, scenarios: opts.scenarios, skipped: undefined }),
+  ]
+    .filter((l, i, all) => l !== "" || all[i - 1] !== "")
+    .join("\n")
+}
+
+export async function writeApiTests(opts: {
+  config: AppConfig
+  repo: string
+  set: ContractSet
+  setDir: string
+  baseUrl?: string
+  writes?: boolean
+  /** Write these scenarios as end-to-end tests instead of one test per contract. */
+  scenarios?: Scenario[]
+  /** Link scenario tests to their Azure DevOps test cases (Associated Automation). Default: sources.write_back. */
+  writeBack?: boolean
+  signal?: AbortSignal
+  onEvent?: (e: CodeEvent) => void
+  open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession>
+  runTest?: typeof runTestCommand
+}): Promise<ApiTestsResult> {
+  const given = path.resolve(opts.repo)
+  if (!existsSync(given) || !statSync(given).isDirectory()) throw new Error(`Repo not found: ${given}`)
+  const repo = realpathSync(given)
+  const { config } = opts
+  const emit = opts.onEvent || (() => {})
+  const runTest = opts.runTest || runTestCommand
+  const maxAttempts = Math.max(1, config.code.maxAttempts)
+  const baseUrl = opts.baseUrl || config.api?.baseUrl || opts.set.baseUrl
+  const result: ApiTestsResult = { status: "failed", repo, attempts: 0, changed: [], reused: [], notes: "", output: "", usage: { input: 0, output: 0 }, model: "", mapped: [], writeBack: [], helpersChanged: [] }
+  const own = ownPaths(repo, [config.export.outDir, config.memory.dir])
+  const session = await (opts.open || openSession)({
+    repo,
+    profile: config.llm,
+    domain: domainNotes(config),
+    model: config.code.model,
+    bin: config.code.opencode,
+    readDirs: [repo, opts.setDir],
+    hide: privateEnvNames(config),
+    signal: opts.signal,
+    onEvent: emit,
+    protect: protectPatterns(repo, own, []),
+    stallAfter: config.code.stallAfter ?? 5,
+    agent: { ...API_AGENT, steps: config.code.maxSteps ?? 50 },
+    skills: loadSkills(config.skills, repo).skills,
+    schema: API_RESULT_SCHEMA as unknown as Record<string, unknown>,
+    title: opts.scenarios?.length ? "ForgeQE API scenarios" : "ForgeQE API tests",
+  })
+  result.model = session.model
+  const budget = config.code.maxTokens
+  const credentials = credentialNames(config)
+  const env = { ...(baseUrl ? { API_BASE_URL: baseUrl } : {}) }
+  try {
+    const profile = buildProfile(repo)
+    let message = apiTestsPrompt(opts.set, { baseUrl, profile: profile ? profileSection(profile) : undefined, credentials, headers: Object.keys(config.api?.headers || {}), writes: Boolean(opts.writes), scenarios: opts.scenarios })
+    emit({ type: "code", kind: "info", text: opts.scenarios?.length ? `Scribe (${session.model}) is writing ${opts.scenarios.length} API scenario test${opts.scenarios.length === 1 ? "" : "s"} in ${repo}, on its existing API helpers` : `Scribe (${session.model}) is writing API tests for ${opts.set.contracts.length} operations in ${repo}` })
+    let report: ApiReport | null = null
+    for (let attempt = 1; attempt <= maxAttempts && !opts.signal?.aborted; attempt++) {
+      result.attempts = attempt
+      const reply = await session.prompt(message)
+      report = reply.report as ApiReport | null
+      if (!report) {
+        result.notes = reply.error ? `Scribe stopped: ${reply.error}` : "Scribe gave no result."
+        break
+      }
+      result.testFile = report.test_file
+      result.command = report.command
+      result.changed = report.changed || []
+      result.reused = report.reused || []
+      result.notes = report.notes || ""
+      if (report.status === "blocked") {
+        result.status = "blocked"
+        result.notes = report.blocked_reason || result.notes
+        break
+      }
+      if (!report.command) {
+        message = "Give the command that runs only these API tests from the repository root, in the structured result."
+        continue
+      }
+      emit({ type: "code", kind: "info", text: `ForgeQE runs the API tests (attempt ${attempt} of ${maxAttempts}): ${report.command}` })
+      const prefix = Object.entries(env).map(([k, v]) => `${k}=${JSON.stringify(v)} `).join("")
+      const run = await runTest({ command: process.platform === "win32" ? report.command : `${prefix}${report.command}`, cwd: repo, timeoutSeconds: config.code.testTimeoutSeconds, signal: opts.signal, save: path.join(opts.setDir, `tests-attempt-${attempt}`), hide: privateEnvNames(config) })
+      result.output = run.output
+      result.helpersChanged = report.helpers_changed || []
+      // Changed shared helpers: the tests that already use them must still pass.
+      const regression = run.ok && report.regression_command?.trim() && report.regression_command.trim() !== report.command.trim() ? report.regression_command.trim() : ""
+      if (regression) {
+        emit({ type: "code", kind: "info", text: `ForgeQE runs the tests that use the changed helpers: ${regression}` })
+        const again = await runTest({ command: process.platform === "win32" ? regression : `${prefix}${regression}`, cwd: repo, timeoutSeconds: config.code.testTimeoutSeconds, signal: opts.signal, save: path.join(opts.setDir, `regression-attempt-${attempt}`), hide: privateEnvNames(config) })
+        if (!again.ok) {
+          emit({ type: "code", kind: "info", text: `Tests that use the changed helpers fail now (exit ${again.exitCode ?? "timeout"})` })
+          result.output = again.output
+          message = [
+            `The new tests pass, but tests that already used the helpers you changed now fail. ForgeQE ran: ${regression}`,
+            "",
+            "```",
+            headAndTail(again.output),
+            "```",
+            "",
+            "Change the helpers so the existing tests pass as before (add, do not change what they rely on), then give the structured result again.",
+          ].join("\n")
+          continue
+        }
+      }
+      if (run.ok) {
+        result.status = "passed"
+        emit({ type: "code", kind: "info", text: `API tests passed in ${(run.ms / 1000).toFixed(1)}s` })
+        break
+      }
+      emit({ type: "code", kind: "info", text: `API tests failed (exit ${run.exitCode ?? "timeout"})` })
+      if (budget && session.usage.input + session.usage.output > budget) {
+        result.notes = `Stopped: the job reached its token budget (code.max_tokens ${budget}).`
+        break
+      }
+      message = [
+        `ForgeQE ran: ${report.command}`,
+        `It failed (exit ${run.exitCode ?? "timeout: the command did not finish"}). The output${run.log ? ` (full output in ${run.log})` : ""}:`,
+        "",
+        "```",
+        headAndTail(run.output),
+        "```",
+        "",
+        "Fix the tests (or say in notes where the API itself is wrong), then give the structured result again.",
+      ].join("\n")
+    }
+    if (opts.signal?.aborted) result.status = "stopped"
+    if (result.status === "passed" && report?.test_file) await mapScenarios(opts, repo, report, result)
+  } finally {
+    result.usage = addUsage(result.usage, session.usage)
+    await session.close()
+  }
+  return result
+}
+
+/** Scenario tests that automate a test case: recorded in the repository's mapping file, and in Azure DevOps when write-back is on. */
+async function mapScenarios(opts: { config: AppConfig; scenarios?: Scenario[]; writeBack?: boolean }, repo: string, report: ApiReport, result: ApiTestsResult): Promise<void> {
+  for (const sc of opts.scenarios || []) {
+    if (!sc.caseId) continue
+    const testName = report.scenario_tests?.find((t) => t.scenario_id === sc.id)?.test_name || `${sc.caseId} ${sc.name}`
+    recordMapping(repo, { caseId: sc.caseId, title: sc.caseTitle || sc.name, source: sc.caseSource || "text", ...(sc.caseUrl ? { url: sc.caseUrl } : {}), status: "automated", testFile: report.test_file, testName, command: report.command })
+    result.mapped.push({ caseId: sc.caseId, testName })
+    if (sc.caseSource === "ado" && (opts.writeBack ?? opts.config.sources.writeBack)) {
+      result.writeBack.push(
+        await associateAdoAutomation(opts.config.sources, { id: sc.caseId, title: sc.caseTitle || sc.name, source: "ado", steps: [], ...(sc.caseUrl ? { url: sc.caseUrl } : {}), ...(sc.caseRef ? { ref: sc.caseRef } : {}) }, { name: testName, file: report.test_file!, command: report.command }),
+      )
+    }
+  }
+}