@capagents/forgeqe 1.2.1 → 1.3.0

package/src/pack/pipeline.ts

@@ -0,0 +1,373 @@
+import { spawn } from "node:child_process"
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs"
+import path from "node:path"
+
+import type { AppConfig, BrowserRemoteConfig } from "../config.ts"
+import { adoSettings } from "../cases/ado.ts"
+import { http } from "../cases/http.ts"
+import { parseResults, type TestOutcome, type TestsRun } from "../code/find.ts"
+import { walk } from "../code/profile.ts"
+import { runDirectory } from "../run.ts"
+import { loadPack, savePack, type Pack, type PackRun } from "./pack.ts"
+import { WS_ENDPOINT_ENV } from "./run.ts"
+
+/**
+ * A pack in CI: a GitHub Actions workflow or an Azure DevOps pipeline in the test repository runs the pack's tests,
+ * selected by one input, on the runner's browser or on TestMu AI. The pipeline builds the TestMu AI endpoint itself
+ * from its own secrets (LT_USERNAME, LT_ACCESS_KEY): no key passes through ForgeQE. ForgeQE starts the run, waits for
+ * it, and reads each test's result (the TRX or JUnit artifact on GitHub, the Test Runs API on Azure DevOps).
+ */
+
+export type Framework = "dotnet" | "playwright" | "pytest"
+export const WORKFLOW_FILE = "forgeqe-pack.yml"
+export const ADO_PIPELINE_FILE = path.join("pipelines", "forgeqe-pack.yml")
+const ARTIFACT = "forgeqe-pack-results"
+
+/** What selects the pack's tests in the framework's own terms, as one pipeline input. */
+export function packSelector(pack: Pick<Pack, "tests">): { framework: Framework; selector: string } {
+  const kinds = new Set(pack.tests.map((t) => (t.file.endsWith(".cs") ? "dotnet" : t.file.endsWith(".py") ? "pytest" : /\.[cm]?[jt]sx?$/.test(t.file) ? "playwright" : "other")))
+  if (kinds.size !== 1 || kinds.has("other")) throw new Error(`A pipeline runs one framework's tests; this pack has ${[...kinds].join(" and ")} tests. Run it on local or testmu.`)
+  const framework = [...kinds][0] as Framework
+  if (framework === "dotnet") return { framework, selector: pack.tests.map((t) => `FullyQualifiedName~${t.test}`).join("|") }
+  if (framework === "playwright") return { framework, selector: [...new Set(pack.tests.map((t) => t.test.split(".").pop()!.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")))].join("|") }
+  return { framework, selector: pack.tests.map((t) => (t.test === t.file ? t.file : `${t.file}::${t.test.replace(/\./g, "::")}`)).join(" ") }
+}
+
+/** The repository's test project and CI settings file, for the .NET pipeline. */
+export function dotnetProject(repo: string, pack: Pick<Pack, "tests">, files = walk(repo)): { project: string; settings?: string } {
+  const projects = files.filter((f) => f.endsWith(".csproj"))
+  const project = projects.filter((p) => pack.tests.some((t) => t.file.startsWith(`${path.posix.dirname(p)}/`))).sort((a, b) => b.length - a.length)[0]
+  if (!project) throw new Error("The pack's tests are in no .NET test project.")
+  const settings = files.find((f) => /(^|\/)ci\.runsettings$/i.test(f)) || files.find((f) => f.endsWith(".runsettings"))
+  return { project, ...(settings ? { settings } : {}) }
+}
+
+/** The script that sets the TestMu AI endpoint from the pipeline's secrets, with the browser from forgeqe.yaml. */
+function endpointScript(remote: BrowserRemoteConfig | undefined, envFile: string): string {
+  const browser = remote?.browser || "Chrome"
+  const version = remote?.browserVersion || "latest"
+  const platform = remote?.platform || "Windows 11"
+  return [
+    `python3 - <<'PY' >> ${envFile}`,
+    "import json, os, urllib.parse",
+    `caps = {"browserName": ${JSON.stringify(browser)}, "browserVersion": ${JSON.stringify(version)}, "LT:Options": {"platform": ${JSON.stringify(platform)}, "build": "ForgeQE pack " + os.environ["PACK"], "name": os.environ["PACK"] + " " + os.environ["RUN_ID"], "user": os.environ["LT_USERNAME"], "accessKey": os.environ["LT_ACCESS_KEY"]}}`,
+    `print("${WS_ENDPOINT_ENV}=wss://cdp.lambdatest.com/playwright?capabilities=" + urllib.parse.quote(json.dumps(caps)))`,
+    "PY",
+  ].join("\n")
+}
+
+const indent = (text: string, n: number) => text.split("\n").map((l) => (l ? " ".repeat(n) + l : l)).join("\n")
+
+/** The test step's commands, reading the selector from $TESTS (never pasted into the script). */
+function testCommands(framework: Framework, dotnet?: { project: string; settings?: string }): { setup: string[]; run: string } {
+  if (framework === "dotnet") {
+    const p = dotnet!.project
+    return {
+      setup: [`dotnet build ${p} -c Release`, `pwsh "$(find ${path.posix.dirname(p)}/bin/Release -name playwright.ps1 | head -n 1)" install --with-deps chromium`],
+      run: `dotnet test ${p} -c Release --no-build${dotnet!.settings ? ` --settings ${dotnet!.settings}` : ""} --filter "$TESTS" --logger "trx;LogFileName=pack.trx" --logger "console;verbosity=normal" --results-directory pack-results`,
+    }
+  }
+  if (framework === "playwright") return { setup: ["npm ci", "npx playwright install --with-deps chromium"], run: 'PLAYWRIGHT_JUNIT_OUTPUT_NAME=pack-results/results.xml npx playwright test --grep "$TESTS" --reporter=line,junit' }
+  return { setup: ["python -m pip install -r requirements.txt", "python -m playwright install --with-deps chromium"], run: "python -m pytest $TESTS --junitxml=pack-results/results.xml" }
+}
+
+/** A GitHub Actions workflow that runs a pack: dispatched by ForgeQE with the pack's selector and target. */
+export function githubWorkflow(framework: Framework, remote: BrowserRemoteConfig | undefined, dotnet?: { project: string; settings?: string }): string {
+  const t = testCommands(framework, dotnet)
+  const setupStep = framework === "dotnet" ? `      - uses: actions/setup-dotnet@v4\n        with:\n          dotnet-version: "9.0.x"` : framework === "playwright" ? `      - uses: actions/setup-node@v4\n        with:\n          node-version: 22` : `      - uses: actions/setup-python@v5\n        with:\n          python-version: "3.12"`
+  return `# Written by ForgeQE (forgeqe pack pipeline init). ForgeQE dispatches it to run a regression pack.
+# For target testmu, add the repository secrets LT_USERNAME and LT_ACCESS_KEY.
+name: ForgeQE regression pack
+run-name: ForgeQE pack \${{ inputs.pack }} (\${{ inputs.run_id }})
+on:
+  workflow_dispatch:
+    inputs:
+      pack:
+        description: Pack name
+        required: true
+      run_id:
+        description: ForgeQE's id for this run
+        required: true
+      tests:
+        description: The pack's tests, in the framework's own selector
+        required: true
+      target:
+        description: Where the browsers run
+        type: choice
+        options: [local, testmu]
+        default: local
+jobs:
+  pack:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+${setupStep}
+      - name: Prepare
+        run: |
+${indent(t.setup.join("\n"), 10)}
+      - name: TestMu AI endpoint
+        if: inputs.target == 'testmu'
+        env:
+          LT_USERNAME: \${{ secrets.LT_USERNAME }}
+          LT_ACCESS_KEY: \${{ secrets.LT_ACCESS_KEY }}
+          PACK: \${{ inputs.pack }}
+          RUN_ID: \${{ inputs.run_id }}
+        run: |
+${indent(endpointScript(remote, '"$GITHUB_ENV"'), 10)}
+      - name: Run the pack
+        env:
+          TESTS: \${{ inputs.tests }}
+        run: ${t.run}
+      # The results are also in the log, one line per test: a full artifact store must not fail the pack.
+      - uses: actions/upload-artifact@v4
+        if: always()
+        continue-on-error: true
+        with:
+          name: ${ARTIFACT}
+          path: pack-results
+          retention-days: 1
+`
+}
+
+/** An Azure DevOps pipeline that runs a pack: queued by ForgeQE with the pack's selector and target. */
+export function adoPipeline(framework: Framework, remote: BrowserRemoteConfig | undefined, dotnet?: { project: string; settings?: string }): string {
+  const t = testCommands(framework, dotnet)
+  const tool = framework === "dotnet" ? `  - task: UseDotNet@2\n    inputs:\n      packageType: sdk\n      version: 9.0.x` : framework === "playwright" ? `  - task: NodeTool@0\n    inputs:\n      versionSpec: "22.x"` : `  - task: UsePythonVersion@0\n    inputs:\n      versionSpec: "3.12"`
+  return `# Written by ForgeQE (forgeqe pack pipeline init). ForgeQE queues it to run a regression pack.
+# For target testmu, add the secret pipeline variables LT_USERNAME and LT_ACCESS_KEY.
+trigger: none
+pr: none
+name: ForgeQE pack \${{ parameters.pack }} (\${{ parameters.run_id }})
+parameters:
+  - name: pack
+    type: string
+  - name: run_id
+    type: string
+  - name: tests
+    type: string
+  - name: target
+    type: string
+    default: local
+    values: [local, testmu]
+pool:
+  vmImage: ubuntu-latest
+steps:
+${tool}
+  - script: |
+${indent(t.setup.join("\n"), 6)}
+    displayName: Prepare
+  - script: |
+${indent(endpointScript(remote, "endpoint.env"), 6)}
+      echo "##vso[task.setvariable variable=${WS_ENDPOINT_ENV};issecret=true]$(sed -n 's/^${WS_ENDPOINT_ENV}=//p' endpoint.env)"
+      rm endpoint.env
+    displayName: TestMu AI endpoint
+    condition: eq('\${{ parameters.target }}', 'testmu')
+    env:
+      LT_USERNAME: $(LT_USERNAME)
+      LT_ACCESS_KEY: $(LT_ACCESS_KEY)
+      PACK: \${{ parameters.pack }}
+      RUN_ID: \${{ parameters.run_id }}
+  - script: ${t.run}
+    displayName: Run the pack
+    env:
+      TESTS: \${{ parameters.tests }}
+      ${WS_ENDPOINT_ENV}: $(${WS_ENDPOINT_ENV})
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFormat: ${framework === "dotnet" ? "VSTest" : "JUnit"}
+      testResultsFiles: "pack-results/**/*.${framework === "dotnet" ? "trx" : "xml"}"
+      testRunTitle: ForgeQE pack \${{ parameters.pack }}
+`
+}
+
+export type Exec = (args: string[], cwd: string) => Promise<{ code: number; out: string; err: string }>
+
+const ghExec: Exec = (args, cwd) =>
+  new Promise((resolve) => {
+    const p = spawn("gh", args, { cwd, stdio: ["ignore", "pipe", "pipe"] })
+    let out = ""
+    let err = ""
+    p.stdout.on("data", (d) => (out += d))
+    p.stderr.on("data", (d) => (err += d))
+    p.on("close", (code) => resolve({ code: code ?? 1, out, err }))
+    p.on("error", (e) => resolve({ code: 1, out, err: e.message }))
+  })
+
+const sleep = (ms: number, signal?: AbortSignal) =>
+  new Promise<void>((resolve) => {
+    const t = setTimeout(resolve, ms)
+    signal?.addEventListener("abort", () => (clearTimeout(t), resolve()), { once: true })
+  })
+
+function filesUnder(dir: string): string[] {
+  if (!existsSync(dir)) return []
+  return readdirSync(dir).flatMap((e) => {
+    const f = path.join(dir, e)
+    return statSync(f).isDirectory() ? filesUnder(f) : /\.(trx|xml)$/i.test(e) ? [f] : []
+  })
+}
+
+/** Per-test results from a run's log: dotnet's console logger ("Passed Name [2 s]"), when there is no results file. */
+export function testsFromLog(log: string): TestOutcome[] {
+  const out: TestOutcome[] = []
+  const seen = new Set<string>()
+  for (const line of log.split("\n")) {
+    // "  Passed text#1 Wikipedia: search finds … [7 s]": the duration at the end tells it from the summary line.
+    const m = /(?:^|\s)(Passed|Failed|Skipped)\s+(.+?)\s+\[(?:< ?)?([\d.]+)\s*(ms|s|m)\]\s*$/.exec(line.replace(/\x1b\[[0-9;]*m/g, ""))
+    if (!m) continue
+    const name = m[2]!.trim()
+    if (seen.has(name)) continue
+    seen.add(name)
+    const secs = m[4] === "ms" ? Number(m[3]) / 1000 : m[4] === "m" ? Number(m[3]) * 60 : Number(m[3])
+    out.push({ name, status: m[1] === "Passed" ? "passed" : m[1] === "Failed" ? "failed" : "skipped", seconds: Math.round(secs) })
+  }
+  return out
+}
+
+function runOf(command: string, tests: TestOutcome[], status: TestsRun["status"], seconds: number, dir: string, output: string): TestsRun {
+  const count = (s: TestOutcome["status"]) => tests.filter((t) => t.status === s).length
+  return { status, command, exitCode: status === "passed" ? 0 : 1, seconds, tests, counts: { passed: count("passed"), failed: count("failed"), skipped: count("skipped") }, resultFiles: filesUnder(dir), browsers: [], output, outputFile: "", dir }
+}
+
+export type PipelineOpts = {
+  config: AppConfig
+  repo: string
+  name: string
+  kind?: "github" | "azure_devops"
+  target: "local" | "testmu"
+  signal?: AbortSignal
+  exec?: Exec
+  pollMs?: number
+  onProgress?: (text: string) => void
+}
+
+/** Starts the pack in a pipeline, waits for it, reads each test's result, and keeps the run with the pack. */
+export async function runPackOnPipeline(opts: PipelineOpts): Promise<{ pack: Pack; run: TestsRun; record: PackRun; notes: string[] }> {
+  const cfg = opts.config
+  const pipe = cfg.packs?.pipeline
+  const kind = opts.kind || pipe?.default || (pipe?.github ? "github" : pipe?.azureDevops ? "azure_devops" : undefined)
+  if (!kind || (kind === "github" && !pipe?.github) || (kind === "azure_devops" && !pipe?.azureDevops)) {
+    throw new Error("No pipeline for packs: set packs.pipeline.github (repo, workflow) or packs.pipeline.azure_devops (project, pipeline_id) in forgeqe.yaml, after forgeqe pack pipeline init writes the pipeline file.")
+  }
+  const pack = loadPack(opts.repo, opts.name)
+  if (!pack.tests.length) throw new Error(`Pack ${pack.name} has no tests to run.`)
+  const { selector } = packSelector(pack)
+  const dir = runDirectory(path.join(cfg.export.outDir, "packs", pack.name))
+  const runId = path.basename(dir)
+  const started = Date.now()
+  const deadline = started + (pipe!.timeoutMinutes || 60) * 60_000
+  const poll = opts.pollMs ?? 15_000
+  const say = opts.onProgress || (() => {})
+  const notes: string[] = []
+  let tests: TestOutcome[] = []
+  let status: TestsRun["status"] = "failed"
+  let url = ""
+  let command = ""
+
+  if (kind === "github") {
+    const g = pipe!.github!
+    const exec = opts.exec || ghExec
+    const inputs = { pack: pack.name, run_id: runId, tests: selector, target: opts.target }
+    command = `gh workflow run ${g.workflow} -R ${g.repo} --ref ${g.ref} ${Object.keys(inputs).map((k) => `-f ${k}=…`).join(" ")}`
+    const fired = await exec(["workflow", "run", g.workflow, "-R", g.repo, "--ref", g.ref, ...Object.entries(inputs).flatMap(([k, v]) => ["-f", `${k}=${v}`])], opts.repo)
+    if (fired.code !== 0) throw new Error(`GitHub did not start ${g.workflow} in ${g.repo}: ${(fired.err || fired.out).trim().slice(0, 300)}`)
+    say(`Started ${g.workflow} in ${g.repo}; waiting for the run`)
+    let id = ""
+    while (!id && Date.now() < deadline && !opts.signal?.aborted) {
+      await sleep(Math.min(poll, 5000), opts.signal)
+      const list = await exec(["run", "list", "-R", g.repo, "-w", g.workflow, "--event", "workflow_dispatch", "--limit", "20", "--json", "databaseId,displayTitle,url"], opts.repo)
+      const hit = (JSON.parse(list.out || "[]") as { databaseId: number; displayTitle: string; url: string }[]).find((r) => r.displayTitle.includes(runId))
+      if (hit) {
+        id = String(hit.databaseId)
+        url = hit.url
+      }
+    }
+    if (!id) throw new Error(`The run of ${g.workflow} for ${runId} did not appear in ${g.repo}.`)
+    say(`Run ${url}`)
+    let conclusion = ""
+    while (Date.now() < deadline && !opts.signal?.aborted) {
+      const view = await exec(["run", "view", id, "-R", g.repo, "--json", "status,conclusion"], opts.repo)
+      const v = JSON.parse(view.out || "{}") as { status?: string; conclusion?: string }
+      if (v.status === "completed") {
+        conclusion = v.conclusion || ""
+        break
+      }
+      await sleep(poll, opts.signal)
+    }
+    if (opts.signal?.aborted) status = "stopped"
+    else if (!conclusion) status = "timed out"
+    else {
+      mkdirSync(path.join(dir, "results"), { recursive: true })
+      const dl = await exec(["run", "download", id, "-R", g.repo, "-n", ARTIFACT, "-D", path.join(dir, "results")], opts.repo)
+      if (dl.code !== 0) notes.push(`The results artifact could not be downloaded: ${(dl.err || dl.out).trim().slice(0, 200)}`)
+      for (const f of filesUnder(path.join(dir, "results"))) tests.push(...parseResults(readFileSync(f, "utf8")))
+      if (!tests.length) {
+        // No artifact (a full artifact store, an upload that failed): the test step's log has a line per test.
+        const log = await exec(["run", "view", id, "-R", g.repo, "--log"], opts.repo)
+        tests = testsFromLog(log.out)
+        if (tests.length) notes.push("The results artifact was missing; the results come from the run's log.")
+      }
+      status = conclusion === "cancelled" ? "stopped" : tests.length ? (tests.some((t) => t.status === "failed") ? "failed" : "passed") : conclusion === "success" ? "passed" : "failed"
+    }
+  } else {
+    const a = pipe!.azureDevops!
+    const { org, auth } = adoSettings(cfg.sources)
+    const base = `${org}/${encodeURIComponent(a.project)}`
+    command = `Azure DevOps pipeline ${a.pipelineId} in ${a.project} (${a.ref})`
+    const { data: queued } = await http<{ id: number; _links?: { web?: { href?: string } } }>(`${base}/_apis/pipelines/${a.pipelineId}/runs?api-version=7.1`, {
+      auth,
+      body: { resources: { repositories: { self: { refName: a.ref } } }, templateParameters: { pack: pack.name, run_id: runId, tests: selector, target: opts.target } },
+    })
+    url = queued?._links?.web?.href || ""
+    say(`Queued pipeline run ${queued.id}${url ? ` (${url})` : ""}`)
+    let result = ""
+    let misses = 0
+    while (Date.now() < deadline && !opts.signal?.aborted) {
+      let data: { state?: string; result?: string } | undefined
+      try {
+        data = (await http<{ state?: string; result?: string }>(`${base}/_apis/pipelines/${a.pipelineId}/runs/${queued.id}?api-version=7.1`, { auth })).data
+        misses = 0
+      } catch (err) {
+        // A dropped connection while waiting is not the run's result: try again, a few times.
+        if (++misses > 5) throw err
+        await sleep(poll, opts.signal)
+        continue
+      }
+      if (data?.state === "completed") {
+        result = data.result || ""
+        break
+      }
+      await sleep(poll, opts.signal)
+    }
+    if (opts.signal?.aborted) status = "stopped"
+    else if (!result) status = "timed out"
+    else {
+      const { data: runs } = await http<{ value?: { id: number }[] }>(`${base}/_apis/test/runs?buildUri=${encodeURIComponent(`vstfs:///Build/Build/${queued.id}`)}&api-version=7.1`, { auth })
+      for (const r of runs?.value || []) {
+        const { data: res } = await http<{ value?: { automatedTestName?: string; testCaseTitle?: string; outcome?: string; errorMessage?: string; durationInMs?: number }[] }>(`${base}/_apis/test/Runs/${r.id}/results?$top=1000&api-version=7.1`, { auth })
+        for (const t of res?.value || []) {
+          const o = String(t.outcome || "").toLowerCase()
+          tests.push({ name: t.automatedTestName || t.testCaseTitle || "", status: o === "passed" ? "passed" : o === "failed" ? "failed" : "skipped", ...(t.errorMessage ? { message: t.errorMessage } : {}), ...(t.durationInMs ? { seconds: Math.round(t.durationInMs / 1000) } : {}) } as TestOutcome)
+        }
+      }
+      status = result === "canceled" ? "stopped" : tests.length ? (tests.some((t) => t.status === "failed") ? "failed" : "passed") : result === "succeeded" ? "passed" : "failed"
+      if (!tests.length && result !== "succeeded") {
+        // Why it failed before any test ran: the run's own errors (no agent, a YAML error, a build that broke).
+        try {
+          const { data: tl } = await http<{ records?: { issues?: { type?: string; message?: string }[] }[] }>(`${base}/_apis/build/builds/${queued.id}/timeline?api-version=7.1`, { auth })
+          const errors = [...new Set((tl?.records || []).flatMap((r) => (r.issues || []).filter((i) => i.type === "error").map((i) => String(i.message || "").trim())))].filter(Boolean)
+          if (errors.length) notes.push(`The pipeline failed before reporting tests: ${errors.slice(0, 3).join(" | ").slice(0, 600)}`)
+        } catch {}
+      }
+    }
+  }
+  if (status === "passed" && tests.some((t) => t.status === "failed")) status = "failed"
+  if (!tests.length && status !== "stopped" && status !== "timed out") notes.push("The pipeline finished but gave no per-test results.")
+  const run = runOf(command, tests, status, Math.round((Date.now() - started) / 1000), dir, url ? `Pipeline run: ${url}` : "")
+  const record: PackRun = { at: new Date().toISOString(), on: "pipeline", target: `${kind === "github" ? "GitHub Actions" : "Azure DevOps"}${opts.target === "testmu" ? " on TestMu AI" : ""}`, status, counts: run.counts, seconds: run.seconds, failed: tests.filter((t) => t.status === "failed").map((t) => t.name), dir, ...(url ? { url } : {}) }
+  pack.runs = [...pack.runs, record].slice(-20)
+  savePack(pack)
+  return { pack, run, record, notes }
+}

package/src/pack/run.ts

@@ -0,0 +1,155 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { BrowserRemoteConfig, AppConfig } from "../config.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import { runSelection, type TestSelection, type TestsRun } from "../code/find.ts"
+import { writtenTests } from "../code/notes.ts"
+import { walk } from "../code/profile.ts"
+import type { runTestCommand } from "../code/testrun.ts"
+import { runDirectory } from "../run.ts"
+import { loadPack, packCommand, savePack, type Pack, type PackRun } from "./pack.ts"
+
+/**
+ * Runs a pack's tests where the release needs them: here (local), on TestMu AI's browsers (the repository's own tests,
+ * connected to a remote browser through FORGEQE_PW_WS_ENDPOINT), or in a pipeline (pipeline.ts). Each run is kept with
+ * the pack, so the next run of the same pack can be compared with it.
+ */
+
+/** The variable a repository's tests read to connect to a remote browser instead of launching one. */
+export const WS_ENDPOINT_ENV = "FORGEQE_PW_WS_ENDPOINT"
+
+/**
+ * TestMu AI's (LambdaTest's) Playwright endpoint for a run: its capabilities from forgeqe.yaml (browser.remote for
+ * testmu), the build and test names, and the account from the environment. `secret` is the access key, for redacting.
+ */
+export function testmuEndpoint(remote: BrowserRemoteConfig | undefined, labels: { build: string; name: string }, env: Record<string, string | undefined> = process.env): { endpoint: string; secret: string; describe: string } {
+  if (!remote) throw new Error("No TestMu AI settings: add browser.remote for testmu in forgeqe.yaml (or a testmu.yaml), with usernameEnv and accessKeyEnv.")
+  const user = env[remote.usernameEnv]
+  const key = env[remote.accessKeyEnv]
+  if (!user || !key) throw new Error(`Set ${remote.usernameEnv} and ${remote.accessKeyEnv} to run on TestMu AI.`)
+  const browser = /^pw-/.test(remote.browser) ? remote.browser : remote.browser || "Chrome"
+  const caps = {
+    browserName: browser,
+    browserVersion: remote.browserVersion || "latest",
+    "LT:Options": {
+      platform: remote.platform || [remote.os, remote.osVersion].filter(Boolean).join(" ") || "Windows 11",
+      build: labels.build,
+      name: labels.name,
+      ...(remote.project ? { project: remote.project } : {}),
+      user,
+      accessKey: key,
+    },
+  }
+  return {
+    endpoint: `wss://cdp.lambdatest.com/playwright?capabilities=${encodeURIComponent(JSON.stringify(caps))}`,
+    secret: key,
+    describe: `${browser} ${caps.browserVersion} on ${caps["LT:Options"].platform}`,
+  }
+}
+
+/** The pack as a selection run_tests' runner can run: its command, and where its results land. */
+export function packSelection(cfg: AppConfig, pack: Pack, dir: string): { selection: TestSelection; notes: string[] } {
+  const resultsDir = path.join(dir, "results")
+  const known = cfg.memory.enabled ? writtenTests(cfg.memory.dir, pack.repo).map((t) => t.command) : []
+  const cmd = packCommand(pack.repo, pack.tests, resultsDir, known)
+  if (!cmd) throw new Error(`ForgeQE cannot build one command for the tests of pack ${pack.name} (${[...new Set(pack.tests.map((t) => path.extname(t.file)))].join(", ")} files).`)
+  return {
+    notes: cmd.notes,
+    selection: {
+      id: path.basename(dir),
+      dir,
+      repo: pack.repo,
+      request: `regression pack ${pack.name}`,
+      framework: cmd.framework,
+      tests: pack.tests.map((t) => ({ name: t.test, file: t.file, reason: t.reasons.map((r) => r.text).join("; ") })),
+      related: [],
+      listCommand: "",
+      listed: false,
+      runCommand: cmd.command,
+      results: [resultsDir],
+      notes: "",
+      model: "",
+      usage: { input: 0, output: 0 },
+      createdAt: new Date().toISOString(),
+    },
+  }
+}
+
+/** How a repository's tests connect to a remote browser when ForgeQE gives them one, by framework. */
+export const REMOTE_HOWTO = [
+  `Make the test base class connect to the endpoint in ${WS_ENDPOINT_ENV} when it is set, and launch a browser as before when it is not.`,
+  `NUnit (Microsoft.Playwright.NUnit 1.46+): override ConnectOptionsAsync to return (endpoint, new BrowserTypeConnectOptions()) when the variable is set, else null.`,
+  `Playwright Test: in playwright.config, use: { connectOptions: process.env.${WS_ENDPOINT_ENV} ? { wsEndpoint: process.env.${WS_ENDPOINT_ENV} } : undefined }.`,
+  `pytest-playwright: a browser fixture that calls playwright.chromium.connect(os.environ["${WS_ENDPOINT_ENV}"]) when it is set.`,
+].join(" ")
+
+/** Whether the repository's test code reads the remote endpoint variable. */
+export function readsEndpoint(repo: string, files = walk(repo)): boolean {
+  return files
+    .filter((f) => /\.(cs|[cm]?[jt]sx?|py|java|kt)$/.test(f))
+    .some((f) => {
+      try {
+        return readFileSync(path.join(repo, f), "utf8").includes(WS_ENDPOINT_ENV)
+      } catch {
+        return false
+      }
+    })
+}
+
+const redact = (text: string, secrets: string[]) => secrets.filter((s) => s.length >= 6).reduce((t, s) => t.split(s).join("***").split(encodeURIComponent(s)).join("***"), text)
+
+/** Runs a pack here or on TestMu AI, and keeps the run with the pack. */
+export async function runPack(opts: {
+  config: AppConfig
+  repo: string
+  name: string
+  on: "local" | "testmu"
+  signal?: AbortSignal
+  runTest?: typeof runTestCommand
+  env?: Record<string, string | undefined>
+}): Promise<{ pack: Pack; run: TestsRun; record: PackRun; notes: string[] }> {
+  const cfg = opts.config
+  const pack = loadPack(opts.repo, opts.name)
+  if (!pack.tests.length) throw new Error(`Pack ${pack.name} has no tests to run.`)
+  const dir = runDirectory(path.join(cfg.export.outDir, "packs", pack.name))
+  const { selection, notes } = packSelection(cfg, pack, dir)
+  let env: Record<string, string> | undefined
+  let secrets: string[] = []
+  let target = "this machine"
+  if (opts.on === "testmu") {
+    // Without it the tests would launch a local browser and the run would only look remote.
+    if (!readsEndpoint(pack.repo)) throw new Error(`The tests in ${pack.repo} do not read ${WS_ENDPOINT_ENV}, so they cannot connect to TestMu AI's browsers yet. ${REMOTE_HOWTO}`)
+    const t = testmuEndpoint(cfg.browser.remotes?.testmu, { build: `ForgeQE pack ${pack.name}`, name: `${pack.name} ${path.basename(dir)}` }, opts.env || process.env)
+    env = { [WS_ENDPOINT_ENV]: t.endpoint }
+    secrets = [t.secret]
+    target = `TestMu AI (${t.describe})`
+  }
+  const run = await runSelection({
+    selection,
+    timeoutSeconds: Math.max(cfg.code.testTimeoutSeconds, 300) * Math.max(1, Math.ceil(pack.tests.length / 10)),
+    signal: opts.signal,
+    runTest: opts.runTest,
+    hide: privateEnvNames(cfg),
+    flaky: cfg.flaky,
+    ...(env ? { env } : {}),
+  })
+  if (secrets.length) {
+    run.output = redact(run.output, secrets)
+    run.command = redact(run.command, secrets)
+    if (run.outputFile && existsSync(run.outputFile)) writeFileSync(run.outputFile, redact(readFileSync(run.outputFile, "utf8"), secrets))
+  }
+  const record: PackRun = {
+    at: new Date().toISOString(),
+    on: opts.on,
+    target,
+    status: run.status,
+    counts: run.counts,
+    seconds: run.seconds,
+    failed: run.tests.filter((t) => t.status === "failed").map((t) => t.name),
+    dir,
+  }
+  pack.runs = [...pack.runs, record].slice(-20)
+  savePack(pack)
+  return { pack, run, record, notes }
+}