@capagents/forgeqe 1.3.1 → 2.0.0

package/src/containers/run.ts

@@ -0,0 +1,362 @@
+import { createHash } from "node:crypto"
+import { existsSync, readdirSync, realpathSync, statSync } from "node:fs"
+import net from "node:net"
+import path from "node:path"
+
+import { allowListEnv, privateEnvNames } from "../childEnv.ts"
+import type { AppConfig } from "../config.ts"
+import { containersOf, type ContainerRuntime, type ServiceSpec, type ServiceVolume } from "./config.ts"
+
+/**
+ * Docker and Podman for test runs (docs/CONTAINERS.md). Every command is an argument array ForgeQE builds from a service
+ * forgeqe.yaml declares; every container runs the same restricted way: ports bound to 127.0.0.1, read-only mounts from
+ * inside the project, no new privileges, a process limit, and only the environment it lists. Containers carry labels
+ * with the project and service, so ForgeQE stops only its own.
+ */
+
+export type ExecResult = { code: number; stdout: string; stderr: string }
+export type Exec = (bin: string, args: string[], opts: { env: Record<string, string>; timeoutMs?: number; signal?: AbortSignal }) => Promise<ExecResult>
+
+export const spawnExec: Exec = async (bin, args, opts) => {
+  const proc = Bun.spawn([bin, ...args], { env: opts.env, stdout: "pipe", stderr: "pipe", stdin: "ignore", ...(opts.signal ? { signal: opts.signal } : {}) })
+  const timer = opts.timeoutMs ? setTimeout(() => proc.kill(), opts.timeoutMs) : undefined
+  const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])
+  if (timer) clearTimeout(timer)
+  return { code, stdout, stderr }
+}
+
+export type Runtime = { name: ContainerRuntime; bin: string }
+
+/** The container runtime to use: containers.runtime, else docker, else podman, on PATH. */
+export function findRuntime(cfg: Pick<AppConfig, "containers">, which: (cmd: string) => string | null = (c) => Bun.which(c)): Runtime | null {
+  const order: ContainerRuntime[] = cfg.containers?.runtime ? [cfg.containers.runtime] : ["docker", "podman"]
+  for (const name of order) {
+    const bin = which(name)
+    if (bin) return { name, bin }
+  }
+  return null
+}
+
+/** Settings the docker and podman programs read to find their daemon; nothing else of ForgeQE's environment. */
+const RUNTIME_ENV = ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY", "CONTAINER_HOST", "CONTAINER_CONNECTION", "CONTAINERS_CONF", "CONTAINERS_REGISTRIES_CONF", "CONTAINER_SSHKEY"]
+
+/** The project's label value: one project's containers are never another's. */
+export function projectKey(root: string): string {
+  const real = existsSync(root) ? realpathSync(root) : path.resolve(root)
+  return createHash("sha256").update(real).digest("hex").slice(0, 10)
+}
+
+export const containerName = (key: string, service: string) => `forgeqe-${key}-${service}`
+/** The project's own network: its containers reach each other by service name (db:5432), and nothing else joins it. */
+export const projectNetwork = (key: string) => `forgeqe-${key}`
+
+/** Files whose names say they hold credentials or keys (as for uploads, src/mcp/attachtools.ts). */
+const SECRET_FILE = /^(?:forgeqe\.ya?ml|credentials\.json|auth\.json|llms\.json|id_(?:rsa|dsa|ecdsa|ed25519)|.*\.(?:pem|key|pfx|p12|kdbx))$/i
+const MAX_WALK = 5000
+
+/**
+ * A mount's folder or file, resolved: inside the project; not hidden and holding nothing hidden (.env, .git, .npmrc)
+ * or named like a key; a container can send what it reads anywhere, so this is checked as for an upload.
+ */
+export function checkVolume(v: ServiceVolume, root: string): string {
+  const top = realpathSync(root)
+  // A path outside the project is refused by its name first, without touching it: host paths such as the Docker
+  // socket can fail to stat (EOPNOTSUPP) and must be refused with the reason, not an OS error.
+  const lexical = path.resolve(root, v.host)
+  const inside = (base: string) => {
+    const rel = path.relative(base, lexical)
+    return !rel.startsWith("..") && !path.isAbsolute(rel)
+  }
+  if (!inside(path.resolve(root)) && !inside(top)) throw new Error(`The mount ${v.host} is outside the project folder (${top}); services mount only folders inside it.`)
+  if (!existsSync(v.host)) throw new Error(`The mount ${v.host} does not exist.`)
+  let real: string
+  try {
+    real = realpathSync(v.host)
+  } catch (e) {
+    throw new Error(`The mount ${v.host} cannot be read (${(e as NodeJS.ErrnoException).code || (e as Error).message}), so it is not mounted.`)
+  }
+  const rel = path.relative(top, real)
+  if (rel.startsWith("..") || path.isAbsolute(rel)) throw new Error(`The mount ${v.host} is outside the project folder (${top}); services mount only folders inside it.`)
+  if (!rel) throw new Error(`The mount ${v.host} is the whole project folder, which holds its settings and secrets; mount the folder the service needs.`)
+  if (rel.split(path.sep).some((p) => p.startsWith("."))) throw new Error(`The mount ${v.host} is a hidden file or in a hidden folder, which can hold secrets, so it is not mounted.`)
+  if (SECRET_FILE.test(path.basename(real))) throw new Error(`The mount ${v.host} is named like a file of credentials or keys, so it is not mounted.`)
+  if (statSync(real).isDirectory()) {
+    let seen = 0
+    const walk = (dir: string) => {
+      for (const e of readdirSync(dir, { withFileTypes: true })) {
+        if (++seen > MAX_WALK) throw new Error(`The mount ${v.host} holds more than ${MAX_WALK} files; mount a smaller folder.`)
+        const at = path.join(dir, e.name)
+        if (e.name.startsWith(".")) throw new Error(`The mount ${v.host} holds ${path.relative(real, at)}, a hidden file or folder that can hold secrets; mount a folder without it.`)
+        if (SECRET_FILE.test(e.name)) throw new Error(`The mount ${v.host} holds ${path.relative(real, at)}, named like a file of credentials or keys; mount a folder without it.`)
+        // A link could point outside the project: refused rather than followed.
+        if (e.isSymbolicLink()) throw new Error(`The mount ${v.host} holds a link (${path.relative(real, at)}); mount a folder without links.`)
+        if (e.isDirectory()) walk(at)
+      }
+    }
+    walk(real)
+  }
+  return real
+}
+
+/** The secrets a service may be given: listed by name, set, and never one of ForgeQE's own keys. */
+export function serviceSecrets(spec: Pick<ServiceSpec, "name" | "secrets">, cfg: AppConfig, env: Record<string, string | undefined> = process.env): Record<string, string> {
+  const own = new Set(privateEnvNames(cfg))
+  const out: Record<string, string> = {}
+  for (const name of spec.secrets) {
+    if (own.has(name)) throw new Error(`${spec.name}: ${name} is one of ForgeQE's own keys and is never passed to a container.`)
+    const v = env[name]
+    if (v === undefined || v === "") throw new Error(`${spec.name} needs ${name}, which is not set: set it in the environment or the project's .env.`)
+    out[name] = v
+  }
+  return out
+}
+
+export type Labels = { key: string; service: string; kind?: "service" | "mock" }
+
+/** `run` arguments for a service: ForgeQE's own restrictions, then the service's settings, never a flag from elsewhere. */
+export function runArgs(spec: ServiceSpec, labels: Labels, mounts: { host: string; container: string }[]): string[] {
+  const name = containerName(labels.key, labels.service)
+  return [
+    "run",
+    "--detach",
+    "--name", name,
+    "--label", "forgeqe=1",
+    "--label", `forgeqe.project=${labels.key}`,
+    "--label", `forgeqe.service=${labels.service}`,
+    "--label", `forgeqe.kind=${labels.kind || "service"}`,
+    "--security-opt", "no-new-privileges",
+    "--pids-limit", "1024",
+    "--network", projectNetwork(labels.key),
+    "--network-alias", labels.service,
+    ...spec.ports.flatMap((p) => ["--publish", `127.0.0.1:${p.host}:${p.container}`]),
+    ...Object.entries(spec.env).flatMap(([k, v]) => ["--env", `${k}=${v}`]),
+    // By name only: the runtime reads the value from its own environment, so it is never on a command line.
+    ...spec.secrets.flatMap((n) => ["--env", n]),
+    ...mounts.flatMap((m) => ["--volume", `${m.host}:${m.container}:ro`]),
+    spec.image,
+    ...spec.command,
+  ]
+}
+
+/** Replace secret values in output with ***. */
+export function scrub(text: string, values: string[]): string {
+  let out = text
+  for (const v of values) if (v && v.length >= 4) out = out.split(v).join("***")
+  return out
+}
+
+const clip = (s: string, n: number) => (s.length <= n ? s : `…${s.slice(-n)}`)
+
+export type Ctx = { cfg: AppConfig; runtime: Runtime; exec: Exec; env?: Record<string, string | undefined>; signal?: AbortSignal; log?: (text: string) => void }
+
+function runtimeEnv(ctx: Ctx, secrets: Record<string, string> = {}): Record<string, string> {
+  return { ...allowListEnv(ctx.env || process.env, RUNTIME_ENV), ...secrets }
+}
+
+/** Values to keep out of output: every listed secret and ForgeQE's own keys. */
+function secretValues(ctx: Ctx, extra: Record<string, string> = {}): string[] {
+  const env = ctx.env || process.env
+  const names = [...privateEnvNames(ctx.cfg), ...Object.values(ctx.cfg.containers?.services || {}).flatMap((s) => s.secrets)]
+  return [...Object.values(extra), ...names.map((n) => env[n] || "")]
+}
+
+/** Containers this process started, so they are stopped when it ends (unless kept). */
+const started = new Map<string, { runtime: Runtime; exec: Exec; env: Record<string, string> }>()
+
+export type StartedService = {
+  service: string
+  container: string
+  image: string
+  ready: boolean
+  /** this machine's address for each container port */
+  ports: { container: number; address: string }[]
+  url?: string
+  error?: string
+  logs?: string
+  ms: number
+}
+
+/** Whether the runtime answers (its daemon or machine runs). */
+export async function runtimeVersion(ctx: Ctx): Promise<{ ok: boolean; version?: string; error?: string }> {
+  const r = await ctx.exec(ctx.runtime.bin, ["version", "--format", ctx.runtime.name === "podman" ? "{{.Client.Version}}" : "{{.Server.Version}}"], { env: runtimeEnv(ctx), timeoutMs: 15000 })
+  if (r.code === 0 && r.stdout.trim()) return { ok: true, version: r.stdout.trim() }
+  return { ok: false, error: clip((r.stderr || r.stdout).trim() || `${ctx.runtime.name} version exited with ${r.code}`, 400) }
+}
+
+/**
+ * Whether the service accepts connections on the port. Docker's port proxy accepts every connection on this machine's
+ * side and drops it when nothing listens in the container, so a connection counts only when it stays open a moment.
+ */
+function tcpOpen(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const s = net.connect({ host: "127.0.0.1", port })
+    let settled = false
+    const done = (ok: boolean) => {
+      if (settled) return
+      settled = true
+      s.destroy()
+      resolve(ok)
+    }
+    s.setTimeout(1500, () => done(false))
+    s.once("connect", () => setTimeout(() => done(true), 400))
+    s.once("end", () => done(false))
+    s.once("close", () => done(false))
+    s.once("error", () => done(false))
+  })
+}
+
+async function httpReady(port: number, p: string): Promise<boolean> {
+  try {
+    const res = await fetch(`http://127.0.0.1:${port}${p}`, { signal: AbortSignal.timeout(2000), redirect: "manual" })
+    return res.status < 500
+  } catch {
+    return false
+  }
+}
+
+async function running(ctx: Ctx, name: string): Promise<boolean> {
+  const r = await ctx.exec(ctx.runtime.bin, ["inspect", "--format", "{{.State.Running}}", name], { env: runtimeEnv(ctx), timeoutMs: 15000 })
+  return r.code === 0 && r.stdout.trim() === "true"
+}
+
+async function logsOf(ctx: Ctx, name: string, lines: number, secrets: string[]): Promise<string> {
+  const r = await ctx.exec(ctx.runtime.bin, ["logs", "--tail", String(lines), name], { env: runtimeEnv(ctx), timeoutMs: 15000 })
+  return clip(scrub(`${r.stdout}${r.stderr}`.trim(), secrets), 6000)
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+
+/** Start one service and wait until it is ready; a service that does not get ready is removed, with its last logs. */
+export async function startService(ctx: Ctx, spec: ServiceSpec, opts: { kind?: "service" | "mock"; trustedMounts?: { host: string; container: string }[]; keep?: boolean } = {}): Promise<StartedService> {
+  const t0 = Date.now()
+  const key = projectKey(containersOf(ctx.cfg).root)
+  const name = containerName(key, spec.name)
+  const secrets = serviceSecrets(spec, ctx.cfg, ctx.env || process.env)
+  const mounts = [...spec.volumes.map((v) => ({ host: checkVolume(v, containersOf(ctx.cfg).root), container: v.container })), ...(opts.trustedMounts || [])]
+  const env = runtimeEnv(ctx, secrets)
+  const hide = secretValues(ctx, secrets)
+  await ensureNetwork(ctx, key)
+  // A container of the same name is this service's from an earlier run: replaced, never reused with old settings.
+  await ctx.exec(ctx.runtime.bin, ["rm", "--force", "--volumes", name], { env, timeoutMs: 30000 })
+  ctx.log?.(`Starting ${spec.name} (${spec.image})`)
+  const run = await ctx.exec(ctx.runtime.bin, runArgs(spec, { key, service: spec.name, kind: opts.kind }, mounts), { env, timeoutMs: 600000, ...(ctx.signal ? { signal: ctx.signal } : {}) })
+  const ports = spec.ports.map((p) => ({ container: p.container, address: `127.0.0.1:${p.host}` }))
+  const base = { service: spec.name, container: name, image: spec.image, ports }
+  if (run.code !== 0) return { ...base, ready: false, error: clip(scrub((run.stderr || run.stdout).trim(), hide), 1500) || `${ctx.runtime.name} run exited with ${run.code}`, ms: Date.now() - t0 }
+  if (!opts.keep) started.set(name, { runtime: ctx.runtime, exec: ctx.exec, env: runtimeEnv(ctx) })
+  const ready = spec.ready
+  const hostPort = (c: number) => spec.ports.find((p) => p.container === c)!.host
+  const url = ready && "http" in ready ? `http://127.0.0.1:${hostPort(ready.port)}` : undefined
+  const deadline = Date.now() + spec.timeout * 1000
+  for (;;) {
+    if (ctx.signal?.aborted) break
+    const ok = !ready ? await running(ctx, name) : "tcp" in ready ? await tcpOpen(hostPort(ready.tcp)) : await httpReady(hostPort(ready.port), ready.http)
+    if (ok) return { ...base, ready: true, ...(url ? { url } : {}), ms: Date.now() - t0 }
+    if (!(await running(ctx, name))) {
+      const logs = await logsOf(ctx, name, 60, hide)
+      await removeContainers(ctx, [name])
+      return { ...base, ready: false, error: `${spec.name} stopped while starting.`, logs, ms: Date.now() - t0 }
+    }
+    if (Date.now() > deadline) break
+    await sleep(500)
+  }
+  const logs = await logsOf(ctx, name, 60, hide)
+  await removeContainers(ctx, [name])
+  return { ...base, ready: false, error: ctx.signal?.aborted ? "Stopped." : `${spec.name} was not ready after ${spec.timeout} s (${ready ? ("tcp" in ready ? `port ${ready.tcp}` : `GET ${ready.http}`) : "running"}).`, logs, ms: Date.now() - t0 }
+}
+
+/** Create the project's network when it is not there yet. */
+export async function ensureNetwork(ctx: Ctx, key: string): Promise<void> {
+  const net = projectNetwork(key)
+  const env = runtimeEnv(ctx)
+  const have = await ctx.exec(ctx.runtime.bin, ["network", "inspect", net], { env, timeoutMs: 15000 })
+  if (have.code === 0) return
+  const made = await ctx.exec(ctx.runtime.bin, ["network", "create", "--label", "forgeqe=1", "--label", `forgeqe.project=${key}`, net], { env, timeoutMs: 30000 })
+  if (made.code !== 0 && !/already exists/i.test(made.stderr)) throw new Error(`Could not create the network ${net}: ${clip(made.stderr.trim(), 300)}`)
+}
+
+async function removeContainers(ctx: Ctx, names: string[]): Promise<void> {
+  if (!names.length) return
+  await ctx.exec(ctx.runtime.bin, ["rm", "--force", "--volumes", ...names], { env: runtimeEnv(ctx), timeoutMs: 60000 })
+  for (const n of names) started.delete(n)
+}
+
+export type ContainerRow = { name: string; service: string; kind: string; state: string; status: string; image: string; ports: string }
+
+/** This project's containers (only those ForgeQE labelled). */
+export async function listContainers(ctx: Ctx): Promise<ContainerRow[]> {
+  const key = projectKey(containersOf(ctx.cfg).root)
+  const r = await ctx.exec(
+    ctx.runtime.bin,
+    ["ps", "--all", "--filter", `label=forgeqe.project=${key}`, "--format", "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}\t{{.Ports}}"],
+    { env: runtimeEnv(ctx), timeoutMs: 30000 },
+  )
+  if (r.code !== 0) throw new Error(`${ctx.runtime.name} ps failed: ${clip((r.stderr || r.stdout).trim(), 400)}`)
+  return r.stdout
+    .split("\n")
+    .filter((l) => l.trim())
+    .map((l) => {
+      const [name = "", state = "", status = "", image = "", ports = ""] = l.split("\t")
+      // The name is forgeqe-<project>-<service> (both docker and podman print it the same way, unlike labels).
+      const service = name.replace(/^\//, "").replace(`forgeqe-${key}-`, "")
+      return { name, state, status, image, ports, service, kind: service.startsWith("mock-") ? "mock" : "service" }
+    })
+}
+
+/** Start the services (and what they depend on) that are not running yet; stops at the first that does not start. */
+export async function ensureServices(ctx: Ctx, specs: ServiceSpec[]): Promise<{ started: StartedService[]; running: string[]; failed?: StartedService }> {
+  const up = new Set((await listContainers(ctx)).filter((r) => r.state === "running").map((r) => r.service))
+  const started: StartedService[] = []
+  const running: string[] = []
+  for (const s of specs) {
+    if (up.has(s.name)) {
+      running.push(s.name)
+      continue
+    }
+    const r = await startService(ctx, s)
+    started.push(r)
+    if (!r.ready) return { started, running, failed: r }
+  }
+  return { started, running }
+}
+
+/** Stop and remove this project's containers: the named services, or all of them. */
+export async function stopServices(ctx: Ctx, services?: string[]): Promise<string[]> {
+  const rows = await listContainers(ctx)
+  const pick = services?.length ? rows.filter((r) => services.includes(r.service)) : rows
+  await removeContainers(ctx, pick.map((r) => r.name))
+  // With everything stopped, the project's network goes too (it is created again on the next start).
+  if (!services?.length) await ctx.exec(ctx.runtime.bin, ["network", "rm", projectNetwork(projectKey(containersOf(ctx.cfg).root))], { env: runtimeEnv(ctx), timeoutMs: 30000 })
+  return pick.map((r) => r.service)
+}
+
+export async function serviceLogs(ctx: Ctx, service: string, lines = 100): Promise<string> {
+  const rows = await listContainers(ctx)
+  const row = rows.find((r) => r.service === service)
+  if (!row) throw new Error(`${service} is not running (forgeqe services status lists what is).`)
+  return logsOf(ctx, row.name, Math.min(Math.max(1, lines), 1000), secretValues(ctx))
+}
+
+/** Containers this process started and has not stopped. */
+export const startedContainers = () => [...started.keys()]
+
+/** Stop what this process started: the session's services end with it. */
+export async function stopStartedContainers(): Promise<string[]> {
+  const names = [...started.keys()]
+  for (const [name, s] of started) await s.exec(s.runtime.bin, ["rm", "--force", "--volumes", name], { env: s.env, timeoutMs: 60000 }).catch(() => undefined)
+  started.clear()
+  return names
+}
+
+/** A port on 127.0.0.1 that nothing listens on now. */
+export function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const s = net.createServer()
+    s.once("error", reject)
+    s.listen(0, "127.0.0.1", () => {
+      const a = s.address()
+      const p = typeof a === "object" && a ? a.port : 0
+      s.close(() => resolve(p))
+    })
+  })
+}

package/src/containers/tests.ts

@@ -0,0 +1,80 @@
+import { randomBytes } from "node:crypto"
+import path from "node:path"
+
+import type { runTestCommand, TestRun } from "../code/testrun.ts"
+import { promptOutput } from "../code/testrun.ts"
+import { redactSecrets, sessionLoginEnv } from "../sessionLogins.ts"
+import { containersOf, type TestsContainer } from "./config.ts"
+import { ensureNetwork, projectKey, projectNetwork, scrub, serviceSecrets, type Ctx } from "./run.ts"
+import { allowListEnv, privateEnvNames } from "../childEnv.ts"
+
+/**
+ * The repository's tests in a container (docs/CONTAINERS.md): run_tests runs the confirmed command there instead of on
+ * this machine. The repository is mounted at its own path (so result files land where ForgeQE reads them), the
+ * container joins the project's network (the services by name: db:5432), runs as this machine's user, and gets only the
+ * environment's FORGEQE_* variables, the saved logins' variables, and containers.tests' env and secrets, by name.
+ */
+
+/** Variables ForgeQE sets for test runs, and the environment's FORGEQE_* (src/environments.ts). */
+const EXPORTED = /^FORGEQE_(ACTIVE_ENV$|APP_|VAR_)/
+
+export function testContainerArgs(t: TestsContainer, o: { key: string; name: string; repo: string; cwd: string; mounts: string[]; command: string; byName: string[]; uid?: string }): string[] {
+  return [
+    "run",
+    "--rm",
+    "--name", o.name,
+    "--label", "forgeqe=1",
+    "--label", `forgeqe.project=${o.key}`,
+    "--label", "forgeqe.kind=tests",
+    "--security-opt", "no-new-privileges",
+    "--pids-limit", "4096",
+    "--network", projectNetwork(o.key),
+    ...(o.uid ? ["--user", o.uid] : []),
+    ...[o.repo, ...o.mounts].flatMap((m) => ["--volume", `${m}:${m}`]),
+    "--workdir", o.cwd,
+    ...Object.entries({ CI: "1", NO_COLOR: "1", FORCE_COLOR: "0", PLAYWRIGHT_HTML_OPEN: "never", HOME: "/tmp", ...t.env }).flatMap(([k, v]) => ["--env", `${k}=${v}`]),
+    // By name: the values come from the runtime's own environment, never the command line.
+    ...o.byName.flatMap((n) => ["--env", n]),
+    t.image,
+    // The command the user confirmed, run by the container's shell, inside the container.
+    "sh", "-c", o.command,
+  ]
+}
+
+/** A runTestCommand that runs the command in containers.tests' image. `writable`: folders outside the repo the results go to. */
+export function containerTestRunner(ctx: Ctx, repo: string, writable: string[] = []): typeof runTestCommand {
+  const t = containersOf(ctx.cfg).tests
+  if (!t) throw new Error("containers.tests is not set in forgeqe.yaml: it names the image the tests run in.")
+  if (process.platform === "win32") throw new Error("Tests in a container need a Linux or macOS machine (the repository is mounted at its own path).")
+  return async (opts): Promise<TestRun> => {
+    const started = Date.now()
+    const key = projectKey(containersOf(ctx.cfg).root)
+    await ensureNetwork(ctx, key)
+    const source = ctx.env || process.env
+    const own = new Set([...privateEnvNames(ctx.cfg), ...(opts.hide || [])])
+    const secretValues = serviceSecrets({ name: "containers.tests", secrets: t.secrets }, ctx.cfg, source)
+    const logins = sessionLoginEnv()
+    const extra = opts.env || {}
+    const exported = Object.fromEntries(Object.entries(source).filter(([k, v]) => EXPORTED.test(k) && v !== undefined) as [string, string][])
+    const pass: Record<string, string> = { ...exported, ...logins, ...extra, ...secretValues }
+    for (const k of Object.keys(pass)) if (own.has(k)) delete pass[k]
+    const name = `forgeqe-${key}-tests-${randomBytes(3).toString("hex")}`
+    const uid = typeof process.getuid === "function" ? `${process.getuid()}:${process.getgid?.() ?? process.getuid()}` : undefined
+    const mounts = [...new Set(writable.map((w) => path.resolve(w)))].filter((w) => path.relative(repo, w).startsWith(".."))
+    const args = testContainerArgs(t, { key, name, repo, cwd: opts.cwd, mounts, command: opts.command, byName: Object.keys(pass), uid })
+    const env = { ...allowListEnv(source, ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY", "CONTAINER_HOST", "CONTAINER_CONNECTION"]), ...pass }
+    const timeoutMs = opts.timeoutSeconds * 1000
+    let r
+    try {
+      r = await ctx.exec(ctx.runtime.bin, args, { env, timeoutMs, ...(opts.signal ? { signal: opts.signal } : {}) })
+    } finally {
+      // Stopping the runtime's client leaves the container running: remove it by name.
+      await ctx.exec(ctx.runtime.bin, ["rm", "--force", "--volumes", name], { env, timeoutMs: 30000 }).catch(() => undefined)
+    }
+    const timedOut = Date.now() - started >= timeoutMs
+    const hide = [...Object.values(secretValues), ...Object.values(logins), ...[...own].map((n) => source[n] || "")]
+    const full = scrub(redactSecrets([r.stdout, r.stderr].filter((s) => s.trim()).join("\n").replace(/\x1b\[[0-9;]*m/g, "")), hide)
+    const ok = !timedOut && r.code === 0
+    return { ok, exitCode: timedOut ? null : r.code, ...promptOutput(full, ok ? undefined : opts.save), ms: Date.now() - started, browsers: [] }
+  }
+}

package/src/context.ts

@@ -1,5 +1,7 @@
-import { AGENTS, type AgentEntry } from "./agents.ts"
+import { AGENTS, CAPABILITY_TOOL_BUDGET, type AgentEntry } from "./agents.ts"
+import { CAPABILITIES } from "./capabilities/registry.ts"
 import { createRouter, tokens } from "./mcp/routing.ts"
+import { capabilityIndex, CORE, MAX_PICKED } from "./mcp/scope.ts"
 import { TOOLS } from "./mcp/tools.ts"
 import { prompt } from "./prompts.ts"
 
@@ -14,6 +16,12 @@ export type AgentContext = {
   over: boolean
   /** ForgeQE's agent only: what it gets on demand, and what the fixed context would be with everything always on. */
   onDemand?: { playbooks: number; playbookTokens: number; everything: number }
+  /**
+   * ForgeQE's agent only: each capability's tool definitions, which a turn carries when it picks the capability
+   * (src/mcp/scope.ts), and a turn with the largest ones it picks from words alone.
+   */
+  capabilities?: { id: string; tokens: number; over: boolean }[]
+  largestTurn?: number
 }
 
 const expanded = (name: string) => {
@@ -32,20 +40,32 @@ export function agentContexts(): AgentContext[] {
       return { agent, prompt: p, total: p, over: p > agent.budget }
     }
     const router = createRouter(TOOLS)
-    const p = tokens(router.parts.core)
-    const t = tokens(JSON.stringify(TOOLS.map((x) => ({ name: x.name, description: x.description, inputSchema: x.inputSchema }))))
-    const full = tokens(expanded("brain")) + t
+    const toolTokens = (names: string[]) => {
+      const list = TOOLS.filter((x) => names.includes(x.name)).map((x) => ({ name: x.name, description: x.description, inputSchema: x.inputSchema }))
+      return list.length ? tokens(JSON.stringify(list)) : 0
+    }
+    // Every turn carries the core prompt, the capability index and the core capabilities' tools.
+    const p = tokens(router.parts.core) + tokens(capabilityIndex())
+    const t = toolTokens(CAPABILITIES.filter((c) => CORE.includes(c.id)).flatMap((c) => c.tools))
+    const capabilities = CAPABILITIES.filter((c) => c.tools.length && !CORE.includes(c.id)).map((c) => {
+      const n = toolTokens(c.tools)
+      return { id: c.id, tokens: n, over: n > CAPABILITY_TOOL_BUDGET }
+    })
+    const largest = [...capabilities].sort((a, b) => b.tokens - a.tokens).slice(0, MAX_PICKED)
+    const full = tokens(expanded("brain")) + toolTokens(TOOLS.map((x) => x.name))
     return {
       agent,
       prompt: p,
       tools: t,
       total: p + t,
-      over: p + t > agent.budget,
+      over: p + t > agent.budget || capabilities.some((c) => c.over),
       onDemand: {
         playbooks: router.parts.playbooks.length,
         playbookTokens: router.parts.playbooks.reduce((s, b) => s + tokens(b.text), 0),
         everything: full,
       },
+      capabilities,
+      largestTurn: p + t + largest.reduce((s, c) => s + c.tokens, 0),
     }
   })
 }
@@ -68,8 +88,15 @@ export function contextReport(rows = agentContexts()): string[] {
       `ForgeQE's agent loads on demand: ${d.playbooks} playbooks (${k(d.playbookTokens)}), each with the first result of its tools or through the playbook tool.`,
       `With everything always loaded it would carry ${k(d.everything)} on every call; it carries ${k(f.total)}.`,
     )
+    if (f.capabilities) {
+      out.push(
+        "",
+        `A turn adds the tools of the capabilities its request is about (at most ${k(CAPABILITY_TOOL_BUDGET)} each); with the ${MAX_PICKED} largest it carries ${k(f.largestTurn || 0)}:`,
+        ...[...f.capabilities].sort((a, b) => b.tokens - a.tokens).map((c) => `  ${c.over ? "✗" : " "} ${c.id.padEnd(28)} ${k(c.tokens).padStart(6)}`),
+      )
+    }
   }
   const over = rows.filter((r) => r.over)
-  if (over.length) out.push("", `Over budget: ${over.map((r) => r.agent.name).join(", ")}. Shorten what they load, or raise the budget in src/agents.ts with the reason.`)
+  if (over.length) out.push("", `Over budget: ${over.map((r) => r.agent.name).join(", ")}. Shorten what they load (a capability's tool descriptions, or split the capability), or raise the budget in src/agents.ts with the reason.`)
   return out
 }