@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/engine/setup.ts

@@ -0,0 +1,298 @@
+import { spawn, spawnSync } from "node:child_process"
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import { allowListEnv } from "../childEnv.ts"
+import { EXE, fetchBytes, installTool, isMusl, toolPath, toolsDir, forgeqeHome, WINDOWS, type Log } from "../tools.ts"
+
+export const BROWSER_USE_VERSION = "0.13.10"
+// azure-identity: keyless sign-in to the model with Microsoft Entra ID (auth: entra in llms.json).
+const IDENTITY_PACKAGE = "azure-identity>=1.19"
+const PACKAGES = [`browser-use==${BROWSER_USE_VERSION}`, "pyyaml>=6", IDENTITY_PACKAGE]
+
+export const PACKAGE_ROOT = path.resolve(import.meta.dir, "..", "..")
+export const ENGINE_SOURCE = path.join(PACKAGE_ROOT, "py")
+
+export type { Log }
+
+export function engineHome(): string {
+  return forgeqeHome()
+}
+
+function venvDir(): string {
+  return path.join(engineHome(), "engine", ".venv")
+}
+
+function venvPython(): string {
+  return process.platform === "win32"
+    ? path.join(venvDir(), "Scripts", "python.exe")
+    : path.join(venvDir(), "bin", "python")
+}
+
+function markerFile(): string {
+  return path.join(engineHome(), "engine", "installed.json")
+}
+
+function installedVersion(): string {
+  try {
+    return String(JSON.parse(readFileSync(markerFile(), "utf8")).browser_use || "")
+  } catch {
+    return ""
+  }
+}
+
+/** FORGEQE_PYTHON without the quotes Windows users often keep around paths such as C:\\Program Files\\... */
+function pythonOverride(): string | null {
+  const raw = (process.env.FORGEQE_PYTHON || "").trim().replace(/^"(.*)"$/, "$1").replace(/^'(.*)'$/, "$1")
+  return raw || null
+}
+
+const skippedOverride = new Set<string>()
+let identityTried = false
+
+function managedPython(): string | null {
+  return existsSync(venvPython()) && installedVersion() === BROWSER_USE_VERSION ? venvPython() : null
+}
+
+/** Python that runs the engine: FORGEQE_PYTHON, else the managed venv once the engine library is installed there. */
+export function enginePython(): string | null {
+  return pythonOverride() || managedPython()
+}
+
+/**
+ * The engine's environment: system settings, engine tuning, and the variables in `allow` (the model key and saved-login
+ * variables a run reads). Other secrets in this terminal, such as test case system tokens, never reach the browser agent.
+ */
+export function engineEnv(extra?: Record<string, string>, allow: string[] = []): Record<string, string> {
+  const env = allowListEnv(process.env, allow)
+  env.PYTHONPATH = [ENGINE_SOURCE, env.PYTHONPATH].filter(Boolean).join(path.delimiter)
+  env.PYTHONUNBUFFERED = "1"
+  return { ...env, ...extra }
+}
+
+export const UV_VERSION = "0.11.16"
+
+function managedUv(): string {
+  return toolPath("uv", UV_VERSION, `uv${EXE}`)
+}
+
+/** A uv ForgeQE downloaded keeps its Pythons under ~/.forgeqe too, so deleting that folder removes everything. */
+function uvEnv(uv: string): Record<string, string> {
+  return uv === managedUv() ? { UV_PYTHON_INSTALL_DIR: path.join(toolsDir(), "python") } : {}
+}
+
+function findUv(): string | null {
+  const home = os.homedir()
+  for (const c of ["uv", path.join(home, ".local", "bin", `uv${EXE}`), path.join(home, ".cargo", "bin", `uv${EXE}`), managedUv()]) {
+    if (c !== "uv" && !existsSync(c)) continue
+    const r = spawnSync(c, ["--version"], { encoding: "utf8" })
+    if (!r.error && r.status === 0) return c
+  }
+  return null
+}
+
+/** uv release asset for this machine, or null when uv has no build for it. */
+export function uvAsset(platform: string = process.platform, arch: string = process.arch, musl = platform === "linux" && isMusl()): string | null {
+  const cpu = ({ x64: "x86_64", arm64: "aarch64" } as Record<string, string>)[arch]
+  if (!cpu) return null
+  if (platform === "darwin") return `uv-${cpu}-apple-darwin.tar.gz`
+  if (platform === "linux") return `uv-${cpu}-unknown-linux-${musl ? "musl" : "gnu"}.tar.gz`
+  if (platform === "win32") return `uv-${cpu}-pc-windows-msvc.zip`
+  return null
+}
+
+/** System uv, else a uv ForgeQE downloaded earlier, else download it now. Null only when the download fails. */
+async function ensureUv(log: Log): Promise<string | null> {
+  const found = findUv()
+  if (found) return found
+  try {
+    const asset = uvAsset()
+    if (!asset) throw new Error(`no uv build for ${process.platform}/${process.arch}`)
+    const url = `https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${asset}`
+    const sha256 = (await fetchBytes(`${url}.sha256`)).toString("utf8").trim().split(/\s+/)[0]?.toLowerCase() || null
+    return await installTool({ name: "uv", version: UV_VERSION, asset, url, binary: `uv${EXE}`, sha256, log })
+  } catch (err) {
+    log(`Could not download uv (${err instanceof Error ? err.message : String(err)}); falling back to a system Python 3.11+.`)
+    return null
+  }
+}
+
+function pythonVersion(cmd: string): [number, number] | null {
+  const r = spawnSync(cmd, ["-c", "import sys; print(sys.version_info[0], sys.version_info[1])"], { encoding: "utf8" })
+  if (r.error || r.status !== 0) return null
+  const [major, minor] = r.stdout.trim().split(/\s+/).map(Number)
+  return Number.isFinite(major) && Number.isFinite(minor) ? [major, minor] : null
+}
+
+/** The Windows `py` launcher picks the newest installed Python; resolve it to that python.exe. */
+function pyLauncherPython(): string | null {
+  const r = spawnSync("py", ["-3", "-c", "import sys; print(sys.executable)"], { encoding: "utf8" })
+  if (r.error || r.status !== 0) return null
+  const exe = r.stdout.trim()
+  return exe && existsSync(exe) ? exe : null
+}
+
+function findSystemPython(): string | null {
+  const launcher = WINDOWS ? pyLauncherPython() : null
+  for (const c of [...(launcher ? [launcher] : []), "python3.13", "python3.12", "python3.11", "python3", "python"]) {
+    const v = pythonVersion(c)
+    if (v && (v[0] > 3 || (v[0] === 3 && v[1] >= 11))) return c
+  }
+  return null
+}
+
+function run(cmd: string, args: string[], log: Log, env: Record<string, string> = {}, shown?: string): Promise<void> {
+  log(`$ ${shown || [path.basename(cmd), ...args].join(" ")}`)
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, {
+      env: engineEnv({ NO_COLOR: "1", UV_NO_PROGRESS: "1", ...env }),
+      stdio: ["ignore", "pipe", "pipe"],
+    })
+    const tail: string[] = []
+    const onData = (buf: Buffer) => {
+      tail.push(buf.toString("utf8"))
+      if (tail.length > 40) tail.shift()
+    }
+    child.stdout.on("data", onData)
+    child.stderr.on("data", onData)
+    child.on("error", reject)
+    child.on("close", (code) => {
+      if (code === 0) resolve()
+      else {
+        const lines = tail.join("").split(/\r?\n/).filter((l) => l.trim())
+        reject(new Error(`${path.basename(cmd)} exited with ${code}\n${lines.slice(-15).join("\n")}`))
+      }
+    })
+  })
+}
+
+export type Doctor = {
+  python?: string
+  browser_use?: string
+  browser_use_error?: string
+  /** Installed azure-identity version; missing in environments set up before auth: entra existed. */
+  azure_identity?: string
+  channel?: string
+  executable_path?: string
+  /** Why the engine cannot run with this Python; set whenever browser_use is missing. */
+  error?: string
+}
+
+function doctorProblem(stderr: string): string {
+  const lines = stderr.split(/\r?\n/).map((l) => l.trim()).filter(Boolean)
+  const missing = lines.find((l) => /No module named '?(browser_use|yaml|forgeqe_engine)/.test(l))
+  if (missing?.includes("browser_use")) return `the engine's browser library is not installed in this Python (${missing})`
+  if (missing?.includes("yaml")) return `pyyaml is not installed in this Python (${missing})`
+  return lines.slice(-3).join(" | ") || "the engine check printed nothing"
+}
+
+export function engineDoctor(python: string): Doctor {
+  const r = spawnSync(python, ["-m", "forgeqe_engine", "doctor"], { encoding: "utf8", env: engineEnv() })
+  if (r.error) {
+    const code = (r.error as NodeJS.ErrnoException).code
+    return { error: code === "ENOENT" ? `${python} does not exist or is not a program` : r.error.message }
+  }
+  const line = (r.stdout || "").trim().split(/\r?\n/).pop() || ""
+  let doc: Doctor & { engine_error?: string } = {}
+  try {
+    doc = JSON.parse(line) as Doctor
+  } catch {}
+  const [major, minor] = String(doc.python || "").split(".").map(Number)
+  if (major === 3 && minor < 11) return { ...doc, browser_use: undefined, error: `Python ${doc.python} is too old; the engine needs 3.11+` }
+  if (doc.browser_use && !doc.engine_error) return doc
+  const error = doc.engine_error
+    ? `the engine failed to load (${doc.engine_error})`
+    : doc.browser_use_error
+      ? `the engine's browser library is not installed in this Python (${doc.browser_use_error})`
+      : doctorProblem(r.stderr || "")
+  return { ...doc, browser_use: undefined, error }
+}
+
+async function ensureBrowser(python: string, uv: string | null, log: Log): Promise<void> {
+  const doc = engineDoctor(python)
+  if (doc?.executable_path) {
+    log(`Browser: ${doc.executable_path}`)
+    return
+  }
+  log("Google Chrome not found; installing Chromium for the engine.")
+  if (uv) {
+    await run(uv, ["tool", "run", "playwright", "install", "chromium", "--no-shell"], log, uvEnv(uv))
+    return
+  }
+  await run(python, ["-m", "pip", "install", "-q", "playwright"], log)
+  await run(python, ["-m", "playwright", "install", "chromium", "--no-shell"], log)
+}
+
+/**
+ * Create ~/.forgeqe/engine/.venv with the engine library. Uses uv (system, or downloaded into ~/.forgeqe/tools),
+ * which fetches Python 3.11+ itself; a system Python 3.11+ is the fallback when uv cannot be downloaded.
+ * A FORGEQE_PYTHON that cannot run the engine is skipped, and a broken environment is rebuilt, without asking.
+ * Safe to call repeatedly; it only installs when the pinned version is missing.
+ */
+export async function ensureEngine(log: Log, opts?: { force?: boolean }): Promise<string> {
+  const override = pythonOverride()
+  if (override) {
+    const doc = engineDoctor(override)
+    if (doc.browser_use) {
+      if (doc.browser_use !== BROWSER_USE_VERSION) log(`Note: ${override} has engine library ${doc.browser_use}; ForgeQE is tested with ${BROWSER_USE_VERSION}.`)
+      return override
+    }
+    if (!skippedOverride.has(override)) {
+      skippedOverride.add(override)
+      log(`FORGEQE_PYTHON (${override}) cannot run the engine: ${doc.error}. Using ForgeQE's own environment instead.`)
+    }
+  }
+  let rebuild = Boolean(opts?.force)
+  const existing = rebuild ? null : managedPython()
+  if (existing) {
+    const doc = engineDoctor(existing)
+    if (doc.browser_use && !doc.azure_identity && !identityTried) {
+      // Set up before auth: entra existed: add the one package rather than rebuild the environment (once per process).
+      identityTried = true
+      log("Adding azure-identity to the engine environment (for auth: entra).")
+      const uv = findUv()
+      try {
+        if (uv) await run(uv, ["pip", "install", "--python", existing, IDENTITY_PACKAGE], log, uvEnv(uv))
+        else await run(existing, ["-m", "pip", "install", "-q", IDENTITY_PACKAGE], log)
+      } catch (err) {
+        log(`Could not add azure-identity (${err instanceof Error ? err.message.split("\n")[0] : String(err)}); auth: entra profiles will not work until forgeqe setup succeeds.`)
+      }
+      return existing
+    }
+    if (doc.browser_use) return existing
+    log(`The engine environment is broken (${doc.error}); rebuilding it.`)
+    rebuild = true
+  }
+
+  const venv = venvDir()
+  mkdirSync(path.dirname(venv), { recursive: true })
+  if (rebuild && existsSync(venv)) rmSync(venv, { recursive: true, force: true })
+  rmSync(markerFile(), { force: true })
+  log(`Setting up the ForgeQE engine in ${venv} (one time).`)
+  const uv = await ensureUv(log)
+
+  if (uv) {
+    if (!existsSync(venvPython())) await run(uv, ["venv", "--python", ">=3.11", venv], log, uvEnv(uv))
+    await run(uv, ["pip", "install", "--python", venvPython(), ...PACKAGES], log, uvEnv(uv), "uv pip install <engine packages>")
+  } else {
+    const py = findSystemPython()
+    if (!py) {
+      const uvInstall = WINDOWS
+        ? `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`
+        : "curl -LsSf https://astral.sh/uv/install.sh | sh"
+      throw new Error(
+        `Could not download uv and found no Python 3.11+. Check the network or proxy, or install uv (${uvInstall}) ` +
+          "or Python 3.11+ yourself, then run: forgeqe setup.",
+      )
+    }
+    if (!existsSync(venvPython())) await run(py, ["-m", "venv", venv], log)
+    await run(venvPython(), ["-m", "pip", "install", "-q", "--upgrade", "pip"], log)
+    await run(venvPython(), ["-m", "pip", "install", "-q", ...PACKAGES], log, {}, "pip install <engine packages>")
+  }
+
+  await ensureBrowser(venvPython(), uv, log)
+  writeFileSync(markerFile(), JSON.stringify({ browser_use: BROWSER_USE_VERSION }, null, 2) + "\n")
+  log("Engine ready.")
+  return venvPython()
+}

package/src/entra.ts

@@ -0,0 +1,154 @@
+import { spawn } from "node:child_process"
+import { randomBytes, timingSafeEqual } from "node:crypto"
+
+import { ENTRA_PROVIDERS, type LlmProfile } from "./llm.ts"
+import { enginePython, engineEnv } from "./engine/setup.ts"
+
+/**
+ * Keyless model access with Microsoft Entra ID (`auth: entra` in llms.json). The engine signs in itself with
+ * azure-identity; the coding agent's runtime only takes a fixed key, so it talks to a local gateway that adds a current
+ * Entra ID token to every request. Neither ever holds a long-lived secret.
+ */
+
+/**
+ * The variables azure-identity reads to find a credential: managed identity (App Service, Functions, Container Apps,
+ * VMs, Arc), workload identity (AKS, federated CI), a service principal, and the Azure CLI's folder. The engine sees
+ * them only for an `auth: entra` profile.
+ */
+export const AZURE_IDENTITY_ENV = [
+  "AZURE_CLIENT_ID",
+  "AZURE_TENANT_ID",
+  "AZURE_CLIENT_SECRET",
+  "AZURE_CLIENT_CERTIFICATE_PATH",
+  "AZURE_CLIENT_CERTIFICATE_PASSWORD",
+  "AZURE_CLIENT_SEND_CERTIFICATE_CHAIN",
+  "AZURE_FEDERATED_TOKEN_FILE",
+  "AZURE_AUTHORITY_HOST",
+  "AZURE_REGIONAL_AUTHORITY_NAME",
+  "AZURE_ADDITIONALLY_ALLOWED_TENANTS",
+  "AZURE_TOKEN_CREDENTIALS",
+  "AZURE_POD_IDENTITY_AUTHORITY_HOST",
+  "AZURE_CONFIG_DIR",
+  "IDENTITY_ENDPOINT",
+  "IDENTITY_HEADER",
+  "IDENTITY_SERVER_THUMBPRINT",
+  "IMDS_ENDPOINT",
+  "MSI_ENDPOINT",
+  "MSI_SECRET",
+]
+
+/** The profile fields the engine's credential reads, in llms.json's spelling. */
+export function entraPayload(p: LlmProfile): Record<string, string> {
+  return {
+    auth: "entra",
+    ...(p.managedIdentityClientId ? { managed_identity_client_id: p.managedIdentityClientId } : {}),
+    ...(p.tokenScope ? { token_scope: p.tokenScope } : {}),
+  }
+}
+
+export type Token = { token: string; expiresOn: number }
+
+/** One token from the engine's credential chain (`python -m forgeqe_engine token`). */
+export function engineToken(p: LlmProfile, python = enginePython()): Promise<Token> {
+  if (!python) return Promise.reject(new Error("auth: entra signs in through the ForgeQE engine, which is not set up yet. Run forgeqe setup."))
+  return new Promise((resolve, reject) => {
+    const child = spawn(python, ["-m", "forgeqe_engine", "token"], { env: engineEnv(undefined, AZURE_IDENTITY_ENV), stdio: ["pipe", "pipe", "pipe"] })
+    let out = ""
+    child.stdout.setEncoding("utf8")
+    child.stdout.on("data", (c: string) => (out += c))
+    child.stderr.resume()
+    child.on("error", reject)
+    child.on("close", () => {
+      let r: { token?: string; expires_on?: number; error?: string } = {}
+      try {
+        r = JSON.parse(out.trim().split(/\r?\n/).pop() || "{}")
+      } catch {}
+      if (r.token) resolve({ token: r.token, expiresOn: Number(r.expires_on || 0) * 1000 || Date.now() + 30 * 60_000 })
+      else reject(new Error(`Could not get a Microsoft Entra ID token for the model: ${r.error || "the engine printed none"}`))
+    })
+    child.stdin.end(JSON.stringify(entraPayload(p)) + "\n")
+  })
+}
+
+/** A token source that reuses a token until five minutes before it expires, and asks once however many callers wait. */
+export function cachedToken(fetchToken: () => Promise<Token>, now = Date.now): () => Promise<string> {
+  let current: Token | null = null
+  let pending: Promise<Token> | null = null
+  return async () => {
+    if (current && now() < current.expiresOn - 5 * 60_000) return current.token
+    pending ||= fetchToken().finally(() => (pending = null))
+    current = await pending
+    return current.token
+  }
+}
+
+/** The OpenAI-style base URL behind a profile: Azure's v1 API (which takes Entra ID tokens), or the gateway's URL. */
+export function upstreamURL(p: LlmProfile): string {
+  if (!p.endpoint) throw new Error(`The ${p.provider} profile needs an endpoint.`)
+  const base = p.endpoint.replace(/\/+$/, "")
+  return p.provider === "azure" ? `${base.replace(/\/openai(\/v1)?$/, "")}/openai/v1` : base
+}
+
+/** The variable the coding agent's runtime reads the gateway's key from. */
+export const GATEWAY_KEY_ENV = "FORGEQE_ENTRA_GATEWAY_KEY"
+
+export type EntraGateway = {
+  /** The profile the runtime uses instead: an OpenAI-compatible endpoint on this machine. */
+  profile: LlmProfile
+  /** Values for the runtime's {env:} references. */
+  secrets: Record<string, string>
+  close(): void
+}
+
+/**
+ * A gateway on 127.0.0.1 for the coding agent's runtime. It accepts only a one-time random key (never valid anywhere
+ * else), and forwards each request to the model with a current Entra ID token in place of that key.
+ */
+export function entraGateway(p: LlmProfile, token: () => Promise<string> = cachedToken(() => engineToken(p))): EntraGateway {
+  if (!ENTRA_PROVIDERS.includes(p.provider)) throw new Error(`auth: entra works with the azure and openai_compatible providers, not ${p.provider}.`)
+  const upstream = upstreamURL(p)
+  const key = randomBytes(24).toString("hex")
+  const expected = Buffer.from(`Bearer ${key}`)
+  const server = Bun.serve({
+    hostname: "127.0.0.1",
+    port: 0,
+    // Model responses stream for minutes; Bun's default idle timeout would cut them off.
+    idleTimeout: 0,
+    async fetch(req) {
+      const got = Buffer.from(req.headers.get("authorization") || "")
+      if (got.length !== expected.length || !timingSafeEqual(got, expected)) return Response.json({ error: { message: "Unauthorized" } }, { status: 401 })
+      let bearer: string
+      try {
+        bearer = await token()
+      } catch (err) {
+        return Response.json({ error: { message: err instanceof Error ? err.message : String(err) } }, { status: 401 })
+      }
+      const url = new URL(req.url)
+      const headers = new Headers(req.headers)
+      for (const h of ["host", "authorization", "api-key", "content-length", "connection"]) headers.delete(h)
+      headers.set("authorization", `Bearer ${bearer}`)
+      // Bun decompresses fetched bodies, so ask for none rather than pass on a content-encoding that no longer holds.
+      headers.set("accept-encoding", "identity")
+      const body = req.method === "GET" || req.method === "HEAD" ? undefined : await req.arrayBuffer()
+      const res = await fetch(`${upstream}${url.pathname}${url.search}`, { method: req.method, headers, body, redirect: "manual", signal: req.signal })
+      const out = new Headers(res.headers)
+      out.delete("content-encoding")
+      out.delete("content-length")
+      return new Response(res.body, { status: res.status, statusText: res.statusText, headers: out })
+    },
+  })
+  return {
+    profile: {
+      ...p,
+      provider: "openai_compatible",
+      // Azure's v1 API names the deployment as the model.
+      model: p.provider === "azure" ? p.deployment || p.model : p.model,
+      endpoint: `http://127.0.0.1:${server.port}`,
+      auth: "key",
+      apiKey: undefined,
+      apiKeyEnv: GATEWAY_KEY_ENV,
+    },
+    secrets: { [GATEWAY_KEY_ENV]: key },
+    close: () => server.stop(true),
+  }
+}

package/src/env.ts

@@ -0,0 +1,24 @@
+import { existsSync, readFileSync } from "node:fs"
+import path from "node:path"
+
+/** Load KEY=VALUE lines. Existing process env wins. */
+export function loadDotEnv(dir: string): void {
+  const file = path.join(dir, ".env")
+  if (!existsSync(file)) return
+  const text = readFileSync(file, "utf8")
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.trim()
+    if (!line || line.startsWith("#")) continue
+    const eq = line.indexOf("=")
+    if (eq <= 0) continue
+    const key = line.slice(0, eq).trim()
+    let value = line.slice(eq + 1).trim()
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1)
+    }
+    if (process.env[key] === undefined) process.env[key] = value
+  }
+}