firegoat 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +16 -2
  2. package/dist/cli.js +306 -44
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -2,15 +2,29 @@
2
2
 
3
3
  Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
4
4
 
5
- Status: early (v0.1). See `plan.md`.
5
+ Status: early (v0.2). See `plan.md`.
6
6
 
7
7
  ## Use
8
8
 
9
9
  ```bash
10
10
  npx firegoat init # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
11
- npx firegoat eval # runs each test with your local `claude` in a fresh empty folder
11
+ npx firegoat login # once per machine: signs claude in to ~/.firegoat/claude
12
+ npx firegoat eval # runs each test with your local `claude`, sandboxed, in a fresh empty folder
12
13
  ```
13
14
 
15
+ ### Isolation
16
+
17
+ On macOS, `eval` runs `claude` inside a Seatbelt (`sandbox-exec`) profile:
18
+
19
+ - it signs in with `~/.firegoat/claude` (from `firegoat login`), never your `~/.claude`;
20
+ - it cannot read your home dir or the repo, except toolchains (node, bun, cargo, ...) it needs to run;
21
+ - it can write only in its run dir (`work/`, its own `TMPDIR` and package caches) and system temp;
22
+ - `ANTHROPIC_API_KEY` is removed, so runs bill your subscription, not the API.
23
+
24
+ A probe checks the sandbox before every run; if it does not hold, the run is not graded. Network is open.
25
+
26
+ Off macOS there is no sandbox yet: run inside a throwaway container and pass `--host`.
27
+
14
28
  `eval` saves each run's transcript to `.firegoat/runs/<runId>/<test>/<n>/session.jsonl` and a `summary.json`, then runs the test's `checks` in the agent's folder. Nothing is uploaded. `firegoat.yaml` is looked up in the current directory, then parents up to the git root.
15
29
 
16
30
  Exit codes:
package/dist/cli.js CHANGED
@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
6947
6947
  // package.json
6948
6948
  var package_default = {
6949
6949
  name: "firegoat",
6950
- version: "0.1.0",
6950
+ version: "0.2.0",
6951
6951
  repository: {
6952
6952
  type: "git",
6953
6953
  url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -7142,20 +7142,164 @@ import {
7142
7142
  accessSync,
7143
7143
  constants,
7144
7144
  createWriteStream as createWriteStream2,
7145
- existsSync as existsSync2,
7146
- mkdirSync,
7145
+ existsSync as existsSync3,
7146
+ mkdirSync as mkdirSync2,
7147
7147
  mkdtempSync,
7148
7148
  readFileSync as readFileSync2,
7149
+ realpathSync as realpathSync2,
7149
7150
  statSync,
7150
7151
  writeFileSync
7151
7152
  } from "node:fs";
7152
7153
  import { tmpdir } from "node:os";
7153
- import { delimiter, join as join2, resolve } from "node:path";
7154
+ import { delimiter, join as join3, resolve } from "node:path";
7154
7155
 
7155
7156
  // src/claude.ts
7156
- import { spawn } from "node:child_process";
7157
+ import { spawn, spawnSync as spawnSync2 } from "node:child_process";
7157
7158
  import { createWriteStream } from "node:fs";
7158
7159
  import { createInterface } from "node:readline";
7160
+
7161
+ // src/sandbox.ts
7162
+ import { spawnSync } from "node:child_process";
7163
+ import { existsSync as existsSync2, mkdirSync, realpathSync } from "node:fs";
7164
+ import { homedir } from "node:os";
7165
+ import { basename, dirname as dirname2, join as join2 } from "node:path";
7166
+ var SANDBOX_EXEC = "/usr/bin/sandbox-exec";
7167
+ function firegoatHome() {
7168
+ return process.env.FIREGOAT_HOME || join2(homedir(), ".firegoat");
7169
+ }
7170
+ function claudeConfigDir() {
7171
+ return join2(firegoatHome(), "claude");
7172
+ }
7173
+ var TOOLCHAINS = [
7174
+ ".nvm",
7175
+ ".fnm",
7176
+ ".volta",
7177
+ ".bun",
7178
+ ".deno",
7179
+ ".cargo",
7180
+ ".rustup",
7181
+ ".pyenv",
7182
+ ".rbenv",
7183
+ ".asdf",
7184
+ ".sdkman",
7185
+ ".local/share/mise",
7186
+ ".local/share/fnm",
7187
+ ".local/share/claude",
7188
+ "go",
7189
+ "Library/pnpm"
7190
+ ];
7191
+ var TOOLCHAIN_SECRETS = [".cargo/credentials", ".cargo/credentials.toml"];
7192
+ function real(p) {
7193
+ return realpathSync(p);
7194
+ }
7195
+ function under(p, dir) {
7196
+ return p === dir || p.startsWith(`${dir}/`);
7197
+ }
7198
+ function readableToolDirs(home, claudeBin) {
7199
+ const dirs = new Set;
7200
+ const add = (p) => {
7201
+ if (existsSync2(p))
7202
+ dirs.add(real(p));
7203
+ };
7204
+ for (const t of TOOLCHAINS)
7205
+ add(join2(home, t));
7206
+ for (const p of (process.env.PATH ?? "").split(":")) {
7207
+ if (!p.startsWith("/") || !existsSync2(p))
7208
+ continue;
7209
+ const r = real(p);
7210
+ if (!under(r, home) || r === home)
7211
+ continue;
7212
+ const prefix = dirname2(r);
7213
+ dirs.add(basename(r) === "bin" && prefix !== home && prefix !== join2(home, ".local") ? prefix : r);
7214
+ }
7215
+ add(dirname2(dirname2(process.execPath)));
7216
+ add(dirname2(claudeBin));
7217
+ add(dirname2(real(claudeBin)));
7218
+ return [...dirs];
7219
+ }
7220
+ var q = (s) => JSON.stringify(s);
7221
+ function sandboxProfile(opts) {
7222
+ const home = real(homedir());
7223
+ const reads = readableToolDirs(home, opts.claudeBin).map((d) => `(subpath ${q(d)})`);
7224
+ return `(version 1)
7225
+ (allow default)
7226
+
7227
+ (deny file-read* (subpath ${q(home)}))
7228
+ (allow file-read-metadata)
7229
+ (allow file-read*
7230
+ ${reads.join(`
7231
+ `)}
7232
+ (subpath ${q(join2(home, "Library/Keychains"))})
7233
+ (regex ${q(`^${home}/Library/Preferences/com\\.apple\\.security`)})
7234
+ (regex ${q(`^${home}/Library/Preferences/\\.GlobalPreferences`)}))
7235
+ (deny file-read* file-write*
7236
+ ${TOOLCHAIN_SECRETS.map((s) => `(literal ${q(join2(home, s))})`).join(`
7237
+ `)})
7238
+
7239
+ (deny file-write* (subpath "/"))
7240
+ (allow file-write*
7241
+ (subpath "/dev")
7242
+ (subpath "/private/tmp")
7243
+ (subpath "/private/var/folders"))
7244
+ (allow file-read* file-write*
7245
+ (subpath ${q(real(opts.configDir))})
7246
+ (subpath ${q(real(opts.runRoot))}))
7247
+
7248
+ (deny file-read* file-write* (subpath ${q(real(opts.repo))}))
7249
+ `;
7250
+ }
7251
+ function agentEnv(runRoot) {
7252
+ const env = { ...process.env };
7253
+ delete env.ANTHROPIC_API_KEY;
7254
+ const cache = join2(runRoot, "cache");
7255
+ const tmp = join2(runRoot, "tmp");
7256
+ mkdirSync(cache, { recursive: true });
7257
+ mkdirSync(tmp, { recursive: true });
7258
+ return {
7259
+ ...env,
7260
+ CLAUDE_CONFIG_DIR: claudeConfigDir(),
7261
+ TMPDIR: tmp,
7262
+ XDG_CACHE_HOME: cache,
7263
+ npm_config_cache: join2(cache, "npm"),
7264
+ npm_config_store_dir: join2(cache, "pnpm-store"),
7265
+ YARN_CACHE_FOLDER: join2(cache, "yarn"),
7266
+ BUN_INSTALL_CACHE_DIR: join2(cache, "bun"),
7267
+ PIP_CACHE_DIR: join2(cache, "pip"),
7268
+ UV_CACHE_DIR: join2(cache, "uv"),
7269
+ GOMODCACHE: join2(cache, "gomod"),
7270
+ GOCACHE: join2(cache, "gobuild"),
7271
+ CARGO_HOME: join2(cache, "cargo")
7272
+ };
7273
+ }
7274
+ function wrap(profile, cmd, args) {
7275
+ if (profile === null)
7276
+ return { cmd, args };
7277
+ return { cmd: SANDBOX_EXEC, args: ["-p", profile, cmd, ...args] };
7278
+ }
7279
+ function probeSandbox(opts) {
7280
+ const outside = join2(homedir(), `.firegoat-escape-probe-${process.pid}-${Date.now()}`);
7281
+ const script = 'touch "$1" 2>/dev/null && echo home-write; cat "$2" >/dev/null 2>&1 && echo repo-read; touch "$3" 2>/dev/null && echo work-write; rm -f "$1" "$3"';
7282
+ const { cmd, args } = wrap(opts.profile, "/bin/sh", [
7283
+ "-c",
7284
+ script,
7285
+ "probe",
7286
+ outside,
7287
+ opts.repoFile,
7288
+ join2(opts.workdir, ".firegoat-probe")
7289
+ ]);
7290
+ const r = spawnSync(cmd, args, { encoding: "utf8", cwd: opts.workdir });
7291
+ if (r.error)
7292
+ return `sandbox probe could not start: ${r.error.message}`;
7293
+ const got = r.stdout.trim().split(`
7294
+ `).filter(Boolean);
7295
+ if (got.includes("home-write") || got.includes("repo-read"))
7296
+ return `sandbox does not hold (escaped: ${got.filter((g) => g !== "work-write").join(", ")}); refusing to run the agent`;
7297
+ if (!got.includes("work-write"))
7298
+ return `sandbox blocks writes in the workdir (stderr: ${r.stderr.trim() || "none"})`;
7299
+ return;
7300
+ }
7301
+
7302
+ // src/claude.ts
7159
7303
  var CLAUDE_FLAGS = [
7160
7304
  "--output-format",
7161
7305
  "stream-json",
@@ -7169,11 +7313,6 @@ var CLAUDE_FLAGS = [
7169
7313
  function claudeArgv(prompt) {
7170
7314
  return ["-p", prompt, ...CLAUDE_FLAGS];
7171
7315
  }
7172
- function agentEnv() {
7173
- const env = { ...process.env };
7174
- delete env.ANTHROPIC_API_KEY;
7175
- return env;
7176
- }
7177
7316
  var live = new Set;
7178
7317
  function killGroup(child, signal) {
7179
7318
  if (child.pid === undefined)
@@ -7303,10 +7442,9 @@ async function runClaude(opts) {
7303
7442
  let badStart;
7304
7443
  let result;
7305
7444
  const spawned = await spawnWatched({
7306
- cmd: opts.bin,
7307
- args: claudeArgv(opts.prompt),
7445
+ ...wrap(opts.profile, opts.bin, claudeArgv(opts.prompt)),
7308
7446
  cwd: opts.cwd,
7309
- env: agentEnv(),
7447
+ env: opts.env,
7310
7448
  timeoutMs: opts.timeoutMs,
7311
7449
  onStderr: (c) => stderr.write(c),
7312
7450
  onLine: (line) => {
@@ -7405,6 +7543,26 @@ async function runCheck(opts) {
7405
7543
  out.notStarted = true;
7406
7544
  return out;
7407
7545
  }
7546
+ function checkLogin(opts) {
7547
+ const { cmd, args } = wrap(opts.profile, opts.bin, ["auth", "status"]);
7548
+ const r = spawnSync2(cmd, args, {
7549
+ cwd: opts.cwd,
7550
+ env: opts.env,
7551
+ encoding: "utf8",
7552
+ timeout: 30000
7553
+ });
7554
+ if (r.error)
7555
+ return `could not run claude auth status: ${r.error.message}`;
7556
+ let status;
7557
+ try {
7558
+ status = JSON.parse(r.stdout);
7559
+ } catch {}
7560
+ if (status?.loggedIn === true)
7561
+ return;
7562
+ if (status?.loggedIn === false)
7563
+ return `claude is not logged in to firegoat's config dir (${opts.env.CLAUDE_CONFIG_DIR}). Run: npx firegoat login`;
7564
+ return `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`;
7565
+ }
7408
7566
 
7409
7567
  // src/exit.ts
7410
7568
  var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
@@ -7418,6 +7576,7 @@ function parseArgs(args) {
7418
7576
  const opts = {
7419
7577
  names: [],
7420
7578
  json: false,
7579
+ host: false,
7421
7580
  claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
7422
7581
  };
7423
7582
  for (let i = 0;i < args.length; i++) {
@@ -7435,6 +7594,8 @@ function parseArgs(args) {
7435
7594
  };
7436
7595
  if (flag === "--json")
7437
7596
  opts.json = true;
7597
+ else if (flag === "--host")
7598
+ opts.host = true;
7438
7599
  else if (flag === "--runs") {
7439
7600
  const v = value();
7440
7601
  const n = Number(v);
@@ -7455,8 +7616,8 @@ function newRunId() {
7455
7616
  return `${ts}-${randomBytes(2).toString("hex")}`;
7456
7617
  }
7457
7618
  function gitignoreHasFiregoat(cwd) {
7458
- const file = join2(cwd, ".gitignore");
7459
- if (!existsSync2(file))
7619
+ const file = join3(cwd, ".gitignore");
7620
+ if (!existsSync3(file))
7460
7621
  return false;
7461
7622
  return readFileSync2(file, "utf8").split(`
7462
7623
  `).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
@@ -7473,17 +7634,35 @@ function resolveBin(bin) {
7473
7634
  if (bin.includes("/"))
7474
7635
  return executable(bin) ? resolve(bin) : undefined;
7475
7636
  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
7476
- if (dir && executable(join2(dir, bin)))
7477
- return join2(dir, bin);
7637
+ if (dir && executable(join3(dir, bin)))
7638
+ return join3(dir, bin);
7478
7639
  }
7479
7640
  return;
7480
7641
  }
7642
+ function newRunRoot(label) {
7643
+ const runRoot = realpathSync2(mkdtempSync(join3(tmpdir(), `firegoat-${label}-`)));
7644
+ const workdir = join3(runRoot, "work");
7645
+ mkdirSync2(workdir);
7646
+ return { runRoot, workdir };
7647
+ }
7648
+ function isolate(iso, claudeBin, runRoot, workdir) {
7649
+ if (iso.host)
7650
+ return { profile: null };
7651
+ const profile = sandboxProfile({
7652
+ claudeBin,
7653
+ runRoot,
7654
+ configDir: claudeConfigDir(),
7655
+ repo: iso.repo
7656
+ });
7657
+ const broken = probeSandbox({ profile, workdir, repoFile: iso.repoFile });
7658
+ return broken ? { error: broken } : { profile };
7659
+ }
7481
7660
  async function runOne(opts) {
7482
7661
  const { test, n } = opts;
7483
7662
  const started = Date.now();
7484
- const dir = join2(opts.runDir, test.name, String(n));
7485
- mkdirSync(dir, { recursive: true });
7486
- const session = join2(dir, "session.jsonl");
7663
+ const dir = join3(opts.runDir, test.name, String(n));
7664
+ mkdirSync2(dir, { recursive: true });
7665
+ const session = join3(dir, "session.jsonl");
7487
7666
  const summary = {
7488
7667
  test: test.name,
7489
7668
  n,
@@ -7502,14 +7681,19 @@ async function runOne(opts) {
7502
7681
  summary.duration_ms = Date.now() - started;
7503
7682
  return summary;
7504
7683
  };
7505
- const workdir = mkdtempSync(join2(tmpdir(), `firegoat-${test.name}-${n}-`));
7684
+ const { runRoot, workdir } = newRunRoot(`${test.name}-${n}`);
7506
7685
  summary.workdir = workdir;
7686
+ const sandbox = isolate(opts.iso, opts.claudeBin, runRoot, workdir);
7687
+ if ("error" in sandbox)
7688
+ return done("error", `not graded: ${sandbox.error}`);
7507
7689
  const outcome = await runClaude({
7690
+ env: agentEnv(runRoot),
7691
+ profile: sandbox.profile,
7508
7692
  bin: opts.claudeBin,
7509
7693
  prompt: test.prompt,
7510
7694
  cwd: workdir,
7511
7695
  sessionPath: session,
7512
- stderrPath: join2(dir, "stderr.txt"),
7696
+ stderrPath: join3(dir, "stderr.txt"),
7513
7697
  timeoutMs: test.timeout * 1000
7514
7698
  });
7515
7699
  if (outcome.result) {
@@ -7522,7 +7706,7 @@ async function runOne(opts) {
7522
7706
  opts.warn(`warning: ${test.name} has no checks, so its goal is unverified; counted as pass`);
7523
7707
  return done("pass", "no checks (goal unverified)");
7524
7708
  }
7525
- const log = createWriteStream2(join2(dir, "checks.txt"));
7709
+ const log = createWriteStream2(join3(dir, "checks.txt"));
7526
7710
  const env = {
7527
7711
  ...process.env,
7528
7712
  FIREGOAT_SESSION: resolve(session),
@@ -7582,19 +7766,49 @@ async function evalCommand(args, cwd) {
7582
7766
  console.error(`firegoat eval: not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
7583
7767
  return EXIT.notGraded;
7584
7768
  }
7585
- mkdirSync(join2(root, ".firegoat"), { recursive: true });
7769
+ if (opts.host)
7770
+ warn("warning: --host: the agent runs without isolation and can read and write your files");
7771
+ else if (process.platform !== "darwin" || !existsSync3(SANDBOX_EXEC)) {
7772
+ console.error(`firegoat eval: isolation needs macOS (${SANDBOX_EXEC}); this is ${process.platform}. In CI or Linux, run inside a throwaway container and pass --host.`);
7773
+ return EXIT.usage;
7774
+ }
7775
+ const iso = {
7776
+ host: opts.host,
7777
+ repo: root,
7778
+ repoFile: join3(root, "firegoat.yaml")
7779
+ };
7780
+ mkdirSync2(claudeConfigDir(), { recursive: true });
7781
+ const pre = newRunRoot("preflight");
7782
+ const preSandbox = isolate(iso, claudePath, pre.runRoot, pre.workdir);
7783
+ const loginProblem = "error" in preSandbox ? preSandbox.error : checkLogin({
7784
+ bin: claudePath,
7785
+ env: agentEnv(pre.runRoot),
7786
+ profile: preSandbox.profile,
7787
+ cwd: pre.workdir
7788
+ });
7789
+ if (loginProblem) {
7790
+ console.error(`firegoat eval: not graded: ${loginProblem}`);
7791
+ return EXIT.notGraded;
7792
+ }
7793
+ mkdirSync2(join3(root, ".firegoat"), { recursive: true });
7586
7794
  if (!gitignoreHasFiregoat(root))
7587
7795
  warn("hint: add .firegoat/ to .gitignore (it holds run traces)");
7588
7796
  const runId = newRunId();
7589
- const runDir = join2(root, ".firegoat", "runs", runId);
7590
- mkdirSync(runDir, { recursive: true });
7591
- const summaryPath = join2(runDir, "summary.json");
7797
+ const runDir = join3(root, ".firegoat", "runs", runId);
7798
+ mkdirSync2(runDir, { recursive: true });
7799
+ const summaryPath = join3(runDir, "summary.json");
7592
7800
  const results = [];
7593
7801
  const writeSummary = () => {
7594
7802
  const totals = { pass: 0, fail: 0, error: 0 };
7595
7803
  for (const r of results)
7596
7804
  totals[r.status]++;
7597
- const doc = { runId, runsDir: runDir, totals, runs: results };
7805
+ const doc = {
7806
+ runId,
7807
+ runsDir: runDir,
7808
+ isolation: opts.host ? "host" : "seatbelt",
7809
+ totals,
7810
+ runs: results
7811
+ };
7598
7812
  writeFileSync(summaryPath, `${JSON.stringify(doc, null, 2)}
7599
7813
  `);
7600
7814
  return doc;
@@ -7603,7 +7817,14 @@ async function evalCommand(args, cwd) {
7603
7817
  for (let n = 1;n <= runs; n++) {
7604
7818
  let r;
7605
7819
  try {
7606
- r = await runOne({ test, n, runDir, claudeBin: claudePath, warn });
7820
+ r = await runOne({
7821
+ test,
7822
+ n,
7823
+ runDir,
7824
+ claudeBin: claudePath,
7825
+ iso,
7826
+ warn
7827
+ });
7607
7828
  } catch (e) {
7608
7829
  r = {
7609
7830
  test: test.name,
@@ -7611,7 +7832,7 @@ async function evalCommand(args, cwd) {
7611
7832
  status: "error",
7612
7833
  reason: `not graded: firegoat crashed: ${e.message}`,
7613
7834
  workdir: null,
7614
- session: join2(runDir, test.name, String(n), "session.jsonl"),
7835
+ session: join3(runDir, test.name, String(n), "session.jsonl"),
7615
7836
  checks: [],
7616
7837
  num_turns: null,
7617
7838
  cost: null,
@@ -7634,8 +7855,8 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
7634
7855
  }
7635
7856
 
7636
7857
  // src/init.ts
7637
- import { cpSync, existsSync as existsSync3, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
7638
- import { basename, join as join3 } from "node:path";
7858
+ import { cpSync, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
7859
+ import { basename as basename2, join as join4 } from "node:path";
7639
7860
  import { createInterface as createInterface2 } from "node:readline/promises";
7640
7861
  import { fileURLToPath } from "node:url";
7641
7862
  var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
@@ -7674,9 +7895,9 @@ function tomlSections(text) {
7674
7895
  return sections;
7675
7896
  }
7676
7897
  function detect(cwd) {
7677
- const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync3(join3(cwd, f)));
7678
- const fallback = { name: basename(cwd), library: true, readme };
7679
- const pkgText = readText(join3(cwd, "package.json"));
7898
+ const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join4(cwd, f)));
7899
+ const fallback = { name: basename2(cwd), library: true, readme };
7900
+ const pkgText = readText(join4(cwd, "package.json"));
7680
7901
  if (pkgText !== undefined) {
7681
7902
  try {
7682
7903
  const pkg = JSON.parse(pkgText);
@@ -7691,7 +7912,7 @@ function detect(cwd) {
7691
7912
  return { ecosystem: "npm", name, bin, library, readme, docsUrl };
7692
7913
  } catch {}
7693
7914
  }
7694
- const pyText = readText(join3(cwd, "pyproject.toml"));
7915
+ const pyText = readText(join4(cwd, "pyproject.toml"));
7695
7916
  if (pyText !== undefined) {
7696
7917
  const s = tomlSections(pyText);
7697
7918
  const name = s.get("project")?.get("name") ?? s.get("tool.poetry")?.get("name") ?? fallback.name;
@@ -7701,12 +7922,12 @@ function detect(cwd) {
7701
7922
  const docsUrl = urls?.get("Documentation") ?? urls?.get("documentation") ?? urls?.get("Homepage") ?? urls?.get("homepage");
7702
7923
  return { ecosystem: "python", name, bin, library: true, readme, docsUrl };
7703
7924
  }
7704
- const goText = readText(join3(cwd, "go.mod"));
7925
+ const goText = readText(join4(cwd, "go.mod"));
7705
7926
  if (goText !== undefined) {
7706
7927
  const name = goText.match(/^module\s+(\S+)/m)?.[1] ?? fallback.name;
7707
7928
  return { ecosystem: "go", name, library: true, readme };
7708
7929
  }
7709
- const cargoText = readText(join3(cwd, "Cargo.toml"));
7930
+ const cargoText = readText(join4(cwd, "Cargo.toml"));
7710
7931
  if (cargoText !== undefined) {
7711
7932
  const name = tomlSections(cargoText).get("package")?.get("name");
7712
7933
  return {
@@ -7809,7 +8030,7 @@ function renderYaml(p, tests) {
7809
8030
  `;
7810
8031
  }
7811
8032
  function ensureGitignore(cwd) {
7812
- const file = join3(cwd, ".gitignore");
8033
+ const file = join4(cwd, ".gitignore");
7813
8034
  const text = readText(file);
7814
8035
  if (text === undefined) {
7815
8036
  writeFileSync2(file, `${GITIGNORE_LINE}
@@ -7876,8 +8097,8 @@ ${INIT_USAGE}`);
7876
8097
  return EXIT.usage;
7877
8098
  }
7878
8099
  }
7879
- const configPath = join3(cwd, CONFIG_FILE);
7880
- const keepConfig = existsSync3(configPath) && !force;
8100
+ const configPath = join4(cwd, CONFIG_FILE);
8101
+ const keepConfig = existsSync4(configPath) && !force;
7881
8102
  if (keepConfig && !skills) {
7882
8103
  console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
7883
8104
  return EXIT.usage;
@@ -7896,7 +8117,7 @@ ${INIT_USAGE}`);
7896
8117
  if (!skills && interactive)
7897
8118
  skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
7898
8119
  if (skills) {
7899
- const dest = join3(cwd, ".claude", "skills", SKILL);
8120
+ const dest = join4(cwd, ".claude", "skills", SKILL);
7900
8121
  cpSync(skillSource(), dest, { recursive: true });
7901
8122
  console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
7902
8123
  } else {
@@ -7906,19 +8127,58 @@ ${INIT_USAGE}`);
7906
8127
  return 0;
7907
8128
  }
7908
8129
 
8130
+ // src/login.ts
8131
+ import { spawnSync as spawnSync3 } from "node:child_process";
8132
+ import { mkdirSync as mkdirSync3 } from "node:fs";
8133
+ function loginCommand(args) {
8134
+ let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
8135
+ for (let i = 0;i < args.length; i++) {
8136
+ const a = args[i];
8137
+ if (a === "--claude-bin" && args[i + 1] !== undefined)
8138
+ claudeBin = args[++i];
8139
+ else if (a.startsWith("--claude-bin="))
8140
+ claudeBin = a.slice(13);
8141
+ else {
8142
+ console.error(`firegoat login: unknown argument: ${a}`);
8143
+ return EXIT.usage;
8144
+ }
8145
+ }
8146
+ const configDir = claudeConfigDir();
8147
+ mkdirSync3(configDir, { recursive: true });
8148
+ const env = {
8149
+ ...process.env,
8150
+ CLAUDE_CONFIG_DIR: configDir
8151
+ };
8152
+ delete env.ANTHROPIC_API_KEY;
8153
+ console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
8154
+ const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
8155
+ if (r.error) {
8156
+ console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
8157
+ return EXIT.notGraded;
8158
+ }
8159
+ if (r.status !== 0) {
8160
+ console.error(`firegoat login: claude auth login exited ${r.status}`);
8161
+ return EXIT.notGraded;
8162
+ }
8163
+ console.log("logged in. next: npx firegoat eval");
8164
+ return EXIT.pass;
8165
+ }
8166
+
7909
8167
  // src/cli.ts
7910
8168
  var USAGE = `firegoat ${package_default.version}: local-first AX evals
7911
8169
 
7912
8170
  Usage:
7913
8171
  firegoat init [--yes] [--force] [--skills] write firegoat.yaml with starter evals
7914
- firegoat eval [test..] [--runs N] [--json] [--claude-bin PATH]
7915
- run evals with claude in an empty workdir
8172
+ firegoat login [--claude-bin PATH] sign claude in to firegoat's own config dir (once)
8173
+ firegoat eval [test..] [--runs N] [--json] [--host] [--claude-bin PATH]
8174
+ run evals with claude in a sandboxed empty workdir
7916
8175
  firegoat --version
7917
8176
 
7918
8177
  eval options:
7919
8178
  test.. only run these tests (by name)
7920
8179
  --runs N runs per test (overrides runs: in firegoat.yaml)
7921
8180
  --json JSON summary on stdout, progress on stderr
8181
+ --host no sandbox (needed off macOS; use a throwaway container)
7922
8182
  --claude-bin PATH claude binary to run (env: FIREGOAT_CLAUDE_BIN; default: claude on PATH)
7923
8183
 
7924
8184
  firegoat.yaml is looked up in the current dir, then parents up to the git root.
@@ -7943,6 +8203,8 @@ async function main(argv) {
7943
8203
  return 0;
7944
8204
  case "init":
7945
8205
  return initCommand(rest, process.cwd());
8206
+ case "login":
8207
+ return loginCommand(rest);
7946
8208
  case "eval":
7947
8209
  return evalCommand(rest, process.cwd());
7948
8210
  default:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firegoat",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "repository": {
5
5
  "type": "git",
6
6
  "url": "git+https://github.com/team2027/firegoat-cli.git"