firegoat 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/cli.js +306 -44
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -2,15 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
|
|
4
4
|
|
|
5
|
-
Status: early (v0.
|
|
5
|
+
Status: early (v0.2). See `plan.md`.
|
|
6
6
|
|
|
7
7
|
## Use
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
npx firegoat init # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
|
|
11
|
-
npx firegoat
|
|
11
|
+
npx firegoat login # once per machine: signs claude in to ~/.firegoat/claude
|
|
12
|
+
npx firegoat eval # runs each test with your local `claude`, sandboxed, in a fresh empty folder
|
|
12
13
|
```
|
|
13
14
|
|
|
15
|
+
### Isolation
|
|
16
|
+
|
|
17
|
+
On macOS, `eval` runs `claude` inside a Seatbelt (`sandbox-exec`) profile:
|
|
18
|
+
|
|
19
|
+
- it signs in with `~/.firegoat/claude` (from `firegoat login`), never your `~/.claude`;
|
|
20
|
+
- it cannot read your home dir or the repo, except toolchains (node, bun, cargo, ...) it needs to run;
|
|
21
|
+
- it can write only in its run dir (`work/`, its own `TMPDIR` and package caches) and system temp;
|
|
22
|
+
- `ANTHROPIC_API_KEY` is removed, so runs bill your subscription, not the API.
|
|
23
|
+
|
|
24
|
+
A probe checks the sandbox before every run; if it does not hold, the run is not graded. Network is open.
|
|
25
|
+
|
|
26
|
+
Off macOS there is no sandbox yet: run inside a throwaway container and pass `--host`.
|
|
27
|
+
|
|
14
28
|
`eval` saves each run's transcript to `.firegoat/runs/<runId>/<test>/<n>/session.jsonl` and a `summary.json`, then runs the test's `checks` in the agent's folder. Nothing is uploaded. `firegoat.yaml` is looked up in the current directory, then parents up to the git root.
|
|
15
29
|
|
|
16
30
|
Exit codes:
|
package/dist/cli.js
CHANGED
|
@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
|
|
|
6947
6947
|
// package.json
|
|
6948
6948
|
var package_default = {
|
|
6949
6949
|
name: "firegoat",
|
|
6950
|
-
version: "0.
|
|
6950
|
+
version: "0.2.0",
|
|
6951
6951
|
repository: {
|
|
6952
6952
|
type: "git",
|
|
6953
6953
|
url: "git+https://github.com/team2027/firegoat-cli.git"
|
|
@@ -7142,20 +7142,164 @@ import {
|
|
|
7142
7142
|
accessSync,
|
|
7143
7143
|
constants,
|
|
7144
7144
|
createWriteStream as createWriteStream2,
|
|
7145
|
-
existsSync as
|
|
7146
|
-
mkdirSync,
|
|
7145
|
+
existsSync as existsSync3,
|
|
7146
|
+
mkdirSync as mkdirSync2,
|
|
7147
7147
|
mkdtempSync,
|
|
7148
7148
|
readFileSync as readFileSync2,
|
|
7149
|
+
realpathSync as realpathSync2,
|
|
7149
7150
|
statSync,
|
|
7150
7151
|
writeFileSync
|
|
7151
7152
|
} from "node:fs";
|
|
7152
7153
|
import { tmpdir } from "node:os";
|
|
7153
|
-
import { delimiter, join as
|
|
7154
|
+
import { delimiter, join as join3, resolve } from "node:path";
|
|
7154
7155
|
|
|
7155
7156
|
// src/claude.ts
|
|
7156
|
-
import { spawn } from "node:child_process";
|
|
7157
|
+
import { spawn, spawnSync as spawnSync2 } from "node:child_process";
|
|
7157
7158
|
import { createWriteStream } from "node:fs";
|
|
7158
7159
|
import { createInterface } from "node:readline";
|
|
7160
|
+
|
|
7161
|
+
// src/sandbox.ts
|
|
7162
|
+
import { spawnSync } from "node:child_process";
|
|
7163
|
+
import { existsSync as existsSync2, mkdirSync, realpathSync } from "node:fs";
|
|
7164
|
+
import { homedir } from "node:os";
|
|
7165
|
+
import { basename, dirname as dirname2, join as join2 } from "node:path";
|
|
7166
|
+
var SANDBOX_EXEC = "/usr/bin/sandbox-exec";
|
|
7167
|
+
function firegoatHome() {
|
|
7168
|
+
return process.env.FIREGOAT_HOME || join2(homedir(), ".firegoat");
|
|
7169
|
+
}
|
|
7170
|
+
function claudeConfigDir() {
|
|
7171
|
+
return join2(firegoatHome(), "claude");
|
|
7172
|
+
}
|
|
7173
|
+
var TOOLCHAINS = [
|
|
7174
|
+
".nvm",
|
|
7175
|
+
".fnm",
|
|
7176
|
+
".volta",
|
|
7177
|
+
".bun",
|
|
7178
|
+
".deno",
|
|
7179
|
+
".cargo",
|
|
7180
|
+
".rustup",
|
|
7181
|
+
".pyenv",
|
|
7182
|
+
".rbenv",
|
|
7183
|
+
".asdf",
|
|
7184
|
+
".sdkman",
|
|
7185
|
+
".local/share/mise",
|
|
7186
|
+
".local/share/fnm",
|
|
7187
|
+
".local/share/claude",
|
|
7188
|
+
"go",
|
|
7189
|
+
"Library/pnpm"
|
|
7190
|
+
];
|
|
7191
|
+
var TOOLCHAIN_SECRETS = [".cargo/credentials", ".cargo/credentials.toml"];
|
|
7192
|
+
function real(p) {
|
|
7193
|
+
return realpathSync(p);
|
|
7194
|
+
}
|
|
7195
|
+
function under(p, dir) {
|
|
7196
|
+
return p === dir || p.startsWith(`${dir}/`);
|
|
7197
|
+
}
|
|
7198
|
+
function readableToolDirs(home, claudeBin) {
|
|
7199
|
+
const dirs = new Set;
|
|
7200
|
+
const add = (p) => {
|
|
7201
|
+
if (existsSync2(p))
|
|
7202
|
+
dirs.add(real(p));
|
|
7203
|
+
};
|
|
7204
|
+
for (const t of TOOLCHAINS)
|
|
7205
|
+
add(join2(home, t));
|
|
7206
|
+
for (const p of (process.env.PATH ?? "").split(":")) {
|
|
7207
|
+
if (!p.startsWith("/") || !existsSync2(p))
|
|
7208
|
+
continue;
|
|
7209
|
+
const r = real(p);
|
|
7210
|
+
if (!under(r, home) || r === home)
|
|
7211
|
+
continue;
|
|
7212
|
+
const prefix = dirname2(r);
|
|
7213
|
+
dirs.add(basename(r) === "bin" && prefix !== home && prefix !== join2(home, ".local") ? prefix : r);
|
|
7214
|
+
}
|
|
7215
|
+
add(dirname2(dirname2(process.execPath)));
|
|
7216
|
+
add(dirname2(claudeBin));
|
|
7217
|
+
add(dirname2(real(claudeBin)));
|
|
7218
|
+
return [...dirs];
|
|
7219
|
+
}
|
|
7220
|
+
var q = (s) => JSON.stringify(s);
|
|
7221
|
+
function sandboxProfile(opts) {
|
|
7222
|
+
const home = real(homedir());
|
|
7223
|
+
const reads = readableToolDirs(home, opts.claudeBin).map((d) => `(subpath ${q(d)})`);
|
|
7224
|
+
return `(version 1)
|
|
7225
|
+
(allow default)
|
|
7226
|
+
|
|
7227
|
+
(deny file-read* (subpath ${q(home)}))
|
|
7228
|
+
(allow file-read-metadata)
|
|
7229
|
+
(allow file-read*
|
|
7230
|
+
${reads.join(`
|
|
7231
|
+
`)}
|
|
7232
|
+
(subpath ${q(join2(home, "Library/Keychains"))})
|
|
7233
|
+
(regex ${q(`^${home}/Library/Preferences/com\\.apple\\.security`)})
|
|
7234
|
+
(regex ${q(`^${home}/Library/Preferences/\\.GlobalPreferences`)}))
|
|
7235
|
+
(deny file-read* file-write*
|
|
7236
|
+
${TOOLCHAIN_SECRETS.map((s) => `(literal ${q(join2(home, s))})`).join(`
|
|
7237
|
+
`)})
|
|
7238
|
+
|
|
7239
|
+
(deny file-write* (subpath "/"))
|
|
7240
|
+
(allow file-write*
|
|
7241
|
+
(subpath "/dev")
|
|
7242
|
+
(subpath "/private/tmp")
|
|
7243
|
+
(subpath "/private/var/folders"))
|
|
7244
|
+
(allow file-read* file-write*
|
|
7245
|
+
(subpath ${q(real(opts.configDir))})
|
|
7246
|
+
(subpath ${q(real(opts.runRoot))}))
|
|
7247
|
+
|
|
7248
|
+
(deny file-read* file-write* (subpath ${q(real(opts.repo))}))
|
|
7249
|
+
`;
|
|
7250
|
+
}
|
|
7251
|
+
function agentEnv(runRoot) {
|
|
7252
|
+
const env = { ...process.env };
|
|
7253
|
+
delete env.ANTHROPIC_API_KEY;
|
|
7254
|
+
const cache = join2(runRoot, "cache");
|
|
7255
|
+
const tmp = join2(runRoot, "tmp");
|
|
7256
|
+
mkdirSync(cache, { recursive: true });
|
|
7257
|
+
mkdirSync(tmp, { recursive: true });
|
|
7258
|
+
return {
|
|
7259
|
+
...env,
|
|
7260
|
+
CLAUDE_CONFIG_DIR: claudeConfigDir(),
|
|
7261
|
+
TMPDIR: tmp,
|
|
7262
|
+
XDG_CACHE_HOME: cache,
|
|
7263
|
+
npm_config_cache: join2(cache, "npm"),
|
|
7264
|
+
npm_config_store_dir: join2(cache, "pnpm-store"),
|
|
7265
|
+
YARN_CACHE_FOLDER: join2(cache, "yarn"),
|
|
7266
|
+
BUN_INSTALL_CACHE_DIR: join2(cache, "bun"),
|
|
7267
|
+
PIP_CACHE_DIR: join2(cache, "pip"),
|
|
7268
|
+
UV_CACHE_DIR: join2(cache, "uv"),
|
|
7269
|
+
GOMODCACHE: join2(cache, "gomod"),
|
|
7270
|
+
GOCACHE: join2(cache, "gobuild"),
|
|
7271
|
+
CARGO_HOME: join2(cache, "cargo")
|
|
7272
|
+
};
|
|
7273
|
+
}
|
|
7274
|
+
function wrap(profile, cmd, args) {
|
|
7275
|
+
if (profile === null)
|
|
7276
|
+
return { cmd, args };
|
|
7277
|
+
return { cmd: SANDBOX_EXEC, args: ["-p", profile, cmd, ...args] };
|
|
7278
|
+
}
|
|
7279
|
+
function probeSandbox(opts) {
|
|
7280
|
+
const outside = join2(homedir(), `.firegoat-escape-probe-${process.pid}-${Date.now()}`);
|
|
7281
|
+
const script = 'touch "$1" 2>/dev/null && echo home-write; cat "$2" >/dev/null 2>&1 && echo repo-read; touch "$3" 2>/dev/null && echo work-write; rm -f "$1" "$3"';
|
|
7282
|
+
const { cmd, args } = wrap(opts.profile, "/bin/sh", [
|
|
7283
|
+
"-c",
|
|
7284
|
+
script,
|
|
7285
|
+
"probe",
|
|
7286
|
+
outside,
|
|
7287
|
+
opts.repoFile,
|
|
7288
|
+
join2(opts.workdir, ".firegoat-probe")
|
|
7289
|
+
]);
|
|
7290
|
+
const r = spawnSync(cmd, args, { encoding: "utf8", cwd: opts.workdir });
|
|
7291
|
+
if (r.error)
|
|
7292
|
+
return `sandbox probe could not start: ${r.error.message}`;
|
|
7293
|
+
const got = r.stdout.trim().split(`
|
|
7294
|
+
`).filter(Boolean);
|
|
7295
|
+
if (got.includes("home-write") || got.includes("repo-read"))
|
|
7296
|
+
return `sandbox does not hold (escaped: ${got.filter((g) => g !== "work-write").join(", ")}); refusing to run the agent`;
|
|
7297
|
+
if (!got.includes("work-write"))
|
|
7298
|
+
return `sandbox blocks writes in the workdir (stderr: ${r.stderr.trim() || "none"})`;
|
|
7299
|
+
return;
|
|
7300
|
+
}
|
|
7301
|
+
|
|
7302
|
+
// src/claude.ts
|
|
7159
7303
|
var CLAUDE_FLAGS = [
|
|
7160
7304
|
"--output-format",
|
|
7161
7305
|
"stream-json",
|
|
@@ -7169,11 +7313,6 @@ var CLAUDE_FLAGS = [
|
|
|
7169
7313
|
function claudeArgv(prompt) {
|
|
7170
7314
|
return ["-p", prompt, ...CLAUDE_FLAGS];
|
|
7171
7315
|
}
|
|
7172
|
-
function agentEnv() {
|
|
7173
|
-
const env = { ...process.env };
|
|
7174
|
-
delete env.ANTHROPIC_API_KEY;
|
|
7175
|
-
return env;
|
|
7176
|
-
}
|
|
7177
7316
|
var live = new Set;
|
|
7178
7317
|
function killGroup(child, signal) {
|
|
7179
7318
|
if (child.pid === undefined)
|
|
@@ -7303,10 +7442,9 @@ async function runClaude(opts) {
|
|
|
7303
7442
|
let badStart;
|
|
7304
7443
|
let result;
|
|
7305
7444
|
const spawned = await spawnWatched({
|
|
7306
|
-
|
|
7307
|
-
args: claudeArgv(opts.prompt),
|
|
7445
|
+
...wrap(opts.profile, opts.bin, claudeArgv(opts.prompt)),
|
|
7308
7446
|
cwd: opts.cwd,
|
|
7309
|
-
env:
|
|
7447
|
+
env: opts.env,
|
|
7310
7448
|
timeoutMs: opts.timeoutMs,
|
|
7311
7449
|
onStderr: (c) => stderr.write(c),
|
|
7312
7450
|
onLine: (line) => {
|
|
@@ -7405,6 +7543,26 @@ async function runCheck(opts) {
|
|
|
7405
7543
|
out.notStarted = true;
|
|
7406
7544
|
return out;
|
|
7407
7545
|
}
|
|
7546
|
+
function checkLogin(opts) {
|
|
7547
|
+
const { cmd, args } = wrap(opts.profile, opts.bin, ["auth", "status"]);
|
|
7548
|
+
const r = spawnSync2(cmd, args, {
|
|
7549
|
+
cwd: opts.cwd,
|
|
7550
|
+
env: opts.env,
|
|
7551
|
+
encoding: "utf8",
|
|
7552
|
+
timeout: 30000
|
|
7553
|
+
});
|
|
7554
|
+
if (r.error)
|
|
7555
|
+
return `could not run claude auth status: ${r.error.message}`;
|
|
7556
|
+
let status;
|
|
7557
|
+
try {
|
|
7558
|
+
status = JSON.parse(r.stdout);
|
|
7559
|
+
} catch {}
|
|
7560
|
+
if (status?.loggedIn === true)
|
|
7561
|
+
return;
|
|
7562
|
+
if (status?.loggedIn === false)
|
|
7563
|
+
return `claude is not logged in to firegoat's config dir (${opts.env.CLAUDE_CONFIG_DIR}). Run: npx firegoat login`;
|
|
7564
|
+
return `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`;
|
|
7565
|
+
}
|
|
7408
7566
|
|
|
7409
7567
|
// src/exit.ts
|
|
7410
7568
|
var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
|
|
@@ -7418,6 +7576,7 @@ function parseArgs(args) {
|
|
|
7418
7576
|
const opts = {
|
|
7419
7577
|
names: [],
|
|
7420
7578
|
json: false,
|
|
7579
|
+
host: false,
|
|
7421
7580
|
claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
|
|
7422
7581
|
};
|
|
7423
7582
|
for (let i = 0;i < args.length; i++) {
|
|
@@ -7435,6 +7594,8 @@ function parseArgs(args) {
|
|
|
7435
7594
|
};
|
|
7436
7595
|
if (flag === "--json")
|
|
7437
7596
|
opts.json = true;
|
|
7597
|
+
else if (flag === "--host")
|
|
7598
|
+
opts.host = true;
|
|
7438
7599
|
else if (flag === "--runs") {
|
|
7439
7600
|
const v = value();
|
|
7440
7601
|
const n = Number(v);
|
|
@@ -7455,8 +7616,8 @@ function newRunId() {
|
|
|
7455
7616
|
return `${ts}-${randomBytes(2).toString("hex")}`;
|
|
7456
7617
|
}
|
|
7457
7618
|
function gitignoreHasFiregoat(cwd) {
|
|
7458
|
-
const file =
|
|
7459
|
-
if (!
|
|
7619
|
+
const file = join3(cwd, ".gitignore");
|
|
7620
|
+
if (!existsSync3(file))
|
|
7460
7621
|
return false;
|
|
7461
7622
|
return readFileSync2(file, "utf8").split(`
|
|
7462
7623
|
`).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
|
|
@@ -7473,17 +7634,35 @@ function resolveBin(bin) {
|
|
|
7473
7634
|
if (bin.includes("/"))
|
|
7474
7635
|
return executable(bin) ? resolve(bin) : undefined;
|
|
7475
7636
|
for (const dir of (process.env.PATH ?? "").split(delimiter)) {
|
|
7476
|
-
if (dir && executable(
|
|
7477
|
-
return
|
|
7637
|
+
if (dir && executable(join3(dir, bin)))
|
|
7638
|
+
return join3(dir, bin);
|
|
7478
7639
|
}
|
|
7479
7640
|
return;
|
|
7480
7641
|
}
|
|
7642
|
+
function newRunRoot(label) {
|
|
7643
|
+
const runRoot = realpathSync2(mkdtempSync(join3(tmpdir(), `firegoat-${label}-`)));
|
|
7644
|
+
const workdir = join3(runRoot, "work");
|
|
7645
|
+
mkdirSync2(workdir);
|
|
7646
|
+
return { runRoot, workdir };
|
|
7647
|
+
}
|
|
7648
|
+
function isolate(iso, claudeBin, runRoot, workdir) {
|
|
7649
|
+
if (iso.host)
|
|
7650
|
+
return { profile: null };
|
|
7651
|
+
const profile = sandboxProfile({
|
|
7652
|
+
claudeBin,
|
|
7653
|
+
runRoot,
|
|
7654
|
+
configDir: claudeConfigDir(),
|
|
7655
|
+
repo: iso.repo
|
|
7656
|
+
});
|
|
7657
|
+
const broken = probeSandbox({ profile, workdir, repoFile: iso.repoFile });
|
|
7658
|
+
return broken ? { error: broken } : { profile };
|
|
7659
|
+
}
|
|
7481
7660
|
async function runOne(opts) {
|
|
7482
7661
|
const { test, n } = opts;
|
|
7483
7662
|
const started = Date.now();
|
|
7484
|
-
const dir =
|
|
7485
|
-
|
|
7486
|
-
const session =
|
|
7663
|
+
const dir = join3(opts.runDir, test.name, String(n));
|
|
7664
|
+
mkdirSync2(dir, { recursive: true });
|
|
7665
|
+
const session = join3(dir, "session.jsonl");
|
|
7487
7666
|
const summary = {
|
|
7488
7667
|
test: test.name,
|
|
7489
7668
|
n,
|
|
@@ -7502,14 +7681,19 @@ async function runOne(opts) {
|
|
|
7502
7681
|
summary.duration_ms = Date.now() - started;
|
|
7503
7682
|
return summary;
|
|
7504
7683
|
};
|
|
7505
|
-
const workdir =
|
|
7684
|
+
const { runRoot, workdir } = newRunRoot(`${test.name}-${n}`);
|
|
7506
7685
|
summary.workdir = workdir;
|
|
7686
|
+
const sandbox = isolate(opts.iso, opts.claudeBin, runRoot, workdir);
|
|
7687
|
+
if ("error" in sandbox)
|
|
7688
|
+
return done("error", `not graded: ${sandbox.error}`);
|
|
7507
7689
|
const outcome = await runClaude({
|
|
7690
|
+
env: agentEnv(runRoot),
|
|
7691
|
+
profile: sandbox.profile,
|
|
7508
7692
|
bin: opts.claudeBin,
|
|
7509
7693
|
prompt: test.prompt,
|
|
7510
7694
|
cwd: workdir,
|
|
7511
7695
|
sessionPath: session,
|
|
7512
|
-
stderrPath:
|
|
7696
|
+
stderrPath: join3(dir, "stderr.txt"),
|
|
7513
7697
|
timeoutMs: test.timeout * 1000
|
|
7514
7698
|
});
|
|
7515
7699
|
if (outcome.result) {
|
|
@@ -7522,7 +7706,7 @@ async function runOne(opts) {
|
|
|
7522
7706
|
opts.warn(`warning: ${test.name} has no checks, so its goal is unverified; counted as pass`);
|
|
7523
7707
|
return done("pass", "no checks (goal unverified)");
|
|
7524
7708
|
}
|
|
7525
|
-
const log = createWriteStream2(
|
|
7709
|
+
const log = createWriteStream2(join3(dir, "checks.txt"));
|
|
7526
7710
|
const env = {
|
|
7527
7711
|
...process.env,
|
|
7528
7712
|
FIREGOAT_SESSION: resolve(session),
|
|
@@ -7582,19 +7766,49 @@ async function evalCommand(args, cwd) {
|
|
|
7582
7766
|
console.error(`firegoat eval: not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
|
|
7583
7767
|
return EXIT.notGraded;
|
|
7584
7768
|
}
|
|
7585
|
-
|
|
7769
|
+
if (opts.host)
|
|
7770
|
+
warn("warning: --host: the agent runs without isolation and can read and write your files");
|
|
7771
|
+
else if (process.platform !== "darwin" || !existsSync3(SANDBOX_EXEC)) {
|
|
7772
|
+
console.error(`firegoat eval: isolation needs macOS (${SANDBOX_EXEC}); this is ${process.platform}. In CI or Linux, run inside a throwaway container and pass --host.`);
|
|
7773
|
+
return EXIT.usage;
|
|
7774
|
+
}
|
|
7775
|
+
const iso = {
|
|
7776
|
+
host: opts.host,
|
|
7777
|
+
repo: root,
|
|
7778
|
+
repoFile: join3(root, "firegoat.yaml")
|
|
7779
|
+
};
|
|
7780
|
+
mkdirSync2(claudeConfigDir(), { recursive: true });
|
|
7781
|
+
const pre = newRunRoot("preflight");
|
|
7782
|
+
const preSandbox = isolate(iso, claudePath, pre.runRoot, pre.workdir);
|
|
7783
|
+
const loginProblem = "error" in preSandbox ? preSandbox.error : checkLogin({
|
|
7784
|
+
bin: claudePath,
|
|
7785
|
+
env: agentEnv(pre.runRoot),
|
|
7786
|
+
profile: preSandbox.profile,
|
|
7787
|
+
cwd: pre.workdir
|
|
7788
|
+
});
|
|
7789
|
+
if (loginProblem) {
|
|
7790
|
+
console.error(`firegoat eval: not graded: ${loginProblem}`);
|
|
7791
|
+
return EXIT.notGraded;
|
|
7792
|
+
}
|
|
7793
|
+
mkdirSync2(join3(root, ".firegoat"), { recursive: true });
|
|
7586
7794
|
if (!gitignoreHasFiregoat(root))
|
|
7587
7795
|
warn("hint: add .firegoat/ to .gitignore (it holds run traces)");
|
|
7588
7796
|
const runId = newRunId();
|
|
7589
|
-
const runDir =
|
|
7590
|
-
|
|
7591
|
-
const summaryPath =
|
|
7797
|
+
const runDir = join3(root, ".firegoat", "runs", runId);
|
|
7798
|
+
mkdirSync2(runDir, { recursive: true });
|
|
7799
|
+
const summaryPath = join3(runDir, "summary.json");
|
|
7592
7800
|
const results = [];
|
|
7593
7801
|
const writeSummary = () => {
|
|
7594
7802
|
const totals = { pass: 0, fail: 0, error: 0 };
|
|
7595
7803
|
for (const r of results)
|
|
7596
7804
|
totals[r.status]++;
|
|
7597
|
-
const doc = {
|
|
7805
|
+
const doc = {
|
|
7806
|
+
runId,
|
|
7807
|
+
runsDir: runDir,
|
|
7808
|
+
isolation: opts.host ? "host" : "seatbelt",
|
|
7809
|
+
totals,
|
|
7810
|
+
runs: results
|
|
7811
|
+
};
|
|
7598
7812
|
writeFileSync(summaryPath, `${JSON.stringify(doc, null, 2)}
|
|
7599
7813
|
`);
|
|
7600
7814
|
return doc;
|
|
@@ -7603,7 +7817,14 @@ async function evalCommand(args, cwd) {
|
|
|
7603
7817
|
for (let n = 1;n <= runs; n++) {
|
|
7604
7818
|
let r;
|
|
7605
7819
|
try {
|
|
7606
|
-
r = await runOne({
|
|
7820
|
+
r = await runOne({
|
|
7821
|
+
test,
|
|
7822
|
+
n,
|
|
7823
|
+
runDir,
|
|
7824
|
+
claudeBin: claudePath,
|
|
7825
|
+
iso,
|
|
7826
|
+
warn
|
|
7827
|
+
});
|
|
7607
7828
|
} catch (e) {
|
|
7608
7829
|
r = {
|
|
7609
7830
|
test: test.name,
|
|
@@ -7611,7 +7832,7 @@ async function evalCommand(args, cwd) {
|
|
|
7611
7832
|
status: "error",
|
|
7612
7833
|
reason: `not graded: firegoat crashed: ${e.message}`,
|
|
7613
7834
|
workdir: null,
|
|
7614
|
-
session:
|
|
7835
|
+
session: join3(runDir, test.name, String(n), "session.jsonl"),
|
|
7615
7836
|
checks: [],
|
|
7616
7837
|
num_turns: null,
|
|
7617
7838
|
cost: null,
|
|
@@ -7634,8 +7855,8 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
|
|
|
7634
7855
|
}
|
|
7635
7856
|
|
|
7636
7857
|
// src/init.ts
|
|
7637
|
-
import { cpSync, existsSync as
|
|
7638
|
-
import { basename, join as
|
|
7858
|
+
import { cpSync, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
7859
|
+
import { basename as basename2, join as join4 } from "node:path";
|
|
7639
7860
|
import { createInterface as createInterface2 } from "node:readline/promises";
|
|
7640
7861
|
import { fileURLToPath } from "node:url";
|
|
7641
7862
|
var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
|
|
@@ -7674,9 +7895,9 @@ function tomlSections(text) {
|
|
|
7674
7895
|
return sections;
|
|
7675
7896
|
}
|
|
7676
7897
|
function detect(cwd) {
|
|
7677
|
-
const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) =>
|
|
7678
|
-
const fallback = { name:
|
|
7679
|
-
const pkgText = readText(
|
|
7898
|
+
const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join4(cwd, f)));
|
|
7899
|
+
const fallback = { name: basename2(cwd), library: true, readme };
|
|
7900
|
+
const pkgText = readText(join4(cwd, "package.json"));
|
|
7680
7901
|
if (pkgText !== undefined) {
|
|
7681
7902
|
try {
|
|
7682
7903
|
const pkg = JSON.parse(pkgText);
|
|
@@ -7691,7 +7912,7 @@ function detect(cwd) {
|
|
|
7691
7912
|
return { ecosystem: "npm", name, bin, library, readme, docsUrl };
|
|
7692
7913
|
} catch {}
|
|
7693
7914
|
}
|
|
7694
|
-
const pyText = readText(
|
|
7915
|
+
const pyText = readText(join4(cwd, "pyproject.toml"));
|
|
7695
7916
|
if (pyText !== undefined) {
|
|
7696
7917
|
const s = tomlSections(pyText);
|
|
7697
7918
|
const name = s.get("project")?.get("name") ?? s.get("tool.poetry")?.get("name") ?? fallback.name;
|
|
@@ -7701,12 +7922,12 @@ function detect(cwd) {
|
|
|
7701
7922
|
const docsUrl = urls?.get("Documentation") ?? urls?.get("documentation") ?? urls?.get("Homepage") ?? urls?.get("homepage");
|
|
7702
7923
|
return { ecosystem: "python", name, bin, library: true, readme, docsUrl };
|
|
7703
7924
|
}
|
|
7704
|
-
const goText = readText(
|
|
7925
|
+
const goText = readText(join4(cwd, "go.mod"));
|
|
7705
7926
|
if (goText !== undefined) {
|
|
7706
7927
|
const name = goText.match(/^module\s+(\S+)/m)?.[1] ?? fallback.name;
|
|
7707
7928
|
return { ecosystem: "go", name, library: true, readme };
|
|
7708
7929
|
}
|
|
7709
|
-
const cargoText = readText(
|
|
7930
|
+
const cargoText = readText(join4(cwd, "Cargo.toml"));
|
|
7710
7931
|
if (cargoText !== undefined) {
|
|
7711
7932
|
const name = tomlSections(cargoText).get("package")?.get("name");
|
|
7712
7933
|
return {
|
|
@@ -7809,7 +8030,7 @@ function renderYaml(p, tests) {
|
|
|
7809
8030
|
`;
|
|
7810
8031
|
}
|
|
7811
8032
|
function ensureGitignore(cwd) {
|
|
7812
|
-
const file =
|
|
8033
|
+
const file = join4(cwd, ".gitignore");
|
|
7813
8034
|
const text = readText(file);
|
|
7814
8035
|
if (text === undefined) {
|
|
7815
8036
|
writeFileSync2(file, `${GITIGNORE_LINE}
|
|
@@ -7876,8 +8097,8 @@ ${INIT_USAGE}`);
|
|
|
7876
8097
|
return EXIT.usage;
|
|
7877
8098
|
}
|
|
7878
8099
|
}
|
|
7879
|
-
const configPath =
|
|
7880
|
-
const keepConfig =
|
|
8100
|
+
const configPath = join4(cwd, CONFIG_FILE);
|
|
8101
|
+
const keepConfig = existsSync4(configPath) && !force;
|
|
7881
8102
|
if (keepConfig && !skills) {
|
|
7882
8103
|
console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
|
|
7883
8104
|
return EXIT.usage;
|
|
@@ -7896,7 +8117,7 @@ ${INIT_USAGE}`);
|
|
|
7896
8117
|
if (!skills && interactive)
|
|
7897
8118
|
skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
|
|
7898
8119
|
if (skills) {
|
|
7899
|
-
const dest =
|
|
8120
|
+
const dest = join4(cwd, ".claude", "skills", SKILL);
|
|
7900
8121
|
cpSync(skillSource(), dest, { recursive: true });
|
|
7901
8122
|
console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
|
|
7902
8123
|
} else {
|
|
@@ -7906,19 +8127,58 @@ ${INIT_USAGE}`);
|
|
|
7906
8127
|
return 0;
|
|
7907
8128
|
}
|
|
7908
8129
|
|
|
8130
|
+
// src/login.ts
|
|
8131
|
+
import { spawnSync as spawnSync3 } from "node:child_process";
|
|
8132
|
+
import { mkdirSync as mkdirSync3 } from "node:fs";
|
|
8133
|
+
function loginCommand(args) {
|
|
8134
|
+
let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
|
|
8135
|
+
for (let i = 0;i < args.length; i++) {
|
|
8136
|
+
const a = args[i];
|
|
8137
|
+
if (a === "--claude-bin" && args[i + 1] !== undefined)
|
|
8138
|
+
claudeBin = args[++i];
|
|
8139
|
+
else if (a.startsWith("--claude-bin="))
|
|
8140
|
+
claudeBin = a.slice(13);
|
|
8141
|
+
else {
|
|
8142
|
+
console.error(`firegoat login: unknown argument: ${a}`);
|
|
8143
|
+
return EXIT.usage;
|
|
8144
|
+
}
|
|
8145
|
+
}
|
|
8146
|
+
const configDir = claudeConfigDir();
|
|
8147
|
+
mkdirSync3(configDir, { recursive: true });
|
|
8148
|
+
const env = {
|
|
8149
|
+
...process.env,
|
|
8150
|
+
CLAUDE_CONFIG_DIR: configDir
|
|
8151
|
+
};
|
|
8152
|
+
delete env.ANTHROPIC_API_KEY;
|
|
8153
|
+
console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
|
|
8154
|
+
const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
|
|
8155
|
+
if (r.error) {
|
|
8156
|
+
console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
|
|
8157
|
+
return EXIT.notGraded;
|
|
8158
|
+
}
|
|
8159
|
+
if (r.status !== 0) {
|
|
8160
|
+
console.error(`firegoat login: claude auth login exited ${r.status}`);
|
|
8161
|
+
return EXIT.notGraded;
|
|
8162
|
+
}
|
|
8163
|
+
console.log("logged in. next: npx firegoat eval");
|
|
8164
|
+
return EXIT.pass;
|
|
8165
|
+
}
|
|
8166
|
+
|
|
7909
8167
|
// src/cli.ts
|
|
7910
8168
|
var USAGE = `firegoat ${package_default.version}: local-first AX evals
|
|
7911
8169
|
|
|
7912
8170
|
Usage:
|
|
7913
8171
|
firegoat init [--yes] [--force] [--skills] write firegoat.yaml with starter evals
|
|
7914
|
-
firegoat
|
|
7915
|
-
|
|
8172
|
+
firegoat login [--claude-bin PATH] sign claude in to firegoat's own config dir (once)
|
|
8173
|
+
firegoat eval [test..] [--runs N] [--json] [--host] [--claude-bin PATH]
|
|
8174
|
+
run evals with claude in a sandboxed empty workdir
|
|
7916
8175
|
firegoat --version
|
|
7917
8176
|
|
|
7918
8177
|
eval options:
|
|
7919
8178
|
test.. only run these tests (by name)
|
|
7920
8179
|
--runs N runs per test (overrides runs: in firegoat.yaml)
|
|
7921
8180
|
--json JSON summary on stdout, progress on stderr
|
|
8181
|
+
--host no sandbox (needed off macOS; use a throwaway container)
|
|
7922
8182
|
--claude-bin PATH claude binary to run (env: FIREGOAT_CLAUDE_BIN; default: claude on PATH)
|
|
7923
8183
|
|
|
7924
8184
|
firegoat.yaml is looked up in the current dir, then parents up to the git root.
|
|
@@ -7943,6 +8203,8 @@ async function main(argv) {
|
|
|
7943
8203
|
return 0;
|
|
7944
8204
|
case "init":
|
|
7945
8205
|
return initCommand(rest, process.cwd());
|
|
8206
|
+
case "login":
|
|
8207
|
+
return loginCommand(rest);
|
|
7946
8208
|
case "eval":
|
|
7947
8209
|
return evalCommand(rest, process.cwd());
|
|
7948
8210
|
default:
|