firegoat 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +211 -160
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
|
|
|
6947
6947
|
// package.json
|
|
6948
6948
|
var package_default = {
|
|
6949
6949
|
name: "firegoat",
|
|
6950
|
-
version: "0.
|
|
6950
|
+
version: "0.29.0",
|
|
6951
6951
|
repository: {
|
|
6952
6952
|
type: "git",
|
|
6953
6953
|
url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
|
|
@@ -7265,8 +7265,146 @@ import { homedir } from "node:os";
|
|
|
7265
7265
|
import { join as join3 } from "node:path";
|
|
7266
7266
|
|
|
7267
7267
|
// src/home.ts
|
|
7268
|
+
import { execFile } from "node:child_process";
|
|
7268
7269
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
7269
7270
|
import { join as join2 } from "node:path";
|
|
7271
|
+
import { promisify } from "node:util";
|
|
7272
|
+
|
|
7273
|
+
// src/process.ts
|
|
7274
|
+
import { execFileSync, spawn } from "node:child_process";
|
|
7275
|
+
var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
|
|
7276
|
+
var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
|
|
7277
|
+
var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
|
|
7278
|
+
function hostEnv() {
|
|
7279
|
+
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
|
|
7280
|
+
}
|
|
7281
|
+
var GRACE_MS = 5000;
|
|
7282
|
+
var liveGroups = new Set;
|
|
7283
|
+
var signalsHooked = false;
|
|
7284
|
+
function hookSignals() {
|
|
7285
|
+
if (signalsHooked)
|
|
7286
|
+
return;
|
|
7287
|
+
signalsHooked = true;
|
|
7288
|
+
const onSignal = (signal) => {
|
|
7289
|
+
for (const g of liveGroups)
|
|
7290
|
+
try {
|
|
7291
|
+
process.kill(-g, "SIGKILL");
|
|
7292
|
+
} catch {}
|
|
7293
|
+
process.kill(process.pid, signal);
|
|
7294
|
+
};
|
|
7295
|
+
for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
|
|
7296
|
+
process.once(s, onSignal);
|
|
7297
|
+
}
|
|
7298
|
+
function forgetGroup(pgid) {
|
|
7299
|
+
liveGroups.delete(pgid);
|
|
7300
|
+
}
|
|
7301
|
+
function processTree(pid) {
|
|
7302
|
+
const children = new Map;
|
|
7303
|
+
const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
|
|
7304
|
+
encoding: "utf8"
|
|
7305
|
+
});
|
|
7306
|
+
for (const line of ps.split(`
|
|
7307
|
+
`)) {
|
|
7308
|
+
const [child, parent] = line.trim().split(/\s+/).map(Number);
|
|
7309
|
+
if (!child || parent === undefined)
|
|
7310
|
+
continue;
|
|
7311
|
+
children.set(parent, [...children.get(parent) ?? [], child]);
|
|
7312
|
+
}
|
|
7313
|
+
const tree = [pid];
|
|
7314
|
+
for (let i = 0;i < tree.length; i++)
|
|
7315
|
+
tree.push(...children.get(tree[i]) ?? []);
|
|
7316
|
+
return tree;
|
|
7317
|
+
}
|
|
7318
|
+
function killTree(pid, group) {
|
|
7319
|
+
if (group)
|
|
7320
|
+
try {
|
|
7321
|
+
process.kill(-pid, "SIGKILL");
|
|
7322
|
+
} catch {}
|
|
7323
|
+
let tree = [pid];
|
|
7324
|
+
let problem;
|
|
7325
|
+
try {
|
|
7326
|
+
tree = processTree(pid);
|
|
7327
|
+
} catch (e) {
|
|
7328
|
+
problem = e.message;
|
|
7329
|
+
}
|
|
7330
|
+
for (const p of tree)
|
|
7331
|
+
try {
|
|
7332
|
+
process.kill(p, "SIGKILL");
|
|
7333
|
+
} catch {}
|
|
7334
|
+
return problem;
|
|
7335
|
+
}
|
|
7336
|
+
function run(opts) {
|
|
7337
|
+
return new Promise((resolve) => {
|
|
7338
|
+
const detached = opts.detached === true;
|
|
7339
|
+
if (detached)
|
|
7340
|
+
hookSignals();
|
|
7341
|
+
const child = spawn(opts.cmd, opts.args, {
|
|
7342
|
+
cwd: opts.cwd,
|
|
7343
|
+
env: opts.env,
|
|
7344
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
7345
|
+
detached
|
|
7346
|
+
});
|
|
7347
|
+
if (child.pid !== undefined) {
|
|
7348
|
+
if (detached)
|
|
7349
|
+
liveGroups.add(child.pid);
|
|
7350
|
+
opts.onSpawn?.(child.pid);
|
|
7351
|
+
}
|
|
7352
|
+
let timedOut = false;
|
|
7353
|
+
let settled = false;
|
|
7354
|
+
const timer = setTimeout(() => {
|
|
7355
|
+
timedOut = true;
|
|
7356
|
+
if (child.pid === undefined)
|
|
7357
|
+
return;
|
|
7358
|
+
const problem = killTree(child.pid, detached);
|
|
7359
|
+
if (problem)
|
|
7360
|
+
opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
|
|
7361
|
+
`));
|
|
7362
|
+
}, opts.timeoutMs);
|
|
7363
|
+
const done = (r) => {
|
|
7364
|
+
if (settled)
|
|
7365
|
+
return;
|
|
7366
|
+
settled = true;
|
|
7367
|
+
clearTimeout(timer);
|
|
7368
|
+
resolve(r);
|
|
7369
|
+
};
|
|
7370
|
+
child.stdout.on("data", opts.onStdout);
|
|
7371
|
+
child.stderr.on("data", opts.onStderr);
|
|
7372
|
+
opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
|
|
7373
|
+
child.on("error", (error) => done({ code: null, timedOut, error }));
|
|
7374
|
+
child.on("close", (code) => done({ code, timedOut }));
|
|
7375
|
+
child.on("exit", (code) => setTimeout(() => {
|
|
7376
|
+
child.stdout.destroy();
|
|
7377
|
+
child.stderr.destroy();
|
|
7378
|
+
done({ code, timedOut });
|
|
7379
|
+
}, 2000).unref());
|
|
7380
|
+
});
|
|
7381
|
+
}
|
|
7382
|
+
async function runShell(opts) {
|
|
7383
|
+
opts.log.write(`$ ${opts.run}
|
|
7384
|
+
`);
|
|
7385
|
+
const r = await run({
|
|
7386
|
+
cmd: "/bin/sh",
|
|
7387
|
+
args: ["-c", opts.run],
|
|
7388
|
+
cwd: opts.cwd,
|
|
7389
|
+
env: opts.env,
|
|
7390
|
+
timeoutMs: opts.timeoutMs,
|
|
7391
|
+
onStdout: (c) => opts.log.write(c),
|
|
7392
|
+
onStderr: (c) => opts.log.write(c)
|
|
7393
|
+
});
|
|
7394
|
+
let error;
|
|
7395
|
+
if (r.error)
|
|
7396
|
+
error = `could not start /bin/sh: ${r.error.message}`;
|
|
7397
|
+
else if (r.timedOut)
|
|
7398
|
+
error = `timed out after ${opts.timeoutMs / 1000}s`;
|
|
7399
|
+
else if (r.code === null)
|
|
7400
|
+
error = "killed by a signal";
|
|
7401
|
+
opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
|
|
7402
|
+
|
|
7403
|
+
`);
|
|
7404
|
+
return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
|
|
7405
|
+
}
|
|
7406
|
+
|
|
7407
|
+
// src/home.ts
|
|
7270
7408
|
function homeEnv(home) {
|
|
7271
7409
|
return {
|
|
7272
7410
|
HOME: home,
|
|
@@ -7287,6 +7425,23 @@ function makeRunHome(dir) {
|
|
|
7287
7425
|
`).join(""));
|
|
7288
7426
|
return { home, envFile };
|
|
7289
7427
|
}
|
|
7428
|
+
async function makeRunKeychain(home) {
|
|
7429
|
+
const keychain = join2(home, "Library", "Keychains", "login.keychain-db");
|
|
7430
|
+
const env = { ...hostEnv(), HOME: home };
|
|
7431
|
+
mkdirSync(join2(home, "Library", "Keychains"), { recursive: true });
|
|
7432
|
+
for (const args of [
|
|
7433
|
+
["create-keychain", "-p", "", keychain],
|
|
7434
|
+
["set-keychain-settings", keychain]
|
|
7435
|
+
]) {
|
|
7436
|
+
try {
|
|
7437
|
+
await promisify(execFile)("security", args, { env, timeout: 30000 });
|
|
7438
|
+
} catch (e) {
|
|
7439
|
+
const err = e;
|
|
7440
|
+
return `could not create the run's login keychain (security ${args[0]} ${keychain}): ${err.stderr?.trim() || err.message}`;
|
|
7441
|
+
}
|
|
7442
|
+
}
|
|
7443
|
+
return null;
|
|
7444
|
+
}
|
|
7290
7445
|
|
|
7291
7446
|
// src/harnesses/codex.ts
|
|
7292
7447
|
var FLAGS2 = [
|
|
@@ -7879,6 +8034,7 @@ function validHarnesses() {
|
|
|
7879
8034
|
// src/config.ts
|
|
7880
8035
|
var CONFIG_FILE = "firegoat.yaml";
|
|
7881
8036
|
var DEFAULT_TIMEOUT = 600;
|
|
8037
|
+
var DEFAULT_PARALLEL = 4;
|
|
7882
8038
|
|
|
7883
8039
|
class ConfigError extends Error {
|
|
7884
8040
|
}
|
|
@@ -7916,6 +8072,7 @@ function parseConfig(text) {
|
|
|
7916
8072
|
fail("(root)", "must be a mapping");
|
|
7917
8073
|
onlyKeys(raw, "(root)", [
|
|
7918
8074
|
"runs",
|
|
8075
|
+
"parallel",
|
|
7919
8076
|
"harness",
|
|
7920
8077
|
"model",
|
|
7921
8078
|
"build",
|
|
@@ -7927,6 +8084,7 @@ function parseConfig(text) {
|
|
|
7927
8084
|
"tests"
|
|
7928
8085
|
]);
|
|
7929
8086
|
const runs = posInt(raw.runs, "runs", 1);
|
|
8087
|
+
const parallel = posInt(raw.parallel, "parallel", DEFAULT_PARALLEL);
|
|
7930
8088
|
const harnesses = raw.harness === undefined ? [DEFAULT_HARNESS] : nameList(raw.harness, "harness", "a harness name");
|
|
7931
8089
|
harnesses.forEach((h, i) => {
|
|
7932
8090
|
if (!HARNESSES.has(h))
|
|
@@ -7990,6 +8148,7 @@ function parseConfig(text) {
|
|
|
7990
8148
|
});
|
|
7991
8149
|
return {
|
|
7992
8150
|
runs,
|
|
8151
|
+
parallel,
|
|
7993
8152
|
harnesses,
|
|
7994
8153
|
models,
|
|
7995
8154
|
build,
|
|
@@ -8173,6 +8332,7 @@ import { randomBytes as randomBytes2 } from "node:crypto";
|
|
|
8173
8332
|
import {
|
|
8174
8333
|
appendFileSync,
|
|
8175
8334
|
createWriteStream as createWriteStream2,
|
|
8335
|
+
existsSync as existsSync6,
|
|
8176
8336
|
mkdirSync as mkdirSync5,
|
|
8177
8337
|
writeFileSync as writeFileSync5
|
|
8178
8338
|
} from "node:fs";
|
|
@@ -8530,142 +8690,6 @@ function ensureFiregoatDir(root) {
|
|
|
8530
8690
|
// src/git.ts
|
|
8531
8691
|
import { spawnSync as spawnSync3 } from "node:child_process";
|
|
8532
8692
|
import { basename, dirname } from "node:path";
|
|
8533
|
-
|
|
8534
|
-
// src/process.ts
|
|
8535
|
-
import { execFileSync, spawn } from "node:child_process";
|
|
8536
|
-
var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
|
|
8537
|
-
var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
|
|
8538
|
-
var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
|
|
8539
|
-
function hostEnv() {
|
|
8540
|
-
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
|
|
8541
|
-
}
|
|
8542
|
-
var GRACE_MS = 5000;
|
|
8543
|
-
var liveGroups = new Set;
|
|
8544
|
-
var signalsHooked = false;
|
|
8545
|
-
function hookSignals() {
|
|
8546
|
-
if (signalsHooked)
|
|
8547
|
-
return;
|
|
8548
|
-
signalsHooked = true;
|
|
8549
|
-
const onSignal = (signal) => {
|
|
8550
|
-
for (const g of liveGroups)
|
|
8551
|
-
try {
|
|
8552
|
-
process.kill(-g, "SIGKILL");
|
|
8553
|
-
} catch {}
|
|
8554
|
-
process.kill(process.pid, signal);
|
|
8555
|
-
};
|
|
8556
|
-
for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
|
|
8557
|
-
process.once(s, onSignal);
|
|
8558
|
-
}
|
|
8559
|
-
function forgetGroup(pgid) {
|
|
8560
|
-
liveGroups.delete(pgid);
|
|
8561
|
-
}
|
|
8562
|
-
function processTree(pid) {
|
|
8563
|
-
const children = new Map;
|
|
8564
|
-
const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
|
|
8565
|
-
encoding: "utf8"
|
|
8566
|
-
});
|
|
8567
|
-
for (const line of ps.split(`
|
|
8568
|
-
`)) {
|
|
8569
|
-
const [child, parent] = line.trim().split(/\s+/).map(Number);
|
|
8570
|
-
if (!child || parent === undefined)
|
|
8571
|
-
continue;
|
|
8572
|
-
children.set(parent, [...children.get(parent) ?? [], child]);
|
|
8573
|
-
}
|
|
8574
|
-
const tree = [pid];
|
|
8575
|
-
for (let i = 0;i < tree.length; i++)
|
|
8576
|
-
tree.push(...children.get(tree[i]) ?? []);
|
|
8577
|
-
return tree;
|
|
8578
|
-
}
|
|
8579
|
-
function killTree(pid, group) {
|
|
8580
|
-
if (group)
|
|
8581
|
-
try {
|
|
8582
|
-
process.kill(-pid, "SIGKILL");
|
|
8583
|
-
} catch {}
|
|
8584
|
-
let tree = [pid];
|
|
8585
|
-
let problem;
|
|
8586
|
-
try {
|
|
8587
|
-
tree = processTree(pid);
|
|
8588
|
-
} catch (e) {
|
|
8589
|
-
problem = e.message;
|
|
8590
|
-
}
|
|
8591
|
-
for (const p of tree)
|
|
8592
|
-
try {
|
|
8593
|
-
process.kill(p, "SIGKILL");
|
|
8594
|
-
} catch {}
|
|
8595
|
-
return problem;
|
|
8596
|
-
}
|
|
8597
|
-
function run(opts) {
|
|
8598
|
-
return new Promise((resolve) => {
|
|
8599
|
-
const detached = opts.detached === true;
|
|
8600
|
-
if (detached)
|
|
8601
|
-
hookSignals();
|
|
8602
|
-
const child = spawn(opts.cmd, opts.args, {
|
|
8603
|
-
cwd: opts.cwd,
|
|
8604
|
-
env: opts.env,
|
|
8605
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
8606
|
-
detached
|
|
8607
|
-
});
|
|
8608
|
-
if (child.pid !== undefined) {
|
|
8609
|
-
if (detached)
|
|
8610
|
-
liveGroups.add(child.pid);
|
|
8611
|
-
opts.onSpawn?.(child.pid);
|
|
8612
|
-
}
|
|
8613
|
-
let timedOut = false;
|
|
8614
|
-
let settled = false;
|
|
8615
|
-
const timer = setTimeout(() => {
|
|
8616
|
-
timedOut = true;
|
|
8617
|
-
if (child.pid === undefined)
|
|
8618
|
-
return;
|
|
8619
|
-
const problem = killTree(child.pid, detached);
|
|
8620
|
-
if (problem)
|
|
8621
|
-
opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
|
|
8622
|
-
`));
|
|
8623
|
-
}, opts.timeoutMs);
|
|
8624
|
-
const done = (r) => {
|
|
8625
|
-
if (settled)
|
|
8626
|
-
return;
|
|
8627
|
-
settled = true;
|
|
8628
|
-
clearTimeout(timer);
|
|
8629
|
-
resolve(r);
|
|
8630
|
-
};
|
|
8631
|
-
child.stdout.on("data", opts.onStdout);
|
|
8632
|
-
child.stderr.on("data", opts.onStderr);
|
|
8633
|
-
opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
|
|
8634
|
-
child.on("error", (error) => done({ code: null, timedOut, error }));
|
|
8635
|
-
child.on("close", (code) => done({ code, timedOut }));
|
|
8636
|
-
child.on("exit", (code) => setTimeout(() => {
|
|
8637
|
-
child.stdout.destroy();
|
|
8638
|
-
child.stderr.destroy();
|
|
8639
|
-
done({ code, timedOut });
|
|
8640
|
-
}, 2000).unref());
|
|
8641
|
-
});
|
|
8642
|
-
}
|
|
8643
|
-
async function runShell(opts) {
|
|
8644
|
-
opts.log.write(`$ ${opts.run}
|
|
8645
|
-
`);
|
|
8646
|
-
const r = await run({
|
|
8647
|
-
cmd: "/bin/sh",
|
|
8648
|
-
args: ["-c", opts.run],
|
|
8649
|
-
cwd: opts.cwd,
|
|
8650
|
-
env: opts.env,
|
|
8651
|
-
timeoutMs: opts.timeoutMs,
|
|
8652
|
-
onStdout: (c) => opts.log.write(c),
|
|
8653
|
-
onStderr: (c) => opts.log.write(c)
|
|
8654
|
-
});
|
|
8655
|
-
let error;
|
|
8656
|
-
if (r.error)
|
|
8657
|
-
error = `could not start /bin/sh: ${r.error.message}`;
|
|
8658
|
-
else if (r.timedOut)
|
|
8659
|
-
error = `timed out after ${opts.timeoutMs / 1000}s`;
|
|
8660
|
-
else if (r.code === null)
|
|
8661
|
-
error = "killed by a signal";
|
|
8662
|
-
opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
|
|
8663
|
-
|
|
8664
|
-
`);
|
|
8665
|
-
return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
|
|
8666
|
-
}
|
|
8667
|
-
|
|
8668
|
-
// src/git.ts
|
|
8669
8693
|
function git(dir, args) {
|
|
8670
8694
|
const r = spawnSync3("git", args, {
|
|
8671
8695
|
cwd: dir,
|
|
@@ -8834,6 +8858,9 @@ firegoat.yaml, committed.
|
|
|
8834
8858
|
|
|
8835
8859
|
firegoat.yaml reference:
|
|
8836
8860
|
runs: 1 Runs per test
|
|
8861
|
+
parallel: 4 Runs at once (default 4; 1 runs them one after
|
|
8862
|
+
another, e.g. when tests share one cloud
|
|
8863
|
+
account); eval --parallel overrides
|
|
8837
8864
|
harness: <name> | [<name>, ...]
|
|
8838
8865
|
The agent CLI: claude-code (default), codex-cli
|
|
8839
8866
|
or grok-cli.
|
|
@@ -8946,7 +8973,11 @@ firegoat.yaml reference:
|
|
|
8946
8973
|
Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
|
|
8947
8974
|
transcript, greppable line by line (format: eval --help, Transcript). So
|
|
8948
8975
|
'^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
|
|
8949
|
-
$FIREGOAT_SESSION (the harness's full session file, JSON lines).
|
|
8976
|
+
$FIREGOAT_SESSION (the harness's full session file, JSON lines). cleanup gets
|
|
8977
|
+
$FIREGOAT_TRANSCRIPT once the agent has run (also after a claude error or a
|
|
8978
|
+
timeout), and $FIREGOAT_SESSION when the harness saved a session file; neither
|
|
8979
|
+
is set when install failed before the agent started, so a cleanup that reads
|
|
8980
|
+
them should fail on an unset one: grep ... "\${FIREGOAT_TRANSCRIPT:?}".
|
|
8950
8981
|
|
|
8951
8982
|
The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
|
|
8952
8983
|
(npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
|
|
@@ -9102,10 +9133,13 @@ Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
|
|
|
9102
9133
|
$FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
|
|
9103
9134
|
collide. claude itself keeps your real HOME (its login is there) and sources
|
|
9104
9135
|
home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
|
|
9105
|
-
checks and cleanup get the run's HOME directly; build keeps yours.
|
|
9106
|
-
|
|
9107
|
-
|
|
9108
|
-
|
|
9136
|
+
checks and cleanup get the run's HOME directly; build keeps yours. On macOS the run's
|
|
9137
|
+
home has its own empty, password-less login keychain (Library/Keychains), so a
|
|
9138
|
+
browser the agent starts doesn't ask you for one; your keychain is never touched.
|
|
9139
|
+
Limits: claude's own Read/Write tools expand ~ to your real home; getpwuid lookups
|
|
9140
|
+
(os.userInfo, python pwd) still see it; keychain logins (gh, docker, git credential
|
|
9141
|
+
helpers) look logged out inside a run, so log tools in from install: or with env:
|
|
9142
|
+
tokens; the
|
|
9109
9143
|
fresh home has no .gitconfig, so set user.name/user.email in install: if the agent
|
|
9110
9144
|
commits; claude's shell snapshot still sources your real ~/.zshrc.
|
|
9111
9145
|
|
|
@@ -9129,14 +9163,16 @@ firegoat.yaml, fixtures and build: stay in the current folder.
|
|
|
9129
9163
|
--model NAME The harness's --model, replacing model: in firegoat.yaml;
|
|
9130
9164
|
repeat it (--model a --model b) to run each test runs times
|
|
9131
9165
|
per model
|
|
9132
|
-
--parallel N Runs at once (
|
|
9166
|
+
--parallel N Runs at once (overrides parallel: in firegoat.yaml;
|
|
9167
|
+
default 4; 1 runs them one after another).
|
|
9133
9168
|
Each run has its own HOME; state outside it (a fixed port,
|
|
9134
9169
|
/tmp paths) can still collide: use 1 then
|
|
9135
9170
|
--var KEY=value Fills {{vars.KEY}} (prompts, checks, build, install,
|
|
9136
9171
|
cleanup); env for build, install, checks and cleanup
|
|
9137
9172
|
--json The summary as JSON on stdout (or {error, exitCode} if eval
|
|
9138
9173
|
stops early); progress goes to stderr. The verdict comes
|
|
9139
|
-
last, after runs: totals {pass, fail, error},
|
|
9174
|
+
last, after runs: totals {pass, fail, error}, tier
|
|
9175
|
+
(S/A/B/C, as printed; null when no run was graded), status
|
|
9140
9176
|
("pass", "fail" or "error" = not graded) and exitCode
|
|
9141
9177
|
--no-upload Keep runs local even when logged in to firegoat.ai
|
|
9142
9178
|
--dry-run Check firegoat.yaml as eval would (test names, --var,
|
|
@@ -9174,7 +9210,7 @@ Output: .firegoat/runs/<runId>/
|
|
|
9174
9210
|
ran), git {branch, commit, dirty} (no git outside a
|
|
9175
9211
|
repo), harnesses, upload {origin, email} (null when
|
|
9176
9212
|
not uploading), runs (every run.json), then the
|
|
9177
|
-
verdict: totals, status and exitCode
|
|
9213
|
+
verdict: totals, tier, status and exitCode
|
|
9178
9214
|
../index.jsonl One line per eval: runId, startedAt, dir, branch,
|
|
9179
9215
|
commit, harnesses, passed, failed, notGraded, cost (null when
|
|
9180
9216
|
any run's cost is unknown), uploaded, uploadFailed
|
|
@@ -9191,7 +9227,8 @@ Output: .firegoat/runs/<runId>/
|
|
|
9191
9227
|
claudeSession (the session.jsonl uploaded: claude's
|
|
9192
9228
|
own file, else <test>/<n>/session.jsonl made from the
|
|
9193
9229
|
stream) and sessionId (its id, the trace's), stderr,
|
|
9194
|
-
checksLog, cleanupLog;
|
|
9230
|
+
checksLog, cleanupLog; turns and cost (null when the
|
|
9231
|
+
harness reports none), durationMs, startedAt; upload ({status: "uploaded",
|
|
9195
9232
|
url} | {status: "failed", error} | {status:
|
|
9196
9233
|
"skipped", reason}, null when not uploading)
|
|
9197
9234
|
<test>/<n>/transcript.txt What the agent did, below
|
|
@@ -10543,9 +10580,9 @@ async function uploadRun(t, r) {
|
|
|
10543
10580
|
run: clip(c.run, MAX_TEXT, "…"),
|
|
10544
10581
|
code: c.code
|
|
10545
10582
|
})),
|
|
10546
|
-
num_turns: r.
|
|
10583
|
+
num_turns: r.turns,
|
|
10547
10584
|
cost: r.cost,
|
|
10548
|
-
duration_ms: r.
|
|
10585
|
+
duration_ms: r.durationMs,
|
|
10549
10586
|
harness: r.harness,
|
|
10550
10587
|
model,
|
|
10551
10588
|
...git && { git }
|
|
@@ -10624,7 +10661,6 @@ function parseArgs(args) {
|
|
|
10624
10661
|
names: [],
|
|
10625
10662
|
models: [],
|
|
10626
10663
|
harnesses: [],
|
|
10627
|
-
parallel: 4,
|
|
10628
10664
|
json: false,
|
|
10629
10665
|
noUpload: false,
|
|
10630
10666
|
dryRun: false,
|
|
@@ -10835,9 +10871,9 @@ async function runOne(opts) {
|
|
|
10835
10871
|
cleanup: [],
|
|
10836
10872
|
cleanupLog: join10(dir, "cleanup.txt"),
|
|
10837
10873
|
leftover: [],
|
|
10838
|
-
|
|
10874
|
+
turns: null,
|
|
10839
10875
|
cost: null,
|
|
10840
|
-
|
|
10876
|
+
durationMs: 0,
|
|
10841
10877
|
startedAt: new Date(started).toISOString(),
|
|
10842
10878
|
upload: null
|
|
10843
10879
|
};
|
|
@@ -10858,13 +10894,17 @@ async function runOne(opts) {
|
|
|
10858
10894
|
r.cleanup = await runAllSteps({
|
|
10859
10895
|
commands: steps.cleanup,
|
|
10860
10896
|
cwd: workdir,
|
|
10861
|
-
env:
|
|
10897
|
+
env: {
|
|
10898
|
+
...stepEnv,
|
|
10899
|
+
...existsSync6(r.transcript) && { FIREGOAT_TRANSCRIPT: r.transcript },
|
|
10900
|
+
...r.session !== null && { FIREGOAT_SESSION: r.session }
|
|
10901
|
+
},
|
|
10862
10902
|
logPath: r.cleanupLog
|
|
10863
10903
|
});
|
|
10864
10904
|
r.leftover = await stopLeftovers({ group: agent.group, workdir });
|
|
10865
10905
|
r.status = status;
|
|
10866
10906
|
r.reason = reason;
|
|
10867
|
-
r.
|
|
10907
|
+
r.durationMs = Date.now() - started;
|
|
10868
10908
|
writeRunJson(r);
|
|
10869
10909
|
return r;
|
|
10870
10910
|
}
|
|
@@ -10874,6 +10914,11 @@ function withoutOldPwd(env) {
|
|
|
10874
10914
|
async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
|
|
10875
10915
|
const { test, n } = opts;
|
|
10876
10916
|
const { dir, workdir } = r;
|
|
10917
|
+
if (process.platform === "darwin") {
|
|
10918
|
+
const error = await makeRunKeychain(r.home);
|
|
10919
|
+
if (error)
|
|
10920
|
+
return ["error", `not graded: ${error}`];
|
|
10921
|
+
}
|
|
10877
10922
|
if (steps.install.length > 0) {
|
|
10878
10923
|
const error = await runSteps({
|
|
10879
10924
|
what: "install",
|
|
@@ -10947,7 +10992,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
|
|
|
10947
10992
|
final: outcome.final,
|
|
10948
10993
|
workdir
|
|
10949
10994
|
});
|
|
10950
|
-
r.
|
|
10995
|
+
r.turns = outcome.turns;
|
|
10951
10996
|
r.cost = outcome.cost;
|
|
10952
10997
|
if (!outcome.ok)
|
|
10953
10998
|
return ["error", `not graded: ${outcome.reason}`];
|
|
@@ -11045,6 +11090,7 @@ async function evalCommand(args, root) {
|
|
|
11045
11090
|
tests = tests.filter((t) => opts.names.includes(t.name));
|
|
11046
11091
|
}
|
|
11047
11092
|
const runs = opts.runs ?? config.runs;
|
|
11093
|
+
const parallel = opts.parallel ?? config.parallel;
|
|
11048
11094
|
const models = opts.models.length > 0 ? opts.models : config.models;
|
|
11049
11095
|
const harnessNames = opts.harnesses.length > 0 ? opts.harnesses : config.harnesses;
|
|
11050
11096
|
if (opts.harnessBin !== undefined && harnessNames.length !== 1)
|
|
@@ -11055,7 +11101,7 @@ async function evalCommand(args, root) {
|
|
|
11055
11101
|
testCount: tests.length,
|
|
11056
11102
|
runCount: tests.length * runs * Math.max(models.length, 1) * harnesses.length,
|
|
11057
11103
|
modelCount: models.length,
|
|
11058
|
-
parallel
|
|
11104
|
+
parallel
|
|
11059
11105
|
});
|
|
11060
11106
|
const vars = { ...config.vars, ...opts.vars };
|
|
11061
11107
|
const missing = [
|
|
@@ -11215,11 +11261,11 @@ async function evalCommand(args, root) {
|
|
|
11215
11261
|
results[i] = r;
|
|
11216
11262
|
Object.assign(row, {
|
|
11217
11263
|
state: r.status,
|
|
11218
|
-
reportedTurns: r.
|
|
11264
|
+
reportedTurns: r.turns,
|
|
11219
11265
|
reason: r.reason,
|
|
11220
11266
|
checks: r.checks,
|
|
11221
11267
|
cost: r.cost,
|
|
11222
|
-
seconds: Math.round(r.
|
|
11268
|
+
seconds: Math.round(r.durationMs / 1000),
|
|
11223
11269
|
command: undefined
|
|
11224
11270
|
});
|
|
11225
11271
|
ui.update();
|
|
@@ -11237,7 +11283,7 @@ async function evalCommand(args, root) {
|
|
|
11237
11283
|
while (next < rows.length)
|
|
11238
11284
|
await runAt(next++);
|
|
11239
11285
|
};
|
|
11240
|
-
await Promise.all(Array.from({ length: Math.min(
|
|
11286
|
+
await Promise.all(Array.from({ length: Math.min(parallel, rows.length) }, worker));
|
|
11241
11287
|
const totals = { pass: 0, fail: 0, error: 0 };
|
|
11242
11288
|
for (const r of results)
|
|
11243
11289
|
totals[r.status]++;
|
|
@@ -11266,6 +11312,7 @@ async function evalCommand(args, root) {
|
|
|
11266
11312
|
upload: upload && { origin: upload.origin, email: upload.email },
|
|
11267
11313
|
runs: results,
|
|
11268
11314
|
totals,
|
|
11315
|
+
tier: tier(rows),
|
|
11269
11316
|
status,
|
|
11270
11317
|
exitCode
|
|
11271
11318
|
};
|
|
@@ -11586,7 +11633,7 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
|
|
|
11586
11633
|
|
|
11587
11634
|
// src/init.ts
|
|
11588
11635
|
import { spawn as spawn4, spawnSync as spawnSync4 } from "node:child_process";
|
|
11589
|
-
import { existsSync as
|
|
11636
|
+
import { existsSync as existsSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
11590
11637
|
import { basename as basename3, join as join11 } from "node:path";
|
|
11591
11638
|
|
|
11592
11639
|
// src/login.ts
|
|
@@ -11900,6 +11947,10 @@ function template(product) {
|
|
|
11900
11947
|
# How many times to run each test.
|
|
11901
11948
|
runs: 1
|
|
11902
11949
|
|
|
11950
|
+
# Optional: how many runs at once (default 4). 1 runs them one after another,
|
|
11951
|
+
# e.g. when tests share one cloud account. eval --parallel N overrides it.
|
|
11952
|
+
# parallel: 1
|
|
11953
|
+
|
|
11903
11954
|
# Optional: the agent CLI each test runs on: claude-code (default), codex-cli or grok-cli.
|
|
11904
11955
|
# A list runs every test \`runs\` times per harness. eval --harness NAME overrides it.
|
|
11905
11956
|
# harness: [claude-code, codex-cli]
|
|
@@ -12162,7 +12213,7 @@ ${INIT_HELP}`);
|
|
|
12162
12213
|
}
|
|
12163
12214
|
ensureFiregoatDir(cwd);
|
|
12164
12215
|
const configPath = join11(cwd, CONFIG_FILE);
|
|
12165
|
-
const keep =
|
|
12216
|
+
const keep = existsSync7(configPath) && !force;
|
|
12166
12217
|
if (!keep)
|
|
12167
12218
|
writeFileSync6(configPath, template(basename3(cwd)));
|
|
12168
12219
|
if (ui) {
|