firegoat 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/cli.js +211 -160
  2. package/package.json +1 -1
package/dist/cli.js CHANGED
@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
6947
6947
  // package.json
6948
6948
  var package_default = {
6949
6949
  name: "firegoat",
6950
- version: "0.28.0",
6950
+ version: "0.29.0",
6951
6951
  repository: {
6952
6952
  type: "git",
6953
6953
  url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7265,8 +7265,146 @@ import { homedir } from "node:os";
7265
7265
  import { join as join3 } from "node:path";
7266
7266
 
7267
7267
  // src/home.ts
7268
+ import { execFile } from "node:child_process";
7268
7269
  import { mkdirSync, writeFileSync } from "node:fs";
7269
7270
  import { join as join2 } from "node:path";
7271
+ import { promisify } from "node:util";
7272
+
7273
+ // src/process.ts
7274
+ import { execFileSync, spawn } from "node:child_process";
7275
+ var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
7276
+ var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
7277
+ var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
7278
+ function hostEnv() {
7279
+ return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
7280
+ }
7281
+ var GRACE_MS = 5000;
7282
+ var liveGroups = new Set;
7283
+ var signalsHooked = false;
7284
+ function hookSignals() {
7285
+ if (signalsHooked)
7286
+ return;
7287
+ signalsHooked = true;
7288
+ const onSignal = (signal) => {
7289
+ for (const g of liveGroups)
7290
+ try {
7291
+ process.kill(-g, "SIGKILL");
7292
+ } catch {}
7293
+ process.kill(process.pid, signal);
7294
+ };
7295
+ for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
7296
+ process.once(s, onSignal);
7297
+ }
7298
+ function forgetGroup(pgid) {
7299
+ liveGroups.delete(pgid);
7300
+ }
7301
+ function processTree(pid) {
7302
+ const children = new Map;
7303
+ const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
7304
+ encoding: "utf8"
7305
+ });
7306
+ for (const line of ps.split(`
7307
+ `)) {
7308
+ const [child, parent] = line.trim().split(/\s+/).map(Number);
7309
+ if (!child || parent === undefined)
7310
+ continue;
7311
+ children.set(parent, [...children.get(parent) ?? [], child]);
7312
+ }
7313
+ const tree = [pid];
7314
+ for (let i = 0;i < tree.length; i++)
7315
+ tree.push(...children.get(tree[i]) ?? []);
7316
+ return tree;
7317
+ }
7318
+ function killTree(pid, group) {
7319
+ if (group)
7320
+ try {
7321
+ process.kill(-pid, "SIGKILL");
7322
+ } catch {}
7323
+ let tree = [pid];
7324
+ let problem;
7325
+ try {
7326
+ tree = processTree(pid);
7327
+ } catch (e) {
7328
+ problem = e.message;
7329
+ }
7330
+ for (const p of tree)
7331
+ try {
7332
+ process.kill(p, "SIGKILL");
7333
+ } catch {}
7334
+ return problem;
7335
+ }
7336
+ function run(opts) {
7337
+ return new Promise((resolve) => {
7338
+ const detached = opts.detached === true;
7339
+ if (detached)
7340
+ hookSignals();
7341
+ const child = spawn(opts.cmd, opts.args, {
7342
+ cwd: opts.cwd,
7343
+ env: opts.env,
7344
+ stdio: ["ignore", "pipe", "pipe"],
7345
+ detached
7346
+ });
7347
+ if (child.pid !== undefined) {
7348
+ if (detached)
7349
+ liveGroups.add(child.pid);
7350
+ opts.onSpawn?.(child.pid);
7351
+ }
7352
+ let timedOut = false;
7353
+ let settled = false;
7354
+ const timer = setTimeout(() => {
7355
+ timedOut = true;
7356
+ if (child.pid === undefined)
7357
+ return;
7358
+ const problem = killTree(child.pid, detached);
7359
+ if (problem)
7360
+ opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
7361
+ `));
7362
+ }, opts.timeoutMs);
7363
+ const done = (r) => {
7364
+ if (settled)
7365
+ return;
7366
+ settled = true;
7367
+ clearTimeout(timer);
7368
+ resolve(r);
7369
+ };
7370
+ child.stdout.on("data", opts.onStdout);
7371
+ child.stderr.on("data", opts.onStderr);
7372
+ opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
7373
+ child.on("error", (error) => done({ code: null, timedOut, error }));
7374
+ child.on("close", (code) => done({ code, timedOut }));
7375
+ child.on("exit", (code) => setTimeout(() => {
7376
+ child.stdout.destroy();
7377
+ child.stderr.destroy();
7378
+ done({ code, timedOut });
7379
+ }, 2000).unref());
7380
+ });
7381
+ }
7382
+ async function runShell(opts) {
7383
+ opts.log.write(`$ ${opts.run}
7384
+ `);
7385
+ const r = await run({
7386
+ cmd: "/bin/sh",
7387
+ args: ["-c", opts.run],
7388
+ cwd: opts.cwd,
7389
+ env: opts.env,
7390
+ timeoutMs: opts.timeoutMs,
7391
+ onStdout: (c) => opts.log.write(c),
7392
+ onStderr: (c) => opts.log.write(c)
7393
+ });
7394
+ let error;
7395
+ if (r.error)
7396
+ error = `could not start /bin/sh: ${r.error.message}`;
7397
+ else if (r.timedOut)
7398
+ error = `timed out after ${opts.timeoutMs / 1000}s`;
7399
+ else if (r.code === null)
7400
+ error = "killed by a signal";
7401
+ opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
7402
+
7403
+ `);
7404
+ return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
7405
+ }
7406
+
7407
+ // src/home.ts
7270
7408
  function homeEnv(home) {
7271
7409
  return {
7272
7410
  HOME: home,
@@ -7287,6 +7425,23 @@ function makeRunHome(dir) {
7287
7425
  `).join(""));
7288
7426
  return { home, envFile };
7289
7427
  }
7428
+ async function makeRunKeychain(home) {
7429
+ const keychain = join2(home, "Library", "Keychains", "login.keychain-db");
7430
+ const env = { ...hostEnv(), HOME: home };
7431
+ mkdirSync(join2(home, "Library", "Keychains"), { recursive: true });
7432
+ for (const args of [
7433
+ ["create-keychain", "-p", "", keychain],
7434
+ ["set-keychain-settings", keychain]
7435
+ ]) {
7436
+ try {
7437
+ await promisify(execFile)("security", args, { env, timeout: 30000 });
7438
+ } catch (e) {
7439
+ const err = e;
7440
+ return `could not create the run's login keychain (security ${args[0]} ${keychain}): ${err.stderr?.trim() || err.message}`;
7441
+ }
7442
+ }
7443
+ return null;
7444
+ }
7290
7445
 
7291
7446
  // src/harnesses/codex.ts
7292
7447
  var FLAGS2 = [
@@ -7879,6 +8034,7 @@ function validHarnesses() {
7879
8034
  // src/config.ts
7880
8035
  var CONFIG_FILE = "firegoat.yaml";
7881
8036
  var DEFAULT_TIMEOUT = 600;
8037
+ var DEFAULT_PARALLEL = 4;
7882
8038
 
7883
8039
  class ConfigError extends Error {
7884
8040
  }
@@ -7916,6 +8072,7 @@ function parseConfig(text) {
7916
8072
  fail("(root)", "must be a mapping");
7917
8073
  onlyKeys(raw, "(root)", [
7918
8074
  "runs",
8075
+ "parallel",
7919
8076
  "harness",
7920
8077
  "model",
7921
8078
  "build",
@@ -7927,6 +8084,7 @@ function parseConfig(text) {
7927
8084
  "tests"
7928
8085
  ]);
7929
8086
  const runs = posInt(raw.runs, "runs", 1);
8087
+ const parallel = posInt(raw.parallel, "parallel", DEFAULT_PARALLEL);
7930
8088
  const harnesses = raw.harness === undefined ? [DEFAULT_HARNESS] : nameList(raw.harness, "harness", "a harness name");
7931
8089
  harnesses.forEach((h, i) => {
7932
8090
  if (!HARNESSES.has(h))
@@ -7990,6 +8148,7 @@ function parseConfig(text) {
7990
8148
  });
7991
8149
  return {
7992
8150
  runs,
8151
+ parallel,
7993
8152
  harnesses,
7994
8153
  models,
7995
8154
  build,
@@ -8173,6 +8332,7 @@ import { randomBytes as randomBytes2 } from "node:crypto";
8173
8332
  import {
8174
8333
  appendFileSync,
8175
8334
  createWriteStream as createWriteStream2,
8335
+ existsSync as existsSync6,
8176
8336
  mkdirSync as mkdirSync5,
8177
8337
  writeFileSync as writeFileSync5
8178
8338
  } from "node:fs";
@@ -8530,142 +8690,6 @@ function ensureFiregoatDir(root) {
8530
8690
  // src/git.ts
8531
8691
  import { spawnSync as spawnSync3 } from "node:child_process";
8532
8692
  import { basename, dirname } from "node:path";
8533
-
8534
- // src/process.ts
8535
- import { execFileSync, spawn } from "node:child_process";
8536
- var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
8537
- var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
8538
- var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
8539
- function hostEnv() {
8540
- return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
8541
- }
8542
- var GRACE_MS = 5000;
8543
- var liveGroups = new Set;
8544
- var signalsHooked = false;
8545
- function hookSignals() {
8546
- if (signalsHooked)
8547
- return;
8548
- signalsHooked = true;
8549
- const onSignal = (signal) => {
8550
- for (const g of liveGroups)
8551
- try {
8552
- process.kill(-g, "SIGKILL");
8553
- } catch {}
8554
- process.kill(process.pid, signal);
8555
- };
8556
- for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
8557
- process.once(s, onSignal);
8558
- }
8559
- function forgetGroup(pgid) {
8560
- liveGroups.delete(pgid);
8561
- }
8562
- function processTree(pid) {
8563
- const children = new Map;
8564
- const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
8565
- encoding: "utf8"
8566
- });
8567
- for (const line of ps.split(`
8568
- `)) {
8569
- const [child, parent] = line.trim().split(/\s+/).map(Number);
8570
- if (!child || parent === undefined)
8571
- continue;
8572
- children.set(parent, [...children.get(parent) ?? [], child]);
8573
- }
8574
- const tree = [pid];
8575
- for (let i = 0;i < tree.length; i++)
8576
- tree.push(...children.get(tree[i]) ?? []);
8577
- return tree;
8578
- }
8579
- function killTree(pid, group) {
8580
- if (group)
8581
- try {
8582
- process.kill(-pid, "SIGKILL");
8583
- } catch {}
8584
- let tree = [pid];
8585
- let problem;
8586
- try {
8587
- tree = processTree(pid);
8588
- } catch (e) {
8589
- problem = e.message;
8590
- }
8591
- for (const p of tree)
8592
- try {
8593
- process.kill(p, "SIGKILL");
8594
- } catch {}
8595
- return problem;
8596
- }
8597
- function run(opts) {
8598
- return new Promise((resolve) => {
8599
- const detached = opts.detached === true;
8600
- if (detached)
8601
- hookSignals();
8602
- const child = spawn(opts.cmd, opts.args, {
8603
- cwd: opts.cwd,
8604
- env: opts.env,
8605
- stdio: ["ignore", "pipe", "pipe"],
8606
- detached
8607
- });
8608
- if (child.pid !== undefined) {
8609
- if (detached)
8610
- liveGroups.add(child.pid);
8611
- opts.onSpawn?.(child.pid);
8612
- }
8613
- let timedOut = false;
8614
- let settled = false;
8615
- const timer = setTimeout(() => {
8616
- timedOut = true;
8617
- if (child.pid === undefined)
8618
- return;
8619
- const problem = killTree(child.pid, detached);
8620
- if (problem)
8621
- opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
8622
- `));
8623
- }, opts.timeoutMs);
8624
- const done = (r) => {
8625
- if (settled)
8626
- return;
8627
- settled = true;
8628
- clearTimeout(timer);
8629
- resolve(r);
8630
- };
8631
- child.stdout.on("data", opts.onStdout);
8632
- child.stderr.on("data", opts.onStderr);
8633
- opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
8634
- child.on("error", (error) => done({ code: null, timedOut, error }));
8635
- child.on("close", (code) => done({ code, timedOut }));
8636
- child.on("exit", (code) => setTimeout(() => {
8637
- child.stdout.destroy();
8638
- child.stderr.destroy();
8639
- done({ code, timedOut });
8640
- }, 2000).unref());
8641
- });
8642
- }
8643
- async function runShell(opts) {
8644
- opts.log.write(`$ ${opts.run}
8645
- `);
8646
- const r = await run({
8647
- cmd: "/bin/sh",
8648
- args: ["-c", opts.run],
8649
- cwd: opts.cwd,
8650
- env: opts.env,
8651
- timeoutMs: opts.timeoutMs,
8652
- onStdout: (c) => opts.log.write(c),
8653
- onStderr: (c) => opts.log.write(c)
8654
- });
8655
- let error;
8656
- if (r.error)
8657
- error = `could not start /bin/sh: ${r.error.message}`;
8658
- else if (r.timedOut)
8659
- error = `timed out after ${opts.timeoutMs / 1000}s`;
8660
- else if (r.code === null)
8661
- error = "killed by a signal";
8662
- opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
8663
-
8664
- `);
8665
- return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
8666
- }
8667
-
8668
- // src/git.ts
8669
8693
  function git(dir, args) {
8670
8694
  const r = spawnSync3("git", args, {
8671
8695
  cwd: dir,
@@ -8834,6 +8858,9 @@ firegoat.yaml, committed.
8834
8858
 
8835
8859
  firegoat.yaml reference:
8836
8860
  runs: 1 Runs per test
8861
+ parallel: 4 Runs at once (default 4; 1 runs them one after
8862
+ another, e.g. when tests share one cloud
8863
+ account); eval --parallel overrides
8837
8864
  harness: <name> | [<name>, ...]
8838
8865
  The agent CLI: claude-code (default), codex-cli
8839
8866
  or grok-cli.
@@ -8946,7 +8973,11 @@ firegoat.yaml reference:
8946
8973
  Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
8947
8974
  transcript, greppable line by line (format: eval --help, Transcript). So
8948
8975
  '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
8949
- $FIREGOAT_SESSION (the harness's full session file, JSON lines).
8976
+ $FIREGOAT_SESSION (the harness's full session file, JSON lines). cleanup gets
8977
+ $FIREGOAT_TRANSCRIPT once the agent has run (also after a claude error or a
8978
+ timeout), and $FIREGOAT_SESSION when the harness saved a session file; neither
8979
+ is set when install failed before the agent started, so a cleanup that reads
8980
+ them should fail on an unset one: grep ... "\${FIREGOAT_TRANSCRIPT:?}".
8950
8981
 
8951
8982
  The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
8952
8983
  (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
@@ -9102,10 +9133,13 @@ Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
9102
9133
  $FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
9103
9134
  collide. claude itself keeps your real HOME (its login is there) and sources
9104
9135
  home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
9105
- checks and cleanup get the run's HOME directly; build keeps yours. Limits: claude's
9106
- own Read/Write tools expand ~ to your real home; getpwuid lookups (os.userInfo,
9107
- python pwd) still see it; keychain logins (gh, docker, git credential helpers) look
9108
- logged out inside a run, so log tools in from install: or with env: tokens; the
9136
+ checks and cleanup get the run's HOME directly; build keeps yours. On macOS the run's
9137
+ home has its own empty, password-less login keychain (Library/Keychains), so a
9138
+ browser the agent starts doesn't ask you for one; your keychain is never touched.
9139
+ Limits: claude's own Read/Write tools expand ~ to your real home; getpwuid lookups
9140
+ (os.userInfo, python pwd) still see it; keychain logins (gh, docker, git credential
9141
+ helpers) look logged out inside a run, so log tools in from install: or with env:
9142
+ tokens; the
9109
9143
  fresh home has no .gitconfig, so set user.name/user.email in install: if the agent
9110
9144
  commits; claude's shell snapshot still sources your real ~/.zshrc.
9111
9145
 
@@ -9129,14 +9163,16 @@ firegoat.yaml, fixtures and build: stay in the current folder.
9129
9163
  --model NAME The harness's --model, replacing model: in firegoat.yaml;
9130
9164
  repeat it (--model a --model b) to run each test runs times
9131
9165
  per model
9132
- --parallel N Runs at once (default 4; 1 runs them one after another).
9166
+ --parallel N Runs at once (overrides parallel: in firegoat.yaml;
9167
+ default 4; 1 runs them one after another).
9133
9168
  Each run has its own HOME; state outside it (a fixed port,
9134
9169
  /tmp paths) can still collide: use 1 then
9135
9170
  --var KEY=value Fills {{vars.KEY}} (prompts, checks, build, install,
9136
9171
  cleanup); env for build, install, checks and cleanup
9137
9172
  --json The summary as JSON on stdout (or {error, exitCode} if eval
9138
9173
  stops early); progress goes to stderr. The verdict comes
9139
- last, after runs: totals {pass, fail, error}, status
9174
+ last, after runs: totals {pass, fail, error}, tier
9175
+ (S/A/B/C, as printed; null when no run was graded), status
9140
9176
  ("pass", "fail" or "error" = not graded) and exitCode
9141
9177
  --no-upload Keep runs local even when logged in to firegoat.ai
9142
9178
  --dry-run Check firegoat.yaml as eval would (test names, --var,
@@ -9174,7 +9210,7 @@ Output: .firegoat/runs/<runId>/
9174
9210
  ran), git {branch, commit, dirty} (no git outside a
9175
9211
  repo), harnesses, upload {origin, email} (null when
9176
9212
  not uploading), runs (every run.json), then the
9177
- verdict: totals, status and exitCode
9213
+ verdict: totals, tier, status and exitCode
9178
9214
  ../index.jsonl One line per eval: runId, startedAt, dir, branch,
9179
9215
  commit, harnesses, passed, failed, notGraded, cost (null when
9180
9216
  any run's cost is unknown), uploaded, uploadFailed
@@ -9191,7 +9227,8 @@ Output: .firegoat/runs/<runId>/
9191
9227
  claudeSession (the session.jsonl uploaded: claude's
9192
9228
  own file, else <test>/<n>/session.jsonl made from the
9193
9229
  stream) and sessionId (its id, the trace's), stderr,
9194
- checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
9230
+ checksLog, cleanupLog; turns and cost (null when the
9231
+ harness reports none), durationMs, startedAt; upload ({status: "uploaded",
9195
9232
  url} | {status: "failed", error} | {status:
9196
9233
  "skipped", reason}, null when not uploading)
9197
9234
  <test>/<n>/transcript.txt What the agent did, below
@@ -10543,9 +10580,9 @@ async function uploadRun(t, r) {
10543
10580
  run: clip(c.run, MAX_TEXT, "…"),
10544
10581
  code: c.code
10545
10582
  })),
10546
- num_turns: r.num_turns,
10583
+ num_turns: r.turns,
10547
10584
  cost: r.cost,
10548
- duration_ms: r.duration_ms,
10585
+ duration_ms: r.durationMs,
10549
10586
  harness: r.harness,
10550
10587
  model,
10551
10588
  ...git && { git }
@@ -10624,7 +10661,6 @@ function parseArgs(args) {
10624
10661
  names: [],
10625
10662
  models: [],
10626
10663
  harnesses: [],
10627
- parallel: 4,
10628
10664
  json: false,
10629
10665
  noUpload: false,
10630
10666
  dryRun: false,
@@ -10835,9 +10871,9 @@ async function runOne(opts) {
10835
10871
  cleanup: [],
10836
10872
  cleanupLog: join10(dir, "cleanup.txt"),
10837
10873
  leftover: [],
10838
- num_turns: null,
10874
+ turns: null,
10839
10875
  cost: null,
10840
- duration_ms: 0,
10876
+ durationMs: 0,
10841
10877
  startedAt: new Date(started).toISOString(),
10842
10878
  upload: null
10843
10879
  };
@@ -10858,13 +10894,17 @@ async function runOne(opts) {
10858
10894
  r.cleanup = await runAllSteps({
10859
10895
  commands: steps.cleanup,
10860
10896
  cwd: workdir,
10861
- env: stepEnv,
10897
+ env: {
10898
+ ...stepEnv,
10899
+ ...existsSync6(r.transcript) && { FIREGOAT_TRANSCRIPT: r.transcript },
10900
+ ...r.session !== null && { FIREGOAT_SESSION: r.session }
10901
+ },
10862
10902
  logPath: r.cleanupLog
10863
10903
  });
10864
10904
  r.leftover = await stopLeftovers({ group: agent.group, workdir });
10865
10905
  r.status = status;
10866
10906
  r.reason = reason;
10867
- r.duration_ms = Date.now() - started;
10907
+ r.durationMs = Date.now() - started;
10868
10908
  writeRunJson(r);
10869
10909
  return r;
10870
10910
  }
@@ -10874,6 +10914,11 @@ function withoutOldPwd(env) {
10874
10914
  async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
10875
10915
  const { test, n } = opts;
10876
10916
  const { dir, workdir } = r;
10917
+ if (process.platform === "darwin") {
10918
+ const error = await makeRunKeychain(r.home);
10919
+ if (error)
10920
+ return ["error", `not graded: ${error}`];
10921
+ }
10877
10922
  if (steps.install.length > 0) {
10878
10923
  const error = await runSteps({
10879
10924
  what: "install",
@@ -10947,7 +10992,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
10947
10992
  final: outcome.final,
10948
10993
  workdir
10949
10994
  });
10950
- r.num_turns = outcome.turns;
10995
+ r.turns = outcome.turns;
10951
10996
  r.cost = outcome.cost;
10952
10997
  if (!outcome.ok)
10953
10998
  return ["error", `not graded: ${outcome.reason}`];
@@ -11045,6 +11090,7 @@ async function evalCommand(args, root) {
11045
11090
  tests = tests.filter((t) => opts.names.includes(t.name));
11046
11091
  }
11047
11092
  const runs = opts.runs ?? config.runs;
11093
+ const parallel = opts.parallel ?? config.parallel;
11048
11094
  const models = opts.models.length > 0 ? opts.models : config.models;
11049
11095
  const harnessNames = opts.harnesses.length > 0 ? opts.harnesses : config.harnesses;
11050
11096
  if (opts.harnessBin !== undefined && harnessNames.length !== 1)
@@ -11055,7 +11101,7 @@ async function evalCommand(args, root) {
11055
11101
  testCount: tests.length,
11056
11102
  runCount: tests.length * runs * Math.max(models.length, 1) * harnesses.length,
11057
11103
  modelCount: models.length,
11058
- parallel: opts.parallel
11104
+ parallel
11059
11105
  });
11060
11106
  const vars = { ...config.vars, ...opts.vars };
11061
11107
  const missing = [
@@ -11215,11 +11261,11 @@ async function evalCommand(args, root) {
11215
11261
  results[i] = r;
11216
11262
  Object.assign(row, {
11217
11263
  state: r.status,
11218
- reportedTurns: r.num_turns,
11264
+ reportedTurns: r.turns,
11219
11265
  reason: r.reason,
11220
11266
  checks: r.checks,
11221
11267
  cost: r.cost,
11222
- seconds: Math.round(r.duration_ms / 1000),
11268
+ seconds: Math.round(r.durationMs / 1000),
11223
11269
  command: undefined
11224
11270
  });
11225
11271
  ui.update();
@@ -11237,7 +11283,7 @@ async function evalCommand(args, root) {
11237
11283
  while (next < rows.length)
11238
11284
  await runAt(next++);
11239
11285
  };
11240
- await Promise.all(Array.from({ length: Math.min(opts.parallel, rows.length) }, worker));
11286
+ await Promise.all(Array.from({ length: Math.min(parallel, rows.length) }, worker));
11241
11287
  const totals = { pass: 0, fail: 0, error: 0 };
11242
11288
  for (const r of results)
11243
11289
  totals[r.status]++;
@@ -11266,6 +11312,7 @@ async function evalCommand(args, root) {
11266
11312
  upload: upload && { origin: upload.origin, email: upload.email },
11267
11313
  runs: results,
11268
11314
  totals,
11315
+ tier: tier(rows),
11269
11316
  status,
11270
11317
  exitCode
11271
11318
  };
@@ -11586,7 +11633,7 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
11586
11633
 
11587
11634
  // src/init.ts
11588
11635
  import { spawn as spawn4, spawnSync as spawnSync4 } from "node:child_process";
11589
- import { existsSync as existsSync6, writeFileSync as writeFileSync6 } from "node:fs";
11636
+ import { existsSync as existsSync7, writeFileSync as writeFileSync6 } from "node:fs";
11590
11637
  import { basename as basename3, join as join11 } from "node:path";
11591
11638
 
11592
11639
  // src/login.ts
@@ -11900,6 +11947,10 @@ function template(product) {
11900
11947
  # How many times to run each test.
11901
11948
  runs: 1
11902
11949
 
11950
+ # Optional: how many runs at once (default 4). 1 runs them one after another,
11951
+ # e.g. when tests share one cloud account. eval --parallel N overrides it.
11952
+ # parallel: 1
11953
+
11903
11954
  # Optional: the agent CLI each test runs on: claude-code (default), codex-cli or grok-cli.
11904
11955
  # A list runs every test \`runs\` times per harness. eval --harness NAME overrides it.
11905
11956
  # harness: [claude-code, codex-cli]
@@ -12162,7 +12213,7 @@ ${INIT_HELP}`);
12162
12213
  }
12163
12214
  ensureFiregoatDir(cwd);
12164
12215
  const configPath = join11(cwd, CONFIG_FILE);
12165
- const keep = existsSync6(configPath) && !force;
12216
+ const keep = existsSync7(configPath) && !force;
12166
12217
  if (!keep)
12167
12218
  writeFileSync6(configPath, template(basename3(cwd)));
12168
12219
  if (ui) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firegoat",
3
- "version": "0.28.0",
3
+ "version": "0.29.0",
4
4
  "repository": {
5
5
  "type": "git",
6
6
  "url": "git+https://github.com/firegoat-ai/firegoat-cli.git"