nubos-pilot 1.2.2 → 1.2.4

package/bin/np-tools/skill-audit.test.cjs

@@ -0,0 +1,86 @@
+'use strict';
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { run } = require('./skill-audit.cjs');
+const checkpoint = require('../../lib/checkpoint.cjs');
+
+function _mkRoot() {
+  const r = fs.mkdtempSync(path.join(os.tmpdir(), 'np-skill-cli-'));
+  fs.mkdirSync(path.join(r, '.nubos-pilot', 'checkpoints'), { recursive: true });
+  fs.writeFileSync(
+    path.join(r, '.nubos-pilot', 'STATE.md'),
+    '---\nschema_version: 2\ncurrent_phase: null\ncurrent_plan: null\ncurrent_task: null\n---\n',
+    'utf-8',
+  );
+  return r;
+}
+function _cap(cwd) {
+  const out = { text: '' }; const err = { text: '' };
+  return { cwd, stdout: { write: (s) => { out.text += s; return true; } }, stderr: { write: (s) => { err.text += s; return true; } }, out, err };
+}
+const TID = 'M001-S001-T0001';
+
+test('SC-1: expect then findings reports the unacked skill', () => {
+  const r = _mkRoot();
+  try {
+    checkpoint.startTask({ id: TID }, r);
+    assert.equal(run(['expect', '--task', TID, '--skills', 'np-api-design,np-encryption'], _cap(r)), 0);
+    const c = _cap(r);
+    assert.equal(run(['findings', '--task', TID], c), 0);
+    const parsed = JSON.parse(c.out.text);
+    assert.equal(parsed.findings.length, 1);
+    assert.deepEqual(parsed.findings[0].raw.missing_skills.sort(), ['np-api-design', 'np-encryption']);
+  } finally { fs.rmSync(r, { recursive: true, force: true }); }
+});
+
+test('SC-2: ack clears the finding for that skill', () => {
+  const r = _mkRoot();
+  try {
+    checkpoint.startTask({ id: TID }, r);
+    run(['expect', '--task', TID, '--skills', 'np-api-design'], _cap(r));
+    run(['ack', '--task', TID, '--skill', 'np-api-design'], _cap(r));
+    const c = _cap(r);
+    run(['findings', '--task', TID], c);
+    assert.equal(JSON.parse(c.out.text).findings.length, 0);
+  } finally { fs.rmSync(r, { recursive: true, force: true }); }
+});
+
+test('SC-3: invalid task id → error envelope exit 1', () => {
+  const c = _cap(process.cwd());
+  assert.equal(run(['ack', '--task', 'bogus', '--skill', 'x'], c), 1);
+  assert.match(c.err.text, /skill-audit-invalid-task-id/);
+});
+
+test('SC-4: ack without --skill → error envelope exit 1', () => {
+  const r = _mkRoot();
+  try {
+    checkpoint.startTask({ id: TID }, r);
+    const c = _cap(r);
+    assert.equal(run(['ack', '--task', TID], c), 1);
+    assert.match(c.err.text, /skill-audit-missing-skill/);
+  } finally { fs.rmSync(r, { recursive: true, force: true }); }
+});
+
+test('SC-5: unknown verb → exit 1; --help → exit 0', () => {
+  const c1 = _cap(process.cwd());
+  assert.equal(run(['bogus'], c1), 1);
+  assert.match(c1.err.text, /skill-audit-unknown-verb/);
+  const c2 = _cap(process.cwd());
+  assert.equal(run(['--help'], c2), 0);
+  assert.match(c2.out.text, /skill-audit/);
+});
+
+test('SC-6: expect with empty skills is a no-op (no findings)', () => {
+  const r = _mkRoot();
+  try {
+    checkpoint.startTask({ id: TID }, r);
+    assert.equal(run(['expect', '--task', TID, '--skills', ''], _cap(r)), 0);
+    const c = _cap(r);
+    run(['findings', '--task', TID], c);
+    assert.equal(JSON.parse(c.out.text).findings.length, 0);
+  } finally { fs.rmSync(r, { recursive: true, force: true }); }
+});

package/bin/np-tools/spawn-headless.cjs

@@ -8,6 +8,7 @@ const child_process = require('node:child_process');
 const { NubosPilotError, atomicWriteFileSync, appendJsonl, findProjectRoot } = require('../../lib/core.cjs');
 const runContext = require('../../lib/run-context.cjs');
 const safePath = require('../../lib/safe-path.cjs');
+const headlessGuard = require('../../lib/headless-guard.cjs');
 const args = require('./_args.cjs');
 
 function _sha256(s) {
@@ -171,6 +172,22 @@ function run(argv, ctx) {
   const stdout = context.stdout || process.stdout;
   const list = Array.isArray(argv) ? argv : [];
 
+  if (headlessGuard.isHeadless(process.env)) {
+    throw new NubosPilotError(
+      'spawn-headless-reentrant',
+      'refusing to spawn a nested headless `claude` (NUBOS_PILOT_HEADLESS is set) — recursion guard',
+      { depth: headlessGuard.currentDepth(process.env) },
+    );
+  }
+  if (headlessGuard.depthExceeded(process.env)) {
+    throw new NubosPilotError(
+      'spawn-headless-depth-exceeded',
+      'refusing to spawn headless `claude`: hook depth ' + headlessGuard.currentDepth(process.env)
+        + ' has reached the cap ' + headlessGuard.maxDepth(process.env) + ' (recursion guard)',
+      { depth: headlessGuard.currentDepth(process.env), max: headlessGuard.maxDepth(process.env) },
+    );
+  }
+
   const agent = args.getFlag(list, '--agent');
   if (!agent) {
     throw new NubosPilotError(
@@ -213,6 +230,21 @@ function run(argv, ctx) {
 
   const runId = runContext.getRunId();
 
+  let lockRoot;
+  try { lockRoot = findProjectRoot(cwd); }
+  catch { lockRoot = cwd; }
+  const lock = headlessGuard.tryAcquireSpawnLock(lockRoot, agent);
+  if (!lock.acquired) {
+    throw new NubosPilotError(
+      'spawn-headless-locked',
+      'another headless run for agent `' + agent + '` is already active in this project (concurrency guard)',
+      { agent, holder: lock.holder || null },
+    );
+  }
+
+  const childEnv = _filterSpawnEnv(process.env);
+  Object.assign(childEnv, headlessGuard.childSpawnEnv(process.env));
+
   const bin = _claudeBinary();
   const claudeArgs = ['-p', '--output-format', 'json'];
   const startedAt = new Date().toISOString();
@@ -224,7 +256,7 @@ function run(argv, ctx) {
       timeout: timeoutMs,
       maxBuffer: 64 * 1024 * 1024,
       encoding: 'utf-8',
-      env: _filterSpawnEnv(process.env),
+      env: childEnv,
       killSignal: 'SIGKILL',
     });
   } catch (err) {
@@ -233,6 +265,8 @@ function run(argv, ctx) {
       'failed to spawn `' + bin + '`: ' + (err && err.message),
       { bin, cause: err && err.code },
     );
+  } finally {
+    lock.release();
   }
   if (result.error && result.error.code === 'ENOENT') {
     throw new NubosPilotError(

package/bin/np-tools/spawn-headless.test.cjs

@@ -8,6 +8,14 @@ const assert = require('node:assert/strict');
 
 const spawnHeadless = require('./spawn-headless.cjs');
 const runContext = require('../../lib/run-context.cjs');
+const headlessGuard = require('../../lib/headless-guard.cjs');
+
+function _mockClaude(r, name, body) {
+  const p = path.join(r, name);
+  fs.writeFileSync(p, body, 'utf-8');
+  fs.chmodSync(p, 0o755);
+  return p;
+}
 
 const _sandboxes = [];
 const _envBackup = {};
@@ -488,6 +496,133 @@ test('SH-TRAIL-2 two sequential spawns append two parseable trail lines (jsonl i
   for (const l of lines) JSON.parse(l);
 });
 
+test('SH-GUARD-1 refuses to spawn when NUBOS_PILOT_HEADLESS=1 (reentrancy guard)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh', '#!/bin/sh\ncat > /dev/null\necho "{}"\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  _setEnv('NUBOS_PILOT_HEADLESS', '1');
+  const cap = _cap();
+  assert.throws(
+    () => spawnHeadless.run(
+      ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+      { cwd: r, stdout: cap.stub },
+    ),
+    (err) => err && err.code === 'spawn-headless-reentrant',
+  );
+  assert.equal(fs.existsSync(path.join(r, 'out.json')), false, 'no claude must be spawned inside a headless run');
+});
+
+test('SH-GUARD-2 refuses to spawn when hook depth has reached the cap (depth guard)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh', '#!/bin/sh\ncat > /dev/null\necho "{}"\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  _setEnv('NUBOS_PILOT_HOOK_DEPTH', '1');
+  const cap = _cap();
+  assert.throws(
+    () => spawnHeadless.run(
+      ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+      { cwd: r, stdout: cap.stub },
+    ),
+    (err) => err && err.code === 'spawn-headless-depth-exceeded',
+  );
+});
+
+test('SH-GUARD-3 child env carries NUBOS_PILOT_HEADLESS=1 and depth=1 (one level deep only)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh',
+    '#!/bin/sh\ncat > /dev/null\nprintf \'{"hl":"\'$NUBOS_PILOT_HEADLESS\'","depth":"\'$NUBOS_PILOT_HOOK_DEPTH\'"}\\n\'\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  const cap = _cap();
+  const rc = spawnHeadless.run(
+    ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+    { cwd: r, stdout: cap.stub },
+  );
+  assert.equal(rc, 0);
+  const child = JSON.parse(fs.readFileSync(path.join(r, 'out.json'), 'utf-8'));
+  assert.equal(child.hl, '1', 'child claude must run with NUBOS_PILOT_HEADLESS=1');
+  assert.equal(child.depth, '1', 'child claude must run at hook depth 1');
+});
+
+test('SH-GUARD-4 refuses to spawn while a live lock for the same agent is held (concurrency guard)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh', '#!/bin/sh\ncat > /dev/null\necho "{}"\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  const held = headlessGuard.tryAcquireSpawnLock(r, 'np-test-critic');
+  assert.equal(held.acquired, true);
+  const cap = _cap();
+  try {
+    assert.throws(
+      () => spawnHeadless.run(
+        ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+        { cwd: r, stdout: cap.stub },
+      ),
+      (err) => err && err.code === 'spawn-headless-locked',
+    );
+  } finally {
+    held.release();
+  }
+});
+
+test('SH-GUARD-5 lock is released after a successful spawn (re-spawnable)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh', '#!/bin/sh\ncat > /dev/null\necho "{}"\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  const cap = _cap();
+  for (let i = 0; i < 2; i++) {
+    const rc = spawnHeadless.run(
+      ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out' + i + '.json'],
+      { cwd: r, stdout: cap.stub },
+    );
+    assert.equal(rc, 0, 'sequential spawns must each acquire and release the lock');
+  }
+  assert.equal(fs.existsSync(headlessGuard._lockPath(r, 'np-test-critic')), false, 'no lock residue after spawns');
+});
+
+test('SH-GUARD-6 a held lock for one agent does NOT block a different agent (per-agent scope)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(
+    path.join(r, '.nubos-pilot', 'agents', 'np-other-critic.md'),
+    '---\nname: np-other-critic\n---\n\n# Role\n',
+    'utf-8',
+  );
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  const mockBin = _mockClaude(r, 'mock.sh', '#!/bin/sh\ncat > /dev/null\necho "{}"\n');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', mockBin);
+  const held = headlessGuard.tryAcquireSpawnLock(r, 'np-test-critic');
+  assert.equal(held.acquired, true);
+  const cap = _cap();
+  try {
+    const rc = spawnHeadless.run(
+      ['--agent', 'np-other-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+      { cwd: r, stdout: cap.stub },
+    );
+    assert.equal(rc, 0, 'a different agent must spawn while np-test-critic is locked');
+  } finally {
+    held.release();
+  }
+});
+
+test('SH-GUARD-7 lock is released even when the spawn errors (claude-not-found)', () => {
+  const r = _mkRoot();
+  fs.writeFileSync(path.join(r, 'p.md'), 'audit', 'utf-8');
+  _setEnv('NUBOS_PILOT_CLAUDE_BIN', path.join(r, 'no-such-binary'));
+  const cap = _cap();
+  assert.throws(
+    () => spawnHeadless.run(
+      ['--agent', 'np-test-critic', '--prompt-path', 'p.md', '--output-path', 'out.json'],
+      { cwd: r, stdout: cap.stub },
+    ),
+    (err) => err && err.code === 'spawn-headless-claude-not-found',
+  );
+  assert.equal(fs.existsSync(headlessGuard._lockPath(r, 'np-test-critic')), false,
+    'the per-agent lock must not leak when the spawn fails');
+});
+
 test('SH-ENV-4 NUBOS_PILOT_/CLAUDE_/ANTHROPIC_ prefixed vars pass through (whitelisted prefix)', () => {
   const parent = {
     PATH: '/usr/bin',

package/bin/np-tools/verify-reliability.cjs

@@ -0,0 +1,65 @@
+'use strict';
+
+const { summarize, describe } = require('../../lib/eval-reliability.cjs');
+const { emitErrorEnvelope } = require('./_args.cjs');
+
+function _usage() {
+  return [
+    'Usage:',
+    '  np-tools.cjs verify-reliability --codes <c1,c2,...>',
+    '',
+    'pass@k reliability: the orchestrator runs a task\'s <verify> command k times',
+    'and passes the collected exit codes (0 = pass). Emits a JSON summary whose',
+    '`aggregate_exit_code` is 0 only when every run passed (pass^k) — feed it to',
+    '`loop-run-round --phase post-executor --verify-exit-code`. A flaky task',
+    'aggregates to red and flows through the normal build-fixer path.',
+  ].join('\n');
+}
+
+function run(argv, ctx) {
+  const context = ctx || {};
+  const stdout = context.stdout || process.stdout;
+  const stderr = context.stderr || process.stderr;
+  const args = Array.isArray(argv) ? argv.slice() : [];
+
+  let codesRaw = null;
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === '-h' || a === '--help') { stdout.write(_usage() + '\n'); return 0; }
+    else if (a === '--codes') { codesRaw = args[++i] || ''; }
+    else if (a.startsWith('--codes=')) { codesRaw = a.slice('--codes='.length); }
+    else {
+      stderr.write(JSON.stringify({
+        code: 'verify-reliability-unknown-arg',
+        message: 'Unknown argument: ' + a,
+        details: { arg: a },
+      }) + '\n');
+      return 1;
+    }
+  }
+
+  if (codesRaw == null) {
+    stderr.write(JSON.stringify({
+      code: 'verify-reliability-missing-codes',
+      message: '--codes <c1,c2,...> is required',
+      details: {},
+    }) + '\n');
+    return 1;
+  }
+
+  try {
+    const codes = String(codesRaw).split(',').map((s) => s.trim()).filter((s) => s !== '').map(Number);
+    const summary = summarize(codes);
+    stdout.write(JSON.stringify(Object.assign({}, summary, { description: describe(summary) })) + '\n');
+    return 0;
+  } catch (err) {
+    emitErrorEnvelope(err, stderr, 'verify-reliability-internal-error');
+    return 1;
+  }
+}
+
+module.exports = { run };
+
+if (require.main === module) {
+  process.exit(run(process.argv.slice(2)));
+}

package/bin/np-tools/verify-reliability.test.cjs

@@ -0,0 +1,69 @@
+'use strict';
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { run } = require('./verify-reliability.cjs');
+
+function _capture() {
+  const out = { text: '' };
+  const err = { text: '' };
+  return {
+    stdout: { write: (s) => { out.text += s; return true; } },
+    stderr: { write: (s) => { err.text += s; return true; } },
+    out, err,
+  };
+}
+
+test('VR-1: all-pass codes → aggregate 0, reliable-pass', () => {
+  const c = _capture();
+  const code = run(['--codes', '0,0,0'], c);
+  assert.strictEqual(code, 0);
+  const r = JSON.parse(c.out.text);
+  assert.strictEqual(r.aggregate_exit_code, 0);
+  assert.strictEqual(r.verdict, 'reliable-pass');
+});
+
+test('VR-2: flaky codes → aggregate 1, flaky verdict + loud description', () => {
+  const c = _capture();
+  const code = run(['--codes', '0,1,0'], c);
+  assert.strictEqual(code, 0);
+  const r = JSON.parse(c.out.text);
+  assert.strictEqual(r.aggregate_exit_code, 1);
+  assert.strictEqual(r.flaky, true);
+  assert.match(r.description, /FLAKY/);
+});
+
+test('VR-3: --codes= form supported', () => {
+  const c = _capture();
+  const code = run(['--codes=1,1'], c);
+  assert.strictEqual(code, 0);
+  assert.strictEqual(JSON.parse(c.out.text).verdict, 'reliable-fail');
+});
+
+test('VR-4: missing --codes → error envelope, exit 1', () => {
+  const c = _capture();
+  const code = run([], c);
+  assert.strictEqual(code, 1);
+  assert.match(c.err.text, /verify-reliability-missing-codes/);
+});
+
+test('VR-5: unknown arg → error envelope, exit 1', () => {
+  const c = _capture();
+  const code = run(['--bogus'], c);
+  assert.strictEqual(code, 1);
+  assert.match(c.err.text, /verify-reliability-unknown-arg/);
+});
+
+test('VR-6: empty codes → internal error envelope, exit 1', () => {
+  const c = _capture();
+  const code = run(['--codes', ''], c);
+  assert.strictEqual(code, 1);
+  assert.match(c.err.text, /eval-reliability-no-runs/);
+});
+
+test('VR-7: --help → usage exit 0', () => {
+  const c = _capture();
+  const code = run(['--help'], c);
+  assert.strictEqual(code, 0);
+  assert.match(c.out.text, /verify-reliability/);
+});

package/lib/agents.test.cjs

@@ -247,6 +247,7 @@ const NP_AGENTS = [
   { file: 'np-nyquist-auditor', expected_tier: 'haiku' },
   { file: 'np-sc-extractor', expected_tier: 'haiku' },
   { file: 'np-critic', expected_tier: 'sonnet' },
+  { file: 'np-learnings-extractor', expected_tier: 'haiku' },
 ];
 
 // Audit-surface modules — files in agents/ that carry agent-shaped frontmatter

package/lib/config-defaults.cjs

@@ -9,6 +9,7 @@ const DEFAULT_WORKFLOW = Object.freeze({
   commit_docs: true,
   commit_artifacts: true,
   worktree_isolation: false,
+  tier_routing: false,
   research_tools: DEFAULT_RESEARCH_TOOLS,
 });
 
@@ -21,6 +22,7 @@ const DEFAULT_AGENTS = Object.freeze({
 
 const DEFAULT_LOOP = Object.freeze({
   maxRounds: 3,
+  verify_runs: 1,
 });
 
 const DEFAULT_SWARM_RESEARCH = Object.freeze({
@@ -58,6 +60,14 @@ const DEFAULT_CONFORMANCE = Object.freeze({
   inject_criteria: true,
 });
 
+const DEFAULT_LEARNINGS = Object.freeze({
+  auto_capture: true,
+  max_captures_per_hour: 10,
+  max_in_a_row: 3,
+  timeout_ms: 120000,
+  max_files: 30,
+});
+
 const DEFAULT_AUTO_LOG_LEARNING = true;
 
 const DEFAULT_SPAWN_HEADLESS = Object.freeze({
@@ -86,6 +96,7 @@ const DEFAULT_CONFIG_TREE = Object.freeze({
   spawn: DEFAULT_SPAWN,
   security: DEFAULT_SECURITY,
   conformance: DEFAULT_CONFORMANCE,
+  learnings: DEFAULT_LEARNINGS,
   auto_log_learning: DEFAULT_AUTO_LOG_LEARNING,
 });
 
@@ -119,6 +130,7 @@ function buildInstallConfig(answers) {
     },
     security: { ...DEFAULT_SECURITY },
     conformance: { ...DEFAULT_CONFORMANCE },
+    learnings: { ...DEFAULT_LEARNINGS },
     auto_log_learning: DEFAULT_AUTO_LOG_LEARNING,
   };
 }
@@ -135,6 +147,7 @@ module.exports = {
   DEFAULT_SPAWN_HEADLESS,
   DEFAULT_SECURITY,
   DEFAULT_CONFORMANCE,
+  DEFAULT_LEARNINGS,
   DEFAULT_AUTO_LOG_LEARNING,
   DEFAULT_MODEL_PROFILE,
   DEFAULT_SCOPE,

package/lib/config-schema.cjs

@@ -21,6 +21,7 @@ const SCHEMA = Object.freeze({
       worktree_isolation: { type: 'boolean', optional: true },
       research_tools:     { type: 'object', shape: 'any', optional: true },
       text_mode:          { type: 'boolean', optional: true },
+      tier_routing:       { type: 'boolean', optional: true },
     },
   },
   agents: {
@@ -34,6 +35,7 @@ const SCHEMA = Object.freeze({
   loop: {
     type: 'object', optional: true, shape: {
       maxRounds: { type: 'number', optional: true },
+      verify_runs: { type: 'number', optional: true },
     },
   },
   swarm: {
@@ -86,6 +88,15 @@ const SCHEMA = Object.freeze({
       inject_criteria: { type: 'boolean', optional: true },
     },
   },
+  learnings: {
+    type: 'object', optional: true, shape: {
+      auto_capture:          { type: 'boolean', optional: true },
+      max_captures_per_hour: { type: 'number', optional: true },
+      max_in_a_row:          { type: 'number', optional: true },
+      timeout_ms:            { type: 'number', optional: true },
+      max_files:             { type: 'number', optional: true },
+    },
+  },
 });
 
 function _typeOf(v) {

package/lib/eval-reliability.cjs

@@ -0,0 +1,63 @@
+'use strict';
+
+// pass@k reliability: the orchestrator runs a task's <verify> command k times
+// and feeds the collected exit codes here. A task that passes only sometimes is
+// FLAKY — not green. summarize() folds k runs into a single aggregate exit code
+// (0 only when every run passed — pass^k semantics) so flakiness flows through
+// the EXISTING verify-red → build-fixer path. No new critic category is
+// introduced (that would risk the unknown-category spurious-stuck trap).
+
+const { NubosPilotError } = require('./core.cjs');
+
+/**
+ * @param {number[]} exitCodes  one exit code per verify run (0 = pass)
+ * @returns {{runs:number, passes:number, fails:number, pass_at_1:boolean, pass_at_k:boolean, flaky:boolean, verdict:string, aggregate_exit_code:number}}
+ */
+function summarize(exitCodes) {
+  if (!Array.isArray(exitCodes) || exitCodes.length === 0) {
+    throw new NubosPilotError(
+      'eval-reliability-no-runs',
+      'summarize requires a non-empty array of exit codes',
+      { got: exitCodes },
+    );
+  }
+  const codes = exitCodes.map((c) => Number(c));
+  if (codes.some((c) => !Number.isInteger(c))) {
+    throw new NubosPilotError(
+      'eval-reliability-bad-code',
+      'every exit code must be an integer',
+      { codes: exitCodes },
+    );
+  }
+
+  const runs = codes.length;
+  const passes = codes.filter((c) => c === 0).length;
+  const fails = runs - passes;
+  const passAt1 = codes[0] === 0;
+  const passAtK = passes === runs;
+  const flaky = passes > 0 && fails > 0;
+
+  let verdict;
+  if (passAtK) verdict = 'reliable-pass';
+  else if (passes === 0) verdict = 'reliable-fail';
+  else verdict = 'flaky';
+
+  // pass^k: green only if every run passed. Flaky and all-fail both aggregate
+  // to non-zero so the loop treats them as verify-red.
+  const aggregate_exit_code = passAtK ? 0 : 1;
+
+  return { runs, passes, fails, pass_at_1: passAt1, pass_at_k: passAtK, flaky, verdict, aggregate_exit_code };
+}
+
+/** One-line human summary for the verify log the build-fixer reads. */
+function describe(s) {
+  if (s.runs === 1) {
+    return s.pass_at_k ? 'verify passed (1 run)' : 'verify failed (1 run)';
+  }
+  if (s.verdict === 'reliable-pass') return 'verify reliably passed (' + s.passes + '/' + s.runs + ' runs)';
+  if (s.verdict === 'reliable-fail') return 'verify reliably failed (0/' + s.runs + ' runs passed)';
+  return 'FLAKY: verify passed only ' + s.passes + '/' + s.runs + ' runs — non-deterministic, treated as red. '
+    + 'Make the verified behaviour deterministic (no sleeps/real clock/network/ordering) before this task can go green.';
+}
+
+module.exports = { summarize, describe };

package/lib/eval-reliability.test.cjs

@@ -0,0 +1,56 @@
+'use strict';
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { summarize, describe } = require('./eval-reliability.cjs');
+
+test('ER-1: all pass → reliable-pass, aggregate 0', () => {
+  const s = summarize([0, 0, 0]);
+  assert.strictEqual(s.verdict, 'reliable-pass');
+  assert.strictEqual(s.pass_at_k, true);
+  assert.strictEqual(s.pass_at_1, true);
+  assert.strictEqual(s.flaky, false);
+  assert.strictEqual(s.aggregate_exit_code, 0);
+});
+
+test('ER-2: all fail → reliable-fail, aggregate non-zero', () => {
+  const s = summarize([1, 1, 1]);
+  assert.strictEqual(s.verdict, 'reliable-fail');
+  assert.strictEqual(s.pass_at_k, false);
+  assert.strictEqual(s.flaky, false);
+  assert.strictEqual(s.aggregate_exit_code, 1);
+});
+
+test('ER-3: mixed → flaky, aggregate non-zero (pass^k)', () => {
+  const s = summarize([0, 1, 0]);
+  assert.strictEqual(s.verdict, 'flaky');
+  assert.strictEqual(s.flaky, true);
+  assert.strictEqual(s.pass_at_1, true);
+  assert.strictEqual(s.pass_at_k, false);
+  assert.strictEqual(s.aggregate_exit_code, 1);
+});
+
+test('ER-4: first-run-fail-then-pass is still flaky and red', () => {
+  const s = summarize([1, 0, 0]);
+  assert.strictEqual(s.flaky, true);
+  assert.strictEqual(s.pass_at_1, false);
+  assert.strictEqual(s.aggregate_exit_code, 1);
+});
+
+test('ER-5: single run preserves classic behaviour', () => {
+  assert.strictEqual(summarize([0]).aggregate_exit_code, 0);
+  assert.strictEqual(summarize([2]).aggregate_exit_code, 1);
+  assert.strictEqual(summarize([0]).verdict, 'reliable-pass');
+});
+
+test('ER-6: empty/invalid input throws', () => {
+  assert.throws(() => summarize([]), (e) => e.code === 'eval-reliability-no-runs');
+  assert.throws(() => summarize('nope'), (e) => e.code === 'eval-reliability-no-runs');
+  assert.throws(() => summarize([0, 1.5]), (e) => e.code === 'eval-reliability-bad-code');
+});
+
+test('ER-7: describe is human-readable and flags flaky loudly', () => {
+  assert.match(describe(summarize([0])), /passed \(1 run\)/);
+  assert.match(describe(summarize([0, 0, 0])), /reliably passed/);
+  assert.match(describe(summarize([0, 1, 0])), /FLAKY/);
+});