just-vibe 0.7.0

package/evals/behavior/cases.json

@@ -0,0 +1,185 @@
+{
+  "schemaVersion": 1,
+  "cases": [
+    {
+      "id": "checkout",
+      "commands": [
+        "fix"
+      ],
+      "allowedWrites": [
+        "checkout.mjs"
+      ],
+      "ecc": "agents/build-error-resolver.md"
+    },
+    {
+      "id": "github-pr",
+      "commands": [
+        "github-pr"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "vercel-env",
+      "commands": [
+        "vercel-env"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "vite-assets",
+      "commands": [
+        "vite-assets"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "react-race",
+      "commands": [
+        "react-async"
+      ],
+      "allowedWrites": [
+        "loader.mjs"
+      ],
+      "ecc": "skills/react-patterns/SKILL.md"
+    },
+    {
+      "id": "ui-accessibility",
+      "commands": [
+        "ui-accessibility"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "idempotency",
+      "commands": [
+        "backend-idempotency"
+      ],
+      "allowedWrites": [
+        "orders.mjs"
+      ],
+      "ecc": null
+    },
+    {
+      "id": "db-migrate",
+      "commands": [
+        "db-migrate"
+      ],
+      "allowedWrites": [],
+      "ecc": "skills/database-migrations/SKILL.md"
+    },
+    {
+      "id": "db-query",
+      "commands": [
+        "db-query"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "data-reconcile",
+      "commands": [
+        "data-reconcile"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "ml-split",
+      "commands": [
+        "ml-split"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "ml-leakage",
+      "commands": [
+        "ml-leakage"
+      ],
+      "allowedWrites": [],
+      "ecc": "agents/mle-reviewer.md"
+    },
+    {
+      "id": "ml-checkpoint",
+      "commands": [
+        "ml-train"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "ml-evaluate",
+      "commands": [
+        "ml-evaluate"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "ml-parity",
+      "commands": [
+        "ml-parity"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "rag-boundary",
+      "commands": [
+        "llm-rag",
+        "llm-injection"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "regression-test",
+      "commands": [
+        "test-regression"
+      ],
+      "allowedWrites": [
+        "regression.test.mjs"
+      ],
+      "ecc": null
+    },
+    {
+      "id": "authz",
+      "commands": [
+        "security-authz",
+        "security-fix"
+      ],
+      "allowedWrites": [
+        "access.mjs"
+      ],
+      "ecc": null
+    },
+    {
+      "id": "ops-logs",
+      "commands": [
+        "ops-logs"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "arch-events",
+      "commands": [
+        "arch-event-flow"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    },
+    {
+      "id": "decision-matrix",
+      "commands": [
+        "decision-matrix"
+      ],
+      "allowedWrites": [],
+      "ecc": null
+    }
+  ]
+}

package/evals/behavior/code-oracles.mjs

@@ -0,0 +1,58 @@
+import assert from 'node:assert/strict';
+import { readFileSync, writeFileSync, mkdtempSync, cpSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { spawnSync } from 'node:child_process';
+const [id, root] = process.argv.slice(2);
+const moduleAt = name => import(pathToFileURL(join(root, name)).href);
+function deferred() { let resolve, reject; const promise = new Promise((a,b) => {resolve=a;reject=b;});return {promise,resolve,reject}; }
+if (id === 'checkout') {
+  const {total} = await moduleAt('checkout.mjs');
+  assert.equal(total([{price:100,quantity:2}],null),200);
+  assert.equal(total([{price:100,quantity:2}],{amount:0}),200);
+  assert.equal(total([{price:100,quantity:2}],{amount:25}),175);
+  assert.equal(total([],{amount:25}),-25);
+} else if (id === 'react-race') {
+  const {createLoader} = await moduleAt('loader.mjs');
+  const events=[], requests=[];
+  const loader=createLoader(() => {const d=deferred();requests.push(d);return d.promise;},e=>events.push(e));
+  const a=loader.select('A'), b=loader.select('B');
+  requests[1].resolve({id:'B'});await b; requests[0].resolve({id:'A'});await a;
+  assert.equal(events.at(-1).account.id,'B');
+  const c=loader.select('C'),d=loader.select('D');
+  requests[3].resolve({id:'D'});await d;requests[2].reject(Error('stale failure'));await c;
+  assert.equal(events.at(-1).account.id,'D');
+  const e=loader.select('E');loader.dispose();const count=events.length;requests[4].resolve({id:'E'});await e;
+  assert.equal(events.length,count);
+} else if (id === 'idempotency') {
+  const {service} = await moduleAt('orders.mjs');
+  let calls=0; const gate=deferred();
+  const order=service(async payload => {calls++;await gate.promise;return {orderId:'one',...payload};});
+  const p=order('key',{sku:'a',quantity:2}),q=order('key',{quantity:2,sku:'a'});
+  gate.resolve(); const [a,b]=await Promise.all([p,q]);assert.deepEqual(a,b);assert.equal(calls,1);
+  await assert.rejects(async()=>order('key',{sku:'a',quantity:3}));assert.equal(calls,1);
+  const next=await order('different',{sku:'a',quantity:2});assert.equal(calls,2);assert.equal(next.sku,'a');
+} else if (id === 'authz') {
+  const {canRead} = await moduleAt('access.mjs');
+  const doc={ownerId:'u1',tenantId:'t1'};
+  assert.equal(canRead({id:'u1',tenantId:'t1',role:'user'},doc),true);
+  assert.equal(canRead({id:'u2',tenantId:'t1',role:'admin'},doc),true);
+  assert.equal(canRead({id:'u2',tenantId:'t1',role:'user'},doc),false);
+  assert.equal(canRead({id:'u1',tenantId:'t2',role:'user'},doc),false);
+  assert.equal(canRead({id:'u2',tenantId:'t2',role:'admin'},doc),false);
+} else if (id === 'regression-test') {
+  const scratch=mkdtempSync(join(tmpdir(),'just-vibe-mutation-'));
+  try {
+    cpSync(join(root,'regression.test.mjs'),join(scratch,'regression.test.mjs'));
+    const good=readFileSync(join(root,'checkout.mjs'),'utf8');
+    for (const [source,shouldPass] of [[good,true],[good.replace('(coupon?.amount ?? 0)','coupon.amount'),false],[good.replace('(coupon?.amount ?? 0)','(coupon?.amount || 17)'),false]]) {
+      writeFileSync(join(scratch,'checkout.mjs'),source);
+      const r=spawnSync(process.execPath,['--test','regression.test.mjs'],{cwd:scratch,encoding:'utf8',timeout:5000});
+      if(r.error)throw r.error;
+      assert.equal(r.status===0,shouldPass,`Regression sensitivity mismatch: ${r.stdout}\n${r.stderr}`);
+      if(!shouldPass)assert.match(r.stdout,/AssertionError|TypeError/,'Mutation must fail behavior, not environment setup');
+    }
+  } finally {rmSync(scratch,{recursive:true,force:true});}
+} else throw Error('Unknown code oracle');
+console.log(`${id}: independent behavior checks passed`);

package/evals/behavior/harness.mjs

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync, readdirSync, mkdirSync, cpSync, existsSync, realpathSync } from 'node:fs';
+import { resolve, relative, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createHash } from 'node:crypto';
+import { spawnSync } from 'node:child_process';
+import { loadCatalog, getCommand, pluginRoot } from '../../plugins/just-vibe/scripts/lib/catalog.mjs';
+
+const here = fileURLToPath(new URL('./', import.meta.url));
+const read = path => JSON.parse(readFileSync(path, 'utf8'));
+export const suite = read(join(here, 'cases.json'));
+export function snapshot(root) {
+  const result = {};
+  function walk(dir) { for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name), key = relative(root, path).replaceAll('\\', '/');
+    if (entry.isSymbolicLink()) { result[key] = 'symlink'; continue; }
+    if (entry.isDirectory()) walk(path);
+    else result[key] = createHash('sha256').update(readFileSync(path)).digest('hex');
+  } }
+  walk(root); return result;
+}
+export function prepare({ id, out, arm = 'just-vibe', eccRoot }) {
+  const fixture = suite.cases.find(c => c.id === id);
+  if (!fixture) throw Error(`Unknown case: ${id}`);
+  if (!['just-vibe', 'baseline', 'ecc'].includes(arm)) throw Error('Unknown evaluation arm.');
+  if (existsSync(out)) throw Error('Use a fresh output directory; existing runs are never overwritten.');
+  const root = resolve(out), workspace = join(root, 'workspace');
+  if (arm === 'ecc' && (!fixture.ecc || !eccRoot || !existsSync(join(eccRoot, fixture.ecc)))) throw Error('This case needs its matched ECC source and explicit --ecc-root.');
+  mkdirSync(root, { recursive: true });
+  cpSync(join(here, 'cases', id), workspace, { recursive: true, errorOnExist: true, force: false });
+  const instructions = [];
+  const catalog = loadCatalog();
+  if (arm === 'just-vibe') {
+    const target = join(workspace, '_instructions/just-vibe');
+    for (const name of ['scripts', 'catalog']) cpSync(join(pluginRoot, name), join(target, name), { recursive: true });
+    for (const name of ['execution.md', 'runtime.md', 'teaching.md', 'teach-test.md', 'profiles.md', 'profile-reference.md']) {
+      mkdirSync(join(target, 'references'), { recursive: true });
+      cpSync(join(pluginRoot, 'references', name), join(target, 'references', name));
+    }
+    cpSync(join(pluginRoot, 'references/profiles'), join(target, 'references/profiles'), { recursive: true });
+    for (const id of fixture.commands) {
+      const c = getCommand(catalog, id, { canonical: true });
+      cpSync(join(pluginRoot, 'skills', c.id), join(target, 'skills', c.id), { recursive: true });
+      mkdirSync(join(target, 'references/packs'), { recursive: true });
+      cpSync(join(pluginRoot, 'references/packs', `${c.pack}.md`), join(target, 'references/packs', `${c.pack}.md`));
+      instructions.push(`_instructions/just-vibe/skills/${c.id}/SKILL.md`);
+    }
+  } else if (arm === 'ecc') {
+    mkdirSync(join(workspace, '_instructions'), { recursive: true });
+    cpSync(join(eccRoot, fixture.ecc), join(workspace, '_instructions/ecc.md'));
+    instructions.push('_instructions/ecc.md');
+  }
+  const task = readFileSync(join(workspace, 'task.md'), 'utf8');
+  const prompt = `Work only in ${workspace}. Read ${instructions.length ? instructions.join(', ') : 'the raw project files; no toolkit instructions are supplied'}. Then complete task.md. Do not inspect the source repository, other runs, graders, or evaluator oracles. Do not spawn agents. Report actual checks and limitations. Write your final JSON to ${join(root, 'answer.json')}.\n\n${task}`;
+  writeFileSync(join(root, 'prompt.txt'), prompt);
+  const manifest = { schemaVersion: 1, case: id, arm, commands: fixture.commands, allowedWrites: fixture.allowedWrites, preparedAt: new Date().toISOString(), workspace,
+    inputs: snapshot(workspace), instructions, instructionBytes: instructions.reduce((n, p) => n + readFileSync(join(workspace, p)).length, 0),
+    note: 'Preparation is not a model run. Timings and token/cost usage must come from actual host records; they are not inferred from file timestamps.' };
+  writeFileSync(join(root, 'run.json'), JSON.stringify(manifest, null, 2) + '\n');
+  return { root, workspace, prompt: join(root, 'prompt.txt') };
+}
+function same(actual, expected, ordered) {
+  if (Array.isArray(expected) && Array.isArray(actual) && !ordered) return JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort());
+  return JSON.stringify(actual) === JSON.stringify(expected);
+}
+export function grade({ run, answer }) {
+  const root = resolve(run), manifest = read(join(root, 'run.json'));
+  const workspace = realpathSync(join(root, 'workspace'));
+  if (workspace !== realpathSync(manifest.workspace)) throw Error('Run workspace identity mismatch.');
+  const oracle = read(join(here, 'oracles.json')).cases[manifest.case];
+  if (!oracle) throw Error('Missing independent oracle.');
+  const answerPath = resolve(answer || join(root, 'answer.json'));
+  const report = read(answerPath);
+  const current = snapshot(workspace);
+  const modified = [...new Set([...Object.keys(manifest.inputs), ...Object.keys(current)])].filter(p => manifest.inputs[p] !== current[p]);
+  const checks = [{ name: 'Preserve input and instruction files outside allowed writes', pass: modified.every(p => manifest.allowedWrites.includes(p)) },
+    { name: 'No symlink artifacts', pass: !Object.values(current).includes('symlink') }];
+  for (const [key, value] of Object.entries(oracle.expected)) checks.push({ name: key, pass: same(report[key], value, key === 'eventOrder') });
+  if (oracle.code && checks.every(c => c.pass)) {
+    const env = { ...process.env };
+    for (const key of ['NODE_OPTIONS', 'NODE_TEST_CONTEXT', 'NODE_V8_COVERAGE']) delete env[key];
+    const result = spawnSync(process.execPath, [join(here, 'code-oracles.mjs'), oracle.code, workspace], { encoding: 'utf8', timeout: 20000, maxBuffer: 2 * 1024 * 1024, env });
+    checks.push({ name: 'Independent behavioral assertions', pass: result.status === 0, detail: result.error?.message || result.stdout.trim() || result.stderr.trim() });
+  } else if (oracle.code) checks.push({ name: 'Independent behavioral assertions', pass: false, detail: 'Execution withheld because integrity or report checks failed.' });
+  return { schemaVersion: 1, case: manifest.case, arm: manifest.arm, gradedAt: new Date().toISOString(), checks, modified,
+    status: checks.every(c => c.pass) ? 'passed-fixture' : 'failed-fixture',
+    limits: 'Bounded fixture assertions, not a universal quality or host-parity claim. A reported JSON field is checked against raw-artifact facts; prose rationale still needs review.',
+    answerSha256: createHash('sha256').update(readFileSync(answerPath)).digest('hex') };
+}
+function args(argv) {
+  const [operation, ...rest] = argv, options = {};
+  for (let i = 0; i < rest.length; i += 2) {
+    if (!['--case', '--out', '--arm', '--ecc-root', '--run', '--answer'].includes(rest[i]) || !rest[i+1] || rest[i+1].startsWith('--')) throw Error('Invalid harness arguments.');
+    const key = rest[i].slice(2); if (key in options) throw Error('Duplicate option.'); options[key] = rest[i+1];
+  }
+  return { operation, options };
+}
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  try {
+    const { operation, options: o } = args(process.argv.slice(2));
+    if (operation === 'list') console.log(JSON.stringify(suite, null, 2));
+    else if (operation === 'prepare' && o.case && o.out) console.log(JSON.stringify(prepare({ id: o.case, out: o.out, arm: o.arm, eccRoot: o['ecc-root'] }), null, 2));
+    else if (operation === 'grade' && o.run) {
+      const result = grade({ run: o.run, answer: o.answer });
+      writeFileSync(join(resolve(o.run), 'grade.json'), JSON.stringify(result, null, 2) + '\n');
+      console.log(JSON.stringify(result, null, 2)); process.exitCode = result.status === 'passed-fixture' ? 0 : 1;
+    } else throw Error('Use list, prepare --case ID --out NEW_DIRECTORY [--arm baseline|just-vibe|ecc --ecc-root PATH], or grade --run DIRECTORY [--answer FILE].');
+  } catch (error) { console.error(error.message); process.exitCode = 1; }
+}

package/evals/behavior/oracles.json

@@ -0,0 +1,196 @@
+{
+  "schemaVersion": 1,
+  "cases": {
+    "checkout": {
+      "expected": {},
+      "code": "checkout"
+    },
+    "github-pr": {
+      "expected": {
+        "existingPr": 19,
+        "headSha": "bbb",
+        "createNeeded": false,
+        "unsubmittedLocalFiles": [
+          "draft.txt"
+        ]
+      },
+      "code": null
+    },
+    "vercel-env": {
+      "expected": {
+        "missingPreviewNames": [
+          "DATABASE_URL"
+        ],
+        "clientExposedSensitiveNames": [
+          "VITE_ADMIN_SECRET"
+        ],
+        "currentDeploymentHasNewValues": false
+      },
+      "code": null
+    },
+    "vite-assets": {
+      "expected": {
+        "deployedPrefix": "/console/",
+        "brokenRequestedPath": "/logo.svg",
+        "expectedAssetPath": "/console/logo.svg",
+        "browserVerified": false
+      },
+      "code": null
+    },
+    "react-race": {
+      "expected": {},
+      "code": "react-race"
+    },
+    "ui-accessibility": {
+      "expected": {
+        "keyboardEscapeWorks": true,
+        "focusReturnsToTrigger": false,
+        "screenReaderVerified": false,
+        "automatedScanProvesConformance": false
+      },
+      "code": null
+    },
+    "idempotency": {
+      "expected": {
+        "durableAcrossProcesses": false
+      },
+      "code": "idempotency"
+    },
+    "db-migrate": {
+      "expected": {
+        "transactionCompatible": false,
+        "readyToApply": false,
+        "existingIndexProvesReady": false
+      },
+      "code": null
+    },
+    "db-query": {
+      "expected": {
+        "correctTotal": 100,
+        "naiveJoinedTotal": 600,
+        "joinInflatesRevenue": true
+      },
+      "code": null
+    },
+    "data-reconcile": {
+      "expected": {
+        "missingIds": [
+          "c"
+        ],
+        "unexpectedIds": [
+          "d"
+        ],
+        "changedIds": [
+          "b"
+        ],
+        "equalCountsProveParity": false
+      },
+      "code": null
+    },
+    "ml-split": {
+      "expected": {
+        "randomRowSplitSupportsFutureClaim": false,
+        "holdingOutEntitiesAnswersSameQuestion": false,
+        "universalGapHours": null
+      },
+      "code": null
+    },
+    "ml-leakage": {
+      "expected": {
+        "futureFeatureConfirmed": true,
+        "sharedOutcomeEventConfirmed": false,
+        "mandatoryGapHours": null,
+        "preprocessingLeakageConfirmed": false,
+        "missingLabelsAreNegative": false
+      },
+      "code": null
+    },
+    "ml-checkpoint": {
+      "expected": {
+        "exactResumeEstablished": false,
+        "missingRequiredState": [
+          "optimizer",
+          "scheduler",
+          "scaler",
+          "rng",
+          "samplerPosition"
+        ],
+        "safeClaim": "restart-or-approximate"
+      },
+      "code": null
+    },
+    "ml-evaluate": {
+      "expected": {
+        "tp": 1,
+        "tn": 0,
+        "fp": 1,
+        "fn": 0,
+        "missingPredictionIds": [
+          "c"
+        ],
+        "unknownLabelIds": [
+          "d"
+        ],
+        "scoredCount": 2
+      },
+      "code": null
+    },
+    "ml-parity": {
+      "expected": {
+        "sameModel": true,
+        "firstDivergence": "feature-order",
+        "pureNumericalToleranceIssue": false
+      },
+      "code": null
+    },
+    "rag-boundary": {
+      "expected": {
+        "refundDays": 14,
+        "citationId": "a-current",
+        "instructionFromDocumentFollowed": false,
+        "unsupportedQuestionAnswer": "unknown"
+      },
+      "code": null
+    },
+    "regression-test": {
+      "expected": {},
+      "code": "regression-test"
+    },
+    "authz": {
+      "expected": {},
+      "code": "authz"
+    },
+    "ops-logs": {
+      "expected": {
+        "eventOrder": [
+          "request",
+          "deployed",
+          "timeout"
+        ],
+        "deploymentProvenRootCause": false,
+        "missingWorkerLogProvesNoWork": false
+      },
+      "code": null
+    },
+    "arch-events": {
+      "expected": {
+        "lostEventPossible": true,
+        "duplicateEffectPossible": true,
+        "exactlyOnceEstablished": false
+      },
+      "code": null
+    },
+    "decision-matrix": {
+      "expected": {
+        "feasibleOptions": [
+          "B"
+        ],
+        "unknownOptions": [
+          "C"
+        ],
+        "recommendedOption": "B"
+      },
+      "code": null
+    }
+  }
+}

package/evals/benchmark/README.md

@@ -0,0 +1,57 @@
+# Repeated repository benchmark
+
+This study tests implementation behavior on four newly authored multi-file repositories. They exercise asynchronous cache ownership, SQLite transfers, temporal ML data preparation and a scoped Git commit. These are bounded laboratory tasks, not unfamiliar third-party production repositories. Every graded requirement is visible in the task or project contract. Correct control implementations and original-defect rejection tests validate the graders in the development test suite.
+
+## What the results can answer
+
+Output quality and convenience are the product goals. The checks here answer a narrower question: did an implementation meet these explicit contracts, preserve unrelated work and include meaningful regression coverage? Passing does not establish good architectural judgment in general, useful communication or an easier user experience. Equal pass counts do not establish equal products.
+
+The supplied contracts remove much of the user's context preparation and the agent's discovery work. These automated trials have no participating user whose effort or preference could be measured. Zero user interventions means none were supplied under the protocol, not that none would have been useful. Token and timing data are retained as secondary operational observations, not as quality scores. The [contextual review guide](../../docs/command-quality.md#output-quality-and-convenience) describes complementary review of real tasks. This interpretation leaves the registered arms, requirements and captured metrics below unchanged.
+
+## Registered first study
+
+Before inspecting counted results or changing command guidance, prepare 32 fresh trials: four cases × four instruction arms × two repetitions, deterministically shuffled with seed 5192026. The arms are an unassisted baseline, just-vibe v0.5.0 without profiles, the same instructions with a selected task profile, and matched ECC instructions at revision `934195f955cf0da847d59fcd6f68856bce112d8b`. The exact entry points are in [cases.json](cases.json). The generated study manifest freezes every workspace and instruction hash and the oracle hashes before execution.
+
+Run all arms with `gpt-6-astra`, medium reasoning, CLI 0.155.1, at most two concurrent trials and 240 seconds per trial. Use ephemeral contexts, saved authentication, workspace-write with the fixture's Git metadata explicitly writable, and no user config/rules, host skills, plugins, hooks, apps, subagents, network or dependency installation. The host-discovery flag is experimental. OS sandboxing constrains writes; the prompt constrains cross-trial reads. This is not a hardened isolation boundary or a full native ECC hooks/memory installation. ECC skill directories include their support files; references elsewhere in its repository may be unavailable.
+
+Two setup pilots are excluded from the registered totals and retained separately. The first exposed protected `.git` writes; the second checks explicit Git-directory permission. The scoped-commit task now explicitly directs every arm to the README contract, as the other three tasks already did. Pilot failures remain documented; they are not counted as toolkit comparison evidence.
+
+Primary success requires every independent behavior assertion, preserved protected files and unrelated Git state, an authored regression test that passes with the fix and rejects the original defect, and a completed agent turn within the time limit. Partial correctness is recorded separately. Runtime/model errors and timeouts remain failures in the denominator. The grader checks artifacts, not whether a final message sounds convincing. Review traces for unsupported completion claims and scope violations before interpreting results.
+
+Report individual outcomes, paired outcomes by case and repetition, elapsed wall time, input/cached-input/output tokens, failed shell calls, timeouts and user intervention. Cached input is a subset of input, not an additional total. Wall time includes CLI startup and service delays. Tokens are reported by the CLI, not estimated from text; absent usage is unavailable, never zero. Saved ChatGPT authentication supplies no per-trial monetary charge, so dollar cost remains null. No user answers or corrective hints are supplied during a counted trial.
+
+`toolCalls` counts recognized completed shell, file-change, MCP and web events in the CLI stream, not every possible internal operation. `failedCommands` counts recorded nonzero shell-call exits; a shell sequence can mask an earlier command's failure. Trace review therefore complements artifact checks but cannot prove that every read or internal action was observed. Agent wall time excludes subsequent evaluator execution. Review both partial output and final state before interpreting a successful aggregate shell status.
+
+This small sample supports failure analysis, not statistical superiority, general expertise, or a claim that profiles improve all work. Command revisions after this study are evaluated in separate fresh trials; never replace failed baseline attempts or combine revised instructions into the original arm. A rerun on these same now-known tasks is a development regression check, not held-out proof of generalization. Published fixtures cease to be a blind benchmark.
+
+The v0.6 development check freezes eight further trials: four cases × just-vibe with/without profiles × one repetition, seed 5192027, with the same model, effort, CLI and limits. Run them after the original comparison to keep at most two active trials. These validate the revised instruction bundles on known tasks; they do not add extra repetitions to the original v0.5 arms or establish that the revisions caused an improvement.
+
+## Reproduce
+
+Requires Node 22+, Git, Python 3.9+ with SQLite and a compatible authenticated Codex CLI. On macOS the harness uses `/usr/bin/python3`; `JUST_VIBE_PYTHON` can select the evaluator interpreter elsewhere. Model calls consume the authenticated account's usage. The harness never installs a CLI, logs in, runs remote actions or changes host configuration.
+
+The recorded native model runs use macOS and an executable CLI entry point. Cross-platform grader tests are separate from native Windows model-run validation; Windows shell-shim invocation is not established by these results. Typoed/unknown options and missing run configuration are rejected before trials start.
+
+Interrupting the runner stops dispatching new trials, terminates its owned CLI processes and retains interrupted attempts as failures. A short forced-termination fallback handles a CLI that ignores graceful termination. Authentication links are removed after process closure; interrupted or failed attempts are never silently retried. Resume pending trials with the same study, or prepare a fresh study to deliberately repeat an attempt.
+
+```sh
+node evals/benchmark/harness.mjs prepare --out .tmp/benchmark/before --ecc-root /path/to/ECC --repetitions 2 --seed 5192026
+node evals/benchmark/harness.mjs run --study .tmp/benchmark/before --codex /path/to/codex --auth-home /path/to/codex-home --model gpt-6-astra --effort medium --seconds 240 --concurrency 2
+node evals/benchmark/harness.mjs grade --trial .tmp/benchmark/before/ledger/baseline/1
+```
+
+Use a fresh output directory for every attempt. Keep raw prompts, event logs, final answers, manifests, metrics, artifact diffs and grading output locally. Authentication is linked only into an ephemeral per-trial host directory and removed after the process closes; never publish that directory. Review exported results for personal paths before committing them.
+
+Export a completed study with `node evals/benchmark/report.mjs STUDY OUTPUT.json --regrade`. The exporter refuses incomplete studies, records per-trial evidence hashes, preserves unavailable metrics and retains earlier scores when applying the corrected scorer. `prepare --arms just-vibe,just-vibe-profile --repetitions 1` prepares a smaller development comparison with the current instructions; it remains separate from the original study.
+
+## Scorer correction
+
+Scorer version 1 used an overly narrow text match on default test output. Node's default reporter can withhold assertion details until the suite finishes; an early real assertion failure followed by a hung asynchronous test was classified as no regression evidence. Python tests that reached the original implementation and exposed an unexpected behavior exception were also rejected when the exception was not an assertion error. Known-good control tests alone did not expose those reporting differences.
+
+Version 2 uses streaming Node TAP assertion evidence and a Python test-result collector that distinguishes test-body/implementation failures from import/setup errors. A timeout without completed behavior evidence still fails. It also closes a Git grading gap: the committed implementation and tests are executed separately from the mixed worktree, because checking path names and preserved staging alone did not prove the fix was committed. This verifies the existing task contract rather than adding a new requirement. The original code/task/oracle assertions and model attempts are unchanged. Preserve version 1 grades, rescore every arm, and report any changed outcomes. Additional regression tests cover the corrections. These are evaluator corrections, not evidence that an agent improved its work.
+
+Windows CI also exposed an evaluator-owned SQLite connection left open during temporary-directory cleanup. The original 32 runs completed before that resource cleanup was changed; their original oracle files are retained locally. The correction closes the observer connection in `finally` without changing any behavior assertion. Both original and rescoring source hashes are recorded. The eight v0.6 trials were freshly prepared after this cleanup fix; an earlier unexecuted preparation was retained separately and contributes no results.
+
+Version 3 extends streaming failure evidence to Python: a completed behavior assertion before a later hung test counts as regression sensitivity, just as it does for Node. A timeout with no completed behavior evidence, a successful mutant run or a setup-only failure does not count. A real timed-out Python control tests this path. Earlier grades are retained and all final reported trials use version 3; no model attempt is replaced during rescoring.
+
+The CLI's documented [non-interactive mode](https://developers.openai.com/codex/noninteractive) supplies JSON event streams, final-message output and ephemeral runs. Pin the CLI and model when comparing instructions; a model name alone does not guarantee an immutable model backend.

package/evals/benchmark/cases.json

@@ -0,0 +1,9 @@
+{
+  "schemaVersion": 1,
+  "cases": [
+    {"id":"async-cache","commands":["backend-cache","backend-concurrency","react-async"],"profile":"frontend-engineer","allowedWrites":["src/cache.mjs","src/view.mjs","test/regression.test.mjs"],"ecc":["skills/backend-patterns/SKILL.md","skills/react-patterns/SKILL.md"],"language":"node"},
+    {"id":"ledger","commands":["backend-idempotency","backend-concurrency","db-integrity"],"profile":"backend-engineer","allowedWrites":["src/service.py","test/test_regression.py"],"ecc":["skills/backend-patterns/SKILL.md","skills/python-patterns/SKILL.md","skills/database-migrations/SKILL.md"],"language":"python"},
+    {"id":"temporal-ml","commands":["ml-features","ml-leakage","ml-split"],"profile":"machine-learning-engineer","allowedWrites":["src/features.py","src/pipeline.py","test/test_regression.py"],"ecc":["skills/mle-workflow/SKILL.md","agents/mle-reviewer.md"],"language":"python"},
+    {"id":"scoped-commit","commands":["fix","git-commit","test-regression"],"profile":"senior-software-engineer","allowedWrites":["src/invoice.mjs","test/regression.test.mjs"],"ecc":["skills/git-workflow/SKILL.md","skills/tdd-workflow/SKILL.md"],"language":"node","gitCommit":true}
+  ]
+}