just-vibe 0.7.0

package/evals/benchmark/harness.mjs

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync, readdirSync, mkdirSync, cpSync, existsSync, realpathSync, symlinkSync, rmSync } from 'node:fs';
+import { resolve, relative, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createHash } from 'node:crypto';
+import { spawnSync, execFileSync, spawn } from 'node:child_process';
+import { performance } from 'node:perf_hooks';
+import { pluginRoot } from '../../plugins/just-vibe/scripts/lib/catalog.mjs';
+
+const here=fileURLToPath(new URL('./',import.meta.url));
+const json=path=>JSON.parse(readFileSync(path,'utf8'));
+export const cases=json(join(here,'cases.json')).cases;
+export const arms=['baseline','just-vibe','just-vibe-profile','ecc'];
+const hash=value=>createHash('sha256').update(value).digest('hex');
+const git=(root,args)=>execFileSync('git',args,{cwd:root,encoding:'utf8',env:{...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:process.platform==='win32'?'NUL':'/dev/null'}}).trimEnd();
+function sourceIdentity(){
+  const root=fileURLToPath(new URL('../../',import.meta.url));
+  try{if(realpathSync(git(root,['rev-parse','--show-toplevel']))!==realpathSync(root))return {revision:null,dirty:null};return {revision:git(root,['rev-parse','HEAD']),dirty:Boolean(git(root,['status','--porcelain']))};}
+  catch{return {revision:null,dirty:null};}
+}
+export function snapshot(root) {
+  const result={};
+  function walk(dir){for(const e of readdirSync(dir,{withFileTypes:true})){
+    if(e.name==='.git'||e.name==='__pycache__')continue;
+    const path=join(dir,e.name),key=relative(root,path).replaceAll('\\','/');
+    if(e.isSymbolicLink())result[key]='symlink';else if(e.isDirectory())walk(path);else result[key]=hash(readFileSync(path));
+  }}walk(root);return result;
+}
+function write(path,body){mkdirSync(dirname(path),{recursive:true});writeFileSync(path,body);}
+function save(path,value){write(path,JSON.stringify(value,null,2)+'\n');}
+function shuffle(values,seed){let n=seed>>>0;const a=[...values];for(let i=a.length-1;i>0;i--){n=(Math.imul(1664525,n)+1013904223)>>>0;const j=n%(i+1);[a[i],a[j]]=[a[j],a[i]];}return a;}
+export function prepareTrial({out,id,arm,eccRoot,repetition=1}){
+  if(existsSync(out))throw Error('Trial output must be fresh.');
+  const fixture=cases.find(c=>c.id===id);if(!fixture||!arms.includes(arm))throw Error('Unknown case or arm.');
+  if(arm==='ecc'&&(!eccRoot||fixture.ecc.some(p=>!existsSync(join(eccRoot,p)))))throw Error('Pinned ECC sources are required.');
+  const root=resolve(out),workspace=join(root,'workspace');mkdirSync(root,{recursive:true});cpSync(join(here,'repos',id),workspace,{recursive:true});
+  git(workspace,['init','--quiet']);git(workspace,['config','user.name','Fixture User']);git(workspace,['config','user.email','fixture@example.invalid']);
+  git(workspace,['add','--all']);git(workspace,['commit','--quiet','-m','Initial fixture']);
+  const base=git(workspace,['rev-parse','HEAD']);
+  if(fixture.gitCommit){
+    const source=join(workspace,'src/invoice.mjs');writeFileSync(source,readFileSync(source,'utf8').replace('`Invoice ${id}`','`Receipt ${id}`'));
+    writeFileSync(join(workspace,'notes.md'),'Existing staged planning note. Preserve it.\n');git(workspace,['add','src/invoice.mjs','notes.md']);
+    writeFileSync(source,readFileSync(source,'utf8').replace("return 'USD'","return 'EUR'"));
+    writeFileSync(join(workspace,'scratch.txt'),'Existing untracked scratch work. Preserve it.\n');
+  }
+  write(join(workspace,'.git/info/exclude'),'_instructions/\n__pycache__/\n');
+  const instructions=[];
+  if(arm.startsWith('just-vibe')){
+    cpSync(pluginRoot,join(workspace,'_instructions/just-vibe'),{recursive:true});
+    for(const id of fixture.commands)instructions.push(`_instructions/just-vibe/skills/${id}/SKILL.md`);
+    if(arm==='just-vibe-profile')instructions.push(`_instructions/just-vibe/references/profiles/${fixture.profile}.md`);
+  }else if(arm==='ecc'){
+    for(const path of fixture.ecc){
+      const target=join(workspace,'_instructions/ecc',path);mkdirSync(dirname(target),{recursive:true});
+      if(path.startsWith('skills/'))cpSync(dirname(join(eccRoot,path)),dirname(target),{recursive:true});else cpSync(join(eccRoot,path),target);
+      instructions.push(`_instructions/ecc/${path}`);
+    }
+    if(existsSync(join(eccRoot,'LICENSE')))cpSync(join(eccRoot,'LICENSE'),join(workspace,'_instructions/ecc/LICENSE'));
+  }
+  const task=readFileSync(join(workspace,'TASK.md'),'utf8');
+  const role=arm==='just-vibe-profile'?`Use ${fixture.profile} as the explicitly selected primary task profile. Preserve the task's scope and constraints.`:arm==='just-vibe'?'For this controlled trial, profiles are disabled: do not select or read a role profile.':'';
+  const prompt=`Complete the task in this fresh repository: ${workspace}.\n${instructions.length?`Read these supplied instructions, then inspect the repository: ${instructions.join(', ')}.`:'No toolkit instructions are supplied; use the task and project evidence.'}\n${role}\n\n${task}\n\nTrial constraints: work only in this repository. Do not read other trials, source toolkit checkouts, evaluator files, authentication/config directories or prior answers. Do not use network, install dependencies, spawn agents, change host settings or perform external actions. You may run local tests and create temporary test artifacts, removing only artifacts you create. Preserve user files. Python is available at ${python()} on this trial host; do not use a no-op shim on PATH. Git commits are authorized only when TASK.md explicitly requests one. Never add agent self-attribution. Finish with a concise report of changes, actual checks and limitations. Do not claim hidden checks ran. No follow-up user answers will be supplied; make reasonable decisions within the written contract or report a concrete blocker.\n`;
+  write(join(root,'prompt.txt'),prompt);
+  const manifest={schemaVersion:1,case:id,arm,repetition,workspace,base,allowedWrites:fixture.allowedWrites,gitCommit:Boolean(fixture.gitCommit),preparedAt:new Date().toISOString(),instructions,inputs:snapshot(workspace),promptSha256:hash(prompt),profile:arm==='just-vibe-profile'?fixture.profile:null,sourceVersion:json(join(pluginRoot,'.codex-plugin/plugin.json')).version};
+  if(arm==='ecc')manifest.eccRevision=git(eccRoot,['rev-parse','HEAD']);
+  save(join(root,'run.json'),manifest);return manifest;
+}
+export function prepareStudy({out,eccRoot,repetitions=2,seed=5192026,selectedCases=cases.map(c=>c.id),selectedArms=arms}){
+  if(existsSync(out))throw Error('Study output must be fresh.');
+  if(!Number.isInteger(repetitions)||repetitions<1||repetitions>10)throw Error('Repetitions must be 1–10.');
+  if(!Number.isInteger(seed)||!selectedCases.length||!selectedArms.length||new Set(selectedCases).size!==selectedCases.length||new Set(selectedArms).size!==selectedArms.length||selectedCases.some(id=>!cases.some(c=>c.id===id))||selectedArms.some(arm=>!arms.includes(arm)))throw Error('Unique known cases/arms and an integer seed are required.');
+  const jobs=[];for(const id of selectedCases)for(let repetition=1;repetition<=repetitions;repetition++)for(const arm of selectedArms)jobs.push({id,arm,repetition});
+  mkdirSync(out,{recursive:true});const order=shuffle(jobs,seed);
+  for(const job of order){job.directory=`${job.id}/${job.arm}/${job.repetition}`;prepareTrial({out:join(out,job.directory),id:job.id,arm:job.arm,repetition:job.repetition,eccRoot});}
+  const source=sourceIdentity();
+  const plan={schemaVersion:1,preparedAt:new Date().toISOString(),seed,repetitions,sourceRevision:source.revision,sourceDirty:source.dirty,eccRevision:selectedArms.includes('ecc')?git(eccRoot,['rev-parse','HEAD']):null,cases:selectedCases,arms:selectedArms,order,oracleHashes:snapshot(join(here,'oracles')),fixtureHashes:snapshot(join(here,'repos')),harnessSha256:hash(readFileSync(fileURLToPath(import.meta.url))),note:'Author-created multi-file fixture repositories; supplied instructions, not full native plugin hooks. Fresh contexts, randomized order and immutable trial inputs. Public tasks are not a permanent blind benchmark.'};
+  save(join(out,'study.json'),plan);return plan;
+}
+export function parseEvents(text){
+  const events=[],invalid=[];for(const line of text.split('\n').filter(Boolean)){try{events.push(JSON.parse(line))}catch{invalid.push(line)}}
+  const completed=events.filter(e=>e.type==='turn.completed'),failed=events.some(e=>e.type==='turn.failed');
+  const usage=completed.length?{}:null;
+  if(usage)for(const e of completed)for(const [key,value]of Object.entries(e.usage||{}))if(typeof value==='number')usage[key]=(usage[key]||0)+value;
+  const items=events.filter(e=>e.type==='item.completed').map(e=>e.item);
+  return {usage,completed:completed.length>0&&!failed,invalidLines:invalid.length,toolCalls:items.filter(i=>['command_execution','file_change','mcp_tool_call','web_search'].includes(i?.type)).length,
+    failedCommands:items.filter(i=>i?.type==='command_execution'&&typeof i.exit_code==='number'&&i.exit_code!==0).length,
+    errors:events.filter(e=>['error','turn.failed'].includes(e.type)).map(e=>e.message||e.error?.message),
+    finalMessages:items.filter(i=>i?.type==='agent_message').map(i=>i.text)};
+}
+function cleanEnv(){const env={...process.env,PYTHONDONTWRITEBYTECODE:'1'};for(const key of ['NODE_OPTIONS','NODE_TEST_CONTEXT','NODE_V8_COVERAGE'])delete env[key];return env;}
+function python(){return process.env.JUST_VIBE_PYTHON||(process.platform==='darwin'?'/usr/bin/python3':process.platform==='win32'?'python':'python3');}
+export function regressionSensitivity(language,result){
+  if(result.status===0||result.status===undefined)return false;
+  if(language==='python'){
+    for(const line of (result.stdout||'').split('\n'))try{
+      const report=JSON.parse(line);
+      if(report.type==='regression-failure'&&report.behavior_failure===true)return true;
+      if(report.tests>0&&report.failures?.some(f=>f.behavior_failure===true))return true;
+    }catch{}
+    return false;
+  }
+  // TAP emits completed assertion evidence immediately, even if a later test hangs.
+  return result.status!==0&&/code: ['"]ERR_ASSERTION['"]/.test(result.stdout||'');
+}
+export function gradeTrial(directory){
+  const root=resolve(directory),manifest=json(join(root,'run.json')),workspace=realpathSync(join(root,'workspace'));
+  if(workspace!==realpathSync(manifest.workspace))throw Error('Workspace identity mismatch.');
+  const current=snapshot(workspace),changed=[...new Set([...Object.keys(current),...Object.keys(manifest.inputs)])].filter(p=>current[p]!==manifest.inputs[p]);
+  const checks=[{name:'only permitted files changed',pass:changed.every(p=>manifest.allowedWrites.includes(p))},{name:'no symlink artifacts',pass:!Object.values(current).includes('symlink')}];
+  const fixture=cases.find(c=>c.id===manifest.case);const head=git(workspace,['rev-parse','HEAD']);
+  if(!fixture.gitCommit)checks.push({name:'no unrequested commits',pass:head===manifest.base});
+  if(checks.every(c=>c.pass)){
+    const file=fixture.language==='python'?'python.py':'node.mjs';const binary=fixture.language==='python'?python():process.execPath;
+    const result=spawnSync(binary,[join(here,'oracles',file),fixture.id,workspace],{encoding:'utf8',timeout:20000,env:cleanEnv(),maxBuffer:1024*1024});
+    try{const evidence=JSON.parse(result.stdout);if(!Array.isArray(evidence.checks)||!evidence.checks.length||evidence.checks.some(c=>typeof c.name!=='string'||typeof c.pass!=='boolean'))throw Error('Invalid oracle report');checks.push(...evidence.checks);checks.push({name:'oracle process completed consistently',pass:!result.error&&result.status===(evidence.checks.every(c=>c.pass)?0:1),detail:result.error?.message});}
+    catch{checks.push({name:'independent behavior assertions',pass:false,detail:result.error?.message||result.stderr||'Oracle produced no report'});}
+  }else checks.push({name:'behavior execution withheld for input integrity failure',pass:false});
+  const regression=fixture.language==='python'?'test/test_regression.py':'test/regression.test.mjs';
+  checks.push({name:'regression test artifact exists',pass:existsSync(join(workspace,regression))});
+  if(existsSync(join(workspace,regression))&&checks.slice(0,2).every(c=>c.pass)){
+    const args=fixture.language==='python'?['-m','unittest','discover','-s','test']:['--test','test/'];
+    // Node's explicit file list works consistently across supported versions.
+    if(fixture.language==='node')args.splice(1,1,...readdirSync(join(workspace,'test')).filter(p=>p.endsWith('.test.mjs')).map(p=>`test/${p}`));
+    if(fixture.language==='node')args.splice(1,0,'--test-reporter=tap');
+    const result=spawnSync(fixture.language==='python'?python():process.execPath,args,{cwd:workspace,encoding:'utf8',timeout:15000,env:cleanEnv()});
+    checks.push({name:'project and authored regression tests pass',pass:result.status===0,detail:result.status===0?undefined:(result.error?.message||result.stdout+result.stderr)});
+    const mutation=join(root,'mutation-workspace');if(existsSync(mutation))rmSync(mutation,{recursive:true,force:true});
+    cpSync(workspace,mutation,{recursive:true,filter:src=>!['.git','_instructions'].includes(src.split(/[\\/]/).at(-1))});
+    for(const p of fixture.allowedWrites.filter(p=>p.startsWith('src/'))){
+      let original=readFileSync(join(here,'repos',fixture.id,p),'utf8');
+      if(fixture.gitCommit)original=original.replace('`Invoice ${id}`','`Receipt ${id}`').replace("return 'USD'","return 'EUR'");
+      writeFileSync(join(mutation,p),original);
+    }
+    const mutantArgs=fixture.language==='python'?[join(here,'support/python-test-report.py'),'--stream']:args;
+    const mutant=spawnSync(fixture.language==='python'?python():process.execPath,mutantArgs,{cwd:mutation,encoding:'utf8',timeout:15000,env:cleanEnv()});
+    save(join(root,'test-evidence.json'),{scorerVersion:3,fixed:{status:result.status,error:result.error?.message,stdout:result.stdout,stderr:result.stderr},original:{status:mutant.status,error:mutant.error?.message,stdout:mutant.stdout,stderr:mutant.stderr}});
+    checks.push({name:'regression tests detect original defect',pass:regressionSensitivity(fixture.language,mutant),detail:mutant.status===0?'Original defective code passed authored tests':mutant.error?.message});
+    rmSync(mutation,{recursive:true,force:true});
+  }
+  if(fixture.gitCommit){
+    try{
+    checks.push({name:'exactly one requested commit',pass:git(workspace,['rev-list','--count',`${manifest.base}..HEAD`])==='1'});
+    const paths=git(workspace,['diff','--name-only',manifest.base,'HEAD']).split('\n').filter(Boolean).sort();
+    checks.push({name:'commit includes only requested fix and new test',pass:JSON.stringify(paths)===JSON.stringify(['src/invoice.mjs','test/regression.test.mjs'])});
+    const committed=git(workspace,['show','HEAD:src/invoice.mjs']),index=git(workspace,['show',':src/invoice.mjs']),worktree=readFileSync(join(workspace,'src/invoice.mjs'),'utf8');
+    checks.push({name:'same-file staged and unstaged distinctions survive',pass:committed.includes('`Invoice ${id}`')&&committed.includes("return 'USD'")&&index.includes('`Receipt ${id}`')&&index.includes("return 'USD'")&&worktree.includes('`Receipt ${id}`')&&worktree.includes("return 'EUR'")});
+    const candidate=join(root,'committed-workspace');
+    try{
+      if(existsSync(candidate))rmSync(candidate,{recursive:true,force:true});mkdirSync(candidate);
+      for(const path of git(workspace,['ls-tree','-r','--name-only','-z','HEAD']).split('\0').filter(Boolean)){
+        if(relative(candidate,resolve(candidate,path)).startsWith('..'))throw Error('Invalid committed path');
+        write(join(candidate,path),execFileSync('git',['show',`HEAD:${path}`],{cwd:workspace}));
+      }
+      const behavior=spawnSync(process.execPath,[join(here,'support/commit-tree.mjs'),candidate],{encoding:'utf8',timeout:10000,env:cleanEnv()});
+      checks.push({name:'committed implementation satisfies the contract independently',pass:behavior.status===0,detail:behavior.status===0?undefined:behavior.error?.message||behavior.stderr});
+      const candidateTests=spawnSync(process.execPath,['--test',...readdirSync(join(candidate,'test')).filter(p=>p.endsWith('.test.mjs')).map(p=>`test/${p}`)],{cwd:candidate,encoding:'utf8',timeout:15000,env:cleanEnv()});
+      checks.push({name:'committed tests pass without unrelated worktree edits',pass:candidateTests.status===0,detail:candidateTests.status===0?undefined:candidateTests.error?.message||candidateTests.stdout+candidateTests.stderr});
+    }finally{rmSync(candidate,{recursive:true,force:true});}
+    const message=git(workspace,['log','-1','--format=%B']),identity=git(workspace,['log','-1','--format=%an <%ae>|%cn <%ce>']);
+    checks.push({name:'user identity and no agent attribution',pass:identity==='Fixture User <fixture@example.invalid>|Fixture User <fixture@example.invalid>'&&!/co-authored-by|generated.by|\b(?:claude|codex|openai)\b/i.test(message)});
+    checks.push({name:'unrelated staged note and untracked file remain',pass:git(workspace,['status','--porcelain','--','notes.md','scratch.txt'])==='A  notes.md\n?? scratch.txt'});
+    }catch(error){checks.push({name:'requested Git state is readable',pass:false,detail:error.message});}
+  }
+  const metrics=existsSync(join(root,'metrics.json'))?json(join(root,'metrics.json')):null;
+  const artifactCorrect=checks.every(c=>c.pass);
+  checks.push({name:'agent turn completed within limit',pass:Boolean(metrics?.turnCompleted&&metrics.exitCode===0&&!metrics.timedOut&&!metrics.cancelled&&!metrics.spawnError)});
+  const result={schemaVersion:1,scorerVersion:3,case:manifest.case,arm:manifest.arm,repetition:manifest.repetition,changed,checks,artifactCorrect,correct:checks.every(c=>c.pass),metrics,gradedAt:new Date().toISOString(),promptSha256:manifest.promptSha256,instructionHashes:Object.fromEntries(Object.entries(manifest.inputs).filter(([p])=>p.startsWith('_instructions/')))};
+  if(existsSync(join(root,'grade.json'))){const version=json(join(root,'grade.json')).scorerVersion??1;if(version<3&&!existsSync(join(root,`grade-v${version}.json`)))cpSync(join(root,'grade.json'),join(root,`grade-v${version}.json`));}
+  save(join(root,'grade.json'),result);return result;
+}
+export async function runTrial(directory,{codex,authHome,model,effort='medium',seconds=240,signal}){
+  const root=resolve(directory),manifest=json(join(root,'run.json'));
+  if(JSON.stringify(snapshot(manifest.workspace))!==JSON.stringify(manifest.inputs))throw Error('Prepared trial inputs changed.');
+  if(['metrics.json','events.jsonl','stderr.log'].some(p=>existsSync(join(root,p))))throw Error('Attempts are immutable; prepare a fresh trial to repeat.');
+  if(!model||!codex||!authHome||!Number.isFinite(seconds)||seconds<1)throw Error('Explicit model, CLI, auth home and time limit required.');
+  if(signal?.aborted)throw Error('Trial cancelled before launch.');
+  const auth=join(resolve(authHome),'auth.json');if(!existsSync(auth))throw Error('Saved CLI authentication is unavailable.');
+  const cliVersion=execFileSync(codex,['--version'],{encoding:'utf8'}).trim();
+  const host=join(root,'host');mkdirSync(host,{recursive:true});
+  symlinkSync(auth,join(host,'auth.json'));write(join(host,'empty-gitconfig'),'');
+  const args=['exec','--ignore-user-config','--ignore-rules','--ephemeral','--json','--sandbox','workspace-write','--add-dir',join(manifest.workspace,'.git'),'--enable','skip_host_skill_discovery','--disable','plugins','--disable','apps','--disable','multi_agent','--disable','hooks','-m',model,'-c',`model_reasoning_effort=${JSON.stringify(effort)}`,'-c','web_search="disabled"','-C',manifest.workspace,'-o',join(root,'answer.md'),'-'];
+  const env={};for(const key of ['PATH','HOME','USER','LOGNAME','TMPDIR','LANG','LC_ALL','SHELL','SystemRoot','ComSpec','PATHEXT'])if(process.env[key])env[key]=process.env[key];
+  Object.assign(env,{CODEX_HOME:host,PYTHONDONTWRITEBYTECODE:'1',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:join(host,'empty-gitconfig')});
+  const startedAt=new Date().toISOString(),start=performance.now();let stdout='',stderr='',timedOut=false,cancelled=false;
+  const child=spawn(codex,args,{cwd:manifest.workspace,env,stdio:['pipe','pipe','pipe'],detached:process.platform!=='win32'});
+  const kill=signal=>{try{if(process.platform!=='win32')process.kill(-child.pid,signal);else child.kill(signal)}catch{}};
+  let hardTimer;const terminate=()=>{kill('SIGTERM');hardTimer??=setTimeout(()=>kill('SIGKILL'),3000);};
+  const abort=()=>{cancelled=true;terminate();};signal?.addEventListener('abort',abort,{once:true});
+  const timer=setTimeout(()=>{timedOut=true;terminate();},seconds*1000);
+  child.stdout.on('data',b=>{stdout+=b;writeFileSync(join(root,'events.jsonl'),stdout)});child.stderr.on('data',b=>{stderr+=b;writeFileSync(join(root,'stderr.log'),stderr)});
+  let spawnError;child.on('error',e=>{spawnError=e.message});child.stdin.on('error',()=>{});child.stdin.end(readFileSync(join(root,'prompt.txt')));
+  const exitCode=await new Promise(done=>child.on('close',done));clearTimeout(timer);clearTimeout(hardTimer);signal?.removeEventListener('abort',abort);
+  const parsed=parseEvents(stdout);const metrics={model,effort,startedAt,finishedAt:new Date().toISOString(),wallMs:Math.round(performance.now()-start),exitCode,exitSignal:child.signalCode??null,timedOut,cancelled,spawnError,usage:parsed.usage,turnCompleted:parsed.completed,invalidEventLines:parsed.invalidLines,toolCalls:parsed.toolCalls,failedCommands:parsed.failedCommands,errors:parsed.errors,userInterventions:0,costUsd:null,costReason:'CLI reports tokens, not a verified per-run monetary charge.',host:{cli:cliVersion,sandbox:'workspace-write',writableGitMetadata:true,externalTools:false,plugins:false,hostSkillDiscovery:false}};
+  try{save(join(root,'metrics.json'),metrics);}finally{rmSync(host,{recursive:true,force:true});}
+  return gradeTrial(root);
+}
+export async function runStudy(out,options){
+  const plan=json(join(out,'study.json'));if(JSON.stringify(plan.oracleHashes)!==JSON.stringify(snapshot(join(here,'oracles'))))throw Error('Oracle changed after study preparation.');
+  if(JSON.stringify(plan.fixtureHashes)!==JSON.stringify(snapshot(join(here,'repos'))))throw Error('Fixture changed after study preparation.');
+  if(!options.model||!options.codex||!options.authHome||!Number.isFinite(options.seconds)||options.seconds<1)throw Error('Explicit model, CLI, auth home and positive time limit required before starting a study.');
+  if(!existsSync(join(resolve(options.authHome),'auth.json')))throw Error('Saved CLI authentication is unavailable; no trials started.');
+  execFileSync(options.codex,['--version'],{encoding:'utf8',stdio:['ignore','pipe','pipe']});
+  let next=0;const results=[];async function worker(){while(next<plan.order.length&&!options.signal?.aborted){
+    const job=plan.order[next++],directory=join(out,job.directory);let result;
+    try{result=existsSync(join(directory,'metrics.json'))?gradeTrial(directory):await runTrial(directory,options);}
+    catch(error){result={schemaVersion:1,case:job.id,arm:job.arm,repetition:job.repetition,correct:false,artifactCorrect:false,metrics:existsSync(join(directory,'metrics.json'))?json(join(directory,'metrics.json')):null,checks:[{name:'harness execution completed',pass:false,detail:error.message}]};save(join(directory,'harness-error.json'),result);}
+    results.push(result);console.log(JSON.stringify({case:job.id,arm:job.arm,repetition:job.repetition,correct:result.correct,wallMs:result.metrics?.wallMs,tokens:result.metrics?.usage,failed:result.checks.filter(c=>!c.pass).map(c=>c.name)}));
+  }}
+  const concurrency=options.concurrency??2;if(!Number.isInteger(concurrency)||concurrency<1||concurrency>4)throw Error('Concurrency must be 1–4.');
+  await Promise.all(Array.from({length:concurrency},worker));save(join(out,'results.json'),results);return results;
+}
+export function options(argv){
+  const [operation,...rest]=argv,result={};
+  const allowed={prepare:['out','ecc-root','repetitions','seed','cases','arms'],run:['study','codex','auth-home','model','effort','seconds','concurrency'],grade:['trial']}[operation];
+  if(!allowed)throw Error('Use prepare, run or grade.');
+  for(let i=0;i<rest.length;i+=2){if(!rest[i].startsWith('--')||!rest[i+1]||rest[i+1].startsWith('--'))throw Error('Options require values.');const key=rest[i].slice(2);if(!allowed.includes(key))throw Error(`Unknown ${operation} option: ${rest[i]}`);if(key in result)throw Error('Duplicate option');result[key]=rest[i+1];}
+  for(const key of {prepare:['out'],run:['study','codex','auth-home','model'],grade:['trial']}[operation])if(!result[key])throw Error(`Missing --${key}`);
+  return {operation,o:result};
+}
+if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url)){
+  try{const {operation,o}=options(process.argv.slice(2));
+    if(operation==='prepare')console.log(JSON.stringify(prepareStudy({out:o.out,eccRoot:o['ecc-root'],repetitions:Number(o.repetitions??2),seed:Number(o.seed??5192026),...(o.cases?{selectedCases:o.cases.split(',')}:{}),...(o.arms?{selectedArms:o.arms.split(',')}:{})}),null,2));
+    else if(operation==='run'){
+      const controller=new AbortController(),abort=()=>controller.abort();process.on('SIGINT',abort);process.on('SIGTERM',abort);
+      try{await runStudy(o.study,{codex:resolve(o.codex),authHome:resolve(o['auth-home']),model:o.model,effort:o.effort||'medium',seconds:Number(o.seconds??240),concurrency:Number(o.concurrency??2),signal:controller.signal});}
+      finally{process.removeListener('SIGINT',abort);process.removeListener('SIGTERM',abort);if(controller.signal.aborted)process.exitCode=130;}
+    }
+    else if(operation==='grade')console.log(JSON.stringify(gradeTrial(o.trial),null,2));
+    else throw Error('Use prepare --out DIR --ecc-root DIR, run --study DIR --codex PATH --auth-home DIR --model ID, or grade --trial DIR.');
+  }catch(error){console.error(error.stack);process.exitCode=1;}
+}

package/evals/benchmark/oracles/node.mjs

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict';
+import { pathToFileURL } from 'node:url';
+import { join } from 'node:path';
+const [id, root] = process.argv.slice(2), checks = [];
+async function check(name, fn) { try { await fn(); checks.push({name,pass:true}); } catch(error) { checks.push({name,pass:false,detail:error.stack}); } }
+const deferred = () => { let resolve,reject;const promise=new Promise((a,b)=>{resolve=a;reject=b});return {promise,resolve,reject}; };
+const tick = () => new Promise(resolve=>setImmediate(resolve));
+const bounded = promise => Promise.race([promise,new Promise((_,reject)=>{const timer=setTimeout(()=>reject(Error('Promise did not settle promptly')),300);})]);
+if (id === 'async-cache') {
+  const {createCache}=await import(pathToFileURL(join(root,'src/cache.mjs')));
+  const {createView}=await import(pathToFileURL(join(root,'src/view.mjs')));
+  await check('tenant identity, delimiter collisions and falsey values',async()=>{
+    const cache=createCache(async(t,k)=>`${t}|${k}`);
+    assert.equal(await cache.get('a','x'),'a|x');assert.equal(await cache.get('b','x'),'b|x');
+    assert.equal(await cache.get('a:b','c'),'a:b|c');assert.equal(await cache.get('a','b:c'),'a|b:c');
+    let calls=0;const empty=createCache(async()=>{calls++;return undefined;});
+    assert.equal(await empty.get('a','e'),undefined);assert.equal(await empty.get('a','e'),undefined);assert.equal(calls,1);
+  });
+  await check('pending coalescing and expiry measured from completion',async()=>{
+    let now=0,calls=0;const gate=deferred();const cache=createCache(()=>{calls++;return gate.promise;},{ttl:10,now:()=>now});
+    const first=cache.get('t','x'),second=cache.get('t','x');await tick();assert.equal(calls,1);
+    now=50;gate.resolve(0);assert.deepEqual(await Promise.all([first,second]),[0,0]);
+    now=59;assert.equal(await cache.get('t','x'),0);assert.equal(calls,1);
+    now=60;await cache.get('t','x');assert.equal(calls,2);
+    let n=0;const zero=createCache(async()=>++n,{ttl:0,now:()=>0});
+    assert.deepEqual(await Promise.all([zero.get('t','z'),zero.get('t','z')]),[1,1]);await zero.get('t','z');assert.equal(n,2);
+  });
+  await check('one waiter cancellation cannot poison shared work',async()=>{
+    const gate=deferred(),controller=new AbortController();let calls=0;
+    const cache=createCache((t,k,options)=>{calls++;assert.ok(!options?.signal);return gate.promise;});
+    const first=cache.get('t','k',{signal:controller.signal});const second=cache.get('t','k');
+    const rejected=assert.rejects(bounded(first),e=>e.name==='AbortError');controller.abort();await rejected;
+    gate.resolve('value');assert.equal(await second,'value');assert.equal(await cache.get('t','k'),'value');assert.equal(calls,1);
+  });
+  await check('pre-abort and listener cleanup on settlement',async()=>{
+    let calls=0;const cache=createCache(()=>++calls);const aborted=new AbortController();aborted.abort();
+    await assert.rejects(()=>cache.get('t','k',{signal:aborted.signal}),e=>e.name==='AbortError');assert.equal(calls,0);
+    let listeners=new Set();const signal={aborted:false,addEventListener(type,fn){assert.equal(type,'abort');listeners.add(fn)},removeEventListener(type,fn){listeners.delete(fn)}};
+    assert.equal(await cache.get('t','k',{signal}),1);assert.equal(listeners.size,0);
+    const failed=createCache(()=>Promise.reject(Error('backend')));await assert.rejects(()=>failed.get('t','k',{signal}),/backend/);assert.equal(listeners.size,0);
+  });
+  await check('detached old success cannot overwrite or delete new generation',async()=>{
+    const a=deferred(),b=deferred();let n=0;const cache=createCache(()=>++n===1?a.promise:b.promise);
+    const old=cache.get('t','k');await tick();cache.invalidate('t','k');const fresh=cache.get('t','k');await tick();
+    b.resolve('new');assert.equal(await fresh,'new');a.resolve('old');assert.equal(await old,'old');assert.equal(await cache.get('t','k'),'new');assert.equal(n,2);
+  });
+  await check('detached old failure cannot clear newer pending work',async()=>{
+    const a=deferred(),b=deferred();let n=0;const cache=createCache(()=>++n===1?a.promise:b.promise);
+    const old=cache.get('t','k');const rejection=assert.rejects(old,/old failure/);await tick();cache.invalidate('t','k');const fresh=cache.get('t','k');await tick();
+    a.reject(Error('old failure'));await rejection;const joined=cache.get('t','k');await tick();assert.equal(n,2);b.resolve('fresh');assert.deepEqual(await Promise.all([fresh,joined]),['fresh','fresh']);
+  });
+  await check('synchronous failures and asynchronous failures are retryable',async()=>{
+    let calls=0;const cache=createCache(()=>{calls++;if(calls===1)throw Error('sync');if(calls===2)return Promise.reject(Error('async'));return null;});
+    await assert.rejects(()=>cache.get('t','k'),/sync/);await assert.rejects(()=>cache.get('t','k'),/async/);assert.equal(await cache.get('t','k'),null);assert.equal(calls,3);
+  });
+  await check('latest view result, stale failure and permanent disposal',async()=>{
+    const requests=[],events=[];const view=createView({get(){const d=deferred();requests.push(d);return d.promise;}},value=>events.push(value));
+    const a=view.select('t','a'),b=view.select('t','b');requests[1].resolve('B');await b;requests[0].resolve('A');await a;assert.equal(events.at(-1).value,'B');
+    const c=view.select('t','c'),d=view.select('t','d');requests[3].resolve('D');await d;requests[2].reject(Error('old'));await c;assert.equal(events.at(-1).value,'D');
+    const e=view.select('t','e');view.dispose();const length=events.length;requests[4].resolve('E');await e;assert.equal(events.length,length);
+    await bounded(view.select('t','after-dispose'));assert.equal(requests.length,5);assert.equal(events.length,length);
+  });
+} else if(id==='scoped-commit') {
+  const {total,formatLabel,currency}=await import(pathToFileURL(join(root,'src/invoice.mjs')));
+  await check('missing and zero adjustment',()=>{assert.equal(total([{price:20,quantity:2}],null),40);assert.equal(total([{price:20,quantity:2}]),40);assert.equal(total([{price:20,quantity:2}],{amount:0}),40)});
+  await check('negative surcharge and no-clamp compatibility',()=>{assert.equal(total([{price:20,quantity:2}],{amount:-7}),47);assert.equal(total([],{amount:7}),-7)});
+  await check('unrelated worktree behavior preserved',()=>{assert.equal(formatLabel('a'),'Receipt a');assert.equal(currency(),'EUR')});
+} else throw Error('Unknown Node oracle');
+console.log(JSON.stringify({checks}));process.exitCode=checks.every(c=>c.pass)?0:1;

package/evals/benchmark/oracles/python.py

@@ -0,0 +1,117 @@
+import sys, json, tempfile, sqlite3, threading, copy, math
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+case, root = sys.argv[1:]
+sys.path.insert(0,root)
+checks=[]
+def check(name,fn):
+    try:
+        fn(); checks.append({'name':name,'pass':True})
+    except Exception as error:
+        checks.append({'name':name,'pass':False,'detail':type(error).__name__+': '+str(error)})
+def equal(actual,expected):
+    assert actual==expected,(actual,expected)
+def raises(kind,fn):
+    try: fn()
+    except kind: return
+    raise AssertionError('Expected '+kind.__name__)
+if case=='ledger':
+    from src.store import connect,initialize
+    from src.service import transfer
+    def db(path=':memory:'):
+        conn=connect(path);initialize(conn)
+        conn.executemany('INSERT INTO accounts VALUES(?,?,?)',[('t','a',100),('t','b',0),('other','a',100),('other','b',0)])
+        conn.commit();return conn
+    def balances(conn): return conn.execute('SELECT tenant,id,balance FROM accounts ORDER BY tenant,id').fetchall()
+    def atomic():
+        conn=db();before=balances(conn)
+        def fail(): raise LookupError('interrupted')
+        raises(LookupError,lambda:transfer(conn,'t','k','a','b',25,after_debit=fail))
+        equal(balances(conn),before);equal(conn.execute('SELECT count(*) FROM transfers').fetchone()[0],0);equal(conn.in_transaction,False)
+        equal(transfer(conn,'t','k','a','b',25)['amount'],25)
+    check('interruption rolls back all state and permits retry',atomic)
+    def replay():
+        conn=db();first=transfer(conn,'t','k','a','b',25);before=balances(conn)
+        equal(transfer(conn,'t','k','a','b',25,after_debit=lambda:(_ for _ in ()).throw(AssertionError('replayed effect'))),first)
+        equal(balances(conn),before)
+        raises(ValueError,lambda:transfer(conn,'t','k','a','b',30));raises(ValueError,lambda:transfer(conn,'t','k','b','a',25))
+        equal(balances(conn),before)
+        equal(transfer(conn,'other','k','a','b',7)['amount'],7)
+    check('payload conflicts and tenant-scoped replay',replay)
+    def invalid():
+        for amount in [True,False,'3',3.1,0,-1,2**63]:
+            conn=db();before=balances(conn);raises(ValueError,lambda:transfer(conn,'t','k','a','b',amount));equal(balances(conn),before);equal(conn.in_transaction,False)
+    check('exact integer amount validation without writes',invalid)
+    def accounts():
+        for source,dest,amount in [('missing','b',1),('a','missing',1),('a','a',1),('a','b',101)]:
+            conn=db();before=balances(conn);raises(ValueError,lambda:transfer(conn,'t','k',source,dest,amount));equal(balances(conn),before);equal(conn.in_transaction,False)
+        conn=db();conn.execute('UPDATE accounts SET balance=? WHERE tenant=? AND id=?',(2**63-1,'t','b'));conn.commit();before=balances(conn)
+        raises(ValueError,lambda:transfer(conn,'t','k','a','b',1));equal(balances(conn),before);equal(conn.in_transaction,False)
+    check('account ownership, funds and credit overflow',accounts)
+    def caller():
+        conn=db();conn.execute('UPDATE accounts SET balance=99 WHERE tenant="t" AND id="a"')
+        before=balances(conn);raises(RuntimeError,lambda:transfer(conn,'t','k','a','b',1));equal(balances(conn),before);equal(conn.in_transaction,True)
+        conn.rollback();equal(conn.execute('SELECT balance FROM accounts WHERE tenant="t" AND id="a"').fetchone()[0],100)
+    check('caller-owned transaction remains untouched',caller)
+    def concurrent():
+        with tempfile.TemporaryDirectory() as temp:
+            path=str(Path(temp)/'ledger.sqlite');conn=db(path);conn.close();barrier=threading.Barrier(4)
+            def worker(_):
+                c=connect(path);barrier.wait()
+                try:return transfer(c,'t','shared','a','b',20)
+                finally:c.close()
+            with ThreadPoolExecutor(4) as pool:results=list(pool.map(worker,range(4)))
+            equal(results,[{'tenant':'t','key':'shared','source':'a','destination':'b','amount':20}]*4)
+            c=connect(path)
+            try:
+                equal(c.execute('SELECT balance FROM accounts WHERE tenant="t" AND id="a"').fetchone()[0],80)
+                equal(c.execute('SELECT count(*) FROM transfers').fetchone()[0],1)
+            finally:
+                c.close()
+    check('concurrent connections serialize one effect',concurrent)
+elif case=='temporal-ml':
+    from src.features import features_for
+    from src.pipeline import prepare
+    from src.report import summarize
+    def e(identity,entity,at,available,value,revision=1):
+        return dict(event_id=identity,entity_id=entity,event_at=at,available_at=available,value=value,revision=revision)
+    def row(identity,entity,at,label=0,observed='2026-01-09T00:00:00Z'):
+        return dict(example_id=identity,entity_id=entity,prediction_at=at,label=label,label_observed_at=observed)
+    def boundaries():
+        examples=[row('x','a','2026-01-08T00:00:00Z')]
+        events=[e('lower','a','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z',99),e('upper','a','2026-01-08T00:00:00Z','2026-01-08T00:00:00Z',2),e('future','a','2026-01-08T00:00:01Z','2026-01-08T00:00:00Z',100),e('late','a','2026-01-07T00:00:00Z','2026-01-09T00:00:00Z',100)]
+        equal(features_for(examples,events),[{'example_id':'x','count':1,'sum':2}])
+    check('window bounds and event availability',boundaries)
+    def versions():
+        examples=[row('x','a','2026-01-08T00:00:00Z')]
+        events=[e('x','a','2026-01-07T00:00:00Z','2026-01-07T00:00:00Z',2),e('x','a','2025-12-20T00:00:00Z','2026-01-07T12:00:00Z',20,2),e('y','a','2026-01-07T00:00:00Z','2026-01-07T00:00:00Z',-2),e('y','b','2026-01-07T00:00:00Z','2026-01-07T00:00:00Z',100)]
+        events.append(copy.deepcopy(events[2]));before=copy.deepcopy((examples,events))
+        equal(features_for(examples,events),[{'example_id':'x','count':1,'sum':-2}]);equal((examples,events),before)
+    check('revision selection before window filtering and deduplication',versions)
+    def zones():
+        events=[e('x','a','2026-01-07T23:00:00+01:00','2026-01-07T22:00:00Z',3),e('future','a','2026-01-07T23:30:00Z','2026-01-07T20:00:00Z',50)]
+        equal(features_for([row('x','a','2026-01-08T00:00:00+02:00')],events),[{'example_id':'x','count':1,'sum':3}])
+        raises(ValueError,lambda:features_for([row('x','a','2026-01-08T00:00:00')],events))
+    check('offset-aware instant comparisons and naive rejection',zones)
+    examples=[row('one','a','2026-01-08T00:00:00Z'),row('two','b','2026-01-09T00:00:00Z',1),row('immature','c','2026-01-09T00:00:00Z',1,'2026-01-11T00:00:00Z'),row('v','a','2026-01-10T00:00:00Z',0,'2026-01-20T00:00:00Z'),row('test','b','2026-01-12T00:00:00Z',0,'2026-01-13T00:00:00Z'),row('future','x','2026-01-20T00:00:00Z',1)]
+    events=[e('a','a','2026-01-07T00:00:00Z','2026-01-07T00:00:00Z',2),e('a','a','2026-01-07T00:00:00Z','2026-01-09T00:00:00Z',20,2),e('b','b','2026-01-08T00:00:00Z','2026-01-08T00:00:00Z',4),e('c','c','2026-01-08T00:00:00Z','2026-01-08T00:00:00Z',100)]
+    def output():return prepare(examples,events,'2026-01-10T00:00:00Z','2026-01-12T00:00:00Z','2026-01-15T00:00:00Z')
+    def training():
+        result=output();equal([r['example_id'] for r in result['train']],['one','two']);equal(result['scaler'],{'mean':3,'scale':1});equal([r['normalized'] for r in result['train']],[-1,1]);equal(result['validation'][0]['normalized'],17)
+    check('mature training membership and train-only scaler',training)
+    def labels():
+        result=output();equal(result['train'][0]['label'],0);equal(result['validation'][0]['label'],None);equal(result['test'][0]['label'],0)
+        equal(summarize(result),{'train':{'rows':2,'labeled':2},'validation':{'rows':1,'labeled':0},'test':{'rows':1,'labeled':1}})
+    check('zero labels, unknown labels and future examples',labels)
+    def ordering():
+        before=copy.deepcopy((examples,events));original=output()
+        reverse=prepare(list(reversed(examples)),list(reversed(events)),'2026-01-10T00:00:00Z','2026-01-12T00:00:00Z','2026-01-15T00:00:00Z')
+        equal(reverse['train'],list(reversed(original['train'])));equal(reverse['scaler'],original['scaler']);equal((examples,events),before)
+    check('input order and immutability',ordering)
+    def empty():
+        result=prepare([],[],'2026-01-10T00:00:00Z','2026-01-12T00:00:00Z','2026-01-15T00:00:00Z');equal(result,{'train':[],'validation':[],'test':[],'scaler':{'mean':0,'scale':1}})
+        result=prepare(examples[:1],events,'2026-01-10T00:00:00Z','2026-01-12T00:00:00Z','2026-01-15T00:00:00Z');equal(result['scaler'],{'mean':2,'scale':1})
+    check('empty and constant training fallback',empty)
+else: raise RuntimeError('Unknown oracle')
+print(json.dumps({'checks':checks}))
+sys.exit(0 if all(c['pass'] for c in checks) else 1)

package/evals/benchmark/report.mjs

@@ -0,0 +1,62 @@
+import {readFileSync,writeFileSync,existsSync} from 'node:fs';
+import {join,resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {createHash} from 'node:crypto';
+import {gradeTrial} from './harness.mjs';
+const json=path=>JSON.parse(readFileSync(path,'utf8'));
+const hash=value=>createHash('sha256').update(value).digest('hex');
+const median=values=>{if(!values.length)return null;const a=[...values].sort((a,b)=>a-b),i=Math.floor(a.length/2);return a.length%2?a[i]:(a[i-1]+a[i])/2;};
+
+export function summarize(rows){
+  return Object.fromEntries([...new Set(rows.map(r=>r.arm))].map(arm=>{
+    const trials=rows.filter(r=>r.arm===arm),metrics=trials.map(r=>r.metrics).filter(Boolean);
+    const totalMetric=key=>{const available=metrics.filter(m=>Number.isFinite(m[key]));return available.length?available.reduce((n,m)=>n+m[key],0):null;};
+    const usageFields=['input_tokens','cached_input_tokens','output_tokens','reasoning_output_tokens'];
+    const usage=Object.fromEntries(usageFields.map(key=>{
+      const available=metrics.filter(m=>Number.isFinite(m.usage?.[key]));
+      return [key,{sumAvailable:available.length?available.reduce((n,m)=>n+m.usage[key],0):null,availableTrials:available.length}];
+    }));
+    return [arm,{passed:trials.filter(r=>r.correct).length,trials:trials.length,artifactPassed:trials.filter(r=>r.artifactCorrect).length,
+      completed:metrics.filter(m=>m.turnCompleted&&m.exitCode===0&&!m.timedOut).length,timeouts:metrics.filter(m=>m.timedOut).length,
+      medianWallMs:median(metrics.map(m=>m.wallMs).filter(Number.isFinite)),wallTimeAvailableTrials:metrics.filter(m=>Number.isFinite(m.wallMs)).length,
+      toolCalls:totalMetric('toolCalls'),failedCommands:totalMetric('failedCommands'),userInterventions:totalMetric('userInterventions'),
+      countMetricsAvailableTrials:Object.fromEntries(['toolCalls','failedCommands','userInterventions'].map(key=>[key,metrics.filter(m=>Number.isFinite(m[key])).length])),usage,costUsd:null}];
+  }));
+}
+export function paired(rows,left,right){
+  const comparisons=[];
+  for(const a of rows.filter(r=>r.arm===left)){
+    const b=rows.find(r=>r.arm===right&&r.case===a.case&&r.repetition===a.repetition);
+    if(b)comparisons.push({case:a.case,repetition:a.repetition,left:a.correct,right:b.correct});
+  }
+  return {left,right,pairs:comparisons.length,bothPass:comparisons.filter(p=>p.left&&p.right).length,leftOnly:comparisons.filter(p=>p.left&&!p.right).length,rightOnly:comparisons.filter(p=>!p.left&&p.right).length,neither:comparisons.filter(p=>!p.left&&!p.right).length,comparisons};
+}
+
+export function exportStudy(directory,{regrade=false}={}){
+  const plan=json(join(directory,'study.json')),trials=[];
+  for(const job of plan.order){
+    const root=join(directory,job.directory);
+    if(!existsSync(join(root,'metrics.json'))&&!existsSync(join(root,'harness-error.json')))throw Error(`Study unfinished: ${job.directory}`);
+    const grade=existsSync(join(root,'harness-error.json'))?json(join(root,'harness-error.json')):regrade?gradeTrial(root):json(join(root,'grade.json'));
+    const manifest=json(join(root,'run.json'));
+    const hashes={};for(const name of ['prompt.txt','events.jsonl','answer.md','run.json','test-evidence.json'])if(existsSync(join(root,name)))hashes[name]=hash(readFileSync(join(root,name)));
+    const instructionHashes=Object.fromEntries(Object.entries(manifest.inputs).filter(([p])=>p.startsWith('_instructions/')));
+    trials.push({case:job.id,arm:job.arm,repetition:job.repetition,sourceVersion:manifest.sourceVersion,profile:manifest.profile,
+      correct:grade.correct,artifactCorrect:grade.artifactCorrect,scorerVersion:grade.scorerVersion??1,
+      priorScore:existsSync(join(root,'grade-v1.json'))?{scorerVersion:1,correct:json(join(root,'grade-v1.json')).correct}:null,
+      priorScores:[1,2].filter(v=>existsSync(join(root,`grade-v${v}.json`))).map(v=>({scorerVersion:v,correct:json(join(root,`grade-v${v}.json`)).correct})),
+      checks:grade.checks.map(({name,pass})=>({name,pass})),metrics:grade.metrics,
+      instructionBundleSha256:hash(JSON.stringify(instructionHashes)),instructionFileCount:Object.keys(instructionHashes).length,
+      selectedInstructions:manifest.instructions.map(path=>({path,sha256:manifest.inputs[path]})),artifactHashes:hashes});
+  }
+  const source=fileURLToPath(new URL('./',import.meta.url));
+  return {schemaVersion:1,exportedAt:new Date().toISOString(),scorer:{version:3,files:Object.fromEntries(['harness.mjs','oracles/node.mjs','oracles/python.py','support/python-test-report.py','support/commit-tree.mjs'].map(p=>[p,hash(readFileSync(join(source,p)))]))},protocol:{...plan,order:undefined},summary:summarize(trials),
+    paired:[['just-vibe','baseline'],['just-vibe','ecc'],['just-vibe-profile','just-vibe']].map(([a,b])=>paired(trials,a,b)).filter(p=>p.pairs),trials,
+    limitations:['Newly authored bounded fixture repositories, not third-party production repositories.','Complete written contracts remove much of the context and discovery burden; user convenience and preference were not measured.','Bounded correctness checks do not rank overall output quality; equal pass counts do not establish equal products.','Matched supplied ECC instructions at a pinned snapshot; native hooks/memory integrations are excluded.','Two repeats per original case/arm are insufficient to establish general superiority.','Token and timing metrics are supporting operational observations, not quality scores. Zero interventions means none were supplied under the protocol.','No monetary charge is reported by this authenticated CLI; cached input is a subset of input.','A stable model name does not guarantee an immutable service backend.','See the protocol for pilots and the uniform scorer correction; original scores are retained.']};
+}
+
+if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url)){
+  const [directory,output,...flags]=process.argv.slice(2);
+  if(!directory||!output||flags.some(f=>f!=='--regrade'))throw Error('Usage: node report.mjs STUDY OUTPUT.json [--regrade]');
+  const report=exportStudy(directory,{regrade:flags.includes('--regrade')});writeFileSync(output,JSON.stringify(report,null,2)+'\n');console.log(JSON.stringify(report.summary,null,2));
+}

package/evals/benchmark/repos/async-cache/README.md

@@ -0,0 +1,12 @@
+# Directory cache
+
+A framework-independent data layer used by two account-picker views. `src/cache.mjs` supplies `createCache(fetchValue, {ttl, now})`. `src/view.mjs` publishes `{value,error}` after a selection. There are no dependencies. Run `npm test` or `node --test test/*.test.mjs`.
+
+Contract:
+- Keys are the ordered pair (tenant, key), each an arbitrary string. Equal pairs share one pending fetch; distinct pairs must never collide, including delimiter characters.
+- `fetchValue(tenant,key,options)` may resolve synchronously, throw synchronously, or return a promise. A failed fetch is not cached; the next request may retry.
+- A settled value, including undefined or null, is reused until `now() >= completionTime + ttl`. Zero TTL means no settled reuse, but concurrent pending requests still coalesce.
+- Each `get` caller has independent optional AbortSignal cancellation. An already-aborted caller receives an AbortError and starts no fetch. Cancelling one waiter rejects that waiter promptly, while other waiters and the underlying shared work remain usable. Never pass one caller's signal into the shared fetch. Remove each abort listener after that waiter's promise settles.
+- `invalidate(tenant,key)` detaches the existing pending/settled entry. Existing waiters can still settle; new callers start fresh work. The detached result must neither replace nor delete a newer entry, even if the old work rejects or resolves later.
+- In the view, only the latest selection may publish either success or failure. Disposal permanently stops publication and future fetch requests. `select` must fulfill after handling a fetch failure or cancellation.
+- Preserve the exported API. No dependencies, network calls, persistent cache or framework changes are requested.

package/evals/benchmark/repos/async-cache/TASK.md

@@ -0,0 +1 @@
+Repair the directory cache and view layer. Switching accounts sometimes shows a stale result; two views can cancel each other's request, and invalidation can resurrect old data. Implement the README contract and add meaningful regression tests. Modify only src/cache.mjs, src/view.mjs, and test/regression.test.mjs. Do not change the supplied smoke test or README. Do not commit.

package/evals/benchmark/repos/async-cache/package.json

@@ -0,0 +1 @@
+{"name":"fixture-directory-cache","private":true,"type":"module","scripts":{"test":"node --test test/*.test.mjs"}}

package/evals/benchmark/repos/async-cache/src/cache.mjs

@@ -0,0 +1,13 @@
+export function createCache(fetchValue, { ttl = 100, now = Date.now } = {}) {
+  const cache = new Map();
+  return {
+    async get(tenant, key, { signal } = {}) {
+      const prior = cache.get(key);
+      if (prior && now() < prior.expires) return prior.value;
+      const value = await fetchValue(tenant, key, { signal });
+      cache.set(key, { value, expires: now() + ttl });
+      return value;
+    },
+    invalidate(tenant, key) { cache.delete(key); },
+  };
+}

package/evals/benchmark/repos/async-cache/src/view.mjs

@@ -0,0 +1,9 @@
+export function createView(cache, publish) {
+  return {
+    async select(tenant, key) {
+      try { publish({ value: await cache.get(tenant, key), error: null }); }
+      catch (error) { publish({ value: null, error: error.message }); }
+    },
+    dispose() {},
+  };
+}

package/evals/benchmark/repos/async-cache/test/smoke.test.mjs

@@ -0,0 +1,9 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createCache } from '../src/cache.mjs';
+test('settled values are reused before expiry', async () => {
+  let calls = 0;
+  const cache = createCache(async () => ++calls, { now: () => 0 });
+  assert.equal(await cache.get('t', 'k'), 1);
+  assert.equal(await cache.get('t', 'k'), 1);
+});

package/evals/benchmark/repos/ledger/README.md

@@ -0,0 +1,11 @@
+# Local transfer ledger
+
+Python standard library only. Use `/usr/bin/python3 -m unittest discover -s test` on this machine. Store schema lives in src/store.py; service API is `transfer(conn, tenant, key, source, destination, amount, *, after_debit=lambda: None)`. This is a local SQLite exercise, with no external payment provider.
+
+Contract:
+- A tenant owns both accounts. Source and destination must exist in that tenant and differ. Amount must have exactly Python type int, be positive, and be <= 2**63-1; bool, strings and floats are invalid. Reject invalid input with ValueError without writes.
+- Idempotency key scope is (tenant,key). An exact replay returns the original result and performs no balance changes or hook call. Same key with different source/destination/amount is ValueError, also with no writes. Another tenant may use the same key independently.
+- Debit, credit and recorded result are one atomic durable transaction. Check sufficient funds and destination overflow before writing. The after_debit hook models interruption: if it raises, roll back the entire operation and re-raise its exception. Do not cache failures. A later retry may succeed.
+- A call owns its transaction only if the supplied connection has no active transaction. If the caller already has a transaction, reject with RuntimeError and preserve all caller state. Never commit or roll back a transaction owned by the caller.
+- Distinct connections may execute equal keys concurrently. They must observe one effect and the same result after normal SQLite lock serialization. Database/locking exceptions must not be mislabeled as business success. Leave the connection usable and outside your transaction after success or failure.
+- Return the dict {tenant,key,source,destination,amount} using the committed request values. Schema changes, new libraries, distributed exactly-once claims, or external calls are out of scope.

package/evals/benchmark/repos/ledger/TASK.md

@@ -0,0 +1 @@
+Repair transfer() so interruption and retries cannot lose or duplicate local balances, and enforce the README contract. Add regression coverage. Modify only src/service.py and test/test_regression.py. Preserve the schema and smoke test. Do not commit.

package/evals/benchmark/repos/ledger/src/service.py

@@ -0,0 +1,14 @@
+import json
+
+def transfer(conn, tenant, key, source, destination, amount, *, after_debit=lambda: None):
+    prior = conn.execute('SELECT amount FROM transfers WHERE key=?', (key,)).fetchone()
+    if prior:
+        return {'tenant': tenant, 'key': key, 'source': source, 'destination': destination, 'amount': prior[0]}
+    amount = int(amount)
+    conn.execute('UPDATE accounts SET balance=balance-? WHERE tenant=? AND id=?', (amount, tenant, source))
+    conn.commit()
+    after_debit()
+    conn.execute('UPDATE accounts SET balance=balance+? WHERE tenant=? AND id=?', (amount, tenant, destination))
+    conn.execute('INSERT INTO transfers VALUES(?,?,?,?,?)', (tenant,key,source,destination,amount))
+    conn.commit()
+    return {'tenant': tenant, 'key': key, 'source': source, 'destination': destination, 'amount': amount}