carthagent 0.0.0-stage → 1.0.7

package/tools/benchmark/guard.test.mjs

@@ -0,0 +1,133 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import guard from './guard.ts';
+
+const key = sessionID => `benchmark/${sessionID}/tools`;
+function harness(storage = new Map()) {
+  const hooks = new Map();
+  const eventQueues=[];
+  const ctx = {
+    session:{hook:async(name,callback)=>{hooks.set(`session:${name}`,callback);return{dispose:async()=>{}};}},
+    tool:{hook:async(name,callback)=>{hooks.set(`tool:${name}`,callback);return{dispose:async()=>{}};}},
+    storage:{
+      get:async name=>storage.get(name),
+      set:async(name,value)=>{storage.set(name,structuredClone(value));},
+    },
+    event:{subscribe:({signal})=>({[Symbol.asyncIterator]:async function*(){
+      const queue=[];const waiters=[];eventQueues.push({queue,waiters});
+      const onAbort=()=>{for(const waiter of waiters.splice(0))waiter();};
+      signal.addEventListener('abort',onAbort,{once:true});
+      try {
+        while(!signal.aborted) {
+          if(queue.length){yield queue.shift();continue;}
+          await new Promise(resolve=>waiters.push(resolve));
+        }
+      } finally {signal.removeEventListener('abort',onAbort);}
+    }})},
+  };
+  return { hooks, storage, ctx, emit:event=>{for(const {queue,waiters} of eventQueues){queue.push(event);for(const waiter of waiters.splice(0))waiter();}} };
+}
+
+test('parallel per-session admissions reserve the final slot once, replay IDs idempotently and persist across guard restart',async()=>{
+  const temp=mkdtempSync(join(tmpdir(),'benchmark-guard-'));
+  const old={limit:process.env.BENCH_TOOL_LIMIT,log:process.env.BENCH_GUARD_LOG,home:process.env.CTG_OPENCODE_HOME};
+  process.env.BENCH_TOOL_LIMIT='2';
+  process.env.BENCH_GUARD_LOG=join(temp,'guard.jsonl');
+  process.env.CTG_OPENCODE_HOME=temp;
+  const state=harness();
+  let dispose=await guard(state.ctx);
+  try {
+    state.storage.set(key('session-a'),[{id:'already-used',tool:'read',admitted:true,at:1}]);
+    const before=state.hooks.get('tool:execute.before');
+    const results=await Promise.allSettled(Array.from({length:12},(_,index)=>before({id:`call-${index}`,tool:'read',sessionID:'session-a'})));
+    assert.equal(results.filter(result=>result.status==='fulfilled').length,1);
+    assert.equal(results.filter(result=>result.status==='rejected').length,11);
+    assert.equal(state.storage.get(key('session-a')).filter(row=>row.admitted).length,2);
+
+    // A duplicate delivery of the admitted call ID is idempotent, including
+    // after a plugin restart which discards in-memory state.
+    const admittedID=state.storage.get(key('session-a')).find(row=>row.id.startsWith('call-')&&row.admitted).id;
+    await assert.rejects(before({id:admittedID,tool:'read',sessionID:'session-a'}),/budget exhausted/);
+    assert.equal(state.storage.get(key('session-a')).filter(row=>row.admitted).length,2);
+    await dispose();
+    const other=harness(state.storage);
+    dispose=await guard(other.ctx);
+    await assert.rejects(other.hooks.get('tool:execute.before')({id:admittedID,tool:'read',sessionID:'session-a'}),/budget exhausted/);
+    await other.hooks.get('tool:execute.before')({id:'neighbor',tool:'read',sessionID:'session-b'});
+    assert.equal(other.storage.get(key('session-b')).filter(row=>row.admitted).length,1);
+    await assert.rejects(other.hooks.get('tool:execute.before')({id:'new-after-restart',tool:'read',sessionID:'session-a'}),/budget exhausted/);
+    assert.equal(other.storage.get(key('session-a')).filter(row=>row.admitted).length,2);
+    await other.hooks.get('tool:execute.before')({id:'neighbor-after-rejection',tool:'read',sessionID:'session-b'});
+    await assert.rejects(other.hooks.get('tool:execute.before')({id:'neighbor-at-cap',tool:'read',sessionID:'session-b'}),/budget exhausted/);
+    assert.equal(other.storage.get(key('session-b')).filter(row=>row.admitted).length,2);
+    assert.equal(JSON.parse(readFileSync(process.env.BENCH_GUARD_LOG,'utf8').trim().split('\n').at(-1)).sessionID,'session-b');
+  } finally {
+    await dispose();
+    for(const [name,value] of Object.entries({BENCH_TOOL_LIMIT:old.limit,BENCH_GUARD_LOG:old.log,CTG_OPENCODE_HOME:old.home})) {
+      if(value===undefined) delete process.env[name]; else process.env[name]=value;
+    }
+    rmSync(temp,{recursive:true,force:true});
+  }
+});
+
+test('failed durable reservation is fail-closed and a cancelled queued call consumes no slot',async()=>{
+  const temp=mkdtempSync(join(tmpdir(),'benchmark-guard-fail-'));
+  const old={limit:process.env.BENCH_TOOL_LIMIT,log:process.env.BENCH_GUARD_LOG,home:process.env.CTG_OPENCODE_HOME};
+  process.env.BENCH_TOOL_LIMIT='1';
+  process.env.BENCH_GUARD_LOG=join(temp,'guard.jsonl');
+  process.env.CTG_OPENCODE_HOME=temp;
+  const storage=new Map();
+  const state=harness(storage);
+  let failWrite=true, releaseRead, enteredRead;
+  const entered=new Promise(resolve=>{enteredRead=resolve;});
+  let holdNextRead=false;
+  state.ctx.storage.get=async name=>{
+    if(name===key('queued-session')&&holdNextRead){
+      holdNextRead=false;
+      await new Promise(resolve=>{releaseRead=resolve;enteredRead();});
+    }
+    return storage.get(name);
+  };
+  state.ctx.storage.set=async(name,value)=>{
+    if(failWrite){failWrite=false;throw Error('durable storage unavailable');}
+    storage.set(name,structuredClone(value));
+  };
+  let dispose=await guard(state.ctx);
+  try {
+    const before=state.hooks.get('tool:execute.before');
+    await assert.rejects(before({id:'write-fails',tool:'read',sessionID:'session-write-fails'}),/durable storage unavailable/);
+    assert.equal(storage.has(key('session-write-fails')),false);
+    holdNextRead=true;
+    const owner=before({id:'slow-owner',tool:'read',sessionID:'queued-session'});
+    await entered;
+    state.emit({type:'session.execution.interrupted',data:{sessionID:'queued-session'}});
+    await new Promise(resolve=>setImmediate(resolve));
+    const cancelled=before({id:'cancelled',tool:'read',sessionID:'queued-session'});
+    await before({id:'independent-session',tool:'read',sessionID:'independent'});
+    assert.equal(storage.get(key('independent')).filter(row=>row.admitted).length,1,'different session admissions do not wait on the held session queue');
+    releaseRead();
+    await assert.rejects(owner,/cancelled before reservation/);
+    await assert.rejects(cancelled,/cancelled before reservation/);
+    assert.equal(storage.has(key('queued-session')),false,'interrupted in-flight admission must not write a reservation');
+    await before({id:'neighbor',tool:'read',sessionID:'neighbor'});
+    assert.equal(storage.get(key('neighbor')).filter(row=>row.admitted).length,1);
+    holdNextRead=true;
+    const draining=before({id:'drain-owner',tool:'read',sessionID:'draining'});
+    await entered;
+    const unloading=dispose();
+    releaseRead();
+    await assert.rejects(draining,/cancelled before reservation/);
+    await unloading;
+    assert.equal(storage.has(key('draining')),false,'unload waits for queued callbacks without admitting late work');
+  } finally {
+    releaseRead?.();
+    await dispose();
+    for(const [name,value] of Object.entries({BENCH_TOOL_LIMIT:old.limit,BENCH_GUARD_LOG:old.log,CTG_OPENCODE_HOME:old.home})) {
+      if(value===undefined) delete process.env[name]; else process.env[name]=value;
+    }
+    rmSync(temp,{recursive:true,force:true});
+  }
+});

package/tools/benchmark/guard.ts

@@ -0,0 +1,136 @@
+import { appendFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { withFileLock } from '../../lib/opencode/runtime.mjs';
+
+type Registration = { dispose(): Promise<void> };
+const guardLedger = (sessionID: string) => `benchmark/${sessionID}/tools`;
+
+/** Identical native preventive tool gate in every condition. */
+export default async function guard(ctx: any): Promise<() => Promise<void>> {
+  const limit = Number(process.env.BENCH_TOOL_LIMIT || 120);
+  const retryLimit = Number(process.env.BENCH_RETRY_LIMIT || 3);
+  if (!Number.isInteger(limit) || limit < 1 || limit > 120) throw Error('Invalid tool limit');
+  if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 20) throw Error('Invalid benchmark retry limit');
+
+  const registrations: Registration[] = [];
+  let disposed = false;
+  let disposing: Promise<void> | undefined;
+  const tails = new Map<string, Promise<void>>();
+  const admitted = new Map<string, boolean>();
+  const cancelledSessions = new Set<string>();
+  let cancellationEventsAvailable = true;
+  const eventAbort = new AbortController();
+  const eventTask = (async () => {
+    try {
+      for await (const event of ctx.event.subscribe({ signal:eventAbort.signal })) {
+        if (!event.type.startsWith('session.') || !('sessionID' in event.data)) continue;
+        if (event.type === 'session.execution.interrupted' || event.type === 'session.deleted') {
+          cancelledSessions.add(event.data.sessionID);
+        } else if (event.type === 'session.execution.started') {
+          cancelledSessions.delete(event.data.sessionID);
+        }
+      }
+      if (!eventAbort.signal.aborted) cancellationEventsAvailable = false;
+    } catch {
+      if (!eventAbort.signal.aborted) cancellationEventsAvailable = false;
+    }
+  })();
+  void eventTask.catch(() => undefined);
+  const callKey = (sessionID: string, callID: string) => `${sessionID}\0${callID}`;
+  const lockPath = (sessionID: string) => {
+    const safeID = createHash('sha256').update(sessionID).digest('hex');
+    const root = process.env.CTG_OPENCODE_HOME;
+    return root ? join(root, 'state', 'benchmark-admission', `${safeID}.lock`)
+      : join(tmpdir(), 'ctg-opencode-benchmark-admission', `${safeID}.lock`);
+  };
+  const serializeSession = async (sessionID: string, action: () => Promise<void>) => {
+    const previous = tails.get(sessionID) ?? Promise.resolve();
+    let release!: () => void;
+    const current = new Promise<void>(resolve => { release = resolve; });
+    tails.set(sessionID, current);
+    try {
+      await previous;
+      await withFileLock(lockPath(sessionID), action);
+    } finally {
+      release();
+      if (tails.get(sessionID) === current) tails.delete(sessionID);
+    }
+  };
+  const register = async (pending: Promise<Registration>) => {
+    const registration = await pending;
+    registrations.push(registration);
+    return registration;
+  };
+  const dispose = () => {
+    if (disposing) return disposing;
+    disposed = true;
+    disposing = (async () => {
+      // Stop new hook dispatch first. Callbacks already queued re-check
+      // disposed after acquiring their session lock; drain until every such
+      // callback has released its queue before plugin unload completes.
+      await Promise.allSettled(registrations.splice(0).reverse().map(registration => registration.dispose()));
+      while (tails.size) await Promise.allSettled([...tails.values()]);
+      eventAbort.abort();
+      await eventTask.catch(() => undefined);
+    })();
+    return disposing;
+  };
+  try {
+    await register(ctx.session.hook('context', async (event: any) => {
+      if (event.kind !== 'primary') return;
+      const rows = await ctx.storage.get(guardLedger(event.sessionID));
+      const attempts = Array.isArray(rows) ? rows.filter(row => row && typeof row === 'object' && row.admitted === true).length : 0;
+      if (attempts >= limit) {
+        event.tools = {};
+        event.system.push({ type:'text', text:'Native benchmark tool budget is exhausted. Stop issuing tool calls, summarize verified status and remaining work.' });
+      }
+    }));
+    await register(ctx.session.hook('retry', async (event: any) => {
+      if (event.attempt >= retryLimit) event.decision = { retry:false };
+    }));
+    await register(ctx.tool.hook('execute.before', async (event: any) => {
+      let blocked = false;
+      let attempts = 0;
+      await serializeSession(event.sessionID, async () => {
+        if (disposed || !cancellationEventsAvailable || cancelledSessions.has(event.sessionID)) throw Error('Benchmark tool admission cancelled before reservation.');
+        const currentCallKey = callKey(event.sessionID, event.id);
+        const previous = await ctx.storage.get(guardLedger(event.sessionID));
+        const rows = Array.isArray(previous) ? previous : [];
+        const existing = rows.find(row => row && typeof row === 'object' && row.id === event.id);
+        if (existing) {
+          if (existing.tool !== event.tool || typeof existing.admitted !== 'boolean') throw Error('Conflicting benchmark tool reservation ID.');
+          attempts = rows.filter(row => row && typeof row === 'object' && row.admitted === true).length;
+          blocked = true; // Duplicate invocation never executes twice.
+        } else {
+          attempts = rows.filter(row => row && typeof row === 'object' && row.admitted === true).length;
+          blocked = attempts >= limit;
+          if (disposed || !cancellationEventsAvailable || cancelledSessions.has(event.sessionID)) throw Error('Benchmark tool admission cancelled before reservation.');
+          // Persist a reservation (or rejection) while holding both the
+          // per-session queue and inter-process file lock. No tool is admitted
+          // until this durable write has succeeded.
+          await ctx.storage.set(guardLedger(event.sessionID), [...rows, { id:event.id, tool:event.tool, admitted:!blocked, at:Date.now() }]);
+          attempts += Number(!blocked);
+          appendFileSync(process.env.BENCH_GUARD_LOG!, JSON.stringify({ type:'tool-attempt', sessionID:event.sessionID, callID:event.id, tool:event.tool, attempts, blocked })+'\n');
+        }
+        if (!existing) admitted.set(currentCallKey, !blocked);
+      });
+      if (blocked) throw Error('Benchmark tool budget exhausted before execution.');
+    }));
+    await register(ctx.tool.hook('execute.after', async (event: any) => {
+      const currentCallKey=callKey(event.sessionID,event.id);
+      if (admitted.get(currentCallKey) === false) return;
+      if (event.status !== 'completed' || event.tool !== 'delivery_finish') return;
+      const raw = event.result.content.filter((item: any) => item.type === 'text').map((item: any) => item.text).join('\n');
+      try {
+        const result = JSON.parse(raw);
+        if (result.status !== 'verified') event.result.content.push({ type:'text', text:'Benchmark verification policy: only verified status is accepted after the delivery extension checks.' });
+      } catch { event.result.content.push({ type:'text', text:'Benchmark verification policy: result shape was not recognized.' }); }
+    }));
+  } catch (error) {
+    await dispose();
+    throw error;
+  }
+  return dispose;
+}

package/tools/benchmark/native-plugin/index.ts

@@ -0,0 +1,24 @@
+import { Plugin } from '@opencode/plugin';
+import guard from '../guard.js';
+import advisor from '../advisor.js';
+
+export default Plugin.define({
+  id:'ctg.benchmark',
+  async setup(ctx) {
+    const cleanups: Array<() => Promise<void>> = [];
+    let disposed = false;
+    const dispose = async () => {
+      if (disposed) return;
+      disposed = true;
+      await Promise.allSettled(cleanups.reverse().map(cleanup => cleanup()));
+    };
+    try {
+      cleanups.push(await guard(ctx));
+      cleanups.push(await advisor(ctx));
+    } catch (error) {
+      await dispose();
+      throw error;
+    }
+    return dispose;
+  },
+});

package/tools/benchmark/planning-startup.test.mjs

@@ -0,0 +1,47 @@
+// Offline scripted provider: exercises actual installed pi plumbing, not model efficacy.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { runCandidate } from './runner.mjs';
+import { initializeCampaign } from './campaign.mjs';
+import { MODEL } from './gateway.mjs';
+import { fixtureDesign, fixtureReview } from '../../tests/helpers/tested-design.mjs';
+
+test('installed engine traverses discovery guide and full delivery with offline scripted responses', async t => {
+ const root=mkdtempSync(join(tmpdir(),'ctg-startup-engine-'));t.after(()=>rmSync(root,{recursive:true,force:true}));
+ const campaignPath=join(root,'campaign.json');initializeCampaign(campaignPath,{historicalCandidate:0,historicalSmoke:0});
+ const requirement='Writes the exact expected file';
+ const plan={goal:'Implement fixture output',assumptions:[],artifacts:['.'],outputs:['output.txt'],steps:[{id:'build',title:'Write and verify',checks:['output']}],acceptance:[{requirement,checks:['output']}],checks:[{id:'output',kind:'test',argv:[process.execPath,'-e',"require('node:assert/strict').equal(require('node:fs').readFileSync('output.txt','utf8'),'ok')"],timeoutSeconds:5}]};plan.design=fixtureDesign(plan);
+ const lastTool=payload=>{
+  const raw=payload.messages.filter(m=>m.role==='tool').at(-1)?.content;
+  return JSON.parse(typeof raw==='string'?raw:raw.map(b=>b.text||'').join('\n'));
+ };
+ const sequence=[...Array.from({length:8},()=>()=>({name:'read',args:{path:'TASK.md'}})),
+ ()=>({name:'delivery_design',args:{action:'guide'}}),
+ ()=>({name:'delivery_plan',args:plan}),
+ ()=>({name:'delivery_design',args:{action:'validate'}}),
+ ()=>({name:'delivery_design',args:{action:'inspect'}}),
+ p=>({name:'delivery_design',args:{action:'review',review:{...fixtureReview(plan),captureId:lastTool(p).result.id}}}),
+ ()=>({name:'delivery_design',args:{action:'approve'}}),
+ ()=>({name:'write',args:{path:'output.txt',content:'ok'}}),
+ ()=>({name:'delivery_check',args:{id:'all'}}),
+ ()=>({name:'delivery_progress',args:{step:'build',status:'done'}}),
+ ()=>({name:'delivery_review',args:{action:'inspect'}}),
+ p=>({name:'delivery_review',args:{action:'record',captureId:lastTool(p).id,coverage:[{requirement,assertions:'Exact file content asserted by Node strict equality'}],probes:['Declared output check exited zero'],findings:[],limitations:['Scripted fixture; no semantic review claim']}}),
+ ()=>({name:'delivery_finish',args:{status:'verified',review:'Reviewed fixture evidence',launch:'n/a',limitations:['Offline scripted provider']}}),
+ ];
+ let i=0,sawCheckpoint=false;
+ const r=await runCandidate('smoke','harness',{smoke:true,key:'dummy',root,campaignPath,promptOverride:'Implement a fixture output.txt containing exactly ok. Use the delivery contract and finish verified.',fetchImpl:async(_url,options)=>{
+  const payload=JSON.parse(options.body);sawCheckpoint ||= JSON.stringify(payload.messages).includes('Pre-plan checkpoint: 8 discovery');
+  const call=i<sequence.length?sequence[i++](payload):null;
+  return new Response('data: '+JSON.stringify({id:'offline',model:MODEL,choices:[{index:0,delta:{role:'assistant',content:call?'':'SMOKE_OK',...(call?{tool_calls:[{index:0,id:'call'+i,type:'function',function:{name:call.name,arguments:JSON.stringify(call.args)}}]}:{})},finish_reason:call?'tool_calls':'stop'}],usage:{cost:0,prompt_tokens:1,completion_tokens:1}})+'\n\ndata: [DONE]\n\n');
+ }});
+ assert.equal(r.smokePassed,true);assert.equal(i,sequence.length);assert.ok(sawCheckpoint);
+ const events=readFileSync(join(root,'smoke-harness/events.jsonl'),'utf8').trim().split('\n').map(JSON.parse);
+ assert.deepEqual(events.filter(e=>e.type==='tool_execution_end'&&e.isError),[]);
+ assert.equal(readFileSync(join(root,'smoke-harness/workspace/output.txt'),'utf8'),'ok');
+ const finish=events.findLast(e=>e.type==='tool_execution_end'&&e.toolName==='delivery_finish');
+ assert.match(JSON.stringify(finish.result),/verified/);
+});

package/tools/benchmark/preflight-live.mjs

@@ -0,0 +1,31 @@
+// Explicit paid opt-in only; all calls debit the original shared smoke allowance.
+import { readFileSync,writeFileSync,mkdirSync,existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { startGateway,MODEL } from './gateway.mjs';
+import { campaignAccount,initializeCampaign } from './campaign.mjs';
+import { CAMPAIGN,ROOT,runCandidate } from './runner.mjs';
+export async function livePreflight(){
+ if(!existsSync(CAMPAIGN))initializeCampaign(CAMPAIGN);
+ const dir=join(ROOT,'live-preflight');mkdirSync(ROOT,{recursive:true});mkdirSync(dir); // exclusive: preserve every attempt
+ const key=JSON.parse(readFileSync('/home/baissi/.pi/agent/auth.json','utf8')).openrouter.key;
+ const gateway=await startGateway({key,limit:.09,allowJev:true,outputTokens:32,account:campaignAccount(CAMPAIGN,'smoke',dir),receiptPath:join(dir,'gateway.jsonl')});
+ const checks=[];let failure;
+ try{
+  // Multiple calls exceed the earlier failing request counts. Two larger payloads
+  // exercise TLS record boundaries; output length is intentionally bounded.
+  for(let i=0;i<24;i++){
+   const content='Reply OK. The following repeated filler is inert data: '+ 'a'.repeat([8,19].includes(i)?60000:1000);
+   const r=await fetch(gateway.url+'/v1/chat/completions',{method:'POST',body:JSON.stringify({model:MODEL,messages:[{role:'user',content}]})});await r.text();
+   if(r.status!==200)throw Error(`Transport probe ${i} failed; no automatic retry`);checks.push({kind:'generation',index:i,passed:true});
+  }
+  const r=await fetch(gateway.url+'/jev',{method:'POST',body:JSON.stringify({state:{goal:'Transport readiness',acceptance:[],design:{components:[],ports:[],scenarios:[]}}})});await r.text();if(r.status!==200)throw Error('Jev connectivity failed');checks.push({kind:'jev',passed:true});
+ }catch(e){failure=e.message;}finally{await gateway.close();}
+ const result={passed:!failure,checks,failure,budget:gateway.budget,at:new Date().toISOString()};writeFileSync(join(dir,'result.json'),JSON.stringify(result,null,2));
+ if(failure)throw Error(failure);
+ const smoke=await runCandidate('smoke','plain',{smoke:true,key});
+ if(!smoke.smokePassed)throw Error('Live installed-engine smoke failed');
+ writeFileSync(join(dir,'readiness.json'),JSON.stringify({passed:true,transport:result,engineSmoke:smoke,limitations:['Finite live soak does not establish TLS root cause or guarantee future availability.']},null,2));
+ return result;
+}
+if(process.argv[1]===fileURLToPath(import.meta.url)){if(process.argv[2]!=='--paid')throw Error('Explicit --paid required');console.log(JSON.stringify(await livePreflight()));}

package/tools/benchmark/preflight.mjs

@@ -0,0 +1,22 @@
+import { readdirSync,mkdirSync,writeFileSync,readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { command } from './eval-common.mjs';
+import { ROOT } from './runner.mjs';
+import { REPO,createFreeze,frozenFiles } from './freeze.mjs';
+import { sha } from './runtime.mjs';
+// No paid calls. Rebuild readiness after any code change. Output directory is exclusive.
+export async function preflight(out){
+ mkdirSync(out,{recursive:false});
+ const files=Object.fromEntries(frozenFiles().map(f=>[f,sha(readFileSync(f))]));
+ const tests=readdirSync(join(REPO,'tools/benchmark')).filter(f=>f.endsWith('.test.mjs')).map(f=>'tools/benchmark/'+f);
+ const focused=await command(process.execPath,['--test',...tests,'tools/benchmark/selftest.mjs'],REPO,{},240000);writeFileSync(join(out,'focused.log'),focused.stdout+focused.stderr);
+ const quality=await command('npm',['run','quality'],REPO,{},240000);writeFileSync(join(out,'quality.log'),quality.stdout+quality.stderr);
+ const livePath=join(ROOT,'live-preflight/readiness.json');const live=JSON.parse(readFileSync(livePath,'utf8'));
+ const result={passed:focused.code===0&&!focused.error&&quality.code===0&&!quality.error&&live.passed===true,focused:{code:focused.code,error:focused.error},quality:{code:quality.code,error:quality.error},live:{path:livePath,sha256:sha(readFileSync(livePath))},files,at:new Date().toISOString()};
+ const path=join(out,'readiness.json');writeFileSync(path,JSON.stringify(result,null,2));if(!result.passed)throw Error('Readiness failed; inspect logs');return path;
+}
+if(process.argv[1]===fileURLToPath(import.meta.url)){
+ if(!process.argv[2])throw Error('Usage: node tools/benchmark/preflight.mjs NEW_OUTPUT_DIRECTORY [--freeze]');
+ const path=await preflight(process.argv[2]);if(process.argv[3]==='--freeze')createFreeze(ROOT,path);console.log(path);
+}

package/tools/benchmark/replay-diagnostic.mjs

@@ -0,0 +1,23 @@
+// Explicit one-shot paid diagnostic. Never executes returned tools or resumes a candidate.
+import{readFileSync,writeFileSync}from'node:fs';import{join}from'node:path';import{fileURLToPath}from'node:url';
+import{curlFetch}from'./transport.mjs';import{upstreamError}from'./upstream-error.mjs';import{campaignAccount}from'./campaign.mjs';import{sha}from'./runtime.mjs';
+export async function replay(dir){
+ const original=readFileSync(join(dir,'payload.json'),'utf8'),inspection=JSON.parse(readFileSync(join(dir,'inspection.json'),'utf8'));
+ if(!inspection.hashMatches||sha(original)!==inspection.originalHash)throw Error('Exact reconstruction required');
+ const path=join(dir,'replay.json');writeFileSync(path,JSON.stringify({status:'reserved-pending',at:new Date().toISOString()}),{flag:'wx',mode:0o600});
+ const payload=JSON.parse(original);payload.max_tokens=32;const wire=JSON.stringify(payload);
+ if(Buffer.byteLength(wire)>100000)throw Error('Diagnostic input exceeds reservation envelope');
+ const account=campaignAccount('/home/baissi/benchmarks/deepseek-jev-campaign.json','smoke',dir+'/one-shot-replay');const hold=account.reserve(.04);
+ const key=JSON.parse(readFileSync('/home/baissi/.pi/agent/auth.json','utf8')).openrouter.key;
+ const result={originalHash:sha(original),replayHash:sha(wire),changedFields:{max_tokens:{before:16384,after:32}},reserved:.04,hold,at:new Date().toISOString()};
+ try{
+  const response=await curlFetch('https://openrouter.ai/api/v1/chat/completions',{method:'POST',headers:{Authorization:`Bearer ${key}`,'Content-Type':'application/json'},body:wire,signal:AbortSignal.timeout(180000)});
+  result.httpStatus=response.status;
+  if(!response.ok){result.error=await upstreamError(response,{secrets:[key]});result.billing='unknown-hold-retained';}
+  else{const text=await response.text();let usage;for(const line of text.split('\n'))if(line.startsWith('data: {')){const c=JSON.parse(line.slice(6));if(c.usage)usage=c.usage;if(c.id)result.id=c.id;if(c.provider)result.provider=c.provider;if(c.model)result.model=c.model;}
+   if(!Number.isFinite(usage?.cost)||usage.cost<0||usage.cost>.04)throw Error('Unknown diagnostic billing');account.reconcile(hold,usage.cost);result.usage=usage;result.billing='reconciled';result.outputExecuted=false;
+  }
+ }catch(e){result.failure={name:e.name,code:e.code||null};result.billing='unknown-hold-retained';}
+ writeFileSync(path,JSON.stringify(result,null,2),{mode:0o600});return result;
+}
+if(process.argv[1]===fileURLToPath(import.meta.url)){if(process.argv[2]!=='--paid'||!process.argv[3])throw Error('Usage: replay-diagnostic.mjs --paid DIAGNOSTIC_DIR');console.log(JSON.stringify(await replay(process.argv[3])));}

package/tools/benchmark/runner.mjs

@@ -0,0 +1,141 @@
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, writeFileSync, existsSync, appendFileSync } from 'node:fs';
+import { resolve, join, dirname } from 'node:path';
+import { startGateway, MODEL } from './gateway.mjs';
+import { TASKS, MATRIX } from './tasks.mjs';
+import { safeEnv, provisionTools, ENGINE_RUNTIME, isolatedHome } from './runtime.mjs';
+import { campaignAccount } from './campaign.mjs';
+import { verifyFreeze } from './freeze.mjs';
+import { runNativeEvaluation } from '../../lib/opencode/evaluation.mjs';
+import { createServer } from 'node:http';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { ensureService, createSession, listPlugins, stopService } from '../../lib/opencode/service.mjs';
+import { provisionRuntime } from '../../lib/opencode/runtime.mjs';
+
+const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+export const ROOT = process.env.CTG_BENCHMARK_ROOT || join(REPO, '.harness/deepseek-jev-v4');
+export const CAMPAIGN = process.env.CTG_BENCHMARK_CAMPAIGN || join(REPO, '.harness/deepseek-jev-campaign.json');
+export function terminalStatus(events, exit, stopReason, budget, outcome = null) {
+  const last = events.filter(e=>e.type==='session.step.ended').at(-1)?.data;
+  const status = budget.stopped ? 'infrastructure-error' : stopReason ? stopReason : exit.code!==0 ? 'process-error' : outcome !== 'succeeded' ? 'non-successful-session-outcome' : !last ? 'missing-terminal-message' : last.finish==='error' ? 'model-error' : last.finish!=='stop' ? 'incomplete' : 'completed';
+  return { status, terminalModelReason:last?.finish || null, terminalError:last?.error?.message || null, outcome, finalText:'' };
+}
+export function childEnv(config, gateway, log) {
+  const home = isolatedHome(config);
+  return safeEnv({ CTG_OPENCODE_HOME: home, CTG_OPENCODE_URL: process.env.CTG_OPENCODE_URL || '', BENCH_GATEWAY: gateway, BENCH_ADVISOR_LOG: log, BENCH_GUARD_LOG: join(dirname(config),'guard.jsonl') }, home);
+}
+async function waitForService(url, { timeoutMs = 30000, signal } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline && !signal?.aborted) {
+    try { const response = await fetch(`${url}/api/info`, { signal: AbortSignal.timeout(1000) }); if (response.ok) return; } catch {}
+    await new Promise(resolve => setTimeout(resolve, 150));
+  }
+  throw Error('Isolated OpenCode 2 service readiness timeout');
+}
+async function startFixtureService(directory) {
+  const server = createServer((req,res) => {
+    if (req.url === '/api/info') { res.writeHead(200,{'content-type':'application/json'}); res.end(JSON.stringify({version:'2.0.32'})); return; }
+    res.writeHead(404); res.end();
+  });
+  await new Promise(resolve => server.listen(0,'127.0.0.1',resolve));
+  const url = `http://127.0.0.1:${server.address().port}`;
+  return { url, close: () => new Promise(resolve => server.close(resolve)), directory };
+}
+async function startNativeService({ workspace, config, mode, arm, smoke, toolLimit, gateway, apiKey, serviceManager }) {
+  const home = isolatedHome(config), configDir = join(home,'.config','opencode');
+  mkdirSync(configDir,{recursive:true});
+  const provider = arm === 'plain' ? 'benchmark-fixture' : 'benchmark';
+  const model = arm === 'plain' ? 'fixture' : MODEL;
+  const plugins = [join(REPO,'lib/opencode/evaluation-budget-plugin'), ...(arm === 'plain' ? [] : [join(REPO,'tools/benchmark/native-plugin')])];
+  const configJson = {
+    '$schema':'https://opencode.ai/config.json', update:'disable', model:`${provider}/${model}`,
+    plugins:plugins.filter((_,index)=>mode!=='smoke' || index===0),
+    providers:{[provider]:{
+      package:'@opencode/ai/providers/openai-compatible',
+      settings:{baseURL:`${gateway}/v1`,apiKey:apiKey || 'fixture-no-key'},
+      models:{[model]:{limit:{context:100000,output:16384},cost:{input:.3,output:1.2,cacheRead:.006,cacheWrite:0}}},
+    }},
+  };
+  await provisionRuntime({ root:home, profile:configJson });
+  const service=await serviceManager.ensureService({root:home,env:safeEnv({
+    BENCH_TOOL_LIMIT:String(toolLimit),BENCH_GUARD_LOG:join(dirname(config),'guard.jsonl'),
+    BENCH_GATEWAY:gateway,BENCH_ADVISOR_LOG:join(dirname(config),'advisor.jsonl'),
+    CTG_EVALUATION_MAX_TURNS:String(toolLimit),CTG_EVALUATION_MAX_COST:String(smoke ? .08 : 2.79),
+    OPENROUTER_API_KEY:apiKey || '',
+  },home)});
+  const endpoint=service.url,auth=service.headers;
+  const request={location:{directory:workspace},signal:AbortSignal.timeout(15000)};
+  try {
+  if (configJson.plugins.length) {
+    await createSession(service,workspace,{model:{providerID:provider,id:model},title:'native plugin activation probe',signal:request.signal});
+    const deadline=Date.now()+15000;
+    while(Date.now()<deadline){
+      const loaded=await listPlugins(service,request).catch(()=>null);
+      const expected = ['ctg.evaluation-budget', ...(arm === 'plain' ? [] : ['ctg.benchmark'])];
+      const active = expected.every(id=>loaded?.some(plugin=>plugin.id===id&&plugin.state?.status==='active'));
+      const failure=loaded?.find(plugin=>expected.includes(plugin.id)&&plugin.state?.status==='failed');
+      if(active) break;
+      if(failure) throw Error(`Native OpenCode plugin failed to activate: ${JSON.stringify(failure)}`);
+      if(Date.now()+100>=deadline) throw Error(`Native benchmark plugin did not become active: ${JSON.stringify(loaded)}`);
+      await new Promise(resolve=>setTimeout(resolve,100));
+    }
+  }
+  return {endpoint,auth,version:'2.0.32',close:()=>serviceManager.stopService({root:home})};
+  } catch(error) {
+    await serviceManager.stopService({root:home}).catch(()=>{});
+    throw error;
+  }
+}
+export async function runCandidate(task, arm, { smoke = false, key, root = ROOT, campaignPath = CAMPAIGN, fetchImpl, promptOverride, toolLimit = 120, nativeRunner = runNativeEvaluation, serviceManager = { ensureService, stopService } } = {}) {
+  if ((!TASKS[task] && !smoke) || !['plain','harness','jev'].includes(arm)) throw Error('Invalid run');
+  if (process.env.CTG_OPENCODE_URL) throw Error('CTG_OPENCODE_URL is not accepted by the benchmark: service ownership and authentication must be established by this runner.');
+  if (!smoke) { if (promptOverride || toolLimit !== 120 || fetchImpl) throw Error('Candidate overrides forbidden'); verifyFreeze(root); }
+  const apiKey = key || process.env.OPENROUTER_API_KEY;
+  if (!apiKey && !nativeRunner.fixture) throw Error('Set OPENROUTER_API_KEY explicitly; credential discovery is disabled.');
+  const run = join(root, smoke ? `smoke-${arm}` : `${task}-${arm}`), workspace = join(run, 'workspace'), config = join(run, 'config');
+  if (existsSync(run)) throw Error(`Run exists; refusing overwrite: ${run}`);
+  mkdirSync(root, { recursive: true }); mkdirSync(run); mkdirSync(workspace); mkdirSync(config);
+  if (!Number.isInteger(toolLimit) || toolLimit < 1 || toolLimit > 120) throw Error('Invalid tool limit');
+  const toolsReceipt = provisionTools(config);
+  const gateway = await startGateway({ key: apiKey || 'fixture-only-no-upstream', limit: smoke ? 0.08 : 2.79, allowJev: arm === 'jev', receiptPath: join(run, 'gateway.jsonl'), fetchImpl, account: campaignAccount(campaignPath, smoke ? 'smoke' : 'candidate', run) });
+  let fixtureService, nativeService;
+  try {
+    const env = childEnv(config, gateway.url, join(run,'advisor.jsonl'));
+    const prompt = promptOverride || (smoke ? 'Infrastructure smoke only. Read TASK.md and report SMOKE_OK. Do not create a plan or change files.' : TASKS[task]);
+    writeFileSync(join(workspace, 'TASK.md'), prompt);
+    writeFileSync(join(workspace, '.gitignore'), 'node_modules/\ntarget/\nartifacts/\n.harness/\ndata/\n');
+    for (const args of [['init','-q'],['add','.'],['-c','user.name=Benchmark','-c','user.email=benchmark@localhost','commit','-qm','Frozen task']]) {
+      const result = spawnSync('git', args, { cwd: workspace, env }); if (result.status) throw Error('Git initialization failed');
+    }
+    if (nativeRunner.fixture) fixtureService = await startFixtureService(workspace);
+    else if (!process.env.CTG_OPENCODE_URL) nativeService = await startNativeService({workspace,config,mode:arm,arm,smoke,toolLimit,gateway:gateway.url,apiKey,serviceManager});
+    env.CTG_OPENCODE_URL = process.env.CTG_OPENCODE_URL || nativeService?.endpoint || fixtureService?.url;
+    env.CTG_OPENCODE_AUTHORIZATION = nativeService ? nativeService.auth.authorization : '';
+    const args = { task, arm, smoke, runtime: ENGINE_RUNTIME, openCodeService: nativeService ? 'dedicated-isolated-service' : process.env.CTG_OPENCODE_URL ? 'explicit-isolated-endpoint' : 'injected-fixture', toolsReceipt, harnessCommit: spawnSync('git',['rev-parse','HEAD'],{cwd:REPO,encoding:'utf8'}).stdout.trim(), ceilings: { seconds: smoke ? 180 : 900, tools:toolLimit, usd:smoke ? .08 : 2.79 }, startedAt:new Date().toISOString() };
+    writeFileSync(join(run,'invocation.json'), JSON.stringify(args,null,2));
+    let stopReason = null; const events = [], started = Date.now();
+    const controller = new AbortController();
+    const timer = setTimeout(() => { stopReason = 'wall-time'; controller.abort(); }, (smoke ? 180 : 900) * 1000);
+    let result;
+    try { await waitForService(env.CTG_OPENCODE_URL,{signal:controller.signal}); result = await nativeRunner({ directory: workspace, prompt, systemPrompt: arm === 'plain' ? '' : `Use the Carthagent delivery workflow. Required checks: ${JSON.stringify({ toolLimit })}.`, provider:arm === 'plain' ? 'benchmark-fixture' : 'benchmark', model:arm === 'plain' ? 'fixture' : MODEL, timeoutSeconds: smoke ? 180 : 900, maxTurns: toolLimit, maxCost: smoke ? .08 : 2.79, signal: controller.signal, baseUrl: env.CTG_OPENCODE_URL, headers:nativeService?.auth || {}, onEvent: event => { events.push(event); appendFileSync(join(run,'events.jsonl'),JSON.stringify(event)+'\n'); } }); }
+    catch (error) { result = { exitCode:1, errors:[error.message], turns:0, productiveTurns:0, tools:{} }; }
+    finally { clearTimeout(timer); }
+    if (result.outcome !== 'succeeded') {
+      result.exitCode = 1;
+      result.budgetReason ||= 'missing terminal session outcome';
+      result.errors = [...(result.errors || []), 'Native run cannot be successful without outcome=succeeded from session.get().'];
+    }
+  const classification = terminalStatus(events,{code:result.exitCode},stopReason,gateway.budget,result.outcome);
+    if (result.errors?.some(error => /usage|cost/i.test(error)) || result.budgetReason === 'unknown usage; fail-closed') gateway.budget.stopped = 'unknown-or-excess-billing';
+    const guardRows=existsSync(join(run,'guard.jsonl'))?readFileSync(join(run,'guard.jsonl'),'utf8').trim().split('\n').filter(Boolean).map(JSON.parse):[];
+    const final = { task,arm,smoke,...classification,status:classification.status,smokePassed:smoke ? result.exitCode===0 && result.outcome==='succeeded' && !result.budgetReason : undefined, stopReason:result.budgetReason || stopReason, outcome:result.outcome||null, interruptionStatus:result.interruptionStatus||'unknown', terminalAuthority:'session.outcome', costScope:'session', usageSource:'session.get', toolStarts:Object.values(result.tools||{}).reduce((a,b)=>a+b,0), toolAdmittedByBudget:guardRows.filter(row=>!row.blocked).length, toolBlockedByBudget:guardRows.filter(row=>row.blocked).length, wallMs:Date.now()-started,budget:gateway.budget,runtime:ENGINE_RUNTIME,openCodeVersion:result.opencodeVersion||nativeService?.version||'2.0.32',productiveTurns:result.productiveTurns||0, productiveTurnEvidence:'paginated-session-transcript',reportedCost:result.cost,finishedAt:new Date().toISOString() };
+    writeFileSync(join(run,'result.json'),JSON.stringify(final,null,2)); console.log(JSON.stringify(final)); return final;
+  } finally { await nativeService?.close(); await fixtureService?.close(); await gateway.close(); }
+}
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  const key = process.env.OPENROUTER_API_KEY;
+  const [task,arm]=process.argv.slice(2);
+  if(task==='matrix') { verifyFreeze(ROOT); for(const [t,a] of MATRIX) { const result = await runCandidate(t,a,{key}); if(['infrastructure-error','process-error','missing-terminal-message','model-error'].includes(result.status)) throw Error('Matrix stopped on infrastructure/billing failure; retain results and diagnose before any further runs.'); } }
+  else await runCandidate(task,arm,{key,smoke:task==='smoke'});
+}