carthagent 0.0.0-stage → 1.0.7

package/tools/benchmark/runner.test.mjs

@@ -0,0 +1,93 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { initializeCampaign } from './campaign.mjs';
+import { runCandidate, terminalStatus } from './runner.mjs';
+import { createFreeze, verifyFreeze, frozenFiles } from './freeze.mjs';
+import { sha } from './runtime.mjs';
+import { nativeFixture } from '../native-fixture.mjs';
+const mk = () => mkdtempSync(join(tmpdir(), 'benchmark-runner-'));
+
+test('native terminal classification does not mistake process success for model success', () => {
+  assert.equal(terminalStatus([{ type:'session.step.ended', data:{ finish:'error' } }], {code:0}, null, {},'failed').status, 'non-successful-session-outcome');
+  assert.equal(terminalStatus([], {code:0}, null, {},'succeeded').status, 'missing-terminal-message');
+});
+test('native runtime freeze includes source and package lock but no Pi install or user auth', () => {
+  assert.ok(frozenFiles().some(path => path.endsWith('/lib/opencode/evaluation.mjs')));
+  assert.ok(frozenFiles().some(path => path.endsWith('/package-lock.json')));
+  assert.ok(frozenFiles().every(path => !path.includes('/.pi/') && !path.includes('/node_modules/')));
+  const root = mk(), receipt = join(root,'ready.json'), live = join(root,'live.json');
+  writeFileSync(live,'{"passed":true}');
+  writeFileSync(receipt,JSON.stringify({passed:true,files:Object.fromEntries(frozenFiles().map(path=>[path,sha(readFileSync(path))])),live:{path:live,sha256:sha(readFileSync(live))}}));
+  createFreeze(root,receipt); assert.equal(verifyFreeze(root).runtime,'opencode-2');
+});
+test('offline fixture runner logs runtime and uses no provider credentials', async () => {
+  const root=mk(), campaignPath=join(root,'campaign.json');
+  initializeCampaign(campaignPath,{historicalCandidate:0,historicalSmoke:0});
+  const fake = async options => {
+    assert.equal(options.directory,join(root,'smoke-plain','workspace'));
+    options.onEvent({type:'session.step.ended',data:{sessionID:'fixture',assistantMessageID:'m1',finish:'stop'}});
+    return {exitCode:0,turns:1,productiveTurns:1,tools:{read:1},cost:0,outcome:'succeeded',lastStop:'stop',opencodeVersion:'2.0.32'};
+  };
+  fake.fixture = true;
+  const result=await runCandidate('smoke','plain',{smoke:true,root,campaignPath,nativeRunner:fake});
+  assert.equal(result.runtime,'opencode-2');
+  assert.equal(result.status,'completed');
+  assert.equal(result.outcome,'succeeded');
+  assert.equal(JSON.parse(readFileSync(join(root,'smoke-plain','invocation.json'))).runtime,'opencode-2');
+  assert.equal(existsSync(join(root,'smoke-plain','config','runtime.json')),true);
+});
+
+test('benchmark refuses arbitrary service endpoints before creating a run', async () => {
+  const root=mk(), campaignPath=join(root,'campaign.json');
+  initializeCampaign(campaignPath,{historicalCandidate:0,historicalSmoke:0});
+  const before=process.env.CTG_OPENCODE_URL;
+  process.env.CTG_OPENCODE_URL='http://127.0.0.1:1';
+  try {
+    await assert.rejects(runCandidate('smoke','plain',{smoke:true,root,campaignPath,nativeRunner:Object.assign(async()=>{throw Error('must not execute');},{fixture:true})}),/ownership and authentication/);
+    assert.equal(existsSync(join(root,'smoke-plain')),false,'rejected endpoint must be rejected before allocating a run');
+  } finally {
+    if(before===undefined) delete process.env.CTG_OPENCODE_URL;
+    else process.env.CTG_OPENCODE_URL=before;
+  }
+});
+
+test('native benchmark plugin loads in OpenCode 2 with both session and tool hooks', async () => {
+  const root=mk();
+  const host=await nativeFixture({
+    plugin:join(process.cwd(),'lib/opencode/evaluation-budget-plugin'),
+    plugins:[join(process.cwd(),'tools/benchmark/native-plugin'),join(process.cwd(),'plugins/carthagent')],
+    env:{BENCH_TOOL_LIMIT:'1',BENCH_RETRY_LIMIT:'0',BENCH_GUARD_LOG:join(root,'guard.jsonl'),BENCH_ADVISOR_LOG:join(root,'advisor.jsonl'),BENCH_GATEWAY:'http://127.0.0.1:1',CTG_EVALUATION_MAX_TURNS:'4',CTG_EVALUATION_MAX_COST:'1'},
+    respond(body,res) {
+      if(JSON.stringify(body.messages).includes('CTG_PARALLEL_GUARD_PROBE')) {
+        const shell=body.tools.find(tool=>tool.function?.name==='shell');
+        assert.ok(shell,'native host must advertise shell for the concurrent tool admission probe');
+        const calls=Array.from({length:8},(_,index)=>({index,id:`parallel-${index}`,type:'function',function:{name:'shell',arguments:JSON.stringify({command:`printf parallel-${index}`})}}));
+        const chunk=(delta,finish_reason=null)=>`data: ${JSON.stringify({id:'parallel-guard-probe',object:'chat.completion.chunk',created:1,model:'fixture',choices:[{index:0,delta,finish_reason}]})}\n\n`;
+        res.writeHead(200,{'content-type':'text/event-stream'});
+        res.end(`${chunk({role:'assistant',tool_calls:calls})}${chunk({},'tool_calls')}data: [DONE]\n\n`);
+        return;
+      }
+      streamText(res);
+    },
+  });
+  try {
+    await host.active('ctg.benchmark');
+    const inventory=await host.client.plugin.list({location:host.location},host.request());
+    assert.ok(inventory.data.some(plugin=>plugin.id==='ctg.evaluation-budget'&&plugin.state.status==='active'),JSON.stringify(inventory.data));
+    const response=await host.client.session.create({location:host.location,model:host.model,title:'hook initialization probe'},host.request());
+    assert.ok(response.id);
+    const session=await host.client.session.create({location:host.location,model:host.model,title:'parallel admission probe',metadata:{'ctg.evaluation.budget':{maxTurns:4,maxCost:1}}},host.request());
+    await host.client.session.prompt({sessionID:session.id,text:'CTG_PARALLEL_GUARD_PROBE'},host.request());
+    await host.client.session.wait({sessionID:session.id},host.request());
+    assert.ok(existsSync(join(root,'guard.jsonl')),`${host.output}\n${JSON.stringify(host.requests)}`);
+    const rows=readFileSync(join(root,'guard.jsonl'),'utf8').trim().split('\n').map(JSON.parse).filter(row=>row.sessionID===session.id);
+    assert.equal(rows.length,8,JSON.stringify(rows));
+    assert.equal(rows.filter(row=>!row.blocked).length,1,JSON.stringify(rows));
+    assert.equal(rows.filter(row=>row.blocked).length,7,JSON.stringify(rows));
+    const ledger=await host.client.session.context({sessionID:session.id},host.request());
+    assert.ok(ledger.at(-1)?.outcome,JSON.stringify(ledger.at(-1)));
+  } finally { await host.close(); }
+});

package/tools/benchmark/runtime.mjs

@@ -0,0 +1,23 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+export const ENGINE_RUNTIME = 'opencode-2';
+export const isolatedHome = config => join(config, 'home');
+export function safeEnv(extra = {}, home = undefined) {
+  const allowed = { PATH: process.env.PATH || '', LANG: process.env.LANG || 'C.UTF-8', CI: '1', NODE_ENV: 'production' };
+  if (home) { mkdirSync(home, { recursive: true }); allowed.HOME = home; allowed.XDG_CONFIG_HOME = join(home, '.config'); allowed.XDG_DATA_HOME = join(home, '.local/share'); allowed.XDG_CACHE_HOME = join(home, '.cache'); }
+  return { ...allowed, ...extra };
+}
+export const sha = value => createHash('sha256').update(value).digest('hex');
+export function provisionTools(config) {
+  mkdirSync(config, { recursive: true });
+  const receipt = { runtime: ENGINE_RUNTIME, opencodePackage: '@opencode/client@2.0.32', home: isolatedHome(config), noPiDiscovery: true };
+  writeFileSync(join(config, 'runtime.json'), JSON.stringify(receipt, null, 2), { flag: 'wx' });
+  return receipt;
+}
+export async function stopGroup(child, graceMs = 250) {
+  if (!child?.pid) return;
+  try { process.kill(-child.pid, 'SIGTERM'); } catch {}
+  await new Promise(r => setTimeout(r, graceMs));
+  try { process.kill(-child.pid, 'SIGKILL'); } catch {}
+}

package/tools/benchmark/selftest.mjs

@@ -0,0 +1,47 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync,mkdirSync,writeFileSync,readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createServer } from 'node:http';
+import { evaluateLedger } from './eval-ledger.mjs';
+import { quoteOracle } from './eval-shop.mjs';
+import { validWorld,changedBlocks } from './eval-voxel.mjs';
+import { browser } from './eval-common.mjs';
+// Independent test-only reference process. Never copied into any candidate workspace.
+const reference=String.raw`
+import fs from 'node:fs';import readline from 'node:readline';
+const file=process.argv[3], balances={},seen=new Map(),events=[];const max=99999999999999999999n;
+const amount=s=>{if(typeof s!=='string'||! /^[+-]?\d+(\.\d{1,2})?$/.test(s))throw Error('decimal');const neg=s[0]==='-';s=s.replace(/^[+-]/,'');const [a,b='']=s.split('.');const n=BigInt(a)*100n+BigInt(b.padEnd(2,'0'));if(n>max)throw Error('range');return neg?-n:n;};
+const fmt=n=>String(n/100n)+'.'+String(n%100n).padStart(2,'0');
+function apply(c,replay=false){const id=s=>{if(typeof s!=='string'||! /^[A-Za-z0-9_-]{1,40}$/.test(s))throw Error('id');return s;};
+ if(c.op==='snapshot')return {ok:true,balances:Object.fromEntries(Object.entries(balances).map(([k,n])=>[k,fmt(n)]))};
+ if(c.op==='balance'){id(c.account);if(balances[c.account]===undefined)throw Error('missing');return {ok:true,balance:fmt(balances[c.account])};}
+ id(c.key);let norm,n;
+ if(c.op==='open'){id(c.account);n=amount(c.balance);if(n<0n)throw Error('negative');norm=JSON.stringify(['open',c.account,String(n)]);}
+ else if(c.op==='transfer'){id(c.from);id(c.to);n=amount(c.amount);norm=JSON.stringify(['transfer',c.from,c.to,String(n)]);}else throw Error('op');
+ if(seen.has(c.key)){if(seen.get(c.key)!==norm)throw Error('conflict');return {ok:true};}
+ if(c.op==='open'){if(balances[c.account]!==undefined)throw Error('exists');balances[c.account]=n;}
+ else{if(c.from===c.to||n<=0n||balances[c.from]===undefined||balances[c.to]===undefined||balances[c.from]<n||balances[c.to]+n>max)throw Error('transfer');balances[c.from]-=n;balances[c.to]+=n;}
+ seen.set(c.key,norm);if(!replay)fs.appendFileSync(file,JSON.stringify(c)+'\n');return {ok:true};}
+try{if(fs.existsSync(file))for(const line of fs.readFileSync(file,'utf8').split('\n').filter(Boolean)){const c=JSON.parse(line);if(!['open','transfer'].includes(c.op))throw Error('record');apply(c,true);}}catch{process.exit(2);}
+for await(const line of readline.createInterface({input:process.stdin})){try{console.log(JSON.stringify(apply(JSON.parse(line))));}catch(e){console.log(JSON.stringify({ok:false,error:e.message}));}}
+`;
+test('ledger evaluator passes reference and detects float/idempotency mutants through real processes',async()=>{
+ const root=mkdtempSync(join(tmpdir(),'ledger-oracle-'));writeFileSync(join(root,'ledger.mjs'),reference);
+ const positive=await evaluateLedger(root,join(root,'good'));assert.equal(positive.passed,positive.total,JSON.stringify(positive));
+ writeFileSync(join(root,'ledger.mjs'),reference.replace('BigInt(a)*100n+BigInt(b.padEnd(2,\'0\'))','BigInt(Math.round(Number(s)*100))'));
+ const floating=await evaluateLedger(root,join(root,'float'));assert.ok(floating.results.some(r=>r.name.includes('large magnitudes')&&!r.passed));
+ writeFileSync(join(root,'ledger.mjs'),reference.replace("if(seen.has(c.key))","if(false && seen.has(c.key))"));
+ const duplicate=await evaluateLedger(root,join(root,'duplicate'));assert.ok(duplicate.results.some(r=>r.name.includes('idempotency')&&!r.passed));
+});
+test('shop oracle catches rounding/shipping mutants and voxel oracle rejects unchanged/malformed state',()=>{
+ assert.deepEqual(quoteOracle([{id:'mug',quantity:1}],'WELCOME10'),{subtotalCents:1299,discountCents:129,shippingCents:599,totalCents:1769});
+ assert.notEqual(quoteOracle([{id:'mug',quantity:1}],'WELCOME10').discountCents,Math.round(1299*.1));
+ assert.equal(quoteOracle([{id:'mat',quantity:2},{id:'mug',quantity:1}],'WELCOME10').shippingCents,599);
+ const s={seed:1,player:{x:1,y:3,z:1,yaw:0,pitch:0},blocks:Array.from({length:256},(_,i)=>[i%16,0,Math.floor(i/16),i%3+1])};validWorld(s);assert.equal(changedBlocks(s,structuredClone(s)),0);assert.throws(()=>validWorld({...s,blocks:[]}));assert.throws(()=>validWorld({...s,player:{...s.player,y:0}}));
+});
+test('real Firefox infrastructure and native click screenshot',async()=>{
+ const base='/home/baissi/benchmarks/infrastructure-tests';mkdirSync(base,{recursive:true});const root=mkdtempSync(join(base,'browser-'));const server=createServer((req,res)=>{res.setHeader('Content-Type','text/html');res.end('<button onclick="this.textContent=\'clicked\'" id="go">ready</button>');});await new Promise(r=>server.listen(0,'127.0.0.1',r));let b;
+ try{b=await browser(root);await b.call('/url',{url:`http://127.0.0.1:${server.address().port}`});const el=await b.call('/element',{using:'css selector',value:'#go'});await b.call('/element/'+el['element-6066-11e4-a52e-4f735466cecf']+'/click',{});assert.equal(await b.exec('document.querySelector("#go").textContent'),'clicked');await b.shot(root+'/smoke.png');assert.equal(readFileSync(root+'/smoke.png').subarray(1,4).toString(),'PNG');}finally{await b?.close();await new Promise(r=>server.close(r));}
+});

package/tools/benchmark/smoke-jev.mjs

@@ -0,0 +1,5 @@
+// Retained entry point; the old standalone smoke bypassed campaign accounting.
+// All new live checks now use the exclusive, centrally accounted preflight.
+import { livePreflight } from './preflight-live.mjs';
+if (process.argv[2] !== '--paid') throw Error('Use node tools/benchmark/preflight-live.mjs --paid (shared smoke budget, no overwrite/retry).');
+console.log(JSON.stringify(await livePreflight()));

package/tools/benchmark/tasks.mjs

@@ -0,0 +1,20 @@
+export const COMMON = `Implement the full task in this initially empty Git workspace. Do not modify TASK.md. Include README launch instructions and meaningful automated tests. Work only within this workspace; do not inspect sibling benchmark runs, supervisor scripts/evaluators or host credentials, and do not call external AI services. Dependencies may be installed. Node 26, Rust 1.98/cargo, cc and Firefox are available. No real payments, deployments or external accounts. Budget: 15 minutes and 120 tool starts, so deliver a runnable vertical slice early. Independent evaluation runs after you finish; you will not receive its source or results during implementation. Avoid leaving development servers running. Final response: list implemented features, commands actually tested and remaining limitations.\n`;
+export const TASKS = {
+  voxel: `${COMMON}
+Build a small playable Minecraft-like voxel sandbox in Rust, not merely a data model or static picture. Rust must own terrain, physics, interactions, saves AND first-person software rendering. A thin browser JS input/display frontend is allowed. Do not require GPU/display or downloaded art. Use distinct block colors, sky, depth shading, crosshair, instructions and a visible selected-block indicator. WASD moves, arrow keys turn/look, Space jumps, left click removes and right click places a selected block; keys 1/2/3 choose material. Collisions prevent walking through solid terrain, gravity settles the player, reach <=5 blocks, and placement cannot intersect player. Deterministic finite terrain >=16x16 horizontally, >=8 high, with at least grass/dirt/stone. Same seed => same world; different seeds => different terrain. Save and reload preserve blocks and player; malformed/out-of-bounds saves are rejected atomically.
+Public interface (required for independent evaluation): Cargo binary named voxel. cargo build --release; launch target/release/voxel --port PORT --save FILE (bind 127.0.0.1). Serve browser UI at /. GET /api/state -> JSON {seed,player:{x,y,z,yaw,pitch},selected,blocks:[[x,y,z,material],...]}; occupied blocks only, integer coordinates, materials 1/2/3. POST /api/action JSON actions below returns {ok:true,state:<same shape>} or {ok:false,error:string}; HTTP 4xx is acceptable for rejected actions. GET /api/frame -> Rust-rendered PPM P6 image >=160x100 (browser may convert to canvas). Coordinates: y up, yaw 0 looks +z; pitch 0 horizontal. APIs must power normal browser controls, not evaluator-only stand-ins.
+Actions: {type:"reset",seed:integer}; {type:"look",yaw:number,pitch:number} sets absolute view radians (clamp pitch safely); {type:"move",forward:number,right:number,jump:boolean,dt:number} with forward/right in [-1,1], dt in (0,0.25], move uses current yaw, applies physics; {type:"select",material:1|2|3}; {type:"remove"}; {type:"place"}; {type:"save"}; {type:"load"}. Unknown/invalid actions must not mutate. Finite world boundary must be collision-safe. On reset spawn in free space above terrain, near its center. Include unit tests and HTTP integration tests. No crafting, mobs, infinite streaming or multiplayer required.
+`,
+  shop: `${COMMON}
+Build a polished responsive outdoor-products sales website with a real Node backend, no framework required. Launch node server.mjs; PORT and DATA_DIR environment variables choose bind port and persistence directory (127.0.0.1). npm test runs your automated tests. Serve / with semantic labeled controls, keyboard accessibility, visible loading/error/success feedback, attractive desktop/mobile layout, product cards, search/category/sort, cart quantity/remove, promo, checkout form and order confirmation. Use local CSS/SVG/emoji rather than unavailable remote assets. Cart survives page reload via localStorage; server owns pricing, inventory and orders. No actual payment.
+Catalog on first launch, exact fields {id,name,category,priceCents,stock}: tent / Alpine Tent / shelter / 12999 / 3; mug / Trail Mug / accessories / 1299 / 8; lamp / Camp Lantern / lighting / 3499 / 4; pack / Ridge Pack / bags / 7999 / 2; rope / Utility Rope / accessories / 999 / 0; mat / Sleeping Mat / shelter / 4599 / 5. All prices/totals integer cents. WELCOME10 gives floor(subtotalCents/10) discount. Shipping 599 unless post-discount subtotal >=10000, then 0. Unknown nonempty coupon rejected; empty cart rejected. Quantities positive integers, merge duplicate product IDs before validating aggregate stock. UI must not add out-of-stock products.
+Public JSON APIs: GET /api/products?q=<case-insensitive-name-substring>&category=<exact-category>&sort=price-asc|price-desc|name -> {products:[...]}, optional filters compose. POST /api/quote {items:[{id,quantity}],coupon:""} -> {subtotalCents,discountCents,shippingCents,totalCents}. POST /api/orders same plus {key:string,customer:{name,email,address}} -> {order:{id,items,subtotalCents,discountCents,shippingCents,totalCents}}. Trimmed name>=2 chars, conventional nonblank email containing @ with domain dot, trimmed address>=8 chars required. Invalid input returns HTTP 4xx JSON {error:string} with NO mutation. Same key+same normalized purchase/customer request => same order exactly and no stock decrement; same key+different request =>409. GET /api/orders/:id -> {order}. Orders and stock survive backend restart, including idempotency. Check and decrement inventory atomically even for concurrent orders.
+Stable UI test hooks (data-testid values): search (input), category (select), sort (select), products (container), add-<id> (button), cart (container), qty-<id> (numeric input inside cart), remove-<id> (button), coupon (input), total (text with formatted dollars), customer-name/email/address (inputs), checkout (button), status (aria-live status). Updates may be async, <=2 seconds. Cart quantity input change recalculates totals. Include meaningful backend and browser-oriented tests.
+`,
+  ledger: `${COMMON}
+Implement an exact-money transfer ledger in Node, using only built-in dependencies. Launch node ledger.mjs --journal PATH. Read one JSON object per stdin line and emit exactly one JSON object per line (no other stdout); process until EOF. Each request is atomic. Persist accepted mutations in append-only JSONL; restart replays balances and idempotency. Missing journal creates empty state. Malformed JSON or invalid records anywhere in existing journal => exit nonzero before reading commands, never truncate/repair it silently. Handle malformed request lines with {ok:false,error:string} and continue.
+Commands: {op:"open",account:string,balance:string,key:string}; {op:"transfer",from:string,to:string,amount:string,key:string}; {op:"balance",account:string}; {op:"snapshot"}. Account/key IDs must match /^[A-Za-z0-9_-]{1,40}$/. open creates a previously absent account with nonnegative balance; transfer requires distinct existing accounts, strictly positive amount, enough funds. Mutations have globally unique idempotency keys: semantically identical retry returns its original result without reapplying; same key with a different normalized command => error and no mutation. Decimal money syntax /^[+-]?\\d+(\\.\\d{1,2})?$/ with no whitespace, exponent or rounding. Leading zeroes and + accepted; negative zero equivalent to zero. Normalize equivalent decimal spellings for idempotency. Amount range absolute <=999999999999999999.99 (cents <=99999999999999999999); reject source/destination overflow atomically. No IEEE-754 parsing/arithmetic for money.
+Responses: mutations {ok:true}, balance {ok:true,balance:"canonical decimal with exactly two places"}, snapshot {ok:true,balances:{account:"canonical amount",...}}, all failures {ok:false,error:nonempty string}. No timestamps/extra nondeterministic fields needed. Unknown operations rejected. Failed mutations neither consume key nor modify durable state; rejected transfer can be retried successfully later using same key. Balances always nonnegative and transfers conserve exact total. Supply npm test with adversarial and restart integration coverage; README describes crash/concurrency limits honestly. One process writer at a time is sufficient; no multi-process locking required.
+`,
+};
+export const MATRIX = [['voxel','plain'],['voxel','harness'],['voxel','jev'],['shop','harness'],['shop','jev'],['shop','plain'],['ledger','jev'],['ledger','plain'],['ledger','harness']];

package/tools/benchmark/transport.mjs

@@ -0,0 +1,65 @@
+import { spawn } from 'node:child_process';
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { safeEnv } from './runtime.mjs';
+// Credentials travel through stdin config, never argv or files. Request BODY uses
+// a private 0700 directory / 0600 file: curl's inline config has a line-size limit.
+// No Node TLS / connection reuse; body files are removed on close/spawn failure.
+// Abrupt supervisor termination may leave private files; this is not zero retention.
+export function createCurlTransport({ executable = '/usr/bin/curl', caFile, maxBytes = 4000000, timeoutMs = 180000, allowLoopbackHttp = false } = {}) {
+  return async function curlFetch(url, options = {}) {
+    const target = new URL(url);
+    if (target.protocol !== 'https:' && !(allowLoopbackHttp && target.protocol === 'http:' && target.hostname === '127.0.0.1')) throw Error('HTTPS required');
+    if (target.username || target.password) throw Error('URL credentials forbidden');
+    options.signal?.throwIfAborted();
+    const headers = Object.entries(options.headers || {}).map(([k,v]) => {
+      if (!/^[A-Za-z0-9-]+$/.test(k) || /[\r\n\0]/.test(String(v))) throw Error('Invalid HTTP header');
+      return `header = ${JSON.stringify(`${k}: ${v}`)}\n`;
+    }).join('');
+    const directory = mkdtempSync(join(tmpdir(), 'ctg-http-body-'));
+    const bodyPath = join(directory, 'body');
+    try { writeFileSync(bodyPath, options.body || '', { mode: 0o600 }); }
+    catch (error) { rmSync(directory,{recursive:true,force:true}); throw error; }
+    const args = ['-q', '--silent', '--show-error', '--http1.1', '--proto', allowLoopbackHttp ? '=http,https' : '=https', '--max-redirs', '0', '--connect-timeout', '15', '--max-time', String(timeoutMs / 1000), '--retry', '0', '--request', options.method || 'POST', '--config', '-', '--data-binary', '@' + bodyPath, '--include', '--write-out', '\n%{http_code}', ...(caFile ? ['--cacert', caFile] : []), '--url', target.href];
+    return new Promise((resolve, reject) => {
+      const child = spawn(executable, args, { env: safeEnv(), stdio: ['pipe','pipe','pipe'] });
+      let chunks = [], bytes = 0, failure;
+      const fail = (code) => { failure ||= Object.assign(Error('HTTPS transport failed'), { code }); child.kill('SIGKILL'); };
+      const abort = () => fail('ABORTED');
+      const timer = setTimeout(() => fail('TRANSPORT_TIMEOUT'), timeoutMs + 1000);
+      options.signal?.addEventListener('abort', abort, { once: true });
+      child.stdout.on('data', b => { bytes += b.length; if (bytes > maxBytes + 65536 + 4) fail('RESPONSE_TOO_LARGE'); else chunks.push(b); });
+      // Drain but do not expose curl stderr, which can include server-controlled content.
+      child.stderr.on('data', () => {});
+      child.on('error', () => { failure ||= Object.assign(Error('HTTPS transport unavailable'), { code: 'TRANSPORT_SPAWN' }); });
+      child.stdin.on('error', () => {});
+      child.on('close', code => {
+        clearTimeout(timer); options.signal?.removeEventListener('abort', abort);
+        rmSync(directory,{recursive:true,force:true});
+        if (failure || code !== 0) return reject(failure || Object.assign(Error('HTTPS transport failed'), { code: `CURL_${code}` }));
+        const data = Buffer.concat(chunks), status = Number(data.subarray(-3).toString());
+        if (data.at(-4) !== 10 || status < 200 || status > 599) return reject(Error('Invalid HTTPS status'));
+        let body = data.subarray(0,-4), headerBytes = 0, fields;
+        do {
+          const end = body.indexOf('\r\n\r\n');
+          if (end < 0 || !body.subarray(0,5).equals(Buffer.from('HTTP/'))) return reject(Error('Invalid HTTPS headers'));
+          headerBytes += end + 4;
+          if (headerBytes > 65536) return reject(Error('HTTPS headers too large'));
+          const lines = body.subarray(0,end).toString().split('\r\n');
+          const interim = /^HTTP\/\S+ 1\d\d\b/.test(lines[0]) || / 200 Connection established$/i.test(lines[0]);
+          fields = new Headers();
+          for (const line of lines.slice(1)) { const colon = line.indexOf(':'); if(colon>0 && ['x-request-id','request-id','cf-ray'].includes(line.slice(0,colon).toLowerCase())) fields.set(line.slice(0,colon),line.slice(colon+1).trim()); }
+          body = body.subarray(end+4);
+          if (!interim) break;
+        } while (body.length);
+        if (body.length > maxBytes) return reject(Object.assign(Error('HTTPS transport failed'),{code:'RESPONSE_TOO_LARGE'}));
+        resolve(new Response([204,205,304].includes(status) ? null : body, { status, headers:fields }));
+      });
+      // Only short header lines enter curl's config parser. Body is byte-preserving.
+      child.stdin.end(headers);
+      if (options.signal?.aborted) abort();
+    });
+  };
+}
+export const curlFetch = createCurlTransport();

package/tools/benchmark/transport.test.mjs

@@ -0,0 +1,44 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createServer } from 'node:https';
+import { createServer as httpServer } from 'node:http';
+import { sha } from './runtime.mjs';
+import { mkdtempSync,readFileSync,existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { createCurlTransport } from './transport.mjs';
+test('real TLS: trusted local certificate, fresh requests, no retry, redirect, bounds, deadline and cancellation',async()=>{
+ const dir=mkdtempSync(join(tmpdir(),'benchmark-tls-')),key=join(dir,'key.pem'),cert=join(dir,'cert.pem');
+ const r=spawnSync('openssl',['req','-x509','-newkey','rsa:2048','-nodes','-keyout',key,'-out',cert,'-days','1','-subj','/CN=localhost','-addext','subjectAltName=IP:127.0.0.1'],{encoding:'utf8'});assert.equal(r.status,0,r.stderr);
+ let calls=0;const sockets=new Set();
+ const server=createServer({key:readFileSync(key),cert:readFileSync(cert)},async(req,res)=>{calls++;sockets.add(req.socket);let body='';for await(const chunk of req)body+=chunk;
+  if(req.url==='/stall')return;
+  if(req.url==='/redirect'){res.writeHead(302,{location:'/ok'});return res.end();}
+  if(req.url==='/big')return res.end('x'.repeat(10000));
+  if(req.url==='/disconnect'){req.socket.destroy();return;}
+  res.setHeader('x-request-id','tls-fixture');
+  assert.equal(req.headers.authorization,'Bearer private-test');res.write(Buffer.from('héllo 🌍').subarray(0,3));res.end(Buffer.concat([Buffer.from('héllo 🌍').subarray(3),Buffer.from(body)]));
+ });await new Promise(r=>server.listen(0,'127.0.0.1',r));const url=`https://127.0.0.1:${server.address().port}`;
+ const options={method:'POST',headers:{Authorization:'Bearer private-test'},body:'payload'};
+ try{
+  await assert.rejects(createCurlTransport()(url,options),e=>e.code==='CURL_60');assert.equal(calls,0,'untrusted cert must fail');
+  const fetch=createCurlTransport({caFile:cert});
+  for(let i=0;i<25;i++)assert.equal(await(await fetch(url,options)).text(),'héllo 🌍payload');assert.equal(sockets.size,25,'fresh TLS connections');
+  assert.equal((await fetch(url+'/redirect',options)).status,302);assert.equal(calls,26,'no redirect follow');
+  await assert.rejects(fetch(url+'/disconnect',options));assert.equal(calls,27,'no retry of ambiguous POST');
+  await assert.rejects(createCurlTransport({caFile:cert,maxBytes:100})(url+'/big',options),e=>e.code==='RESPONSE_TOO_LARGE');
+  await assert.rejects(createCurlTransport({caFile:cert,timeoutMs:100})(url+'/stall',options));
+  const ac=new AbortController();setTimeout(()=>ac.abort(),50);await assert.rejects(fetch(url+'/stall',{...options,signal:ac.signal}),e=>e.code==='ABORTED');
+  await assert.rejects(fetch('http://example.com',options),/HTTPS required/);
+  await assert.rejects(fetch(url,{headers:{Authorization:'x\nInjected: y'}}),/Invalid HTTP header/);
+  await assert.rejects(createCurlTransport({executable:'/missing-curl'})(url,options),e=>e.code==='TRANSPORT_SPAWN');
+  const large=JSON.stringify({content:'héllo\\\"\n'.repeat(16000)});const largeResponse=await fetch(url,{...options,body:large});assert.equal(largeResponse.headers.get('x-request-id'),'tls-fixture');assert.equal(await largeResponse.text(),'héllo 🌍'+large);
+ }finally{server.closeAllConnections();await new Promise(r=>server.close(r));}
+});
+test('real curl wire body remains byte-exact across former config limits and historical multi-turn request',async()=>{
+ const received=[];const server=httpServer(async(req,res)=>{const chunks=[];for await(const c of req)chunks.push(c);const bytes=Buffer.concat(chunks);received.push(bytes);res.end(JSON.stringify({bytes:bytes.length,hash:sha(bytes)}));});await new Promise(r=>server.listen(0,'127.0.0.1',r));
+ const send=createCurlTransport({allowLoopbackHttp:true});const payloads=[...Array.from([60000,90000,97377,130000,399000],n=>JSON.stringify({content:'x'.repeat(n)})),JSON.stringify({content:'line\nquote"slash\\tab\té🌍'.repeat(9000)})];
+ const historical='/home/baissi/benchmarks/deepseek-jev-v3/http400-diagnostic/payload.json';if(existsSync(historical))payloads.push(readFileSync(historical,'utf8'));
+ try{for(const body of payloads){const r=await send(`http://127.0.0.1:${server.address().port}`,{body,headers:{'Content-Type':'application/json'}});const d=await r.json();assert.equal(d.hash,sha(body));assert.equal(d.bytes,Buffer.byteLength(body));assert.deepEqual(JSON.parse(received.at(-1)),JSON.parse(body));}}finally{server.closeAllConnections();await new Promise(r=>server.close(r));}
+});

package/tools/benchmark/upstream-error.mjs

@@ -0,0 +1,18 @@
+import { createHash } from 'node:crypto';
+// Known credentials are removed BEFORE truncation. This is not general-purpose
+// secret detection: provider errors may echo submitted source or planning data.
+export function redactError(value, secrets = []) {
+  let text=String(value);
+  for(const secret of secrets.filter(s=>typeof s==='string'&&s.length)){
+    const variants=[secret,JSON.stringify(secret).slice(1,-1),encodeURIComponent(secret),Buffer.from(secret).toString('base64')];
+    for(const v of variants)text=text.split(v).join('[REDACTED]');
+  }
+  return text.replace(/Bearer\s+[^\s"'<>]+/gi,'Bearer [REDACTED]').replace(/sk-or-v1-[A-Za-z0-9_-]+/g,'[REDACTED]').replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g,'?');
+}
+export async function upstreamError(response,{secrets=[],maxBytes=16384}={}){
+  const reader=response.body?.getReader();const chunks=[];let bytes=0,truncated=false;
+  try{if(reader)for(;;){const {done,value}=await reader.read();if(done)break;const take=Math.min(value.byteLength,maxBytes-bytes);chunks.push(value.subarray(0,take));bytes+=take;if(value.byteLength>take||bytes===maxBytes){truncated=true;await reader.cancel();break;}}}finally{reader?.releaseLock();}
+  const body=Buffer.concat(chunks).toString('utf8');let parsed;try{parsed=JSON.parse(body);}catch{}
+  const clean=v=>typeof v==='string'||typeof v==='number'?redactError(v,secrets).slice(0,2000):null;
+  return {status:response.status,code:clean(parsed?.error?.code),message:clean(parsed?.error?.message || parsed?.message),provider:clean(parsed?.error?.metadata?.provider_name),requestId:clean(response.headers.get('x-request-id')||parsed?.request_id||parsed?.id),bodyExcerpt:truncated?'[Oversized upstream error omitted]':redactError(body,secrets).slice(0,8000),truncated:truncated||body.length>8000,bodyPrefixSha256:createHash('sha256').update(body).digest('hex')};
+}

package/tools/benchmark/upstream-error.test.mjs

@@ -0,0 +1,10 @@
+import test from 'node:test';import assert from 'node:assert/strict';
+import {upstreamError}from'./upstream-error.mjs';import{startGateway,MODEL}from'./gateway.mjs';
+import{reconstruct,inspectRequest}from'./diagnose-request.mjs';import{existsSync}from'node:fs';
+test('bounded error receipts redact reflected credentials and preserve diagnostic fields',async()=>{
+ const secret='private+key/test';const r=await upstreamError(Response.json({error:{code:400,message:'Invalid tool history: '+secret,metadata:{provider_name:'fixture',raw:'Bearer token '+encodeURIComponent(secret)}}},{status:400,headers:{'x-request-id':'req123'}}),{secrets:[secret]});assert.equal(r.status,400);assert.equal(r.requestId,'req123');assert.equal(r.provider,'fixture');assert.match(r.message,/Invalid tool history/);assert.ok(!JSON.stringify(r).includes(secret));assert.ok(!JSON.stringify(r).includes(encodeURIComponent(secret)));
+ let canceled=false;const over=await upstreamError(new Response(new ReadableStream({start(c){c.enqueue(new TextEncoder().encode('x'.repeat(17000)));},cancel(){canceled=true;}}),{status:400}));assert.equal(canceled,true);assert.equal(over.bodyExcerpt,'[Oversized upstream error omitted]');
+ const html=await upstreamError(new Response('<html>bad request</html>',{status:400}));assert.match(html.bodyExcerpt,/bad request/);
+});
+test('gateway records upstream rejection without replay or credential leak',async()=>{let calls=0;const g=await startGateway({key:'reflected-secret',fetchImpl:async()=>{calls++;return Response.json({error:{code:400,message:'Invalid input reflected-secret'}},{status:400});}});try{for(let i=0;i<2;i++){const r=await fetch(g.url+'/v1/chat/completions',{method:'POST',body:JSON.stringify({model:MODEL,messages:[]})});await r.text();assert.equal(r.status,400);}assert.equal(calls,1);assert.match(g.receipts.find(r=>r.type==='http-failure').message,/Invalid input/);assert.ok(!JSON.stringify(g.receipts).includes('reflected-secret'));assert.equal(g.budget.reserved,.15);}finally{await g.close();}});
+test('offline reconstruction matches original failed multi-turn wire payload, detects orphan results',async(t)=>{const run='/home/baissi/benchmarks/deepseek-jev-v3/voxel-plain';if(!existsSync(run)){t.skip('Historical session not installed');return;}const {payload,report}=await reconstruct(run);assert.equal(report.hashMatches,true);assert.equal(report.toolCalls,11);assert.equal(report.toolResults,11);assert.deepEqual(report.issues,[]);payload.messages.push({role:'tool',tool_call_id:'missing',content:'x'});assert.ok(inspectRequest(payload).issues.some(s=>s.includes('unmatched')));});

package/tools/browser-check.mjs

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+/**
+ * browser-check — smoke + DOM + screenshot checks for delivered web pages.
+ *
+ * Declared in delivery_plan checks argv, e.g.:
+ *   ["node","tools/browser-check.mjs","--page","index.html",
+ *    "--assert","#keyboard button","--assert-count","#keyboard button:26",
+ *    "--click","#key-a","--then-text","#word:_A",
+ *    "--console-clean","--screenshot","artifacts/shot.png"]
+ *
+ * Serves the workspace over a local HTTP server (avoids file:// CORS),
+ * runs jsdom assertions with real inline-script execution, and optionally
+ * captures a real screenshot into artifacts/ using the bundled headless
+ * shell (auto-downloaded), a system Chrome/Chromium, or Firefox as a last
+ * resort.
+ *
+ * Exit 0 = all assertions pass (screenshot failures are warnings unless
+ * --require-screenshot is set). Prints a JSON summary to stdout.
+ */
+import { resolve, isAbsolute } from 'node:path';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { serveDir, domCheck, browserScreenshot } from '../lib/browser.mjs';
+import { findBrowser, ensureBundledBrowser } from '../lib/browser-bin.mjs';
+
+const args = process.argv.slice(2);
+const opt = (name, def) => {
+  const i = args.indexOf(name);
+  return i === -1 ? def : args[i + 1];
+};
+const optAll = name => {
+  const out = [];
+  for (let i = 0; i < args.length; i++) if (args[i] === name && args[i + 1]) out.push(args[i + 1]);
+  return out;
+};
+const has = name => args.includes(name);
+
+const page = opt('--page', 'index.html');
+const root = resolve(opt('--root', '.'));
+const assertions = [];
+
+for (const s of optAll('--assert')) assertions.push({ selector: s });
+for (const s of optAll('--assert-count')) {
+  const m = s.match(/^(.+):(\d+)$/);
+  if (!m) { console.error(`bad --assert-count: ${s} (want "selector:N")`); process.exit(2); }
+  assertions.push({ selector: m[1], count: +m[2] });
+}
+for (const s of optAll('--assert-text')) {
+  const m = s.match(/^([^:]+):(.+)$/s);
+  if (!m) { console.error(`bad --assert-text: ${s} (want "selector:text")`); process.exit(2); }
+  assertions.push({ selector: m[1], text: m[2] });
+}
+// --click '#a' --then-text '#word:_A' — click then assert text on another element
+const clicks = optAll('--click');
+const thenTexts = optAll('--then-text');
+if (clicks.length) {
+  clicks.forEach((sel, i) => {
+    const t = thenTexts[i];
+    const m = t && t.match(/^([^:]+):(.+)$/s);
+    assertions.push(m
+      ? { action: 'click', selector: sel, then: { selector: m[1], text: m[2] } }
+      : { action: 'click', selector: sel, then: { selector: sel } });
+  });
+}
+for (const s of optAll('--eval')) assertions.push({ eval: s });
+if (has('--console-clean')) assertions.push({ console: 'error-free' });
+
+const srv = await serveDir(root);
+const url = `${srv.url}/${page}`;
+let summary = { url, http: null, dom: null, screenshot: null };
+
+try {
+  const res = await fetch(url);
+  const body = await res.text();
+  summary.http = { status: res.status, bytes: body.length, contentType: res.headers.get('content-type') };
+  if (res.status !== 200) {
+    console.log(JSON.stringify({ ...summary, pass: false, error: `HTTP ${res.status}` }, null, 2));
+    process.exit(1);
+  }
+
+  if (assertions.length) {
+    const r = await domCheck(body, assertions, { url });
+    summary.dom = { pass: r.pass, failures: r.failures, consoleErrors: r.consoleErrors.slice(0, 5) };
+  }
+
+  const shot = opt('--screenshot');
+  if (shot) {
+    const abs = isAbsolute(shot) ? shot : resolve(root, shot);
+    mkdirSync(dirname(abs), { recursive: true });
+    const wait = opt('--wait');
+    const win = (opt('--window', '1280x800').split('x').map(Number));
+    const browser = findBrowser() ?? await ensureBundledBrowser().catch(() => null);
+    summary.screenshot = await browserScreenshot(url, abs, {
+      width: win[0] || 1280, height: win[1] || 800,
+      waitSeconds: wait != null ? +wait : undefined,
+      browser,
+    });
+    if (!summary.screenshot.ok && has('--require-screenshot')) {
+      console.log(JSON.stringify({ ...summary, pass: false }, null, 2));
+      process.exit(1);
+    }
+  }
+
+  const pass = (summary.dom ? summary.dom.pass : true);
+  console.log(JSON.stringify({ ...summary, pass }, null, 2));
+  process.exit(pass ? 0 : 1);
+} finally {
+  srv.close();
+}

package/tools/check-syntax.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { readdirSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const root = resolve(import.meta.dirname, '..');
+const paths = ['bin', 'lib', 'tools', 'evaluate.mjs', 'server.mjs'];
+const files = [];
+
+function collect(path) {
+  if (path.endsWith('.mjs')) {
+    files.push(path);
+    return;
+  }
+  for (const entry of readdirSync(path, { withFileTypes: true })) {
+    const child = join(path, entry.name);
+    if (entry.isDirectory()) collect(child);
+    else if (entry.name.endsWith('.mjs')) files.push(child);
+  }
+}
+
+for (const path of paths) collect(join(root, path));
+for (const file of files.sort()) {
+  const result = spawnSync(process.execPath, ['--check', file], { stdio: 'inherit' });
+  if (result.status !== 0) process.exit(result.status || 1);
+}
+console.log(`Checked ${files.length} JavaScript files.`);

package/tools/evaluate-plan-advisor.mjs

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+import { evaluateAdvisor } from '../lib/plan-advisor.mjs';
+import { createJevAdvisor } from '../lib/jev-advisor.mjs';
+
+// Separate explicit evaluation entry point; never invoked by the delivery gate.
+const args = process.argv.slice(2);
+if (!args[0] || args.some((arg, i) => i > 0 && arg !== '--jev')) {
+  console.error('Usage: node tools/evaluate-plan-advisor.mjs corpus.json [--jev]');
+  process.exitCode = 2;
+} else {
+  try {
+    const cases = JSON.parse(await readFile(args[0], 'utf8'));
+    const advisor = args.includes('--jev') ? createJevAdvisor({ apiKey: process.env.TYPESAFE_API_KEY }) : undefined;
+    const baseline = await evaluateAdvisor(cases);
+    const shadow = advisor ? await evaluateAdvisor(cases, { advisor }) : null;
+    console.log(JSON.stringify({ version: 1, baseline, shadow, liveProviderEvaluated: Boolean(advisor), note: 'Explicit --jev transmits design data to TypeSafe. Cost remains unknown without injected pricing. Classifiers never authorize code.' }, null, 2));
+    if (shadow?.unavailable) process.exitCode = 1;
+  } catch (error) { console.error(error.message); process.exitCode = 1; }
+}