agent-doc-system 1.4.0 → 1.5.0

package/evals/agent/results/smoke-gpt-6.1-sol.json

@@ -0,0 +1,81 @@
+{
+  "schemaVersion": "agentdoc.eval.smoke/v1",
+  "purpose": "Real-model functional smoke; not a performance adoption experiment",
+  "model": "gpt-6.1-sol",
+  "modelVersion": "configured-default/codex-cli-0.167.2",
+  "sourceState": "Development worktree during implementation; no immutable release claim",
+  "task": "money-1",
+  "repeats": 1,
+  "budget": {
+    "timeoutMs": 90000,
+    "maxTokens": 300000,
+    "maxToolCalls": 20,
+    "maxCostUsd": null
+  },
+  "conditions": {
+    "baseline": "No preloaded AgentDoc context; identical filesystem task and constraints",
+    "reference": "Current CLI compile followed by query service/src/logic.mjs",
+    "candidate": "Current CLI prepare --request request.json"
+  },
+  "attempts": [
+    {
+      "condition": "baseline",
+      "status": "failed",
+      "usage": {
+        "inputTokens": 157019,
+        "outputTokens": 1003,
+        "cachedInputTokens": 142208,
+        "toolCalls": 6,
+        "costUsd": null
+      },
+      "durationMs": 54914.645124,
+      "fixtureHash": "9d718cd543878efd8f8670eb12c5036f0446024e3185e2d9082836caa7066800",
+      "validation": {
+        "passed": false,
+        "failure": "Behavior assertion: per-unit decimal rounding returned 200 instead of 202 cents."
+      }
+    },
+    {
+      "condition": "reference",
+      "status": "passed",
+      "usage": {
+        "inputTokens": 131257,
+        "outputTokens": 937,
+        "cachedInputTokens": 116608,
+        "toolCalls": 4,
+        "costUsd": null
+      },
+      "durationMs": 54258.42732899999,
+      "fixtureHash": "9d718cd543878efd8f8670eb12c5036f0446024e3185e2d9082836caa7066800",
+      "validation": {
+        "passed": true
+      }
+    },
+    {
+      "condition": "candidate",
+      "status": "passed",
+      "usage": {
+        "inputTokens": 109042,
+        "outputTokens": 978,
+        "cachedInputTokens": 93568,
+        "toolCalls": 3,
+        "costUsd": null
+      },
+      "durationMs": 44541.01249600001,
+      "fixtureHash": "9d718cd543878efd8f8670eb12c5036f0446024e3185e2d9082836caa7066800",
+      "validation": {
+        "passed": true
+      }
+    }
+  ],
+  "limitations": [
+    "One synthetic development task, one repetition, one configured model alias.",
+    "Provider dollar cost is unavailable; no cost-reduction or adoption claim is supported.",
+    "Token usage includes cached input and local Codex harness instructions; it is reported at turn boundaries, not enforced before each provider call.",
+    "Comparison ran against a changing development worktree; rerun a pinned campaign for release performance claims."
+  ],
+  "decision": {
+    "adopt": false,
+    "reason": "Functional smoke only; larger independent held-out campaign required."
+  }
+}

package/evals/agent/run.mjs

@@ -0,0 +1,163 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {spawn, execFileSync} from 'node:child_process';
+import {pathToFileURL, fileURLToPath} from 'node:url';
+import {createHash,randomUUID} from 'node:crypto';
+import {performance} from 'node:perf_hooks';
+import {tasks, publicTask} from './tasks/corpus.mjs';
+
+const RUNNER_HASH=createHash('sha256').update(fs.readFileSync(fileURLToPath(import.meta.url))).digest('hex');
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+export function runProcess(command, {cwd, input='', timeoutMs=120000, maxOutputBytes=1048576, onEvent}={}) {
+  if (!Array.isArray(command) || !command.length || command.some(x=>typeof x!=='string')) throw new Error('command must be a nonempty argv array');
+  return new Promise(resolve => {
+    let stdout='',stderr='',line='',bytes=0,reason=null,done=false;
+    const child=spawn(command[0],command.slice(1),{cwd,detached:process.platform!=='win32',stdio:['pipe','pipe','pipe']});
+    const kill = why => { reason ||= why; try { process.kill(process.platform==='win32'?child.pid:-child.pid,'SIGKILL'); } catch {} };
+    const timer=setTimeout(()=>kill('timeout'),timeoutMs);
+    const finish=(code,error)=>{if(done)return;done=true;clearTimeout(timer);if(child.pid&&process.platform!=='win32')try{process.kill(-child.pid,'SIGKILL');}catch{}resolve({code,stdout,stderr,reason,error});};
+    child.on('error',e=>finish(null,e.message));
+    child.stdout.on('data',chunk=>{
+      bytes+=chunk.length;if(bytes>maxOutputBytes){kill('output-limit');return;}
+      const s=chunk.toString();stdout+=s;line+=s;
+      let i;while((i=line.indexOf('\n'))>=0){const item=line.slice(0,i);line=line.slice(i+1);if(onEvent&&item.trim())try{onEvent(JSON.parse(item),kill);}catch{kill('invalid-driver-event');}}
+    });
+    child.stderr.on('data',chunk=>{bytes+=chunk.length;if(bytes>maxOutputBytes)kill('output-limit');else stderr+=chunk.toString();});
+    child.on('close',code=>{if(line.trim()&&onEvent&&!reason)try{onEvent(JSON.parse(line),kill);}catch{reason='invalid-driver-event';}finish(code);});
+    child.stdin.on('error',()=>{});child.stdin.end(input);
+  });
+}
+
+export function materialize(task, workspace) {
+  const hash=createHash('sha256').update(JSON.stringify(Object.entries(task.files).sort())).digest('hex');
+  if(hash!==task.fixtureHash)throw new Error('fixture pin mismatch: '+task.id);
+  for(const [name,content] of Object.entries(task.files)){
+    const dest=path.resolve(workspace,name);if(!dest.startsWith(workspace+path.sep))throw new Error('fixture path escape');
+    fs.mkdirSync(path.dirname(dest),{recursive:true});fs.writeFileSync(dest,content);
+  }
+  execFileSync('git',['init','-q','-b','main'],{cwd:workspace});
+  execFileSync('git',['add','.'],{cwd:workspace});
+  execFileSync('git',['-c','user.name=AgentDoc Eval','-c','user.email=eval@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','pinned fixture '+task.fixtureHash],{cwd:workspace});
+}
+
+export async function validate(task, workspace) {
+  // The validator is never installed in the agent's repository or sent in its prompt.
+  const completed='agentdoc-validator-complete:'+randomUUID();
+  const script=`import assert from 'node:assert/strict';
+const {run}=await import(${JSON.stringify(pathToFileURL(path.join(workspace,'service/src/consumer.mjs')).href)});
+for(const [input,expected] of ${JSON.stringify(task.cases)}){
+ const original=JSON.stringify(input);let result,thrown=false;try{result=await run(input);}catch{thrown=true;}
+ if(expected?.throws){assert.equal(thrown,true);continue;}
+ assert.equal(thrown,false);assert.deepEqual(result,expected);assert.equal(JSON.stringify(input),original,'input mutated');
+ if(${JSON.stringify(task.family==='merge-config')}){const refs=new Set();const visit=x=>{if(x&&typeof x==='object'){refs.add(x);for(const v of Object.values(x))visit(v);}};visit(input);const check=x=>{if(x&&typeof x==='object'){assert.equal(refs.has(x),false,'output aliases input');for(const v of Object.values(x))check(v);}};check(result);}
+}
+process.stdout.write(${JSON.stringify(completed)});
+`;
+  const r=await runProcess([process.execPath,'--input-type=module','-e',script],{cwd:workspace,timeoutMs:10000,maxOutputBytes:32768});
+  return {passed:r.code===0&&!r.reason&&!r.error&&r.stdout===completed,diagnostic:r.stderr.slice(-4000),reason:r.reason||r.error||null};
+}
+
+function checkUsage(usage,budget){
+  if(!usage||!['inputTokens','outputTokens','toolCalls'].every(k=>Number.isFinite(usage[k])&&usage[k]>=0))throw new Error('missing or invalid usage');
+  if(usage.costUsd!==undefined&&usage.costUsd!==null&&(!Number.isFinite(usage.costUsd)||usage.costUsd<0))throw new Error('invalid cost usage');
+  if(usage.inputTokens+usage.outputTokens>budget.maxTokens)return 'token-limit';
+  if(usage.toolCalls>budget.maxToolCalls)return 'tool-limit';
+  if(budget.maxCostUsd!==null){if(!Number.isFinite(usage.costUsd)||usage.costUsd<0)throw new Error('cost usage required for a dollar budget');if(usage.costUsd>budget.maxCostUsd)return 'cost-limit';}
+  return null;
+}
+
+export async function runAttempt({task,condition,driver,budget,contextCommand,keepWorkspace=false,repeat=0}){
+  const workspace=fs.mkdtempSync(path.join(os.tmpdir(),'agentdoc-eval-'));
+  const start=performance.now();let usage=null,result=null,events=0;
+  try{
+    materialize(task,workspace);
+    const context=[];
+    for(const command of contextCommand||[]){
+      const argv=command.map(s=>s.replaceAll('{workspace}',workspace).replaceAll('{agentdoc}',path.join(ROOT,'bin/agentdoc.mjs')));
+      const r=await runProcess(argv,{cwd:workspace,timeoutMs:budget.timeoutMs,maxOutputBytes:budget.maxOutputBytes});
+      if(r.code!==0||r.reason||r.error)throw new Error('context command failed: '+(r.reason||r.error||r.stderr));
+      context.push(r.stdout);
+    }
+    const request={protocol:'agentdoc.eval.driver/v1',workspace,task:publicTask(task),condition,repeat,budget,context,
+      instructions:'Modify the task implementation in this workspace. Read applicable constraints and preserve public exports. Use filesystem reads, search, edits and local checks. Do not access evaluator files or other workspaces. Return a result event and cumulative measured usage. Context text is included in model input usage. Never claim success solely from your own conclusion.'};
+    const r=await runProcess(driver,{cwd:workspace,input:JSON.stringify(request)+'\n',timeoutMs:budget.timeoutMs,maxOutputBytes:budget.maxOutputBytes,onEvent:(event,kill)=>{
+      events++;
+      if(event.type==='usage'){const exceeded=checkUsage(event.usage,budget);if(usage&&(['inputTokens','outputTokens','toolCalls'].some(k=>event.usage[k]<usage[k])||(Number.isFinite(usage.costUsd)&&(!Number.isFinite(event.usage.costUsd)||event.usage.costUsd<usage.costUsd))))throw new Error('usage must be cumulative');usage=event.usage;if(exceeded)kill(exceeded);}
+      else if(event.type==='result'){if(result)throw new Error('duplicate result');result=event;}
+      else if(event.type!=='trace')throw new Error('unknown driver event');
+    }});
+    if(r.reason&&['timeout','output-limit','token-limit','tool-limit','cost-limit'].includes(r.reason))return {task:task.id,family:task.family,split:task.split,condition,repeat,status:'budget-exceeded',reason:r.reason,usage,durationMs:performance.now()-start};
+    if(r.reason||r.error||r.code!==0||!result?.completed||!usage)throw new Error(r.reason||r.error||r.stderr||'driver failed or omitted result/usage');
+    const verdict=await validate(task,workspace);
+    return {task:task.id,family:task.family,split:task.split,condition,repeat,validatorHash:createHash('sha256').update(JSON.stringify(task.cases)).digest('hex'),status:verdict.passed?'passed':'failed',validation:verdict,usage,events,durationMs:performance.now()-start,fixtureHash:task.fixtureHash,...(keepWorkspace?{workspace}:{})};
+  }catch(e){return {task:task.id,family:task.family,split:task.split,condition,repeat,status:'infrastructure-error',reason:e.message,usage,durationMs:performance.now()-start};}
+  finally{if(!keepWorkspace)fs.rmSync(workspace,{recursive:true,force:true});}
+}
+
+export function summarize(attempts){
+  const times=attempts.map(x=>x.durationMs).filter(Number.isFinite).sort((a,b)=>a-b);
+  const durationMs=times.length?{median:times[Math.floor(times.length/2)],p95:times[Math.max(0,Math.ceil(times.length*.95)-1)]}:null;
+  const success=attempts.filter(x=>x.status==='passed').length;
+  const known=attempts.every(x=>Number.isFinite(x.usage?.costUsd));
+  const cost=known?attempts.reduce((n,x)=>n+x.usage.costUsd,0):null;
+  return {durationMs,attempts:attempts.length,successes:success,successRate:attempts.length?success/attempts.length:null,infrastructureErrors:attempts.filter(x=>x.status==='infrastructure-error').length,budgetExceeded:attempts.filter(x=>x.status==='budget-exceeded').length,totalCostUsd:cost,costPerSuccess:success&&cost!==null?cost/success:null,totalTokens:attempts.every(x=>x.usage)?attempts.reduce((n,x)=>n+x.usage.inputTokens+x.usage.outputTokens,0):null};
+}
+
+export function selectCandidate(reference,candidate,{minCostReduction=.2,nonInferiorityMargin=.02}={}){
+  const key=x=>x.task+'#'+x.repeat;
+  const ref=new Map(reference.map(x=>[key(x),x]));
+  if(!reference.length||ref.size!==reference.length||reference.length!==candidate.length||new Set(candidate.map(key)).size!==candidate.length||candidate.some(x=>!ref.has(key(x))))return {adopt:false,reason:'unpaired-data'};
+  if([...reference,...candidate].some(x=>x.status==='infrastructure-error'))return {adopt:false,reason:'infrastructure-errors'};
+  const groups=new Map();
+  for(const c of candidate){const r=ref.get(key(c));if(c.fixtureHash!==r.fixtureHash||c.validatorHash!==r.validatorHash)return {adopt:false,reason:'fixture-mismatch'};const k=c.family||c.task;if(!groups.has(k))groups.set(k,[]);groups.get(k).push((c.status==='passed'?1:0)-(r.status==='passed'?1:0));}
+  const differences=[...groups.values()].map(values=>values.reduce((a,b)=>a+b,0)/values.length);
+  const mean=differences.reduce((a,b)=>a+b,0)/differences.length;
+  // Family-level bounded-variable interval; variants/repeats are not independent
+  // samples. A tiny pilot cannot manufacture zero uncertainty from identical outcomes.
+  const halfWidth=Math.sqrt(2*Math.log(2/(1-.95))/differences.length);
+  const lower=mean-halfWidth;
+  const a=summarize(reference),b=summarize(candidate);
+  if([...reference,...candidate].some(x=>x.split!=='holdout'))return {adopt:false,reason:'holdout-required',lower};
+  if(candidate.some(c=>c.criticalError||c.validation?.criticalError))return {adopt:false,reason:'critical-error',lower};
+  if(lower<=-nonInferiorityMargin)return {adopt:false,reason:'insufficient-noninferiority-evidence',lower};
+  if(a.costPerSuccess===null||b.costPerSuccess===null||a.costPerSuccess<=0)return {adopt:false,reason:'cost-unmeasured',lower};
+  const reduction=1-b.costPerSuccess/a.costPerSuccess;
+  return {adopt:reduction>=minCostReduction,reason:reduction>=minCostReduction?'measured-improvement':'cost-target-missed',lower,costReduction:reduction};
+}
+
+export async function campaign(config){
+  if(!config.driver)throw new Error('A model driver command is required; no model campaign was run.');
+  if(!config.model||!config.modelVersion)throw new Error('Pin model and modelVersion in campaign configuration.');
+  if(!Number.isInteger(config.repeats)||config.repeats<1)throw new Error('repeats must be a positive integer');
+  const budget={timeoutMs:120000,maxTokens:16000,maxToolCalls:40,maxCostUsd:null,maxOutputBytes:1048576,...config.budget};
+  for(const k of ['timeoutMs','maxTokens','maxToolCalls','maxOutputBytes'])if(!Number.isFinite(budget[k])||budget[k]<=0)throw new Error('invalid budget '+k);
+  if(budget.maxCostUsd!==null&&(!Number.isFinite(budget.maxCostUsd)||budget.maxCostUsd<=0))throw new Error('invalid dollar budget');
+  const selected=tasks.filter(t=>(!config.split||t.split===config.split)&&(!config.tasks||config.tasks.includes(t.id)));
+  if(!selected.length)throw new Error('no tasks selected');
+  const conditions=config.conditions||{baseline:{contextCommands:[]}};
+  const attempts=[];let totalCost=0;
+  for(const task of selected)for(let repeat=0;repeat<config.repeats;repeat++)for(const [condition,settings] of Object.entries(conditions)){
+    if(config.maxCampaignCostUsd!==undefined){
+      if(budget.maxCostUsd===null)throw new Error('campaign dollar cap requires a per-attempt dollar cap');
+      if(totalCost+budget.maxCostUsd>config.maxCampaignCostUsd)return {schemaVersion:'agentdoc.eval.campaign/v1',config,attempts,stopped:'campaign-cost-limit',summary:summarize(attempts)};
+    }
+    const attempt=await runAttempt({task,condition,repeat,driver:config.driver,budget,contextCommand:settings.contextCommands,keepWorkspace:config.keepWorkspaces});attempts.push(attempt);
+    if(config.maxCampaignCostUsd!==undefined&&attempt.usage?.costUsd===undefined)return {schemaVersion:'agentdoc.eval.campaign/v1',config,attempts,stopped:'cost-unmeasured',summary:summarize(attempts)};
+    totalCost+=attempt.usage?.costUsd||0;
+  }
+  return {schemaVersion:'agentdoc.eval.campaign/v1',runtime:{node:process.version,runnerHash:RUNNER_HASH,corpusHash:createHash('sha256').update(JSON.stringify(tasks.map(t=>({id:t.id,pin:t.fixtureHash,cases:t.cases})))).digest('hex'),driverFiles:Object.fromEntries(config.driver.filter(p=>path.isAbsolute(p)&&fs.existsSync(p)&&fs.statSync(p).isFile()).map(p=>[p,createHash('sha256').update(fs.readFileSync(p)).digest('hex')]))},config,attempts,summary:summarize(attempts),conditions:Object.fromEntries(Object.keys(conditions).map(k=>[k,summarize(attempts.filter(x=>x.condition===k))]))};
+}
+
+if(process.argv[1]&&path.resolve(process.argv[1])===fileURLToPath(import.meta.url)){
+  try{
+    if(process.argv.includes('--list'))process.stdout.write(JSON.stringify(tasks.map(t=>({...publicTask(t),family:t.family,split:t.split})),null,2)+'\n');
+    else{
+      const args=process.argv.slice(2),configIndex=args.indexOf('--config');if(configIndex<0||!args[configIndex+1])throw new Error('usage: node evals/agent/run.mjs --config campaign.json [--out report.json] | --list');
+      const report=await campaign(JSON.parse(fs.readFileSync(args[configIndex+1],'utf8')));const out=args.indexOf('--out');const text=JSON.stringify(report,null,2)+'\n';
+      if(out>=0)fs.writeFileSync(args[out+1],text);else process.stdout.write(text);
+      if(report.attempts.some(x=>x.status==='infrastructure-error')||report.stopped)process.exitCode=1;
+    }
+  }catch(e){console.error(e.message);process.exitCode=2;}
+}

package/evals/agent/tasks/corpus.mjs

@@ -0,0 +1,53 @@
+// Bootstrap coding tasks. Validation cases stay in the evaluator, outside the agent workspace.
+import fs from 'node:fs';
+const pins=JSON.parse(fs.readFileSync(new URL('./pins.json',import.meta.url),'utf8'));
+
+const families = [
+  { id: 'money', split: 'development', prompt: 'Order totals drift by one cent. Return the sum of line items as integer cents, rounding each unit price to cents before multiplying its integer quantity.', rules: 'Prices are decimal currency units; quantities are nonnegative integers. Round each unit price first, then sum. Empty orders total zero. Do not mutate input.', code: 'return rows.reduce((sum, row) => sum + row.price * row.quantity, 0);', cases: [[[],0],[[{price:1.005,quantity:2}],202],[[{price:0.1,quantity:3},{price:0.2,quantity:2}],70]] },
+  { id: 'pagination', split: 'development', prompt: 'The listing API skips its first page. Implement one-based pagination with the configured maximum page size.', rules: 'Input is {items,page,size}. Clamp page to at least one, size to [1,MAX], truncate fractional page and size. Return a new array. Missing page=1 and size=10.', code: 'return input.items.slice(input.page * input.size, (input.page + 1) * input.size);', cases: [[{items:[0,1,2,3,4],page:1,size:2},[0,1]],[{items:[0,1,2],page:0,size:2},[0,1]],[{items:[0,1,2],page:2,size:2},[2]],[{items:[0,1,2]},[0,1,2]]] },
+  { id: 'deduplicate', split: 'development', prompt: 'Webhook redelivery creates duplicate records. Keep the first record for each id while preserving order.', rules: 'Input is an array of records with string id. Identity is exact and case sensitive. Preserve first record and order. Do not mutate input. Empty input gives [].', code: 'return rows.filter((row, i) => rows.findLastIndex(x => x.id === row.id) === i);', cases: [[[],[]],[[{id:'a',v:1},{id:'b',v:2},{id:'a',v:3}],[{id:'a',v:1},{id:'b',v:2}]],[[{id:'A'},{id:'a'}],[{id:'A'},{id:'a'}]]] },
+  { id: 'event-routing', split: 'development', prompt: 'Event routing matches unrelated subject prefixes. Implement the documented segment-aware subject pattern matcher.', rules: 'Input is {pattern,subject}. Dot separates segments; * matches exactly one nonempty segment; > matches one or more trailing segments and is valid only at the end. Other segments match exactly. Empty segments are invalid.', code: 'return input.subject.startsWith(input.pattern.replace(/[*>]/g, ""));', cases: [[{pattern:'orders.*.v1',subject:'orders.created.v1'},true],[{pattern:'orders.*',subject:'orders.created.v1'},false],[{pattern:'orders.>',subject:'orders.created.v1'},true],[{pattern:'orders.>',subject:'orders'},false],[{pattern:'a.*',subject:'a.'},false],[{pattern:'a',subject:'ab'},false]] },
+  { id: 'dependency-order', split: 'development', prompt: 'Deployment ordering sometimes starts dependents first. Return all task names in deterministic dependency-first order, and reject a dependency cycle.', rules: 'Input maps each task to its dependencies. Include names appearing only as dependencies. Sort names and sibling dependencies lexicographically. Throw Error on cycles. Do not mutate input.', code: 'return Object.keys(input).sort();', cases: [[{api:['db'],db:[]},['db','api']],[{z:['a'],b:[],a:[]},['a','b','z']],[{a:['missing']},['missing','a']],[{a:['b'],b:['a']},{throws:true}]] },
+  { id: 'merge-config', split: 'development', prompt: 'Environment overrides overwrite unrelated nested settings. Implement recursive merging of plain JSON objects.', rules: 'Input is {base,override}. Recursively merge object/object pairs. Arrays and scalars replace values. null is a value. Do not mutate either input. Return independent nested structures.', code: 'return {...input.base, ...input.override};', cases: [[{base:{db:{host:'local',port:10}},override:{db:{port:20}}},{db:{host:'local',port:20}}],[{base:{a:[1,2]},override:{a:[3]}},{a:[3]}],[{base:{a:1},override:{a:null}},{a:null}],[{base:{a:{b:2}},override:{c:1}},{a:{b:2},c:1}]] },
+  { id: 'path-owner', split: 'holdout', prompt: 'The ownership resolver assigns paths to similarly named sibling directories. Resolve using complete path segments and the longest owning root.', rules: 'Input is {roots,path}. Roots and paths are canonical relative POSIX paths; . owns every path. A root owns itself and descendants at slash boundaries. Return the longest owner or null. Do not mutate roots.', code: 'return input.roots.find(root => input.path.startsWith(root)) ?? null;', cases: [[{roots:['apps/a','apps/ab'],path:'apps/ab/file'},'apps/ab'],[{roots:['.','apps'],path:'elsewhere/file'},'.'],[{roots:['apps','apps/a'],path:'apps/a/file'},'apps/a'],[{roots:['apps/a'],path:'apps/ab/file'},null]] },
+  { id: 'runtime-freshness', split: 'holdout', prompt: 'Runtime evidence is accepted beyond its validity window. Implement observation freshness against the supplied clock.', rules: 'Input is {capturedAt,now,maxAgeMs}. Return false for invalid timestamps, negative maxAgeMs, or observations in the future. The exact maximum age is still fresh. Do not consult the wall clock.', code: 'return Date.parse(input.now) - Date.parse(input.capturedAt) < input.maxAgeMs;', cases: [[{capturedAt:'2026-01-01T00:00:00Z',now:'2026-01-01T00:00:01Z',maxAgeMs:1000},true],[{capturedAt:'bad',now:'2026-01-01T00:00:00Z',maxAgeMs:1000},false],[{capturedAt:'2026-01-01T00:00:02Z',now:'2026-01-01T00:00:01Z',maxAgeMs:1000},false],[{capturedAt:'2026-01-01T00:00:00Z',now:'2026-01-01T00:00:02Z',maxAgeMs:1000},false]] },
+  { id: 'resource-impact', split: 'holdout', prompt: 'Migration impact reaches consumers of similarly named databases. Follow exact resource identities and reverse build dependencies only.', rules: 'Input is {resource,bindings,dependencies}. bindings maps component names to resource arrays. dependencies maps component names to build-dependency arrays. Start with components bound to the exact resource; include their transitive dependents. Return sorted unique names. Cycles must terminate.', code: 'return Object.keys(input.bindings).filter(c => input.bindings[c].some(r => r.includes(input.resource))).sort();', cases: [[{resource:'db',bindings:{a:['db'],b:['db2']},dependencies:{c:['a'],d:['c']}},['a','c','d']],[{resource:'none',bindings:{a:['db']},dependencies:{}},[]],[{resource:'db',bindings:{a:['db']},dependencies:{a:['b'],b:['a']}},['a','b']]] },
+  { id: 'check-scope', split: 'holdout', prompt: 'Check recommendations collapse equal commands from different components. Deduplicate only by check id and prioritize directly changed components.', rules: 'Input is {checks,changed}. Each check has id,component,command. Keep first record per id. Checks owned by changed components come first; within each group sort by id. Return records without mutation.', code: 'return input.checks.filter((x,i,a) => a.findIndex(y=>y.command===x.command)===i);', cases: [[{checks:[{id:'b',component:'x',command:'test'},{id:'a',component:'y',command:'test'}],changed:['y']},[{id:'a',component:'y',command:'test'},{id:'b',component:'x',command:'test'}]],[{checks:[{id:'a',component:'x',command:'test'},{id:'a',component:'x',command:'other'}],changed:[]},[{id:'a',component:'x',command:'test'}]]] },
+];
+
+// Different initial defects make each family variant a distinct repair task.
+const alternativeBugs = {
+  money: ['return Math.round(rows.reduce((n,r)=>n+r.price*r.quantity,0)*100);','return rows.reduce((n,r)=>n+Math.round((r.price+Number.EPSILON)*100),0);'],
+  pagination: ['return input.items.slice((input.page-1)*input.size,input.page*input.size);','return input.items.slice(0,input.size||10);'],
+  deduplicate: ['return [...new Map(rows.map(r=>[r.id,r])).values()].sort((a,b)=>a.id.localeCompare(b.id));','return rows.filter((r,i)=>rows.findIndex(x=>x.id.toLowerCase()===r.id.toLowerCase())===i);'],
+  'event-routing': ['return input.pattern.split(".").length===input.subject.split(".").length;','return input.pattern===input.subject;'],
+  'dependency-order': ['return Object.keys(input).reverse();','return [...new Set(Object.values(input).flat())].sort().concat(Object.keys(input).sort());'],
+  'merge-config': ['return Object.assign(input.base,input.override);','return JSON.parse(JSON.stringify(input.base));'],
+  'path-owner': ['return input.roots.find(r=>input.path===r||input.path.startsWith(r+"/"))??null;','return input.roots.sort((a,b)=>b.length-a.length).find(r=>input.path.startsWith(r))??null;'],
+  'runtime-freshness': ['return Math.abs(Date.parse(input.now)-Date.parse(input.capturedAt))<=input.maxAgeMs;','return Boolean(Date.parse(input.capturedAt));'],
+  'resource-impact': ['return Object.keys(input.bindings).filter(c=>input.bindings[c].includes(input.resource)).sort();','return [...new Set([...Object.keys(input.bindings),...Object.keys(input.dependencies)])].sort();'],
+  'check-scope': ['return [...new Map(input.checks.map(c=>[c.id,c])).values()].sort((a,b)=>a.id.localeCompare(b.id));','return input.checks.sort((a,b)=>a.id.localeCompare(b.id));'],
+};
+
+export const tasks = families.flatMap(f => [1,2,3].map(variant => {
+  const arg = ['money','deduplicate'].includes(f.id) ? 'rows' : 'input';
+  const files = {
+    'request.json': JSON.stringify({task:f.prompt,paths:['service/src/logic.mjs'],budget:{tokens:4096}})+'\n',
+    'package.json': JSON.stringify({name:`agent-eval-${f.id}-${variant}`,private:true,type:'module',scripts:{test:'node service/smoke.mjs'}})+'\n',
+    '.gitignore': '/.agentdoc/\n',
+    'service/package.json': JSON.stringify({name:`@eval/${f.id}`,type:'module',scripts:{test:'node smoke.mjs'}})+'\n',
+    'service/src/logic.mjs': `const MAX = ${10 * variant};\nexport function solve(${arg}) { ${variant===1?f.code:alternativeBugs[f.id][variant-2]} }\n`,
+    'service/src/consumer.mjs': "import {solve} from './logic.mjs';\nexport const run = input => solve(input);\n",
+    'service/smoke.mjs': "import {run} from './src/consumer.mjs';\nif(typeof run !== 'function') process.exit(1);\n",
+    'service/CONSTRAINTS.md': `# Constraints\n\n${f.rules}\n\nPublic export solve and consumer export run must remain callable.\n`,
+    'service/README.md': `# ${f.id}\nService implementation lives in src/logic.mjs. src/consumer.mjs is a downstream caller.\n`,
+    'agentdoc/agentdoc.config.yaml': 'apiVersion: agentdoc.dev/config/v1\nnamespace: default\ndiscovery:\n  componentDescriptorName: agentdoc.yaml\n  componentDescriptors: [service/agentdoc.yaml]\n  centralDescriptors: [agentdoc/domains.yaml, agentdoc/systems.yaml]\nadapters: [node]\nauthority:\n  rules: []\nreviewedOverrides: []\nwarningAcceptances: []\noutput:\n  graph: .agentdoc/graph.json\n  observations: agentdoc/observations\n',
+    'service/agentdoc.yaml': `apiVersion: agentdoc.dev/v1\nkind: Component\nmetadata:\n  name: service\n  description: "${f.id} implementation and consumer."\nspec:\n  type: service\n  system: evaluation\n  context:\n    constraints: [service/CONSTRAINTS.md]\n`,
+    'agentdoc/domains.yaml': 'apiVersion: agentdoc.dev/v1\nkind: Domain\nmetadata:\n  name: evaluation\n  description: "Agent evaluation domain."\nspec: {}\n',
+    'agentdoc/systems.yaml': 'apiVersion: agentdoc.dev/v1\nkind: System\nmetadata:\n  name: evaluation\n  description: "Task implementation."\nspec:\n  domain: evaluation\n',
+  };
+  const fixtureHash = pins[`${f.id}-${variant}`];
+  return {id:`${f.id}-${variant}`,family:f.id,split:f.split,prompt:f.prompt,variant,files,fixtureHash,cases:f.id==='pagination'?[...f.cases,[{items:Array.from({length:40},(_,i)=>i),page:1,size:999},Array.from({length:Math.min(40,10*variant)},(_,i)=>i)]]:f.cases};
+}));
+
+export function publicTask(task) { return {id:task.id,prompt:task.prompt,paths:['service/src/logic.mjs'],fixtureHash:task.fixtureHash}; }

package/evals/agent/tasks/pins.json

@@ -0,0 +1,32 @@
+{
+  "money-1": "9d718cd543878efd8f8670eb12c5036f0446024e3185e2d9082836caa7066800",
+  "money-2": "27df099ec23abb2d84da3518bf6fd54272a6d574603700995ee48de9ede59e08",
+  "money-3": "a3f75a9ede31a13e96216971f540a4da3476e944290459fd01fdfca75bf4c6ee",
+  "pagination-1": "61665bb1b7db4da915b53040ccfb213508d1fb57f73e6364d587268b3d1ce5e9",
+  "pagination-2": "b51b0bbe8d9d2323eb369d588b3ae417be23d6910721a3928490de092c5a8116",
+  "pagination-3": "3d36d3e21be24fba1403d0ad29ca65a4ef03f8e1fa181cc45c4da2856be1b771",
+  "deduplicate-1": "7cacd3fd3f998b24bbfb5b78edfc954afc1ad1f3377ef3169aa25802c0618b36",
+  "deduplicate-2": "407e5cd438a69013175c8e3c3e3d7876b9834bce5b851a9faab7f0f89c13d387",
+  "deduplicate-3": "b61522162c0bdf0421f0d40366549cdccd795fe2a859a7174e66733433a8672e",
+  "event-routing-1": "f05ab481ea3645c2574fd3aeda1899c41d5a5d9592dd8587ac6e9b3e1757258f",
+  "event-routing-2": "882978f4a235ef0ff23377f112d62494a078c66ec2ebabf44a7f5025155054e0",
+  "event-routing-3": "de1914683e9cca505a25366818df378da893502964570ca58e52a2f0968ae6e2",
+  "dependency-order-1": "a66fd9c360e3d776a74dda055c997c85c99dcf5d9d565898f60fc2b5f618d56f",
+  "dependency-order-2": "104950296c086e76a290144a6be59c3d1af40bbfc25b2bcc7e50ff861564ef06",
+  "dependency-order-3": "37cfdfeeb9fa432885fca06c6ce43757e1c8636103826f1d69ede4e3f1794d88",
+  "merge-config-1": "d6fe5f7189535cacd6314de4ecfcd370163777159882a03477139caefe721d3d",
+  "merge-config-2": "321a45f9eebfb4d9c61cd971e28bc7836c846527eddeeb55cafd88a7257e65a6",
+  "merge-config-3": "5198d65a622851f33f3a1d22d4761e50981d00a48065b0b41cde8d1f6067de19",
+  "path-owner-1": "80c37b741ab9b71286879b346cce1fb4489a6363f4e51531327bf1ec54362f8d",
+  "path-owner-2": "9ddc26c23ad2f88f279c4b5bf40c3f172181c546fb0f0b4195ae1b49a2eed16d",
+  "path-owner-3": "8494a1b255bec4b4a034f8750d4e8646fd5fbef701921b93f8910405df1b6758",
+  "runtime-freshness-1": "c334e9059a24325d44db9708eda97a33ffd399d74bb45d23361e118904558315",
+  "runtime-freshness-2": "66f77a9a59af172f23003c7abac1d93dff12e6269082fcdbc8f06920d59f76e4",
+  "runtime-freshness-3": "0680b26c02ae64187721bfeda8cc2ac3ce1920bd1119d94eea1af0e6fb4a3ad0",
+  "resource-impact-1": "276333e51df632b7b5f8587d3a89163ab7b09fe6870c4dad4a7faf863b84d9c5",
+  "resource-impact-2": "e2d28c85238930842db84dcadd359042f109c7be77dc897d5cfd8cfc38aa6c65",
+  "resource-impact-3": "44980f3221bc58e2f018613ae43d98415436a3cec475f9d50ac08cb88638fa7a",
+  "check-scope-1": "e317e861e62575909dae23d3e2eead3c50f9df8835a16b14fa6a690a6befaaa0",
+  "check-scope-2": "205c7078be6d5d8a9e746caaa99759f807e098f8462b01b741231eb125e61f85",
+  "check-scope-3": "22fb34beb19feb19e6cc07a42c12993d41ee0de47a83ae2d0b398193a31f80f3"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-doc-system",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "type": "module",
   "description": "Portable documentation and software-catalog system with an explicit authority model, for AI coding agents",
   "keywords": [
@@ -27,10 +27,13 @@
     "test": "node --test tests/*.test.mjs",
     "eval": "node evals/run-all.mjs",
     "selfcheck": "node bin/agentdoc.mjs doctor",
-    "compare": "python3 scripts/compare-koda.py"
+    "compare": "python3 scripts/compare-koda.py",
+    "eval:agent": "node evals/agent/run.mjs",
+    "bench:context": "node scripts/bench/context.mjs"
+  },
+  "dependencies": {
+    "js-tiktoken": "1.0.21"
   },
-  "dependencies": {},
-  "devDependencies": {},
   "files": [
     "adapters",
     "bin",
@@ -45,7 +48,8 @@
     "skills",
     "templates",
     "CHANGELOG.md",
-    "INSTALL_FOR_AGENTS.md"
+    "INSTALL_FOR_AGENTS.md",
+    "evals/agent"
   ],
   "license": "MIT"
 }

package/schemas/context.schema.json

@@ -0,0 +1,222 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "schemaVersion": {
+      "const": "agentdoc.dev/context/v1"
+    },
+    "packet": {
+      "type": "string"
+    },
+    "snapshot": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "inputHash": {
+          "type": "string"
+        },
+        "compiler": {
+          "type": "object"
+        },
+        "graphHash": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "inputHash",
+        "compiler",
+        "graphHash"
+      ]
+    },
+    "task": {
+      "type": "string"
+    },
+    "intent": {
+      "enum": [
+        "implement",
+        "change-contract",
+        "debug-runtime",
+        "review"
+      ]
+    },
+    "scope": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "coverage": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "unmatched": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "limitations": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "staleObservations": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "unmatched",
+        "limitations",
+        "staleObservations"
+      ]
+    },
+    "records": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "version": {
+            "type": "string"
+          },
+          "kind": {
+            "type": "string"
+          },
+          "subject": {
+            "type": "string"
+          },
+          "key": {
+            "type": "string"
+          },
+          "value": {},
+          "evidence": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "critical": {
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "id",
+          "version",
+          "kind",
+          "subject",
+          "key",
+          "value",
+          "evidence",
+          "critical"
+        ]
+      }
+    },
+    "delivery": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "offset": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "total": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "omitted": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "criticalOmitted": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "partial": {
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "offset",
+        "total",
+        "omitted",
+        "criticalOmitted",
+        "partial"
+      ]
+    },
+    "continuation": {
+      "anyOf": [
+        {
+          "type": "null"
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "packet": {
+              "type": "string"
+            },
+            "offset": {
+              "type": "integer",
+              "minimum": 0
+            }
+          },
+          "required": [
+            "packet",
+            "offset"
+          ]
+        }
+      ]
+    },
+    "budget": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "limit": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "used": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "encoding": {
+          "enum": [
+            "o200k_base",
+            "cl100k_base"
+          ]
+        },
+        "tokenizerVersion": {
+          "const": "js-tiktoken@1.0.21"
+        }
+      },
+      "required": [
+        "limit",
+        "used",
+        "encoding",
+        "tokenizerVersion"
+      ]
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "packet",
+    "snapshot",
+    "task",
+    "intent",
+    "scope",
+    "coverage",
+    "records",
+    "delivery",
+    "continuation",
+    "budget"
+  ],
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "agentdoc.dev/schema/context.schema.json"
+}

package/skills/agent-doc-system/SKILL.md

@@ -116,3 +116,9 @@ The normative specification (`docs/SPEC.md`, `docs/AUTHORITY.md`,
 `docs/PROVENANCE.md`, `docs/QUERY.md`, `docs/MIGRATION.md`, `docs/CI.md`) lives
 in the agent-doc-system repository — read it when a rule here needs the full
 contract behind it.
+
+## Task context protocol (1.5+)
+
+For an onboarded repository, use `agentdoc prepare` with a JSON request on stdin: `{"task":"Describe the intended change","paths":["relative/path"],"budget":{"tokens":4096,"encoding":"o200k_base"}}`. The response describes the current working tree, with no requirement to compile a committed graph first. Select the encoding your model uses; do not assume all models share one tokenizer.
+
+A partial packet is incomplete: follow its continuation through `agentdoc expand`, increasing the budget if a record does not fit. After edits, use `agentdoc refresh` with the previous packet reference and only record IDs still retained in active context. Apply retractions and changed versions. Reset after losing context. Stale observations and unknown execution directories require resolution before treating facts as current or running checks. The complete protocol is in `docs/CONTEXT.md`. Existing query/impact/check workflows remain available.

package/templates/ci.yml

@@ -17,7 +17,7 @@ jobs:
         with: { node-version: '24' }
       - name: Install the agentdoc CLI
         # Unpinned tracks main. For CI reproducibility pin a tag or commit:
-        #   github:lbframe/agent-doc-system#v1.4.0
+        #   github:lbframe/agent-doc-system#v1.5.0
         # A compiler upgrade intentionally invalidates committed graphs.
         run: npm install -g github:lbframe/agent-doc-system
       - name: Validate and compile