@getopencrawl/cli 0.1.0 → 0.1.1

package/src/research.mjs

@@ -1,5 +1,5 @@
 import fs from 'node:fs/promises';
-import {allowedOptions,decimal,identifier,withInterrupt} from './jobs.mjs';
+import {allowedOptions,decimal,identifier,settle,transient,withInterrupt} from './jobs.mjs';
 import {displayLine,displayText} from './display.mjs';
 import {coverageWarnings} from './context.mjs';
 import {validateEventEvidence,renderEventEvidence} from './research-event.mjs';
@@ -27,8 +27,11 @@ export const RESEARCH_HELP=`OpenCrawl · collect and search public research
   opencrawl research configure COLLECTION_ID --file settings.json --request-key RANDOM_KEY
   opencrawl research job JOB_ID [--wait] [--timeout 300]
   opencrawl research search "question" --collection COLLECTION_ID [--limit 10] [--budget 6000]
-  opencrawl research events COLLECTION_ID [--limit 10] [--cursor OPAQUE_CURSOR]
+  opencrawl research events COLLECTION_ID [--limit 10] [--cursor OPAQUE_CURSOR] [--content-only]
   opencrawl research event COLLECTION_ID EVENT_ID [--budget 16000] [--before-start N] [--after-start N]
+  opencrawl research feed [--customer NAME] [--after CURSOR] [--limit 50]   new change signals across the project
+  opencrawl research subscribe HTTPS_URL [--customer NAME]                 one signed webhook for every watch (secret shown once)
+  opencrawl research subscriptions | unsubscribe SUBSCRIPTION_ID
 
 Use --json for exact receipts. Create/refresh/configure never retry automatically: retain
 both the original JSON descriptor and a random 16–128 character request key.
@@ -55,7 +58,7 @@ export function researchKey(value){if(typeof value!=='string'||!(/^[A-Za-z0-9_-]
 function pageQuery(options,defaultLimit=20,max=100){const limit=decimal(options.limit,'limit',1,max,defaultLimit),query=new URLSearchParams({limit});if(options.cursor!==undefined){if(!text(options.cursor,512))throw new Error('--cursor must be the exact bounded cursor returned by the server.');query.set('cursor',options.cursor)}return {limit,query};}
 export function researchRequest(rest,options){
  const [command,id,...extra]=rest;
- const allowed={create:['file','request-key','wait','timeout'],configure:['file','request-key'],list:['limit','cursor'],show:[],refresh:['request-key','revision','wait','timeout'],pause:['revision'],resume:['revision'],job:['wait','timeout'],search:['collection','limit','budget'],events:['limit','cursor'],event:['budget','before-start','after-start']};
+ const allowed={create:['file','request-key','wait','timeout'],configure:['file','request-key'],list:['limit','cursor'],show:[],refresh:['request-key','revision','wait','timeout'],pause:['revision'],resume:['revision'],job:['wait','timeout'],search:['collection','limit','budget'],events:['limit','cursor','content-only'],event:['budget','before-start','after-start'],feed:['customer','after','limit'],subscribe:['customer'],subscriptions:[],unsubscribe:[]};
  if(!Object.hasOwn(allowed,command))throw new Error('Choose a research command from opencrawl research --help.');
  allowedOptions(options,allowed[command],'research');
  if(command==='event'?extra.length!==1:command!=='search'&&(extra.length||['create','list'].includes(command)&&id!==undefined))throw new Error('Unexpected research argument.');
@@ -63,6 +66,10 @@ export function researchRequest(rest,options){
  const timeout=decimal(options.timeout,'timeout',1,3600,300);
  if(command==='create'){if(!text(options.file,4096))throw new Error('Provide --file with the collection JSON descriptor.');return {command,route:ROOT+'/collections',method:'POST',file:options.file,key:researchKey(options['request-key']),wait:!!options.wait,timeout};}
  if(command==='list'){const {limit,query}=pageQuery(options);return {command,limit,route:ROOT+'/collections?'+query};}
+ if(command==='feed'){if(id!==undefined)throw new Error('Unexpected research argument.');const query=new URLSearchParams({after:decimal(options.after,'after',0,Number.MAX_SAFE_INTEGER,0),limit:decimal(options.limit,'limit',1,100,50)});if(options.customer!==undefined){if(!text(options.customer,120))throw new Error('--customer must be a customer name.');query.set('customer',options.customer);}return {command,route:ROOT+'/feed?'+query};}
+ if(command==='subscribe'){if(!text(id,2048)||!/^https:\/\//.test(id))throw new Error('Give the public HTTPS endpoint to deliver to.');if(options.customer!==undefined&&!text(options.customer,120))throw new Error('--customer must be a customer name.');return {command,route:ROOT+'/subscriptions',method:'POST',body:{endpoint:id,...(options.customer!==undefined?{customer:options.customer}:{})}};}
+ if(command==='subscriptions'){if(id!==undefined)throw new Error('Unexpected research argument.');return {command,route:ROOT+'/subscriptions'};}
+ if(command==='unsubscribe'){if(!/^rsub_[0-9a-f]{32}$/.test(String(id)))throw new Error('Give the subscription ID from research subscriptions.');return {command,route:ROOT+'/subscriptions/'+id,method:'DELETE'};}
  if(command==='search'){const query=rest.slice(1).join(' ').trim();if(!query||Array.from(query).length>1000)throw new Error('Provide a nonempty question of at most 1,000 characters.');const collection=identifier(options.collection,'collection');return {command,collection,route:ROOT+'/search',method:'POST',body:{query,collection_ids:[collection],freshness:'indexed',limit:decimal(options.limit,'limit',1,50,10),budget:decimal(options.budget,'budget',0,32000,6000)}};}
  identifier(id,command==='job'?'job':'collection');
  const route=ROOT+(command==='job'?'/jobs/':'/collections/')+encodeURIComponent(id);
@@ -74,7 +81,7 @@ export function researchRequest(rest,options){
   return {command,id,eventId,budget,beforeStart,afterStart,route:route+'/events/'+encodeURIComponent(eventId)+'?'+query};
  }
  if(command==='configure'){if(!text(options.file,4096))throw new Error('Provide --file with all four reviewed settings.');return {command,id,route:route+'/configure',method:'POST',file:options.file,key:researchKey(options['request-key'])};}
- if(command==='events'){const {limit,query}=pageQuery(options,10,25);return {command,id,limit,route:route+'/events?'+query};}
+ if(command==='events'){const {limit,query}=pageQuery(options,10,25);if(options['content-only'])query.set('include_volatile','false');return {command,id,limit,route:route+'/events?'+query};}
  if(command==='refresh')return {command,id,route:route+'/refresh',method:'POST',body:options.revision===undefined?{}:{expected_revision:decimal(options.revision,'revision',1,Number.MAX_SAFE_INTEGER)},key:researchKey(options['request-key']),wait:!!options.wait,timeout};
  if(command==='pause'||command==='resume')return {command,id,route,method:'PATCH',body:{status:command==='pause'?'paused':'active',expected_revision:decimal(options.revision,'revision',1,Number.MAX_SAFE_INTEGER)}};
  return {command,id,route,wait:!!options.wait,timeout};
@@ -84,7 +91,7 @@ export function validateResearchCollection(value,id=value?.id){if(!value||!text(
 function validatePage(value,limit){if(!value||!Array.isArray(value.items)||value.items.length>limit||!(value.next_cursor===null||text(value.next_cursor,512)))throw new Error('Server returned an invalid research page.');return value;}
 function validateEvents(value,request){validatePage(value,request.limit);for(const item of value.items){if(!item||!text(item.id)||item.collection_id!==request.id||!['baseline','changed','unavailable','recovered'].includes(item.kind)||typeof item.url!=='string'||typeof item.observed_at!=='string'||!item.change_summary||typeof item.gap!=='boolean')throw new Error('Server returned invalid or out-of-scope research observations.');for(const side of ['before','after'])if(item[side]!==null&&(!item[side]||typeof item[side].excerpt!=='string'||Array.from(item[side].excerpt).length>4000))throw new Error('Server returned an oversized observation excerpt.');}return value;}
 function validateSearch(value,request){const expected=request.body.query.split(/[\s\u0085\u001c-\u001f]+/u).filter(Boolean).join(' ');if(!value||value.query!==expected||!Array.isArray(value.results)||value.results.length>request.body.limit||typeof value.context!=='string'||Buffer.byteLength(value.context)>request.body.budget||!Array.isArray(value.evidence?.citations)||Boolean(value.context)!==Boolean(value.evidence.citations.length)||!value.freshness||!['indexed','empty'].includes(value.freshness.status)||!Array.isArray(value.freshness.refresh_jobs)||value.freshness.refresh_jobs.length)throw new Error('Server returned an invalid indexed research response; no context was emitted.');for(const result of value.results)if(!text(result.document_id)||!text(result.title,2000)||typeof result.source_id!=='string'||result.research_observation&&result.research_observation.collection_id!==request.collection)throw new Error('Server returned invalid or out-of-scope research citations.');for(const citation of value.evidence.citations)if(!value.results.some(result=>result.document_id===citation.document_id))throw new Error('Server returned a context citation outside the search results.');return value;}
-export function researchFailure(error,request){const status=Number.isInteger(error?.status)?error.status:undefined;const fixed={401:'Sign in with a valid key for the intended workspace.',403:'This account cannot perform the requested research operation.',404:'This collection, event or job is unavailable. Check the selected workspace and exact identifier.',409:'The collection or request changed, or work remains unfinished. Review its current revision and jobs; preserve the original request key and descriptor before explicitly retrying.',422:'The research selection or descriptor is invalid. Check the approved origins, limits and explicit identifiers.',429:'OpenCrawl is busy or a workspace limit was reached. Check the collection before retrying.'};const message=fixed[status]||'The request outcome could not be confirmed. Recover create/refresh/configure with the same request key and exact descriptor; never start a duplicate blindly.';return {code:status?`http_${status}`:'request_unconfirmed',message,...(status?{status}:{}),...(request.key?{request_key:request.key}:{}),...(request.id?{id:request.id}:{})};}
+export function researchFailure(error,request){const status=Number.isInteger(error?.status)?error.status:undefined;const fixed={401:'Sign in with a valid key for the intended workspace.',403:'This account cannot perform the requested research operation.',404:'This collection, event or job is unavailable. Check the selected workspace and exact identifier. Watches, jobs and research built by the OpenCrawl agent belong to their project: use that project\'s key (project.api_key from the build), not the organization key.',409:'The collection or request changed, or work remains unfinished. Review its current revision and jobs; preserve the original request key and descriptor before explicitly retrying.',422:'The research selection or descriptor is invalid. Check the approved origins, limits and explicit identifiers.',429:'OpenCrawl is busy or a workspace limit was reached. Check the collection before retrying.'};const message=fixed[status]||'The request outcome could not be confirmed. Recover create/refresh/configure with the same request key and exact descriptor; never start a duplicate blindly.';return {code:status?`http_${status}`:'request_unconfirmed',message,...(status?{status}:{}),...(request.key?{request_key:request.key}:{}),...(request.id?{id:request.id}:{})};}
 export function validateResearchSettings(value){
  const fields=['expected_revision','page_limit','schedule_minutes','include_media'];
  if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).length!==fields.length||fields.some(key=>!Object.hasOwn(value,key))||!finite(value.expected_revision,1)||!finite(value.page_limit,1,5000)||!(value.schedule_minutes===0||finite(value.schedule_minutes,5,10080))||typeof value.include_media!=='boolean')throw new Error('Settings JSON must contain exactly expected_revision (positive integer), page_limit (1–5000), schedule_minutes (0 or 5–10080), and include_media (boolean).');
@@ -106,12 +113,13 @@ export function validateConfigureReceipt(value,request){
  return value;
 }
 async function descriptor(file){const handle=await fs.open(file,'r');try{const size=(await handle.stat()).size;if(!finite(size,1,256*1024))throw new Error('Collection JSON must be between 1 byte and 256 KiB.');const bytes=Buffer.alloc(size+1);const {bytesRead}=await handle.read(bytes,0,size+1,0);if(bytesRead!==size)throw new Error('The collection descriptor changed while being read.');let value;try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes.subarray(0,size)))}catch{throw new Error('Invalid collection JSON. Supply a valid UTF-8 collection descriptor.');}const allowed=['name','brief','seed_urls','allowed_origins','sitemap_urls','page_limit','schedule_minutes','include_media'];if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).some(key=>!allowed.includes(key))||!text(value.name,200)||!Array.isArray(value.seed_urls)||!value.seed_urls.length||value.seed_urls.length>50)throw new Error('Collection JSON must contain name and 1–50 seed_urls, with only documented collection settings.');return value;}finally{await handle.close()}}
-export async function waitResearch(client,initial,{timeout=300,signal,now=()=>performance.now(),sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms))}={}){let job=validateResearchJob(initial);const deadline=now()+timeout*1000;while(ACTIVE.has(job.status)){if(signal?.aborted)return {job,wait:{outcome:'interrupted'}};if(now()>=deadline)return {job,wait:{outcome:'timeout'}};try{job=validateResearchJob(await client.request(ROOT+'/jobs/'+encodeURIComponent(job.id),{timeout:Math.max(1,Math.min(30000,Math.ceil(deadline-now()))),signal,maxResponseBytes:1024*1024}),job.id)}catch(error){return {job,wait:{outcome:signal?.aborted?'interrupted':now()>=deadline?'timeout':'unavailable',error:researchFailure(error,{id:job.id})}}}if(!ACTIVE.has(job.status))break;const next=Math.min(deadline,now()+2000);while(!signal?.aborted&&now()<next)await sleep(Math.min(100,next-now()));}return {job,wait:{outcome:job.status}};}
+export async function waitResearch(client,initial,{timeout=300,signal,now=()=>performance.now(),sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms))}={}){let job=validateResearchJob(initial),failures=0;const deadline=now()+timeout*1000;while(ACTIVE.has(job.status)){if(signal?.aborted)return {job,wait:{outcome:'interrupted'}};if(now()>=deadline)return {job,wait:{outcome:'timeout'}};try{job=validateResearchJob(await client.request(ROOT+'/jobs/'+encodeURIComponent(job.id),{timeout:Math.max(1,Math.min(30000,Math.ceil(deadline-now()))),signal,maxResponseBytes:1024*1024}),job.id);failures=0}catch(error){if(!signal?.aborted&&now()<deadline&&transient(error)){await settle(++failures,{signal,now,sleep,deadline});continue;}return {job,wait:{outcome:signal?.aborted?'interrupted':now()>=deadline?'timeout':'unavailable',error:researchFailure(error,{id:job.id})}}}if(!ACTIVE.has(job.status))break;const next=Math.min(deadline,now()+2000);while(!signal?.aborted&&now()<next)await sleep(Math.min(100,next-now()));}return {job,wait:{outcome:job.status}};}
 const renderJob=value=>`${displayLine(value.id)}\t${displayLine(value.status)}\t${value.progress.indexed} indexed, ${value.progress.failed} unsuccessful, ${value.progress.discovered} discovered\n`;
 export async function researchCommand(rest,options,client,{stdout=value=>process.stdout.write(value),stderr=value=>process.stderr.write(value)}={}){
  if(['agent','agent-run'].includes(rest[0]))return agentCommand(rest,options,client,{stdout,stderr});
  const request=researchRequest(rest,options);if(request.file)request.body=await (request.command==='configure'?settingsDescriptor(request.file):descriptor(request.file));
  let value;try{value=await client.request(request.route,{method:request.method,body:request.body,idempotencyKey:request.key,maxResponseBytes:4*1024*1024});}catch(error){const failure=researchFailure(error,request);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+(request.key?`\nOriginal request key: ${request.key}`:'')+'\n');return 1;}
+ if(['feed','subscribe','subscriptions','unsubscribe'].includes(request.command)){stdout(JSON.stringify(value,null,2)+'\n');if(request.command==='subscribe')stderr('Save secret now: it is shown only once. Deliveries are signed (Webhook-Signature, verifyResearchWebhook in @getopencrawl/sdk).\n');if(request.command==='feed'&&value.more)stderr('More signals: continue with --after '+value.next_cursor+'\n');return 0;}
  if(request.command==='event'){validateEventEvidence(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else renderEventEvidence(value,stdout,stderr);return 0;}
  if(request.command==='configure'){
   try{validateConfigureReceipt(value,request)}catch(error){const failure=researchFailure(error,request);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+'\nOriginal request key: '+request.key+'\n');return 1;}
@@ -120,7 +128,7 @@ export async function researchCommand(rest,options,client,{stdout=value=>process
   return 0;
  }
  if(request.command==='list'){validatePage(value,request.limit);value.items.forEach(item=>validateResearchCollection(item));if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(value.items.map(item=>`${displayLine(item.id)}\tr${item.revision}\t${item.stats.indexed} indexed\t${displayLine(item.name)}\n`).join('')||'No research collections on this page.\n');if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');}return 0;}
- if(request.command==='events'){validateEvents(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{for(const event of value.items){stdout(`${displayLine(event.id)}\t${displayLine(event.kind)}\t${displayLine(event.observed_at)}\t${displayLine(event.url)}\n`);for(const side of ['before','after'])if(event[side]){stdout(`${side==='before'?'Previous observation':'This observation'}: ${displayLine(event[side].title)}\n${displayText(event[side].excerpt)}\n`);if(event[side].excerpt_truncated)stderr(`${side==='before'?'Previous':'Current'} observation excerpt is shortened.\n`);}if(event.gap)stderr('Unsuccessful checks separate these observations; the exact change time is unknown.\n');if(event.change_summary.truncated)stderr('Observation excerpts or differences are shortened.\n');}if(!value.items.length)stdout('No retained observations on this page.\n');if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');stderr('Baselines are not new business events. Unavailable checks do not prove removal. Times are observations, not authored modifications.\n');}return 0;}
+ if(request.command==='events'){validateEvents(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{for(const event of value.items){stdout(`${displayLine(event.id)}\t${displayLine(event.kind)}\t${displayLine(event.observed_at)}\t${displayLine(event.url)}\n`);for(const side of ['before','after'])if(event[side]){stdout(`${side==='before'?'Previous observation':'This observation'}: ${displayLine(event[side].title)}\n${displayText(event[side].excerpt)}\n`);if(event[side].excerpt_truncated)stderr(`${side==='before'?'Previous':'Current'} observation excerpt is shortened.\n`);}const summary=event.change_summary;if(Array.isArray(summary.changes)){if(summary.substance==='volatile_only')stdout('Only dates, clock times, relative ages or whitespace changed.\n');for(const hunk of summary.changes.slice(0,20)){if(!hunk||!Array.isArray(hunk.removed)||!Array.isArray(hunk.added))continue;for(const line of hunk.removed.slice(0,20))stdout('- '+displayLine(String(line))+'\n');for(const line of hunk.added.slice(0,20))stdout('+ '+displayLine(String(line))+'\n');}if(summary.changes_truncated)stderr('Changed lines are shortened.\n');}if(event.gap)stderr('Unsuccessful checks separate these observations; the exact change time is unknown.\n');if(event.change_summary.truncated)stderr('Observation excerpts or differences are shortened.\n');}if(!value.items.length)stdout('No retained observations on this page.\n');if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');stderr('Baselines are not new business events. Unavailable checks do not prove removal. Times are observations, not authored modifications.\n');}return 0;}
  if(request.command==='search'){validateSearch(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(value.context);stderr(`${value.evidence.citations.length} context citations from ${value.results.length} matching results in the explicitly selected collection.\n`);if(!value.results.length)stderr('No matching indexed evidence; no broader search or refresh was attempted.\n');for(const warning of coverageWarnings(value))stderr(displayLine(warning)+'\n');}return 0;}
  if(['show','pause','resume'].includes(request.command)){validateResearchCollection(value,request.id);stdout(options.json?JSON.stringify(value,null,2)+'\n':`${displayLine(value.name)}\nCollection: ${displayLine(value.id)}\nRevision: ${value.revision}\nStatus: ${value.status}\nIndexed pages: ${value.stats.indexed}\nApproved origins: ${value.allowed_origins.map(displayLine).join(', ')}\n`);return 0;}
  let collection,job;

package/src/research.test.mjs

@@ -0,0 +1,97 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import {spawn} from 'node:child_process';
+import {fileURLToPath} from 'node:url';
+import {researchRequest,researchCommand,waitResearch,validateResearchJob,validateResearchSettings,validateConfigureReceipt} from './research.mjs';
+import {loggedInConfig} from './client.mjs';
+const collection={id:'rcol_'+ 'a'.repeat(32),source_id:'src_'+'a'.repeat(32),name:'Synthetic research',brief:'Public product pages',status:'active',can_manage:true,seed_urls:['https://example.com/'],allowed_origins:['https://example.com'],sitemap_urls:[],page_limit:100,schedule_minutes:0,include_media:false,revision:1,created_at:'2026-10-23T00:00:00Z',updated_at:'2026-10-23T00:00:00Z',last_checked_at:null,stats:{indexed:1,discovered:2,failed:1,bytes:123}};
+const job={id:'job_'+'b'.repeat(32),collection_id:collection.id,status:'partial',phase:'finished',created_at:'2026-10-23T00:00:00Z',started_at:null,finished_at:null,progress:{discovered:2,fetched:2,indexed:1,failed:1,bytes:123},coverage:{site_complete:false},error:null};
+const search={query:'product',results:[{document_id:'doc_test',title:'Product evidence',source_id:collection.source_id}],context:'[S1] product evidence\né🌲\n',evidence:{citations:[{document_id:'doc_test'}]},freshness:{status:'indexed',refresh_jobs:[]}};
+const key='research-test-key-0001';
+async function execute(rest,options,response){let stdout='',stderr='';const calls=[];const code=await researchCommand(rest,options,{request:async(route,params)=>{calls.push({route,params});if(response instanceof Error)throw response;return response}},{stdout:value=>stdout+=value,stderr:value=>stderr+=value});return {stdout,stderr,code,calls};}
+test('research commands require exact scope, revision and stable mutation key',()=>{assert.deepEqual(researchRequest(['search','release','notes'],{collection:collection.id}).body,{query:'release notes',collection_ids:[collection.id],freshness:'indexed',limit:10,budget:6000});assert.deepEqual(researchRequest(['pause',collection.id],{revision:'3'}).body,{status:'paused',expected_revision:3});assert.equal(researchRequest(['refresh',collection.id],{'request-key':key}).key,key);for(const [rest,options] of [[['search','x'],{}],[['refresh',collection.id],{}],[['pause',collection.id],{}],[['show','../secret'],{}],[['list','extra'],{}],[['events',collection.id],{limit:'100'}],[['search','x'],{collection:collection.id,freshness:'refresh'}],[['job',job.id],{timeout:'1'}]])assert.throws(()=>researchRequest(rest,options));});
+test('opaque event cursor is encoded as data on the selected collection route',()=>{const value=researchRequest(['events',collection.id],{cursor:'opaque&url=https://foreign.example'});const url=new URL(value.route,'https://example.com');assert.equal(url.searchParams.get('cursor'),'opaque&url=https://foreign.example');assert.equal(url.origin,'https://example.com');});
+test('query output is exact evidence and empty responses never broaden or refresh',async()=>{const result=await execute(['search','product'],{collection:collection.id},search);assert.equal(result.stdout,search.context);assert.equal(result.calls.length,1);assert.deepEqual(result.calls[0].params.body.collection_ids,[collection.id]);const empty=await execute(['search','none'],{collection:collection.id},{...search,query:'none',results:[],context:'',evidence:{citations:[]},freshness:{status:'empty',refresh_jobs:[]}});assert.equal(empty.stdout,'');assert.match(empty.stderr,/No matching indexed evidence/);});
+test('terminal partial job is distinct from success and remains recoverable',async()=>{const result=await execute(['job',job.id],{json:true},job);assert.equal(result.code,2);assert.equal(JSON.parse(result.stdout).job.status,'partial');assert.throws(()=>validateResearchJob({...job,status:'done'}));});
+test('bounded waits stop locally and never replay a mutation',async()=>{let time=0,calls=0;const client={request:async()=>{calls++;return {...job,status:'running',phase:'fetching'}}};const result=await waitResearch(client,{...job,status:'queued',phase:'discovery'},{timeout:1,now:()=>time,sleep:async ms=>{time+=ms}});assert.equal(result.wait.outcome,'timeout');assert.equal(result.job.id,job.id);assert.equal(calls,1);const controller=new AbortController();controller.abort();const stopped=await waitResearch(client,{...job,status:'queued'},{signal:controller.signal});assert.equal(stopped.wait.outcome,'interrupted');assert.equal(calls,1);});
+test('unknown refresh outcome keeps key with fixed safe errors and no retry',async()=>{for(const error of [new Error('RAW SECRET'),Object.assign(new Error('RAW SECRET'),{status:409}),Object.assign(new Error('RAW SECRET'),{status:401})]){const result=await execute(['refresh',collection.id],{'request-key':key,json:true},error);assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(result.calls[0].params.idempotencyKey,key);assert.equal(JSON.parse(result.stdout).error.request_key,key);assert.ok(!result.stdout.includes('SECRET'));}});
+test('observation display preserves uncertainty and escapes terminal controls',async()=>{const event={id:'event',collection_id:collection.id,url:'https://example.com/\u001b[2J',kind:'changed',observed_at:'2026-10-23',previous_observed_at:'2026-10-22',before:{title:'Old',excerpt:'Old\u001btext'},after:{title:'New',excerpt:'New'},gap:true,change_summary:{added:[],removed:[],truncated:true}};const result=await execute(['events',collection.id],{}, {items:[event],next_cursor:null});assert.ok(!result.stdout.includes('\u001b'));assert.match(result.stdout,/\\u001b/);assert.match(result.stdout,/^event\tchanged/);assert.match(result.stderr,/exact change time is unknown/);assert.match(result.stderr,/not new business events/);await assert.rejects(execute(['events',collection.id],{}, {items:[{...event,collection_id:'foreign'}],next_cursor:null}));});
+const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
+function run(args,env,cwd){return new Promise((resolve,reject)=>{const child=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>child.kill('SIGKILL'),10000);child.stdout.on('data',value=>stdout+=value);child.stderr.on('data',value=>stderr+=value);child.on('error',reject);child.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr})})});}
+test('actual CLI keeps account-origin binding and keyed recovery headers',async()=>{const folder=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-research-'));const calls=[];const server=http.createServer(async(req,res)=>{let raw='';for await(const part of req)raw+=part;const body=raw?JSON.parse(raw):null;calls.push({url:req.url,authorization:req.headers.authorization,key:req.headers['idempotency-key'],body,method:req.method});res.setHeader('Content-Type','application/json');res.end(JSON.stringify(req.url==='/api/research/collections'?{collection,job,replayed:true}:req.url.startsWith('/api/research/collections?')?{items:[collection],next_cursor:null}:req.url.endsWith('/refresh')?{job,replayed:true}:req.url==='/api/research/search'?search:collection));});await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const url='http://127.0.0.1:'+server.address().port;const env={...process.env};for(const name of Object.keys(env))if(name.startsWith('OPENCRAWL_'))delete env[name];env.OPENCRAWL_CONFIG_DIR=folder;const config=loggedInConfig({},url,'synthetic-pinned-key',{user:{id:'usr_'+'c'.repeat(32)},workspace:{id:'ws_'+'d'.repeat(32)}});await fs.writeFile(path.join(folder,'config.json'),JSON.stringify(config),{mode:0o600});const input=path.join(folder,'collection.json');await fs.writeFile(input,JSON.stringify({name:'Synthetic research',seed_urls:['https://example.com/'],schedule_minutes:0}));try{const help=await run(['research','--help'],env,folder);assert.equal(help.code,0);assert.equal(calls.length,0);const created=await run(['research','create','--file',input,'--request-key',key,'--json','--account',config.activeProfile],env,folder);assert.equal(created.code,0,created.stderr);assert.equal(JSON.parse(created.stdout).job.id,job.id);assert.equal(calls.at(-1).key,key);assert.equal(calls.at(-1).authorization,'Bearer synthetic-pinned-key');const before=calls.length;for(const extra of [['--request-key',key,'--request-key','different-key-000'],['--json=false'],['--source','other']]){const value=await run(['research','refresh',collection.id,...extra,'--account',config.activeProfile],env,folder);assert.equal(value.code,1);}assert.equal(calls.length,before);const wrong=await run(['research','list','--account',config.activeProfile,'--server','https://foreign.example'],env,folder);assert.equal(wrong.code,1);assert.equal(calls.length,before);const paused=await run(['research','pause',collection.id,'--revision','1','--json'],env,folder);assert.equal(paused.code,0,paused.stderr);assert.equal(calls.at(-1).method,'PATCH');assert.deepEqual(calls.at(-1).body,{status:'paused',expected_revision:1});assert.equal(calls.at(-1).key,undefined);}finally{server.closeAllConnections();await new Promise(resolve=>server.close(resolve));await fs.rm(folder,{recursive:true,force:true});}});
+
+test('search context enforces exact normalized query, UTF8 budget and real context citations',async()=>{for(const response of [{...search,query:'another question'},{...search,context:'é'.repeat(4000)},{...search,evidence:{citations:[]}},{...search,evidence:{citations:[{document_id:'foreign'}]}},{...search,results:[{...search.results[0],research_observation:{collection_id:'foreign'}}]}]){let stdout='';await assert.rejects(researchCommand(['search','product'],{collection:collection.id},{request:async()=>response},{stdout:v=>stdout+=v,stderr:()=>{}}));assert.equal(stdout,'');}const result=await execute(['search','product'],{collection:collection.id,budget:'0'},{...search,context:'',evidence:{citations:[]}});assert.equal(result.stdout,'');assert.match(result.stderr,/0 context citations from 1 matching results/);});
+test('invalid JSON descriptor never echoes private file snippets',async()=>{const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-research-invalid-')),file=path.join(dir,'input.json');try{await fs.writeFile(file,'{ PRIVATE_EXAMPLE_SECRET');await assert.rejects(researchCommand(['create'],{file,'request-key':key},{request:async()=>{throw new Error('must not send')} }),error=>error.message==='Invalid collection JSON. Supply a valid UTF-8 collection descriptor.');}finally{await fs.rm(dir,{recursive:true,force:true});}});
+
+const settings={expected_revision:1,page_limit:20,schedule_minutes:60,include_media:false};
+const configured={collection:{...collection,...settings,revision:2},next_scheduled_at:'2026-10-24T02:00:00.123456Z',replayed:false};
+delete configured.collection.expected_revision;
+async function withSettings(value,fn){const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-settings-'));const file=path.join(dir,'settings.json');try{await fs.writeFile(file,typeof value==='string'?value:JSON.stringify(value));return await fn(file);}finally{await fs.rm(dir,{recursive:true,force:true});}}
+test('configure requires four strict reviewed settings and disallows job-following flags',()=>{
+ assert.deepEqual(validateResearchSettings(settings),settings);
+ for(const field of Object.keys(settings)){const value={...settings};delete value[field];assert.throws(()=>validateResearchSettings(value));}
+ for(const change of [{page_limit:0},{page_limit:5001},{page_limit:true},{page_limit:'20'},{expected_revision:0},{expected_revision:1.5},{expected_revision:true},{schedule_minutes:1},{schedule_minutes:10081},{schedule_minutes:'60'},{include_media:'false'},{seed_urls:[]},{status:'active'},{name:'Changed'}])assert.throws(()=>validateResearchSettings({...settings,...change}));
+ for(const cadence of [0,5,10080])assert.equal(validateResearchSettings({...settings,schedule_minutes:cadence}).schedule_minutes,cadence);
+ for(const option of [{wait:true},{timeout:'5'},{revision:'1'},{source:'foreign'}])assert.throws(()=>researchRequest(['configure',collection.id],{file:'settings.json','request-key':key,...option}));
+ assert.throws(()=>researchRequest(['configure',collection.id],{file:'settings.json'}));
+ const request=researchRequest(['configure',collection.id],{file:'settings.json','request-key':key});
+ assert.equal(request.method,'POST');assert.equal(request.route,`/api/research/collections/${collection.id}/configure`);
+});
+test('configure file validation finishes before any network request and does not echo content',async()=>{
+ for(const value of [{...settings,include_media:'true'},{...settings,allowed_origins:['https://foreign.example']},'{ PRIVATE_SECRET',Buffer.from([0xff]).toString()])await withSettings(value,async file=>{
+  let count=0;await assert.rejects(researchCommand(['configure',collection.id],{file,'request-key':key,json:true},{request:async()=>{count++;return configured}}),error=>!error.message.includes('PRIVATE_SECRET'));assert.equal(count,0);
+ });
+});
+test('configure validates changed and no-op scheduling receipts without inventing a job',()=>{
+ const request={id:collection.id,body:settings};assert.equal(validateConfigureReceipt(configured,request),configured);
+ assert.equal(validateConfigureReceipt({...configured,replayed:true},request).replayed,true);
+ const noop={...configured,collection:{...configured.collection,revision:1},next_scheduled_at:null};assert.equal(validateConfigureReceipt(noop,request).next_scheduled_at,null);
+ const paused={...configured,collection:{...configured.collection,status:'paused'},next_scheduled_at:null};assert.equal(validateConfigureReceipt(paused,request).collection.status,'paused');
+ assert.equal(validateConfigureReceipt({...configured,collection:{...configured.collection,schedule_minutes:0},next_scheduled_at:null},{id:collection.id,body:{...settings,schedule_minutes:0}}).next_scheduled_at,null);
+ for(const value of [
+  {...configured,collection:{...configured.collection,id:'rcol_foreign'}},
+  {...configured,collection:{...configured.collection,page_limit:21}},
+  {...configured,collection:{...configured.collection,include_media:true}},
+  {...configured,collection:{...configured.collection,revision:3}},
+  {...configured,collection:{...configured.collection,seed_urls:[]}},
+  {...configured,collection:{...configured.collection,status:'paused'}},
+  {...configured,next_scheduled_at:null},{...configured,next_scheduled_at:'2026-10-24T02:00:00+00:00'},
+  {...configured,next_scheduled_at:'2026-02-30T02:00:00Z'}, {...configured,replayed:'true'},
+  {...configured,job}, {...configured,next_scheduled_at:undefined},
+ ])assert.throws(()=>validateConfigureReceipt(value,request));
+});
+test('configure sends one keyed POST and preserves unknown or invalid acceptance without retry',async()=>{
+ await withSettings(settings,async file=>{
+  const result=await execute(['configure',collection.id],{file,'request-key':key,json:true},configured);
+  assert.equal(result.code,0);assert.deepEqual(JSON.parse(result.stdout),configured);assert.equal(result.calls.length,1);
+  assert.deepEqual(result.calls[0].params.body,settings);assert.equal(result.calls[0].params.idempotencyKey,key);
+  for(const response of [new Error('RAW SECRET'),Object.assign(new Error('RAW SECRET'),{status:409}),Object.assign(new Error('RAW SECRET'),{status:401}),{...configured,job}]){
+   const failed=await execute(['configure',collection.id],{file,'request-key':key,json:true},response);
+   assert.equal(failed.code,1);assert.equal(failed.calls.length,1);const error=JSON.parse(failed.stdout).error;
+   assert.equal(error.request_key,key);assert.equal(error.id,collection.id);assert.ok(!failed.stdout.includes('SECRET'));
+  }
+  const human=await execute(['configure',collection.id],{file,'request-key':key},{...configured,replayed:true});
+  assert.equal(human.code,0);assert.match(human.stderr,/No immediate crawl/);assert.match(human.stderr,/Recovered the original receipt/);assert.match(human.stdout,/Schedule: 60 minutes/);
+ });
+});
+test('installed CLI configure preserves pinned credentials and exact settings on its sole request',async()=>{
+ const folder=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-configure-cli-')),calls=[];
+ const server=http.createServer(async(req,res)=>{let raw='';for await(const chunk of req)raw+=chunk;calls.push({url:req.url,method:req.method,key:req.headers['idempotency-key'],authorization:req.headers.authorization,body:JSON.parse(raw)});res.setHeader('content-type','application/json');res.end(JSON.stringify(configured));});
+ await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const origin='http://127.0.0.1:'+server.address().port;
+ const env={...process.env};for(const name of Object.keys(env))if(name.startsWith('OPENCRAWL_'))delete env[name];env.OPENCRAWL_CONFIG_DIR=folder;
+ const config=loggedInConfig({},origin,'synthetic-configure-key',{user:{id:'usr_'+'c'.repeat(32)},workspace:{id:'ws_'+'d'.repeat(32)}});
+ await fs.writeFile(path.join(folder,'config.json'),JSON.stringify(config),{mode:0o600});const file=path.join(folder,'settings.json');await fs.writeFile(file,JSON.stringify(settings));
+ try{
+  const args=['research','configure',collection.id,'--file',file,'--request-key',key,'--json','--account',config.activeProfile];
+  const result=await run(args,env,folder);assert.equal(result.code,0,result.stderr);assert.deepEqual(JSON.parse(result.stdout),configured);
+  assert.deepEqual(calls,[{url:`/api/research/collections/${collection.id}/configure`,method:'POST',key,authorization:'Bearer synthetic-configure-key',body:settings}]);
+  const mismatch=await run([...args,'--server','https://foreign.example'],env,folder);assert.equal(mismatch.code,1);assert.equal(calls.length,1);
+  await fs.writeFile(file,JSON.stringify({...settings,include_media:1}));const invalid=await run(args,env,folder);assert.equal(invalid.code,1);assert.equal(calls.length,1);
+  assert.deepEqual(JSON.parse(await fs.readFile(path.join(folder,'config.json'),'utf8')),config);
+ }finally{server.closeAllConnections();await new Promise(resolve=>server.close(resolve));await fs.rm(folder,{recursive:true,force:true});}
+});
+test('events can omit volatile-only changes and print changed lines as escaped data',async()=>{const value=researchRequest(['events',collection.id],{'content-only':true});assert.equal(new URL(value.route,'https://example.com').searchParams.get('include_volatile'),'false');assert.equal(new URL(researchRequest(['events',collection.id],{}).route,'https://example.com').searchParams.get('include_volatile'),null);const event={id:'event',collection_id:collection.id,url:'https://example.com/pricing',kind:'changed',observed_at:'2026-10-23',previous_observed_at:'2026-10-22',before:null,after:null,gap:false,change_summary:{added:[],removed:[],truncated:false,changes:[{removed:['Pro $49'],added:['Pro $59\u001b[2J'],volatile:false}],changes_truncated:true,substance:'content'}};const result=await execute(['events',collection.id],{}, {items:[event],next_cursor:null});assert.match(result.stdout,/^- Pro \$49$/m);assert.match(result.stdout,/^\+ Pro \$59\\u001b\[2J$/m);assert.match(result.stderr,/Changed lines are shortened/);const clock=await execute(['events',collection.id],{}, {items:[{...event,change_summary:{...event.change_summary,changes:[],changes_truncated:false,substance:'volatile_only'}}],next_cursor:null});assert.match(clock.stdout,/Only dates, clock times, relative ages or whitespace changed/);});

package/src/sessions.test.mjs

@@ -0,0 +1,151 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { sessionRequest, selections, sessionsCommand } from './sessions.mjs';
+import { loggedInConfig, createClient } from './client.mjs';
+
+const revision='a'.repeat(64);
+const options=()=>({revision,branch:'selected',selection:['t0:0:100'],budget:'256'});
+const receipt=(context='é'.repeat(128))=>({version:revision,context,context_bytes:Buffer.byteLength(context),budget:256,budget_unit:'utf8_bytes',selected:[{ref:'t0',start_char:0,end_char:100}],included:[{ref:'t0',start_char:0,end_char:Math.min(100,Array.from(context).length),role:'user',branch:'selected',excerpt:Array.from(context).slice(0,100).join(''),truncated:Array.from(context).length<100,citation:'C1'}],omitted:[]});
+
+test('export requires explicit revision branch selections and byte budget',()=>{
+  for (const key of ['revision','branch','selection','budget']) { const value=options();delete value[key];assert.throws(()=>sessionRequest(['export','doc_fixture'],value)); }
+  const request=sessionRequest(['export','doc_fixture'],options());
+  assert.deepEqual(request.options.body,{version:revision,branch:'selected',selections:[{ref:'t0',start_char:0,end_char:100}],budget:256});
+  assert.equal(request.route,'/api/v1/sessions/doc_fixture/context');assert.equal(request.options.method,'POST');
+  assert.equal(request.route.includes(revision),false);
+});
+
+test('malformed, duplicate and overlapping ranges never reach the server',()=>{
+  for (const value of [['t0:10:10'],['t0:10:9'],['t0:-1:10'],['t0:1e2:200'],['t0:0x10:20'],['t01:0:10'],['t0:0:01'],['t0:0:NaN'],['p0:0:5'],['t100000:0:5'],['t0:0:20971521'],['t0:0:99999999999999999'],['t0:0:10','t0:0:10'],['t0:0:10','t1:9:20'],Array(21).fill('t0:0:1')]) assert.throws(()=>selections(value));
+  assert.deepEqual(selections(['t0:0:10','t1:10:20']),[{ref:'t0',start_char:0,end_char:10},{ref:'t1',start_char:10,end_char:20}]);
+  for (const value of ['255','32001','1e3','+256',' 256','0x100']) assert.throws(()=>sessionRequest(['export','doc_fixture'],{...options(),budget:value}));
+});
+
+test('list and read pagination stay explicit and scoped',()=>{
+  assert.equal(new URL(sessionRequest(['list'],{provider:'auto'}).route,'https://example.test').searchParams.has('provider'),false);
+  assert.throws(()=>sessionRequest(['export','doc_fixture'],{...options(),revision:'not-a-revision'}));
+  assert.throws(()=>sessionRequest(['read','doc_fixture'],{branch:'all',cursor:'x'.repeat(257)}));
+  assert.match(sessionRequest(['list'],{provider:'claude',limit:'20',offset:'40'}).route,/limit=20&offset=40&provider=claude$/);
+  const request=sessionRequest(['read','doc_fixture'],{branch:'all',cursor:'opaque_value-123',limit:'2'});
+  const url=new URL(request.route,'https://example.test');assert.equal(url.searchParams.get('cursor'),'opaque_value-123');assert.equal(url.searchParams.get('branch'),'all');
+  for (const [rest,values] of [[['read','doc_fixture'],{branch:'all',offset:'0'}],[['read','doc_fixture'],{branch:'active'}],[['read','../secret'],{branch:'all'}],[['list','extra'],{}],[['export','doc_fixture'],{...options(),from:'0'}]]) assert.throws(()=>sessionRequest(rest,values));
+});
+
+test('context stdout is exact UTF8 bytes while JSON preserves full receipts',async()=>{
+  for (const json of [false,true]) {
+    const output=[],errors=[],data=receipt();
+    await sessionsCommand(['export','doc_fixture'],{...options(),...(json?{json:true}:{})},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
+    const text=output.join('');assert.deepEqual(errors,[]);
+    if (json) assert.deepEqual(JSON.parse(text),data);
+    else { assert.equal(Buffer.byteLength(text),256);assert.equal(text,data.context);assert.equal(text.endsWith('\n'),false); }
+  }
+});
+
+test('oversized wrong-revision or changed-selection responses fail before stdout',async()=>{
+  for (const change of [{context:'é'.repeat(129),context_bytes:258},{context_bytes:1},{version:'different'},{selected:[]},{selected:[{ref:'t1',start_char:0,end_char:100}]},{budget_unit:'tokens'}]) {
+    const output=[];await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>({...receipt(),...change})},{stdout:value=>output.push(value),stderr:()=>{}}));assert.deepEqual(output,[]);
+  }
+});
+
+test('canonical server ordering is accepted without changing selected ranges',async()=>{
+  const selected=[{ref:'t1',start_char:100,end_char:200},{ref:'t0',start_char:0,end_char:100}];
+  const output=[];await sessionsCommand(['export','doc_fixture'],{...options(),selection:['t1:100:200','t0:0:100']},{request:async()=>({...receipt(),selected:selected.toReversed(),omitted:[{ref:'t1',start_char:100,end_char:200,reason:'Budget'}]})},{stdout:value=>output.push(value),stderr:()=>{}});
+  assert.equal(output.join(''),receipt().context);
+});
+
+test('partial context reports exact omission and truncation on stderr only',async()=>{
+  const data={...receipt('p'.repeat(40)),omitted:[{ref:'t1',start_char:100,end_char:200,reason:'budget'}]};
+  data.selected.push({ref:'t1',start_char:100,end_char:200});const output=[],errors=[];
+  await sessionsCommand(['export','doc_fixture'],{...options(),selection:['t0:0:100','t1:100:200']},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
+  assert.equal(output.join(''),'p'.repeat(40));assert.match(errors.join(''),/Omitted t1:100:200/);assert.match(errors.join(''),/through character 40/);
+});
+
+test('permission and revision conflicts are actionable without automatic retry',async()=>{
+  for (const [status,pattern] of [[401,/Sign in/],[403,/authorized account/],[404,/sessions list/],[409,/read again/],[422,/current values/]]) {
+    let calls=0;const output=[];
+    await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>{calls++;throw Object.assign(new Error('private raw detail'),{status});}},{stdout:value=>output.push(value),stderr:()=>{}}),pattern);
+    assert.equal(calls,1);assert.deepEqual(output,[]);
+  }
+});
+
+test('plain fallback reads but never invents selectable turns',async()=>{
+  const output=[],errors=[];const data={version:revision,mode:'plain',document:{title:'Legacy'},branch:{requested:'selected',status:'unknown'},fragments:[],plain:{text:'Unstructured import'},next_cursor:null};
+  await sessionsCommand(['read','doc_fixture'],{branch:'selected'},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
+  assert.match(output.join(''),/Unstructured import/);assert.equal(output.join('').includes('Selection:'),false);assert.match(errors.join(''),/selection is unavailable/);
+});
+
+const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
+function run(args,env,cwd) { return new Promise((resolve,reject)=>{
+  const child=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>child.kill('SIGKILL'),10000);
+  child.stdout.on('data',part=>stdout+=part);child.stderr.on('data',part=>stderr+=part);child.on('error',reject);child.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr});});
+}); }
+async function server(digit){
+  const calls=[];const instance=http.createServer(async(request,response)=>{
+    let raw='';for await(const part of request)raw+=part;
+    calls.push({path:request.url,authorization:request.headers.authorization,body:raw?JSON.parse(raw):null});
+    let data={items:[],has_more:false,offset:0};
+    if(request.url.endsWith('/context')) data=receipt();
+    else if(request.url.includes('/turns?')) data={version:revision,mode:'structured',document:{title:'Plan'},branch:{requested:'selected',status:'selected'},fragments:[{ref:'t0',role:'user',branch:'selected',text:'Atlas task',start_char:0,end_char:100,truncated:false}],next_cursor:'next-page'};
+    response.writeHead(200,{'content-type':'application/json'});response.end(JSON.stringify(data));
+  });await new Promise(resolve=>instance.listen(0,'127.0.0.1',resolve));
+  return {calls,url:'http://127.0.0.1:'+instance.address().port,key:'oc_synthetic_'+digit,close:()=>new Promise(resolve=>instance.close(resolve))};
+}
+
+test('real sessions CLI supports help, repeated selections, exact output and pinned accounts',async()=>{
+  const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-sessions-'));const a=await server('a'),b=await server('b');
+  const env={...process.env};for(const key of Object.keys(env))if(key.startsWith('OPENCRAWL_'))delete env[key];env.OPENCRAWL_CONFIG_DIR=dir;
+  try{
+    const identity=digit=>({user:{id:'usr_'+digit.repeat(32)},workspace:{id:'ws_'+digit.repeat(32)}});
+    let config=loggedInConfig({},a.url,a.key,identity('a'));const account=config.activeProfile;config=loggedInConfig(config,b.url,b.key,identity('b'));
+    await fs.writeFile(path.join(dir,'config.json'),JSON.stringify(config),{mode:0o600});
+    const selected={...env,OPENCRAWL_URL:b.url,OPENCRAWL_API_KEY:b.key};
+    const help=await run(['sessions','export','--help'],selected,dir);assert.equal(help.code,0);assert.match(help.stdout,/--selection/);assert.equal(a.calls.length+b.calls.length,0);
+    const read=await run(['sessions','read','doc_fixture','--branch','selected','--account',account],selected,dir);assert.equal(read.code,0,read.stderr);assert.match(read.stdout,/Selection: t0:0:100/);assert.match(read.stderr,/next-page/);
+    const exported=await run(['sessions','export','doc_fixture','--branch','selected','--revision',revision,'--selection','t0:0:100','--budget','256','--account',account],selected,dir);
+    assert.equal(exported.code,0,exported.stderr);assert.equal(exported.stdout,receipt().context);assert.equal(exported.stderr,'');
+    assert.equal(b.calls.length,0);assert.ok(a.calls.every(call=>call.authorization==='Bearer '+a.key));
+    const globalFirst=await run(['--account',account,'sessions','export','doc_fixture','--branch','selected','--revision',revision,'--selection','t0:0:100','--budget','256'],selected,dir);assert.equal(globalFirst.code,0,globalFirst.stderr);assert.equal(globalFirst.stdout,receipt().context);
+    const before=a.calls.length;
+    for(const extra of [['--selection','t0:0:100','--selection','t0:0:100'],['--budget','256','--budget','500'],['--json=false']]) {
+      const result=await run(['sessions','export','doc_fixture','--branch','selected','--revision',revision,'--account',account,...extra],selected,dir);assert.equal(result.code,1);assert.equal(result.stdout,'');
+    }
+    assert.equal(a.calls.length,before);
+    const conflict=await run(['sessions','read','doc_fixture','--branch','selected','--account',account,'--server',b.url],selected,dir);assert.equal(conflict.code,1);assert.equal(b.calls.length,0);
+  }finally{await Promise.all([a.close(),b.close()]);await fs.rm(dir,{recursive:true,force:true});}
+});
+
+test('bounded HTTP reading refuses excessive bytes including streamed responses',async t=>{
+  const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-response-'));const previous=process.env.OPENCRAWL_CONFIG_DIR;process.env.OPENCRAWL_CONFIG_DIR=dir;
+  try{
+    t.mock.method(globalThis,'fetch',async()=>new Response(new ReadableStream({start(controller){controller.enqueue(new TextEncoder().encode('x'.repeat(33)));controller.close();}})));
+    const client=await createClient({server:'https://example.test',token:'oc_fixture'});
+    await assert.rejects(client.request('/api/v1/sessions',{maxResponseBytes:32}),/byte limit/);
+  }finally{if(previous===undefined)delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR=previous;await fs.rm(dir,{recursive:true,force:true});}
+});
+
+
+test('human reader displays control sequences visibly without changing JSON or export',async()=>{
+  const text='safe\n\tline\u001b[2J\rhidden\u009b31m\u202eend';
+  const data={version:revision,mode:'structured',document:{title:'Plan'},branch:{requested:'selected',status:'selected'},fragments:[{ref:'t0',role:'user',branch:'selected',text,start_char:0,end_char:text.length,truncated:false}]};
+  const output=[];await sessionsCommand(['read','doc_fixture'],{branch:'selected'},{request:async()=>data},{stdout:value=>output.push(value),stderr:()=>{}});
+  assert.match(output.join(''),/safe\n\tline\\u001b\[2J\\u000dhidden\\u009b31m\\u202eend/);
+  assert.equal(output.join('').includes('\u001b'),false);
+  const json=[];await sessionsCommand(['read','doc_fixture'],{branch:'selected',json:true},{request:async()=>data},{stdout:value=>json.push(value),stderr:()=>{}});
+  assert.equal(JSON.parse(json.join('')).fragments[0].text,text);
+  const exact=[];await sessionsCommand(['export','doc_fixture'],options(),{request:async()=>receipt(text)},{stdout:value=>exact.push(value),stderr:()=>{}});
+  assert.equal(exact.join(''),text);
+});
+
+
+test('included and omitted receipts must partition the requested ranges exactly',async()=>{
+  const base=receipt();
+  for (const change of [{included:[],omitted:[]},{included:[{...base.included[0],ref:'t1'}]},{included:[{...base.included[0],end_char:101}]},{included:[{...base.included[0],start_char:1}]},{included:[{...base.included[0],truncated:true}]},{included:[{...base.included[0],excerpt:'different'}]},{omitted:[{ref:'t0',start_char:0,end_char:100,reason:'duplicate'}]}]) {
+    const output=[];await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>({...base,...change})},{stdout:value=>output.push(value),stderr:()=>{}}),/receipt/);assert.deepEqual(output,[]);
+  }
+});

package/src/tasks.test.mjs

@@ -0,0 +1,83 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import {spawn} from 'node:child_process';
+import {fileURLToPath} from 'node:url';
+import {taskRequest,tasksCommand,validateTaskRun} from './tasks.mjs';
+import {loggedInConfig} from './client.mjs';
+const id='task_'+'a'.repeat(32);
+const task=()=>({id,title:'Launch evidence',query:'What is the current launch?',scope:{mode:'selected',source_ids:['src_alpha']},limit:5,budget:16384,revision:2,created_at:'2026-01-01T00:00:00Z',updated_at:'2026-01-01T00:00:00Z',scope_status:'available'});
+const receipt=(text='[S1] Launch\né🌲\u001b[2J\n')=>({task:task(),query:'What is the current launch?',context:text,context_bytes:Buffer.byteLength(text),budget_unit:'utf8_bytes',citations:text?[{citation:'S1',document_id:'doc_alpha',chunk_id:1,source_id:'src_alpha',kind:'document',title:'Launch',context_span:{start_char:12,end_char:Array.from(text).length},excerpt:{start_char:0,end_char:Array.from(text).length-12}}]:[],coverage:{warnings:[]}});
+const execute=async(rest,options,data)=>{let stdout='',stderr='';const calls=[];const status=await tasksCommand(rest,options,{request:async(route,opts)=>{calls.push({route,opts});if(data instanceof Error)throw data;return data;}},{stdout:value=>stdout+=value,stderr:value=>stderr+=value});return{stdout,stderr,calls,status};};
+
+test('tasks preserve stored descriptor and positive revision without run overrides',()=>{
+  assert.deepEqual(taskRequest(['run',id],{}).options,{method:'POST',body:{}});
+  assert.deepEqual(taskRequest(['run',id],{revision:'2'}).options,{method:'POST',body:{expected_revision:2}});
+  assert.equal(taskRequest(['list'],{limit:'20',offset:'40'}).route,'/api/v1/tasks?limit=20&offset=40');
+  for(const revision of ['0','-1','1e2','+2','2.0',' 2','02','a'.repeat(64),'9007199254740992'])assert.throws(()=>taskRequest(['run',id],{revision}));
+  for(const key of ['query','scope','source','budget','limit','offset'])assert.throws(()=>taskRequest(['run',id],{[key]:'2'}));
+  for(const rest of [['run'],['show','../secret'],['list',id],['run',id,'extra'],['create']])assert.throws(()=>taskRequest(rest,{}));
+});
+test('run stdout preserves exact context bytes and full JSON receipts',async()=>{
+  const value=receipt();const plain=await execute(['run',id],{revision:'2'},value);
+  assert.equal(plain.status,0);assert.equal(plain.stdout,value.context);assert.match(plain.stderr,/Task revision 2/);
+  assert.deepEqual(plain.calls[0].opts.body,{expected_revision:2});
+  const json=await execute(['run',id],{json:true},value);assert.deepEqual(JSON.parse(json.stdout),value);assert.equal(json.stderr,'');
+});
+test('empty current evidence succeeds without fabricated answer or silent broader scope',async()=>{
+  const value=receipt('');const result=await execute(['run',id],{},value);assert.equal(result.status,0);assert.equal(result.stdout,'');assert.match(result.stderr,/No current evidence/);assert.equal(result.calls.length,1);
+  const json=await execute(['run',id],{json:true},value);assert.equal(JSON.parse(json.stdout).context,'');
+});
+test('all-accessible tasks can cite standalone documents but cannot carry hidden selected scope',async()=>{
+  const value=receipt();value.task.scope={mode:'all_accessible'};value.citations[0].source_id=null;
+  assert.equal((await execute(['run',id],{},value)).stdout,value.context);
+  value.task.scope={mode:'all_accessible',source_ids:['src_alpha']};
+  await assert.rejects(execute(['run',id],{},value));
+});
+test('byte, revision, current scope and exact citation spans fail closed before stdout',async()=>{
+  const base=receipt();
+  const cases=[{task:{...task(),revision:3}},{task:{...task(),scope_status:'blocked'}},{context_bytes:1},{budget_unit:'tokens'},
+    {context:'x'.repeat(16385),context_bytes:16385},{citations:[]},{citations:[{...base.citations[0],source_id:'src_other'}]},
+    {citations:[{...base.citations[0],context_span:{start_char:0,end_char:99999}}]},
+    {citations:[base.citations[0],base.citations[0]]},{task:{...task(),budget:16385}},{task:{...task(),scope:{mode:'selected',source_ids:[]}}}];
+  for(const change of cases){let output='';await assert.rejects(tasksCommand(['run',id],{revision:'2'},{request:async()=>({...base,...change})},{stdout:v=>output+=v,stderr:()=>{}}));assert.equal(output,'');}
+  const full=receipt('[S1] Launch\n'+'é'.repeat(8186));assert.equal(full.context_bytes,16384);validateTaskRun(full,{id,revision:2});
+});
+test('task conflicts distinguish changed descriptor from unavailable source without retry',async()=>{
+  for(const [status,code,pattern]of[[409,'task_changed',/review its current/],[409,'task_scope_unavailable',/No broader search/],[409,'task_evidence_changed',/No automatic retry/],[404,'anything',/tasks list/],[401,null,/Sign in/],[500,'secret',/could not be confirmed/]]){
+    const error=Object.assign(new Error('RAW SECRET\u001b'),{status,code});const result=await execute(['run',id],{json:true},error);
+    assert.equal(result.status,1);assert.equal(result.calls.length,1);assert.equal(result.stderr,'');assert.match(JSON.parse(result.stdout).error.message,pattern);assert.ok(!result.stdout.includes('SECRET'));
+  }
+});
+test('human task descriptions and warnings escape terminal controls; JSON remains exact',async()=>{
+  const value={...task(),title:'Name\u001b[2J',query:'Question\n\ttext\u009b31m'};
+  const shown=await execute(['show',id],{},value);assert.ok(!shown.stdout.includes('\u001b'));assert.match(shown.stdout,/\\u001b/);assert.match(shown.stdout,/Question\n\ttext\\u009b/);
+  const listed=await execute(['list'],{}, {items:[value],limit:20,offset:0});assert.match(listed.stdout,/\\u001b/);
+  const pack=receipt();pack.coverage.warnings=['Fallback\u001b[2J'];const result=await execute(['run',id],{},pack);assert.match(result.stderr,/\\u001b/);assert.equal(result.stdout,pack.context);
+  const json=await execute(['show',id],{json:true},value);assert.deepEqual(JSON.parse(json.stdout),value);
+});
+const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
+function run(args,env,cwd){return new Promise((resolve,reject)=>{const p=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>p.kill('SIGKILL'),10000);p.stdout.on('data',v=>stdout+=v);p.stderr.on('data',v=>stderr+=v);p.on('error',reject);p.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr});});});}
+async function listen(){const calls=[];let conflict=null;const server=http.createServer(async(req,res)=>{let raw='';for await(const v of req)raw+=v;const body=raw?JSON.parse(raw):null;calls.push({path:req.url,body,authorization:req.headers.authorization});const failed=req.url.endsWith('/run')&&conflict;res.writeHead(failed?409:200,{'content-type':'application/json'});res.end(JSON.stringify(failed?{code:conflict,detail:'RAW PRIVATE'}:req.url.startsWith('/api/v1/tasks?')?{items:[task()],limit:20,offset:0}:req.url.endsWith('/run')||req.url.endsWith('/context')?receipt():task()));});await new Promise(r=>server.listen(0,'127.0.0.1',r));return{url:'http://127.0.0.1:'+server.address().port,calls,setConflict:value=>conflict=value,close:()=>new Promise(r=>server.close(r))};}
+test('real CLI keeps account pins, strict options, fixed JSON conflicts, and default search scope',async()=>{
+  const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-tasks-'));const a=await listen(),b=await listen();const env={...process.env};for(const key of Object.keys(env))if(key.startsWith('OPENCRAWL_'))delete env[key];env.OPENCRAWL_CONFIG_DIR=dir;
+  const identity=digit=>({user:{id:'usr_'+digit.repeat(32)},workspace:{id:'ws_'+digit.repeat(32)}});
+  try{
+    let config=loggedInConfig({},a.url,'oc_synthetic_a',identity('a'));const account=config.activeProfile;config=loggedInConfig(config,b.url,'oc_synthetic_b',identity('b'));await fs.writeFile(path.join(dir,'config.json'),JSON.stringify(config),{mode:0o600});
+    const ambient={...env,OPENCRAWL_URL:b.url,OPENCRAWL_API_KEY:'oc_synthetic_b'};
+    const help=await run(['tasks','--help'],ambient,dir);assert.equal(help.code,0);assert.match(help.stdout,/--revision N/);assert.equal(a.calls.length+b.calls.length,0);
+    const exact=await run(['tasks','run',id,'--revision','2','--account',account],ambient,dir);assert.equal(exact.code,0,exact.stderr);assert.equal(exact.stdout,receipt().context);assert.deepEqual(a.calls.at(-1).body,{expected_revision:2});
+    for(const args of [['tasks','list','--account',account],['tasks','show',id,'--account',account],['--account',account,'tasks','run',id,'--json']]){const result=await run(args,ambient,dir);assert.equal(result.code,0,result.stderr);}
+    assert.equal(b.calls.length,0);assert.ok(a.calls.every(c=>c.authorization==='Bearer oc_synthetic_a'));
+    const before=a.calls.length;
+    for(const extra of [['--revision','2','--revision','3'],['--json=false'],['--source','src_other'],['--revision','0']]){const r=await run(['tasks','run',id,'--account',account,...extra],ambient,dir);assert.equal(r.code,1);}
+    assert.equal(a.calls.length,before);
+    const wrong=await run(['tasks','show',id,'--account',account,'--server',b.url],ambient,dir);assert.equal(wrong.code,1);assert.equal(b.calls.length,0);
+    for(const code of ['task_changed','task_scope_unavailable','task_evidence_changed']){a.setConflict(code);const r=await run(['tasks','run',id,'--json','--account',account],ambient,dir);assert.equal(r.code,1);assert.equal(JSON.parse(r.stdout).error.code,code);assert.equal(r.stderr,'');assert.ok(!r.stdout.includes('PRIVATE'));}
+    a.setConflict(null);const r=await run(['context','launch','--account',account],ambient,dir);assert.equal(r.code,0,r.stderr);assert.equal(Object.hasOwn(a.calls.at(-1).body,'source_ids'),false);
+    await run(['context','launch','--source','src_alpha','--account',account],ambient,dir);assert.deepEqual(a.calls.at(-1).body.source_ids,['src_alpha']);
+  }finally{await Promise.all([a.close(),b.close()]);await fs.rm(dir,{recursive:true,force:true});}
+});