@getopencrawl/cli 0.1.1 → 0.2.1

package/src/research-completion.mjs

@@ -1,94 +0,0 @@
-/** Optional, versioned run metadata. It never changes the validity of literal findings. */
-const POLICY_SHA256 = '1e465f634773191ff5e101b9589547d8af81c9c30f63fbc4febea477d58e009f';
-const POLICY_V2_SHA256 = 'e8793d22da1e547e689b17053f3cc7c8094f25ec77add02c79ce11243825dc21';
-const POLICY_V3_SHA256 = '0a8dd4f97f60eea5afb38bfbc83e86fe6446610b2e3aa659e500c2c46c8c513b';
-const LIMITS = { model_calls: 5, web_searches: 5, pages: 9, page_requests: 12, page_bytes: 6291456, work_seconds: 360, crawl_seconds: 60 };
-const LIMITS_V2 = { model_calls: 7, web_searches: 5, pages: 9, page_requests: 12, page_bytes: 6291456, work_seconds: 480, crawl_seconds: 60 };
-const LIMITS_V3 = { ...LIMITS_V2, page_requests: 36 };
-const NOTICE_TEXT = {
-    bounded_sources: 'This run inspected bounded saved source windows. It does not establish whole-question completeness.',
-    evidence_omitted: 'Some discovery, acquisition or source windows were omitted or unavailable. Read the saved gaps and evidence limits before using this answer.',
-    repair_input_bound: 'The completion pass could not use additional material within the input allowance. Earlier validated findings were retained.',
-    unassessed: 'Requested-detail coverage could not be assessed. No completion pass was inferred from that missing assessment.',
-};
-const REASONS = ['admitted', 'completion_reviewed', 'no_eligible_repair', 'no_identified_gap', 'unassessed', 'repair_input_bound', 'no_new_admitted_source', 'failed', 'unknown', 'cancelled'];
-const PHASES = ['completion_discover', 'synthesize', 'completion_search', 'completion_synthesize'];
-const PHASES_V2 = [...PHASES, 'presentation_draft', 'presentation_review'];
-const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
-const integer = (value, max) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 && value <= max;
-const keys = (value, required, optional = []) => required.every(key => Object.hasOwn(value, key)) && Object.keys(value).every(key => required.includes(key) || optional.includes(key));
-/** Missing or malformed optional metadata yields no notices; saved run fields stay untouched. */
-export function readResearchCompletion(run) {
-    const empty = { completion: null, notices: [] };
-    try {
-        if (!object(run?.usage))
-            return empty;
-        const value = run.usage.completion;
-        if (!object(value) || !keys(value, ['version', 'policy', 'policy_sha256', 'limits', 'reason', 'validated_passes', 'whole_request_completeness', 'notices'], ['timing_ms', 'completed_usage_lower_bound', 'acquisition']) || value.whole_request_completeness !== 'not_certified' || !REASONS.includes(value.reason) || !integer(value.validated_passes, 2))
-            return empty;
-        const v3 = value.version === 3, v2 = value.version === 2 || v3;
-        if (!(v3 ? value.policy === 'completion_v3' && value.policy_sha256 === POLICY_V3_SHA256 : v2 ? value.policy === 'completion_v2' && value.policy_sha256 === POLICY_V2_SHA256 : value.version === 1 && value.policy === 'completion_v1' && value.policy_sha256 === POLICY_SHA256))
-            return empty;
-        const limits = v3 ? LIMITS_V3 : v2 ? LIMITS_V2 : LIMITS, phases = v2 ? PHASES_V2 : PHASES;
-        if (!object(value.limits) || !keys(value.limits, Object.keys(limits)) || Object.entries(limits).some(([key, limit]) => value.limits[key] !== limit))
-            return empty;
-        if (!Array.isArray(value.notices) || value.notices.length > 4 || new Set(value.notices).size !== value.notices.length || !value.notices.every(code => typeof code === 'string' && Object.hasOwn(NOTICE_TEXT, code)))
-            return empty;
-        if (Object.hasOwn(value, 'timing_ms')) {
-            const timing = value.timing_ms;
-            if (!object(timing) || !keys(timing, ['total', 'phases']) || !integer(timing.total, v2 ? 600000 : 360000) || !Array.isArray(timing.phases) || timing.phases.length > (v2 ? 6 : 4) || !timing.phases.every(phase => object(phase) && keys(phase, ['phase', 'elapsed_ms']) && phases.includes(phase.phase) && integer(phase.elapsed_ms, v2 ? 300000 : 210000)))
-                return empty;
-        }
-        if (Object.hasOwn(value, 'acquisition')) {
-            const acquisition = value.acquisition;
-            if (!v3 || value.reason === 'admitted' || !object(acquisition) || !keys(acquisition, ['version', 'stages']) || acquisition.version !== 1 || !Array.isArray(acquisition.stages) || acquisition.stages.length < 1 || acquisition.stages.length > 2)
-                return empty;
-            let requests = 0, pages = 0, bytes = 0, elapsed = 0;
-            for (const [index, row] of acquisition.stages.entries()) {
-                if (!object(row) || !keys(row, ['stage', 'requests', 'pages', 'bytes', 'elapsed_ms']) || row.stage !== (index === 0 ? 'initial' : 'repair') || !integer(row.requests, index === 0 ? 24 : 36) || !integer(row.pages, index === 0 ? 6 : 3) || !integer(row.bytes, 6291456) || !integer(row.elapsed_ms, 360000))
-                    return empty;
-                requests += Number(row.requests);
-                pages += Number(row.pages);
-                bytes += Number(row.bytes);
-                elapsed += Number(row.elapsed_ms);
-            }
-            if (requests > 36 || pages > 9 || bytes > 6291456 || elapsed > 360000 || requests !== run.usage.page_requests || pages !== run.usage.pages_fetched || bytes !== run.usage.page_bytes)
-                return empty;
-            if (object(value.timing_ms) && elapsed > Number(value.timing_ms.total))
-                return empty;
-        }
-        if (Object.hasOwn(value, 'completed_usage_lower_bound')) {
-            const usage = value.completed_usage_lower_bound;
-            if (!object(usage) || !keys(usage, ['model_calls', 'web_searches', 'input_tokens', 'output_tokens']) || !integer(usage.model_calls, limits.model_calls) || !integer(usage.web_searches, 5) || !integer(usage.input_tokens, 1000000000) || !integer(usage.output_tokens, 1000000000))
-                return empty;
-        }
-        if (value.reason === 'admitted' && (value.validated_passes !== 0 || value.notices.length !== 0 || Object.hasOwn(value, 'timing_ms') || Object.hasOwn(value, 'completed_usage_lower_bound')))
-            return empty;
-        // Detach the returned view from the saved receipt; only fixed local text is displayed.
-        const completion = structuredClone(value);
-        return { completion, notices: completion.notices.map(code => ({ code, text: NOTICE_TEXT[code] })) };
-    }
-    catch {
-        return empty;
-    }
-}
-const PRESENTATION_STATUSES = ['reviewed', 'partial', 'refused', 'skipped_no_eligible', 'skipped_bound', 'skipped_capacity', 'unavailable', 'unknown', 'deadline', 'interrupted'];
-/** Optional usage only: its digest reports provenance, not a client recomputation of canonical content. */
-export function readResearchPresentation(run) {
-    try {
-        if (!object(run?.usage))
-            return null;
-        const value = run.usage.presentation;
-        if (!object(value) || !keys(value, ['version', 'status', 'canonical_sha256', 'calls_reserved', 'calls_completed', 'calls_uncertain', 'known_input_tokens', 'known_output_tokens', 'usage_is_lower_bound', 'elapsed_ms']) || value.version !== 1 || !PRESENTATION_STATUSES.includes(value.status) || typeof value.canonical_sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(value.canonical_sha256))
-            return null;
-        if (!integer(value.calls_reserved, 2) || !integer(value.calls_completed, 2) || !integer(value.calls_uncertain, 2) || Number(value.calls_completed) + Number(value.calls_uncertain) > Number(value.calls_reserved) || !integer(value.known_input_tokens, 1000000000) || !integer(value.known_output_tokens, 12000) || value.usage_is_lower_bound !== (Number(value.calls_uncertain) > 0) || !integer(value.elapsed_ms, 180000))
-            return null;
-        if (value.calls_completed === 0 && value.known_input_tokens !== 0 || Number(value.known_output_tokens) > Number(value.calls_completed) * 6000)
-            return null;
-        return structuredClone(value);
-    }
-    catch {
-        return null;
-    }
-}
-//# sourceMappingURL=researchCompletion.js.map

package/src/research-completion.test.mjs

@@ -1,110 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import {readResearchCompletion} from './research-completion.mjs';
-import {agentCommand,agentRequest,validateAgentDescriptor} from './research-agent.mjs';
-
-const fixture=JSON.parse(await fs.readFile(new URL('./fixtures/research-completion-v1.json',import.meta.url),'utf8'));
-const actual=JSON.parse(await fs.readFile(new URL('./fixtures/research-completion-hosted.json',import.meta.url),'utf8'));
-const empty={completion:null,notices:[]},metadata=()=>structuredClone(fixture.completion);
-const read=value=>readResearchCompletion({usage:{completion:value}});
-const key='completion-original-key-0001',id=actual.runs.admitted.agent_id;
-const brief={customer:{name:'Northlight',website:'https://northlight.example/',context:''},goal:'Find documented workflow changes.',collection_ids:[],schedule_minutes:0};
-const queued=actual.runs.admitted;
-const receipt={agent:{id,revision:1,status:'active',...brief,created_at:queued.created_at,updated_at:queued.created_at,latest_run:queued},run:queued,replayed:false};
-async function execute(rest,options,result){
- let stdout='',stderr='';const calls=[];
- const code=await agentCommand(rest,options,{request:async(route,args)=>{calls.push({route,args});if(result instanceof Error)throw result;return result;}},{stdout:x=>stdout+=x,stderr:x=>stderr+=x});
- return {code,stdout,stderr,calls};
-}
-
-test('fixed notices preserve order and detach the optional view from saved metadata',()=>{
- const input=metadata(),before=structuredClone(input),view=read(input);
- assert.deepEqual(view.completion,before);
- assert.deepEqual(view.notices,Object.entries(fixture.notice_text).map(([code,text])=>({code,text})));
- view.completion.notices.pop();view.completion.limits.pages=1;
- assert.deepEqual(input,before);
-});
-
-test('absent and queued metadata are quiet and cannot imply completion',()=>{
- for(const run of [{},{usage:{}},{usage:null}])assert.deepEqual(readResearchCompletion(run),empty);
- const admitted=actual.runs.admitted.usage.completion;
- assert.deepEqual(read(admitted),{completion:admitted,notices:[]});
- for(const patch of [{validated_passes:1},{notices:['bounded_sources']},{timing_ms:metadata().timing_ms},{completed_usage_lower_bound:metadata().completed_usage_lower_bound}])assert.deepEqual(read({...admitted,...patch}),empty);
-});
-
-test('malformed, raw or private optional fields never become human notices',()=>{
- const mutations=[v=>{v.version=2},v=>{v.policy='other'},v=>{v.policy_sha256='a'.repeat(64)},v=>{v.whole_request_completeness='complete'},v=>{v.reason='all_answered'},v=>{v.limits.pages=10},v=>{v.limits.extra=1},v=>{delete v.limits.page_bytes},v=>{v.notices=['PRIVATE_RAW\u001b[31m']},v=>{v.notices=['bounded_sources','bounded_sources']},v=>{v.notices=['toString']},v=>{v.notices=['__proto__']},v=>{v.raw='PRIVATE_RAW'},v=>{delete v.validated_passes},v=>{v.timing_ms=null},v=>{v.completed_usage_lower_bound=[]}];
- for(const mutate of mutations){const value=metadata();mutate(value);assert.deepEqual(read(value),empty);}
-});
-
-test('integer limits and finite timing phases reject impossible optional measurements',()=>{
- for(const invalid of [-1,NaN,Infinity,true,'1',0.5])for(const field of ['validated_passes','timing_ms.total','timing_ms.phases.0.elapsed_ms','completed_usage_lower_bound.model_calls','completed_usage_lower_bound.web_searches','completed_usage_lower_bound.input_tokens','completed_usage_lower_bound.output_tokens']){
-  const value=metadata(),parts=field.split('.');let target=value;for(const part of parts.slice(0,-1))target=target[part];target[parts.at(-1)]=invalid;assert.deepEqual(read(value),empty,field);
- }
- const mutations=[v=>{v.validated_passes=3},v=>{v.timing_ms.total=360001},v=>{v.timing_ms.phases[0].elapsed_ms=210001},v=>{v.timing_ms.phases.push(v.timing_ms.phases[0])},v=>{v.timing_ms.phases[0].phase='raw'},v=>{v.timing_ms.phases[0].body='private'},v=>{v.completed_usage_lower_bound.model_calls=6},v=>{v.completed_usage_lower_bound.web_searches=6},v=>{v.completed_usage_lower_bound.input_tokens=1000000001}];
- for(const mutate of mutations){const value=metadata();mutate(value);assert.deepEqual(read(value),empty);}
-});
-
-test('actual API projected admitted repaired unknown and unassessed runs preserve JSON and status on one read',async()=>{
- assert.deepEqual(Object.keys(actual.runs),['admitted','repaired','unknown','unassessed']);
- for(const [name,run] of Object.entries(actual.runs)){
-  const before=JSON.stringify(run),options=['agent-run',run.agent_id,run.id];
-  const json=await execute(options,{json:true},run),human=await execute(options,{},run);
-  assert.deepEqual(JSON.parse(json.stdout),run);assert.equal(json.code,human.code);assert.equal(json.calls.length,1);assert.equal(human.calls.length,1);
-  assert.equal(json.calls[0].args.method,undefined);assert.equal(JSON.stringify(run),before);
-  assert.deepEqual(readResearchCompletion(run).completion,run.usage.completion);
-  for(const code of run.usage.completion.notices)assert.ok(human.stdout.includes(fixture.notice_text[code]),name);
-  if(name==='unknown'){assert.equal(json.code,1);assert.equal(run.findings.length,0);assert.equal(readResearchCompletion(run).completion.validated_passes,1);assert.equal(readResearchCompletion(run).notices.length,0);}
- }
-});
-
-test('all five saved gaps survive, followed by fixed optional notices without changing ordinary findings',async()=>{
- const run=structuredClone(actual.runs.repaired);run.gaps=Array.from({length:5},(_,i)=>`Saved gap ${i+1}.`);run.usage.completion=metadata();
- const result=await execute(['agent-run',run.agent_id,run.id],{},run);assert.equal(result.calls.length,1);
- let previous=-1;for(const text of [...run.gaps,...Object.values(fixture.notice_text)]){const next=result.stdout.indexOf('Gap: '+text);assert.ok(next>previous,text);previous=next;}
- for(const finding of run.findings){assert.ok(result.stdout.includes(finding.claim));for(const citation of finding.citations)assert.ok(result.stdout.includes(citation.quote));}
-});
-
-test('invalid optional metadata is ignored in human output and preserved verbatim in JSON without extra requests',async()=>{
- const run=structuredClone(actual.runs.repaired);run.usage.completion={...metadata(),notices:['PRIVATE_RAW']};
- const human=await execute(['agent-run',run.agent_id,run.id],{},run),json=await execute(['agent-run',run.agent_id,run.id],{json:true},run);
- assert.equal(human.code,json.code);assert.equal(human.calls.length,1);assert.equal(json.calls.length,1);assert.ok(!human.stdout.includes('PRIVATE_RAW'));
- assert.deepEqual(JSON.parse(json.stdout),run);for(const finding of run.findings)assert.ok(human.stdout.includes(finding.claim));
-});
-
-test('ordinary descriptor and manual-run body remain byte-identical when policy is absent',()=>{
- assert.equal(JSON.stringify(validateAgentDescriptor(brief)),JSON.stringify(brief));
- const request=agentRequest(['agent','run',id],{revision:'1','request-key':key});assert.equal(JSON.stringify(request.body),'\u007b"expected_revision":1}');assert.equal(request.key,key);
-});
-
-test('v1 remains explicit at create JSON and manual-run option',()=>{
- assert.deepEqual(validateAgentDescriptor({...brief,research_policy:'completion_v1'}),{...brief,research_policy:'completion_v1'});
- const request=agentRequest(['agent','run',id],{revision:'1','request-key':key,'research-policy':'completion_v1'});assert.deepEqual(request.body,{expected_revision:1,research_policy:'completion_v1'});assert.equal(request.key,key);
- for(const value of [undefined,null,true,1,'ordinary','completion_v4',{},['completion_v1']]){
-  assert.throws(()=>validateAgentDescriptor({...brief,research_policy:value}),/research_policy/);
-  assert.throws(()=>agentRequest(['agent','run',id],{revision:'1','request-key':key,'research-policy':value}),/research-policy/);
- }
- for(const rest of [['agent','show',id],['agent-run',id,queued.id],['agent','schedule',id,'daily'],['agent','create']])assert.throws(()=>agentRequest(rest,{'research-policy':'completion_v1'}),/Unsupported/);
-});
-
-test('create forwards exactly one explicit policy with original key; unavailable policy never falls back',async()=>{
- const directory=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-completion-')),file=path.join(directory,'brief.json');
- try{
-  const body={...brief,research_policy:'completion_v1'};await fs.writeFile(file,JSON.stringify(body));
-  const result=await execute(['agent','create'],{file,'request-key':key,json:true},receipt);
-  assert.equal(result.code,0);assert.equal(result.calls.length,1);assert.deepEqual(result.calls[0].args.body,body);assert.equal(result.calls[0].args.idempotencyKey,key);
-  const error=new Error('PRIVATE_SERVER_REASON');error.status=409;
-  const denied=await execute(['agent','create'],{file,'request-key':key,json:true},error);
-  assert.equal(denied.code,1);assert.equal(denied.calls.length,1);assert.equal(JSON.parse(denied.stdout).error.request_key,key);assert.ok(!denied.stdout.includes('PRIVATE_SERVER_REASON'));
- }finally{await fs.rm(directory,{recursive:true,force:true});}
-});
-
-test('manual admission unknown keeps original policy/key and uses only GET for explicit receipt recovery',async()=>{
- const result=await execute(['agent','run',id],{revision:'1','request-key':key,'research-policy':'completion_v1',json:true},new Error('PRIVATE_LOST_ACK'));
- assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.deepEqual(result.calls[0].args.body,{expected_revision:1,research_policy:'completion_v1'});assert.equal(JSON.parse(result.stdout).error.request_key,key);assert.ok(!result.stdout.includes('PRIVATE_LOST_ACK'));
- const recovered=await execute(['agent','receipt'],{operation:'run:'+id,'request-key':key,json:true},{...receipt,replayed:true});
- assert.equal(recovered.code,0);assert.equal(recovered.calls.length,1);assert.equal(recovered.calls[0].args.method,undefined);assert.equal(recovered.calls[0].args.idempotencyKey,key);assert.equal(recovered.calls[0].args.body,undefined);
-});

package/src/research-event.mjs

@@ -1,39 +0,0 @@
-import {displayLine,displayText} from './display.mjs';
-const integer=(value,min=0,max=2_000_000)=>Number.isSafeInteger(value)&&value>=min&&value<=max;
-const text=(value,max)=>typeof value==='string'&&value.length<=max;
-const hash=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value);
-const invalid=()=>new Error('Server returned invalid or out-of-scope retained event evidence; no source text was emitted.');
-function window(value,offset,comparison){
- if(value===null){if(offset!==undefined)throw invalid();return 0;}
- if(!value||!text(value.version_id,200)||!value.version_id||!hash(value.version_sha256)||!hash(value.body_sha256)||!text(value.title,8192)||!text(value.url,8192)||!(value.final_url===null||text(value.final_url,65536))||!(value.extraction_version===null||text(value.extraction_version,65536))||!text(value.version_observed_at,200)||!value.version_observed_at||!text(value.text,64000)||!integer(value.full_length)||!integer(value.start_char,0,value.full_length)||!integer(value.end_char,value.start_char,value.full_length))throw invalid();
- const points=Array.from(value.text);
- if(points.length!==value.end_char-value.start_char||points.some(point=>point.length===1&&/[\uD800-\uDFFF]/.test(point))||value.next_start_char!==(value.end_char<value.full_length?value.end_char:null)||value.omitted_before!==(value.start_char>0)||value.omitted_after!==(value.end_char<value.full_length))throw invalid();
- if(offset!==undefined){if(value.window_origin!=='explicit_offset'||value.start_char!==offset)throw invalid();}
- else if(value.window_origin!==(comparison==='contiguous_region'?'changed_region':'page_start')||(value.window_origin==='page_start'&&value.start_char!==0))throw invalid();
- if(comparison==='contiguous_region'){
-  if(!integer(value.changed_start_char,0,value.full_length)||!integer(value.changed_end_char,value.changed_start_char,value.full_length))throw invalid();
- }else if(value.changed_start_char!==null||value.changed_end_char!==null)throw invalid();
- return Buffer.byteLength(value.text,'utf8');
-}
-export function validateEventEvidence(value,request){
- if(!value||value.id!==request.eventId||value.collection_id!==request.id||!text(value.url,8192)||!['baseline','changed','unavailable','recovered'].includes(value.kind)||!text(value.observed_at,200)||!value.observed_at||!(value.previous_observed_at===null||text(value.previous_observed_at,200))||typeof value.gap!=='boolean'||!['contiguous_region','equal_body','single_observation','no_observation'].includes(value.comparison)||value.budget!==request.budget||!integer(value.text_bytes,0,request.budget))throw invalid();
- const count=Number(value.before!==null)+Number(value.after!==null);
- if(count!==(value.comparison==='no_observation'?0:value.comparison==='single_observation'?1:2))throw invalid();
- const total=window(value.before,request.beforeStart,value.comparison)+window(value.after,request.afterStart,value.comparison);
- if(total!==value.text_bytes||total>request.budget||(value.comparison==='equal_body'&&value.before?.body_sha256!==value.after?.body_sha256)||(value.comparison==='contiguous_region'&&value.before?.body_sha256===value.after?.body_sha256))throw invalid();
- // Full observation hashes are metadata; a partial window cannot verify the full retained body.
- return value;
-}
-export function renderEventEvidence(value,stdout,stderr){
- stdout(`Event: ${displayLine(value.id)}\nCollection: ${displayLine(value.collection_id)}\nKind: ${displayLine(value.kind)}\nObserved: ${displayLine(value.observed_at)}\nPrevious observation: ${value.previous_observed_at===null?'none':displayLine(value.previous_observed_at)}\nURL: ${displayLine(value.url)}\nComparison: ${displayLine(value.comparison)}\n`);
- for(const side of ['before','after']){
-  const item=value[side],label=side==='before'?'Previous observation':'This observation';
-  if(item===null){stdout(`\n${label}: no retained observation for this side.\n`);continue;}
-  stdout(`\n${label}: ${displayLine(item.title)}\nVersion: ${displayLine(item.version_id)}\nVersion observed: ${displayLine(item.version_observed_at)}\nURL: ${displayLine(item.url)}\nCodepoints: [${item.start_char}, ${item.end_char}) of ${item.full_length}\n${displayText(item.text)}\n`);
-  if(item.omitted_before)stderr(`${label}: earlier source text is omitted; use --${side}-start 0 to read from the beginning.\n`);
-  if(item.next_start_char!==null)stderr(`${label}: later source text is omitted; continue explicitly with --${side}-start ${item.next_start_char}.\n`);
- }
- if(value.gap)stderr('Unsuccessful checks separate these observations; the exact change time is unknown.\n');
- if(value.comparison==='contiguous_region')stderr('This enclosing changed region can contain unchanged or moved text; it is not a minimal diff.\n');
- stderr(`${value.text_bytes} source-text UTF-8 bytes returned within the ${value.budget}-byte budget. Times describe observations, not authored modifications. Baselines are not new business events; unavailable checks do not prove removal. No crawl or search was started.\n`);
-}

package/src/research-event.test.mjs

@@ -1,57 +0,0 @@
-import assert from 'node:assert/strict';
-import test from 'node:test';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import http from 'node:http';
-import {spawn} from 'node:child_process';
-import {fileURLToPath} from 'node:url';
-import {researchCommand,researchRequest} from './research.mjs';
-import {loggedInConfig} from './client.mjs';
-const stamp='2026-10-24T05:00:00Z',url='https://example.com/product';
-const side=(patch={})=>({version_id:'rver_after',version_sha256:'a'.repeat(64),body_sha256:'b'.repeat(64),title:'Public product',url,final_url:url,extraction_version:'static-v3',version_observed_at:stamp,text:'é🌲 value',start_char:0,end_char:8,full_length:20,next_start_char:8,omitted_before:false,omitted_after:true,window_origin:'page_start',changed_start_char:null,changed_end_char:null,...patch});
-const detail=(patch={})=>({id:'rev_one',collection_id:'rcol_one',url,kind:'baseline',observed_at:stamp,previous_observed_at:null,gap:false,comparison:'single_observation',budget:16000,text_bytes:12,before:null,after:side(),...patch});
-async function execute(value=detail(),options={},rest=['event','rcol_one','rev_one']){let stdout='',stderr='';const calls=[];const code=await researchCommand(rest,options,{request:async(route,params)=>{calls.push({route,params});if(value instanceof Error)throw value;return value}},{stdout:x=>stdout+=x,stderr:x=>stderr+=x});return {code,stdout,stderr,calls};}
-test('event command requires exactly two safe IDs and strict bounded numeric options',()=>{
- assert.equal(researchRequest(['event','rcol_one','rev_one'],{}).route,'/api/research/collections/rcol_one/events/rev_one?budget=16000');
- const request=researchRequest(['event','rcol_one','rev_one'],{budget:'256','before-start':'0','after-start':'25'});assert.equal(request.beforeStart,0);assert.equal(request.afterStart,25);assert.match(request.route,/before_start=0&after_start=25$/);
- for(const rest of [['event'],['event','rcol_one'],['event','rcol_one','rev_one','extra'],['event','rcol_one','../bad']])assert.throws(()=>researchRequest(rest,{}));
- for(const options of [{budget:'0'},{budget:'32001'},{budget:'1e3'},{budget:true},{'after-start':'-1'},{'after-start':'01'},{'before-start':'2000001'},{wait:true},{cursor:'x'},{'request-key':'key'},{refresh:true}])assert.throws(()=>researchRequest(['event','rcol_one','rev_one'],options));
-});
-test('event JSON is exact, plain observations label omissions and never fetch another page',async()=>{
- const json=await execute(detail(),{json:true});assert.deepEqual(JSON.parse(json.stdout),detail());assert.equal(json.calls.length,1);assert.equal(json.calls[0].params.method,undefined);assert.equal(json.calls[0].params.body,undefined);
- const human=await execute();assert.match(human.stdout,/Event: rev_one/);assert.match(human.stdout,/Codepoints: \[0, 8\) of 20/);assert.match(human.stdout,/é🌲 value/);assert.match(human.stderr,/--after-start 8/);assert.match(human.stderr,/not authored modifications/);assert.equal(human.calls.length,1);
-});
-test('bad event evidence emits no source text, including JSON mode',async()=>{
- for(const value of [detail({id:'foreign'}),detail({collection_id:'foreign'}),detail({budget:256}),detail({text_bytes:11}),detail({comparison:'equal_body'}),detail({after:side({end_char:9})}),detail({after:side({body_sha256:'bad'})}),detail({after:side({next_start_char:9})}),detail({after:side({omitted_after:false})}),detail({after:side({text:'\ud800',end_char:1})}),detail({text_bytes:20000,after:side({text:'🌲'.repeat(5000),end_char:5000,full_length:5000,next_start_char:null,omitted_after:false})})]){
-  let output='';await assert.rejects(researchCommand(['event','rcol_one','rev_one'],{json:true},{request:async()=>value},{stdout:s=>output+=s,stderr:()=>{}}),/invalid or out-of-scope/);assert.equal(output,'');
- }
-});
-test('explicit offsets include zero/end and absent sides fail closed',async()=>{
- const end=detail({text_bytes:0,after:side({text:'',start_char:20,end_char:20,next_start_char:null,omitted_before:true,omitted_after:false,window_origin:'explicit_offset'})});
- assert.equal((await execute(end,{'after-start':'20'})).code,0);
- await assert.rejects(execute(detail(),{'before-start':'0'}));await assert.rejects(execute(detail(),{'after-start':'1'}));
- const empty=detail({kind:'unavailable',before:null,after:null,text_bytes:0,comparison:'no_observation'});assert.match((await execute(empty)).stdout,/no retained observation/);
-});
-test('event terminal output escapes controls while JSON retains exact source strings',async()=>{
- const value=detail({gap:true,text_bytes:12,after:side({text:'é🌲 valu\u001b',title:'Title\u001b[2J'})});
- const human=await execute(value);assert.ok(!human.stdout.includes('\u001b'));assert.match(human.stdout,/\\u001b/);assert.match(human.stderr,/exact change time is unknown/);
- assert.equal(JSON.parse((await execute(value,{json:true})).stdout).after.text,value.after.text);
-});
-test('event failures do not disclose upstream detail or start fallback operations',async()=>{
- const result=await execute(Object.assign(new Error('PRIVATE_SOURCE_SECRET'),{status:403}),{json:true});assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.ok(!result.stdout.includes('PRIVATE_SOURCE_SECRET'));
-});
-const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
-function run(args,env,cwd){return new Promise((resolve,reject)=>{const child=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>child.kill('SIGKILL'),10000);child.stdout.on('data',x=>stdout+=x);child.stderr.on('data',x=>stderr+=x);child.on('error',reject);child.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr})})});}
-test('actual CLI event command retains selected account/origin and never edits credentials',async()=>{
- const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-event-')),calls=[];
- const server=http.createServer((req,res)=>{calls.push({url:req.url,method:req.method,authorization:req.headers.authorization});res.setHeader('content-type','application/json');res.end(JSON.stringify(detail()));});
- await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const origin='http://127.0.0.1:'+server.address().port;
- const env={...process.env};for(const name of Object.keys(env))if(name.startsWith('OPENCRAWL_'))delete env[name];env.OPENCRAWL_CONFIG_DIR=dir;
- const config=loggedInConfig({},origin,'synthetic-event-key',{user:{id:'usr_'+'c'.repeat(32)},workspace:{id:'ws_'+'d'.repeat(32)}});await fs.writeFile(path.join(dir,'config.json'),JSON.stringify(config),{mode:0o600});
- try{
-  const args=['research','event','rcol_one','rev_one','--account',config.activeProfile,'--json'];const result=await run(args,env,dir);assert.equal(result.code,0,result.stderr);assert.deepEqual(JSON.parse(result.stdout),detail());assert.deepEqual(calls,[{url:'/api/research/collections/rcol_one/events/rev_one?budget=16000',method:'GET',authorization:'Bearer synthetic-event-key'}]);
-  for(const extra of [['--server','https://foreign.example'],['--after-start','-1'],['--budget','256','--budget','32000']])assert.equal((await run([...args,...extra],env,dir)).code,1);
-  assert.equal(calls.length,1);assert.deepEqual(JSON.parse(await fs.readFile(path.join(dir,'config.json'),'utf8')),config);
- }finally{server.closeAllConnections();await new Promise(resolve=>server.close(resolve));await fs.rm(dir,{recursive:true,force:true});}
-});

package/src/research-presentation.test.mjs

@@ -1,56 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import fs from 'node:fs/promises';
-import {readResearchCompletion,readResearchPresentation} from './research-completion.mjs';
-import {agentCommand,agentRequest,validateAgentDescriptor,validateAgentRun,waitAgentRun} from './research-agent.mjs';
-const f=JSON.parse(await fs.readFile(new URL('./fixtures/research-completion-v2.json',import.meta.url),'utf8'));
-const old=JSON.parse(await fs.readFile(new URL('./fixtures/research-completion-hosted.json',import.meta.url),'utf8'));
-const initial=()=>({...structuredClone(old.runs.admitted),status:'running',phase:'presenting',usage:{completion:structuredClone(f.admitted)}});
-const brief={customer:{name:'Northlight',website:'https://northlight.example/',context:''},goal:'Find documented workflow changes.',collection_ids:[],schedule_minutes:0};
-
-test('v2 create/run selection is explicit and omitted request remains byte-identical',()=>{
- assert.equal(JSON.stringify(validateAgentDescriptor(brief)),JSON.stringify(brief));
- assert.deepEqual(validateAgentDescriptor({...brief,research_policy:'completion_v2'}),{...brief,research_policy:'completion_v2'});
- const args=['agent','run',initial().agent_id],options={revision:'1','request-key':'original-key-0000001'};
- assert.deepEqual(agentRequest(args,options).body,{expected_revision:1});
- assert.deepEqual(agentRequest(args,{...options,'research-policy':'completion_v2'}).body,{expected_revision:1,research_policy:'completion_v2'});
- for(const policy of ['completion_v4',null,false])assert.throws(()=>agentRequest(args,{...options,'research-policy':policy}));
-});
-test('CLI v2 readers preserve finite receipt and ignore malformed presentation',()=>{
- assert.deepEqual(readResearchCompletion(initial()).completion,f.admitted);
- assert.deepEqual(readResearchPresentation({usage:{presentation:f.presentation}}),f.presentation);
- for(const presentation of [{...f.presentation,raw:'PRIVATE'},{...f.presentation,calls_completed:2},{...f.presentation,known_output_tokens:12001}])assert.equal(readResearchPresentation({usage:{presentation}}),null);
- assert.deepEqual(validateAgentRun({...initial(),usage:{presentation:{status:'PRIVATE'}}}).phase,'presenting');
-});
-test('CLI default v2 wait passes300 seconds then expires once at540 from original start',async()=>{
- let clock=5000,calls=0;const original=initial();
- const result=await waitAgentRun({request:async(_route,options)=>{assert.ok(options.timeout<=30000);calls++;clock+=Math.min(25000,options.timeout);return initial()}},original,{now:()=>clock,sleep:async ms=>clock+=ms});
- assert.equal(result.wait.outcome,'timeout');assert.equal(calls,20);assert.equal(clock,545000); // original start plus540 seconds, never renewed
-});
-test('CLI explicit and malformed-policy waits keep old timeout and never replay writes',async()=>{
- for(const [usage,timeout,expected] of [[{},undefined,12],[{completion:{policy:'completion_v2'}},undefined,12],[{completion:f.admitted},300,12],[{completion:f.admitted},1,1]]){
-  let clock=0,calls=0;const run={...initial(),usage};
-  const result=await waitAgentRun({request:async(_route,options)=>{assert.equal(options.method,undefined);calls++;clock+=Math.min(25000,options.timeout);return run}},run,{...(timeout===undefined?{}:{timeout}),now:()=>clock,sleep:async ms=>clock+=ms});
-  assert.equal(result.wait.outcome,'timeout');assert.equal(calls,expected);
- }
-});
-test('human fallback is one fixed notice; raw JSON, all gaps, findings and quotes remain intact',async()=>{
- const run=structuredClone(old.runs.repaired);run.usage.presentation=structuredClone(f.presentation);run.gaps=Array.from({length:5},(_,i)=>`Original gap ${i+1}.`);
- const invoke=async json=>{let stdout='',stderr='',calls=0;const code=await agentCommand(['agent-run',run.agent_id,run.id],{json},{request:async()=>{calls++;return run}},{stdout:x=>stdout+=x,stderr:x=>stderr+=x});return {stdout,stderr,calls,code}};
- const json=await invoke(true);assert.deepEqual(JSON.parse(json.stdout),run);assert.equal(json.calls,1);
- const human=await invoke(false);assert.equal((human.stdout.match(/Summary:/g)||[]).length,1);
- for(const text of [...run.gaps,...run.findings.flatMap(v=>[v.claim,...v.qualifiers,...v.citations.map(c=>c.quote)])])assert.ok(human.stdout.includes(text));
- run.usage.presentation.status='skipped_no_eligible';assert.equal((await invoke(false)).stdout.includes('Summary:'),false);
- run.usage.presentation.raw='PRIVATE';assert.equal((await invoke(false)).stdout.includes('Summary:'),false);
-});
-
-test('actual admitted presenting reviewed and fallback public states are unchanged through CLI JSON',async()=>{
- const actual=JSON.parse(await fs.readFile(new URL('./fixtures/research-presentation-hosted.json',import.meta.url),'utf8'));
- assert.equal(actual.presenting.phase,'synthesizing');
- for(const run of Object.values(actual)){
-  let stdout='',calls=0;const before=JSON.stringify(run);
-  const code=await agentCommand(['agent-run',run.agent_id,run.id],{json:true},{request:async()=>{calls++;return run}},{stdout:x=>stdout+=x,stderr:()=>{}});
-  assert.equal(code,0);assert.deepEqual(JSON.parse(stdout),run);assert.equal(calls,1);assert.equal(JSON.stringify(run),before);
-  assert.deepEqual(readResearchCompletion(run).completion,run.usage.completion);assert.deepEqual(readResearchPresentation(run),run.usage.presentation??null);
- }
-});

package/src/research.mjs

@@ -1,141 +0,0 @@
-import fs from 'node:fs/promises';
-import {allowedOptions,decimal,identifier,settle,transient,withInterrupt} from './jobs.mjs';
-import {displayLine,displayText} from './display.mjs';
-import {coverageWarnings} from './context.mjs';
-import {validateEventEvidence,renderEventEvidence} from './research-event.mjs';
-import {agentCommand} from './research-agent.mjs';
-export const RESEARCH_HELP=`OpenCrawl · collect and search public research
-
-  opencrawl research agent create --file brief.json --request-key RANDOM_KEY [--wait]
-  opencrawl research agent list [--limit 25] [--cursor OPAQUE_CURSOR]
-  opencrawl research agent show AGENT_ID
-  opencrawl research agent run AGENT_ID --revision N --request-key RANDOM_KEY [--wait]
-  opencrawl research agent pause|resume AGENT_ID --revision N
-  opencrawl research agent receipt --operation create|run:AGENT_ID --request-key RANDOM_KEY
-  opencrawl research agent schedule AGENT_ID [show|daily|pause|resume] [--revision N]
-  opencrawl research agent repeats AGENT_ID RUN_ID
-  opencrawl research agent source AGENT_ID RUN_ID SOURCE_ID
-  opencrawl research agent feedback AGENT_ID RUN_ID FINDING_ID [show|set|delete] [--revision N] [--rating useful|irrelevant|incorrect|outdated] [--note TEXT]
-  opencrawl research agent runs AGENT_ID [--limit N] [--cursor CURSOR]
-  opencrawl research agent-run AGENT_ID RUN_ID [--wait] [--timeout 300]
-
-  opencrawl research create --file collection.json --request-key RANDOM_KEY [--wait] [--timeout 300]
-  opencrawl research list [--limit 20] [--cursor OPAQUE_CURSOR]
-  opencrawl research show COLLECTION_ID
-  opencrawl research refresh COLLECTION_ID --request-key RANDOM_KEY [--revision N] [--wait] [--timeout 300]
-  opencrawl research pause|resume COLLECTION_ID --revision N
-  opencrawl research configure COLLECTION_ID --file settings.json --request-key RANDOM_KEY
-  opencrawl research job JOB_ID [--wait] [--timeout 300]
-  opencrawl research search "question" --collection COLLECTION_ID [--limit 10] [--budget 6000]
-  opencrawl research events COLLECTION_ID [--limit 10] [--cursor OPAQUE_CURSOR] [--content-only]
-  opencrawl research event COLLECTION_ID EVENT_ID [--budget 16000] [--before-start N] [--after-start N]
-  opencrawl research feed [--customer NAME] [--after CURSOR] [--limit 50]   new change signals across the project
-  opencrawl research subscribe HTTPS_URL [--customer NAME]                 one signed webhook for every watch (secret shown once)
-  opencrawl research subscriptions | unsubscribe SUBSCRIPTION_ID
-
-Use --json for exact receipts. Create/refresh/configure never retry automatically: retain
-both the original JSON descriptor and a random 16–128 character request key.
-After a lost response, repeat the exact command with the same key to recover
-its original receipt. Changing the descriptor with that key fails closed.
---wait follows the accepted job only; Ctrl+C never cancels or resubmits it.
-Completed exits 0, partial exits 2, failed/cancelled/unconfirmed waits exit 1.
-Search reads the current index in one explicit collection; it never refreshes.
-Event reads retained source text only: budget is total UTF-8 source bytes
-(256–32000), offsets are codepoints. Continuation is always explicit.
-Configure requires exactly expected_revision, page_limit (1–5000),
-schedule_minutes (0 for manual, or 5–10080), and include_media (boolean).
-It preserves source scope/history and starts no immediate job. Positive cadence
-permits recurring work. Unfinished crawl/media work blocks changed settings.
-Baseline observations are not new events. Failed checks do not prove removal.
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-const ROOT='/api/research';
-const ACTIVE=new Set(['queued','running']);
-const STATES=new Set(['queued','running','completed','partial','failed','cancelled']);
-const finite=(n,min=0,max=Number.MAX_SAFE_INTEGER)=>Number.isSafeInteger(n)&&n>=min&&n<=max;
-const text=(value,max=200)=>typeof value==='string'&&value.length>0&&value.length<=max;
-export function researchKey(value){if(typeof value!=='string'||!(/^[A-Za-z0-9_-]{16,128}$/).test(value))throw new Error('Supply --request-key with a saved random 16–128 character key; reuse it only for this exact create, refresh or configure.');return value;}
-function pageQuery(options,defaultLimit=20,max=100){const limit=decimal(options.limit,'limit',1,max,defaultLimit),query=new URLSearchParams({limit});if(options.cursor!==undefined){if(!text(options.cursor,512))throw new Error('--cursor must be the exact bounded cursor returned by the server.');query.set('cursor',options.cursor)}return {limit,query};}
-export function researchRequest(rest,options){
- const [command,id,...extra]=rest;
- const allowed={create:['file','request-key','wait','timeout'],configure:['file','request-key'],list:['limit','cursor'],show:[],refresh:['request-key','revision','wait','timeout'],pause:['revision'],resume:['revision'],job:['wait','timeout'],search:['collection','limit','budget'],events:['limit','cursor','content-only'],event:['budget','before-start','after-start'],feed:['customer','after','limit'],subscribe:['customer'],subscriptions:[],unsubscribe:[]};
- if(!Object.hasOwn(allowed,command))throw new Error('Choose a research command from opencrawl research --help.');
- allowedOptions(options,allowed[command],'research');
- if(command==='event'?extra.length!==1:command!=='search'&&(extra.length||['create','list'].includes(command)&&id!==undefined))throw new Error('Unexpected research argument.');
- if(options.timeout!==undefined&&!options.wait)throw new Error('--timeout requires --wait.');
- const timeout=decimal(options.timeout,'timeout',1,3600,300);
- if(command==='create'){if(!text(options.file,4096))throw new Error('Provide --file with the collection JSON descriptor.');return {command,route:ROOT+'/collections',method:'POST',file:options.file,key:researchKey(options['request-key']),wait:!!options.wait,timeout};}
- if(command==='list'){const {limit,query}=pageQuery(options);return {command,limit,route:ROOT+'/collections?'+query};}
- if(command==='feed'){if(id!==undefined)throw new Error('Unexpected research argument.');const query=new URLSearchParams({after:decimal(options.after,'after',0,Number.MAX_SAFE_INTEGER,0),limit:decimal(options.limit,'limit',1,100,50)});if(options.customer!==undefined){if(!text(options.customer,120))throw new Error('--customer must be a customer name.');query.set('customer',options.customer);}return {command,route:ROOT+'/feed?'+query};}
- if(command==='subscribe'){if(!text(id,2048)||!/^https:\/\//.test(id))throw new Error('Give the public HTTPS endpoint to deliver to.');if(options.customer!==undefined&&!text(options.customer,120))throw new Error('--customer must be a customer name.');return {command,route:ROOT+'/subscriptions',method:'POST',body:{endpoint:id,...(options.customer!==undefined?{customer:options.customer}:{})}};}
- if(command==='subscriptions'){if(id!==undefined)throw new Error('Unexpected research argument.');return {command,route:ROOT+'/subscriptions'};}
- if(command==='unsubscribe'){if(!/^rsub_[0-9a-f]{32}$/.test(String(id)))throw new Error('Give the subscription ID from research subscriptions.');return {command,route:ROOT+'/subscriptions/'+id,method:'DELETE'};}
- if(command==='search'){const query=rest.slice(1).join(' ').trim();if(!query||Array.from(query).length>1000)throw new Error('Provide a nonempty question of at most 1,000 characters.');const collection=identifier(options.collection,'collection');return {command,collection,route:ROOT+'/search',method:'POST',body:{query,collection_ids:[collection],freshness:'indexed',limit:decimal(options.limit,'limit',1,50,10),budget:decimal(options.budget,'budget',0,32000,6000)}};}
- identifier(id,command==='job'?'job':'collection');
- const route=ROOT+(command==='job'?'/jobs/':'/collections/')+encodeURIComponent(id);
- if(command==='event'){
-  const eventId=identifier(extra[0],'event'),budget=decimal(options.budget,'budget',256,32000,16000),query=new URLSearchParams({budget});
-  const beforeStart=options['before-start']===undefined?undefined:decimal(options['before-start'],'before-start',0,2_000_000);
-  const afterStart=options['after-start']===undefined?undefined:decimal(options['after-start'],'after-start',0,2_000_000);
-  if(beforeStart!==undefined)query.set('before_start',beforeStart);if(afterStart!==undefined)query.set('after_start',afterStart);
-  return {command,id,eventId,budget,beforeStart,afterStart,route:route+'/events/'+encodeURIComponent(eventId)+'?'+query};
- }
- if(command==='configure'){if(!text(options.file,4096))throw new Error('Provide --file with all four reviewed settings.');return {command,id,route:route+'/configure',method:'POST',file:options.file,key:researchKey(options['request-key'])};}
- if(command==='events'){const {limit,query}=pageQuery(options,10,25);if(options['content-only'])query.set('include_volatile','false');return {command,id,limit,route:route+'/events?'+query};}
- if(command==='refresh')return {command,id,route:route+'/refresh',method:'POST',body:options.revision===undefined?{}:{expected_revision:decimal(options.revision,'revision',1,Number.MAX_SAFE_INTEGER)},key:researchKey(options['request-key']),wait:!!options.wait,timeout};
- if(command==='pause'||command==='resume')return {command,id,route,method:'PATCH',body:{status:command==='pause'?'paused':'active',expected_revision:decimal(options.revision,'revision',1,Number.MAX_SAFE_INTEGER)}};
- return {command,id,route,wait:!!options.wait,timeout};
-}
-export function validateResearchJob(value,id=value?.id){if(!value||!text(value.id)||value.id!==id||!text(value.collection_id)||!STATES.has(value.status)||!['discovery','fetching','finished'].includes(value.phase)||!value.progress||!['discovered','fetched','indexed','failed','bytes'].every(key=>finite(value.progress[key])))throw new Error('Server returned an invalid research job receipt. No completion was assumed.');return value;}
-export function validateResearchCollection(value,id=value?.id){if(!value||!text(value.id)||value.id!==id||!text(value.source_id)||!text(value.name,200)||!['active','paused'].includes(value.status)||!finite(value.revision,1)||!Array.isArray(value.seed_urls)||!Array.isArray(value.allowed_origins)||!value.stats||!['discovered','indexed','failed','bytes'].every(key=>finite(value.stats[key])))throw new Error('Server returned an invalid research collection.');return value;}
-function validatePage(value,limit){if(!value||!Array.isArray(value.items)||value.items.length>limit||!(value.next_cursor===null||text(value.next_cursor,512)))throw new Error('Server returned an invalid research page.');return value;}
-function validateEvents(value,request){validatePage(value,request.limit);for(const item of value.items){if(!item||!text(item.id)||item.collection_id!==request.id||!['baseline','changed','unavailable','recovered'].includes(item.kind)||typeof item.url!=='string'||typeof item.observed_at!=='string'||!item.change_summary||typeof item.gap!=='boolean')throw new Error('Server returned invalid or out-of-scope research observations.');for(const side of ['before','after'])if(item[side]!==null&&(!item[side]||typeof item[side].excerpt!=='string'||Array.from(item[side].excerpt).length>4000))throw new Error('Server returned an oversized observation excerpt.');}return value;}
-function validateSearch(value,request){const expected=request.body.query.split(/[\s\u0085\u001c-\u001f]+/u).filter(Boolean).join(' ');if(!value||value.query!==expected||!Array.isArray(value.results)||value.results.length>request.body.limit||typeof value.context!=='string'||Buffer.byteLength(value.context)>request.body.budget||!Array.isArray(value.evidence?.citations)||Boolean(value.context)!==Boolean(value.evidence.citations.length)||!value.freshness||!['indexed','empty'].includes(value.freshness.status)||!Array.isArray(value.freshness.refresh_jobs)||value.freshness.refresh_jobs.length)throw new Error('Server returned an invalid indexed research response; no context was emitted.');for(const result of value.results)if(!text(result.document_id)||!text(result.title,2000)||typeof result.source_id!=='string'||result.research_observation&&result.research_observation.collection_id!==request.collection)throw new Error('Server returned invalid or out-of-scope research citations.');for(const citation of value.evidence.citations)if(!value.results.some(result=>result.document_id===citation.document_id))throw new Error('Server returned a context citation outside the search results.');return value;}
-export function researchFailure(error,request){const status=Number.isInteger(error?.status)?error.status:undefined;const fixed={401:'Sign in with a valid key for the intended workspace.',403:'This account cannot perform the requested research operation.',404:'This collection, event or job is unavailable. Check the selected workspace and exact identifier. Watches, jobs and research built by the OpenCrawl agent belong to their project: use that project\'s key (project.api_key from the build), not the organization key.',409:'The collection or request changed, or work remains unfinished. Review its current revision and jobs; preserve the original request key and descriptor before explicitly retrying.',422:'The research selection or descriptor is invalid. Check the approved origins, limits and explicit identifiers.',429:'OpenCrawl is busy or a workspace limit was reached. Check the collection before retrying.'};const message=fixed[status]||'The request outcome could not be confirmed. Recover create/refresh/configure with the same request key and exact descriptor; never start a duplicate blindly.';return {code:status?`http_${status}`:'request_unconfirmed',message,...(status?{status}:{}),...(request.key?{request_key:request.key}:{}),...(request.id?{id:request.id}:{})};}
-export function validateResearchSettings(value){
- const fields=['expected_revision','page_limit','schedule_minutes','include_media'];
- if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).length!==fields.length||fields.some(key=>!Object.hasOwn(value,key))||!finite(value.expected_revision,1)||!finite(value.page_limit,1,5000)||!(value.schedule_minutes===0||finite(value.schedule_minutes,5,10080))||typeof value.include_media!=='boolean')throw new Error('Settings JSON must contain exactly expected_revision (positive integer), page_limit (1–5000), schedule_minutes (0 or 5–10080), and include_media (boolean).');
- return value;
-}
-async function settingsDescriptor(file){
- const handle=await fs.open(file,'r');
- try{
-  const size=(await handle.stat()).size;if(!finite(size,1,256*1024))throw new Error('Settings JSON must be between 1 byte and 256 KiB.');
-  const bytes=Buffer.alloc(size+1),{bytesRead}=await handle.read(bytes,0,size+1,0);if(bytesRead!==size)throw new Error('The settings descriptor changed while being read.');
-  let value;try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes.subarray(0,size)))}catch{throw new Error('Invalid settings JSON. Supply the original valid UTF-8 settings descriptor.');}
-  return validateResearchSettings(value);
- }finally{await handle.close()}
-}
-export function validateConfigureReceipt(value,request){
- const collection=validateResearchCollection(value?.collection,request.id),settings=request.body;
- const utc=value.next_scheduled_at===null||typeof value.next_scheduled_at==='string'&&/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z$/.test(value.next_scheduled_at)&&Number.isFinite(Date.parse(value.next_scheduled_at))&&new Date(value.next_scheduled_at).toISOString().slice(0,19)===value.next_scheduled_at.slice(0,19);
- if(Object.keys(value).some(key=>!['collection','next_scheduled_at','replayed'].includes(key))||typeof value.replayed!=='boolean'||!utc||!['page_limit','schedule_minutes','include_media'].every(key=>collection[key]===settings[key])||![settings.expected_revision,settings.expected_revision+1].includes(collection.revision)||!collection.seed_urls.length||collection.seed_urls.length>50||!collection.allowed_origins.length||collection.allowed_origins.length>20||![...collection.seed_urls,...collection.allowed_origins].every(url=>text(url,2048))||((collection.status==='paused'||settings.schedule_minutes===0)&&value.next_scheduled_at!==null)||(collection.revision!==settings.expected_revision&&collection.status==='active'&&settings.schedule_minutes>0&&value.next_scheduled_at===null))throw new Error('The settings receipt could not be verified. Retain the original request key and descriptor.');
- return value;
-}
-async function descriptor(file){const handle=await fs.open(file,'r');try{const size=(await handle.stat()).size;if(!finite(size,1,256*1024))throw new Error('Collection JSON must be between 1 byte and 256 KiB.');const bytes=Buffer.alloc(size+1);const {bytesRead}=await handle.read(bytes,0,size+1,0);if(bytesRead!==size)throw new Error('The collection descriptor changed while being read.');let value;try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes.subarray(0,size)))}catch{throw new Error('Invalid collection JSON. Supply a valid UTF-8 collection descriptor.');}const allowed=['name','brief','seed_urls','allowed_origins','sitemap_urls','page_limit','schedule_minutes','include_media'];if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).some(key=>!allowed.includes(key))||!text(value.name,200)||!Array.isArray(value.seed_urls)||!value.seed_urls.length||value.seed_urls.length>50)throw new Error('Collection JSON must contain name and 1–50 seed_urls, with only documented collection settings.');return value;}finally{await handle.close()}}
-export async function waitResearch(client,initial,{timeout=300,signal,now=()=>performance.now(),sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms))}={}){let job=validateResearchJob(initial),failures=0;const deadline=now()+timeout*1000;while(ACTIVE.has(job.status)){if(signal?.aborted)return {job,wait:{outcome:'interrupted'}};if(now()>=deadline)return {job,wait:{outcome:'timeout'}};try{job=validateResearchJob(await client.request(ROOT+'/jobs/'+encodeURIComponent(job.id),{timeout:Math.max(1,Math.min(30000,Math.ceil(deadline-now()))),signal,maxResponseBytes:1024*1024}),job.id);failures=0}catch(error){if(!signal?.aborted&&now()<deadline&&transient(error)){await settle(++failures,{signal,now,sleep,deadline});continue;}return {job,wait:{outcome:signal?.aborted?'interrupted':now()>=deadline?'timeout':'unavailable',error:researchFailure(error,{id:job.id})}}}if(!ACTIVE.has(job.status))break;const next=Math.min(deadline,now()+2000);while(!signal?.aborted&&now()<next)await sleep(Math.min(100,next-now()));}return {job,wait:{outcome:job.status}};}
-const renderJob=value=>`${displayLine(value.id)}\t${displayLine(value.status)}\t${value.progress.indexed} indexed, ${value.progress.failed} unsuccessful, ${value.progress.discovered} discovered\n`;
-export async function researchCommand(rest,options,client,{stdout=value=>process.stdout.write(value),stderr=value=>process.stderr.write(value)}={}){
- if(['agent','agent-run'].includes(rest[0]))return agentCommand(rest,options,client,{stdout,stderr});
- const request=researchRequest(rest,options);if(request.file)request.body=await (request.command==='configure'?settingsDescriptor(request.file):descriptor(request.file));
- let value;try{value=await client.request(request.route,{method:request.method,body:request.body,idempotencyKey:request.key,maxResponseBytes:4*1024*1024});}catch(error){const failure=researchFailure(error,request);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+(request.key?`\nOriginal request key: ${request.key}`:'')+'\n');return 1;}
- if(['feed','subscribe','subscriptions','unsubscribe'].includes(request.command)){stdout(JSON.stringify(value,null,2)+'\n');if(request.command==='subscribe')stderr('Save secret now: it is shown only once. Deliveries are signed (Webhook-Signature, verifyResearchWebhook in @getopencrawl/sdk).\n');if(request.command==='feed'&&value.more)stderr('More signals: continue with --after '+value.next_cursor+'\n');return 0;}
- if(request.command==='event'){validateEventEvidence(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else renderEventEvidence(value,stdout,stderr);return 0;}
- if(request.command==='configure'){
-  try{validateConfigureReceipt(value,request)}catch(error){const failure=researchFailure(error,request);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+'\nOriginal request key: '+request.key+'\n');return 1;}
-  if(options.json)stdout(JSON.stringify(value,null,2)+'\n');
-  else{stdout(`Collection: ${displayLine(value.collection.id)}\nRevision: ${value.collection.revision}\nPage limit: ${value.collection.page_limit}\nSchedule: ${value.collection.schedule_minutes===0?'manual':value.collection.schedule_minutes+' minutes'}\nMedia: ${value.collection.include_media?'enabled':'disabled'}\nNext scheduled check: ${value.next_scheduled_at??'none'}\n`);stderr('No immediate crawl was started. Source scope and evidence history are preserved.\n'+(value.replayed?'Recovered the original receipt; read current state before another change.\n':'')+'Original request key: '+request.key+'\n');}
-  return 0;
- }
- if(request.command==='list'){validatePage(value,request.limit);value.items.forEach(item=>validateResearchCollection(item));if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(value.items.map(item=>`${displayLine(item.id)}\tr${item.revision}\t${item.stats.indexed} indexed\t${displayLine(item.name)}\n`).join('')||'No research collections on this page.\n');if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');}return 0;}
- if(request.command==='events'){validateEvents(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{for(const event of value.items){stdout(`${displayLine(event.id)}\t${displayLine(event.kind)}\t${displayLine(event.observed_at)}\t${displayLine(event.url)}\n`);for(const side of ['before','after'])if(event[side]){stdout(`${side==='before'?'Previous observation':'This observation'}: ${displayLine(event[side].title)}\n${displayText(event[side].excerpt)}\n`);if(event[side].excerpt_truncated)stderr(`${side==='before'?'Previous':'Current'} observation excerpt is shortened.\n`);}const summary=event.change_summary;if(Array.isArray(summary.changes)){if(summary.substance==='volatile_only')stdout('Only dates, clock times, relative ages or whitespace changed.\n');for(const hunk of summary.changes.slice(0,20)){if(!hunk||!Array.isArray(hunk.removed)||!Array.isArray(hunk.added))continue;for(const line of hunk.removed.slice(0,20))stdout('- '+displayLine(String(line))+'\n');for(const line of hunk.added.slice(0,20))stdout('+ '+displayLine(String(line))+'\n');}if(summary.changes_truncated)stderr('Changed lines are shortened.\n');}if(event.gap)stderr('Unsuccessful checks separate these observations; the exact change time is unknown.\n');if(event.change_summary.truncated)stderr('Observation excerpts or differences are shortened.\n');}if(!value.items.length)stdout('No retained observations on this page.\n');if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');stderr('Baselines are not new business events. Unavailable checks do not prove removal. Times are observations, not authored modifications.\n');}return 0;}
- if(request.command==='search'){validateSearch(value,request);if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(value.context);stderr(`${value.evidence.citations.length} context citations from ${value.results.length} matching results in the explicitly selected collection.\n`);if(!value.results.length)stderr('No matching indexed evidence; no broader search or refresh was attempted.\n');for(const warning of coverageWarnings(value))stderr(displayLine(warning)+'\n');}return 0;}
- if(['show','pause','resume'].includes(request.command)){validateResearchCollection(value,request.id);stdout(options.json?JSON.stringify(value,null,2)+'\n':`${displayLine(value.name)}\nCollection: ${displayLine(value.id)}\nRevision: ${value.revision}\nStatus: ${value.status}\nIndexed pages: ${value.stats.indexed}\nApproved origins: ${value.allowed_origins.map(displayLine).join(', ')}\n`);return 0;}
- let collection,job;
- if(request.command==='create'){collection=validateResearchCollection(value.collection);job=validateResearchJob(value.job);if(job.collection_id!==collection.id)throw new Error('Collection receipt and job do not match.');}
- else if(request.command==='refresh'){job=validateResearchJob(value.job);if(job.collection_id!==request.id)throw new Error('Refresh receipt belongs to another collection.');}
- else job=validateResearchJob(value,request.id);
- let receipt=request.command==='job'?{job}:{...value};if(request.wait){const waited=await withInterrupt(signal=>waitResearch(client,job,{timeout:request.timeout,signal}));job=waited.job;receipt={...receipt,...waited};}
- if(options.json)stdout(JSON.stringify(receipt,null,2)+'\n');else{if(collection)stdout('Collection: '+displayLine(collection.id)+'\n');stdout(renderJob(job));if(request.key)stderr('Original request key: '+request.key+'\n');if(receipt.wait&&!['completed','partial'].includes(receipt.wait.outcome))stderr(`Wait outcome: ${receipt.wait.outcome}. Follow the same job with research job ${displayLine(job.id)} --wait.\n`);stderr('Job counts describe bounded configured coverage, not the entire website. Media discovery is not successful parsing.\n');}
- return request.wait?receipt.wait.outcome==='completed'?0:receipt.wait.outcome==='partial'?2:1:request.command==='job'&&!ACTIVE.has(job.status)?job.status==='completed'?0:job.status==='partial'?2:1:0;
-}