@getopencrawl/cli 0.1.0 → 0.2.0

package/src/product-monitors.mjs

@@ -1,313 +0,0 @@
-/** Private local browser observations. No browser state is sent to OpenCrawl. */
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { createInterface } from 'node:readline/promises';
-import { configDir } from './client.mjs';
-import { isReadOnlyRequest } from '../browser.mjs';
-import { displayLine } from './display.mjs';
-import { allowedOptions } from './jobs.mjs';
-import {LETTER_ADAPTER,isLetterPolicy,LetterReadAttempt} from './product-letter-read.mjs';
-import {LETTER_CAPABILITIES_ADAPTER,isLetterCapabilitiesPolicy,letterPickerReady,extractLetterCapabilities} from './product-letter-capabilities.mjs';
-
-const API='/api/v1/product-monitors';
-const UUID=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
-const HASH=/^[0-9a-f]{64}$/;
-const sha=value=>crypto.createHash('sha256').update(value).digest('hex');
-const same=(a,b)=>canonical(a)===canonical(b);
-export function canonical(value) {return JSON.stringify(value,(_,v)=>v&&typeof v==='object'&&!Array.isArray(v)?Object.fromEntries(Object.keys(v).sort().map(k=>[k,v[k]])):v);}
-export class MonitorFailure extends Error {constructor(code){super(code);this.code=code;}}
-const fail=code=>{throw new MonitorFailure(code);};
-const id=(value,prefix)=>{if(typeof value!=='string'||!new RegExp(`^${prefix}_[a-f0-9]{32}$`).test(value))fail('invalid_id');return value;};
-const monitorId=value=>{if(typeof value!=='string'||!/^src_[a-f0-9]{32}$/.test(value))fail('invalid_id');return value;};
-const uuid=value=>{if(typeof value!=='string'||!UUID.test(value))fail('invalid_capture_id');return value;};
-const count=(value,min,max)=>Number.isSafeInteger(value)&&value>=min&&value<=max;
-export function exactUrl(value) {
-  if(typeof value!=='string'||value.length>2048||/[\x00-\x20\x7f\\]/.test(value)||/%0[0-9a-f]|%1[0-9a-f]|%7f/i.test(value))fail('invalid_url');
-  let u;try{u=new URL(value);}catch{fail('invalid_url');}
-  if(u.protocol!=='https:'||u.username||u.password||u.href!==value)fail('invalid_url');
-  let pathname;try{pathname=decodeURIComponent(u.pathname);}catch{fail('invalid_url');}
-  if(/(?:^|\/)(?:logout|log-out|signout|sign-out|delete|remove|destroy|unsubscribe|deactivate|terminate)(?:\/|$)/i.test(pathname)||[...u.searchParams.keys()].some(k=>/^(action|delete|remove|logout|unsubscribe)$/i.test(k)))fail('action_url');
-  return u;
-}
-export function readMonitor(m,expected) {
-  if(!m||m.version!==1||monitorId(m.id)!==m.id||expected&&m.id!==expected||m.visibility!=='private'||!count(m.generation,1,Number.MAX_SAFE_INTEGER)||!['active','paused','revoked'].includes(m.status)||!Array.isArray(m.pages)||!count(m.pages.length,1,5)||!HASH.test(m.policy_sha256)||typeof m.account_label!=='string')fail('invalid_monitor');
-  const p=m.policy,v=p?.viewport;
-  if(sha(canonical(p))!==m.policy_sha256)fail('invalid_policy');
-  if(!(p?.version===1&&!Object.hasOwn(p,'request_adapter')||p?.version===2&&p.request_adapter===LETTER_ADAPTER||p?.version===3&&p.request_adapter===LETTER_CAPABILITIES_ADAPTER)||p.extractor!=='product-screen-v1'||p.screenshot_mode!=='viewport'||!count(v?.width,320,1920)||!count(v?.height,240,1080)||v.device_scale_factor!==1||!['light','dark'].includes(p.color_scheme)||typeof p.locale!=='string'||typeof p.timezone_id!=='string'||![0,60,360,1440].includes(m.cadence_minutes))fail('invalid_policy');
-  const selector=s=>typeof s==='string'&&s.length>0&&s.length<=400&&!/[\x00-\x1f\x7f]/.test(s);
-  for(const rows of [p.masks,p.noise_regions])if(!Array.isArray(rows)||rows.length>16||rows.some(r=>!selector(r?.selector)||typeof r.label!=='string'))fail('invalid_policy');
-  const seen=new Set();for(const page of m.pages){id(page.id,'ppg');if(seen.has(page.id))fail('invalid_monitor');seen.add(page.id);if(exactUrl(page.url).origin!==m.origin||!selector(page.ready_selector)||!selector(page.scope_selector)||page.login_selector!==null&&!selector(page.login_selector)||page.account_marker!==null&&(!selector(page.account_marker?.selector)||page.account_marker.required!==true))fail('invalid_monitor');}
-  if(p.version>=2&&(m.pages.length!==1||!(isLetterPolicy(p,m.pages[0])||isLetterCapabilitiesPolicy(p,m.pages[0]))))fail('invalid_policy');
-  return m;
-}
-async function request(client,route,options={}) {return client.request(route,{timeout:30000,maxResponseBytes:1024*1024,...options});}
-async function identity(client,signal) {
-  const me=await request(client,'/api/auth/me',{signal});
-  if(!/^usr_[0-9a-f]{32}$/.test(me?.user?.id)||!/^ws_[0-9a-f]{32}$/.test(me?.workspace?.id))fail('identity_unavailable');
-  if(client.profile&&(client.profile.userId!==me.user.id||client.profile.workspaceId!==me.workspace.id))fail('identity_changed');
-  return {server:client.baseUrl,user_id:me.user.id,workspace_id:me.workspace.id};
-}
-async function current(client,monitor,scope,signal) {
-  const who=await identity(client,signal);if(scope&&!same(who,scope))fail('identity_changed');
-  const m=readMonitor(await request(client,`${API}/${monitor.id}`,{signal}),monitor.id);
-  if(m.generation!==monitor.generation||m.policy_sha256!==monitor.policy_sha256||m.status!=='active')fail('monitor_changed');return m;
-}
-async function privateDir(directory){await fs.mkdir(directory,{recursive:true,mode:0o700});if((await fs.lstat(directory)).isSymbolicLink())fail('unsafe_local_path');await fs.chmod(directory,0o700);}
-async function write(file,value,beforeCommit=()=>{}){const temp=file+'.'+crypto.randomUUID()+'.tmp';try{const h=await fs.open(temp,'wx',0o600);try{await h.writeFile(JSON.stringify(value)+'\n');await h.sync();}finally{await h.close();}beforeCommit();await fs.rename(temp,file);if(process.platform!=='win32'){const d=await fs.open(path.dirname(file),'r');try{await d.sync();}finally{await d.close();}}}finally{await fs.unlink(temp).catch(e=>{if(e.code!=='ENOENT')throw e;});}}
-async function read(file,max=4*1024*1024){try{const h=await fs.open(file,'r');try{const s=await h.stat();if(!s.isFile()||s.size>max)fail('invalid_local_state');return JSON.parse(await h.readFile('utf8'));}finally{await h.close();}}catch(e){if(e.code==='ENOENT')return null;throw e;}}
-async function lock(directory,fn){await privateDir(directory);let h;try{h=await fs.open(path.join(directory,'lock'),'wx',0o600);}catch(e){if(e.code==='EEXIST')fail('profile_locked');throw e;}try{await h.writeFile(String(process.pid));return await fn();}finally{await h.close();await fs.unlink(path.join(directory,'lock'));}}
-function profilePath(name){if(typeof name!=='string'||!/^[A-Za-z0-9_-]{1,64}$/.test(name))fail('invalid_profile');return path.join(configDir(),'product-monitor-profiles',name);}
-function operationPath(scope,monitor){return path.join(configDir(),'product-monitor-operations',sha(canonical({...scope,monitor})));}
-function checkUpload(u,m) {
-  if(u.outcome==='captured'){
-    const png=Buffer.from(u.screenshot_png_base64,'base64');
-    if(Buffer.byteLength(u.text)>131072||png.length>2097152||png.length<24||!png.subarray(0,8).equals(Buffer.from([137,80,78,71,13,10,26,10]))||png.readUInt32BE(16)!==m.policy.viewport.width||png.readUInt32BE(20)!==m.policy.viewport.height)fail('capture_too_large_or_invalid');
-    if(Array.from(u.title).length>256||Buffer.byteLength(u.title)>1024)fail('title_too_large');
-  }
-  if(Buffer.byteLength(JSON.stringify(u))>3145728)fail('capture_too_large');return u;
-}
-export function receipt(value,u,monitor) {
-  if(value?.version!==1||id(value.id,'pcap')!==value.id||value.client_capture_id!==u.client_capture_id||value.monitor_id!==monitor||value.page_id!==u.page_id||value.generation!==u.expected_generation||value.outcome!==u.outcome||value.observed_at!==u.observed_at||value.manifest_sha256!==sha(canonical(u)))fail('receipt_mismatch');
-  if(u.outcome==='captured'){
-    const s=value.snapshot;id(s?.id,'psnap');
-    if(!HASH.test(s.version)||s.text_sha256!==sha(u.text)||s.text_bytes!==Buffer.byteLength(u.text)||s.screenshot_sha256!==sha(Buffer.from(u.screenshot_png_base64,'base64'))||s.screenshot_bytes!==Buffer.from(u.screenshot_png_base64,'base64').length||s.policy_sha256!==u.policy_sha256)fail('receipt_mismatch');
-  }else if(value.snapshot!==null||value.comparison!==null)fail('receipt_mismatch');
-  return value;
-}
-export async function sendCapture(client,monitor,u,directory,signal){
-  const file=path.join(directory,u.client_capture_id+'.json');
-  if(await read(file))fail('capture_already_attempted');
-  await unresolved(directory,Buffer.byteLength(JSON.stringify({version:1,state:'intended',monitor,upload:u}))+1);
-  await write(file,{version:1,state:'intended',monitor,upload:u});
-  let result;
-  try{result=await request(client,`${API}/${monitor}/captures`,{method:'POST',body:u,signal});}
-  catch{try{result=await request(client,`${API}/${monitor}/captures/by-client-id/${u.client_capture_id}`,{signal});}catch{fail('upload_unconfirmed');}}
-  result=receipt(result,u,monitor);await write(file,{version:1,state:'confirmed',monitor,manifest_sha256:result.manifest_sha256,receipt:result});return result;
-}
-async function unresolved(directory,additionalBytes=0){const names=await fs.readdir(directory);if(names.length+(additionalBytes?2:0)>256)fail('local_history_full');let bytes=additionalBytes*2;for(const name of names){bytes+=(await fs.lstat(path.join(directory,name))).size;if(bytes>64*1024*1024)fail('local_history_full');if(!UUID.test(name.slice(0,-5))||!name.endsWith('.json'))continue;const record=await read(path.join(directory,name));if(record?.state!=='confirmed')fail('upload_unconfirmed');}}
-export async function loginPrompt(message,{signal,input=process.stdin,output=process.stderr}={}){if(!input.isTTY)fail('interactive_login_required');const r=createInterface({input,output});try{await r.question(message,{signal});}catch(error){if(error?.name==='AbortError')fail('cancelled');throw error;}finally{r.close();}}
-async function browser(overrides){try{return overrides.playwright||await import('playwright');}catch{fail('browser_setup_required');}}
-async function launch(pw,directory,m,headed=false,timeout){
-  const context=await pw.chromium.launchPersistentContext(path.join(directory,'chromium'),{headless:!headed,acceptDownloads:false,permissions:[],serviceWorkers:'block',viewport:{width:m.policy.viewport.width,height:m.policy.viewport.height},deviceScaleFactor:1,locale:m.policy.locale,timezoneId:m.policy.timezone_id,colorScheme:m.policy.color_scheme,reducedMotion:'reduce',...(timeout===undefined?{}:{timeout})});
-  if(!headed){try{await context.route('**/*',route=>route.abort('blockedbyclient'));if(typeof context.routeWebSocket!=='function')fail('browser_version_unsupported');await context.routeWebSocket('**/*',socket=>socket.close());for(const page of context.pages())await page.close();}catch(error){await context.close();throw error;}}
-  return context;
-}
-
-/** A Boolean readiness hint only; headless capture remains authoritative. */
-export function loginScreenReady({screen,expected,capabilities=false}) {
-  if(location.href!==screen.url||!screen.account_marker||typeof expected!=='string'||!expected)return false;
-  const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
-  const all=s=>Array.from(document.querySelectorAll(s)).filter(visible);
-  const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
-  if(screen.login_selector&&all(screen.login_selector).length)return false;
-  return Boolean(one(screen.ready_selector)&&(capabilities||one(screen.scope_selector))&&one(screen.account_marker.selector)?.innerText?.trim()===expected);
-}
-function loginActive(signal){if(signal.aborted)throw signal.reason instanceof MonitorFailure?signal.reason:new MonitorFailure('cancelled');}
-export async function waitForLogin(page,screen,markers,signal,manual=loginPrompt,capabilities=false){
-  const waiting=new AbortController();const abort=()=>waiting.abort(signal.reason);signal.addEventListener('abort',abort,{once:true});
-  try{
-    loginActive(signal);
-    const automatic=(async()=>{while(true){
-      loginActive(waiting.signal);if(page.isClosed?.())fail('login_window_closed');
-      if(screen.account_marker&&page.url()===screen.url){
-        try{if(await page.evaluate(loginScreenReady,{screen,expected:markers[screen.id],...(capabilities?{capabilities:true}:{})})===true)return;}
-        catch{loginActive(waiting.signal);if(page.isClosed?.())fail('login_window_closed');} // Normal authentication navigation can replace the DOM.
-      }
-      await sleep(1000,waiting.signal);
-    }})();
-    await Promise.race([automatic,Promise.resolve().then(()=>manual('Sign in in the browser; we\u2019ll continue when ready. If needed, return here and press Enter. (10-minute limit; Ctrl+C cancels.) ',{signal:waiting.signal}))]);
-    loginActive(signal);
-  }finally{waiting.abort();signal.removeEventListener('abort',abort);}
-}
-
-/** Runs synchronously in the page. Neither hidden text nor excluded text is returned. */
-export function extractScreen({screen,policy,expected,freeze=false}) {
-  const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
-  const all=s=>Array.from(document.querySelectorAll(s)).filter(visible);
-  const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
-  const login=screen.login_selector&&all(screen.login_selector).length>0;
-  if(login)return {error:'session_expired',login:true};
-  if(!one(screen.ready_selector))return {error:'not_ready'};
-  const scope=one(screen.scope_selector);if(!scope)return {error:'scope_mismatch'};
-  let marker=null,markerNode=null;
-  if(screen.account_marker){const node=one(screen.account_marker.selector);markerNode=node;marker=node?.innerText?.trim();if(!marker||typeof expected!=='string'||marker!==expected)return {error:'context_mismatch'};}
-  const selectors=[...policy.masks,...policy.noise_regions].map(r=>r.selector);
-  const masks=[];for(const s of selectors){const a=all(s);if(!a.length||a.length>100)return {error:'mask_unresolved'};masks.push(...a);}
-  masks.push(...document.querySelectorAll('input,textarea,select,[contenteditable="true"]'));
-  if(markerNode)masks.push(markerNode); // Only the marker digest is eligible for upload.
-  const excluded=node=>masks.some(m=>m===node||m.contains(node));
-  let nodes=0;
-  const walk=node=>{if(++nodes>20000)throw Error('bounds');if(node.nodeType===Node.TEXT_NODE)return node.textContent;if(node.nodeType!==Node.ELEMENT_NODE)return '';if(excluded(node)||!visible(node)||['SCRIPT','STYLE','NOSCRIPT','TEMPLATE'].includes(node.tagName))return '';return Array.from(node.childNodes).map(walk).join('')+(/^(P|DIV|SECTION|ARTICLE|H[1-6]|LI|TR|BR)$/.test(node.tagName)?'\n':' ');};
-  const text=walk(scope).replace(/[ \t]+/g,' ').replace(/ *\n */g,'\n').trim();
-  if(!text)return {error:'not_ready'};
-  if(freeze){
-    // Hide the entire composited subtree; descendants cannot override parent opacity.
-    // This happens before observation, and no unmasked screenshot is ever taken.
-    for(const node of masks)node.style.setProperty('opacity','0','important');
-    window.__opencrawlMasks=masks;window.__opencrawlChanged=false;
-    window.__opencrawlObserver?.disconnect();
-    window.__opencrawlObserver=new MutationObserver(rows=>{if(rows.some(r=>!excluded(r.target)||r.type==='attributes'&&masks.includes(r.target)))window.__opencrawlChanged=true;});
-    window.__opencrawlObserver.observe(document.documentElement,{attributes:true,childList:true,characterData:true,subtree:true});
-  }
-  return {text,title:screen.label,marker,maskCount:masks.length};
-}
-export async function captureScreen(context,m,screen,binding,{signal,onDiagnostics}={}) {
-  if(m.policy.version!==1&&!isLetterPolicy(m.policy,screen)&&!isLetterCapabilitiesPolicy(m.policy,screen))fail('invalid_policy');
-  const page=await context.newPage();page.on('popup',popup=>popup.close().catch(()=>{}));let blocked=0,captureGuard='not_reached';const outerBlocks={navigation_scope:0,approved_read_redirect:0,non_read:0};const outerBlock=kind=>{outerBlocks[kind]=Math.min(64,outerBlocks[kind]+1);};
-  const base={version:1,client_capture_id:crypto.randomUUID(),page_id:screen.id,expected_generation:m.generation,policy_sha256:m.policy_sha256,expected_previous_snapshot_id:screen.latest_snapshot?.id||null,session_epoch:binding.session_epoch,observed_at:new Date().toISOString(),requested_url:screen.url,final_url:null,outcome:'capture_failed',account:{label:m.account_label,assurance:'declared_only',marker_sha256:null},ready_matched:false,login_marker_matched:false,title:null,text:null,screenshot_png_base64:null};
-  const capabilities=isLetterCapabilitiesPolicy(m.policy,screen),deadline=performance.now()+60000;
-  const adapter=isLetterPolicy(m.policy,screen)||capabilities?new LetterReadAttempt(page,deadline,capabilities):null;
-  const abort=()=>{if(capabilities)adapter.close();return page.close().catch(()=>{});};signal?.addEventListener('abort',abort,{once:true});const timer=setTimeout(abort,60000);
-  try{
-    await page.route('**/*',async route=>{const req=route.request();const navigation=req.isNavigationRequest()&&req.frame()===page.mainFrame();const requested=req.url();const wanted=new URL(screen.url);wanted.hash='';if(adapter&&req.redirectedFrom()&&adapter.requests.has(req.redirectedFrom())){adapter.reject(req,'redirect','approved_read_redirect');blocked++;outerBlock('approved_read_redirect');await route.abort('blockedbyclient');}
-      else if(navigation&&requested!==wanted.href&&requested!==screen.url){blocked++;outerBlock('navigation_scope');adapter?.recordDenial(req,'selected_page','navigation_scope');await route.abort('blockedbyclient');}
-      else if(adapter?!['GET','HEAD','OPTIONS'].includes(req.method().toUpperCase()):!isReadOnlyRequest(req.method(),req.postData())){const decision=adapter?.request(req);if(decision==='allow')await adapter.forward(route);else{if(decision!=='block_known'){blocked++;outerBlock('non_read');}await route.abort('blockedbyclient');}}
-      else await route.continue();});
-    if(typeof page.routeWebSocket!=='function')fail('browser_version_unsupported');await page.routeWebSocket('**/*',socket=>socket.close());
-    if(signal?.aborted)fail('cancelled');
-    const response=await page.goto(screen.url,{waitUntil:'domcontentloaded',timeout:30000});
-    if(signal?.aborted)fail('cancelled');
-    if(page.url()!==screen.url){base.outcome='scope_mismatch';return base;}
-    if(response&&response.status()>=400){base.outcome='capture_failed';return base;}
-    await page.locator(screen.ready_selector).waitFor({state:'visible',timeout:5000}).catch(()=>{});
-    if(adapter&&screen.login_selector&&await page.evaluate(selector=>Array.from(document.querySelectorAll(selector)).some(node=>{const style=getComputedStyle(node),rect=node.getBoundingClientRect();return style.display!=='none'&&style.visibility!=='hidden'&&style.opacity!=='0'&&rect.width>0&&rect.height>0;}),screen.login_selector)){base.outcome='session_expired';base.login_marker_matched=true;return base;}
-    if(adapter&&!await adapter.ready(signal)){if(signal?.aborted)fail('cancelled');captureGuard='readiness_refused';base.outcome='request_blocked';return base;}
-    if(capabilities){
-      const ready=await page.evaluate(letterPickerReady,{screen,expected:binding.markers[screen.id]});
-      if(ready.error){base.outcome=ready.error;base.login_marker_matched=ready.login===true;return base;}
-      const remaining=()=>{const ms=Math.floor(deadline-performance.now());if(ms<=0)fail('timeout');return ms;};
-      // The only interaction this adapter permits: open the observed provider picker once.
-      await page.locator('[data-testid="connect-integration-open"]').click({timeout:remaining()});
-      await page.locator('[data-testid="connect-integration"]').waitFor({state:'visible',timeout:remaining()});
-      // Playwright visibility permits opacity zero during Letter's entrance animation.
-      // Wait within the original capture deadline before freezing the unchanged mask set.
-      await page.waitForFunction(()=>{
-        const nodes=document.querySelectorAll('[data-testid="connect-integration"]');if(nodes.length!==1)return false;
-        const node=nodes[0],style=getComputedStyle(node),rect=node.getBoundingClientRect();
-        return node.getAttribute('role')==='dialog'&&style.display!=='none'&&style.visibility!=='hidden'&&style.opacity==='1'&&rect.width>0&&rect.height>0&&node.getAnimations().every(animation=>!animation.pending&&['finished','idle'].includes(animation.playState));
-      },null,{timeout:remaining()});
-      if(!await adapter.ready(signal)){captureGuard='readiness_refused';base.outcome='request_blocked';return base;}
-    }
-    await page.evaluate(()=>{window.scrollTo(0,0);});
-    const data=await page.evaluate(capabilities?extractLetterCapabilities:extractScreen,{screen,policy:m.policy,expected:binding.markers[screen.id],freeze:true});
-    if(data.error){base.outcome=data.error;base.login_marker_matched=data.login===true;return base;}
-    const png=await page.screenshot({type:'png',fullPage:false,animations:'disabled',caret:'initial',timeout:10000});
-    const changed=await page.evaluate(()=>window.__opencrawlChanged===true||window.__opencrawlMasks?.some(node=>!node.isConnected||getComputedStyle(node).opacity!=='0')); 
-    if(changed||page.url()!==screen.url){base.outcome='not_ready';return base;}
-    if(blocked||adapter&&(adapter.invalid||adapter.inFlight>0||adapter.required!=='verified')){captureGuard='final_request_refused';base.outcome='request_blocked';return base;}
-    captureGuard='passed';
-    return checkUpload({...base,outcome:'captured',observed_at:new Date().toISOString(),final_url:screen.url,ready_matched:true,title:data.title,text:data.text,screenshot_png_base64:png.toString('base64'),account:{label:m.account_label,assurance:data.marker?'dom_marker_matched':'declared_only',marker_sha256:data.marker?sha(data.marker):null}},m);
-  }catch(e){if(signal?.aborted)fail('cancelled');base.outcome=e?.name==='TimeoutError'?'timeout':'capture_failed';return base;}
-  finally{clearTimeout(timer);signal?.removeEventListener('abort',abort);adapter?.close();await page.close().catch(()=>{});if(adapter)await onDiagnostics?.(base.client_capture_id,{...adapter.report(),capture_guard:captureGuard,outer_blocks:outerBlocks});}
-}
-async function sleep(ms,signal){if(signal?.aborted)fail('cancelled');await new Promise((resolve,reject)=>{const stop=()=>{clearTimeout(t);signal?.removeEventListener('abort',stop);reject(new MonitorFailure('cancelled'));};const t=setTimeout(()=>{signal?.removeEventListener('abort',stop);resolve();},ms);signal?.addEventListener('abort',stop,{once:true});});}
-export async function monitorRunner(action,m,client,options={},overrides={}){
-  const scope=await identity(client,overrides.signal);const directory=profilePath(options.profile);
-  return lock(directory,async()=>{
-    const file=path.join(directory,'binding.json');let binding=await read(file);
-    const expected={...scope,monitor_id:m.id,generation:m.generation,policy_sha256:m.policy_sha256,origin:m.origin};
-    if(binding&&!same(binding.scope,expected))fail('profile_binding_changed');
-    const operations=operationPath(scope,m.id);await privateDir(operations);
-    return lock(operations,async()=>{
-    await unresolved(operations);
-    const onDiagnostics=async(captureId,report)=>{const bytes=Buffer.byteLength(JSON.stringify(report)+'\n');if(bytes>16384)fail('diagnostics_too_large');await unresolved(operations,bytes);await write(path.join(operations,'diagnostics-'+captureId+'.json'),report);};
-    const pw=await browser(overrides);
-    if(action==='login'){
-      let markers=options.markers?await read(path.resolve(options.markers),16384):binding?.markers||{};
-      if(!markers||Array.isArray(markers)||typeof markers!=='object'||Object.entries(markers).some(([k,v])=>!m.pages.some(p=>p.id===k)||typeof v!=='string'||!v.trim()||v.length>500))fail('invalid_markers');
-      if(m.pages.some(p=>p.account_marker&&!markers[p.id]))fail('markers_required');
-      if(binding&&!same(binding.markers,markers))fail('profile_binding_changed');
-      if(!overrides.waitForLogin&&!process.stdin.isTTY)fail('interactive_login_required');
-      const duration=overrides.loginTimeoutMs??600000;if(!count(duration,1,600000))fail('invalid_login_timeout');
-      const controller=new AbortController(),signal=controller.signal,deadline=performance.now()+duration;
-      const stop=()=>controller.abort(new MonitorFailure('cancelled'));
-      const active=()=>{if(performance.now()>=deadline)controller.abort(new MonitorFailure('login_timeout'));loginActive(signal);};
-      const remaining=()=>{active();return Math.max(1,Math.min(30000,Math.ceil(deadline-performance.now())));};
-      let context,closing;const close=()=>{if(context&&!closing)closing=Promise.resolve().then(()=>context.close());return closing||Promise.resolve();};
-      const onAbort=()=>{void close().catch(()=>{});};signal.addEventListener('abort',onAbort,{once:true});
-      overrides.signal?.addEventListener('abort',stop,{once:true});if(overrides.signal?.aborted)stop();
-      process.on('SIGINT',stop);process.on('SIGTERM',stop);
-      const timer=setTimeout(()=>controller.abort(new MonitorFailure('login_timeout')),duration);
-      try{
-        context=await launch(pw,directory,m,true,remaining());active();
-        const page=context.pages()[0]||await context.newPage();active();await page.goto(m.pages[0].url,{waitUntil:'domcontentloaded',timeout:remaining()});active();
-        await waitForLogin(page,m.pages[0],markers,signal,overrides.waitForLogin||loginPrompt,isLetterCapabilitiesPolicy(m.policy,m.pages[0]));active();await current(client,m,scope,signal);active();
-        binding={version:1,scope:expected,markers,session_epoch:crypto.randomUUID()};
-        await close();context=null;closing=null;active();
-        context=await launch(pw,directory,m,false,remaining());active();
-        for(const screen of m.pages){active();const observed=await captureScreen(context,m,screen,binding,{signal,onDiagnostics});active();if(observed.outcome!=='captured')fail(observed.outcome);}
-        await current(client,m,scope,signal);active();await write(file,binding,active);
-      }catch(error){active();throw error;}
-      finally{try{await close();}finally{clearTimeout(timer);signal.removeEventListener('abort',onAbort);overrides.signal?.removeEventListener('abort',stop);process.off('SIGINT',stop);process.off('SIGTERM',stop);}}
-      return {state:'session_saved_locally',monitor_id:m.id,profile:options.profile};
-    }
-    if(!binding)fail('local_login_required');
-    if(Number(options.generation)!==m.generation)fail('generation_required');
-    if(action==='watch'&&m.cadence_minutes===0)fail('cadence_required');
-    const controller=new AbortController();const signal=overrides.signal||controller.signal;const stop=()=>controller.abort();if(!overrides.signal){process.on('SIGINT',stop);process.on('SIGTERM',stop);}
-    let context;try{context=await launch(pw,directory,m,false);}catch(e){if(!overrides.signal){process.off('SIGINT',stop);process.off('SIGTERM',stop);}throw e;}
-    const records=[];
-    try{do{
-      await unresolved(operations);const fresh=await current(client,m,scope,signal);
-      for(const screen of fresh.pages){
-        if(signal.aborted)fail('cancelled');await current(client,m,scope,signal);
-        const upload=await captureScreen(context,m,screen,binding,{signal,onDiagnostics});await current(client,m,scope,signal);
-        if(action==='preview'){
-          if((await fs.readdir(directory)).filter(n=>n.startsWith('preview-')).length>=20)fail('local_preview_full');
-          const preview=path.join(directory,'preview-'+upload.client_capture_id);await privateDir(preview);await write(path.join(preview,'observation.json'),upload);if(upload.outcome==='captured')await fs.writeFile(path.join(preview,'masked.png'),Buffer.from(upload.screenshot_png_base64,'base64'),{mode:0o600});records.push({state:upload.outcome,preview});
-        }else {const r=await sendCapture(client,m.id,upload,operations,signal);records.push(r);overrides.onReceipt?.(r);}
-        if(upload.outcome!=='captured')fail(upload.outcome);
-      }
-      if(action!=='watch')break;
-      await (overrides.sleep||sleep)(m.cadence_minutes*60000,signal);
-      records.splice(0); // A long-running watch never accumulates unbounded output in memory.
-    }while(!signal.aborted);
-    return {state:'complete',captures:records};
-    }finally{await context.close();if(!overrides.signal){process.off('SIGINT',stop);process.off('SIGTERM',stop);}}
-    });
-  });
-}
-export const PRODUCT_MONITORS_HELP=`Local private product monitoring\n\n  opencrawl browser monitor create --spec FILE\n  opencrawl browser monitor list|show [MONITOR_ID]\n  opencrawl browser monitor login MONITOR_ID --profile NAME [--markers FILE]\n  opencrawl browser monitor preview|check|watch MONITOR_ID --profile NAME --generation N\n  opencrawl browser monitor recover MONITOR_ID --capture UUID\n  opencrawl browser monitor changes MONITOR_ID\n  opencrawl browser monitor compare COMPARISON_ID --revision HASH\n  opencrawl browser monitor image SNAPSHOT_ID --revision HASH --output NEW.png\n  opencrawl browser monitor context COMPARISON_ID --revision HASH [--budget 4096]\n\nUse --account PROFILE and --server ORIGIN for the intended OpenCrawl account.\nmarkers FILE is private JSON {page_id: expected visible account/workspace text}.\nBrowser sessions stay here. Watch runs in the foreground; this computer must stay on.\nUnknown uploads stop without resend. Failures retain the last good snapshot.\n`;
-async function command(rest,options,client,overrides={}){
-  const [action,value,...extra]=rest;const out=overrides.stdout||((s)=>process.stdout.write(s));
-  const allowed={create:['spec'],list:[],show:[],login:['profile','markers'],preview:['profile','generation'],check:['profile','generation'],watch:['profile','generation'],recover:['capture'],changes:[],compare:['revision'],image:['revision','output'],context:['revision','budget']};
-  if(!Object.hasOwn(allowed,action)||extra.length)fail('invalid_command');allowedOptions(options,allowed[action],'browser monitor');
-  let result;
-  if(action==='list'){result=await request(client,API);if(!Array.isArray(result?.monitors)||result.monitors.length>10)fail('invalid_monitor_list');result.monitors.forEach(m=>readMonitor(m));}
-  else if(action==='create'){
-    if(value||typeof options.spec!=='string')fail('spec_required');const input=await read(path.resolve(options.spec),65536);if(!input)fail('spec_required');uuid(input.client_create_id);const scope=await identity(client);const dir=operationPath(scope,'create');await privateDir(dir);
-    result=await lock(dir,async()=>{const file=path.join(dir,input.client_create_id+'.json');const prior=await read(file);if(prior&&!same(prior.input,input))fail('creation_conflict');let value;
-      if(prior){try{value=await request(client,`${API}/by-client-id/${input.client_create_id}`);}catch{fail('creation_unconfirmed');}}
-      else {await write(file,{state:'intended',input});try{value=await request(client,API,{method:'POST',body:input});}catch{try{value=await request(client,`${API}/by-client-id/${input.client_create_id}`);}catch{fail('creation_unconfirmed');}}}
-      readMonitor(value);const pages=input.pages.map(p=>({login_selector:null,account_marker:null,...p}));const policy={version:1,color_scheme:'light',screenshot_mode:'viewport',extractor:'product-screen-v1',masks:[],noise_regions:[],...input.policy,viewport:{device_scale_factor:1,...input.policy?.viewport}};if(value.name!==input.name||value.account_label!==input.account_label||value.cadence_minutes!==(input.cadence_minutes??0)||!same(value.policy,policy)||!same(value.pages.map(({id,last_attempt,latest_snapshot,...p})=>p),pages))fail('receipt_mismatch');await write(file,{state:'confirmed',input,monitor:value});return value;});
-  }else if(action==='image'){
-    id(value,'psnap');if(!HASH.test(options.revision)||typeof options.output!=='string')fail('image_output_required');
-    const snapshot=await request(client,`${API}/snapshots/${value}`);if(snapshot?.id!==value||snapshot.version!==options.revision)fail('snapshot_changed');
-    const bytes=await request(client,`${API}/snapshots/${value}/image`,{responseType:'png'});
-    if(bytes.length!==snapshot.screenshot_bytes||sha(bytes)!==snapshot.screenshot_sha256)fail('image_mismatch');
-    const output=path.resolve(options.output),handle=await fs.open(output,'wx',0o600);try{await handle.writeFile(bytes);await handle.sync();}finally{await handle.close();}result={snapshot_id:value,version:options.revision,output};
-  }else if(action==='context'){
-    id(value,'pcmp');if(!HASH.test(options.revision))fail('version_required');const budget=options.budget===undefined?4096:Number(options.budget);if(!count(budget,256,16384))fail('invalid_budget');
-    result=await request(client,'/api/v1/product-monitor-context',{method:'POST',body:{selections:[{comparison_id:value,version:options.revision}],budget}});
-    if(typeof result?.context!=='string'||Buffer.byteLength(result.context)!==result.context_bytes||result.context_bytes>budget||result.budget!==budget||!Array.isArray(result.included)||!Array.isArray(result.omitted)||result.included.some(r=>r.comparison_id!==value||r.version!==options.revision||!['before','after'].includes(r.side))||result.omitted.some(r=>r.comparison_id!==value||!['before','after'].includes(r.side)||!['budget','no_text_region'].includes(r.reason))||result.included.length+result.omitted.length!==2||new Set([...result.included,...result.omitted].map(r=>r.side)).size!==2||result.image_content_included!==false)fail('context_mismatch');
-  }else if(action==='compare'){
-    id(value,'pcmp');if(!HASH.test(options.revision))fail('version_required');result=await request(client,`${API}/comparisons/${value}`);if(result?.id!==value||result.version!==options.revision)fail('comparison_changed');
-  }else{
-    monitorId(value);
-    if(action==='recover'){
-      uuid(options.capture);const scope=await identity(client);const dir=operationPath(scope,value);result=await lock(dir,async()=>{const file=path.join(dir,options.capture+'.json');const saved=await read(file);if(!saved||saved.monitor!==value)fail('capture_unknown');const raw=await request(client,`${API}/${value}/captures/by-client-id/${options.capture}`);let got;if(saved.state==='confirmed'){if(raw?.client_capture_id!==options.capture||raw.monitor_id!==value||raw.manifest_sha256!==saved.manifest_sha256||!same(raw,saved.receipt))fail('receipt_mismatch');got=raw;}else got=receipt(raw,saved.upload,value);await write(file,{version:1,monitor:value,state:'confirmed',manifest_sha256:got.manifest_sha256,receipt:got});return got;});
-    }else if(action==='changes'){result=await request(client,`${API}/${value}/comparisons`);if(!Array.isArray(result?.comparisons))fail('invalid_comparisons');}
-    else{const m=readMonitor(await request(client,`${API}/${value}`),value);if(action==='show')result=m;else{if(m.status!=='active')fail('monitor_not_active');result=await monitorRunner(action,m,client,options,{...overrides,onReceipt:action==='watch'?r=>out(JSON.stringify(r)+'\n'):undefined});}}
-  }
-  if(options.json)out(JSON.stringify(result,null,2)+'\n');else if(result?.comparisons)out(result.comparisons.map(x=>`${displayLine(x.id)} ${displayLine(x.version)} ${displayLine(x.observed_before)} → ${displayLine(x.observed_after)}\n`).join('')||'No retained comparisons.\n');else out(JSON.stringify(result,null,2)+'\n');return 0;
-}
-
-export async function productMonitorsCommand(rest,options,client,overrides={}) {
-  try {return await command(rest,options,client,overrides);}
-  catch(error) {
-    if(error instanceof MonitorFailure)throw error;
-    const code=Number.isInteger(error?.status)&&error.status>=400&&error.status<=599?`http_${error.status}`:error?.code==='EEXIST'?'output_exists':'request_unconfirmed';
-    throw new MonitorFailure(code);
-  }
-}

package/src/research-agent.mjs

@@ -1,164 +0,0 @@
-import {readResearchCompletion,readResearchPresentation} from './research-completion.mjs';
-import {readResearchAnswerContract} from './research-answer-contract.mjs';
-import {readResearchAnswerView} from './research-answer-view.mjs';
-import fs from 'node:fs/promises';
-import {createHash} from 'node:crypto';
-import {allowedOptions,decimal,identifier,withInterrupt} from './jobs.mjs';
-import {displayLine,displayText} from './display.mjs';
-
-const ROOT='/api/research/agents',ACTIVE=new Set(['queued','running']);
-const object=v=>v!==null&&typeof v==='object'&&!Array.isArray(v);
-const text=(v,max,min=0)=>typeof v==='string'&&Array.from(v).length>=min&&Array.from(v).length<=max&&!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(v);
-const integer=(v,min=0,max=Number.MAX_SAFE_INTEGER)=>Number.isSafeInteger(v)&&v>=min&&v<=max;
-const stamp=v=>typeof v==='string'&&/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|\+00:00)$/.test(v)&&Number.isFinite(Date.parse(v))&&new Date(v).toISOString().slice(0,19)===v.slice(0,19);
-const fields=(v,keys)=>Object.keys(v).every(key=>keys.includes(key));
-function canonicalWebsite(value){if(/[\\\s\u0000-\u001f\u007f]/.test(value)||/%(?![0-9a-f]{2})/i.test(value))throw new Error('Use a valid public website URL.');const u=new URL(value);u.hash='';return new URL(u.href.replace(/%([0-9a-f]{2})/gi,(_all,hex)=>{const ch=String.fromCharCode(parseInt(hex,16));return /^[A-Za-z0-9._~-]$/.test(ch)?ch:'%'+hex.toUpperCase()})).href;}
-function publicUrl(v){if(!text(v,2048,1))return false;try{const u=new URL(v);return ['http:','https:'].includes(u.protocol)&&!u.username&&!u.password&&!!u.hostname}catch{return false}}
-function pageEvidence(cite){
- if(cite.source_kind!=='web_page')return !Object.hasOwn(cite,'page');
- const p=cite.page;
- return object(p)&&fields(p,['requested_url','final_url','body_sha256','start','end','total_chars','extractor'])&&publicUrl(p.requested_url)&&publicUrl(p.final_url)&&p.final_url===cite.url&&typeof p.body_sha256==='string'&&/^[0-9a-f]{64}$/.test(p.body_sha256)&&integer(p.start,0,131071)&&integer(p.end,1,131072)&&integer(p.total_chars,1,131072)&&p.start<p.end&&p.end<=p.total_chars&&p.end-p.start<=16000&&typeof cite.end==='number'&&cite.end<=p.end-p.start&&p.extractor==='public-static-v3';
-}
-const invalid=()=>{throw new Error('Server returned invalid or out-of-scope saved research findings. No result was emitted.');};
-const feedbackRatings=['useful','irrelevant','incorrect','outdated'];
-const feedbackNote=value=>value===null||text(value,1000)&&!value.includes('\u0000')&&Buffer.byteLength(value)<=4096;
-function confirmedReadFailure(error,request){
- if((request.method??'GET')!=='GET'||!Number.isInteger(error?.status))return null;
- const message={
-  401:'Saved research could not be read. Check the selected CLI account, server and workspace and its existing credential.',
-  403:'This account cannot read the requested research. Check the selected account and workspace and ask its administrator to confirm access.',
-  404:request.command==='receipt'?'The original admission receipt is unavailable to this account. Check the selected workspace and exact operation. A missing receipt does not authorize a new admission.':'The saved research is unavailable to this account. Check the selected workspace and exact agent, run or source identifiers. Do not substitute a current webpage for retained evidence.',
-  409:'The saved research could not be read in its current state. Read the same agent, schedule or feedback and reconcile its current revision before any write.'
- }[error.status];
- return message?{code:`http_${error.status}`,message}:null;
-}
-export function validateAgentFeedback(value,agentId,runId,findingId){
- if(!object(value)||!fields(value,['agent_id','run_id','finding_id','feedback'])||value.agent_id!==agentId||value.run_id!==runId||value.finding_id!==findingId||!object(value.feedback))invalid();
- const f=value.feedback;if(!fields(f,['version','revision','state','rating','note','updated_at'])||f.version!==1||!integer(f.revision)||!feedbackNote(f.note)||Buffer.byteLength(JSON.stringify(value))>8192)invalid();
- if(f.state==='absent'){if(f.revision!==0||f.rating!==null||f.note!==null||f.updated_at!==null)invalid()}
- else if(f.state==='deleted'){if(f.revision<1||f.rating!==null||f.note!==null||!stamp(f.updated_at))invalid()}
- else if(f.state==='active'){if(f.revision<1||!feedbackRatings.includes(f.rating)||!stamp(f.updated_at))invalid()}
- else invalid();return value;
-}
-export function validateAgentSource(value,agentId,runId,sourceId){
- if(!object(value)||!fields(value,['agent_id','run_id','source'])||value.agent_id!==agentId||value.run_id!==runId||!object(value.source))invalid();
- const source=value.source;
- if(!fields(source,['id','url','title','text','body_sha256','observed_at','published_at','source_kind','collection_id','document_id','version_id','page'])||source.id!==sourceId||!text(source.id,80,1)||!publicUrl(source.url)||!text(source.title,2048,1)||!text(source.text,16000,1)||!source.text.trim()||source.text.includes('\u0000')||!stamp(source.observed_at)||source.published_at!==null&&!text(source.published_at,100,1)||!['indexed','web_page'].includes(source.source_kind)||!pageEvidence({...source,end:Array.from(source.text).length})||typeof source.body_sha256!=='string'||!/^[0-9a-f]{64}$/.test(source.body_sha256))invalid();
- for(const key of ['collection_id','document_id','version_id'])if(source.source_kind==='indexed'?!text(source[key],200,1):source[key]!==null)invalid();
- if(source.source_kind==='web_page'&&source.page.end-source.page.start!==Array.from(source.text).length)invalid();
- if(Buffer.byteLength(JSON.stringify(value))>131072||createHash('sha256').update(source.text,'utf8').digest('hex')!==source.body_sha256)invalid();
- return value;
-}
-export function validateAgentDescriptor(value){
- if(!object(value)||!fields(value,['customer','goal','collection_ids','schedule_minutes','research_policy'])||!object(value.customer)||!fields(value.customer,['name','website','context'])||!text(value.customer.name,200,1)||!value.customer.name.trim()||!publicUrl(value.customer.website)||!text(value.customer.context??'',4000)||!text(value.goal,4000,1)||!value.goal.trim()||value.schedule_minutes!==undefined&&value.schedule_minutes!==0||!Array.isArray(value.collection_ids??[])||(value.collection_ids??[]).length>8||!(value.collection_ids??[]).every(v=>text(v,200,1))||new Set(value.collection_ids??[]).size!==(value.collection_ids??[]).length)throw new Error('Agent JSON must contain a customer name and public website, a goal, optional context and up to eight unique collection_ids; schedule_minutes must be 0.');
- if(Object.hasOwn(value,'research_policy')&&!['completion_v1','completion_v2','completion_v3'].includes(value.research_policy))throw new Error('research_policy must be completion_v1, completion_v2 or completion_v3 when explicitly supplied.');
- const result={customer:{name:value.customer.name.trim(),website:canonicalWebsite(value.customer.website),context:value.customer.context??''},goal:value.goal.trim(),collection_ids:value.collection_ids??[],schedule_minutes:0,...(Object.hasOwn(value,'research_policy')?{research_policy:value.research_policy}:{})};if(Buffer.byteLength(JSON.stringify(result))>32768)throw new Error('The research brief exceeds 32 KiB.');return result;
-}
-export function validateAgentRun(value,agentId=value?.agent_id,runId=value?.id){
- if(!object(value)||!fields(value,['id','agent_id','agent_revision','status','phase','created_at','started_at','finished_at','outcome','findings','gaps','usage','error'])||!text(value.id,200,1)||value.id!==runId||!text(value.agent_id,200,1)||value.agent_id!==agentId||!integer(value.agent_revision,1)||!['queued','running','completed','partial','needs_input','failed','unknown','cancelled'].includes(value.status)||!['queued','researching','synthesizing','presenting','finished'].includes(value.phase)||!stamp(value.created_at)||value.started_at!==null&&!stamp(value.started_at)||value.finished_at!==null&&!stamp(value.finished_at)||!['findings','no_supported_findings','needs_input',null].includes(value.outcome)||!Array.isArray(value.findings)||value.findings.length>5||!Array.isArray(value.gaps)||value.gaps.length>5||!value.gaps.every(v=>text(v,500,1))||!object(value.usage)||Buffer.byteLength(JSON.stringify(value.usage))>16384||value.error!==null&&(!object(value.error)||!text(value.error.code,100,1)||!text(value.error.message,1000,1)))invalid();
- const seen=new Set();for(const finding of value.findings){if(!object(finding)||!fields(finding,['id','claim','relevance','qualifiers','citations'])||!text(finding.id,200,1)||seen.has(finding.id)||!text(finding.claim,1500,1)||!text(finding.relevance,1500,1)||!Array.isArray(finding.qualifiers)||finding.qualifiers.length>5||!finding.qualifiers.every(v=>text(v,300,1))||!Array.isArray(finding.citations)||finding.citations.length<1||finding.citations.length>3)invalid();seen.add(finding.id);
-  for(const cite of finding.citations){if(!object(cite)||!fields(cite,['id','url','title','quote','start','end','body_sha256','observed_at','published_at','source_kind','collection_id','document_id','version_id','page'])||!text(cite.id,80,1)||!publicUrl(cite.url)||!text(cite.title,2048)||!text(cite.quote,1200,1)||!integer(cite.start,0,16000)||!integer(cite.end,1,16000)||cite.end-cite.start!==Array.from(cite.quote).length||typeof cite.body_sha256!=='string'||!/^[0-9a-f]{64}$/.test(cite.body_sha256)||!stamp(cite.observed_at)||cite.published_at!==null&&!text(cite.published_at,100,1)||!['web_search','indexed','web_page'].includes(cite.source_kind)||!pageEvidence(cite))invalid();for(const key of ['collection_id','document_id','version_id'])if(cite.source_kind==='indexed'?!text(cite[key],200,1):cite[key]!==null)invalid();}
- }
- if((value.outcome==='findings')!==Boolean(value.findings.length)||ACTIVE.has(value.status)&&(value.phase==='finished'||value.outcome!==null||value.findings.length||value.finished_at!==null)||!ACTIVE.has(value.status)&&value.phase!=='finished'||Buffer.byteLength(JSON.stringify({findings:value.findings,gaps:value.gaps}))>32768)invalid();return value;
-}
-export function validateResearchAgent(value,id=value?.id){if(!object(value)||!fields(value,['id','customer','goal','collection_ids','revision','status','schedule_minutes','created_at','updated_at','latest_run'])||!text(value.id,200,1)||value.id!==id||!integer(value.revision,1)||!['active','paused'].includes(value.status)||![0,1440].includes(value.schedule_minutes)||!stamp(value.created_at)||!stamp(value.updated_at)||!text(value.customer?.context,4000))invalid();try{validateAgentDescriptor({customer:value.customer,goal:value.goal,collection_ids:value.collection_ids,schedule_minutes:0})}catch{invalid()}if(value.latest_run!==null)validateAgentRun(value.latest_run,value.id);return value;}
-export function validateAgentSchedule(value,agentId){
- if(!object(value)||!fields(value,['agent_id','revision','enabled','schedule_minutes','next_scheduled_at','blocked_reason','blocked_run_id','deferred_reason'])||value.agent_id!==agentId||!integer(value.revision)||typeof value.enabled!=='boolean'||value.schedule_minutes!==(value.enabled?1440:0)||!['unknown','failed','needs_input','cancelled','access_changed','agent_paused',null].includes(value.blocked_reason)||!['capacity','daily_budget','storage','run_active','history_capacity',null].includes(value.deferred_reason)||value.blocked_run_id!==null&&!text(value.blocked_run_id,200,1)||value.next_scheduled_at!==null&&!stamp(value.next_scheduled_at))invalid();
- if((!value.enabled||value.blocked_reason!==null)&&value.deferred_reason!==null||!value.enabled&&(value.next_scheduled_at!==null||value.blocked_reason!==null||value.blocked_run_id!==null)||value.enabled&&(value.revision===0||(value.blocked_reason===null)!==(value.next_scheduled_at!==null))||value.blocked_reason===null&&value.blocked_run_id!==null)invalid();
- return value;
-}
-export function validateAgentRepeats(value,agentId,runId){
- if(!object(value)||!fields(value,['agent_id','run_id','status','items'])||value.agent_id!==agentId||value.run_id!==runId||!['available','not_assessed'].includes(value.status)||!Array.isArray(value.items)||value.items.length>5||value.status==='not_assessed'&&value.items.length)invalid();
- const seen=new Set();for(const item of value.items){
-  if(!object(item)||!fields(item,['finding_id','classification','first_seen_run_id','first_seen_finding_id'])||!text(item.finding_id,200,1)||seen.has(item.finding_id)||!text(item.first_seen_run_id,200,1)||!text(item.first_seen_finding_id,200,1)||!['first_seen','exact_repeat'].includes(item.classification))invalid();
-  const same=item.first_seen_run_id===runId&&item.first_seen_finding_id===item.finding_id;if((item.classification==='first_seen')!==same||item.classification==='exact_repeat'&&item.first_seen_run_id===runId)invalid();seen.add(item.finding_id);
- }
- return value;
-}
-export function validateAgentReceipt(value,id){if(!object(value)||!fields(value,['agent','run','replayed'])||typeof value.replayed!=='boolean')invalid();const agent=validateResearchAgent(value.agent,id??value.agent?.id),run=validateAgentRun(value.run,agent.id);if(agent.revision!==run.agent_revision||agent.latest_run?.id!==run.id)invalid();return value;}
-function key(value){if(typeof value!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(value))throw new Error('Retain a random 16–128 character --request-key for this exact research request.');return value;}
-export function agentRequest(rest,options){
- if(rest[0]==='agent'&&rest[1]==='feedback'){
-  const [, ,id,runId,findingId,action='show',...extra]=rest;
-  identifier(id,'agent');identifier(runId,'run');identifier(findingId,'finding');
-  if(extra.length||!['show','set','delete'].includes(action))throw new Error('Use research agent feedback AGENT_ID RUN_ID FINDING_ID [show|set|delete].');
-  allowedOptions(options,action==='show'?[]:action==='set'?['revision','rating','note']:['revision'],'research agent feedback');
-  const route=`${ROOT}/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/findings/${encodeURIComponent(findingId)}/feedback`,base={command:'feedback',id,runId,findingId,action,route};
-  if(action==='show')return base;
-  const revision=decimal(options.revision,'revision',0,Number.MAX_SAFE_INTEGER);
-  if(action==='delete')return {...base,revision,method:'DELETE',route:route+`?expected_revision=${revision}`};
-  if(!feedbackRatings.includes(options.rating)||!feedbackNote(options.note??null))throw new Error('Use --rating useful|irrelevant|incorrect|outdated and an optional note of at most 1000 Unicode characters.');
-  return {...base,revision,method:'PUT',body:{expected_revision:revision,rating:options.rating,note:options.note??null}};
- }
- if(rest[0]==='agent'&&rest[1]==='source'){
-  allowedOptions(options,[],'research agent source');if(rest.length!==5)throw new Error('Use research agent source AGENT_ID RUN_ID SOURCE_ID.');
-  const id=identifier(rest[2],'agent'),runId=identifier(rest[3],'run'),sourceId=identifier(rest[4],'source');return {command:'source',id,runId,sourceId,route:`${ROOT}/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/sources/${encodeURIComponent(sourceId)}`};
- }
- if(rest[0]==='agent'&&rest[1]==='runs'){
-  allowedOptions(options,['limit','cursor'],'research agent runs');if(rest.length!==3)throw new Error('Use research agent runs AGENT_ID [--limit N] [--cursor CURSOR].');
-  const id=identifier(rest[2],'agent'),limit=decimal(options.limit,'limit',1,100,25),query=new URLSearchParams({limit});
-  if(options.cursor!==undefined){if(typeof options.cursor!=='string'||!/^[\x21-\x7e]{1,512}$/.test(options.cursor))throw new Error('Use the returned opaque research history cursor.');query.set('cursor',options.cursor)}
-  return {command:'runs',id,limit,cursor:options.cursor,route:`${ROOT}/${encodeURIComponent(id)}/runs?${query}`};
- }
- if(rest[0]==='agent'&&rest[1]==='schedule'){
-  const [, ,id,action='show',...extra]=rest;identifier(id,'agent');if(extra.length||!['show','daily','pause','resume'].includes(action))throw new Error('Use research agent schedule AGENT_ID [show|daily|pause|resume].');allowedOptions(options,action==='show'?[]:['revision'],'research agent schedule');
-  return {command:'schedule',id,route:`${ROOT}/${encodeURIComponent(id)}/schedule`,...(action==='show'?{}:{method:'PATCH',body:{expected_revision:decimal(options.revision,'revision',0,Number.MAX_SAFE_INTEGER),enabled:action!=='pause'}})};
- }
- if(rest[0]==='agent'&&rest[1]==='repeats'){
-  allowedOptions(options,[],'research agent repeats');if(rest.length!==4)throw new Error('Use research agent repeats AGENT_ID RUN_ID.');const id=identifier(rest[2],'agent'),runId=identifier(rest[3],'run');return {command:'repeats',id,runId,route:`${ROOT}/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/repeats`};
- }
- if(rest[0]==='agent-run'){
-  allowedOptions(options,['wait','timeout'],'research agent-run');if(rest.length!==3)throw new Error('Use research agent-run AGENT_ID RUN_ID.');
-  const id=identifier(rest[1],'agent'),runId=identifier(rest[2],'run');if(options.timeout!==undefined&&!options.wait)throw new Error('--timeout requires --wait.');
-  return {command:'agent-run',id,runId,route:`${ROOT}/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}`,wait:!!options.wait,timeout:decimal(options.timeout,'timeout',1,3600,300)};
- }
- const [,command,id,...extra]=rest,allowed={create:['file','request-key','wait','timeout'],list:['limit','cursor'],show:[],run:['revision','request-key','wait','timeout','research-policy'],pause:['revision'],resume:['revision'],receipt:['operation','request-key']};
- if(!Object.hasOwn(allowed,command))throw new Error('Choose research agent create, list, show, run, pause, resume or receipt.');allowedOptions(options,allowed[command],'research agent');
- if(extra.length||['create','list','receipt'].includes(command)&&id!==undefined)throw new Error('Unexpected research agent argument.');
- if(options.timeout!==undefined&&!options.wait)throw new Error('--timeout requires --wait.');
- const base={command,wait:!!options.wait,timeout:decimal(options.timeout,'timeout',1,3600,300)};
- if(command==='create'){if(!text(options.file,4096,1))throw new Error('Provide --file with the research brief JSON.');return {...base,route:ROOT,method:'POST',file:options.file,key:key(options['request-key'])};}
- if(command==='list'){const limit=decimal(options.limit,'limit',1,100,25),q=new URLSearchParams({limit});if(options.cursor!==undefined){if(!text(options.cursor,512,1))throw new Error('Use the returned bounded cursor.');q.set('cursor',options.cursor)}return {...base,limit,route:ROOT+'?'+q};}
- if(command==='receipt'){if(options.operation!=='create'&&!/^run:ragent_[0-9a-f]{32}$/.test(options.operation??''))throw new Error('--operation must be create or run:AGENT_ID.');return {...base,route:ROOT+'/receipts/by-key?operation='+encodeURIComponent(options.operation),key:key(options['request-key']),id:options.operation==='create'?undefined:options.operation.slice(4)};}
- identifier(id,'agent');const route=ROOT+'/'+encodeURIComponent(id);
- if(command==='show')return {...base,id,route};
- const revision=decimal(options.revision,'revision',1,Number.MAX_SAFE_INTEGER);
- if(command==='run'){if(Object.hasOwn(options,'research-policy')&&!['completion_v1','completion_v2','completion_v3'].includes(options['research-policy']))throw new Error('--research-policy must be completion_v1, completion_v2 or completion_v3 when explicitly supplied.');return {...base,id,route:route+'/runs',method:'POST',body:{expected_revision:revision,...(Object.hasOwn(options,'research-policy')?{research_policy:options['research-policy']}:{})},key:key(options['request-key'])};}
- return {...base,id,route,method:'PATCH',body:{expected_revision:revision,status:command==='pause'?'paused':'active'}};
-}
-async function readDescriptor(file){const handle=await fs.open(file,'r');try{const size=(await handle.stat()).size;if(!integer(size,1,65536))throw new Error('Research brief JSON must be between 1 byte and 64 KiB.');const bytes=Buffer.alloc(size+1),read=await handle.read(bytes,0,size+1,0);if(read.bytesRead!==size)throw new Error('Research brief changed while reading.');let value;try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes.subarray(0,size)))}catch{throw new Error('Invalid research brief JSON. Supply valid UTF-8 JSON without extra fields.');}return validateAgentDescriptor(value)}finally{await handle.close()}}
-export async function waitAgentRun(client,initial,{timeout,signal,now=()=>performance.now(),sleep=ms=>new Promise(resolve=>setTimeout(resolve,ms))}={}){let run=validateAgentRun(initial);const started=now();let deadline=started+(timeout??([2,3].includes(readResearchCompletion(run).completion?.version??0)?540:300))*1000;while(ACTIVE.has(run.status)){if(signal?.aborted)return {run,wait:{outcome:'interrupted'}};if(now()>=deadline)return {run,wait:{outcome:'timeout'}};try{run=validateAgentRun(await client.request(`${ROOT}/${encodeURIComponent(run.agent_id)}/runs/${encodeURIComponent(run.id)}`,{signal,timeout:Math.max(1,Math.min(30000,Math.ceil(deadline-now()))),maxResponseBytes:262144}),run.agent_id,run.id)}catch(error){const outcome=signal?.aborted?'interrupted':now()>=deadline?'timeout':'unavailable',failure=outcome==='unavailable'?confirmedReadFailure(error,{command:'agent-run'}):null;return {run,wait:{outcome,...(failure?{error:failure}:{})}}}if(timeout===undefined&&[2,3].includes(readResearchCompletion(run).completion?.version??0))deadline=started+540000;if(ACTIVE.has(run.status)){const next=Math.min(deadline,now()+2000);while(!signal?.aborted&&now()<next)await sleep(Math.min(100,next-now()));}}return {run,wait:{outcome:run.status}};}
-function renderRun(run,stdout){stdout(`Run: ${displayLine(run.id)}\nAgent: ${displayLine(run.agent_id)}\nStatus: ${displayLine(run.status)}\n`);const answer=readResearchAnswerView(run);if(answer.state==='valid'&&answer.sections.some(s=>s.assertions.length)){stdout('\nConcise answer\n');for(const section of answer.sections){stdout('\n'+displayText(section.component.text)+'\n');for(const assertion of section.assertions)stdout(displayText(assertion.text)+'\n');if(section.notice)stdout(displayText(section.notice)+'\n');const evidenceIds=section.assertions.length?[...new Set(section.assertions.flatMap(a=>a.fact_refs.flatMap(r=>run.findings.find(f=>f.id===r.finding_id)?.citations.map(c=>c.id)??[])))]:section.component.evidence_ids;if(evidenceIds.length)stdout('Evidence: '+evidenceIds.map(displayLine).join(', ')+'\n');}stdout('\nExact findings and qualifications\n');}for(const finding of run.findings){stdout(`\n${displayLine(finding.id)}\nObserved statement: ${displayText(finding.claim)}\n`);if(finding.relevance!=='Selected for this research brief.')stdout(`Relevance (inference): ${displayText(finding.relevance)}\n`);for(const qualifier of finding.qualifiers)stdout('Qualification: '+displayText(qualifier)+'\n');for(const cite of finding.citations)stdout(`[${displayLine(cite.id)}] ${displayLine(cite.title)}\n${displayLine(cite.url)}\nObserved: ${displayLine(cite.observed_at)}; source positions ${cite.start}–${cite.end}\n${displayText(cite.quote)}\n`);}for(const gap of [...run.gaps,...readResearchAnswerContract(run).notices.map(notice=>notice.text),...readResearchCompletion(run).notices.map(notice=>notice.text)])stdout('Gap: '+displayText(gap)+'\n');const presentation=readResearchPresentation(run);if(presentation&&!['reviewed','skipped_no_eligible'].includes(presentation.status))stdout('Summary: A concise presentation was not completed for every part. The saved findings and gaps remain available.\n');if(run.error)stdout('Run error: '+displayLine(run.error.message)+'\n');}
-export async function agentCommand(rest,options,client,{stdout=value=>process.stdout.write(value),stderr=value=>process.stderr.write(value)}={}){
- const request=agentRequest(rest,options);if(request.file)request.body=await readDescriptor(request.file);
- let value;try{value=await client.request(request.route,{method:request.method,body:request.body,idempotencyKey:request.key,maxResponseBytes:request.command==='source'?131072:request.command==='runs'?1048576:4*1024*1024});
-  if(request.command==='runs'){
-   if(!object(value)||!fields(value,['items','next_cursor','has_more'])||!Array.isArray(value.items)||value.items.length>request.limit||typeof value.next_cursor!=='string'||!/^[\x21-\x7e]{1,512}$/.test(value.next_cursor)||typeof value.has_more!=='boolean'||value.has_more&&!value.items.length||request.cursor!==undefined&&(value.items.length?value.next_cursor===request.cursor:value.next_cursor!==request.cursor)||Buffer.byteLength(JSON.stringify(value))>1048576)invalid();
-   const seen=new Set();for(const item of value.items){const run=validateAgentRun(item,request.id);if(seen.has(run.id))invalid();seen.add(run.id)}
-  }
-  else if(request.command==='list'){if(!object(value)||!Array.isArray(value.items)||value.items.length>request.limit||value.next_cursor!==null&&!text(value.next_cursor,512,1))invalid();value.items.forEach(item=>validateResearchAgent(item));}
-  else if(['create','run','receipt'].includes(request.command)){validateAgentReceipt(value,request.id);if(request.command==='create'&&(value.agent.revision!==1||value.agent.schedule_minutes!==0||value.agent.goal!==request.body.goal||['name','website','context'].some(k=>value.agent.customer[k]!==request.body.customer[k])||JSON.stringify(value.agent.collection_ids)!==JSON.stringify(request.body.collection_ids)))invalid();if(request.command==='run'&&value.run.agent_revision!==request.body.expected_revision)invalid();}
-  else if(request.command==='schedule'){validateAgentSchedule(value,request.id);if(request.body&&(value.enabled!==request.body.enabled||![request.body.expected_revision,request.body.expected_revision+1].includes(value.revision)))invalid();}
-  else if(request.command==='feedback'){
-   validateAgentFeedback(value,request.id,request.runId,request.findingId);const f=value.feedback;
-   if(request.action!=='show'&&![request.revision,request.revision+1].includes(f.revision))invalid();
-   if(request.action==='set'&&(f.state!=='active'||f.rating!==request.body.rating||f.note!==request.body.note)||request.action==='delete'&&f.state==='active')invalid();
-  }
-  else if(request.command==='source')validateAgentSource(value,request.id,request.runId,request.sourceId);
-  else if(request.command==='repeats')validateAgentRepeats(value,request.id,request.runId);
-  else if(request.command==='agent-run')validateAgentRun(value,request.id,request.runId);
-  else {validateResearchAgent(value,request.id);if(['pause','resume'].includes(request.command)&&(value.status!==request.body.status||![request.body.expected_revision,request.body.expected_revision+1].includes(value.revision)))invalid();}
- }catch(error){const status=Number.isInteger(error?.status)?error.status:undefined,message=confirmedReadFailure(error,request)?.message??(request.command==='feedback'?'Feedback was not confirmed. Read the same finding feedback and reconcile its revision before another write.':request.command==='source'?'Saved source context could not be read. Do not substitute a current webpage for this saved observation.':request.command==='schedule'?'Schedule request was not confirmed. Read the same schedule and reconcile its setting and revision before another write.':'Research request was not confirmed. Read the same agent/run or recover the original admission key; never start a duplicate blindly.');const result={error:{code:status?`http_${status}`:'request_unconfirmed',message,...(request.key?{request_key:request.key}:{})}};if(options.json)stdout(JSON.stringify(result,null,2)+'\n');else stderr(message+(request.key?'\nOriginal request key: '+request.key:'')+'\n');return 1;}
- if(request.command==='feedback'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else stdout(`Feedback: ${displayLine(value.feedback.state)}\nRevision: ${value.feedback.revision}\nRating: ${displayLine(value.feedback.rating??'none')}\n${value.feedback.note===null?'':displayText(value.feedback.note)+'\n'}Customer feedback is not independent factual proof.\n`);return 0;}
- if(request.command==='source'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else stdout(`Saved source context: ${displayLine(value.source.id)}\n${displayLine(value.source.title)}\n${displayLine(value.source.url)}\nObserved: ${displayLine(value.source.observed_at)}\nThis is the admitted evidence window, which may be part of a page.\n\n${displayText(value.source.text)}\n`);return 0;}
- if(request.command==='runs'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{for(const run of value.items)stdout(`${displayLine(run.id)}\t${displayLine(run.status)}\t${displayLine(run.created_at)}\n`);stderr(`Next cursor: ${displayLine(value.next_cursor)}\nMore admitted runs: ${value.has_more?'yes':'no'}\nRetain pending run IDs and read each exact run until terminal.\n`);}return 0;}
- if(request.command==='schedule'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else stdout(`Agent: ${displayLine(value.agent_id)}\nSchedule revision: ${value.revision}\nDaily research: ${value.enabled?'enabled':'disabled'}\nNext scheduled: ${displayLine(value.next_scheduled_at??'none')}\nBlocked: ${displayLine(value.blocked_reason??'no')}\nDeferred: ${displayLine(value.deferred_reason??'no')}\n`);return 0;}
- if(request.command==='repeats'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(`Run: ${displayLine(value.run_id)}\nExact-repeat assessment: ${displayLine(value.status)}\n`);for(const item of value.items)stdout(`${displayLine(item.finding_id)}\t${displayLine(item.classification)}\tfirst seen ${displayLine(item.first_seen_run_id)} / ${displayLine(item.first_seen_finding_id)}\n`);stdout('Exact repetition is not a judgment of business novelty; all saved findings remain available.\n');}return 0;}
- if(request.command==='list'){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{for(const agent of value.items)stdout(`${displayLine(agent.id)}\tr${agent.revision}\t${displayLine(agent.status)}\t${displayLine(agent.customer.name)}\n`);if(value.next_cursor)stderr('Next cursor: '+displayLine(value.next_cursor)+'\n');}return 0;}
- if(['show','pause','resume'].includes(request.command)){if(options.json)stdout(JSON.stringify(value,null,2)+'\n');else{stdout(`Agent: ${displayLine(value.id)}\nCustomer: ${displayLine(value.customer.name)}\nGoal: ${displayText(value.goal)}\nRevision: ${value.revision}\nStatus: ${displayLine(value.status)}\n`);if(value.latest_run)renderRun(value.latest_run,stdout);}return 0;}
- let run=request.command==='agent-run'?value:value.run,output=value;
- if(request.wait){const waited=await withInterrupt(signal=>waitAgentRun(client,run,{...(options.timeout===undefined?{}:{timeout:request.timeout}),signal}));run=waited.run;output=request.command==='agent-run'?waited:{...value,...waited};}
- if(options.json)stdout(JSON.stringify(output,null,2)+'\n');else{renderRun(run,stdout);if(value.replayed)stderr('Recovered the original admission receipt; read its run for current findings.\n');if(request.key)stderr('Original request key: '+request.key+'\n');}
- if(output.wait&&!['completed','partial'].includes(output.wait.outcome)){if(output.wait.error&&!options.json)stderr(output.wait.error.message+'\n');stderr('Follow the same agent and run to continue. Waiting does not cancel server work.\n');return 1;}
- return request.wait||request.command==='agent-run'?run.status==='completed'?0:run.status==='partial'?2:ACTIVE.has(run.status)&&!request.wait?0:1:0;
-}

package/src/research-answer-contract.mjs

@@ -1,3 +0,0 @@
-import {createHash} from 'node:crypto';
-import {readAnswerContract} from './answer-contract.mjs';
-export const readResearchAnswerContract=run=>readAnswerContract(run,text=>createHash('sha256').update(text,'utf8').digest('hex'));

package/src/research-answer-view.mjs

@@ -1,7 +0,0 @@
-import {createHash} from 'node:crypto';
-import {readResearchAnswerContract} from './research-answer-contract.mjs';
-import {readAnswerView} from './answer-view.mjs';
-export function readResearchAnswerView(run){
- if(!Object.hasOwn(run.usage,'answer_view'))return {state:'absent',view:null,sections:[]};
- return readAnswerView(run,readResearchAnswerContract(run),text=>createHash('sha256').update(text,'utf8').digest('hex'));
-}