@getopencrawl/cli 0.1.1 → 0.2.1

package/src/product-monitors.mjs

@@ -1,321 +0,0 @@
-/** Private local browser observations. No browser state is sent to OpenCrawl. */
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { createInterface } from 'node:readline/promises';
-import { configDir } from './client.mjs';
-import { isReadOnlyRequest } from '../browser.mjs';
-import { displayLine } from './display.mjs';
-import { allowedOptions } from './jobs.mjs';
-import {LETTER_ADAPTER,isLetterPolicy,LetterReadAttempt} from './product-letter-read.mjs';
-import {LETTER_CAPABILITIES_ADAPTER,isLetterCapabilitiesPolicy,letterPickerReady,extractLetterCapabilities} from './product-letter-capabilities.mjs';
-
-const API='/api/v1/product-monitors';
-const UUID=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
-const HASH=/^[0-9a-f]{64}$/;
-const sha=value=>crypto.createHash('sha256').update(value).digest('hex');
-const same=(a,b)=>canonical(a)===canonical(b);
-export function canonical(value) {return JSON.stringify(value,(_,v)=>v&&typeof v==='object'&&!Array.isArray(v)?Object.fromEntries(Object.keys(v).sort().map(k=>[k,v[k]])):v);}
-export class MonitorFailure extends Error {constructor(code){super(code);this.code=code;}}
-const fail=code=>{throw new MonitorFailure(code);};
-const id=(value,prefix)=>{if(typeof value!=='string'||!new RegExp(`^${prefix}_[a-f0-9]{32}$`).test(value))fail('invalid_id');return value;};
-const monitorId=value=>{if(typeof value!=='string'||!/^src_[a-f0-9]{32}$/.test(value))fail('invalid_id');return value;};
-const uuid=value=>{if(typeof value!=='string'||!UUID.test(value))fail('invalid_capture_id');return value;};
-const count=(value,min,max)=>Number.isSafeInteger(value)&&value>=min&&value<=max;
-export function exactUrl(value) {
-  if(typeof value!=='string'||value.length>2048||/[\x00-\x20\x7f\\]/.test(value)||/%0[0-9a-f]|%1[0-9a-f]|%7f/i.test(value))fail('invalid_url');
-  let u;try{u=new URL(value);}catch{fail('invalid_url');}
-  if(u.protocol!=='https:'||u.username||u.password||u.href!==value)fail('invalid_url');
-  let pathname;try{pathname=decodeURIComponent(u.pathname);}catch{fail('invalid_url');}
-  if(/(?:^|\/)(?:logout|log-out|signout|sign-out|delete|remove|destroy|unsubscribe|deactivate|terminate)(?:\/|$)/i.test(pathname)||[...u.searchParams.keys()].some(k=>/^(action|delete|remove|logout|unsubscribe)$/i.test(k)))fail('action_url');
-  return u;
-}
-export function readMonitor(m,expected) {
-  if(!m||m.version!==1||monitorId(m.id)!==m.id||expected&&m.id!==expected||m.visibility!=='private'||!count(m.generation,1,Number.MAX_SAFE_INTEGER)||!['active','paused','revoked'].includes(m.status)||!Array.isArray(m.pages)||!count(m.pages.length,1,5)||!HASH.test(m.policy_sha256)||typeof m.account_label!=='string')fail('invalid_monitor');
-  const p=m.policy,v=p?.viewport;
-  if(sha(canonical(p))!==m.policy_sha256)fail('invalid_policy');
-  if(!(p?.version===1&&!Object.hasOwn(p,'request_adapter')||p?.version===2&&p.request_adapter===LETTER_ADAPTER||p?.version===3&&p.request_adapter===LETTER_CAPABILITIES_ADAPTER)||p.extractor!=='product-screen-v1'||p.screenshot_mode!=='viewport'||!count(v?.width,320,1920)||!count(v?.height,240,1080)||v.device_scale_factor!==1||!['light','dark'].includes(p.color_scheme)||typeof p.locale!=='string'||typeof p.timezone_id!=='string'||![0,60,360,1440].includes(m.cadence_minutes))fail('invalid_policy');
-  const selector=s=>typeof s==='string'&&s.length>0&&s.length<=400&&!/[\x00-\x1f\x7f]/.test(s);
-  for(const rows of [p.masks,p.noise_regions])if(!Array.isArray(rows)||rows.length>16||rows.some(r=>!selector(r?.selector)||typeof r.label!=='string'))fail('invalid_policy');
-  const seen=new Set();for(const page of m.pages){id(page.id,'ppg');if(seen.has(page.id))fail('invalid_monitor');seen.add(page.id);if(exactUrl(page.url).origin!==m.origin||!selector(page.ready_selector)||!selector(page.scope_selector)||page.login_selector!==null&&!selector(page.login_selector)||page.account_marker!==null&&(!selector(page.account_marker?.selector)||page.account_marker.required!==true))fail('invalid_monitor');}
-  if(p.version>=2&&(m.pages.length!==1||!(isLetterPolicy(p,m.pages[0])||isLetterCapabilitiesPolicy(p,m.pages[0]))))fail('invalid_policy');
-  return m;
-}
-async function request(client,route,options={}) {return client.request(route,{timeout:30000,maxResponseBytes:1024*1024,...options});}
-async function identity(client,signal) {
-  const me=await request(client,'/api/auth/me',{signal});
-  if(!/^usr_[0-9a-f]{32}$/.test(me?.user?.id)||!/^ws_[0-9a-f]{32}$/.test(me?.workspace?.id))fail('identity_unavailable');
-  if(client.profile&&(client.profile.userId!==me.user.id||client.profile.workspaceId!==me.workspace.id))fail('identity_changed');
-  return {server:client.baseUrl,user_id:me.user.id,workspace_id:me.workspace.id};
-}
-async function current(client,monitor,scope,signal) {
-  const who=await identity(client,signal);if(scope&&!same(who,scope))fail('identity_changed');
-  const m=readMonitor(await request(client,`${API}/${monitor.id}`,{signal}),monitor.id);
-  if(m.generation!==monitor.generation||m.policy_sha256!==monitor.policy_sha256||m.status!=='active')fail('monitor_changed');return m;
-}
-async function privateDir(directory){await fs.mkdir(directory,{recursive:true,mode:0o700});if((await fs.lstat(directory)).isSymbolicLink())fail('unsafe_local_path');await fs.chmod(directory,0o700);}
-async function write(file,value,beforeCommit=()=>{}){const temp=file+'.'+crypto.randomUUID()+'.tmp';try{const h=await fs.open(temp,'wx',0o600);try{await h.writeFile(JSON.stringify(value)+'\n');await h.sync();}finally{await h.close();}beforeCommit();await fs.rename(temp,file);if(process.platform!=='win32'){const d=await fs.open(path.dirname(file),'r');try{await d.sync();}finally{await d.close();}}}finally{await fs.unlink(temp).catch(e=>{if(e.code!=='ENOENT')throw e;});}}
-async function read(file,max=4*1024*1024){try{const h=await fs.open(file,'r');try{const s=await h.stat();if(!s.isFile()||s.size>max)fail('invalid_local_state');return JSON.parse(await h.readFile('utf8'));}finally{await h.close();}}catch(e){if(e.code==='ENOENT')return null;throw e;}}
-async function lock(directory,fn){await privateDir(directory);let h;try{h=await fs.open(path.join(directory,'lock'),'wx',0o600);}catch(e){if(e.code==='EEXIST')fail('profile_locked');throw e;}try{await h.writeFile(String(process.pid));return await fn();}finally{await h.close();await fs.unlink(path.join(directory,'lock'));}}
-function profilePath(name){if(typeof name!=='string'||!/^[A-Za-z0-9_-]{1,64}$/.test(name))fail('invalid_profile');return path.join(configDir(),'product-monitor-profiles',name);}
-function operationPath(scope,monitor){return path.join(configDir(),'product-monitor-operations',sha(canonical({...scope,monitor})));}
-function checkUpload(u,m) {
-  if(u.outcome==='captured'){
-    const png=Buffer.from(u.screenshot_png_base64,'base64');
-    if(Buffer.byteLength(u.text)>131072||png.length>2097152||png.length<24||!png.subarray(0,8).equals(Buffer.from([137,80,78,71,13,10,26,10]))||png.readUInt32BE(16)!==m.policy.viewport.width||png.readUInt32BE(20)!==m.policy.viewport.height)fail('capture_too_large_or_invalid');
-    if(Array.from(u.title).length>256||Buffer.byteLength(u.title)>1024)fail('title_too_large');
-  }
-  if(Buffer.byteLength(JSON.stringify(u))>3145728)fail('capture_too_large');return u;
-}
-export function receipt(value,u,monitor) {
-  if(value?.version!==1||id(value.id,'pcap')!==value.id||value.client_capture_id!==u.client_capture_id||value.monitor_id!==monitor||value.page_id!==u.page_id||value.generation!==u.expected_generation||value.outcome!==u.outcome||value.observed_at!==u.observed_at||value.manifest_sha256!==sha(canonical(u)))fail('receipt_mismatch');
-  if(u.outcome==='captured'){
-    const s=value.snapshot;id(s?.id,'psnap');
-    if(!HASH.test(s.version)||s.text_sha256!==sha(u.text)||s.text_bytes!==Buffer.byteLength(u.text)||s.screenshot_sha256!==sha(Buffer.from(u.screenshot_png_base64,'base64'))||s.screenshot_bytes!==Buffer.from(u.screenshot_png_base64,'base64').length||s.policy_sha256!==u.policy_sha256)fail('receipt_mismatch');
-  }else if(value.snapshot!==null||value.comparison!==null)fail('receipt_mismatch');
-  return value;
-}
-export async function sendCapture(client,monitor,u,directory,signal){
-  const file=path.join(directory,u.client_capture_id+'.json');
-  if(await read(file))fail('capture_already_attempted');
-  await unresolved(directory,Buffer.byteLength(JSON.stringify({version:1,state:'intended',monitor,upload:u}))+1);
-  await write(file,{version:1,state:'intended',monitor,upload:u});
-  let result;
-  try{result=await request(client,`${API}/${monitor}/captures`,{method:'POST',body:u,signal});}
-  catch{try{result=await request(client,`${API}/${monitor}/captures/by-client-id/${u.client_capture_id}`,{signal});}catch{fail('upload_unconfirmed');}}
-  result=receipt(result,u,monitor);await write(file,{version:1,state:'confirmed',monitor,manifest_sha256:result.manifest_sha256,receipt:result});return result;
-}
-async function unresolved(directory,additionalBytes=0){const names=await fs.readdir(directory);if(names.length+(additionalBytes?2:0)>256)fail('local_history_full');let bytes=additionalBytes*2;for(const name of names){bytes+=(await fs.lstat(path.join(directory,name))).size;if(bytes>64*1024*1024)fail('local_history_full');if(!UUID.test(name.slice(0,-5))||!name.endsWith('.json'))continue;const record=await read(path.join(directory,name));if(record?.state!=='confirmed')fail('upload_unconfirmed');}}
-export async function loginPrompt(message,{signal,input=process.stdin,output=process.stderr}={}){if(!input.isTTY)fail('interactive_login_required');const r=createInterface({input,output});try{await r.question(message,{signal});}catch(error){if(error?.name==='AbortError')fail('cancelled');throw error;}finally{r.close();}}
-async function browser(overrides){try{return overrides.playwright||await import('playwright');}catch{fail('browser_setup_required');}}
-async function launch(pw,directory,m,headed=false,timeout){
-  const context=await pw.chromium.launchPersistentContext(path.join(directory,'chromium'),{headless:!headed,acceptDownloads:false,permissions:[],serviceWorkers:'block',viewport:{width:m.policy.viewport.width,height:m.policy.viewport.height},deviceScaleFactor:1,locale:m.policy.locale,timezoneId:m.policy.timezone_id,colorScheme:m.policy.color_scheme,reducedMotion:'reduce',...(timeout===undefined?{}:{timeout})});
-  if(!headed){try{await context.route('**/*',route=>route.abort('blockedbyclient'));if(typeof context.routeWebSocket!=='function')fail('browser_version_unsupported');await context.routeWebSocket('**/*',socket=>socket.close());for(const page of context.pages())await page.close();}catch(error){await context.close();throw error;}}
-  return context;
-}
-
-/** A Boolean readiness hint only; headless capture remains authoritative. */
-export function loginScreenReady({screen,expected,capabilities=false}) {
-  if(location.href!==screen.url||!screen.account_marker||typeof expected!=='string'||!expected)return false;
-  const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
-  const all=s=>Array.from(document.querySelectorAll(s)).filter(visible);
-  const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
-  if(screen.login_selector&&all(screen.login_selector).length)return false;
-  return Boolean(one(screen.ready_selector)&&(capabilities||one(screen.scope_selector))&&one(screen.account_marker.selector)?.innerText?.trim()===expected);
-}
-function loginActive(signal){if(signal.aborted)throw signal.reason instanceof MonitorFailure?signal.reason:new MonitorFailure('cancelled');}
-export async function waitForLogin(page,screen,markers,signal,manual=loginPrompt,capabilities=false){
-  const waiting=new AbortController();const abort=()=>waiting.abort(signal.reason);signal.addEventListener('abort',abort,{once:true});
-  try{
-    loginActive(signal);
-    const automatic=(async()=>{while(true){
-      loginActive(waiting.signal);if(page.isClosed?.())fail('login_window_closed');
-      if(screen.account_marker&&page.url()===screen.url){
-        try{if(await page.evaluate(loginScreenReady,{screen,expected:markers[screen.id],...(capabilities?{capabilities:true}:{})})===true)return;}
-        catch{loginActive(waiting.signal);if(page.isClosed?.())fail('login_window_closed');} // Normal authentication navigation can replace the DOM.
-      }
-      await sleep(1000,waiting.signal);
-    }})();
-    await Promise.race([automatic,Promise.resolve().then(()=>manual('Sign in in the browser; we\u2019ll continue when ready. If needed, return here and press Enter. (10-minute limit; Ctrl+C cancels.) ',{signal:waiting.signal}))]);
-    loginActive(signal);
-  }finally{waiting.abort();signal.removeEventListener('abort',abort);}
-}
-
-/** Runs synchronously in the page. Neither hidden text nor excluded text is returned. */
-export function extractScreen({screen,policy,expected,freeze=false}) {
-  const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
-  const all=s=>Array.from(document.querySelectorAll(s)).filter(visible);
-  const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
-  const login=screen.login_selector&&all(screen.login_selector).length>0;
-  if(login)return {error:'session_expired',login:true};
-  if(!one(screen.ready_selector))return {error:'not_ready'};
-  const scope=one(screen.scope_selector);if(!scope)return {error:'scope_mismatch'};
-  let marker=null,markerNode=null;
-  if(screen.account_marker){const node=one(screen.account_marker.selector);markerNode=node;marker=node?.innerText?.trim();if(!marker||typeof expected!=='string'||marker!==expected)return {error:'context_mismatch'};}
-  const selectors=[...policy.masks,...policy.noise_regions].map(r=>r.selector);
-  const masks=[];for(const s of selectors){const a=all(s);if(!a.length||a.length>100)return {error:'mask_unresolved'};masks.push(...a);}
-  masks.push(...document.querySelectorAll('input,textarea,select,[contenteditable="true"]'));
-  if(markerNode)masks.push(markerNode); // Only the marker digest is eligible for upload.
-  const excluded=node=>masks.some(m=>m===node||m.contains(node));
-  let nodes=0;
-  const walk=node=>{if(++nodes>20000)throw Error('bounds');if(node.nodeType===Node.TEXT_NODE)return node.textContent;if(node.nodeType!==Node.ELEMENT_NODE)return '';if(excluded(node)||!visible(node)||['SCRIPT','STYLE','NOSCRIPT','TEMPLATE'].includes(node.tagName))return '';return Array.from(node.childNodes).map(walk).join('')+(/^(P|DIV|SECTION|ARTICLE|H[1-6]|LI|TR|BR)$/.test(node.tagName)?'\n':' ');};
-  const text=walk(scope).replace(/[ \t]+/g,' ').replace(/ *\n */g,'\n').trim();
-  if(!text)return {error:'not_ready'};
-  if(freeze){
-    // Hide the entire composited subtree; descendants cannot override parent opacity.
-    // This happens before observation, and no unmasked screenshot is ever taken.
-    for(const node of masks)node.style.setProperty('opacity','0','important');
-    window.__opencrawlMasks=masks;window.__opencrawlChanged=false;
-    window.__opencrawlObserver?.disconnect();
-    window.__opencrawlObserver=new MutationObserver(rows=>{if(rows.some(r=>!excluded(r.target)||r.type==='attributes'&&masks.includes(r.target)))window.__opencrawlChanged=true;});
-    window.__opencrawlObserver.observe(document.documentElement,{attributes:true,childList:true,characterData:true,subtree:true});
-  }
-  return {text,title:screen.label,marker,maskCount:masks.length};
-}
-// deadlineMs can only shorten the 60s capture deadline (tests use it); it never extends it.
-export async function captureScreen(context,m,screen,binding,{signal,onDiagnostics,deadlineMs=60000}={}) {
-  const limit=Number.isFinite(deadlineMs)?Math.min(60000,Math.max(1000,deadlineMs)):60000;
-  if(m.policy.version!==1&&!isLetterPolicy(m.policy,screen)&&!isLetterCapabilitiesPolicy(m.policy,screen))fail('invalid_policy');
-  const page=await context.newPage();page.on('popup',popup=>popup.close().catch(()=>{}));let blocked=0,captureGuard='not_reached';const outerBlocks={navigation_scope:0,approved_read_redirect:0,non_read:0};const outerBlock=kind=>{outerBlocks[kind]=Math.min(64,outerBlocks[kind]+1);};
-  const base={version:1,client_capture_id:crypto.randomUUID(),page_id:screen.id,expected_generation:m.generation,policy_sha256:m.policy_sha256,expected_previous_snapshot_id:screen.latest_snapshot?.id||null,session_epoch:binding.session_epoch,observed_at:new Date().toISOString(),requested_url:screen.url,final_url:null,outcome:'capture_failed',account:{label:m.account_label,assurance:'declared_only',marker_sha256:null},ready_matched:false,login_marker_matched:false,title:null,text:null,screenshot_png_base64:null};
-  const capabilities=isLetterCapabilitiesPolicy(m.policy,screen),deadline=performance.now()+limit;
-  const adapter=isLetterPolicy(m.policy,screen)||capabilities?new LetterReadAttempt(page,deadline,capabilities):null;
-  const abort=()=>{if(capabilities)adapter.close();return page.close().catch(()=>{});};signal?.addEventListener('abort',abort,{once:true});const timer=setTimeout(abort,limit);
-  try{
-    await page.route('**/*',async route=>{const req=route.request();const navigation=req.isNavigationRequest()&&req.frame()===page.mainFrame();const requested=req.url();const wanted=new URL(screen.url);wanted.hash='';if(adapter&&req.redirectedFrom()&&adapter.requests.has(req.redirectedFrom())){adapter.reject(req,'redirect','approved_read_redirect');blocked++;outerBlock('approved_read_redirect');await route.abort('blockedbyclient');}
-      else if(navigation&&requested!==wanted.href&&requested!==screen.url){blocked++;outerBlock('navigation_scope');adapter?.recordDenial(req,'selected_page','navigation_scope');await route.abort('blockedbyclient');}
-      else if(adapter?!['GET','HEAD','OPTIONS'].includes(req.method().toUpperCase()):!isReadOnlyRequest(req.method(),req.postData())){const decision=adapter?.request(req);if(decision==='allow')await adapter.forward(route);else{if(decision!=='block_known'){blocked++;outerBlock('non_read');}await route.abort('blockedbyclient');}}
-      else await route.continue();});
-    if(typeof page.routeWebSocket!=='function')fail('browser_version_unsupported');await page.routeWebSocket('**/*',socket=>socket.close());
-    if(signal?.aborted)fail('cancelled');
-    const response=await page.goto(screen.url,{waitUntil:'domcontentloaded',timeout:30000});
-    if(signal?.aborted)fail('cancelled');
-    if(page.url()!==screen.url){base.outcome='scope_mismatch';return base;}
-    if(response&&response.status()>=400){base.outcome='capture_failed';return base;}
-    await page.locator(screen.ready_selector).waitFor({state:'visible',timeout:5000}).catch(()=>{});
-    if(adapter&&screen.login_selector&&await page.evaluate(selector=>Array.from(document.querySelectorAll(selector)).some(node=>{const style=getComputedStyle(node),rect=node.getBoundingClientRect();return style.display!=='none'&&style.visibility!=='hidden'&&style.opacity!=='0'&&rect.width>0&&rect.height>0;}),screen.login_selector)){base.outcome='session_expired';base.login_marker_matched=true;return base;}
-    if(adapter&&!await adapter.ready(signal)){if(signal?.aborted)fail('cancelled');captureGuard='readiness_refused';base.outcome='request_blocked';return base;}
-    if(capabilities){
-      let ready=await page.evaluate(letterPickerReady,{screen,expected:binding.markers[screen.id]});
-      // Completed reads can precede the page's own re-render that enables the opener.
-      // Re-run the same fail-closed check briefly within the original deadline; never click early.
-      for(const settle=Math.min(performance.now()+2000,deadline);ready.error==='not_ready'&&performance.now()<settle;){
-        await new Promise(resolve=>setTimeout(resolve,100));if(signal?.aborted)fail('cancelled');
-        ready=await page.evaluate(letterPickerReady,{screen,expected:binding.markers[screen.id]});
-      }
-      if(ready.error){base.outcome=ready.error;base.login_marker_matched=ready.login===true;return base;}
-      const remaining=()=>{const ms=Math.floor(deadline-performance.now());if(ms<=0)fail('timeout');return ms;};
-      // The only interaction this adapter permits: open the observed provider picker once.
-      await page.locator('[data-testid="connect-integration-open"]').click({timeout:remaining()});
-      await page.locator('[data-testid="connect-integration"]').waitFor({state:'visible',timeout:remaining()});
-      // Playwright visibility permits opacity zero during Letter's entrance animation.
-      // Wait within the original capture deadline before freezing the unchanged mask set.
-      await page.waitForFunction(()=>{
-        const nodes=document.querySelectorAll('[data-testid="connect-integration"]');if(nodes.length!==1)return false;
-        const node=nodes[0],style=getComputedStyle(node),rect=node.getBoundingClientRect();
-        return node.getAttribute('role')==='dialog'&&style.display!=='none'&&style.visibility!=='hidden'&&style.opacity==='1'&&rect.width>0&&rect.height>0&&node.getAnimations().every(animation=>!animation.pending&&['finished','idle'].includes(animation.playState));
-      },null,{timeout:remaining()});
-      if(!await adapter.ready(signal)){captureGuard='readiness_refused';base.outcome='request_blocked';return base;}
-    }
-    await page.evaluate(()=>{window.scrollTo(0,0);});
-    const data=await page.evaluate(capabilities?extractLetterCapabilities:extractScreen,{screen,policy:m.policy,expected:binding.markers[screen.id],freeze:true});
-    if(data.error){base.outcome=data.error;base.login_marker_matched=data.login===true;return base;}
-    const png=await page.screenshot({type:'png',fullPage:false,animations:'disabled',caret:'initial',timeout:10000});
-    const changed=await page.evaluate(()=>window.__opencrawlChanged===true||window.__opencrawlMasks?.some(node=>!node.isConnected||getComputedStyle(node).opacity!=='0')); 
-    if(changed||page.url()!==screen.url){base.outcome='not_ready';return base;}
-    if(blocked||adapter&&(adapter.invalid||adapter.inFlight>0||adapter.required!=='verified')){captureGuard='final_request_refused';base.outcome='request_blocked';return base;}
-    captureGuard='passed';
-    return checkUpload({...base,outcome:'captured',observed_at:new Date().toISOString(),final_url:screen.url,ready_matched:true,title:data.title,text:data.text,screenshot_png_base64:png.toString('base64'),account:{label:m.account_label,assurance:data.marker?'dom_marker_matched':'declared_only',marker_sha256:data.marker?sha(data.marker):null}},m);
-  }catch(e){if(signal?.aborted)fail('cancelled');base.outcome=e?.name==='TimeoutError'?'timeout':'capture_failed';return base;}
-  finally{clearTimeout(timer);signal?.removeEventListener('abort',abort);adapter?.close();await page.close().catch(()=>{});if(adapter)await onDiagnostics?.(base.client_capture_id,{...adapter.report(),capture_guard:captureGuard,outer_blocks:outerBlocks});}
-}
-async function sleep(ms,signal){if(signal?.aborted)fail('cancelled');await new Promise((resolve,reject)=>{const stop=()=>{clearTimeout(t);signal?.removeEventListener('abort',stop);reject(new MonitorFailure('cancelled'));};const t=setTimeout(()=>{signal?.removeEventListener('abort',stop);resolve();},ms);signal?.addEventListener('abort',stop,{once:true});});}
-export async function monitorRunner(action,m,client,options={},overrides={}){
-  const scope=await identity(client,overrides.signal);const directory=profilePath(options.profile);
-  return lock(directory,async()=>{
-    const file=path.join(directory,'binding.json');let binding=await read(file);
-    const expected={...scope,monitor_id:m.id,generation:m.generation,policy_sha256:m.policy_sha256,origin:m.origin};
-    if(binding&&!same(binding.scope,expected))fail('profile_binding_changed');
-    const operations=operationPath(scope,m.id);await privateDir(operations);
-    return lock(operations,async()=>{
-    await unresolved(operations);
-    const onDiagnostics=async(captureId,report)=>{const bytes=Buffer.byteLength(JSON.stringify(report)+'\n');if(bytes>16384)fail('diagnostics_too_large');await unresolved(operations,bytes);await write(path.join(operations,'diagnostics-'+captureId+'.json'),report);};
-    const pw=await browser(overrides);
-    if(action==='login'){
-      let markers=options.markers?await read(path.resolve(options.markers),16384):binding?.markers||{};
-      if(!markers||Array.isArray(markers)||typeof markers!=='object'||Object.entries(markers).some(([k,v])=>!m.pages.some(p=>p.id===k)||typeof v!=='string'||!v.trim()||v.length>500))fail('invalid_markers');
-      if(m.pages.some(p=>p.account_marker&&!markers[p.id]))fail('markers_required');
-      if(binding&&!same(binding.markers,markers))fail('profile_binding_changed');
-      if(!overrides.waitForLogin&&!process.stdin.isTTY)fail('interactive_login_required');
-      const duration=overrides.loginTimeoutMs??600000;if(!count(duration,1,600000))fail('invalid_login_timeout');
-      const controller=new AbortController(),signal=controller.signal,deadline=performance.now()+duration;
-      const stop=()=>controller.abort(new MonitorFailure('cancelled'));
-      const active=()=>{if(performance.now()>=deadline)controller.abort(new MonitorFailure('login_timeout'));loginActive(signal);};
-      const remaining=()=>{active();return Math.max(1,Math.min(30000,Math.ceil(deadline-performance.now())));};
-      let context,closing;const close=()=>{if(context&&!closing)closing=Promise.resolve().then(()=>context.close());return closing||Promise.resolve();};
-      const onAbort=()=>{void close().catch(()=>{});};signal.addEventListener('abort',onAbort,{once:true});
-      overrides.signal?.addEventListener('abort',stop,{once:true});if(overrides.signal?.aborted)stop();
-      process.on('SIGINT',stop);process.on('SIGTERM',stop);
-      const timer=setTimeout(()=>controller.abort(new MonitorFailure('login_timeout')),duration);
-      try{
-        context=await launch(pw,directory,m,true,remaining());active();
-        const page=context.pages()[0]||await context.newPage();active();await page.goto(m.pages[0].url,{waitUntil:'domcontentloaded',timeout:remaining()});active();
-        await waitForLogin(page,m.pages[0],markers,signal,overrides.waitForLogin||loginPrompt,isLetterCapabilitiesPolicy(m.policy,m.pages[0]));active();await current(client,m,scope,signal);active();
-        binding={version:1,scope:expected,markers,session_epoch:crypto.randomUUID()};
-        await close();context=null;closing=null;active();
-        context=await launch(pw,directory,m,false,remaining());active();
-        for(const screen of m.pages){active();const observed=await captureScreen(context,m,screen,binding,{signal,onDiagnostics});active();if(observed.outcome!=='captured')fail(observed.outcome);}
-        await current(client,m,scope,signal);active();await write(file,binding,active);
-      }catch(error){active();throw error;}
-      finally{try{await close();}finally{clearTimeout(timer);signal.removeEventListener('abort',onAbort);overrides.signal?.removeEventListener('abort',stop);process.off('SIGINT',stop);process.off('SIGTERM',stop);}}
-      return {state:'session_saved_locally',monitor_id:m.id,profile:options.profile};
-    }
-    if(!binding)fail('local_login_required');
-    if(Number(options.generation)!==m.generation)fail('generation_required');
-    if(action==='watch'&&m.cadence_minutes===0)fail('cadence_required');
-    const controller=new AbortController();const signal=overrides.signal||controller.signal;const stop=()=>controller.abort();if(!overrides.signal){process.on('SIGINT',stop);process.on('SIGTERM',stop);}
-    let context;try{context=await launch(pw,directory,m,false);}catch(e){if(!overrides.signal){process.off('SIGINT',stop);process.off('SIGTERM',stop);}throw e;}
-    const records=[];
-    try{do{
-      await unresolved(operations);const fresh=await current(client,m,scope,signal);
-      for(const screen of fresh.pages){
-        if(signal.aborted)fail('cancelled');await current(client,m,scope,signal);
-        const upload=await captureScreen(context,m,screen,binding,{signal,onDiagnostics});await current(client,m,scope,signal);
-        if(action==='preview'){
-          if((await fs.readdir(directory)).filter(n=>n.startsWith('preview-')).length>=20)fail('local_preview_full');
-          const preview=path.join(directory,'preview-'+upload.client_capture_id);await privateDir(preview);await write(path.join(preview,'observation.json'),upload);if(upload.outcome==='captured')await fs.writeFile(path.join(preview,'masked.png'),Buffer.from(upload.screenshot_png_base64,'base64'),{mode:0o600});records.push({state:upload.outcome,preview});
-        }else {const r=await sendCapture(client,m.id,upload,operations,signal);records.push(r);overrides.onReceipt?.(r);}
-        if(upload.outcome!=='captured')fail(upload.outcome);
-      }
-      if(action!=='watch')break;
-      await (overrides.sleep||sleep)(m.cadence_minutes*60000,signal);
-      records.splice(0); // A long-running watch never accumulates unbounded output in memory.
-    }while(!signal.aborted);
-    return {state:'complete',captures:records};
-    }finally{await context.close();if(!overrides.signal){process.off('SIGINT',stop);process.off('SIGTERM',stop);}}
-    });
-  });
-}
-export const PRODUCT_MONITORS_HELP=`Local private product monitoring\n\n  opencrawl browser monitor create --spec FILE\n  opencrawl browser monitor list|show [MONITOR_ID]\n  opencrawl browser monitor login MONITOR_ID --profile NAME [--markers FILE]\n  opencrawl browser monitor preview|check|watch MONITOR_ID --profile NAME --generation N\n  opencrawl browser monitor recover MONITOR_ID --capture UUID\n  opencrawl browser monitor changes MONITOR_ID\n  opencrawl browser monitor compare COMPARISON_ID --revision HASH\n  opencrawl browser monitor image SNAPSHOT_ID --revision HASH --output NEW.png\n  opencrawl browser monitor context COMPARISON_ID --revision HASH [--budget 4096]\n\nUse --account PROFILE and --server ORIGIN for the intended OpenCrawl account.\nmarkers FILE is private JSON {page_id: expected visible account/workspace text}.\nBrowser sessions stay here. Watch runs in the foreground; this computer must stay on.\nUnknown uploads stop without resend. Failures retain the last good snapshot.\n`;
-async function command(rest,options,client,overrides={}){
-  const [action,value,...extra]=rest;const out=overrides.stdout||((s)=>process.stdout.write(s));
-  const allowed={create:['spec'],list:[],show:[],login:['profile','markers'],preview:['profile','generation'],check:['profile','generation'],watch:['profile','generation'],recover:['capture'],changes:[],compare:['revision'],image:['revision','output'],context:['revision','budget']};
-  if(!Object.hasOwn(allowed,action)||extra.length)fail('invalid_command');allowedOptions(options,allowed[action],'browser monitor');
-  let result;
-  if(action==='list'){result=await request(client,API);if(!Array.isArray(result?.monitors)||result.monitors.length>10)fail('invalid_monitor_list');result.monitors.forEach(m=>readMonitor(m));}
-  else if(action==='create'){
-    if(value||typeof options.spec!=='string')fail('spec_required');const input=await read(path.resolve(options.spec),65536);if(!input)fail('spec_required');uuid(input.client_create_id);const scope=await identity(client);const dir=operationPath(scope,'create');await privateDir(dir);
-    result=await lock(dir,async()=>{const file=path.join(dir,input.client_create_id+'.json');const prior=await read(file);if(prior&&!same(prior.input,input))fail('creation_conflict');let value;
-      if(prior){try{value=await request(client,`${API}/by-client-id/${input.client_create_id}`);}catch{fail('creation_unconfirmed');}}
-      else {await write(file,{state:'intended',input});try{value=await request(client,API,{method:'POST',body:input});}catch{try{value=await request(client,`${API}/by-client-id/${input.client_create_id}`);}catch{fail('creation_unconfirmed');}}}
-      readMonitor(value);const pages=input.pages.map(p=>({login_selector:null,account_marker:null,...p}));const policy={version:1,color_scheme:'light',screenshot_mode:'viewport',extractor:'product-screen-v1',masks:[],noise_regions:[],...input.policy,viewport:{device_scale_factor:1,...input.policy?.viewport}};if(value.name!==input.name||value.account_label!==input.account_label||value.cadence_minutes!==(input.cadence_minutes??0)||!same(value.policy,policy)||!same(value.pages.map(({id,last_attempt,latest_snapshot,...p})=>p),pages))fail('receipt_mismatch');await write(file,{state:'confirmed',input,monitor:value});return value;});
-  }else if(action==='image'){
-    id(value,'psnap');if(!HASH.test(options.revision)||typeof options.output!=='string')fail('image_output_required');
-    const snapshot=await request(client,`${API}/snapshots/${value}`);if(snapshot?.id!==value||snapshot.version!==options.revision)fail('snapshot_changed');
-    const bytes=await request(client,`${API}/snapshots/${value}/image`,{responseType:'png'});
-    if(bytes.length!==snapshot.screenshot_bytes||sha(bytes)!==snapshot.screenshot_sha256)fail('image_mismatch');
-    const output=path.resolve(options.output),handle=await fs.open(output,'wx',0o600);try{await handle.writeFile(bytes);await handle.sync();}finally{await handle.close();}result={snapshot_id:value,version:options.revision,output};
-  }else if(action==='context'){
-    id(value,'pcmp');if(!HASH.test(options.revision))fail('version_required');const budget=options.budget===undefined?4096:Number(options.budget);if(!count(budget,256,16384))fail('invalid_budget');
-    result=await request(client,'/api/v1/product-monitor-context',{method:'POST',body:{selections:[{comparison_id:value,version:options.revision}],budget}});
-    if(typeof result?.context!=='string'||Buffer.byteLength(result.context)!==result.context_bytes||result.context_bytes>budget||result.budget!==budget||!Array.isArray(result.included)||!Array.isArray(result.omitted)||result.included.some(r=>r.comparison_id!==value||r.version!==options.revision||!['before','after'].includes(r.side))||result.omitted.some(r=>r.comparison_id!==value||!['before','after'].includes(r.side)||!['budget','no_text_region'].includes(r.reason))||result.included.length+result.omitted.length!==2||new Set([...result.included,...result.omitted].map(r=>r.side)).size!==2||result.image_content_included!==false)fail('context_mismatch');
-  }else if(action==='compare'){
-    id(value,'pcmp');if(!HASH.test(options.revision))fail('version_required');result=await request(client,`${API}/comparisons/${value}`);if(result?.id!==value||result.version!==options.revision)fail('comparison_changed');
-  }else{
-    monitorId(value);
-    if(action==='recover'){
-      uuid(options.capture);const scope=await identity(client);const dir=operationPath(scope,value);result=await lock(dir,async()=>{const file=path.join(dir,options.capture+'.json');const saved=await read(file);if(!saved||saved.monitor!==value)fail('capture_unknown');const raw=await request(client,`${API}/${value}/captures/by-client-id/${options.capture}`);let got;if(saved.state==='confirmed'){if(raw?.client_capture_id!==options.capture||raw.monitor_id!==value||raw.manifest_sha256!==saved.manifest_sha256||!same(raw,saved.receipt))fail('receipt_mismatch');got=raw;}else got=receipt(raw,saved.upload,value);await write(file,{version:1,monitor:value,state:'confirmed',manifest_sha256:got.manifest_sha256,receipt:got});return got;});
-    }else if(action==='changes'){result=await request(client,`${API}/${value}/comparisons`);if(!Array.isArray(result?.comparisons))fail('invalid_comparisons');}
-    else{const m=readMonitor(await request(client,`${API}/${value}`),value);if(action==='show')result=m;else{if(m.status!=='active')fail('monitor_not_active');result=await monitorRunner(action,m,client,options,{...overrides,onReceipt:action==='watch'?r=>out(JSON.stringify(r)+'\n'):undefined});}}
-  }
-  if(options.json)out(JSON.stringify(result,null,2)+'\n');else if(result?.comparisons)out(result.comparisons.map(x=>`${displayLine(x.id)} ${displayLine(x.version)} ${displayLine(x.observed_before)} → ${displayLine(x.observed_after)}\n`).join('')||'No retained comparisons.\n');else out(JSON.stringify(result,null,2)+'\n');return 0;
-}
-
-export async function productMonitorsCommand(rest,options,client,overrides={}) {
-  try {return await command(rest,options,client,overrides);}
-  catch(error) {
-    if(error instanceof MonitorFailure)throw error;
-    const code=Number.isInteger(error?.status)&&error.status>=400&&error.status<=599?`http_${error.status}`:error?.code==='EEXIST'?'output_exists':'request_unconfirmed';
-    throw new MonitorFailure(code);
-  }
-}

package/src/product-monitors.test.mjs

@@ -1,69 +0,0 @@
-import {test} from 'node:test';
-import assert from 'node:assert/strict';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import {canonical,exactUrl,readMonitor,receipt,sendCapture,monitorRunner,productMonitorsCommand} from './product-monitors.mjs';
-const f=JSON.parse(await fs.readFile(new URL('./fixtures/product-monitor-contract.json',import.meta.url),'utf8'));
-const sha=x=>crypto.createHash('sha256').update(x).digest('hex');
-const fresh=()=>structuredClone(f.monitor);
-const identity={user:{id:'usr_'+'1'.repeat(32)},workspace:{id:'ws_'+'2'.repeat(32)}};
-const fakeReceipt=(u,m=f.monitor.id)=>({...structuredClone(f.baseline_receipt),client_capture_id:u.client_capture_id,page_id:u.page_id,monitor_id:m,generation:u.expected_generation,manifest_sha256:sha(canonical(u)),observed_at:u.observed_at,outcome:u.outcome,snapshot:u.outcome==='captured'?{...f.baseline_receipt.snapshot,text_sha256:sha(u.text),text_bytes:Buffer.byteLength(u.text),screenshot_sha256:sha(Buffer.from(u.screenshot_png_base64,'base64')),screenshot_bytes:Buffer.from(u.screenshot_png_base64,'base64').length,policy_sha256:u.policy_sha256}:null,comparison:null});
-async function temp(fn){const dir=await fs.mkdtemp(path.join(os.tmpdir(),'product-monitor-'));const prior=process.env.OPENCRAWL_CONFIG_DIR;process.env.OPENCRAWL_CONFIG_DIR=dir;try{await fn(dir);}finally{if(prior===undefined)delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR=prior;await fs.rm(dir,{recursive:true,force:true});}}
-test('actual backend fixture monitor and canonical manifest match; hashes/failure/scope changes refuse',()=>{
- assert.equal(readMonitor(f.monitor).id,f.monitor.id);assert.equal(sha(canonical(f.capture_request)),f.baseline_receipt.manifest_sha256);assert.deepEqual(receipt(f.baseline_receipt,f.capture_request,f.monitor.id),f.baseline_receipt);
- for(const change of [{manifest_sha256:'0'.repeat(64)},{monitor_id:'src_'+'0'.repeat(32)},{page_id:'ppg_'+'0'.repeat(32)},{generation:2},{outcome:'timeout'}])assert.throws(()=>receipt({...f.baseline_receipt,...change},f.capture_request,f.monitor.id));
-});
-test('fragments retained; action/credential/control/ambiguous URLs refused',()=>{assert.equal(exactUrl('https://product.example/app#overview').hash,'#overview');for(const u of ['https://u:p@product.example/','http://product.example/','https://product.example/logout','https://product.example/?action=x','https://product.example','https://product.example/\n'])assert.throws(()=>exactUrl(u));});
-test('lost upload acknowledgement performs one exact GET and never repeats POST; confirmed local bytes are minimized',()=>temp(async dir=>{
- const calls=[],u=structuredClone(f.capture_request);const client={request:async(r,o)=>{calls.push([r,o.method||'GET']);if(o.method==='POST')throw Error('lost');return fakeReceipt(u);}};
- await sendCapture(client,f.monitor.id,u,dir);assert.deepEqual(calls.map(c=>c[1]),['POST','GET']);const saved=JSON.parse(await fs.readFile(path.join(dir,u.client_capture_id+'.json')));assert.equal(saved.state,'confirmed');assert.equal(saved.upload,undefined);assert.equal(saved.manifest_sha256,sha(canonical(u)));
- await assert.rejects(sendCapture(client,f.monitor.id,u,dir),/capture_already_attempted/);assert.equal(calls.length,2);
-}));
-test('unconfirmed receipt remains durable, with no new POST on repeated use',()=>temp(async dir=>{let calls=0;const c={request:async()=>{calls++;throw Error('network');}};await assert.rejects(sendCapture(c,f.monitor.id,f.capture_request,dir),/upload_unconfirmed/);assert.equal(calls,2);const saved=JSON.parse(await fs.readFile(path.join(dir,f.capture_request.client_capture_id+'.json')));assert.equal(saved.state,'intended');await assert.rejects(sendCapture(c,f.monitor.id,f.capture_request,dir),/capture_already_attempted/);assert.equal(calls,2);}));
-test('failure receipt has no image/text and confirmed mismatch does not become success',()=>temp(async dir=>{const u={...f.capture_request,outcome:'session_expired',final_url:null,text:null,title:null,screenshot_png_base64:null,ready_matched:false,login_marker_matched:true};const r=fakeReceipt(u);assert.equal(receipt(r,u,f.monitor.id).snapshot,null);await assert.rejects(sendCapture({request:async()=>({...r,manifest_sha256:'f'.repeat(64)})},f.monitor.id,u,dir),/receipt_mismatch/);}));
-function fakeBrowser(m,{outcome,mutate,onCapture}={}){
- let closes=0,captures=0;const png=Buffer.from(f.capture_request.screenshot_png_base64,'base64');
- const pw={chromium:{launchPersistentContext:async()=>{const context={route:async()=>{},routeWebSocket:async()=>{},pages:()=>[],close:async()=>{closes++;},newPage:async()=>{let url='about:blank';return {on(){},close:async()=>{},route:async()=>{},routeWebSocket:async()=>{},mainFrame:()=>({}),goto:async u=>{url=u;return{status:()=>200};},url:()=>url,locator:()=>({waitFor:async()=>{}}),evaluate:async(fn,args)=>{if(args){captures++;onCapture?.(captures);return outcome?{error:outcome}:{text:'Observed workflow',title:'Overview',marker:'Example account'};}if(String(fn).includes('__opencrawlChanged'))return false;},screenshot:async()=>{mutate?.();return png;}};}};return context;}}};return {pw,get closes(){return closes;},get captures(){return captures;}};
-}
-async function loginFixture(dir,m,browser,client){const markers=path.join(dir,'markers.json');await fs.writeFile(markers,JSON.stringify({[m.pages[0].id]:'Example account'}));return monitorRunner('login',m,client,{profile:'test',markers},{playwright:browser.pw,waitForLogin:async()=>{}});}
-function fakeClient(m,events=[]){return {baseUrl:'https://opencrawl.example',request:async(r,o={})=>{events.push([r,o.method||'GET']);if(r==='/api/auth/me')return identity;if(r.endsWith('/captures'))return fakeReceipt(o.body,m.id);return m;}};}
-test('login verifies markers without upload; preview saves only private masked data; check uses exact same scope',()=>temp(async dir=>{const m=fresh();m.pages[0].latest_snapshot=null;const events=[],c=fakeClient(m,events),b=fakeBrowser(m);await loginFixture(dir,m,b,c);assert.equal(events.filter(x=>x[1]==='POST').length,0);const result=await monitorRunner('preview',m,c,{profile:'test',generation:1},{playwright:b.pw});assert.equal(result.captures.length,1);assert.equal((await fs.stat(result.captures[0].preview+'/masked.png')).mode&0o777,0o600);await monitorRunner('check',m,c,{profile:'test',generation:1},{playwright:b.pw});assert.equal(events.filter(x=>x[1]==='POST').length,1);assert.equal(b.closes,4);}));
-test('watch runs serially at cadence and no catch-up; abort closes owned context',()=>temp(async dir=>{const m=fresh(),events=[],c=fakeClient(m,events),b=fakeBrowser(m);await loginFixture(dir,m,b,c);const a=new AbortController();let cycles=0;await monitorRunner('watch',m,c,{profile:'test',generation:1},{playwright:b.pw,signal:a.signal,sleep:async ms=>{assert.equal(ms,3600000);cycles++;if(cycles===2)a.abort();}});assert.equal(events.filter(x=>x[1]==='POST').length,2);assert.equal(b.closes,3);}));
-test('changed identity, policy/generation and unavailable marker stop before upload',()=>temp(async dir=>{const m=fresh(),events=[],c=fakeClient(m,events),b=fakeBrowser(m);await loginFixture(dir,m,b,c);await assert.rejects(monitorRunner('check',{...m,generation:2},c,{profile:'test',generation:2},{playwright:b.pw}),/profile_binding_changed/);const other={...c,request:async(r,o)=>r==='/api/auth/me'?{...identity,user:{id:'usr_'+'3'.repeat(32)}}:c.request(r,o)};await assert.rejects(monitorRunner('check',m,other,{profile:'test',generation:1},{playwright:b.pw}),/profile_binding_changed/);const expired=fakeBrowser(m,{outcome:'session_expired'});await assert.rejects(monitorRunner('check',m,c,{profile:'test',generation:1},{playwright:expired.pw}),/session_expired/);assert.equal(events.filter(x=>x[1]==='POST').length,1);}));
-test('authority revoked after browser capture prevents POST',()=>temp(async dir=>{const m=fresh(),events=[],c=fakeClient(m,events),b=fakeBrowser(m);await loginFixture(dir,m,b,c);const changed=fakeBrowser(m,{mutate:()=>{m.generation=2;}});const original=structuredClone(m);await assert.rejects(monitorRunner('check',original,c,{profile:'test',generation:1},{playwright:changed.pw}),/monitor_changed/);assert.equal(events.filter(x=>x[1]==='POST').length,0);}));
-test('image read binds exact snapshot/hash and refuses overwrite',()=>temp(async dir=>{const png=Buffer.from(f.capture_request.screenshot_png_base64,'base64'),s=f.baseline_receipt.snapshot,events=[];const client={request:async(r,o)=>{events.push([r,o]);return r.endsWith('/image')?png:s;}};const args=['image',s.id],opts={revision:s.version,output:path.join(dir,'image.png')};await productMonitorsCommand(args,opts,client,{stdout:()=>{}});assert.deepEqual(await fs.readFile(opts.output),png);assert.equal(events[1][1].responseType,'png');await assert.rejects(productMonitorsCommand(args,opts,client,{stdout:()=>{}}),/output_exists/);}));
-test('create uncertainty recovers by original ID; next invocation is GET only',()=>temp(async dir=>{const m=fresh(),input={client_create_id:crypto.randomUUID(),name:m.name,account_label:m.account_label,pages:m.pages.map(({id,last_attempt,latest_snapshot,...p})=>p),policy:m.policy,cadence_minutes:60};const spec=path.join(dir,'spec.json');await fs.writeFile(spec,JSON.stringify(input));const calls=[];const c={baseUrl:'https://opencrawl.example',request:async(r,o)=>{calls.push(o.method||'GET');if(r==='/api/auth/me')return identity;if(o.method==='POST')throw Error('lost');return m;}};await productMonitorsCommand(['create'],{spec},c,{stdout:()=>{}});await productMonitorsCommand(['create'],{spec},c,{stdout:()=>{}});assert.equal(calls.filter(x=>x==='POST').length,1);}));
-test('monitor operation lock prevents overlap across profiles, and pending upload blocks restart',()=>temp(async dir=>{
- const m=fresh(),c=fakeClient(m),b=fakeBrowser(m);await loginFixture(dir,m,b,c);
- const scope={server:c.baseUrl,user_id:identity.user.id,workspace_id:identity.workspace.id,monitor:m.id};const operations=path.join(dir,'product-monitor-operations',sha(canonical(scope)));
- await fs.writeFile(path.join(operations,'lock'),'fixture process');await assert.rejects(monitorRunner('check',m,c,{profile:'test',generation:1},{playwright:b.pw}),/profile_locked/);await fs.unlink(path.join(operations,'lock'));
- const u={...f.capture_request,client_capture_id:crypto.randomUUID()};await fs.writeFile(path.join(operations,u.client_capture_id+'.json'),JSON.stringify({state:'intended',monitor:m.id,upload:u}));const captures=b.captures;
- await assert.rejects(monitorRunner('watch',m,c,{profile:'test',generation:1},{playwright:b.pw}),/upload_unconfirmed/);assert.equal(b.captures,captures);
-}));
-test('local history cap refuses before any upload, and creation recovery compares complete settings',()=>temp(async dir=>{
- for(let i=0;i<255;i++)await fs.writeFile(path.join(dir,`bounded-${i}`),'');let calls=0;
- await assert.rejects(sendCapture({request:async()=>{calls++;}},f.monitor.id,f.capture_request,dir),/local_history_full/);assert.equal(calls,0);
- const m=fresh(),input={client_create_id:crypto.randomUUID(),name:m.name,account_label:m.account_label,pages:m.pages.map(({id,last_attempt,latest_snapshot,...p})=>p),policy:m.policy,cadence_minutes:60};const spec=path.join(dir,'spec.json');await fs.writeFile(spec,JSON.stringify(input));
- const wrong=structuredClone(m);wrong.pages[0].scope_selector='body';const c={baseUrl:'https://opencrawl.example',request:async r=>r==='/api/auth/me'?identity:wrong};
- await assert.rejects(productMonitorsCommand(['create'],{spec},c,{stdout:()=>{}}),/receipt_mismatch/);
-}));
-test('CLI entrypoint routes product help and revision argument without global version confusion',async()=>{
- const {promisify}=await import('node:util'),{execFile}=await import('node:child_process');const run=promisify(execFile),entry=new URL('../bin/opencrawl.mjs',import.meta.url);
- const help=await run(process.execPath,[entry.pathname,'browser','monitor','--help']);assert.match(help.stdout,/--revision HASH/);assert.match(help.stdout,/preview\|check\|watch/);
- await assert.rejects(run(process.execPath,[entry.pathname,'browser','monitor','compare','pcmp_bad','--revision','bad'],{env:{...process.env,OPENCRAWL_API_KEY:'fixture-token',OPENCRAWL_URL:'http://127.0.0.1:1'}}),e=>/invalid_id/.test(e.stderr)&&!e.stdout.includes('0.1.0'));
-});
-
-test('actual CLI browser login preview and check failures honor --json without a browser or network',async()=>{
- const {promisify}=await import('node:util'),{execFile}=await import('node:child_process');const run=promisify(execFile),entry=new URL('../bin/opencrawl.mjs',import.meta.url);
- for(const action of ['login','preview','check']){
-  const args=[entry.pathname,'--json','browser','monitor',action,'invalid'];
-  await assert.rejects(run(process.execPath,args,{env:{PATH:process.env.PATH,OPENCRAWL_API_KEY:'fixture-token',OPENCRAWL_URL:'http://127.0.0.1:1',OPENCRAWL_CONFIG_DIR:'/nonexistent-opencrawl-test-config'}}),e=>e.code===1&&e.stderr===''&&JSON.stringify(JSON.parse(e.stdout))===JSON.stringify({error:{code:'command_failed',message:'invalid_id'}}));
- }
-});
-test('low-budget context preserves actual omitted rows without inventing their versions',async()=>{
- const context=f.context.context.split('\n[')[0],value={context,context_bytes:Buffer.byteLength(context),budget:256,included:[],omitted:['before','after'].map(side=>({comparison_id:f.comparison.id,side,reason:'budget'})),image_content_included:false,assessment:'observed_interface_difference'};const out=[];
- await productMonitorsCommand(['context',f.comparison.id],{revision:f.comparison.version,budget:256,json:true},{request:async()=>value},{stdout:s=>out.push(s)});assert.deepEqual(JSON.parse(out.join('')),value);
- await assert.rejects(productMonitorsCommand(['context',f.comparison.id],{revision:f.comparison.version,budget:256},{request:async()=>({...value,omitted:[value.omitted[0],value.omitted[0]]})},{stdout:()=>{}}),/context_mismatch/);
-});

package/src/research-agent-errors.test.mjs

@@ -1,110 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import {createClient} from './client.mjs';
-import {agentCommand,waitAgentRun} from './research-agent.mjs';
-
-const id='ragent_'+'a'.repeat(32),runId='job_'+'b'.repeat(32),key='original-request-key-0001',stamp='2026-10-24T08:00:00.000Z';
-const brief={customer:{name:'Synthetic',website:'https://example.com/',context:''},goal:'Read public product changes.',collection_ids:[],schedule_minutes:0};
-const run={id:runId,agent_id:id,agent_revision:1,status:'queued',phase:'queued',created_at:stamp,started_at:null,finished_at:null,outcome:null,findings:[],gaps:[],usage:{},error:null};
-const receipt={agent:{id,...brief,revision:1,status:'active',created_at:stamp,updated_at:stamp,latest_run:run},run,replayed:false};
-const generic='Research request was not confirmed. Read the same agent/run or recover the original admission key; never start a duplicate blindly.';
-const secret='PRIVATE_SERVER_TEXT\u001b[31m';
-const refusal=status=>Object.assign(new Error(secret),{status,code:'private_server_code',cause:new Error(secret)});
-async function execute(rest,options,value){
- let stdout='',stderr='';const calls=[];
- const client=value?.request?value:{request:async(route,args)=>{calls.push({route,args});if(value instanceof Error)throw value;return value;}};
- const code=await agentCommand(rest,options,client,{stdout:v=>stdout+=v,stderr:v=>stderr+=v});return {code,stdout,stderr,calls};
-}
-function noLeak(result){assert(!JSON.stringify(result).includes('PRIVATE_SERVER_TEXT'));assert(!JSON.stringify(result).includes('private_server_code'));}
-
-test('real CLI transport confirmed GET failures use finite guidance without server prose or another request',async()=>{
- const folder=await fs.mkdtemp(path.join(os.tmpdir(),'research-read-guidance-')),priorDir=process.env.OPENCRAWL_CONFIG_DIR,priorFetch=globalThis.fetch;
- process.env.OPENCRAWL_CONFIG_DIR=folder;
- try{
-  const client=await createClient({server:'https://api.example',token:'synthetic-only-token'});
-  for(const [status,pattern] of [[401,/account, server and workspace/],[403,/administrator to confirm access/],[404,/Do not substitute a current webpage/],[409,/current revision before any write/]]){
-   const calls=[];globalThis.fetch=async(url,init)=>{calls.push({url,init});return new Response(JSON.stringify({detail:secret,code:'private_server_code'}),{status,headers:{'content-type':'application/json'}});};
-   const result=await execute(['agent-run',id,runId],{},client);
-   assert.equal(result.code,1);assert.equal(result.stdout,'');assert.match(result.stderr,pattern);assert.equal(calls.length,1);
-   assert.equal(calls[0].url,`https://api.example/api/research/agents/${id}/runs/${runId}`);assert.equal(calls[0].init.method,'GET');noLeak(result);
-  }
- }finally{globalThis.fetch=priorFetch;if(priorDir===undefined)delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR=priorDir;await fs.rm(folder,{recursive:true,force:true});}
-});
-
-test('all saved-read commands retain exit1 and finite JSON on confirmed denial',async()=>{
- const commands=[['agent','list'],['agent','show',id],['agent','runs',id],['agent','schedule',id],['agent','repeats',id,runId],['agent','source',id,runId,'W1'],['agent','feedback',id,runId,'finding_one'],['agent-run',id,runId]];
- for(const rest of commands)for(const status of [401,403,404,409]){
-  const result=await execute(rest,{json:true},refusal(status));const error=JSON.parse(result.stdout).error;
-  assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(result.calls[0].args.method,undefined);
-  assert.deepEqual(Object.keys(error).sort(),['code','message']);assert.equal(error.code,`http_${status}`);assert.notEqual(error.message,generic);assert.equal(result.stderr,'');noLeak(result);
- }
-});
-
-test('unavailable original admission receipt keeps exact key and never authorizes replacement',async()=>{
- for(const operation of ['create','run:'+id])for(const json of [false,true]){
-  const result=await execute(['agent','receipt'],{operation,'request-key':key,json},refusal(404));
-  assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(result.calls[0].args.idempotencyKey,key);assert.equal(result.calls[0].args.method,undefined);
-  assert.equal(result.calls[0].route,'/api/research/agents/receipts/by-key?operation='+encodeURIComponent(operation));
-  assert.match(json?JSON.parse(result.stdout).error.message:result.stderr,/missing receipt does not authorize a new admission/);
-  if(json)assert.equal(JSON.parse(result.stdout).error.request_key,key);else assert.match(result.stderr,new RegExp('Original request key: '+key));noLeak(result);
- }
-});
-
-test('create rerun and other writes retain original uncertain guidance and request keys even on known HTTP errors',async()=>{
- const folder=await fs.mkdtemp(path.join(os.tmpdir(),'research-write-unchanged-')),file=path.join(folder,'brief.json');
- try{
-  await fs.writeFile(file,JSON.stringify(brief));
-  const commands=[
-   [['agent','create'],{file,'request-key':key},generic,'POST'],
-   [['agent','run',id],{revision:'1','request-key':key},generic,'POST'],
-   [['agent','pause',id],{revision:'1'},generic,'PATCH'],
-   [['agent','schedule',id,'daily'],{revision:'0'},'Schedule request was not confirmed. Read the same schedule and reconcile its setting and revision before another write.','PATCH'],
-   [['agent','feedback',id,runId,'finding_one','set'],{revision:'0',rating:'useful'},'Feedback was not confirmed. Read the same finding feedback and reconcile its revision before another write.','PUT'],
-   [['agent','feedback',id,runId,'finding_one','delete'],{revision:'0'},'Feedback was not confirmed. Read the same finding feedback and reconcile its revision before another write.','DELETE']
-  ];
-  for(const [rest,options,message,method] of commands)for(const status of [401,403,404,409,503,undefined]){
-   const result=await execute(rest,{...options,json:true},refusal(status)),error=JSON.parse(result.stdout).error;
-   assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(result.calls[0].args.method,method);assert.equal(error.message,message);
-   assert.equal(error.request_key,options['request-key']);assert.equal(result.calls[0].args.idempotencyKey,options['request-key']);noLeak(result);
-  }
- }finally{await fs.rm(folder,{recursive:true,force:true});}
-});
-
-test('transport server and malformed read results remain uncertain and expose no unvalidated result',async()=>{
- for(const value of [refusal(undefined),refusal('403'),refusal(500),refusal(503),{...run,id:'wrong',status:403,private:secret}]){
-  const result=await execute(['agent-run',id,runId],{json:true},value),error=JSON.parse(result.stdout).error;
-  assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(error.message,generic);noLeak(result);
- }
-});
-
-test('wait read denial retains last validated run and stops after one GET with finite guidance',async()=>{
- for(const status of [401,403,404,409]){
-  const calls=[];const result=await waitAgentRun({request:async(route,args)=>{calls.push({route,args});throw refusal(status);}},run);
-  assert.deepEqual(result.run,run);assert.equal(result.wait.outcome,'unavailable');assert.equal(result.wait.error.code,`http_${status}`);assert.equal(calls.length,1);
-  assert.equal(calls[0].route,`/api/research/agents/${id}/runs/${runId}`);assert.equal(calls[0].args.method,undefined);noLeak(result);
- }
- for(const json of [false,true]){
-  const calls=[];const result=await execute(['agent','run',id],{revision:'1','request-key':key,wait:true,json},{request:async(route,args)=>{calls.push({route,args});if(calls.length===1)return receipt;throw refusal(403);}});
-  assert.equal(result.code,1);assert.equal(calls.length,2);assert.deepEqual(calls.map(c=>c.args.method),['POST',undefined]);assert.equal(calls[0].args.idempotencyKey,key);
-  if(json){const saved=JSON.parse(result.stdout);assert.deepEqual(saved.run,run);assert.equal(saved.wait.outcome,'unavailable');assert.equal(saved.wait.error.code,'http_403');}
-  else{assert.match(result.stderr,/administrator to confirm access/);assert.match(result.stderr,new RegExp('Original request key: '+key));}
-  assert.match(result.stderr,/Waiting does not cancel server work/);noLeak(result);
- }
-});
-
-test('wait deadline cancellation and uncertain responses keep prior outcomes without false confirmed guidance',async()=>{
- for(const mode of ['abort','deadline','transport','malformed','server']){
-  const signal=new AbortController();let clock=0,calls=0;
-  const result=await waitAgentRun({request:async()=>{calls++;if(mode==='abort')signal.abort();if(mode==='deadline')clock=1000;if(mode==='malformed')return {...run,id:'wrong'};throw refusal(mode==='transport'?undefined:mode==='server'?503:401);}},run,{timeout:1,signal:signal.signal,now:()=>clock,sleep:async ms=>{clock+=ms;}});
-  // A 503 (a restarting server) keeps the wait going until its deadline; it never confirms anything.
-  assert.equal(calls,1);assert.deepEqual(result.run,run);assert.deepEqual(result.wait,{outcome:mode==='abort'?'interrupted':mode==='deadline'||mode==='server'?'timeout':'unavailable'});noLeak(result);
- }
-});
-test('a refused schedule change names the server reason, such as a run still in progress',async()=>{
- const refused=Object.assign(new Error('OpenCrawl 409'),{status:409,detail:'Wait for the current run before enabling its schedule.'});
- const result=await execute(['agent','schedule',id,'daily'],{revision:'0',json:true},refused);
- assert.equal(JSON.parse(result.stdout).error.message,'Schedule request was not confirmed. Read the same schedule and reconcile its setting and revision before another write. Server: Wait for the current run before enabling its schedule.');
-});

package/src/research-agent-feedback.test.mjs

@@ -1,28 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import {agentRequest,agentCommand,validateAgentFeedback} from './research-agent.mjs';
-const aid='ragent_'+'a'.repeat(32),rid='job_'+'b'.repeat(32),fid='finding_example';
-const root=`/api/research/agents/${aid}/runs/${rid}/findings/${fid}/feedback`,args=['agent','feedback',aid,rid,fid];
-const receipt=patch=>({agent_id:aid,run_id:rid,finding_id:fid,feedback:{version:1,revision:0,state:'absent',rating:null,note:null,updated_at:null,...patch}});
-test('CLI feedback maps explicit commands to one scoped CAS request and preserves JSON',async()=>{
- for(const [action,options,expected,method] of [
-  ['show',{},receipt(),'GET'],
-  ['set',{revision:'0',rating:'incorrect',note:'  Customer correction 😀  '},receipt({revision:1,state:'active',rating:'incorrect',note:'  Customer correction 😀  ',updated_at:'2026-10-24T00:00:00Z'}),'PUT'],
-  ['delete',{revision:'1'},receipt({revision:2,state:'deleted',updated_at:'2026-10-24T00:00:00Z'}),'DELETE']]){
-  let calls=0,out='',err='';const code=await agentCommand([...args,action],{...options,json:true},{request:async(path,init)=>{
-   calls++;assert.equal(path,root+(action==='delete'?'?expected_revision=1':''));assert.equal(init.method??'GET',method);
-   if(action==='set')assert.deepEqual(init.body,{expected_revision:0,rating:'incorrect',note:options.note});return expected;
-  }},{stdout:v=>out+=v,stderr:v=>err+=v});
-  assert.equal(code,0);assert.equal(calls,1);assert.deepEqual(JSON.parse(out),expected);assert.equal(err,'');
- }
-});
-test('CLI rejects invalid ratings/notes/revisions and mismatched saved identity',()=>{
- for(const options of [{revision:'0',rating:'proven'},{revision:'-1',rating:'useful'},{revision:'0',rating:'useful',note:'x'.repeat(1001)},{revision:'0',rating:'useful',note:'\ud800'},{revision:'0',rating:'useful',note:'bad\u0000note'}])assert.throws(()=>agentRequest([...args,'set'],options));
- assert.throws(()=>agentRequest([...args,'delete'],{}));
- assert.throws(()=>validateAgentFeedback({...receipt(),finding_id:'other'},aid,rid,fid));
- assert.throws(()=>validateAgentFeedback(receipt({state:'deleted'}),aid,rid,fid));
-});
-test('CLI feedback uncertain write returns one error without retry or success text',async()=>{
- let calls=0,out='',err='';const code=await agentCommand([...args,'set'],{revision:'0',rating:'useful'},{request:async()=>{calls++;throw Object.assign(Error('conflict'),{status:409})}},{stdout:v=>out+=v,stderr:v=>err+=v});
- assert.equal(code,1);assert.equal(calls,1);assert.equal(out,'');assert.match(err,/Read the same finding feedback/);
-});