@getopencrawl/cli 0.1.0 → 0.2.0

package/src/jobs.mjs

@@ -1,108 +0,0 @@
-import { displayLine } from './display.mjs';
-
-export const JOBS_HELP = `OpenCrawl · follow background jobs
-
-  opencrawl jobs list [--status active|completed|failed|cancelled|discarded] [--limit 20] [--offset 0]
-  opencrawl jobs show JOB_ID [--json]
-  opencrawl jobs wait JOB_ID [--timeout 300] [--json]
-
-Wait stops after 1–3600 seconds (default 300). Ctrl+C stops waiting locally.
-It never cancels, retries, or resubmits a job. The job ID remains in the receipt
-on timeout, permission failure, or job failure. Only completed exits successfully.
-Completion is for this job; source scan coverage and provider cleanup are separate.
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-export const GLOBAL_OPTIONS = ['account', 'server', 'config-dir', 'json'];
-export const ACTIVE = new Set(['queued', 'running', 'waiting', 'processing', 'pending']);
-export const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'discarded']);
-export function decimal(value, name, min, max, fallback) {
-  if (value === undefined && fallback !== undefined) return fallback;
-  if (typeof value !== 'string' || !/^(0|[1-9]\d*)$/.test(value) || !Number.isSafeInteger(Number(value)) || Number(value) < min || Number(value) > max)
-    throw new Error(`--${name} must be a decimal integer between ${min} and ${max}.`);
-  return Number(value);
-}
-export function allowedOptions(options, allowed, command) {
-  for (const key of Object.keys(options)) if (![...GLOBAL_OPTIONS, ...allowed].includes(key)) throw new Error(`Unsupported ${command} option: --${key}. Run opencrawl ${command} --help.`);
-}
-export function identifier(value, kind = 'job') {
-  if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(value)) throw new Error(`Supply the exact ${kind} ID from its receipt.`);
-  return value;
-}
-export function waitOptions(options) {
-  if (options.timeout !== undefined && !options.wait) throw new Error('--timeout requires --wait. Use jobs wait to follow an existing job.');
-  return decimal(options.timeout, 'timeout', 1, 3600, 300);
-}
-export function requestFailure(error) {
-  const status = Number.isInteger(error?.status) ? error.status : null;
-  const message = ({401: 'Sign in with a valid key for the intended workspace.', 403: 'This account cannot access this operation.',
-    404: 'This job or upload receipt is unavailable to this account. Check the account and saved ID.',
-    409: 'The request conflicts with the saved upload details or current source state. Inspect the receipt and source before explicitly retrying; do not start a duplicate upload.',
-    413: 'The file exceeds an upload limit.', 422: 'The upload details or content failed validation. Check the file and its scope.',
-    429: 'OpenCrawl is busy or a workspace limit was reached. Inspect the receipt before retrying.',
-    507: 'Workspace storage admission failed. Inspect the receipt before retrying.'})[status] || (status ? 'OpenCrawl could not complete this request. Inspect its receipt before retrying.' : 'The request outcome could not be confirmed. Check the connection and inspect the saved receipt.');
-  return { code: status ? `http_${status}` : 'request_unconfirmed', message, ...(status ? { status } : {}) };
-}
-export function validateJob(job, id = job?.id) {
-  if (!job || typeof job.id !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(job.id) || job.id !== id || typeof job.status !== 'string' || !ACTIVE.has(job.status) && !TERMINAL.has(job.status)) throw new Error('Server returned an invalid job receipt. No completion was assumed.');
-  return job;
-}
-export const getJob = async (client, id, options = {}) => validateJob(await client.request(`/api/v1/jobs/${encodeURIComponent(identifier(id))}`, { maxResponseBytes: 512 * 1024, ...options }), id);
-
-export async function waitForJob(client, id, { timeout = 300, signal, now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), onUpdate = () => {}, initial, deadline } = {}) {
-  identifier(id); const started = now(); deadline ??= started + timeout * 1000;
-  let job = initial ? validateJob(initial, id) : null;
-  const result = (outcome, error) => ({ job_id: id, job, wait: { outcome, elapsed_seconds: Math.max(0, Math.round((now() - started) / 1000)), ...(error ? { error } : {}) } });
-  while (true) {
-    if (signal?.aborted) return result('interrupted');
-    if (job && TERMINAL.has(job.status)) return result(job.status);
-    const remaining = deadline - now();
-    if (remaining <= 0) return result('timeout');
-    try { job = await getJob(client, id, { timeout: Math.max(1, Math.min(30000, Math.ceil(remaining))), signal }); }
-    catch (error) { return result(signal?.aborted ? 'interrupted' : now() >= deadline ? 'timeout' : 'unavailable', requestFailure(error)); }
-    onUpdate(job);
-    if (TERMINAL.has(job.status)) return result(job.status);
-    // Short sleeps keep Ctrl+C responsive without extra server polling.
-    const next = Math.min(deadline, now() + 2000);
-    while (!signal?.aborted && now() < next) await sleep(Math.min(100, next - now()));
-  }
-}
-export async function withInterrupt(action) {
-  const controller = new AbortController(); const stop = () => controller.abort();
-  process.on('SIGINT', stop); process.on('SIGTERM', stop);
-  try { return await action(controller.signal); }
-  finally { process.off('SIGINT', stop); process.off('SIGTERM', stop); }
-}
-export function renderJob(job) {
-  let text = `${displayLine(job.id)}\t${displayLine(job.status)}\t${displayLine(job.kind)}${job.filename ? '\t' + displayLine(job.filename) : ''}\n`;
-  if (job.error) text += `Issue: ${displayLine(job.error)}\n`;
-  if (typeof job.indexed_documents === 'number') text += `Indexed documents: ${job.indexed_documents}\n`;
-  for (const doc of job.documents || []) text += `${displayLine(doc.id)}\t${displayLine(doc.title)}\n`;
-  if (job.documents_truncated) text += 'Additional indexed documents are available in OpenCrawl.\n';
-  if (job.coverage) text += `Source listing coverage: ${displayLine(job.coverage.state)} (separate from this job's status).\n`;
-  return text;
-}
-export async function jobsCommand(rest, options, client, { stdout = text => process.stdout.write(text), stderr = text => process.stderr.write(text) } = {}) {
-  const [command, id, ...extra] = rest;
-  if (!['list', 'show', 'wait'].includes(command)) throw new Error('Choose jobs list, show, or wait. Run opencrawl jobs --help.');
-  allowedOptions(options, command === 'list' ? ['status', 'limit', 'offset'] : command === 'wait' ? ['timeout'] : [], 'jobs');
-  if (extra.length || command === 'list' && id !== undefined) throw new Error('Unexpected jobs argument.');
-  if (command === 'list') {
-    if (options.status !== undefined && !['active', ...ACTIVE, ...TERMINAL].includes(options.status)) throw new Error('Choose a job status shown in jobs --help.');
-    const query = new URLSearchParams({ limit: decimal(options.limit, 'limit', 1, 200, 20), offset: decimal(options.offset, 'offset', 0, 1000000, 0) });
-    if (options.status) query.set('status', options.status);
-    const data = await client.request('/api/v1/jobs?' + query, { maxResponseBytes: 2 * 1024 * 1024 });
-    if (!Array.isArray(data?.items)) throw new Error('Server returned an invalid job list.');
-    if (options.json) stdout(JSON.stringify(data, null, 2) + '\n');
-    else { stdout(data.items.map(renderJob).join('') || 'No jobs on this page.\n'); if (data.has_more) stderr(`More jobs are available; use --offset ${Number(data.offset) + data.items.length}.\n`); }
-    return 0;
-  }
-  identifier(id);
-  if (command === 'show') {
-    const job = await getJob(client, id); stdout(options.json ? JSON.stringify(job, null, 2) + '\n' : renderJob(job)); return 0;
-  }
-  const timeout = decimal(options.timeout, 'timeout', 1, 3600, 300);
-  stderr(`Following job ${displayLine(id)} for at most ${timeout} seconds.\n`);
-  const result = await withInterrupt(signal => waitForJob(client, id, { timeout, signal }));
-  stdout(options.json ? JSON.stringify(result, null, 2) + '\n' : (result.job ? renderJob(result.job) : `Job: ${displayLine(id)}\n`) + `Wait: ${displayLine(result.wait.outcome)}. The job was not cancelled or resubmitted.\n` + (result.wait.error ? result.wait.error.message + '\n' : ''));
-  return result.wait.outcome === 'completed' ? 0 : 1;
-}

package/src/monitors.mjs

@@ -1,146 +0,0 @@
-import { allowedOptions, decimal, validateJob, renderJob } from './jobs.mjs';
-import { displayLine } from './display.mjs';
-
-export const MONITORS_HELP = `OpenCrawl · inspect exact public-page observations
-
-  opencrawl monitors list [--limit 20] [--offset 0] [--json]
-  opencrawl monitors show SOURCE_ID [--json]
-  opencrawl monitors check SOURCE_ID --generation N [--json]
-  opencrawl monitors changes [--source SOURCE_ID] [--limit 20] [--cursor CURSOR] [--json]
-  opencrawl monitors context CHANGE_ID --revision HASH [--budget 4096] [--json]
-
-Create monitors in OpenCrawl with 1–5 explicit public static URLs. Checks do not
-follow discovered links or execute JavaScript. A failed or missing page never
-proves a job closed or content was removed. Inspect show for each page's latest
-attempt and last good observation. Check queues work; follow its job separately.
-Context requires an exact change ID and immutable version from changes. It
-exports before/after evidence, including observation times, within 256–16384
-UTF-8 bytes. Truncation and omissions go to stderr; stdout is exact context.
---json preserves the complete receipt. No automatic check or newer substitution.
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-const SOURCE = /^src_[a-f0-9]{32}$/, CHANGE = /^mchg_[a-f0-9]{32}$/, HASH = /^[a-f0-9]{64}$/;
-const integer = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max;
-const id = (value, pattern, kind) => {
-  if (typeof value !== 'string' || !pattern.test(value)) throw new Error(`Supply the exact ${kind} ID from its list.`);
-  return value;
-};
-const cursor = value => {
-  if (typeof value !== 'string' || !value || value.length > 256 || /[\x00-\x20\x7f]/.test(value)) throw new Error('Use the unchanged opaque cursor from the previous changes page.');
-  return value;
-};
-export function monitorRequest(rest, options) {
-  const [command, value, ...extra] = rest;
-  const allowed = {list: ['limit','offset'], show: [], check: ['generation'], changes: ['source','limit','cursor'], context: ['revision','budget']};
-  if (!Object.hasOwn(allowed, command)) throw new Error('Choose monitors list, show, check, changes, or context. Run opencrawl monitors --help.');
-  allowedOptions(options, allowed[command], 'monitors');
-  if (extra.length || ['list','changes'].includes(command) && value !== undefined) throw new Error('Unexpected monitors argument.');
-  if (command === 'list') {
-    const limit=decimal(options.limit,'limit',1,20,20), offset=decimal(options.offset,'offset',0,1000000,0);
-    return {command,limit,offset,route:'/api/v1/monitors?'+new URLSearchParams({limit,offset}),options:{}};
-  }
-  if (command === 'changes') {
-    const limit=decimal(options.limit,'limit',1,20,20), query=new URLSearchParams({limit});
-    if (options.source !== undefined) query.set('source_id',id(options.source,SOURCE,'monitor'));
-    if (options.cursor !== undefined) query.set('cursor',cursor(options.cursor));
-    return {command,limit,source:options.source,route:'/api/v1/monitor-changes?'+query,options:{}};
-  }
-  if (command === 'context') {
-    id(value,CHANGE,'change');
-    if (typeof options.revision !== 'string' || !HASH.test(options.revision)) throw new Error('--revision must be the exact 64-character lowercase version hash from changes.');
-    const budget=decimal(options.budget,'budget',256,16384,4096), selections=[{change_id:value,version:options.revision}];
-    return {command,id:value,selections,budget,route:'/api/v1/monitor-changes/context',options:{method:'POST',body:{selections,budget}}};
-  }
-  id(value,SOURCE,'monitor');
-  return {command,id:value,route:'/api/v1/monitors/'+value+(command==='check'?'/check':''),
-    options:command==='check'?{method:'POST',body:{expected_generation:decimal(options.generation,'generation',1,Number.MAX_SAFE_INTEGER)}}:{}};
-}
-function validateMonitor(value, expected = value?.id) {
-  if (!value || !SOURCE.test(value.id) || value.id!==expected || value.kind!=='monitor' || typeof value.name!=='string' ||
-      !integer(value.generation,1,Number.MAX_SAFE_INTEGER) || !['active','paused','revoked','error'].includes(value.status) ||
-      !Array.isArray(value.targets) || value.targets.length<1 || value.targets.length>5 || value.targets.some(target=>
-        !target || typeof target.requested_url!=='string' || typeof target.state!=='string' || typeof target.enabled!=='boolean'))
-    throw new Error('Server returned an invalid monitor receipt. No check was assumed.');
-  return value;
-}
-function validateChange(value, source) {
-  if (!value || !CHANGE.test(value.id) || !HASH.test(value.version) || !SOURCE.test(value.source_id) || source && value.source_id!==source ||
-      typeof value.title!=='string' || typeof value.final_url!=='string' || typeof value.observed_before!=='string' || typeof value.observed_after!=='string')
-    throw new Error('Server returned an invalid or out-of-scope change receipt.');
-  return value;
-}
-export function validateChangeContext(value, request) {
-  const invalid=()=>{throw new Error('Server returned an invalid historical context receipt. No context was emitted; inspect the change again.');};
-  if (!value || typeof value.context!=='string' || value.budget!==request.budget || value.budget_unit!=='utf8_bytes' ||
-      Buffer.byteLength(value.context)!==value.context_bytes || value.context_bytes>request.budget ||
-      !Array.isArray(value.selections) || value.selections.length!==1 || value.selections[0]?.change_id!==request.id || value.selections[0]?.version!==request.selections[0].version ||
-      !Array.isArray(value.included) || value.included.length>2 || !Array.isArray(value.omitted) || value.omitted.length>2 ||
-      !Array.isArray(value.warnings) || value.warnings.length>20 || value.warnings.some(w=>typeof w!=='string') ||
-      Boolean(value.context)!==Boolean(value.included.length)) invalid();
-  const seen=new Set(), labels=new Set(), chars=Array.from(value.context); let lastEnd=0;
-  for (const entry of [...value.included,...value.omitted]) {
-    if (!entry || entry.change_id!==request.id || entry.version!==request.selections[0].version || !['before','after'].includes(entry.side) || seen.has(entry.side)) invalid();
-    seen.add(entry.side);
-  }
-  if (seen.size!==2 || value.omitted.some(entry=>entry.reason!=='budget')) invalid();
-  for (const entry of value.included) {
-    const span=entry.context_span, excerpt=entry.excerpt;
-    if (typeof excerpt!=='string' || !excerpt || typeof entry.citation!=='string' || !/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(entry.citation) || labels.has(entry.citation) ||
-        !integer(entry.full_length,0,131072) || !integer(entry.start_char,0,entry.full_length) || !integer(entry.end_char,entry.start_char+1,entry.full_length) ||
-        Array.from(excerpt).length!==entry.end_char-entry.start_char || typeof entry.truncated!=='boolean' ||
-        typeof entry.observed_at!=='string' || typeof entry.url!=='string' || !span || !integer(span.start_char,lastEnd,chars.length) ||
-        !integer(span.end_char,span.start_char+1,chars.length) || chars.slice(span.start_char,span.end_char).join('')!==excerpt) invalid();
-    labels.add(entry.citation); lastEnd=span.end_char;
-  }
-  return value;
-}
-export function monitorFailure(error) {
-  const status=Number.isInteger(error?.status)?error.status:undefined;
-  const codes={monitor_changed:'Monitor settings changed. Run monitors show and review the current generation before explicitly checking again.',
-    monitor_not_active:'This monitor is not active. Review its status in OpenCrawl; no check was queued.',
-    monitor_change_changed:'The historical version does not match. Inspect changes and explicitly select the intended immutable version; no newer change was substituted.',
-    monitor_evidence_changed:'Historical evidence or its access changed. Inspect the change again before explicitly requesting context.'};
-  if (status===409 && Object.hasOwn(codes,error.code)) return {code:error.code,status,message:codes[error.code]};
-  const message=({401:'Sign in with a valid key for the intended workspace.',403:'This account cannot perform this monitor operation.',
-    404:'This monitor or historical change is unavailable to this account, or its retained history has expired. No other evidence was substituted.',
-    409:'The monitor or historical selection changed. Inspect it before explicitly trying again.',413:'A monitor or retained-history limit was reached. Review capacity in OpenCrawl.',
-    422:'The monitor request is invalid. Check its exact ID, generation, version and byte budget.',429:'OpenCrawl is busy or a workspace limit was reached. Inspect the monitor before checking again.',
-    507:'Workspace storage admission failed. Review storage before checking again.'})[status] ||
-    'The monitor request outcome could not be confirmed. Inspect its current state before explicitly trying again. No automatic retry was made.';
-  return {code:status?`http_${status}`:'request_unconfirmed',message,...(status?{status}:{})};
-}
-export async function monitorsCommand(rest, options, client, {stdout=value=>process.stdout.write(value),stderr=value=>process.stderr.write(value)}={}) {
-  const request=monitorRequest(rest,options); let data;
-  try {data=await client.request(request.route,{...request.options,maxResponseBytes:1024*1024});}
-  catch(error){const failure=monitorFailure(error);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+'\n');return 1;}
-  if(request.command==='list'){
-    if(!Array.isArray(data?.items)||data.items.length>request.limit||data.limit!==request.limit||data.offset!==request.offset)throw new Error('Server returned an invalid monitor page.');
-    data.items.forEach(value=>validateMonitor(value));
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(data.items.map(value=>`${value.id}\tg${value.generation}\t${value.status}\t${displayLine(value.name)}\n`).join('')||'No monitors on this page.\n');if(data.has_more)stderr(`Next page: --offset ${request.offset+data.items.length}.\n`);}
-  }else if(request.command==='show'){
-    validateMonitor(data,request.id);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(`${displayLine(data.name)}\nMonitor: ${data.id}\nGeneration: ${data.generation}; status: ${data.status}\n`);
-      for(const target of data.targets)stdout(`${displayLine(target.requested_url)}\n  ${displayLine(target.state)}; latest attempt: ${displayLine(target.last_attempt_at||'none')}; last good observation: ${displayLine(target.last_success_at||'none')}\n${target.issue?.message?'  '+displayLine(target.issue.message)+'\n':''}`);
-      stderr('Only selected static pages are checked. Failure or missing content is not evidence of removal.\n');}
-  }else if(request.command==='check'){
-    validateJob(data?.job);
-    stdout(options.json?JSON.stringify(data,null,2)+'\n':renderJob(data.job));
-    if(!options.json)stderr(`Check accepted; this does not confirm page coverage. Follow with jobs wait ${data.job.id}, then monitors show ${request.id}.\n`);
-  }else if(request.command==='changes'){
-    if(!Array.isArray(data?.items)||data.items.length>request.limit||data.limit!==request.limit||!(data.next_cursor===null||typeof data.next_cursor==='string'))throw new Error('Server returned an invalid change page.');
-    data.items.forEach(value=>validateChange(value,request.source));if(data.next_cursor!==null)cursor(data.next_cursor);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(data.items.map(value=>`${value.id}\t${displayLine(value.title)}\nVersion: ${value.version}\nObserved: ${displayLine(value.observed_before)} → ${displayLine(value.observed_after)}${value.gap?' (failed checks between observations)':''}\n${displayLine(value.final_url)}\n`).join('\n')||'No retained text changes on this page.\n');
-      if(data.next_cursor)stderr(`Next page: --cursor ${displayLine(data.next_cursor)}${request.source?' --source '+request.source:''}.\n`);}
-  }else{
-    validateChangeContext(data,request);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stderr(`${data.included.length}/2 sides included; ${data.context_bytes}/${data.budget} UTF-8 bytes. Observation times are not remote publication times.\n`);
-      for(const entry of data.included)if(entry.truncated)stderr(`${entry.side} excerpt is truncated; inspect the complete retained change in OpenCrawl.\n`);
-      for(const entry of data.omitted)stderr(`${entry.side} side omitted because of the byte budget.\n`);
-      for(const warning of data.warnings)stderr(displayLine(warning)+'\n');stdout(data.context);}
-  }
-  return 0;
-}

package/src/product-letter-capabilities.mjs

@@ -1,71 +0,0 @@
-/** Fixed read-only Letter picker scope. No provider selection or neutral re-render. */
-export const LETTER_CAPABILITIES_ADAPTER='letter-capabilities-read-v1';
-export const LETTER_PICKER='[data-testid="connect-integration"]';
-export const LETTER_PICKER_OPEN='[data-testid="connect-integration-open"]';
-export function isLetterCapabilitiesPolicy(policy,screen){return policy?.version===3&&policy.request_adapter===LETTER_CAPABILITIES_ADAPTER&&screen.url==='https://app.letterstory.com/letterpress?view=connections'&&screen.account_marker?.required===true&&screen.ready_selector===LETTER_PICKER_OPEN&&screen.scope_selector===LETTER_PICKER;}
-
-/** Executed in the page before the one fixed opener click. Only finite status leaves. */
-export function letterPickerReady({screen,expected}){
- const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
- const all=s=>[...document.querySelectorAll(s)].filter(visible);
- if(location.href!==screen.url)return {error:'scope_mismatch'};
- if(screen.login_selector&&all(screen.login_selector).length)return {error:'session_expired',login:true};
- const markers=all(screen.account_marker.selector);
- if(markers.length!==1||typeof expected!=='string'||!expected||markers[0].innerText.trim()!==expected)return {error:'context_mismatch'};
- const buttons=all('[data-testid="connect-integration-open"]');
- if(buttons.length!==1||buttons[0].tagName!=='BUTTON'||buttons[0].innerText.trim()!=='Connect'||buttons[0].disabled||buttons[0].closest('form')||document.querySelector('[data-testid="connect-integration"]'))return {error:'not_ready'};
- return {ready:true};
-}
-
-/** All public text is literal in-place UI text; unknown structure refuses before pixels. */
-export function extractLetterCapabilities({screen,policy,expected,freeze=false}){
- const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
- const all=s=>[...document.querySelectorAll(s)].filter(visible);
- const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
- if(location.href!==screen.url)return {error:'scope_mismatch'};
- if(screen.login_selector&&all(screen.login_selector).length)return {error:'session_expired',login:true};
- const marker=one(screen.account_marker.selector)?.innerText.trim();
- if(!marker||marker!==expected)return {error:'context_mismatch'};
- const dialog=one('[data-testid="connect-integration"]');
- if(!dialog||dialog.getAttribute('role')!=='dialog')return {error:'not_ready'};
- const exact=(e,tags)=>e&&[...e.children].map(n=>n.tagName.toUpperCase()).join(',')===tags&&!Array.from(e.childNodes).some(n=>n.nodeType===Node.TEXT_NODE&&n.textContent.trim());
- const leaf=(e,max)=>e&&e.children.length===0&&visible(e)&&e.textContent===e.textContent.trim()&&e.textContent.length>0&&Array.from(e.textContent).length<=max&&!/[\p{C}\n\r]/u.test(e.textContent)&&!/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}|https?:\/\/|\b(?:Bearer |sk-|ghp_|AKIA)/i.test(e.textContent);
- if(!exact(dialog,'DIV,DIV,BUTTON'))return {error:'mask_unresolved'};
- const [header,body,close]=dialog.children;
- if(!exact(header,'H2,P')||!leaf(header.children[0],100)||!leaf(header.children[1],100)||header.children[0].textContent!=='Connect an integration'||header.children[1].textContent!=='Connect where you use it.'||!exact(body,'UL')||!exact(close,'SVG,SPAN')||close.children[1]?.textContent!=='Close')return {error:'mask_unresolved'};
- const list=body.children[0],rows=[...list.children];
- if(rows.length<1||rows.length>32||!exact(list,rows.map(()=> 'LI').join(',')))return {error:'mask_unresolved'};
- const masks=[close],publicNodes=[dialog,header,...header.children,body,list],names=new Set(),text=['Connect an integration','Connect where you use it.'];
- const badges=[],logos=[],missingBadgeSlots=[];
- for(const row of rows){
-  if(!exact(row,'BUTTON'))return {error:'mask_unresolved'};
-  const button=row.children[0];if(!exact(button,'IMG,SPAN')&&!exact(button,'IMG,SPAN,SPAN'))return {error:'mask_unresolved'};
-  const [logo,copy,badge]=button.children;
-  if(!copy.classList.contains('min-w-0')||!exact(copy,'SPAN,SPAN')||!leaf(copy.children[0],100)||!leaf(copy.children[1],400)||names.has(copy.children[0].textContent))return {error:'mask_unresolved'};
-  names.add(copy.children[0].textContent);text.push(copy.children[0].textContent,copy.children[1].textContent);
-  masks.push(logo);logos.push(logo);if(badge){masks.push(badge);badges.push(badge);}else missingBadgeSlots.push(button);publicNodes.push(row,button,copy,...copy.children);
- }
- // Unknown controls or pseudo/image content in the public structure are not public fields.
- if(publicNodes.some(e=>!visible(e)||getComputedStyle(e).backgroundImage!=='none'||getComputedStyle(e).listStyleImage!=='none'||['::before','::after','::marker'].some(p=>!['none','normal'].includes(getComputedStyle(e,p).content))))return {error:'mask_unresolved'};
- // Element masks cannot hide direct ancestor text or ancestor-generated paint.
- // Include html: a body-only boundary leaves document-level paint outside the mask set.
- for(let child=dialog,parent=dialog.parentElement;parent;child=parent,parent=parent.parentElement){
-  if(Array.from(parent.childNodes).some(n=>n.nodeType===Node.TEXT_NODE&&n.textContent.trim())||getComputedStyle(parent).backgroundImage!=='none'||['::before','::after'].some(p=>!['none','normal'].includes(getComputedStyle(parent,p).content)))return {error:'mask_unresolved'};
-  for(const sibling of parent.children)if(sibling!==child)masks.push(sibling);
- }
- for(const selector of [...policy.masks,...policy.noise_regions].map(r=>r.selector)){const matches=all(selector);if(!matches.length||matches.length>100)return {error:'mask_unresolved'};masks.push(...matches);}
- if(masks.some(m=>publicNodes.some(n=>m===n||m.contains(n))))return {error:'mask_unresolved'};
- const value=text.join('\n');if(new TextEncoder().encode(value).length>16384)return {error:'mask_unresolved'};
- if(freeze){
-  // Fixed private slots make connection-status strings/logos unable to change public layout.
-  // Missing status badges receive the same empty masked slot before pixels or observation.
-  for(const button of missingBadgeSlots){const badge=document.createElement('span');badge.style.setProperty('opacity','0','important');button.append(badge);badges.push(badge);masks.push(badge);}
-  for(const badge of badges){badge.style.setProperty('width','100px','important');badge.style.setProperty('flex','0 0 100px','important');badge.style.setProperty('overflow','hidden','important');badge.style.setProperty('height','20px','important');badge.style.setProperty('white-space','nowrap','important');}
-  for(const logo of logos){logo.style.setProperty('width','24px','important');logo.style.setProperty('height','24px','important');logo.style.setProperty('flex','0 0 24px','important');}
-  for(const node of masks)node.style.setProperty('opacity','0','important');
-  window.__opencrawlMasks=masks;window.__opencrawlChanged=false;window.__opencrawlObserver?.disconnect();
-  window.__opencrawlObserver=new MutationObserver(rows=>{if(rows.length)window.__opencrawlChanged=true;});
-  window.__opencrawlObserver.observe(document.documentElement,{attributes:true,childList:true,characterData:true,subtree:true});
- }
- return {text:value,title:screen.label,marker,maskCount:masks.length};
-}

package/src/product-letter-read.mjs

@@ -1,178 +0,0 @@
-/** Explicit read adapter. Diagnostics retain finite labels and query-free target hashes only. */
-import {createHash} from 'node:crypto';
-import {LETTER_CAPABILITIES_ADAPTER} from './product-letter-capabilities.mjs';
-export const LETTER_ADAPTER='letter-connections-read-v1';
-export const LETTER_SCREEN='https://app.letterstory.com/letterpress?view=connections';
-const ORIGIN='https://app.letterstory.com',PREFIX=ORIGIN+'/api/console/tools/';
-const READS=['list_cms_connections','list_sites','list_repo_sites'];
-const BLOCKED=['prepare_account','refresh_site'];
-// Observed unrelated startup inventory. These are denied, never fetched.
-const BACKGROUND=['export_org_config','list_phantoms','list_collections','list_kernels','list_kernel_endpoints','list_shredder_endpoints','list_seers','list_arenas','list_lede_endpoints','list_mailers','list_prospect_lists','list_sequences','list_dispatches','list_simulations','letterprove_get_vendor_link','list_api_keys','list_integrations','visitors_list_sites','list_growth_builds'];
-const ORG_QUERY=/^\?org_id=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$/;
-function toolTarget(url){
-  for(const op of [...READS,...BLOCKED,...BACKGROUND]){
-    const base=PREFIX+op;if(url===base)return {op,org:null,url};
-    if(url.startsWith(base+'?')){const match=ORG_QUERY.exec(url.slice(base.length));if(match)return {op,org:match[1],url};}
-  }
-  return null;
-}
-function blockedTelemetry(url){
-  // Deny-only endpoints: query parameters cannot grant forwarding authority.
-  if(url==='https://app.letterprove.com/api/v1/observe')return true;
-  try{const u=new URL(url);return u.origin===ORIGIN&&u.pathname==='/ingest/flags/'&&!u.hash&&!u.username&&!u.password&&url===u.origin+u.pathname+u.search;}catch{return false;}
-}
-// Diagnostic classification is separate from toolTarget: it never grants forwarding
-// or converts a fatal rejection into a known-background rejection.
-function deniedTarget(req){
-  const value={method:'OTHER',origin_class:'invalid',known_target:'other',target_sha256:null};
-  try{const method=req.method();if(['GET','HEAD','OPTIONS','POST','PUT','PATCH','DELETE'].includes(method))value.method=method;}catch{}
-  try{
-    const raw=req.url();if(typeof raw!=='string'||raw.length>8192)return value;
-    const url=new URL(raw);if(!['http:','https:'].includes(url.protocol))return value;
-    value.origin_class=url.origin===ORIGIN?'letter_app':url.origin==='https://app.letterprove.com'?'letter_telemetry':'other';
-    if(url.origin===ORIGIN){
-      value.known_target=[...READS,...BLOCKED,...BACKGROUND].find(op=>url.pathname==='/api/console/tools/'+op)||'other';
-      if(url.pathname==='/ingest/flags/')value.known_target='telemetry_flags';
-    }else if(url.origin==='https://app.letterprove.com'&&url.pathname==='/api/v1/observe')value.known_target='telemetry_observe';
-    // URL origin excludes credentials; pathname excludes query and fragment.
-    // No percent-decoding: parser normalization is diagnostic, not authority.
-    value.target_sha256=createHash('sha256').update('letter-denied-target-v1\0'+url.origin+url.pathname).digest('hex');
-  }catch{}
-  return value;
-}
-const JSON_TYPE=/^application\/json(?:\s*;\s*charset=utf-8)?$/i;
-const PROVIDERS={webflow:['site_id','site_name'],framer:['project_id','project_url','project_name','auto_deploy'],sanity:['project_id','dataset','project_name','workspace_name'],contentful:['space_id','environment_id','space_name','default_locale'],github:['account_login']};
-const object=v=>v!==null&&typeof v==='object'&&!Array.isArray(v);
-const keys=(v,allowed)=>object(v)&&Object.keys(v).length===allowed.length&&Object.keys(v).every(k=>allowed.includes(k));
-const nullable=v=>v===null||typeof v==='string'&&v.length<=2048;
-export function validConnections(v){
-  if(!keys(v,['connections'])||!Array.isArray(v.connections)||v.connections.length!==5)return false;
-  const seen=new Set();
-  return v.connections.every(row=>{
-    if(!keys(row,['provider','connected','label','created_at','fields'])||!Object.hasOwn(PROVIDERS,row.provider)||seen.has(row.provider)||typeof row.connected!=='boolean'||!nullable(row.label)||!nullable(row.created_at))return false;
-    seen.add(row.provider);
-    return keys(row.fields,PROVIDERS[row.provider])&&Object.entries(row.fields).every(([k,x])=>k==='auto_deploy'?typeof x==='boolean':nullable(x));
-  });
-}
-/** v3 keeps private rows bounded but does not equate capability discovery with five CMS names. */
-export function validCapabilityConnections(v){
-  if(!keys(v,['connections'])||!Array.isArray(v.connections)||v.connections.length<1||v.connections.length>32)return false;
-  const seen=new Set();return v.connections.every(row=>{
-    if(!keys(row,['provider','connected','label','created_at','fields'])||typeof row.provider!=='string'||!(/^[a-z][a-z0-9_-]{0,63}$/).test(row.provider)||seen.has(row.provider)||typeof row.connected!=='boolean'||!nullable(row.label)||!nullable(row.created_at)||!object(row.fields)||Object.keys(row.fields).length>16)return false;
-    seen.add(row.provider);return Object.entries(row.fields).every(([k,x])=>/^[a-z][a-z0-9_]{0,63}$/.test(k)&&(typeof x==='boolean'||nullable(x)));
-  });
-}
-function connectionShape(v){
-  const rows=Array.isArray(v?.connections)?v.connections:[];
-  return {root_keys:keys(v,['connections']),five_rows:Array.isArray(v?.connections)&&rows.length===5,provider_roster:rows.length===5&&new Set(rows.map(r=>r?.provider)).size===5&&rows.every(r=>object(r)&&Object.hasOwn(PROVIDERS,r.provider)),row_keys:rows.length===5&&rows.every(r=>keys(r,['provider','connected','label','created_at','fields'])),row_types:rows.length===5&&rows.every(r=>object(r)&&typeof r.connected==='boolean'&&nullable(r.label)&&nullable(r.created_at)),field_keys:rows.length===5&&rows.every(r=>object(r)&&Object.hasOwn(PROVIDERS,r.provider)&&keys(r.fields,PROVIDERS[r.provider])),field_types:rows.length===5&&rows.every(r=>object(r?.fields)&&Object.entries(r.fields).every(([k,x])=>k==='auto_deploy'?typeof x==='boolean':nullable(x)))};
-}
-export function isLetterPolicy(policy,screen){return policy?.version===2&&policy.request_adapter===LETTER_ADAPTER&&screen.url===LETTER_SCREEN&&screen.account_marker?.required===true;}
-
-export class LetterReadAttempt {
-  constructor(page,deadline,capabilities=false){this.capabilities=capabilities;this.shared=new Map();this.shareTimer=null;this.coalesced=0;this.page=page;this.deadline=deadline;this.requests=new Map();this.seen=new Set();this.scopeSet=false;this.org=null;this.required='missing';this.requiredChecks={};this.readStates=Object.fromEntries(READS.map(op=>[op,{state:'not_requested',phase:null,reason:null,checks:{}}]));this.readiness={state:'not_checked',in_flight:0};this.invalid=false;this.closed=false;this.inFlight=0;this.counts={allowed_reads:0,blocked_setup:0,blocked_refresh:0,blocked_background:0,blocked_other:0};this.denials={version:1,samples:[],omitted:0};}
-  /** Called for each non-read-only request before any forwarding. */
-  request(req){
-    const url=req.url(),target=toolTarget(url),op=target?.op;
-    if(this.capabilities&&(this.closed||this.invalid||performance.now()>=this.deadline))return this.reject(req);
-    let exact=false,reason='unavailable';
-    try{
-      reason=req.method()!=='POST'?'method':req.frame()!==this.page.mainFrame()?'frame':req.redirectedFrom()?'redirect':this.page.url()!==LETTER_SCREEN?'selected_page':req.headers().origin!==ORIGIN?'request_origin':null;
-      exact=reason===null;
-    }catch{}
-    if(exact&&(BLOCKED.includes(op)||BACKGROUND.includes(op)||blockedTelemetry(url))){const key=op==='prepare_account'?'blocked_setup':op==='refresh_site'?'blocked_refresh':'blocked_background';this.counts[key]=Math.min(64,this.counts[key]+1);return 'block_known';}
-    if(!exact)return this.reject(req,reason);
-    if(!READS.includes(op))return this.reject(req,'target');
-    if(req.postData()!=='{}')return this.reject(req,'body_shape');
-    if(!JSON_TYPE.test(req.headers()['content-type']||''))return this.reject(req,'content_type');
-    if(!this.capabilities&&this.seen.has(op))return this.reject(req,'duplicate_read');
-    if(this.scopeSet&&this.org!==target.org)return this.reject(req,'organization_scope');
-    if(this.capabilities){
-      let shared=this.shared.get(op);
-      if(shared){
-        if(shared.url!==url||this.requests.has(req)||shared.followers>=2)return this.reject(req,'duplicate_read');
-        shared.followers++;this.coalesced++;this.requests.set(req,{...target,shared,follower:true,started:false});this.inFlight++;return 'allow';
-      }
-      shared={url,followers:0,value:null};shared.done=new Promise(resolve=>{shared.resolve=resolve});this.shared.set(op,shared);
-      target.shared=shared;target.started=false;this.inFlight++;
-      // One absolute timer also releases followers of a slow or never-started leader.
-      if(this.shareTimer===null)this.shareTimer=setTimeout(()=>{this.invalid=true;this.clearShared();},Math.max(0,this.deadline-performance.now()));
-    }
-    this.scopeSet=true;this.org=target.org;this.seen.add(op);this.requests.set(req,target);this.readStates[op].state='accepted';this.counts.allowed_reads++;return 'allow';
-  }
-  recordDenial(req,reason='unavailable',outerKind='non_read'){
-    try{
-      if(this.closed)return;
-      const sample={...deniedTarget(req),outer_kind:['non_read','navigation_scope','approved_read_redirect'].includes(outerKind)?outerKind:'non_read',reason:['method','frame','redirect','selected_page','request_origin','target','body_shape','content_type','duplicate_read','organization_scope'].includes(reason)?reason:'unavailable'};
-      const match=this.denials.samples.find(row=>Object.entries(sample).every(([key,value])=>row[key]===value));
-      if(match)match.count=Math.min(64,match.count+1);
-      else if(this.denials.samples.length<8)this.denials.samples.push({...sample,count:1});
-      else this.denials.omitted=Math.min(64,this.denials.omitted+1);
-    }catch{} // Diagnostic failure never changes denial or prevents route.abort.
-  }
-  clearShared(){for(const item of this.shared.values()){item.value=null;item.resolve(false);}clearTimeout(this.shareTimer);this.shareTimer=null;}
-  shareActive(req,target){return !this.closed&&!this.invalid&&performance.now()<this.deadline&&this.page.url()===LETTER_SCREEN&&req.frame()===this.page.mainFrame()&&req.url()===target.url;}
-  reject(req,reason='unavailable',outerKind='non_read'){this.invalid=true;if(this.capabilities)this.clearShared();this.counts.blocked_other=Math.min(64,this.counts.blocked_other+1);this.recordDenial(req,reason,outerKind);return 'block';}
-  async replay(route,target){
-    const req=route.request();
-    try{
-      if(!this.shareActive(req,target))throw Error();
-      if(!await target.shared.done)throw Error();
-      const value=target.shared.value;
-      if(!value||!this.shareActive(req,target))throw Error();
-      // Detached validated JSON only: never repeat cookies or other response headers.
-      await route.fulfill({status:value.status,headers:{'content-type':value.contentType},body:value.bytes});
-    }catch{this.invalid=true;this.clearShared();await route.abort('blockedbyclient').catch(()=>{});}
-    finally{this.inFlight--;}
-  }
-  async forward(route){
-    const req=route.request(),target=this.requests.get(req),op=target?.op;
-    if(this.capabilities){
-      if(!target||target.started){this.reject(req);await route.abort('blockedbyclient').catch(()=>{});return;}
-      target.started=true;if(target.follower)return this.replay(route,target);
-    }
-    let response,phase='preflight',reason=null;const diagnostic=this.readStates[op];if(diagnostic)diagnostic.state='in_flight';if(!this.capabilities)this.inFlight++;
-    try{
-      const remaining=Math.floor(this.deadline-performance.now());
-      if(!op||this.closed||remaining<=0||this.capabilities&&!this.shareActive(req,target)){reason=!op?'unaccepted':this.closed?'closed':'deadline';throw Error();}
-      if(op==='list_cms_connections')this.required='pending';
-      // route.continue may follow redirects outside subsequent route handlers.
-      // Fetch exactly once with redirects/retries disabled, then fulfill only a valid response.
-      phase='fetch';response=await route.fetch({maxRedirects:0,maxRetries:0,timeout:remaining});
-      phase='protocol';const headers=response.headers(),status=response.status();
-      const checks={url_exact:response.url()===target.url,status_ok:status>=200&&status<300,content_type_json:JSON_TYPE.test(headers['content-type']||''),declared_size_ok:headers['content-length']===undefined||/^[0-9]+$/.test(headers['content-length'])&&Number(headers['content-length'])<=65536};
-      if(diagnostic)Object.assign(diagnostic.checks,checks);
-      if(op==='list_cms_connections')Object.assign(this.requiredChecks,checks);
-      if(this.closed||performance.now()>=this.deadline||!Object.values(checks).every(Boolean)){reason=this.closed?'closed':performance.now()>=this.deadline?'deadline':!checks.url_exact?'response_url':!checks.status_ok?'http_status':!checks.content_type_json?'content_type':'declared_size';throw Error();}
-      phase='body';const bytes=await response.body();
-      if(diagnostic)diagnostic.checks.body_size_ok=bytes.length<=65536;
-      if(op==='list_cms_connections')this.requiredChecks.body_size_ok=bytes.length<=65536;
-      if(bytes.length>65536){reason='body_size';throw Error();}
-      if(op==='list_cms_connections')this.requiredChecks.json_valid=false;
-      phase='json';if(diagnostic)diagnostic.checks.json_valid=false;const value=JSON.parse(bytes.toString('utf8'));if(diagnostic)diagnostic.checks.json_valid=true;
-      if(op==='list_cms_connections')Object.assign(this.requiredChecks,{json_valid:true,schema_valid:this.capabilities?validCapabilityConnections(value):validConnections(value),schema_shape:this.capabilities?{bounded_rows:Array.isArray(value?.connections)&&value.connections.length>=1&&value.connections.length<=32}:connectionShape(value)});
-      phase='schema';const valid=op==='list_cms_connections'?(this.capabilities?validCapabilityConnections(value):validConnections(value)):object(value)||Array.isArray(value);
-      if(diagnostic)diagnostic.checks.schema_valid=valid;
-      if(!valid){reason='schema';throw Error();}
-      if(this.closed||performance.now()>=this.deadline||this.capabilities&&!this.shareActive(req,target)){reason=this.closed?'closed':performance.now()>=this.deadline?'deadline':'attempt_invalid';throw Error();}
-      phase='fulfill';await route.fulfill({response,body:bytes});if(diagnostic)diagnostic.state='verified';
-      if(op==='list_cms_connections'&&!this.closed)this.required='verified';
-      if(this.capabilities){
-        if(!this.shareActive(req,target)){reason='attempt_invalid';throw Error();}
-        target.shared.value={status,contentType:headers['content-type'],bytes:Buffer.from(bytes)};
-        target.shared.resolve(true);
-      }
-    }catch{
-      if(diagnostic){diagnostic.state='failed';diagnostic.phase=phase;diagnostic.reason=reason||({preflight:'preflight_failed',fetch:'fetch_failed',protocol:'protocol_failed',body:'body_read_failed',json:'invalid_json',schema:'schema_failed',fulfill:'fulfill_failed'})[phase];}
-      this.invalid=true;if(this.capabilities)this.clearShared();if(op==='list_cms_connections')this.required='invalid';
-      await route.abort('blockedbyclient').catch(()=>{});
-    }finally{await response?.dispose().catch(()=>{});this.inFlight--;}
-  }
-  /** Finite wait within capture's original 60s deadline; never reissues a read. */
-  async ready(signal){
-    while(!this.closed&&!this.invalid&&!signal?.aborted&&(this.inFlight>0||['missing','pending'].includes(this.required))&&performance.now()<this.deadline)await new Promise(r=>setTimeout(r,Math.min(25,Math.max(0,this.deadline-performance.now()))));
-    const ready=!this.closed&&!this.invalid&&!signal?.aborted&&performance.now()<this.deadline&&this.inFlight===0&&this.required==='verified';
-    this.readiness={state:ready?'ready':this.closed?'closed':signal?.aborted?'cancelled':this.invalid?'invalid':'capture_deadline',in_flight:Math.min(this.capabilities?9:3,Math.max(0,this.inFlight))};return ready;
-  }
-  report(){return structuredClone({version:1,adapter:this.capabilities?LETTER_CAPABILITIES_ADAPTER:LETTER_ADAPTER,required_read:this.required==='verified'?'verified':this.required==='invalid'?'invalid':'missing',...this.counts,...(this.capabilities?{coalesced_reads:this.coalesced}:{}),required_response_checks:this.requiredChecks,reads:this.readStates,readiness:this.readiness,invalid:this.invalid,in_flight:Math.min(this.capabilities?9:3,Math.max(0,this.inFlight)),...(this.denials.samples.length||this.denials.omitted?{blocked_requests:this.denials}:{})});}
-  close(){this.closed=true;if(this.capabilities)this.clearShared();}
-}