@getopencrawl/cli 0.1.1 → 0.2.1

package/src/jobs.mjs

@@ -1,117 +0,0 @@
-import { displayLine } from './display.mjs';
-
-export const JOBS_HELP = `OpenCrawl · follow background jobs
-
-  opencrawl jobs list [--status active|completed|failed|cancelled|discarded] [--limit 20] [--offset 0]
-  opencrawl jobs show JOB_ID [--json]
-  opencrawl jobs wait JOB_ID [--timeout 300] [--json]
-
-Wait stops after 1–3600 seconds (default 300). Ctrl+C stops waiting locally.
-It never cancels, retries, or resubmits a job. The job ID remains in the receipt
-on timeout, permission failure, or job failure. Only completed exits successfully.
-Completion is for this job; source scan coverage and provider cleanup are separate.
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-export const GLOBAL_OPTIONS = ['account', 'server', 'config-dir', 'json'];
-export const ACTIVE = new Set(['queued', 'running', 'waiting', 'processing', 'pending']);
-export const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'discarded']);
-export function decimal(value, name, min, max, fallback) {
-  if (value === undefined && fallback !== undefined) return fallback;
-  if (typeof value !== 'string' || !/^(0|[1-9]\d*)$/.test(value) || !Number.isSafeInteger(Number(value)) || Number(value) < min || Number(value) > max)
-    throw new Error(`--${name} must be a decimal integer between ${min} and ${max}.`);
-  return Number(value);
-}
-export function allowedOptions(options, allowed, command) {
-  for (const key of Object.keys(options)) if (![...GLOBAL_OPTIONS, ...allowed].includes(key)) throw new Error(`Unsupported ${command} option: --${key}. Run opencrawl ${command} --help.`);
-}
-export function identifier(value, kind = 'job') {
-  if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(value)) throw new Error(`Supply the exact ${kind} ID from its receipt.`);
-  return value;
-}
-export function waitOptions(options) {
-  if (options.timeout !== undefined && !options.wait) throw new Error('--timeout requires --wait. Use jobs wait to follow an existing job.');
-  return decimal(options.timeout, 'timeout', 1, 3600, 300);
-}
-export function requestFailure(error) {
-  const status = Number.isInteger(error?.status) ? error.status : null;
-  const message = ({401: 'Sign in with a valid key for the intended workspace.', 403: 'This account cannot access this operation.',
-    404: 'This job or upload receipt is unavailable to this account. Check the account and saved ID.',
-    409: 'The request conflicts with the saved upload details or current source state. Inspect the receipt and source before explicitly retrying; do not start a duplicate upload.',
-    413: 'The file exceeds an upload limit.', 422: 'The upload details or content failed validation. Check the file and its scope.',
-    429: 'OpenCrawl is busy or a workspace limit was reached. Inspect the receipt before retrying.',
-    507: 'Workspace storage admission failed. Inspect the receipt before retrying.'})[status] || (status ? 'OpenCrawl could not complete this request. Inspect its receipt before retrying.' : 'The request outcome could not be confirmed. Check the connection and inspect the saved receipt.');
-  return { code: status ? `http_${status}` : 'request_unconfirmed', message, ...(status ? { status } : {}) };
-}
-export function validateJob(job, id = job?.id) {
-  if (!job || typeof job.id !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(job.id) || job.id !== id || typeof job.status !== 'string' || !ACTIVE.has(job.status) && !TERMINAL.has(job.status)) throw new Error('Server returned an invalid job receipt. No completion was assumed.');
-  return job;
-}
-export const getJob = async (client, id, options = {}) => validateJob(await client.request(`/api/v1/jobs/${encodeURIComponent(identifier(id))}`, { maxResponseBytes: 512 * 1024, ...options }), id);
-
-// Failures that say nothing about the job itself: the server restarting, a proxy, the network. Polling keeps going.
-const TRANSIENT=new Set([408,425,429,500,502,503,504]);
-export function transient(error){return error?.transient===true||TRANSIENT.has(error?.status);}
-// Wait before the next poll after a transient failure: 2s, 4s, 8s, then every 15s, never past the deadline.
-export async function settle(failures,{signal,now,sleep,deadline}){const next=Math.min(deadline,now()+Math.min(15000,1000*2**failures));while(!signal?.aborted&&now()<next)await sleep(Math.min(100,next-now()));}
-
-export async function waitForJob(client, id, { timeout = 300, signal, now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), onUpdate = () => {}, initial, deadline } = {}) {
-  identifier(id); const started = now(); deadline ??= started + timeout * 1000;
-  let job = initial ? validateJob(initial, id) : null, failures = 0;
-  const result = (outcome, error) => ({ job_id: id, job, wait: { outcome, elapsed_seconds: Math.max(0, Math.round((now() - started) / 1000)), ...(error ? { error } : {}) } });
-  while (true) {
-    if (signal?.aborted) return result('interrupted');
-    if (job && TERMINAL.has(job.status)) return result(job.status);
-    const remaining = deadline - now();
-    if (remaining <= 0) return result('timeout');
-    try { job = await getJob(client, id, { timeout: Math.max(1, Math.min(30000, Math.ceil(remaining))), signal }); failures = 0; }
-    catch (error) {
-      if (!signal?.aborted && now() < deadline && transient(error)) { await settle(++failures, {signal, now, sleep, deadline}); continue; }
-      return result(signal?.aborted ? 'interrupted' : now() >= deadline ? 'timeout' : 'unavailable', requestFailure(error));
-    }
-    onUpdate(job);
-    if (TERMINAL.has(job.status)) return result(job.status);
-    // Short sleeps keep Ctrl+C responsive without extra server polling.
-    const next = Math.min(deadline, now() + 2000);
-    while (!signal?.aborted && now() < next) await sleep(Math.min(100, next - now()));
-  }
-}
-export async function withInterrupt(action) {
-  const controller = new AbortController(); const stop = () => controller.abort();
-  process.on('SIGINT', stop); process.on('SIGTERM', stop);
-  try { return await action(controller.signal); }
-  finally { process.off('SIGINT', stop); process.off('SIGTERM', stop); }
-}
-export function renderJob(job) {
-  let text = `${displayLine(job.id)}\t${displayLine(job.status)}\t${displayLine(job.kind)}${job.filename ? '\t' + displayLine(job.filename) : ''}\n`;
-  if (job.error) text += `Issue: ${displayLine(job.error)}\n`;
-  if (typeof job.indexed_documents === 'number') text += `Indexed documents: ${job.indexed_documents}\n`;
-  for (const doc of job.documents || []) text += `${displayLine(doc.id)}\t${displayLine(doc.title)}\n`;
-  if (job.documents_truncated) text += 'Additional indexed documents are available in OpenCrawl.\n';
-  if (job.coverage) text += `Source listing coverage: ${displayLine(job.coverage.state)} (separate from this job's status).\n`;
-  return text;
-}
-export async function jobsCommand(rest, options, client, { stdout = text => process.stdout.write(text), stderr = text => process.stderr.write(text) } = {}) {
-  const [command, id, ...extra] = rest;
-  if (!['list', 'show', 'wait'].includes(command)) throw new Error('Choose jobs list, show, or wait. Run opencrawl jobs --help.');
-  allowedOptions(options, command === 'list' ? ['status', 'limit', 'offset'] : command === 'wait' ? ['timeout'] : [], 'jobs');
-  if (extra.length || command === 'list' && id !== undefined) throw new Error('Unexpected jobs argument.');
-  if (command === 'list') {
-    if (options.status !== undefined && !['active', ...ACTIVE, ...TERMINAL].includes(options.status)) throw new Error('Choose a job status shown in jobs --help.');
-    const query = new URLSearchParams({ limit: decimal(options.limit, 'limit', 1, 200, 20), offset: decimal(options.offset, 'offset', 0, 1000000, 0) });
-    if (options.status) query.set('status', options.status);
-    const data = await client.request('/api/v1/jobs?' + query, { maxResponseBytes: 2 * 1024 * 1024 });
-    if (!Array.isArray(data?.items)) throw new Error('Server returned an invalid job list.');
-    if (options.json) stdout(JSON.stringify(data, null, 2) + '\n');
-    else { stdout(data.items.map(renderJob).join('') || 'No jobs on this page.\n'); if (data.has_more) stderr(`More jobs are available; use --offset ${Number(data.offset) + data.items.length}.\n`); }
-    return 0;
-  }
-  identifier(id);
-  if (command === 'show') {
-    const job = await getJob(client, id); stdout(options.json ? JSON.stringify(job, null, 2) + '\n' : renderJob(job)); return 0;
-  }
-  const timeout = decimal(options.timeout, 'timeout', 1, 3600, 300);
-  stderr(`Following job ${displayLine(id)} for at most ${timeout} seconds.\n`);
-  const result = await withInterrupt(signal => waitForJob(client, id, { timeout, signal }));
-  stdout(options.json ? JSON.stringify(result, null, 2) + '\n' : (result.job ? renderJob(result.job) : `Job: ${displayLine(id)}\n`) + `Wait: ${displayLine(result.wait.outcome)}. The job was not cancelled or resubmitted.\n` + (result.wait.error ? result.wait.error.message + '\n' : ''));
-  return result.wait.outcome === 'completed' ? 0 : 1;
-}

package/src/jobs.test.mjs

@@ -1,67 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { jobsCommand, waitForJob, waitOptions, renderJob } from './jobs.mjs';
-import { displayText, displayLine } from './display.mjs';
-
-const id = 'job_' + 'a'.repeat(32);
-const job = status => ({ id, status, kind: 'upload', indexed_documents: status === 'completed' ? 1 : 0, documents: [] });
-test('job waiting follows exact IDs and stops on terminal status without any mutation', async () => {
-  let time = 0; const calls = []; const statuses = ['queued', 'running', 'completed'];
-  const result = await waitForJob({ request: async (route, options) => { calls.push([route, options]); return job(statuses.shift()); } }, id, { now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(result.wait.outcome, 'completed'); assert.equal(result.job_id, id); assert.equal(time, 4000);
-  assert.equal(calls.length, 3); assert(calls.every(([route, options]) => route === `/api/v1/jobs/${id}` && !options.method && options.timeout <= 30000));
-});
-test('all unsuccessful terminal states retain the job and return unsuccessful wait receipts', async () => {
-  for (const status of ['failed', 'cancelled', 'discarded']) {
-    const result = await waitForJob({ request: async () => job(status) }, id);
-    assert.equal(result.wait.outcome, status); assert.equal(result.job.id, id);
-  }
-});
-test('finite timeout and interruption do not cancel or retry work', async () => {
-  let time = 0, calls = 0;
-  const result = await waitForJob({ request: async (route, options) => { calls++; assert(options.timeout <= 1000); return job('waiting'); } }, id, { timeout: 1, now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(result.wait.outcome, 'timeout'); assert.equal(calls, 1); assert.equal(result.job_id, id);
-  const controller = new AbortController(); controller.abort();
-  const interrupted = await waitForJob({ request: async () => { throw Error('must not call'); } }, id, { signal: controller.signal });
-  assert.equal(interrupted.wait.outcome, 'interrupted'); assert.equal(interrupted.job_id, id);
-});
-test('permission and unknown failures never imply completion', async () => {
-  for (const status of [401,403,404,422,undefined]) {
-    let calls = 0;
-    const result = await waitForJob({ request: async () => { calls++; const error = Error('RAW PRIVATE BODY'); error.status = status; throw error; } }, id);
-    assert.equal(result.wait.outcome, 'unavailable'); assert.equal(result.job_id, id); assert.equal(calls, 1); assert(!JSON.stringify(result).includes('RAW PRIVATE'));
-  }
-  const result = await waitForJob({ request: async () => job('surprise') }, id);
-  assert.equal(result.wait.outcome, 'unavailable'); assert.equal(result.job, null);
-});
-test('jobs validate option bounds before calling the server', async () => {
-  const client = { request: async () => { throw Error('unexpected call'); } };
-  for (const [rest, options] of [[['wait', id], { timeout: '0' }], [['wait', id], { timeout: '3601' }], [['list'], { limit: '1e2' }], [['show', '../secret'], {}], [['show', id], { retry: true }], [['list', 'extra'], {}]]) {
-    await assert.rejects(jobsCommand(rest, options, client), error => !error.message.includes('unexpected call'));
-  }
-  assert.throws(() => waitOptions({ timeout: '1' }), /requires --wait/);
-});
-test('job human display escapes terminal controls while JSON retains source text', async () => {
-  const value = { ...job('failed'), filename: 'report\u001b]52;copy\u0007.pdf', error: 'error\rreplacement\u202e', documents: [{ id: 'doc', title: 'a\nb' }] };
-  assert(!renderJob(value).includes('\u001b')); assert(renderJob(value).includes('\\u001b')); assert(renderJob(value).includes('a\\nb'));
-  assert.equal(displayText('a\nb\tc\u001b'), 'a\nb\tc\\u001b'); assert.equal(displayLine('a\nb'), 'a\\nb');
-  let output = ''; await jobsCommand(['show', id], { json: true }, { request: async () => value }, { stdout: text => { output += text; } });
-  assert.equal(JSON.parse(output).filename, value.filename);
-});
-
-test('a restart, proxy error or lost connection keeps the wait going until the job finishes', async () => {
-  let time = 0, calls = 0;
-  const failures = [502, 503, 429, 'network'];
-  const result = await waitForJob({ request: async () => {
-    calls++;
-    const next = failures.shift();
-    if (next === undefined) return job('completed');
-    const error = Error('RAW PRIVATE BODY'); if (next === 'network') error.transient = true; else error.status = next; throw error;
-  } }, id, { now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(result.wait.outcome, 'completed'); assert.equal(calls, 5);
-  assert(time >= 2000 + 4000 + 8000 + 15000);  // backs off instead of hammering a restarting server
-  // Still bounded by the caller's deadline.
-  time = 0;
-  const stuck = await waitForJob({ request: async () => { const error = Error('x'); error.status = 503; throw error; } }, id, { timeout: 60, now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(stuck.wait.outcome, 'timeout');
-});

package/src/mcp-conversations.test.mjs

@@ -1,80 +0,0 @@
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import http from 'node:http';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import { spawn } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-
-const script = fileURLToPath(new URL('../bin/opencrawl.mjs', import.meta.url));
-const names = ['opencrawl_list_conversations', 'opencrawl_read_conversation', 'opencrawl_conversation_context'];
-
-test('stdio forwards bounded conversation tools and structured errors without rewriting evidence', async () => {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'oc-mcp-conversation-'));
-  const calls = [];
-  const version = 'a'.repeat(64);
-  const selector = { ref: 't2', start_char: 12, end_char: 24 };
-  const context = 'Quoted source: 日本語 🌲\n\u001b[31m';
-  const receipt = { version, context, selected: [selector], included: [{ ...selector, citation: 'C1' }],
-    omitted: [], context_bytes: Buffer.byteLength(context), budget: 700, budget_unit: 'utf8_bytes' };
-  const error = { code: 'snapshot_changed', message: 'Read the conversation again before selecting evidence.' };
-  const server = http.createServer(async (request, response) => {
-    let raw = '';for await (const part of request) raw += part;
-    const message = JSON.parse(raw);
-    calls.push({ path: request.url, authorization: request.headers.authorization,
-      protocol: request.headers['mcp-protocol-version'], message });
-    if (!Object.hasOwn(message, 'id')) { response.writeHead(202);response.end();return; }
-    let result;
-    if (message.method === 'initialize') result = { protocolVersion: '2025-11-25', capabilities: { tools: {} }, serverInfo: { name: 'fixture', version: '1' } };
-    else if (message.method === 'tools/list') result = { tools: names.map(name => ({ name, inputSchema: { type: 'object' } })) };
-    else {
-      const failed = message.params.arguments.version === 'b'.repeat(64);
-      const value = failed ? error : receipt;
-      result = { content: [{ type: 'text', text: JSON.stringify(value) }], structuredContent: value, isError: failed };
-    }
-    response.writeHead(200, { 'Content-Type': 'application/json' });
-    response.end(JSON.stringify({ jsonrpc: '2.0', id: message.id, result }));
-  });
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
-  try {
-    for (const token of ['oc_synthetic_api_only', 'oca_synthetic_oauth_only']) {
-      const env = { ...process.env };
-      for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
-      Object.assign(env, { OPENCRAWL_CONFIG_DIR: path.join(directory, 'isolated'),
-        OPENCRAWL_URL: `http://127.0.0.1:${server.address().port}`, OPENCRAWL_API_KEY: token });
-      const args = { document_id: 'doc_synthetic', version, branch: 'selected', selections: [selector], budget: 700 };
-      const messages = [
-        { jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2025-11-25' } },
-        { jsonrpc: '2.0', method: 'notifications/initialized' },
-        { jsonrpc: '2.0', id: 2, method: 'tools/list' },
-        ...names.map((name, index) => ({ jsonrpc: '2.0', id: index + 3, method: 'tools/call', params: { name, arguments: args } })),
-        { jsonrpc: '2.0', id: 6, method: 'tools/call', params: { name: names[2], arguments: { ...args, version: 'b'.repeat(64) } } },
-      ];
-      const before = calls.length;
-      const result = await new Promise((resolve, reject) => {
-        const child = spawn(process.execPath, [script, 'mcp'], { cwd: directory, env, stdio: ['pipe', 'pipe', 'pipe'] });
-        let stdout = '', stderr = '';const timer = setTimeout(() => child.kill('SIGKILL'), 10000);
-        child.stdout.on('data', part => { stdout += part; });child.stderr.on('data', part => { stderr += part; });
-        child.once('error', reject);child.once('close', code => { clearTimeout(timer);resolve({ code, stdout, stderr }); });
-        child.stdin.end(messages.map(message => JSON.stringify(message)).join('\n') + '\n');
-      });
-      assert.equal(result.code, 0);assert.equal(result.stderr, '');assert.ok(!result.stdout.includes(token));
-      const responses = result.stdout.trim().split('\n').map(line => JSON.parse(line));
-      assert.deepEqual(responses.map(value => value.id), [1, 2, 3, 4, 5, 6]);
-      assert.deepEqual(responses[1].result.tools.map(tool => tool.name), names);
-      for (const response of responses.slice(2, 5)) {
-        assert.equal(response.result.isError, false);assert.deepEqual(response.result.structuredContent, receipt);
-        assert.deepEqual(JSON.parse(response.result.content[0].text), receipt);
-      }
-      assert.equal(responses[5].result.isError, true);assert.deepEqual(responses[5].result.structuredContent, error);
-      assert.deepEqual(calls.slice(before).map(call => call.message), messages);
-      assert.ok(calls.slice(before).every(call => call.path === '/mcp' && call.authorization === `Bearer ${token}`));
-      assert.ok(calls.slice(before + 1).every(call => call.protocol === '2025-11-25'));
-    }
-    await assert.rejects(fs.stat(path.join(directory, 'isolated')), { code: 'ENOENT' });
-  } finally {
-    await new Promise(resolve => server.close(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
-  }
-});

package/src/monitors.mjs

@@ -1,146 +0,0 @@
-import { allowedOptions, decimal, validateJob, renderJob } from './jobs.mjs';
-import { displayLine } from './display.mjs';
-
-export const MONITORS_HELP = `OpenCrawl · inspect exact public-page observations
-
-  opencrawl monitors list [--limit 20] [--offset 0] [--json]
-  opencrawl monitors show SOURCE_ID [--json]
-  opencrawl monitors check SOURCE_ID --generation N [--json]
-  opencrawl monitors changes [--source SOURCE_ID] [--limit 20] [--cursor CURSOR] [--json]
-  opencrawl monitors context CHANGE_ID --revision HASH [--budget 4096] [--json]
-
-Create monitors in OpenCrawl with 1–5 explicit public static URLs. Checks do not
-follow discovered links or execute JavaScript. A failed or missing page never
-proves a job closed or content was removed. Inspect show for each page's latest
-attempt and last good observation. Check queues work; follow its job separately.
-Context requires an exact change ID and immutable version from changes. It
-exports before/after evidence, including observation times, within 256–16384
-UTF-8 bytes. Truncation and omissions go to stderr; stdout is exact context.
---json preserves the complete receipt. No automatic check or newer substitution.
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-const SOURCE = /^src_[a-f0-9]{32}$/, CHANGE = /^mchg_[a-f0-9]{32}$/, HASH = /^[a-f0-9]{64}$/;
-const integer = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max;
-const id = (value, pattern, kind) => {
-  if (typeof value !== 'string' || !pattern.test(value)) throw new Error(`Supply the exact ${kind} ID from its list.`);
-  return value;
-};
-const cursor = value => {
-  if (typeof value !== 'string' || !value || value.length > 256 || /[\x00-\x20\x7f]/.test(value)) throw new Error('Use the unchanged opaque cursor from the previous changes page.');
-  return value;
-};
-export function monitorRequest(rest, options) {
-  const [command, value, ...extra] = rest;
-  const allowed = {list: ['limit','offset'], show: [], check: ['generation'], changes: ['source','limit','cursor'], context: ['revision','budget']};
-  if (!Object.hasOwn(allowed, command)) throw new Error('Choose monitors list, show, check, changes, or context. Run opencrawl monitors --help.');
-  allowedOptions(options, allowed[command], 'monitors');
-  if (extra.length || ['list','changes'].includes(command) && value !== undefined) throw new Error('Unexpected monitors argument.');
-  if (command === 'list') {
-    const limit=decimal(options.limit,'limit',1,20,20), offset=decimal(options.offset,'offset',0,1000000,0);
-    return {command,limit,offset,route:'/api/v1/monitors?'+new URLSearchParams({limit,offset}),options:{}};
-  }
-  if (command === 'changes') {
-    const limit=decimal(options.limit,'limit',1,20,20), query=new URLSearchParams({limit});
-    if (options.source !== undefined) query.set('source_id',id(options.source,SOURCE,'monitor'));
-    if (options.cursor !== undefined) query.set('cursor',cursor(options.cursor));
-    return {command,limit,source:options.source,route:'/api/v1/monitor-changes?'+query,options:{}};
-  }
-  if (command === 'context') {
-    id(value,CHANGE,'change');
-    if (typeof options.revision !== 'string' || !HASH.test(options.revision)) throw new Error('--revision must be the exact 64-character lowercase version hash from changes.');
-    const budget=decimal(options.budget,'budget',256,16384,4096), selections=[{change_id:value,version:options.revision}];
-    return {command,id:value,selections,budget,route:'/api/v1/monitor-changes/context',options:{method:'POST',body:{selections,budget}}};
-  }
-  id(value,SOURCE,'monitor');
-  return {command,id:value,route:'/api/v1/monitors/'+value+(command==='check'?'/check':''),
-    options:command==='check'?{method:'POST',body:{expected_generation:decimal(options.generation,'generation',1,Number.MAX_SAFE_INTEGER)}}:{}};
-}
-function validateMonitor(value, expected = value?.id) {
-  if (!value || !SOURCE.test(value.id) || value.id!==expected || value.kind!=='monitor' || typeof value.name!=='string' ||
-      !integer(value.generation,1,Number.MAX_SAFE_INTEGER) || !['active','paused','revoked','error'].includes(value.status) ||
-      !Array.isArray(value.targets) || value.targets.length<1 || value.targets.length>5 || value.targets.some(target=>
-        !target || typeof target.requested_url!=='string' || typeof target.state!=='string' || typeof target.enabled!=='boolean'))
-    throw new Error('Server returned an invalid monitor receipt. No check was assumed.');
-  return value;
-}
-function validateChange(value, source) {
-  if (!value || !CHANGE.test(value.id) || !HASH.test(value.version) || !SOURCE.test(value.source_id) || source && value.source_id!==source ||
-      typeof value.title!=='string' || typeof value.final_url!=='string' || typeof value.observed_before!=='string' || typeof value.observed_after!=='string')
-    throw new Error('Server returned an invalid or out-of-scope change receipt.');
-  return value;
-}
-export function validateChangeContext(value, request) {
-  const invalid=()=>{throw new Error('Server returned an invalid historical context receipt. No context was emitted; inspect the change again.');};
-  if (!value || typeof value.context!=='string' || value.budget!==request.budget || value.budget_unit!=='utf8_bytes' ||
-      Buffer.byteLength(value.context)!==value.context_bytes || value.context_bytes>request.budget ||
-      !Array.isArray(value.selections) || value.selections.length!==1 || value.selections[0]?.change_id!==request.id || value.selections[0]?.version!==request.selections[0].version ||
-      !Array.isArray(value.included) || value.included.length>2 || !Array.isArray(value.omitted) || value.omitted.length>2 ||
-      !Array.isArray(value.warnings) || value.warnings.length>20 || value.warnings.some(w=>typeof w!=='string') ||
-      Boolean(value.context)!==Boolean(value.included.length)) invalid();
-  const seen=new Set(), labels=new Set(), chars=Array.from(value.context); let lastEnd=0;
-  for (const entry of [...value.included,...value.omitted]) {
-    if (!entry || entry.change_id!==request.id || entry.version!==request.selections[0].version || !['before','after'].includes(entry.side) || seen.has(entry.side)) invalid();
-    seen.add(entry.side);
-  }
-  if (seen.size!==2 || value.omitted.some(entry=>entry.reason!=='budget')) invalid();
-  for (const entry of value.included) {
-    const span=entry.context_span, excerpt=entry.excerpt;
-    if (typeof excerpt!=='string' || !excerpt || typeof entry.citation!=='string' || !/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(entry.citation) || labels.has(entry.citation) ||
-        !integer(entry.full_length,0,131072) || !integer(entry.start_char,0,entry.full_length) || !integer(entry.end_char,entry.start_char+1,entry.full_length) ||
-        Array.from(excerpt).length!==entry.end_char-entry.start_char || typeof entry.truncated!=='boolean' ||
-        typeof entry.observed_at!=='string' || typeof entry.url!=='string' || !span || !integer(span.start_char,lastEnd,chars.length) ||
-        !integer(span.end_char,span.start_char+1,chars.length) || chars.slice(span.start_char,span.end_char).join('')!==excerpt) invalid();
-    labels.add(entry.citation); lastEnd=span.end_char;
-  }
-  return value;
-}
-export function monitorFailure(error) {
-  const status=Number.isInteger(error?.status)?error.status:undefined;
-  const codes={monitor_changed:'Monitor settings changed. Run monitors show and review the current generation before explicitly checking again.',
-    monitor_not_active:'This monitor is not active. Review its status in OpenCrawl; no check was queued.',
-    monitor_change_changed:'The historical version does not match. Inspect changes and explicitly select the intended immutable version; no newer change was substituted.',
-    monitor_evidence_changed:'Historical evidence or its access changed. Inspect the change again before explicitly requesting context.'};
-  if (status===409 && Object.hasOwn(codes,error.code)) return {code:error.code,status,message:codes[error.code]};
-  const message=({401:'Sign in with a valid key for the intended workspace.',403:'This account cannot perform this monitor operation.',
-    404:'This monitor or historical change is unavailable to this account, or its retained history has expired. No other evidence was substituted.',
-    409:'The monitor or historical selection changed. Inspect it before explicitly trying again.',413:'A monitor or retained-history limit was reached. Review capacity in OpenCrawl.',
-    422:'The monitor request is invalid. Check its exact ID, generation, version and byte budget.',429:'OpenCrawl is busy or a workspace limit was reached. Inspect the monitor before checking again.',
-    507:'Workspace storage admission failed. Review storage before checking again.'})[status] ||
-    'The monitor request outcome could not be confirmed. Inspect its current state before explicitly trying again. No automatic retry was made.';
-  return {code:status?`http_${status}`:'request_unconfirmed',message,...(status?{status}:{})};
-}
-export async function monitorsCommand(rest, options, client, {stdout=value=>process.stdout.write(value),stderr=value=>process.stderr.write(value)}={}) {
-  const request=monitorRequest(rest,options); let data;
-  try {data=await client.request(request.route,{...request.options,maxResponseBytes:1024*1024});}
-  catch(error){const failure=monitorFailure(error);if(options.json)stdout(JSON.stringify({error:failure},null,2)+'\n');else stderr(failure.message+'\n');return 1;}
-  if(request.command==='list'){
-    if(!Array.isArray(data?.items)||data.items.length>request.limit||data.limit!==request.limit||data.offset!==request.offset)throw new Error('Server returned an invalid monitor page.');
-    data.items.forEach(value=>validateMonitor(value));
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(data.items.map(value=>`${value.id}\tg${value.generation}\t${value.status}\t${displayLine(value.name)}\n`).join('')||'No monitors on this page.\n');if(data.has_more)stderr(`Next page: --offset ${request.offset+data.items.length}.\n`);}
-  }else if(request.command==='show'){
-    validateMonitor(data,request.id);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(`${displayLine(data.name)}\nMonitor: ${data.id}\nGeneration: ${data.generation}; status: ${data.status}\n`);
-      for(const target of data.targets)stdout(`${displayLine(target.requested_url)}\n  ${displayLine(target.state)}; latest attempt: ${displayLine(target.last_attempt_at||'none')}; last good observation: ${displayLine(target.last_success_at||'none')}\n${target.issue?.message?'  '+displayLine(target.issue.message)+'\n':''}`);
-      stderr('Only selected static pages are checked. Failure or missing content is not evidence of removal.\n');}
-  }else if(request.command==='check'){
-    validateJob(data?.job);
-    stdout(options.json?JSON.stringify(data,null,2)+'\n':renderJob(data.job));
-    if(!options.json)stderr(`Check accepted; this does not confirm page coverage. Follow with jobs wait ${data.job.id}, then monitors show ${request.id}.\n`);
-  }else if(request.command==='changes'){
-    if(!Array.isArray(data?.items)||data.items.length>request.limit||data.limit!==request.limit||!(data.next_cursor===null||typeof data.next_cursor==='string'))throw new Error('Server returned an invalid change page.');
-    data.items.forEach(value=>validateChange(value,request.source));if(data.next_cursor!==null)cursor(data.next_cursor);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stdout(data.items.map(value=>`${value.id}\t${displayLine(value.title)}\nVersion: ${value.version}\nObserved: ${displayLine(value.observed_before)} → ${displayLine(value.observed_after)}${value.gap?' (failed checks between observations)':''}\n${displayLine(value.final_url)}\n`).join('\n')||'No retained text changes on this page.\n');
-      if(data.next_cursor)stderr(`Next page: --cursor ${displayLine(data.next_cursor)}${request.source?' --source '+request.source:''}.\n`);}
-  }else{
-    validateChangeContext(data,request);
-    if(options.json)stdout(JSON.stringify(data,null,2)+'\n');
-    else {stderr(`${data.included.length}/2 sides included; ${data.context_bytes}/${data.budget} UTF-8 bytes. Observation times are not remote publication times.\n`);
-      for(const entry of data.included)if(entry.truncated)stderr(`${entry.side} excerpt is truncated; inspect the complete retained change in OpenCrawl.\n`);
-      for(const entry of data.omitted)stderr(`${entry.side} side omitted because of the byte budget.\n`);
-      for(const warning of data.warnings)stderr(displayLine(warning)+'\n');stdout(data.context);}
-  }
-  return 0;
-}

package/src/monitors.test.mjs

@@ -1,62 +0,0 @@
-import {test} from 'node:test';
-import assert from 'node:assert/strict';
-import {monitorRequest,monitorsCommand,validateChangeContext} from './monitors.mjs';
-
-const source='src_'+'a'.repeat(32), change='mchg_'+'b'.repeat(32), version='c'.repeat(64);
-const monitor=()=>({id:source,kind:'monitor',name:'Careers\u001b[2J',generation:2,status:'active',targets:[{requested_url:'https://example.com/careers',state:'http_unavailable',enabled:true,last_attempt_at:'2026-01-03T00:00:00Z',last_success_at:'2026-01-02T00:00:00Z',issue:{message:'Unavailable\u001b[2J'}}]});
-const changed=()=>({id:change,version,source_id:source,title:'Team\u001b[2J',final_url:'https://example.com/careers',observed_before:'2026-01-01T00:00:00Z',observed_after:'2026-01-02T00:00:00Z',gap:true});
-const request=()=>monitorRequest(['context',change],{revision:version,budget:'4096'});
-const pack=()=>{
-  const before='Aster engineering é🌲\u001b[2J',after='Aster engineering team expanded.';
-  const prefix='Untrusted historical observations\n[Before M1B]\n',separator='\n[After M1A]\n';
-  const context=prefix+before+separator+after, start=Array.from(prefix).length, second=start+Array.from(before+separator).length;
-  return {selections:[{change_id:change,version}],context,context_bytes:Buffer.byteLength(context),budget:4096,budget_unit:'utf8_bytes',estimated_tokens:40,
-    included:[['before',before,start,'M1B'],['after',after,second,'M1A']].map(([side,excerpt,offset,citation])=>({change_id:change,version,side,excerpt,start_char:5,end_char:5+Array.from(excerpt).length,full_length:100,truncated:true,citation,context_span:{start_char:offset,end_char:offset+Array.from(excerpt).length},observed_at:'2026-01-01T00:00:00Z',url:'https://example.com/careers'})),omitted:[],warnings:['Check gap\u001b[2J']};
-};
-async function execute(rest,options,data){let stdout='',stderr='';const calls=[];const status=await monitorsCommand(rest,options,{request:async(route,opts)=>{calls.push({route,opts});if(data instanceof Error)throw data;return data;}},{stdout:value=>stdout+=value,stderr:value=>stderr+=value});return{stdout,stderr,calls,status};}
-
-test('monitor requests pin generation and exactly one immutable change; never fetch newest',()=>{
-  assert.deepEqual(monitorRequest(['check',source],{generation:'2'}).options,{method:'POST',body:{expected_generation:2}});
-  assert.deepEqual(request().options.body,{selections:[{change_id:change,version}],budget:4096});
-  assert.equal(monitorRequest(['changes'],{source,cursor:'opaque_-',limit:'3'}).route,`/api/v1/monitor-changes?limit=3&source_id=${source}&cursor=opaque_-`);
-  assert.equal(monitorRequest(['list'],{limit:'2',offset:'4'}).route,'/api/v1/monitors?limit=2&offset=4');
-  for(const generation of [undefined,'0','-1','1e2','02','9007199254740992'])assert.throws(()=>monitorRequest(['check',source],{generation}));
-  for(const options of [{},{revision:version.toUpperCase()},{revision:version,budget:'16385'},{revision:version,budget:'255'},{revision:version,source},{revision:version,selection:[]}])assert.throws(()=>monitorRequest(['context',change],options));
-  for(const rest of [['context',change,'extra'],['show','../private'],['check'],['list',source],['create']])assert.throws(()=>monitorRequest(rest,{}));
-  assert.throws(()=>monitorRequest(['changes'],{cursor:'x'.repeat(257)}));
-});
-test('exact historical stdout and complete JSON receipt preserve Unicode/control bytes',async()=>{
-  const data=pack(),result=await execute(['context',change],{revision:version,budget:'4096'},data);
-  assert.equal(result.status,0);assert.equal(result.stdout,data.context);assert.equal(result.calls.length,1);assert.deepEqual(result.calls[0].opts.body,request().options.body);
-  assert.match(result.stderr,/2\/2 sides included/);assert.match(result.stderr,/truncated/);assert.match(result.stderr,/\\u001b/);assert.ok(!result.stderr.includes('\u001b'));
-  const json=await execute(['context',change],{revision:version,json:true},data);assert.deepEqual(JSON.parse(json.stdout),data);assert.equal(json.stderr,'');
-});
-test('empty budget receipt and explicit omitted side do not imply a removal',async()=>{
-  const data=pack();data.context='';data.context_bytes=0;data.included=[];data.omitted=['before','after'].map(side=>({change_id:change,version,side,reason:'budget'}));
-  const result=await execute(['context',change],{revision:version},data);assert.equal(result.status,0);assert.equal(result.stdout,'');assert.match(result.stderr,/0\/2/);assert.match(result.stderr,/before side omitted/);assert.match(result.stderr,/after side omitted/);
-});
-test('receipt validation refuses changed identities, incomplete partition, incorrect Unicode spans and byte overruns before output',async()=>{
-  const original=pack();const cases=[{selections:[]},{selections:[{change_id:change,version:'d'.repeat(64)}]},{budget:5000},{budget_unit:'tokens'},{context_bytes:1},
-    {context:'x'.repeat(4097),context_bytes:4097},{included:[original.included[0]],omitted:[]},
-    {included:[original.included[0],original.included[0]]},{included:[{...original.included[0],change_id:'mchg_'+'f'.repeat(32)},original.included[1]]},
-    {included:[{...original.included[0],context_span:{start_char:0,end_char:2}},original.included[1]]},
-    {included:[{...original.included[0],end_char:9000},original.included[1]]},{included:[],omitted:[{change_id:change,version,side:'before',reason:'missing'}]}];
-  for(const replacement of cases){let out='';await assert.rejects(monitorsCommand(['context',change],{revision:version},{request:async()=>({...pack(),...replacement})},{stdout:v=>out+=v,stderr:()=>{}}));assert.equal(out,'');}
-  validateChangeContext(pack(),request());
-});
-test('monitor displays escape controls and distinguish latest attempt from last good observation',async()=>{
-  const show=await execute(['show',source],{},monitor());assert.ok(!show.stdout.includes('\u001b'));assert.match(show.stdout,/\\u001b/);assert.match(show.stdout,/2026-01-03.*last good observation: 2026-01-02/);assert.match(show.stderr,/not evidence of removal/);
-  const list=await execute(['list'],{}, {items:[monitor()],limit:20,offset:0,has_more:true});assert.match(list.stderr,/--offset 1/);
-  const changes=await execute(['changes'],{source},{items:[changed()],limit:20,next_cursor:'opaque'});assert.match(changes.stdout,/Version: c{64}/);assert.match(changes.stdout,/failed checks/);assert.ok(!changes.stdout.includes('\u001b'));assert.match(changes.stderr,/--source src_/);
-  await assert.rejects(execute(['changes'],{source},{items:[{...changed(),source_id:'src_'+'f'.repeat(32)}],limit:20,next_cursor:null}));
-});
-test('check only returns accepted job and never polls or assumes all pages succeeded',async()=>{
-  const job={id:'job_test',kind:'monitor',status:'queued'};const result=await execute(['check',source],{generation:'2'},{job});
-  assert.equal(result.calls.length,1);assert.match(result.stderr,/does not confirm page coverage/);assert.match(result.stderr,/jobs wait job_test/);assert.match(result.stdout,/queued/);
-});
-test('fixed conflict/auth/retention errors never echo provider payloads or automatically retry',async()=>{
-  for(const [status,code,pattern] of [[409,'monitor_changed',/current generation/],[409,'monitor_change_changed',/no newer change/],[409,'monitor_evidence_changed',/access changed/],[409,'monitor_not_active',/not active/],[404,'arbitrary',/retained history/],[401,null,/Sign in/],[500,'secret',/No automatic retry/]]){
-    const error=Object.assign(new Error('RAW SECRET\u001b[2J'),{status,code});const result=await execute(['context',change],{revision:version,json:true},error);
-    assert.equal(result.status,1);assert.equal(result.calls.length,1);assert.match(JSON.parse(result.stdout).error.message,pattern);assert.equal(result.stderr,'');assert.ok(!result.stdout.includes('SECRET'));
-  }
-});

package/src/product-letter-capabilities.mjs

@@ -1,71 +0,0 @@
-/** Fixed read-only Letter picker scope. No provider selection or neutral re-render. */
-export const LETTER_CAPABILITIES_ADAPTER='letter-capabilities-read-v1';
-export const LETTER_PICKER='[data-testid="connect-integration"]';
-export const LETTER_PICKER_OPEN='[data-testid="connect-integration-open"]';
-export function isLetterCapabilitiesPolicy(policy,screen){return policy?.version===3&&policy.request_adapter===LETTER_CAPABILITIES_ADAPTER&&screen.url==='https://app.letterstory.com/letterpress?view=connections'&&screen.account_marker?.required===true&&screen.ready_selector===LETTER_PICKER_OPEN&&screen.scope_selector===LETTER_PICKER;}
-
-/** Executed in the page before the one fixed opener click. Only finite status leaves. */
-export function letterPickerReady({screen,expected}){
- const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
- const all=s=>[...document.querySelectorAll(s)].filter(visible);
- if(location.href!==screen.url)return {error:'scope_mismatch'};
- if(screen.login_selector&&all(screen.login_selector).length)return {error:'session_expired',login:true};
- const markers=all(screen.account_marker.selector);
- if(markers.length!==1||typeof expected!=='string'||!expected||markers[0].innerText.trim()!==expected)return {error:'context_mismatch'};
- const buttons=all('[data-testid="connect-integration-open"]');
- if(buttons.length!==1||buttons[0].tagName!=='BUTTON'||buttons[0].innerText.trim()!=='Connect'||buttons[0].disabled||buttons[0].closest('form')||document.querySelector('[data-testid="connect-integration"]'))return {error:'not_ready'};
- return {ready:true};
-}
-
-/** All public text is literal in-place UI text; unknown structure refuses before pixels. */
-export function extractLetterCapabilities({screen,policy,expected,freeze=false}){
- const visible=e=>{const s=getComputedStyle(e),r=e.getBoundingClientRect();return s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'&&r.width>0&&r.height>0;};
- const all=s=>[...document.querySelectorAll(s)].filter(visible);
- const one=s=>{const a=all(s);return a.length===1?a[0]:null;};
- if(location.href!==screen.url)return {error:'scope_mismatch'};
- if(screen.login_selector&&all(screen.login_selector).length)return {error:'session_expired',login:true};
- const marker=one(screen.account_marker.selector)?.innerText.trim();
- if(!marker||marker!==expected)return {error:'context_mismatch'};
- const dialog=one('[data-testid="connect-integration"]');
- if(!dialog||dialog.getAttribute('role')!=='dialog')return {error:'not_ready'};
- const exact=(e,tags)=>e&&[...e.children].map(n=>n.tagName.toUpperCase()).join(',')===tags&&!Array.from(e.childNodes).some(n=>n.nodeType===Node.TEXT_NODE&&n.textContent.trim());
- const leaf=(e,max)=>e&&e.children.length===0&&visible(e)&&e.textContent===e.textContent.trim()&&e.textContent.length>0&&Array.from(e.textContent).length<=max&&!/[\p{C}\n\r]/u.test(e.textContent)&&!/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}|https?:\/\/|\b(?:Bearer |sk-|ghp_|AKIA)/i.test(e.textContent);
- if(!exact(dialog,'DIV,DIV,BUTTON'))return {error:'mask_unresolved'};
- const [header,body,close]=dialog.children;
- if(!exact(header,'H2,P')||!leaf(header.children[0],100)||!leaf(header.children[1],100)||header.children[0].textContent!=='Connect an integration'||header.children[1].textContent!=='Connect where you use it.'||!exact(body,'UL')||!exact(close,'SVG,SPAN')||close.children[1]?.textContent!=='Close')return {error:'mask_unresolved'};
- const list=body.children[0],rows=[...list.children];
- if(rows.length<1||rows.length>32||!exact(list,rows.map(()=> 'LI').join(',')))return {error:'mask_unresolved'};
- const masks=[close],publicNodes=[dialog,header,...header.children,body,list],names=new Set(),text=['Connect an integration','Connect where you use it.'];
- const badges=[],logos=[],missingBadgeSlots=[];
- for(const row of rows){
-  if(!exact(row,'BUTTON'))return {error:'mask_unresolved'};
-  const button=row.children[0];if(!exact(button,'IMG,SPAN')&&!exact(button,'IMG,SPAN,SPAN'))return {error:'mask_unresolved'};
-  const [logo,copy,badge]=button.children;
-  if(!copy.classList.contains('min-w-0')||!exact(copy,'SPAN,SPAN')||!leaf(copy.children[0],100)||!leaf(copy.children[1],400)||names.has(copy.children[0].textContent))return {error:'mask_unresolved'};
-  names.add(copy.children[0].textContent);text.push(copy.children[0].textContent,copy.children[1].textContent);
-  masks.push(logo);logos.push(logo);if(badge){masks.push(badge);badges.push(badge);}else missingBadgeSlots.push(button);publicNodes.push(row,button,copy,...copy.children);
- }
- // Unknown controls or pseudo/image content in the public structure are not public fields.
- if(publicNodes.some(e=>!visible(e)||getComputedStyle(e).backgroundImage!=='none'||getComputedStyle(e).listStyleImage!=='none'||['::before','::after','::marker'].some(p=>!['none','normal'].includes(getComputedStyle(e,p).content))))return {error:'mask_unresolved'};
- // Element masks cannot hide direct ancestor text or ancestor-generated paint.
- // Include html: a body-only boundary leaves document-level paint outside the mask set.
- for(let child=dialog,parent=dialog.parentElement;parent;child=parent,parent=parent.parentElement){
-  if(Array.from(parent.childNodes).some(n=>n.nodeType===Node.TEXT_NODE&&n.textContent.trim())||getComputedStyle(parent).backgroundImage!=='none'||['::before','::after'].some(p=>!['none','normal'].includes(getComputedStyle(parent,p).content)))return {error:'mask_unresolved'};
-  for(const sibling of parent.children)if(sibling!==child)masks.push(sibling);
- }
- for(const selector of [...policy.masks,...policy.noise_regions].map(r=>r.selector)){const matches=all(selector);if(!matches.length||matches.length>100)return {error:'mask_unresolved'};masks.push(...matches);}
- if(masks.some(m=>publicNodes.some(n=>m===n||m.contains(n))))return {error:'mask_unresolved'};
- const value=text.join('\n');if(new TextEncoder().encode(value).length>16384)return {error:'mask_unresolved'};
- if(freeze){
-  // Fixed private slots make connection-status strings/logos unable to change public layout.
-  // Missing status badges receive the same empty masked slot before pixels or observation.
-  for(const button of missingBadgeSlots){const badge=document.createElement('span');badge.style.setProperty('opacity','0','important');button.append(badge);badges.push(badge);masks.push(badge);}
-  for(const badge of badges){badge.style.setProperty('width','100px','important');badge.style.setProperty('flex','0 0 100px','important');badge.style.setProperty('overflow','hidden','important');badge.style.setProperty('height','20px','important');badge.style.setProperty('white-space','nowrap','important');}
-  for(const logo of logos){logo.style.setProperty('width','24px','important');logo.style.setProperty('height','24px','important');logo.style.setProperty('flex','0 0 24px','important');}
-  for(const node of masks)node.style.setProperty('opacity','0','important');
-  window.__opencrawlMasks=masks;window.__opencrawlChanged=false;window.__opencrawlObserver?.disconnect();
-  window.__opencrawlObserver=new MutationObserver(rows=>{if(rows.length)window.__opencrawlChanged=true;});
-  window.__opencrawlObserver.observe(document.documentElement,{attributes:true,childList:true,characterData:true,subtree:true});
- }
- return {text:value,title:screen.label,marker,maskCount:masks.length};
-}