@getopencrawl/cli 0.1.1 → 0.2.0

package/browser.mjs

@@ -1,166 +0,0 @@
-/** Explicitly scoped, read-only browser capture using an isolated OpenCrawl profile. */
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { createInterface } from 'node:readline/promises';
-import { configDir } from './src/client.mjs';
-
-export function validateCaptureUrl(value) {
-  let url;
-  try { url = new URL(value); } catch { throw new Error('Browser capture needs an absolute URL.'); }
-  if (url.username || url.password) throw new Error('Browser URLs cannot contain credentials.');
-  if (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) {
-    throw new Error('Browser capture requires HTTPS, or HTTP on localhost.');
-  }
-  const mutationPath = /(?:^|\/)(?:logout|log-out|signout|sign-out|delete|remove|destroy|unsubscribe|deactivate|terminate)(?:\/|$)/i;
-  if (mutationPath.test(decodeURIComponent(url.pathname)) || [...url.searchParams.keys()].some(key => /^(action|delete|remove|logout|unsubscribe)$/i.test(key))) {
-    throw new Error('This URL appears to perform an action. Select a page that displays records.');
-  }
-  url.hash = '';
-  return url;
-}
-
-export function isReadOnlyRequest(method, postData) {
-  if (['GET', 'HEAD', 'OPTIONS'].includes(method.toUpperCase())) return true;
-  if (method.toUpperCase() !== 'POST' || !postData) return false;
-  try {
-    const payload = JSON.parse(postData);
-    const list = Array.isArray(payload) ? payload : [payload];
-    return list.length > 0 && list.every(item => typeof item.query === 'string' && /^\s*(?:#[^\n]*\n\s*)*(?:query\b|\{)/.test(item.query) && !/\bmutation\b/.test(item.query));
-  } catch { return false; }
-}
-
-async function playwrightModule() {
-  try { return await import('playwright'); }
-  catch { throw new Error('Browser capture needs the optional Playwright package. Install the CLI with optional dependencies, then run: opencrawl browser setup'); }
-}
-async function profile(name, origin) {
-  if (!/^[a-zA-Z0-9_-]{1,60}$/.test(name)) throw new Error('Choose a simple profile name using letters, digits, dashes, or underscores.');
-  const directory = path.join(configDir(), 'browser-profiles', name);
-  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
-  await fs.chmod(directory, 0o700);
-  const metadataFile = path.join(directory, 'profile.json');
-  let metadata;
-  try { metadata = JSON.parse(await fs.readFile(metadataFile, 'utf8')); }
-  catch (error) { if (error.code !== 'ENOENT') throw new Error('The OpenCrawl browser profile is invalid. Choose a new profile name.'); }
-  if (metadata && metadata.origin !== origin) throw new Error(`Profile ${name} is scoped to ${metadata.origin}. Choose a separate --profile for another origin.`);
-  if (!metadata) await fs.writeFile(metadataFile, JSON.stringify({ origin, created_at: new Date().toISOString() }), { mode: 0o600 });
-  return directory;
-}
-
-export async function browserCommand(action, target, options, client, overrides = {}) {
-  if (!['setup', 'login', 'capture', 'crawl'].includes(action)) throw new Error('Use browser setup, login URL, capture URL, or crawl URL.');
-  const playwright = overrides.playwright || await playwrightModule();
-  if (action === 'setup') {
-    // Invoke Playwright's own browser installer without a shell.
-    const { spawn } = await import('node:child_process');
-    const { createRequire } = await import('node:module');
-    const require = createRequire(import.meta.url);
-    const entry = path.join(path.dirname(require.resolve('playwright/package.json')), 'cli.js');
-    await new Promise((resolve, reject) => {
-      const child = spawn(process.execPath, [entry, 'install', 'chromium'], { stdio: 'inherit' });
-      child.once('error', reject); child.once('exit', code => code === 0 ? resolve() : reject(new Error('Browser installation failed.')));
-    });
-    return { installed: 'chromium' };
-  }
-  const selected = validateCaptureUrl(target);
-  const directory = await profile(options.profile || 'default', selected.origin);
-  const maxPages = action === 'crawl' ? Number(options['max-pages'] || 10) : 1;
-  if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 100) throw new Error('--max-pages must be between 1 and 100.');
-  const prefix = options['path-prefix'] || (selected.pathname.endsWith('/') ? selected.pathname : selected.pathname.slice(0, selected.pathname.lastIndexOf('/') + 1));
-  if (!prefix.startsWith('/')) throw new Error('--path-prefix must start with /.');
-  const inScope = value => {
-    try { const url = validateCaptureUrl(value); return url.origin === selected.origin && (action !== 'crawl' || url.pathname.startsWith(prefix)); }
-    catch { return false; }
-  };
-  const queue = [selected.href];
-  if (options.urls) {
-    if (action !== 'capture') throw new Error('--urls can be used with browser capture only.');
-    const content = await fs.readFile(path.resolve(options.urls), 'utf8');
-    let urls;
-    try { urls = JSON.parse(content); } catch { urls = content.split('\n').map(line => line.trim()).filter(Boolean); }
-    if (!Array.isArray(urls) || urls.length > 100 || !urls.every(url => typeof url === 'string' && inScope(url))) {
-      throw new Error('--urls must contain at most 100 explicit page URLs on the selected origin.');
-    }
-    queue.push(...urls);
-  }
-  let context;
-  try {
-    context = await playwright.chromium.launchPersistentContext(path.join(directory, 'chromium'), {
-      headless: action === 'login' || options.interactive ? false : Boolean(options.headless), acceptDownloads: false,
-      permissions: [], viewport: { width: 1440, height: 1000 }, serviceWorkers: 'block',
-    });
-  } catch { throw new Error('Could not launch the isolated OpenCrawl browser. Run opencrawl browser setup; close any other command using this profile.'); }
-  const results = [], warnings = [], seen = new Set();
-  let blocked = 0;
-  try {
-    const page = context.pages()[0] || await context.newPage();
-    if (action === 'login' || options.interactive) {
-      await page.goto(selected.href, { waitUntil: 'domcontentloaded', timeout: 45000 });
-      process.stderr.write(`Sign in manually in the isolated OpenCrawl browser. This profile is reserved for ${selected.origin}.\n`);
-      if (!process.stdin.isTTY && !overrides.waitForLogin) throw new Error('Browser login requires an interactive terminal.');
-      if (overrides.waitForLogin) await overrides.waitForLogin(page);
-      else { const input = createInterface({ input: process.stdin, output: process.stderr }); try { await input.question('After signing in, press Enter here to save the isolated session: '); } finally { input.close(); } }
-      if (action === 'login') return { profile: options.profile || 'default', origin: selected.origin, message: 'Session saved locally. Run browser capture with the page URL you want to index.' };
-    }
-    if (!client.configured) throw new Error('Run opencrawl login before capturing pages.');
-    await context.route('**/*', async route => {
-      const request = route.request();
-      if (!isReadOnlyRequest(request.method(), request.postData())) { blocked++; await route.abort('blockedbyclient'); return; }
-      if (request.isNavigationRequest() && request.frame() === page.mainFrame() && !inScope(request.url())) { blocked++; await route.abort('blockedbyclient'); return; }
-      await route.continue();
-    });
-    // Browser popups are not part of the selected capture scope.
-    context.on('page', popup => { if (popup !== page) popup.close().catch(() => {}); });
-    while (queue.length && seen.size < (action === 'crawl' ? maxPages : 101)) {
-      const url = queue.shift();
-      if (seen.has(url) || !inScope(url)) continue;
-      seen.add(url);
-      try {
-        const response = options.interactive && page.url() === url ? null : await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
-        if (response && response.status() >= 400) throw new Error(`HTTP ${response.status()}`);
-        await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});
-        if (!inScope(page.url())) throw new Error('The page redirected outside the selected capture scope.');
-        const extracted = await page.evaluate(() => {
-          const root = (document.querySelector('main, [role="main"], article') || document.body).cloneNode(true);
-          for (const element of root.querySelectorAll('script,style,noscript,template,input,textarea,select,[contenteditable="true"],[aria-hidden="true"],[hidden]')) element.remove();
-          const blocks = new Set(['P','DIV','SECTION','ARTICLE','H1','H2','H3','H4','LI','TR','BR']);
-          const readable = node => node.nodeType === Node.TEXT_NODE ? node.textContent : [...node.childNodes].map(readable).join('') + (blocks.has(node.nodeName) ? '\n' : node.nodeName === 'TD' ? ' | ' : '');
-          const text = readable(root).replace(/\s+\n/g, '\n').replace(/[ \t]{2,}/g, ' ').trim();
-          return { title: document.title, text, links: [...document.querySelectorAll('a[href]')].map(link => link.href),
-            headings: [...document.querySelectorAll('h1,h2,h3')].map(heading => heading.textContent.trim()).filter(Boolean) };
-        });
-        if (!extracted.text || extracted.text.length < 30) throw new Error('No readable page content. Sign in with browser login, then capture the record page.');
-        const document = { title: (extracted.title || url).slice(0, 500), content: extracted.text.slice(0, 2_000_000),
-          external_id: `browser:${url}`, kind: 'web', url, visibility: 'private', metadata: {
-            capture: 'isolated-browser', origin: selected.origin, captured_at: new Date().toISOString(), headings: extracted.headings,
-            content_complete: extracted.text.length <= 2_000_000, blocked_mutating_requests: blocked,
-          } };
-        const imported = await client.request('/api/v1/documents', { method: 'POST', body: document });
-        let screenshot, ocrJob;
-        if (options.screenshots || options.ocr) {
-          const output = options.screenshots ? path.resolve(options.screenshots) : path.join(directory, 'captures'); await fs.mkdir(output, { recursive: true, mode: 0o700 });
-          screenshot = path.join(output, crypto.createHash('sha256').update(url).digest('hex').slice(0, 16) + '.png');
-          await page.screenshot({ path: screenshot, fullPage: true, mask: [page.locator('input,textarea,[contenteditable="true"]')] });
-          await fs.chmod(screenshot, 0o600);
-          if (options.ocr) {
-            const form = new FormData();
-            form.append('file', new Blob([await fs.readFile(screenshot)], { type: 'image/png' }), path.basename(screenshot));
-            form.append('visibility', 'private');
-            const uploaded = await client.request('/api/v1/uploads', { method: 'POST', body: form });
-            ocrJob = uploaded.job?.id;
-          }
-        }
-        results.push({ url, title: document.title, document_id: imported.id || imported.document?.id, characters: document.content.length, screenshot, ocr_job_id: ocrJob });
-        if (action === 'crawl') {
-          for (const link of extracted.links) {
-            if (queue.length >= 1000) break;
-            if (inScope(link)) { const canonical = validateCaptureUrl(link).href; if (!seen.has(canonical)) queue.push(canonical); }
-          }
-        }
-      } catch (error) { warnings.push({ url, error: String(error.message).split('\n')[0].slice(0, 300) }); }
-    }
-    if (blocked) warnings.push({ message: `${blocked} requests outside read-only capture scope were blocked. Some applications require additional supported read endpoints to render fully.` });
-    return { imported: results.length, pages: results, warnings, origin: selected.origin, profile: options.profile || 'default' };
-  } finally { await context.close(); }
-}

package/src/accounts.test.mjs

@@ -1,223 +0,0 @@
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import http from 'node:http';
-import { spawn } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-import { validateBaseUrl, resolveCredentials, loggedInConfig, loggedOutConfig, profileSummaries, createClient } from './client.mjs';
-
-const identity = (digit = 'a') => ({ user: { id: 'usr_' + digit.repeat(32), name: 'Synthetic user' },
-  workspace: { id: 'ws_' + digit.repeat(32), name: 'Synthetic workspace' } });
-const saved = () => loggedInConfig({}, 'https://a.example', 'oc_synthetic_a', identity());
-
-test('origins canonicalize standard ports and reject path/credential surprises', () => {
-  for (const value of ['https://A.example:443/', 'https://a.example', ' https://a.example/ '])
-    assert.equal(validateBaseUrl(value), 'https://a.example');
-  assert.equal(validateBaseUrl('http://localhost:80/'), 'http://localhost');
-  for (const value of ['https://a.example/api', 'https://a.example/..', 'https://a.example/%2e/',
-    'https://a.example\\anything', 'https://user:pass@a.example', 'https://a.example?token=x',
-    'https://a.example/#fragment', 'file:///tmp', 'http://remote.example'])
-    assert.throws(() => validateBaseUrl(value));
-});
-
-test('saved credentials are origin-bound across arguments, environment, and configured defaults', () => {
-  const config = saved();
-  assert.equal(resolveCredentials(config, { server: 'https://A.example:443/' }, {}).token, 'oc_synthetic_a');
-  for (const [value, overrides, environment] of [
-    [config, { server: 'https://b.example' }, {}], [config, {}, { OPENCRAWL_URL: 'https://b.example' }],
-    [{ ...config, server: 'https://b.example' }, {}, {}], [config, { server: 'https://a.example:444' }, {}]]) {
-    const choice = resolveCredentials(value, overrides, environment);
-    assert.equal(choice.token, null);assert.match(choice.credentialError, /another server/);
-  }
-});
-
-test('legacy unbound keys never migrate by guessing their current destination', () => {
-  const config = { server: 'https://b.example', apiKey: 'oc_old_key', otherSetting: 'keep' };
-  for (const overrides of [{}, { server: 'https://a.example' }]) {
-    const choice = resolveCredentials(config, overrides, {});
-    assert.equal(choice.token, null);assert.match(choice.credentialError, /predates server binding/);
-  }
-  const upgraded = loggedInConfig(config, 'https://b.example', 'oc_explicit_new', identity('b'));
-  assert.equal(upgraded.version, 2);assert.equal(upgraded.otherSetting, 'keep');assert.equal(upgraded.apiKey, undefined);
-  assert.equal(config.apiKey, 'oc_old_key');
-});
-
-test('intentional explicit credentials stay supported without empty-value saved fallback', () => {
-  const config = saved();
-  assert.equal(resolveCredentials(config, { server: 'https://b.example', token: 'oc_explicit_b' }, {}).token, 'oc_explicit_b');
-  assert.equal(resolveCredentials(config, {}, { OPENCRAWL_URL: 'https://b.example', OPENCRAWL_API_KEY: 'oc_explicit_b' }).token, 'oc_explicit_b');
-  for (const token of ['', '  ', 'bad\nvalue']) {
-    assert.throws(() => resolveCredentials(config, { token }, {}));
-    assert.throws(() => resolveCredentials(config, {}, { OPENCRAWL_API_KEY: token }));
-  }
-});
-
-test('explicit account pins win over global defaults and ambient URL/key changes', () => {
-  const a = saved(), account = a.activeProfile;
-  const config = loggedInConfig(a, 'https://b.example', 'oc_synthetic_b', identity('b'));
-  const choice = resolveCredentials(config, { account }, { OPENCRAWL_URL: 'https://c.example', OPENCRAWL_API_KEY: 'oc_ambient' });
-  assert.equal(choice.baseUrl, 'https://a.example');assert.equal(choice.token, 'oc_synthetic_a');
-  assert.throws(() => resolveCredentials(config, { account, server: 'https://b.example' }, {}), /bound to another server/);
-  assert.throws(() => resolveCredentials(config, { account: 'missing' }, {}), /not found/);
-});
-
-test('key rotation preserves profile identity and logout leaves other accounts alone', () => {
-  let config = saved();const first = config.activeProfile;
-  config = loggedInConfig(config, 'https://b.example', 'oc_synthetic_b', identity('b'));const second = config.activeProfile;
-  config = loggedInConfig(config, 'https://a.example:443/', 'oc_rotated_a', identity());
-  assert.equal(config.activeProfile, first);assert.equal(Object.keys(config.profiles).length, 2);
-  const removed = loggedOutConfig(config, first);
-  assert.equal(resolveCredentials(removed, { account: first }, {}).token, null);
-  assert.equal(resolveCredentials(removed, { account: second }, {}).token, 'oc_synthetic_b');
-  assert.equal(JSON.stringify(profileSummaries(config)).includes('oc_rotated_a'), false);
-  assert.equal(JSON.stringify(profileSummaries(config)).includes('apiKey'), false);
-});
-
-test('invalid or tampered identity bindings fail before credential selection', () => {
-  assert.throws(() => loggedInConfig({}, 'https://a.example', 'oc_key', { user: { id: 'bad' }, workspace: {} }), /valid workspace identity/);
-  const config = saved();config.profiles[config.activeProfile].origin = 'https://b.example';
-  assert.throws(() => resolveCredentials(config, {}, {}), /binding is invalid/);
-});
-
-test('exported client rejects route-based host and port changes before fetch', async t => {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-route-'));
-  const previous = process.env.OPENCRAWL_CONFIG_DIR;
-  process.env.OPENCRAWL_CONFIG_DIR = directory;
-  const calls = [];
-  t.mock.method(globalThis, 'fetch', async (url, options) => {
-    calls.push({ url, options });return new Response('{}', { status: 200 });
-  });
-  try {
-    const client = await createClient({ server: 'https://a.example', token: 'oc_synthetic_a' });
-    for (const route of ['@b.example/path', ':444/path', 'https://b.example/path', '//b.example/path',
-      '/\\b.example/path', '\\b.example/path', ' /api/v1/usage', '/\n/b.example/path', null])
-      await assert.rejects(client.request(route), /routes must be absolute paths/);
-    assert.equal(calls.length, 0);
-    await client.request('/api/v1/usage?query=synthetic%20value');
-    assert.equal(calls.length, 1);
-    assert.equal(calls[0].url, 'https://a.example/api/v1/usage?query=synthetic%20value');
-    assert.equal(calls[0].options.headers.Authorization, 'Bearer oc_synthetic_a');
-    assert.equal(calls[0].options.redirect, 'error');
-  } finally {
-    if (previous === undefined) delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR = previous;
-    await fs.rm(directory, { recursive: true, force: true });
-  }
-});
-
-const script = fileURLToPath(new URL('../bin/opencrawl.mjs', import.meta.url));
-function run(args, env, cwd, input = '') {
-  return new Promise((resolve, reject) => {
-    const child = spawn(process.execPath, [script, ...args], { env, cwd, stdio: ['pipe', 'pipe', 'pipe'] });
-    let stdout = '', stderr = '';const timer = setTimeout(() => child.kill('SIGKILL'), 10000);
-    child.stdout.on('data', value => { stdout += value });child.stderr.on('data', value => { stderr += value });
-    child.on('error', reject);child.on('close', code => { clearTimeout(timer);resolve({ code, stdout, stderr }) });child.stdin.end(input);
-  });
-}
-async function fixtureServer(digit) {
-  const calls = [], key = 'oc_synthetic_' + digit;
-  const server = http.createServer(async (request, response) => {
-    let raw = '';for await (const part of request) raw += part;
-    calls.push({ path: request.url, authorization: request.headers.authorization });
-    let result = { status: 'ok' };
-    if (request.url !== '/healthz' && request.headers.authorization !== 'Bearer ' + key) {
-      response.writeHead(401, { 'Content-Type': 'application/json' });response.end('{"detail":"Synthetic key rejected"}');return;
-    }
-    if (request.url === '/api/auth/me') result = identity(digit);
-    else if (request.url === '/mcp') { const message = JSON.parse(raw);result = { jsonrpc: '2.0', id: message.id, result: { tools: [], workspace: digit } }; }
-    else if (request.url === '/api/v1/context') result = { context: '[S1] Synthetic workspace ' + digit };
-    else result = { workspace: digit };
-    response.writeHead(200, { 'Content-Type': 'application/json' });response.end(JSON.stringify(result));
-  });
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
-  return { calls, key, url: 'http://127.0.0.1:' + server.address().port,
-    close: () => new Promise(resolve => server.close(resolve)) };
-}
-
-test('real two-origin commands never send a saved key to a changed destination', async () => {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-accounts-'));
-  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
-  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');
-  const a = await fixtureServer('a'), b = await fixtureServer('b');
-  const configPath = path.join(env.OPENCRAWL_CONFIG_DIR, 'config.json');
-  try {
-    let result = await run(['login', '--server', a.url], env, directory, a.key + '\n');
-    assert.equal(result.code, 0, result.stderr);assert.equal(result.stdout.includes(a.key), false);
-    const account = JSON.parse(await fs.readFile(configPath)).activeProfile;
-    assert.equal((await fs.stat(configPath)).mode & 0o777, 0o600);
-    for (const [args, extra] of [[['status', '--server', b.url], {}], [['status'], { OPENCRAWL_URL: b.url }]]) {
-      result = await run(args, { ...env, ...extra }, directory);
-      assert.equal(result.code, 1);assert.equal(b.calls.length, 0);
-    }
-    assert.equal((await run(['configure', '--server', b.url], env, directory)).code, 0);
-    assert.equal((await run(['status'], env, directory)).code, 1);assert.equal(b.calls.length, 0);
-    const doctor = await run(['doctor'], env, directory);
-    assert.equal(doctor.code, 1);assert.equal(b.calls.at(-1).path, '/healthz');assert.equal(b.calls.at(-1).authorization, undefined);
-    result = await run(['status'], { ...env, OPENCRAWL_API_KEY: b.key }, directory);
-    assert.equal(result.code, 0);assert.equal(b.calls.at(-1).authorization, 'Bearer ' + b.key);
-    result = await run(['login', '--account', account], { ...env, OPENCRAWL_URL: b.url }, directory, a.key + '\n');
-    assert.equal(result.code, 0, result.stderr);assert.equal(a.calls.at(-1).path, '/api/auth/me');
-    assert.equal(b.calls.some(call => call.authorization === 'Bearer ' + a.key), false);
-    const before = await fs.readFile(configPath, 'utf8');
-    result = await run(['login', '--account', account, '--server', b.url], env, directory, a.key + '\n');
-    assert.equal(result.code, 1);assert.equal(await fs.readFile(configPath, 'utf8'), before);
-  } finally { await Promise.all([a.close(), b.close()]);await fs.rm(directory, { recursive: true, force: true }); }
-});
-
-test('installed MCP and hook stay pinned after another login and ambient configuration changes', async () => {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-pins-'));
-  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
-  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');
-  const a = await fixtureServer('a'), b = await fixtureServer('b');
-  try {
-    assert.equal((await run(['login', '--server', a.url], env, directory, a.key + '\n')).code, 0);
-    for (const target of ['claude-code', 'cursor', 'codex']) assert.equal((await run(['install', target], env, directory)).code, 0);
-    const mcp = JSON.parse(await fs.readFile(path.join(directory, '.mcp.json'), 'utf8')).mcpServers.opencrawl;
-    const settings = await fs.readFile(path.join(directory, '.claude/settings.local.json'), 'utf8');
-    const codex = await fs.readFile(path.join(directory, '.codex/config.toml'), 'utf8');
-    assert.ok(mcp.args.includes('--account') && mcp.args.includes(a.url) && mcp.args.includes('--config-dir'));
-    for (const value of [JSON.stringify(mcp), settings, codex]) assert.ok(!value.includes(a.key) && value.includes('--account'));
-    assert.equal((await run(['login', '--server', b.url], env, directory, b.key + '\n')).code, 0);
-    const before = b.calls.length;
-    const legacyBridge = await run(['mcp'], env, directory, '{"jsonrpc":"2.0","id":1,"method":"tools/list"}\n');
-    assert.equal(legacyBridge.code, 1);assert.match(legacyBridge.stderr, /pinned account/);assert.equal(b.calls.length, before);
-    assert.equal((await run(['hook'], env, directory, '{"prompt":"Synthetic question"}')).stdout, '');
-    const changedEnv = { ...env, OPENCRAWL_URL: b.url, OPENCRAWL_API_KEY: b.key, OPENCRAWL_CONFIG_DIR: path.join(directory, 'unrelated') };
-    const argumentsAfterScript = mcp.args.slice(1);
-    const response = await run(argumentsAfterScript, changedEnv, directory, '{"jsonrpc":"2.0","id":7,"method":"tools/list"}\n');
-    assert.equal(response.code, 0, response.stderr);assert.equal(JSON.parse(response.stdout).result.workspace, 'a');
-    const hooked = await run(['hook', ...argumentsAfterScript.slice(1)], changedEnv, directory, '{"prompt":"Synthetic question"}');
-    assert.equal(hooked.code, 0);assert.match(JSON.parse(hooked.stdout).hookSpecificOutput.additionalContext, /workspace a/);
-    assert.equal(b.calls.some(call => call.path === '/mcp' || call.path === '/api/v1/context'), false);
-    const account = argumentsAfterScript[argumentsAfterScript.indexOf('--account') + 1];
-    assert.equal((await run(['logout', '--account', account], env, directory)).code, 0);
-    const disconnected = await run(argumentsAfterScript, changedEnv, directory, '{"jsonrpc":"2.0","id":8,"method":"tools/list"}\n');
-    assert.ok(JSON.parse(disconnected.stdout).error);assert.equal(b.calls.some(call => call.path === '/mcp'), false);
-  } finally { await Promise.all([a.close(), b.close()]);await fs.rm(directory, { recursive: true, force: true }); }
-});
-
-test('legacy migration requires explicit login and failed login leaves the old file untouched', async () => {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-legacy-'));
-  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
-  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');const server = await fixtureServer('a');
-  try {
-    await fs.mkdir(env.OPENCRAWL_CONFIG_DIR);
-    const file = path.join(env.OPENCRAWL_CONFIG_DIR, 'config.json');
-    const legacy = JSON.stringify({ server: server.url, apiKey: 'oc_unknown_origin', unrelated: 'preserved' });
-    await fs.writeFile(file, legacy, { mode: 0o600 });
-    const denied = await run(['status'], env, directory);
-    assert.equal(denied.code, 1);assert.match(denied.stderr, /predates server binding/);assert.equal(server.calls.length, 0);
-    assert.equal((await run(['install', 'claude-code'], env, directory)).code, 1);
-    await assert.rejects(fs.stat(path.join(directory, '.mcp.json')), { code: 'ENOENT' });
-    const failed = await run(['login', '--server', server.url], env, directory, 'oc_explicit_invalid\n');
-    assert.equal(failed.code, 1);assert.equal(await fs.readFile(file, 'utf8'), legacy);
-    assert.equal((await run(['login', '--server', server.url], env, directory, server.key + '\n')).code, 0);
-    const upgraded = JSON.parse(await fs.readFile(file));
-    assert.equal(upgraded.version, 2);assert.equal(upgraded.apiKey, undefined);assert.equal(upgraded.unrelated, 'preserved');
-    assert.equal((await run(['status'], env, directory)).code, 0);
-    const inventory = await run(['profiles'], env, directory);
-    assert.equal(JSON.parse(inventory.stdout).accounts.length, 1);assert.equal(inventory.stdout.includes(server.key), false);
-    assert.equal(server.calls.some(call => call.authorization === 'Bearer oc_unknown_origin'), false);
-  } finally { await server.close();await fs.rm(directory, { recursive: true, force: true }); }
-});

package/src/agent.mjs

@@ -1,95 +0,0 @@
-// Talk to the OpenCrawl agent from a terminal or a coding agent. Output is JSON for agents.
-import {settle, transient} from './jobs.mjs';
-
-export const AGENT_HELP = `Usage:
-  opencrawl agent signup --organization NAME --email EMAIL [--create-new]
-  opencrawl agent join POLL_URL                        after signup asks for approval: wait for an admin to approve, then save the key
-  opencrawl agent "what you want to build"            start a session
-  opencrawl agent "answer" --session SESSION_ID        continue it
-  opencrawl agent --session SESSION_ID                 read the session (customer discovery progress, spec, project)
-  opencrawl agent --session SESSION_ID --build         build the project, or apply changes described on its session
-  opencrawl agent --session SESSION_ID --new-key       a new project key (the previous one stops working), if a key was lost
-  opencrawl agent --session SESSION_ID --check         verify app authorizations
-  opencrawl agent --session SESSION_ID --connect APP   add gmail, googledrive, notion, hubspot or granola to a built project
-  opencrawl agent projects                             list built projects
-  opencrawl agent memory                               what the OpenCrawl agent remembers for this organization
-  opencrawl agent remember "note"                      add to that memory (or say "remember ..." in any message)
-Use an organization admin key (OPENCRAWL_API_KEY or opencrawl login). Signup needs no key.
-Signup and build save the keys they return as CLI accounts and owner-only key files, and print where
-instead of the key: use --account ACCOUNT on any command, or read key_file for MCP or your server.
-Signup makes the organization account the default; a project account is used with --account.
-Use the project account to read a project's watches, jobs and research. --print-keys prints keys instead.`;
-
-const SESSION = /^ags_[a-f0-9]{32}$/;
-
-export function agentRequest(rest, options) {
-  const [first, ...more] = rest;
-  if (first === 'signup') {
-    if (!options.organization || !options.email) throw new Error('Use agent signup --organization NAME --email EMAIL.');
-    return {route: '/api/v1/agent/signup', method: 'POST', authenticated: false,
-      body: {organization: String(options.organization), email: String(options.email), agent: 'OpenCrawl CLI', ...(options['create-new'] ? {create_new: true} : {})}};
-  }
-  if (first === 'join') {
-    const token = /^https?:\/\/[^/]+\/api\/v1\/agent\/signup\/([A-Za-z0-9_-]{20,64})$/.exec(String(more[0] ?? ''))?.[1];
-    if (!token || more.length > 1) throw new Error('Use agent join POLL_URL, with the poll_url agent signup returned.');
-    return {route: `/api/v1/agent/signup/${token}`, method: 'GET', authenticated: false, poll: true};
-  }
-  if (first === 'projects' && !more.length) return {route: '/api/v1/agent/projects', method: 'GET'};
-  if (first === 'memory' && !more.length) return {route: '/api/v1/agent/memory', method: 'GET'};
-  if (first === 'remember') {
-    if (!more.join(' ').trim()) throw new Error('Use agent remember "note".');
-    return {route: '/api/v1/agent/memory', method: 'POST', body: {note: more.join(' ').trim()}};
-  }
-  const message = rest.join(' ').trim();
-  const session = options.session;
-  if (session !== undefined && !SESSION.test(String(session))) throw new Error('--session must be a session ID returned by opencrawl agent.');
-  if ([options.build, options.check, options.connect, options['new-key']].filter(Boolean).length > 1) throw new Error('Choose one of --build, --check, --connect or --new-key.');
-  const action = options.build ? 'build' : options.check ? 'check_connections' : options.connect ? 'connect' : options['new-key'] ? 'issue_key' : undefined;
-  if (action && !session) throw new Error(`--${options.build ? 'build' : options.check ? 'check' : options.connect ? 'connect' : 'new-key'} needs --session.`);
-  if (options.connect === true) throw new Error('--connect needs an app name, such as granola.');
-  if (!action && !message && session) return {route: `/api/v1/agent/sessions/${session}`, method: 'GET'};
-  if (!action && !message) throw new Error('Say what you want to build, or use --session with --build or --check.');
-  const body = {...(message ? {message} : {}), ...(action ? {action} : {}), ...(action === 'connect' ? {toolkit: String(options.connect)} : {})};
-  return session ? {route: `/api/v1/agent/sessions/${session}/messages`, method: 'POST', body}
-    : {route: '/api/v1/agent/sessions', method: 'POST', body};
-}
-
-// Keys are shown once. Keep them out of terminal output (and an agent's chat) unless asked: save them instead.
-export async function keepKeys(result, saveKey) {
-  if (!result || typeof result !== 'object') return result;
-  let next = result;
-  if (typeof result.api_key === 'string') {
-    const {api_key: key, ...rest} = result;
-    next = {...rest, api_key_saved: {...await saveKey(key, {activate: true}), kind: 'organization'}};
-  }
-  if (typeof next.project?.api_key === 'string') {
-    const {api_key: key, ...project} = next.project;
-    next = {...next, project: {...project, api_key_saved: {...await saveKey(key, {activate: false}), kind: 'project'}}};
-  }
-  return next;
-}
-
-export async function agentCommand(rest, options, client, out = value => process.stdout.write(JSON.stringify(value, null, 2) + '\n'), saveKey,
-                                   {now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms))} = {}) {
-  const request = agentRequest(rest, options);
-  // A join request waits up to the hour an approval link stays valid.
-  const pollUntil = now() + 3600000;
-  let deadline = now() + 120000, result, failures = 0;
-  while (true) {
-    try {
-      result = await client.request(request.route, {method: request.method, body: request.body, authenticated: request.authenticated !== false, timeout: 120000});
-      if (!request.poll || result?.status !== 'pending' || now() >= pollUntil) break;
-      await sleep(10000); deadline = now() + 120000; failures = 0; continue;
-    }
-    catch (error) {
-      // Reads are safe to repeat while the server restarts. A write's outcome is unknown, so say how to find out.
-      if (request.method === 'GET' && transient(error) && now() < deadline) { await settle(++failures, {now, sleep, deadline}); continue; }
-      if (transient(error) && request.method === 'POST' && options.session)
-        error.message += ` The outcome is unknown: read the session (opencrawl agent --session ${options.session}) before sending again.` +
-          (request.body?.action === 'build' ? ' If it shows the project built but you did not receive its key, use --new-key.' : '');
-      throw error;
-    }
-  }
-  out(saveKey && !options['print-keys'] ? await keepKeys(result, saveKey) : result);
-  return 0;
-}

package/src/agent.test.mjs

@@ -1,58 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import {agentRequest, agentCommand} from './agent.mjs';
-
-const session = 'ags_' + 'a'.repeat(32);
-test('agent requests map messages, actions and signup to the OpenCrawl agent API', () => {
-  assert.deepEqual(agentRequest(['watch', 'rival', 'changelogs'], {}), {route: '/api/v1/agent/sessions', method: 'POST', body: {message: 'watch rival changelogs'}});
-  assert.deepEqual(agentRequest(['hourly'], {session}), {route: `/api/v1/agent/sessions/${session}/messages`, method: 'POST', body: {message: 'hourly'}});
-  assert.deepEqual(agentRequest([], {session, build: true}).body, {action: 'build'});
-  assert.deepEqual(agentRequest([], {session, check: true}).body, {action: 'check_connections'});
-  assert.deepEqual(agentRequest([], {session, connect: 'granola'}).body, {action: 'connect', toolkit: 'granola'});
-  assert.deepEqual(agentRequest(['projects'], {}), {route: '/api/v1/agent/projects', method: 'GET'});
-  assert.deepEqual(agentRequest([], {session}), {route: `/api/v1/agent/sessions/${session}`, method: 'GET'});
-  assert.deepEqual(agentRequest(['memory'], {}), {route: '/api/v1/agent/memory', method: 'GET'});
-  assert.deepEqual(agentRequest(['remember', 'Lantern', 'is', 'lantern.fm'], {}), {route: '/api/v1/agent/memory', method: 'POST', body: {note: 'Lantern is lantern.fm'}});
-  const signup = agentRequest(['signup'], {organization: 'Acme', email: 'a@acme.example', 'create-new': true});
-  assert.equal(signup.authenticated, false); assert.deepEqual(signup.body, {organization: 'Acme', email: 'a@acme.example', agent: 'OpenCrawl CLI', create_new: true});
-  for (const [rest, options] of [[[], {}], [[], {build: true}], [['x'], {session: 'bad'}], [['signup'], {email: 'a@b.c'}], [[], {session, build: true, check: true}], [[], {session, connect: true}], [[], {connect: 'gmail'}], [['remember'], {}]]) assert.throws(() => agentRequest(rest, options));
-});
-test('agent command prints the server response as JSON', async () => {
-  const calls = [], printed = [];
-  await agentCommand(['hello'], {}, {request: async (route, init) => { calls.push({route, init}); return {session_id: session, reply: 'Hi'}; }}, v => printed.push(v));
-  assert.equal(calls[0].init.timeout, 120000); assert.equal(calls[0].init.authenticated, true); assert.deepEqual(printed, [{session_id: session, reply: 'Hi'}]);
-});
-test('research feed and subscription commands map to the project signal routes', async () => {
-  const {researchRequest} = await import('./research.mjs');
-  assert.deepEqual(researchRequest(['feed'], {customer: 'Basedash', after: '7'}), {command: 'feed', route: '/api/research/feed?after=7&limit=50&customer=Basedash'});
-  assert.deepEqual(researchRequest(['subscribe', 'https://hooks.example/oc'], {customer: 'Basedash'}).body, {endpoint: 'https://hooks.example/oc', customer: 'Basedash'});
-  assert.equal(researchRequest(['unsubscribe', 'rsub_' + 'a'.repeat(32)], {}).method, 'DELETE');
-  for (const [rest, options] of [[['subscribe', 'http://insecure.example'], {}], [['unsubscribe', 'bad'], {}], [['feed', 'extra'], {}]]) assert.throws(() => researchRequest(rest, options));
-});
-test('session reads ride out a server restart; a lost build says how to recover the key', async () => {
-  assert.deepEqual(agentRequest([], {session, 'new-key': true}).body, {action: 'issue_key'});
-  let time = 0, calls = 0;
-  const clock = {now: () => time, sleep: async ms => { time += ms; }};
-  const printed = [];
-  await agentCommand([], {session}, {request: async () => { if (++calls < 3) { const e = Error('OpenCrawl 502'); e.status = 502; throw e; } return {state: 'ready'}; }}, v => printed.push(v), undefined, clock);
-  assert.equal(calls, 3); assert.deepEqual(printed, [{state: 'ready'}]);
-  calls = 0;
-  await assert.rejects(agentCommand([], {session, build: true}, {request: async () => { calls++; const e = Error('OpenCrawl 502.'); e.status = 502; throw e; }}, () => {}, undefined, clock),
-    error => error.message.includes('--new-key') && error.message.includes(`--session ${session}`));
-  assert.equal(calls, 1);  // a build is never sent twice on its own
-  await assert.rejects(agentCommand([], {session}, {request: async () => { const e = Error('OpenCrawl 404'); e.status = 404; throw e; }}, () => {}, undefined, clock));
-});
-test('join waits for an admin to approve, then saves the organization key', async () => {
-  const poll = 'https://app.getopencrawl.com/api/v1/agent/signup/' + 'T'.repeat(43);
-  assert.deepEqual(agentRequest(['join', poll], {}), {route: '/api/v1/agent/signup/' + 'T'.repeat(43), method: 'GET', authenticated: false, poll: true});
-  for (const rest of [['join'], ['join', 'https://app.getopencrawl.com/api/v1/agent/sessions'], ['join', poll, 'extra']]) assert.throws(() => agentRequest(rest, {}));
-  let time = 0, calls = 0;
-  const clock = {now: () => time, sleep: async ms => { time += ms; }}, printed = [], saved = [];
-  await agentCommand(['join', poll], {}, {request: async (route, init) => { assert.equal(init.authenticated, false); return ++calls < 3 ? {status: 'pending'} : {status: 'approved', organization: {name: 'Acme'}, api_key: 'oc_secret'}; }},
-    v => printed.push(v), async (key, options) => { saved.push([key, options]); return {account: 'acc_1'}; }, clock);
-  assert.equal(calls, 3); assert.equal(time, 20000); assert.deepEqual(saved, [['oc_secret', {activate: true}]]);
-  assert.deepEqual(printed, [{status: 'approved', organization: {name: 'Acme'}, api_key_saved: {account: 'acc_1', kind: 'organization'}}]);
-  time = 0; printed.length = 0;
-  await agentCommand(['join', poll], {}, {request: async () => ({status: 'pending'})}, v => printed.push(v), undefined, clock);
-  assert.ok(time >= 3600000); assert.deepEqual(printed, [{status: 'pending'}]);
-});