@getopencrawl/cli 0.1.0 → 0.1.1

package/README.md

@@ -4,6 +4,24 @@ Node.js 20 or later. Core CLI commands have no required package dependencies; br
 
 The canonical deployment is live at `https://app.getopencrawl.com`; [STATUS.md](../STATUS.md) records its verification. Copy the exact immutable `npm install -g` command from **Connect an agent**, or use `packages.cli.url` from the [current manifest](https://app.getopencrawl.com/downloads/manifest.json) and use `opencrawl login --server https://app.getopencrawl.com`. Existing installations keep their original origin and account pins. Follow the [domain migration steps](../docs/cli.md#moving-to-the-canonical-domain), including flushing old capture and starting a new native chat after reinstalling.
 
+## Start here: new to OpenCrawl
+
+No account, card or browser is needed. These commands create a free organization, describe the data you need to the OpenCrawl agent, and build a project that crawls and refreshes in the background. The same steps over HTTP or MCP are in [the agent guide](https://app.getopencrawl.com/start.md).
+
+```sh
+npm install -g "$(curl -fsS https://app.getopencrawl.com/downloads/manifest.json | node -pe 'JSON.parse(require("fs").readFileSync(0)).packages.cli.url')"
+opencrawl agent signup --organization "Acme" --email you@acme.com --server https://app.getopencrawl.com
+opencrawl agent "Watch the planning commission agendas at https://example.gov/agendas daily, including linked PDFs, for data-center rezonings"
+opencrawl agent "answers to its questions" --session SESSION_ID
+opencrawl agent --session SESSION_ID --build
+```
+
+- `signup` saves the organization key as the default CLI account and in an owner-only key file, and prints where, never the key. `--build` does the same for the project key.
+- The OpenCrawl agent checks every page with OpenCrawl's crawler before a plan is ready. A page it cannot read (blocked by robots.txt or by the site) is listed in `questions`; replace it or drop it.
+- Read a project's watches, jobs and research with its project account: `opencrawl --account PROJECT_ACCOUNT research list`, `research job FIRST_CHECK_JOB --wait`, `research search "question" --collection WATCH_ID`, `search "question"`. The organization key cannot read inside projects (404).
+- Research questions run after their watches' first check, then daily. To change a built project, send the change on the same session, then `--build` again: new watches and research are added and existing ones keep running.
+- For MCP or server code, use the project key file: `OPENCRAWL_API_KEY="$(cat KEY_FILE)"`. Never paste keys into chat or commit them.
+
 From this repository:
 
 ```sh

package/bin/opencrawl.mjs

@@ -34,9 +34,15 @@ Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
 `;
 const HELP = `OpenCrawl · knowledge that follows your work
 
+New here? Start free, no card or browser needed:
+  opencrawl agent signup --organization NAME --email EMAIL   Create a free organization; its key is saved
+  opencrawl agent "what data you need"   The OpenCrawl agent plans it; then --session ID --build
+  Guide: https://app.getopencrawl.com/start.md
+
   opencrawl login [--server URL]         Save a workspace API key securely
   opencrawl configure --server URL       Change the server
   opencrawl profiles                    List saved workspace account profiles
+  opencrawl agent --help                 Ask the OpenCrawl agent to build a data project
   opencrawl search "question" [--json]    Search connected knowledge
   opencrawl context "task" [--budget N]  Get a compact, cited context pack
   opencrawl research --help              Collect and search public-source research
@@ -63,7 +69,7 @@ const HELP = `OpenCrawl · knowledge that follows your work
   opencrawl hook                        Run Claude Code's UserPromptSubmit hook
   opencrawl logout                       Remove the locally saved API key
 
-Options: --server URL  --account PROFILE_ID  --json  --limit N  --source ID  --once (watch)
+Options: --server URL  --account PROFILE_ID  --json  --limit N  --source ID  --customer NAME (search/context: one customer's index)  --once (watch)
 Providers: auto, chatgpt, claude, claude-code, codex, generic
 Credentials: OPENCRAWL_API_KEY, OPENCRAWL_URL. Node.js 20+.
 Hosted ChatGPT/Claude history is imported from an export; it is not silently read.
@@ -76,8 +82,8 @@ function parseArgs(args) {
     if (!arg.startsWith('--')) { positional.push(arg); continue; }
     const [key, inline] = arg.slice(2).split(/=(.*)/s);
     if (key !== 'selection' && Object.hasOwn(options,key)) duplicates.push(key);
-    if (['json','help','version','wait','once','acknowledge-unknown','capture'].includes(key) && inline !== undefined) valuedFlags.push(key);
-    if (['json', 'once', 'help', 'version', 'headless', 'interactive', 'ocr', 'wait','acknowledge-unknown','capture'].includes(key)) options[key] = true;
+    if (['json','help','version','wait','once','acknowledge-unknown','capture','content-only','build','check','new-key','create-new','print-keys'].includes(key) && inline !== undefined) valuedFlags.push(key);
+    if (['json', 'once', 'help', 'version', 'headless', 'interactive', 'ocr', 'wait','acknowledge-unknown','capture','content-only','build','check','new-key','create-new','print-keys'].includes(key)) options[key] = true;
     else {
       const value = inline ?? args[++index];
       if (value === undefined || value.startsWith('--')) throw new Error(`Missing value for --${key}`);
@@ -426,7 +432,8 @@ async function main() {
   const [command, ...rest] = positional;
   if (options['config-dir']) process.env.OPENCRAWL_CONFIG_DIR = path.resolve(options['config-dir']);
   if (options.token !== undefined || options.key !== undefined) throw new Error('Do not pass credentials in command-line arguments. Use the hidden login prompt or OPENCRAWL_API_KEY.');
-  if (options.version) return print('0.1.0');
+  if (options.version) return print('0.1.1');
+  if (command === 'agent' && (options.help || (!rest.length && !options.session))) { const { AGENT_HELP } = await import('../src/agent.mjs'); return print(AGENT_HELP); }
   if (command === 'sessions' && (options.help || !rest.length)) return print(SESSIONS_HELP);
   if (command === 'research' && (options.help || !rest.length)) return print(RESEARCH_HELP);
   if (command === 'tasks' && (options.help || !rest.length)) return print(TASKS_HELP);
@@ -447,7 +454,7 @@ async function main() {
     // Resolve an explicit account before reading/sending the replacement key.
     const { baseUrl: server } = resolveCredentials(config, { account: options.account, server: options.server }, { ...process.env, OPENCRAWL_API_KEY: undefined });
     const apiKey = process.env.OPENCRAWL_API_KEY ?? await askSecret('Workspace API key (hidden): ');
-    if (!apiKey) throw new Error('Create an API key in OpenCrawl → Agents & API, then paste it here.');
+    if (!apiKey) throw new Error('No API key was given. New to OpenCrawl? Create a free organization and key with: opencrawl agent signup --organization NAME --email EMAIL. Existing account: create a key under Connect an agent → API keys and developer tools, then pipe it into opencrawl login or set OPENCRAWL_API_KEY.');
     const client = await createClient({ server, token: apiKey });
     const identity = await client.request('/api/auth/me');
     const next = loggedInConfig(config, server, apiKey, identity);
@@ -462,6 +469,24 @@ async function main() {
     throw new Error('This agent connection needs a pinned account. Rerun opencrawl install for this project, or use --account PROFILE_ID from opencrawl profiles.');
   if(command==='browser'&&rest[0]==='monitor'){process.exitCode=await productMonitorsCommand(rest.slice(1),options,client);return;}
   if (command === 'browser') { const { browserCommand } = await import('../browser.mjs'); return print(await browserCommand(rest[0], rest[1], options, client)); }
+  if (command === 'agent') {
+    const { agentCommand } = await import('../src/agent.mjs');
+    // Save each returned key as a verified account plus an owner-only key file; the default account changes only at signup.
+    const saveKey = async (apiKey, { activate }) => {
+      const identity = await (await createClient({ server: client.baseUrl, token: apiKey })).request('/api/auth/me');
+      const config = await readConfig();
+      const next = loggedInConfig(config, client.baseUrl, apiKey, identity);
+      const account = next.activeProfile;
+      if (!activate && config.activeProfile) Object.assign(next, { activeProfile: config.activeProfile, server: config.server });
+      await writeConfig(next);
+      const dir = path.join(configDir(), 'keys');
+      await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+      const keyFile = path.join(dir, `${account}.key`);
+      await fs.writeFile(keyFile, apiKey + '\n', { mode: 0o600 }); await fs.chmod(keyFile, 0o600);
+      return { account, key_file: keyFile, use: `opencrawl --account ${account} <command>, or OPENCRAWL_API_KEY="$(cat ${keyFile})" for MCP and server code` };
+    };
+    process.exitCode = await agentCommand(rest, options, client, undefined, saveKey); return;
+  }
   if (command === 'sessions') return sessionsCommand(rest,options,client);
   if (command === 'research') { process.exitCode = await researchCommand(rest, options, client); return; }
   if (command === 'tasks') { process.exitCode = await tasksCommand(rest, options, client); return; }
@@ -476,7 +501,7 @@ async function main() {
   if (command === 'search' || command === 'context') {
     const query = rest.join(' ');
     if (!query.trim()) throw new Error('Enter a query.');
-    const body = { query, limit: integer(options.limit, 8, 1, 50), budget: integer(options.budget, 2000, 256), ...(options.source ? { source_ids: [options.source] } : {}) };
+    const body = { query, limit: integer(options.limit, 8, 1, 50), budget: integer(options.budget, 2000, 256), ...(options.source ? { source_ids: [options.source] } : {}), ...(options.customer ? { customer: String(options.customer) } : {}) };
     const data = await client.request(`/api/v1/${command}`, { method: 'POST', body });
     if (options.json) return print(data);
     for (const warning of coverageWarnings(data)) process.stderr.write(`OpenCrawl coverage: ${displayLine(warning)}\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getopencrawl/cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Search, import and bring OpenCrawl context to your agents",
   "type": "module",
   "bin": {
@@ -10,7 +10,6 @@
   "files": [
     "bin",
     "src",
-    "!src/**/*.test.mjs",
     "README.md",
     "browser.mjs"
   ],
@@ -24,8 +23,5 @@
   "license": "UNLICENSED",
   "optionalDependencies": {
     "playwright": "^1.58.0"
-  },
-  "publishConfig": {
-    "access": "public"
   }
 }

package/src/accounts.test.mjs

@@ -0,0 +1,223 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { validateBaseUrl, resolveCredentials, loggedInConfig, loggedOutConfig, profileSummaries, createClient } from './client.mjs';
+
+const identity = (digit = 'a') => ({ user: { id: 'usr_' + digit.repeat(32), name: 'Synthetic user' },
+  workspace: { id: 'ws_' + digit.repeat(32), name: 'Synthetic workspace' } });
+const saved = () => loggedInConfig({}, 'https://a.example', 'oc_synthetic_a', identity());
+
+test('origins canonicalize standard ports and reject path/credential surprises', () => {
+  for (const value of ['https://A.example:443/', 'https://a.example', ' https://a.example/ '])
+    assert.equal(validateBaseUrl(value), 'https://a.example');
+  assert.equal(validateBaseUrl('http://localhost:80/'), 'http://localhost');
+  for (const value of ['https://a.example/api', 'https://a.example/..', 'https://a.example/%2e/',
+    'https://a.example\\anything', 'https://user:pass@a.example', 'https://a.example?token=x',
+    'https://a.example/#fragment', 'file:///tmp', 'http://remote.example'])
+    assert.throws(() => validateBaseUrl(value));
+});
+
+test('saved credentials are origin-bound across arguments, environment, and configured defaults', () => {
+  const config = saved();
+  assert.equal(resolveCredentials(config, { server: 'https://A.example:443/' }, {}).token, 'oc_synthetic_a');
+  for (const [value, overrides, environment] of [
+    [config, { server: 'https://b.example' }, {}], [config, {}, { OPENCRAWL_URL: 'https://b.example' }],
+    [{ ...config, server: 'https://b.example' }, {}, {}], [config, { server: 'https://a.example:444' }, {}]]) {
+    const choice = resolveCredentials(value, overrides, environment);
+    assert.equal(choice.token, null);assert.match(choice.credentialError, /another server/);
+  }
+});
+
+test('legacy unbound keys never migrate by guessing their current destination', () => {
+  const config = { server: 'https://b.example', apiKey: 'oc_old_key', otherSetting: 'keep' };
+  for (const overrides of [{}, { server: 'https://a.example' }]) {
+    const choice = resolveCredentials(config, overrides, {});
+    assert.equal(choice.token, null);assert.match(choice.credentialError, /predates server binding/);
+  }
+  const upgraded = loggedInConfig(config, 'https://b.example', 'oc_explicit_new', identity('b'));
+  assert.equal(upgraded.version, 2);assert.equal(upgraded.otherSetting, 'keep');assert.equal(upgraded.apiKey, undefined);
+  assert.equal(config.apiKey, 'oc_old_key');
+});
+
+test('intentional explicit credentials stay supported without empty-value saved fallback', () => {
+  const config = saved();
+  assert.equal(resolveCredentials(config, { server: 'https://b.example', token: 'oc_explicit_b' }, {}).token, 'oc_explicit_b');
+  assert.equal(resolveCredentials(config, {}, { OPENCRAWL_URL: 'https://b.example', OPENCRAWL_API_KEY: 'oc_explicit_b' }).token, 'oc_explicit_b');
+  for (const token of ['', '  ', 'bad\nvalue']) {
+    assert.throws(() => resolveCredentials(config, { token }, {}));
+    assert.throws(() => resolveCredentials(config, {}, { OPENCRAWL_API_KEY: token }));
+  }
+});
+
+test('explicit account pins win over global defaults and ambient URL/key changes', () => {
+  const a = saved(), account = a.activeProfile;
+  const config = loggedInConfig(a, 'https://b.example', 'oc_synthetic_b', identity('b'));
+  const choice = resolveCredentials(config, { account }, { OPENCRAWL_URL: 'https://c.example', OPENCRAWL_API_KEY: 'oc_ambient' });
+  assert.equal(choice.baseUrl, 'https://a.example');assert.equal(choice.token, 'oc_synthetic_a');
+  assert.throws(() => resolveCredentials(config, { account, server: 'https://b.example' }, {}), /bound to another server/);
+  assert.throws(() => resolveCredentials(config, { account: 'missing' }, {}), /not found/);
+});
+
+test('key rotation preserves profile identity and logout leaves other accounts alone', () => {
+  let config = saved();const first = config.activeProfile;
+  config = loggedInConfig(config, 'https://b.example', 'oc_synthetic_b', identity('b'));const second = config.activeProfile;
+  config = loggedInConfig(config, 'https://a.example:443/', 'oc_rotated_a', identity());
+  assert.equal(config.activeProfile, first);assert.equal(Object.keys(config.profiles).length, 2);
+  const removed = loggedOutConfig(config, first);
+  assert.equal(resolveCredentials(removed, { account: first }, {}).token, null);
+  assert.equal(resolveCredentials(removed, { account: second }, {}).token, 'oc_synthetic_b');
+  assert.equal(JSON.stringify(profileSummaries(config)).includes('oc_rotated_a'), false);
+  assert.equal(JSON.stringify(profileSummaries(config)).includes('apiKey'), false);
+});
+
+test('invalid or tampered identity bindings fail before credential selection', () => {
+  assert.throws(() => loggedInConfig({}, 'https://a.example', 'oc_key', { user: { id: 'bad' }, workspace: {} }), /valid workspace identity/);
+  const config = saved();config.profiles[config.activeProfile].origin = 'https://b.example';
+  assert.throws(() => resolveCredentials(config, {}, {}), /binding is invalid/);
+});
+
+test('exported client rejects route-based host and port changes before fetch', async t => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-route-'));
+  const previous = process.env.OPENCRAWL_CONFIG_DIR;
+  process.env.OPENCRAWL_CONFIG_DIR = directory;
+  const calls = [];
+  t.mock.method(globalThis, 'fetch', async (url, options) => {
+    calls.push({ url, options });return new Response('{}', { status: 200 });
+  });
+  try {
+    const client = await createClient({ server: 'https://a.example', token: 'oc_synthetic_a' });
+    for (const route of ['@b.example/path', ':444/path', 'https://b.example/path', '//b.example/path',
+      '/\\b.example/path', '\\b.example/path', ' /api/v1/usage', '/\n/b.example/path', null])
+      await assert.rejects(client.request(route), /routes must be absolute paths/);
+    assert.equal(calls.length, 0);
+    await client.request('/api/v1/usage?query=synthetic%20value');
+    assert.equal(calls.length, 1);
+    assert.equal(calls[0].url, 'https://a.example/api/v1/usage?query=synthetic%20value');
+    assert.equal(calls[0].options.headers.Authorization, 'Bearer oc_synthetic_a');
+    assert.equal(calls[0].options.redirect, 'error');
+  } finally {
+    if (previous === undefined) delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR = previous;
+    await fs.rm(directory, { recursive: true, force: true });
+  }
+});
+
+const script = fileURLToPath(new URL('../bin/opencrawl.mjs', import.meta.url));
+function run(args, env, cwd, input = '') {
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [script, ...args], { env, cwd, stdio: ['pipe', 'pipe', 'pipe'] });
+    let stdout = '', stderr = '';const timer = setTimeout(() => child.kill('SIGKILL'), 10000);
+    child.stdout.on('data', value => { stdout += value });child.stderr.on('data', value => { stderr += value });
+    child.on('error', reject);child.on('close', code => { clearTimeout(timer);resolve({ code, stdout, stderr }) });child.stdin.end(input);
+  });
+}
+async function fixtureServer(digit) {
+  const calls = [], key = 'oc_synthetic_' + digit;
+  const server = http.createServer(async (request, response) => {
+    let raw = '';for await (const part of request) raw += part;
+    calls.push({ path: request.url, authorization: request.headers.authorization });
+    let result = { status: 'ok' };
+    if (request.url !== '/healthz' && request.headers.authorization !== 'Bearer ' + key) {
+      response.writeHead(401, { 'Content-Type': 'application/json' });response.end('{"detail":"Synthetic key rejected"}');return;
+    }
+    if (request.url === '/api/auth/me') result = identity(digit);
+    else if (request.url === '/mcp') { const message = JSON.parse(raw);result = { jsonrpc: '2.0', id: message.id, result: { tools: [], workspace: digit } }; }
+    else if (request.url === '/api/v1/context') result = { context: '[S1] Synthetic workspace ' + digit };
+    else result = { workspace: digit };
+    response.writeHead(200, { 'Content-Type': 'application/json' });response.end(JSON.stringify(result));
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  return { calls, key, url: 'http://127.0.0.1:' + server.address().port,
+    close: () => new Promise(resolve => server.close(resolve)) };
+}
+
+test('real two-origin commands never send a saved key to a changed destination', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-accounts-'));
+  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
+  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');
+  const a = await fixtureServer('a'), b = await fixtureServer('b');
+  const configPath = path.join(env.OPENCRAWL_CONFIG_DIR, 'config.json');
+  try {
+    let result = await run(['login', '--server', a.url], env, directory, a.key + '\n');
+    assert.equal(result.code, 0, result.stderr);assert.equal(result.stdout.includes(a.key), false);
+    const account = JSON.parse(await fs.readFile(configPath)).activeProfile;
+    assert.equal((await fs.stat(configPath)).mode & 0o777, 0o600);
+    for (const [args, extra] of [[['status', '--server', b.url], {}], [['status'], { OPENCRAWL_URL: b.url }]]) {
+      result = await run(args, { ...env, ...extra }, directory);
+      assert.equal(result.code, 1);assert.equal(b.calls.length, 0);
+    }
+    assert.equal((await run(['configure', '--server', b.url], env, directory)).code, 0);
+    assert.equal((await run(['status'], env, directory)).code, 1);assert.equal(b.calls.length, 0);
+    const doctor = await run(['doctor'], env, directory);
+    assert.equal(doctor.code, 1);assert.equal(b.calls.at(-1).path, '/healthz');assert.equal(b.calls.at(-1).authorization, undefined);
+    result = await run(['status'], { ...env, OPENCRAWL_API_KEY: b.key }, directory);
+    assert.equal(result.code, 0);assert.equal(b.calls.at(-1).authorization, 'Bearer ' + b.key);
+    result = await run(['login', '--account', account], { ...env, OPENCRAWL_URL: b.url }, directory, a.key + '\n');
+    assert.equal(result.code, 0, result.stderr);assert.equal(a.calls.at(-1).path, '/api/auth/me');
+    assert.equal(b.calls.some(call => call.authorization === 'Bearer ' + a.key), false);
+    const before = await fs.readFile(configPath, 'utf8');
+    result = await run(['login', '--account', account, '--server', b.url], env, directory, a.key + '\n');
+    assert.equal(result.code, 1);assert.equal(await fs.readFile(configPath, 'utf8'), before);
+  } finally { await Promise.all([a.close(), b.close()]);await fs.rm(directory, { recursive: true, force: true }); }
+});
+
+test('installed MCP and hook stay pinned after another login and ambient configuration changes', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-pins-'));
+  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
+  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');
+  const a = await fixtureServer('a'), b = await fixtureServer('b');
+  try {
+    assert.equal((await run(['login', '--server', a.url], env, directory, a.key + '\n')).code, 0);
+    for (const target of ['claude-code', 'cursor', 'codex']) assert.equal((await run(['install', target], env, directory)).code, 0);
+    const mcp = JSON.parse(await fs.readFile(path.join(directory, '.mcp.json'), 'utf8')).mcpServers.opencrawl;
+    const settings = await fs.readFile(path.join(directory, '.claude/settings.local.json'), 'utf8');
+    const codex = await fs.readFile(path.join(directory, '.codex/config.toml'), 'utf8');
+    assert.ok(mcp.args.includes('--account') && mcp.args.includes(a.url) && mcp.args.includes('--config-dir'));
+    for (const value of [JSON.stringify(mcp), settings, codex]) assert.ok(!value.includes(a.key) && value.includes('--account'));
+    assert.equal((await run(['login', '--server', b.url], env, directory, b.key + '\n')).code, 0);
+    const before = b.calls.length;
+    const legacyBridge = await run(['mcp'], env, directory, '{"jsonrpc":"2.0","id":1,"method":"tools/list"}\n');
+    assert.equal(legacyBridge.code, 1);assert.match(legacyBridge.stderr, /pinned account/);assert.equal(b.calls.length, before);
+    assert.equal((await run(['hook'], env, directory, '{"prompt":"Synthetic question"}')).stdout, '');
+    const changedEnv = { ...env, OPENCRAWL_URL: b.url, OPENCRAWL_API_KEY: b.key, OPENCRAWL_CONFIG_DIR: path.join(directory, 'unrelated') };
+    const argumentsAfterScript = mcp.args.slice(1);
+    const response = await run(argumentsAfterScript, changedEnv, directory, '{"jsonrpc":"2.0","id":7,"method":"tools/list"}\n');
+    assert.equal(response.code, 0, response.stderr);assert.equal(JSON.parse(response.stdout).result.workspace, 'a');
+    const hooked = await run(['hook', ...argumentsAfterScript.slice(1)], changedEnv, directory, '{"prompt":"Synthetic question"}');
+    assert.equal(hooked.code, 0);assert.match(JSON.parse(hooked.stdout).hookSpecificOutput.additionalContext, /workspace a/);
+    assert.equal(b.calls.some(call => call.path === '/mcp' || call.path === '/api/v1/context'), false);
+    const account = argumentsAfterScript[argumentsAfterScript.indexOf('--account') + 1];
+    assert.equal((await run(['logout', '--account', account], env, directory)).code, 0);
+    const disconnected = await run(argumentsAfterScript, changedEnv, directory, '{"jsonrpc":"2.0","id":8,"method":"tools/list"}\n');
+    assert.ok(JSON.parse(disconnected.stdout).error);assert.equal(b.calls.some(call => call.path === '/mcp'), false);
+  } finally { await Promise.all([a.close(), b.close()]);await fs.rm(directory, { recursive: true, force: true }); }
+});
+
+test('legacy migration requires explicit login and failed login leaves the old file untouched', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-legacy-'));
+  const env = { ...process.env };for (const key of Object.keys(env)) if (key.startsWith('OPENCRAWL_')) delete env[key];
+  env.OPENCRAWL_CONFIG_DIR = path.join(directory, 'config');const server = await fixtureServer('a');
+  try {
+    await fs.mkdir(env.OPENCRAWL_CONFIG_DIR);
+    const file = path.join(env.OPENCRAWL_CONFIG_DIR, 'config.json');
+    const legacy = JSON.stringify({ server: server.url, apiKey: 'oc_unknown_origin', unrelated: 'preserved' });
+    await fs.writeFile(file, legacy, { mode: 0o600 });
+    const denied = await run(['status'], env, directory);
+    assert.equal(denied.code, 1);assert.match(denied.stderr, /predates server binding/);assert.equal(server.calls.length, 0);
+    assert.equal((await run(['install', 'claude-code'], env, directory)).code, 1);
+    await assert.rejects(fs.stat(path.join(directory, '.mcp.json')), { code: 'ENOENT' });
+    const failed = await run(['login', '--server', server.url], env, directory, 'oc_explicit_invalid\n');
+    assert.equal(failed.code, 1);assert.equal(await fs.readFile(file, 'utf8'), legacy);
+    assert.equal((await run(['login', '--server', server.url], env, directory, server.key + '\n')).code, 0);
+    const upgraded = JSON.parse(await fs.readFile(file));
+    assert.equal(upgraded.version, 2);assert.equal(upgraded.apiKey, undefined);assert.equal(upgraded.unrelated, 'preserved');
+    assert.equal((await run(['status'], env, directory)).code, 0);
+    const inventory = await run(['profiles'], env, directory);
+    assert.equal(JSON.parse(inventory.stdout).accounts.length, 1);assert.equal(inventory.stdout.includes(server.key), false);
+    assert.equal(server.calls.some(call => call.authorization === 'Bearer oc_unknown_origin'), false);
+  } finally { await server.close();await fs.rm(directory, { recursive: true, force: true }); }
+});

package/src/agent.mjs

@@ -0,0 +1,95 @@
+// Talk to the OpenCrawl agent from a terminal or a coding agent. Output is JSON for agents.
+import {settle, transient} from './jobs.mjs';
+
+export const AGENT_HELP = `Usage:
+  opencrawl agent signup --organization NAME --email EMAIL [--create-new]
+  opencrawl agent join POLL_URL                        after signup asks for approval: wait for an admin to approve, then save the key
+  opencrawl agent "what you want to build"            start a session
+  opencrawl agent "answer" --session SESSION_ID        continue it
+  opencrawl agent --session SESSION_ID                 read the session (customer discovery progress, spec, project)
+  opencrawl agent --session SESSION_ID --build         build the project, or apply changes described on its session
+  opencrawl agent --session SESSION_ID --new-key       a new project key (the previous one stops working), if a key was lost
+  opencrawl agent --session SESSION_ID --check         verify app authorizations
+  opencrawl agent --session SESSION_ID --connect APP   add gmail, googledrive, notion, hubspot or granola to a built project
+  opencrawl agent projects                             list built projects
+  opencrawl agent memory                               what the OpenCrawl agent remembers for this organization
+  opencrawl agent remember "note"                      add to that memory (or say "remember ..." in any message)
+Use an organization admin key (OPENCRAWL_API_KEY or opencrawl login). Signup needs no key.
+Signup and build save the keys they return as CLI accounts and owner-only key files, and print where
+instead of the key: use --account ACCOUNT on any command, or read key_file for MCP or your server.
+Signup makes the organization account the default; a project account is used with --account.
+Use the project account to read a project's watches, jobs and research. --print-keys prints keys instead.`;
+
+const SESSION = /^ags_[a-f0-9]{32}$/;
+
+export function agentRequest(rest, options) {
+  const [first, ...more] = rest;
+  if (first === 'signup') {
+    if (!options.organization || !options.email) throw new Error('Use agent signup --organization NAME --email EMAIL.');
+    return {route: '/api/v1/agent/signup', method: 'POST', authenticated: false,
+      body: {organization: String(options.organization), email: String(options.email), agent: 'OpenCrawl CLI', ...(options['create-new'] ? {create_new: true} : {})}};
+  }
+  if (first === 'join') {
+    const token = /^https?:\/\/[^/]+\/api\/v1\/agent\/signup\/([A-Za-z0-9_-]{20,64})$/.exec(String(more[0] ?? ''))?.[1];
+    if (!token || more.length > 1) throw new Error('Use agent join POLL_URL, with the poll_url agent signup returned.');
+    return {route: `/api/v1/agent/signup/${token}`, method: 'GET', authenticated: false, poll: true};
+  }
+  if (first === 'projects' && !more.length) return {route: '/api/v1/agent/projects', method: 'GET'};
+  if (first === 'memory' && !more.length) return {route: '/api/v1/agent/memory', method: 'GET'};
+  if (first === 'remember') {
+    if (!more.join(' ').trim()) throw new Error('Use agent remember "note".');
+    return {route: '/api/v1/agent/memory', method: 'POST', body: {note: more.join(' ').trim()}};
+  }
+  const message = rest.join(' ').trim();
+  const session = options.session;
+  if (session !== undefined && !SESSION.test(String(session))) throw new Error('--session must be a session ID returned by opencrawl agent.');
+  if ([options.build, options.check, options.connect, options['new-key']].filter(Boolean).length > 1) throw new Error('Choose one of --build, --check, --connect or --new-key.');
+  const action = options.build ? 'build' : options.check ? 'check_connections' : options.connect ? 'connect' : options['new-key'] ? 'issue_key' : undefined;
+  if (action && !session) throw new Error(`--${options.build ? 'build' : options.check ? 'check' : options.connect ? 'connect' : 'new-key'} needs --session.`);
+  if (options.connect === true) throw new Error('--connect needs an app name, such as granola.');
+  if (!action && !message && session) return {route: `/api/v1/agent/sessions/${session}`, method: 'GET'};
+  if (!action && !message) throw new Error('Say what you want to build, or use --session with --build or --check.');
+  const body = {...(message ? {message} : {}), ...(action ? {action} : {}), ...(action === 'connect' ? {toolkit: String(options.connect)} : {})};
+  return session ? {route: `/api/v1/agent/sessions/${session}/messages`, method: 'POST', body}
+    : {route: '/api/v1/agent/sessions', method: 'POST', body};
+}
+
+// Keys are shown once. Keep them out of terminal output (and an agent's chat) unless asked: save them instead.
+export async function keepKeys(result, saveKey) {
+  if (!result || typeof result !== 'object') return result;
+  let next = result;
+  if (typeof result.api_key === 'string') {
+    const {api_key: key, ...rest} = result;
+    next = {...rest, api_key_saved: {...await saveKey(key, {activate: true}), kind: 'organization'}};
+  }
+  if (typeof next.project?.api_key === 'string') {
+    const {api_key: key, ...project} = next.project;
+    next = {...next, project: {...project, api_key_saved: {...await saveKey(key, {activate: false}), kind: 'project'}}};
+  }
+  return next;
+}
+
+export async function agentCommand(rest, options, client, out = value => process.stdout.write(JSON.stringify(value, null, 2) + '\n'), saveKey,
+                                   {now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms))} = {}) {
+  const request = agentRequest(rest, options);
+  // A join request waits up to the hour an approval link stays valid.
+  const pollUntil = now() + 3600000;
+  let deadline = now() + 120000, result, failures = 0;
+  while (true) {
+    try {
+      result = await client.request(request.route, {method: request.method, body: request.body, authenticated: request.authenticated !== false, timeout: 120000});
+      if (!request.poll || result?.status !== 'pending' || now() >= pollUntil) break;
+      await sleep(10000); deadline = now() + 120000; failures = 0; continue;
+    }
+    catch (error) {
+      // Reads are safe to repeat while the server restarts. A write's outcome is unknown, so say how to find out.
+      if (request.method === 'GET' && transient(error) && now() < deadline) { await settle(++failures, {now, sleep, deadline}); continue; }
+      if (transient(error) && request.method === 'POST' && options.session)
+        error.message += ` The outcome is unknown: read the session (opencrawl agent --session ${options.session}) before sending again.` +
+          (request.body?.action === 'build' ? ' If it shows the project built but you did not receive its key, use --new-key.' : '');
+      throw error;
+    }
+  }
+  out(saveKey && !options['print-keys'] ? await keepKeys(result, saveKey) : result);
+  return 0;
+}

package/src/agent.test.mjs

@@ -0,0 +1,58 @@
+import assert from 'node:assert/strict';
+import {test} from 'node:test';
+import {agentRequest, agentCommand} from './agent.mjs';
+
+const session = 'ags_' + 'a'.repeat(32);
+test('agent requests map messages, actions and signup to the OpenCrawl agent API', () => {
+  assert.deepEqual(agentRequest(['watch', 'rival', 'changelogs'], {}), {route: '/api/v1/agent/sessions', method: 'POST', body: {message: 'watch rival changelogs'}});
+  assert.deepEqual(agentRequest(['hourly'], {session}), {route: `/api/v1/agent/sessions/${session}/messages`, method: 'POST', body: {message: 'hourly'}});
+  assert.deepEqual(agentRequest([], {session, build: true}).body, {action: 'build'});
+  assert.deepEqual(agentRequest([], {session, check: true}).body, {action: 'check_connections'});
+  assert.deepEqual(agentRequest([], {session, connect: 'granola'}).body, {action: 'connect', toolkit: 'granola'});
+  assert.deepEqual(agentRequest(['projects'], {}), {route: '/api/v1/agent/projects', method: 'GET'});
+  assert.deepEqual(agentRequest([], {session}), {route: `/api/v1/agent/sessions/${session}`, method: 'GET'});
+  assert.deepEqual(agentRequest(['memory'], {}), {route: '/api/v1/agent/memory', method: 'GET'});
+  assert.deepEqual(agentRequest(['remember', 'Lantern', 'is', 'lantern.fm'], {}), {route: '/api/v1/agent/memory', method: 'POST', body: {note: 'Lantern is lantern.fm'}});
+  const signup = agentRequest(['signup'], {organization: 'Acme', email: 'a@acme.example', 'create-new': true});
+  assert.equal(signup.authenticated, false); assert.deepEqual(signup.body, {organization: 'Acme', email: 'a@acme.example', agent: 'OpenCrawl CLI', create_new: true});
+  for (const [rest, options] of [[[], {}], [[], {build: true}], [['x'], {session: 'bad'}], [['signup'], {email: 'a@b.c'}], [[], {session, build: true, check: true}], [[], {session, connect: true}], [[], {connect: 'gmail'}], [['remember'], {}]]) assert.throws(() => agentRequest(rest, options));
+});
+test('agent command prints the server response as JSON', async () => {
+  const calls = [], printed = [];
+  await agentCommand(['hello'], {}, {request: async (route, init) => { calls.push({route, init}); return {session_id: session, reply: 'Hi'}; }}, v => printed.push(v));
+  assert.equal(calls[0].init.timeout, 120000); assert.equal(calls[0].init.authenticated, true); assert.deepEqual(printed, [{session_id: session, reply: 'Hi'}]);
+});
+test('research feed and subscription commands map to the project signal routes', async () => {
+  const {researchRequest} = await import('./research.mjs');
+  assert.deepEqual(researchRequest(['feed'], {customer: 'Basedash', after: '7'}), {command: 'feed', route: '/api/research/feed?after=7&limit=50&customer=Basedash'});
+  assert.deepEqual(researchRequest(['subscribe', 'https://hooks.example/oc'], {customer: 'Basedash'}).body, {endpoint: 'https://hooks.example/oc', customer: 'Basedash'});
+  assert.equal(researchRequest(['unsubscribe', 'rsub_' + 'a'.repeat(32)], {}).method, 'DELETE');
+  for (const [rest, options] of [[['subscribe', 'http://insecure.example'], {}], [['unsubscribe', 'bad'], {}], [['feed', 'extra'], {}]]) assert.throws(() => researchRequest(rest, options));
+});
+test('session reads ride out a server restart; a lost build says how to recover the key', async () => {
+  assert.deepEqual(agentRequest([], {session, 'new-key': true}).body, {action: 'issue_key'});
+  let time = 0, calls = 0;
+  const clock = {now: () => time, sleep: async ms => { time += ms; }};
+  const printed = [];
+  await agentCommand([], {session}, {request: async () => { if (++calls < 3) { const e = Error('OpenCrawl 502'); e.status = 502; throw e; } return {state: 'ready'}; }}, v => printed.push(v), undefined, clock);
+  assert.equal(calls, 3); assert.deepEqual(printed, [{state: 'ready'}]);
+  calls = 0;
+  await assert.rejects(agentCommand([], {session, build: true}, {request: async () => { calls++; const e = Error('OpenCrawl 502.'); e.status = 502; throw e; }}, () => {}, undefined, clock),
+    error => error.message.includes('--new-key') && error.message.includes(`--session ${session}`));
+  assert.equal(calls, 1);  // a build is never sent twice on its own
+  await assert.rejects(agentCommand([], {session}, {request: async () => { const e = Error('OpenCrawl 404'); e.status = 404; throw e; }}, () => {}, undefined, clock));
+});
+test('join waits for an admin to approve, then saves the organization key', async () => {
+  const poll = 'https://app.getopencrawl.com/api/v1/agent/signup/' + 'T'.repeat(43);
+  assert.deepEqual(agentRequest(['join', poll], {}), {route: '/api/v1/agent/signup/' + 'T'.repeat(43), method: 'GET', authenticated: false, poll: true});
+  for (const rest of [['join'], ['join', 'https://app.getopencrawl.com/api/v1/agent/sessions'], ['join', poll, 'extra']]) assert.throws(() => agentRequest(rest, {}));
+  let time = 0, calls = 0;
+  const clock = {now: () => time, sleep: async ms => { time += ms; }}, printed = [], saved = [];
+  await agentCommand(['join', poll], {}, {request: async (route, init) => { assert.equal(init.authenticated, false); return ++calls < 3 ? {status: 'pending'} : {status: 'approved', organization: {name: 'Acme'}, api_key: 'oc_secret'}; }},
+    v => printed.push(v), async (key, options) => { saved.push([key, options]); return {account: 'acc_1'}; }, clock);
+  assert.equal(calls, 3); assert.equal(time, 20000); assert.deepEqual(saved, [['oc_secret', {activate: true}]]);
+  assert.deepEqual(printed, [{status: 'approved', organization: {name: 'Acme'}, api_key_saved: {account: 'acc_1', kind: 'organization'}}]);
+  time = 0; printed.length = 0;
+  await agentCommand(['join', poll], {}, {request: async () => ({status: 'pending'})}, v => printed.push(v), undefined, clock);
+  assert.ok(time >= 3600000); assert.deepEqual(printed, [{status: 'pending'}]);
+});

package/src/browser.test.mjs

@@ -0,0 +1,72 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { validateCaptureUrl, isReadOnlyRequest, browserCommand } from '../browser.mjs';
+
+test('browser scope rejects executable schemes and action links', () => {
+  for (const url of ['file:///etc/passwd', 'javascript:alert(1)', 'https://x.test/logout', 'https://x.test/%64elete/123', 'https://x.test/?action=delete', 'https://user:password@x.test']) {
+    assert.throws(() => validateCaptureUrl(url));
+  }
+  assert.equal(validateCaptureUrl('https://crm.example/contacts#top').href, 'https://crm.example/contacts');
+});
+test('only read HTTP methods and GraphQL queries are accepted', () => {
+  assert.equal(isReadOnlyRequest('GET'), true);
+  assert.equal(isReadOnlyRequest('DELETE'), false);
+  assert.equal(isReadOnlyRequest('POST', '{"name":"Change me"}'), false);
+  assert.equal(isReadOnlyRequest('POST', JSON.stringify({ query: 'query Contacts { contacts { name } }' })), true);
+  assert.equal(isReadOnlyRequest('POST', JSON.stringify({ query: 'mutation DeleteContact { deleteContact(id: 1) }' })), false);
+});
+
+function browserFixture() {
+  const observations = { urls: [], closed: false, documents: [], routes: [] };
+  const frame = {};
+  let current = 'about:blank';
+  const page = {
+    goto: async url => { current = url; observations.urls.push(url); return { status: () => 200 }; },
+    url: () => current, mainFrame: () => frame,
+    waitForLoadState: async () => {},
+    evaluate: async () => ({ title: 'Customer account', text: 'Acme customer account. Renewal is on November 15. Owner is Jamie.', links: [
+      'https://crm.example/accounts/next', 'https://crm.example/logout', 'https://unrelated.example/private',
+    ], headings: ['Acme'] }),
+    locator: () => ({}), screenshot: async ({ path: output }) => fs.writeFile(output, 'test screenshot'),
+  };
+  const context = {
+    pages: () => [page], newPage: async () => page, route: async (match, callback) => observations.routes.push(callback),
+    on: () => {}, close: async () => { observations.closed = true; },
+  };
+  const playwright = { chromium: { launchPersistentContext: async (dir, options) => { observations.profile = dir; observations.options = options; return context; } } };
+  const client = { configured: true, request: async (route, { body }) => { observations.documents.push(body); return { id: 'document-' + observations.documents.length }; } };
+  return { observations, playwright, client };
+}
+
+test('capture imports provenance using isolated profile and blocks mutations', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-browser-test-'));
+  const prior = process.env.OPENCRAWL_CONFIG_DIR; process.env.OPENCRAWL_CONFIG_DIR = directory;
+  try {
+    const { observations, playwright, client } = browserFixture();
+    const result = await browserCommand('capture', 'https://crm.example/accounts/123', { headless: true }, client, { playwright });
+    assert.equal(result.imported, 1);
+    assert.equal(observations.documents[0].visibility, 'private');
+    assert.equal(observations.documents[0].url, 'https://crm.example/accounts/123');
+    assert.ok(observations.profile.startsWith(path.join(directory, 'browser-profiles')));
+    assert.equal(observations.options.serviceWorkers, 'block');
+    assert.equal(observations.closed, true);
+    let aborted = false;
+    await observations.routes[0]({ request: () => ({ method: () => 'DELETE', postData: () => null }), abort: async () => { aborted = true; } });
+    assert.equal(aborted, true);
+    await assert.rejects(browserCommand('capture', 'https://other.example/page', {}, client, { playwright }), /scoped to/);
+  } finally { if (prior === undefined) delete process.env.OPENCRAWL_CONFIG_DIR; else process.env.OPENCRAWL_CONFIG_DIR = prior; await fs.rm(directory, { recursive: true, force: true }); }
+});
+
+test('bounded crawl excludes other origins and action links', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-crawl-test-'));
+  const prior = process.env.OPENCRAWL_CONFIG_DIR; process.env.OPENCRAWL_CONFIG_DIR = directory;
+  try {
+    const { observations, playwright, client } = browserFixture();
+    const result = await browserCommand('crawl', 'https://crm.example/accounts/123', { 'max-pages': '2', 'path-prefix': '/accounts/' }, client, { playwright });
+    assert.equal(result.imported, 2);
+    assert.deepEqual(observations.urls, ['https://crm.example/accounts/123', 'https://crm.example/accounts/next']);
+  } finally { if (prior === undefined) delete process.env.OPENCRAWL_CONFIG_DIR; else process.env.OPENCRAWL_CONFIG_DIR = prior; await fs.rm(directory, { recursive: true, force: true }); }
+});