@getopencrawl/cli 0.1.1 → 0.2.0

package/src/tasks.test.mjs

@@ -1,83 +0,0 @@
-import {test} from 'node:test';
-import assert from 'node:assert/strict';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import http from 'node:http';
-import {spawn} from 'node:child_process';
-import {fileURLToPath} from 'node:url';
-import {taskRequest,tasksCommand,validateTaskRun} from './tasks.mjs';
-import {loggedInConfig} from './client.mjs';
-const id='task_'+'a'.repeat(32);
-const task=()=>({id,title:'Launch evidence',query:'What is the current launch?',scope:{mode:'selected',source_ids:['src_alpha']},limit:5,budget:16384,revision:2,created_at:'2026-01-01T00:00:00Z',updated_at:'2026-01-01T00:00:00Z',scope_status:'available'});
-const receipt=(text='[S1] Launch\né🌲\u001b[2J\n')=>({task:task(),query:'What is the current launch?',context:text,context_bytes:Buffer.byteLength(text),budget_unit:'utf8_bytes',citations:text?[{citation:'S1',document_id:'doc_alpha',chunk_id:1,source_id:'src_alpha',kind:'document',title:'Launch',context_span:{start_char:12,end_char:Array.from(text).length},excerpt:{start_char:0,end_char:Array.from(text).length-12}}]:[],coverage:{warnings:[]}});
-const execute=async(rest,options,data)=>{let stdout='',stderr='';const calls=[];const status=await tasksCommand(rest,options,{request:async(route,opts)=>{calls.push({route,opts});if(data instanceof Error)throw data;return data;}},{stdout:value=>stdout+=value,stderr:value=>stderr+=value});return{stdout,stderr,calls,status};};
-
-test('tasks preserve stored descriptor and positive revision without run overrides',()=>{
-  assert.deepEqual(taskRequest(['run',id],{}).options,{method:'POST',body:{}});
-  assert.deepEqual(taskRequest(['run',id],{revision:'2'}).options,{method:'POST',body:{expected_revision:2}});
-  assert.equal(taskRequest(['list'],{limit:'20',offset:'40'}).route,'/api/v1/tasks?limit=20&offset=40');
-  for(const revision of ['0','-1','1e2','+2','2.0',' 2','02','a'.repeat(64),'9007199254740992'])assert.throws(()=>taskRequest(['run',id],{revision}));
-  for(const key of ['query','scope','source','budget','limit','offset'])assert.throws(()=>taskRequest(['run',id],{[key]:'2'}));
-  for(const rest of [['run'],['show','../secret'],['list',id],['run',id,'extra'],['create']])assert.throws(()=>taskRequest(rest,{}));
-});
-test('run stdout preserves exact context bytes and full JSON receipts',async()=>{
-  const value=receipt();const plain=await execute(['run',id],{revision:'2'},value);
-  assert.equal(plain.status,0);assert.equal(plain.stdout,value.context);assert.match(plain.stderr,/Task revision 2/);
-  assert.deepEqual(plain.calls[0].opts.body,{expected_revision:2});
-  const json=await execute(['run',id],{json:true},value);assert.deepEqual(JSON.parse(json.stdout),value);assert.equal(json.stderr,'');
-});
-test('empty current evidence succeeds without fabricated answer or silent broader scope',async()=>{
-  const value=receipt('');const result=await execute(['run',id],{},value);assert.equal(result.status,0);assert.equal(result.stdout,'');assert.match(result.stderr,/No current evidence/);assert.equal(result.calls.length,1);
-  const json=await execute(['run',id],{json:true},value);assert.equal(JSON.parse(json.stdout).context,'');
-});
-test('all-accessible tasks can cite standalone documents but cannot carry hidden selected scope',async()=>{
-  const value=receipt();value.task.scope={mode:'all_accessible'};value.citations[0].source_id=null;
-  assert.equal((await execute(['run',id],{},value)).stdout,value.context);
-  value.task.scope={mode:'all_accessible',source_ids:['src_alpha']};
-  await assert.rejects(execute(['run',id],{},value));
-});
-test('byte, revision, current scope and exact citation spans fail closed before stdout',async()=>{
-  const base=receipt();
-  const cases=[{task:{...task(),revision:3}},{task:{...task(),scope_status:'blocked'}},{context_bytes:1},{budget_unit:'tokens'},
-    {context:'x'.repeat(16385),context_bytes:16385},{citations:[]},{citations:[{...base.citations[0],source_id:'src_other'}]},
-    {citations:[{...base.citations[0],context_span:{start_char:0,end_char:99999}}]},
-    {citations:[base.citations[0],base.citations[0]]},{task:{...task(),budget:16385}},{task:{...task(),scope:{mode:'selected',source_ids:[]}}}];
-  for(const change of cases){let output='';await assert.rejects(tasksCommand(['run',id],{revision:'2'},{request:async()=>({...base,...change})},{stdout:v=>output+=v,stderr:()=>{}}));assert.equal(output,'');}
-  const full=receipt('[S1] Launch\n'+'é'.repeat(8186));assert.equal(full.context_bytes,16384);validateTaskRun(full,{id,revision:2});
-});
-test('task conflicts distinguish changed descriptor from unavailable source without retry',async()=>{
-  for(const [status,code,pattern]of[[409,'task_changed',/review its current/],[409,'task_scope_unavailable',/No broader search/],[409,'task_evidence_changed',/No automatic retry/],[404,'anything',/tasks list/],[401,null,/Sign in/],[500,'secret',/could not be confirmed/]]){
-    const error=Object.assign(new Error('RAW SECRET\u001b'),{status,code});const result=await execute(['run',id],{json:true},error);
-    assert.equal(result.status,1);assert.equal(result.calls.length,1);assert.equal(result.stderr,'');assert.match(JSON.parse(result.stdout).error.message,pattern);assert.ok(!result.stdout.includes('SECRET'));
-  }
-});
-test('human task descriptions and warnings escape terminal controls; JSON remains exact',async()=>{
-  const value={...task(),title:'Name\u001b[2J',query:'Question\n\ttext\u009b31m'};
-  const shown=await execute(['show',id],{},value);assert.ok(!shown.stdout.includes('\u001b'));assert.match(shown.stdout,/\\u001b/);assert.match(shown.stdout,/Question\n\ttext\\u009b/);
-  const listed=await execute(['list'],{}, {items:[value],limit:20,offset:0});assert.match(listed.stdout,/\\u001b/);
-  const pack=receipt();pack.coverage.warnings=['Fallback\u001b[2J'];const result=await execute(['run',id],{},pack);assert.match(result.stderr,/\\u001b/);assert.equal(result.stdout,pack.context);
-  const json=await execute(['show',id],{json:true},value);assert.deepEqual(JSON.parse(json.stdout),value);
-});
-const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
-function run(args,env,cwd){return new Promise((resolve,reject)=>{const p=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>p.kill('SIGKILL'),10000);p.stdout.on('data',v=>stdout+=v);p.stderr.on('data',v=>stderr+=v);p.on('error',reject);p.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr});});});}
-async function listen(){const calls=[];let conflict=null;const server=http.createServer(async(req,res)=>{let raw='';for await(const v of req)raw+=v;const body=raw?JSON.parse(raw):null;calls.push({path:req.url,body,authorization:req.headers.authorization});const failed=req.url.endsWith('/run')&&conflict;res.writeHead(failed?409:200,{'content-type':'application/json'});res.end(JSON.stringify(failed?{code:conflict,detail:'RAW PRIVATE'}:req.url.startsWith('/api/v1/tasks?')?{items:[task()],limit:20,offset:0}:req.url.endsWith('/run')||req.url.endsWith('/context')?receipt():task()));});await new Promise(r=>server.listen(0,'127.0.0.1',r));return{url:'http://127.0.0.1:'+server.address().port,calls,setConflict:value=>conflict=value,close:()=>new Promise(r=>server.close(r))};}
-test('real CLI keeps account pins, strict options, fixed JSON conflicts, and default search scope',async()=>{
-  const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-tasks-'));const a=await listen(),b=await listen();const env={...process.env};for(const key of Object.keys(env))if(key.startsWith('OPENCRAWL_'))delete env[key];env.OPENCRAWL_CONFIG_DIR=dir;
-  const identity=digit=>({user:{id:'usr_'+digit.repeat(32)},workspace:{id:'ws_'+digit.repeat(32)}});
-  try{
-    let config=loggedInConfig({},a.url,'oc_synthetic_a',identity('a'));const account=config.activeProfile;config=loggedInConfig(config,b.url,'oc_synthetic_b',identity('b'));await fs.writeFile(path.join(dir,'config.json'),JSON.stringify(config),{mode:0o600});
-    const ambient={...env,OPENCRAWL_URL:b.url,OPENCRAWL_API_KEY:'oc_synthetic_b'};
-    const help=await run(['tasks','--help'],ambient,dir);assert.equal(help.code,0);assert.match(help.stdout,/--revision N/);assert.equal(a.calls.length+b.calls.length,0);
-    const exact=await run(['tasks','run',id,'--revision','2','--account',account],ambient,dir);assert.equal(exact.code,0,exact.stderr);assert.equal(exact.stdout,receipt().context);assert.deepEqual(a.calls.at(-1).body,{expected_revision:2});
-    for(const args of [['tasks','list','--account',account],['tasks','show',id,'--account',account],['--account',account,'tasks','run',id,'--json']]){const result=await run(args,ambient,dir);assert.equal(result.code,0,result.stderr);}
-    assert.equal(b.calls.length,0);assert.ok(a.calls.every(c=>c.authorization==='Bearer oc_synthetic_a'));
-    const before=a.calls.length;
-    for(const extra of [['--revision','2','--revision','3'],['--json=false'],['--source','src_other'],['--revision','0']]){const r=await run(['tasks','run',id,'--account',account,...extra],ambient,dir);assert.equal(r.code,1);}
-    assert.equal(a.calls.length,before);
-    const wrong=await run(['tasks','show',id,'--account',account,'--server',b.url],ambient,dir);assert.equal(wrong.code,1);assert.equal(b.calls.length,0);
-    for(const code of ['task_changed','task_scope_unavailable','task_evidence_changed']){a.setConflict(code);const r=await run(['tasks','run',id,'--json','--account',account],ambient,dir);assert.equal(r.code,1);assert.equal(JSON.parse(r.stdout).error.code,code);assert.equal(r.stderr,'');assert.ok(!r.stdout.includes('PRIVATE'));}
-    a.setConflict(null);const r=await run(['context','launch','--account',account],ambient,dir);assert.equal(r.code,0,r.stderr);assert.equal(Object.hasOwn(a.calls.at(-1).body,'source_ids'),false);
-    await run(['context','launch','--source','src_alpha','--account',account],ambient,dir);assert.deepEqual(a.calls.at(-1).body.source_ids,['src_alpha']);
-  }finally{await Promise.all([a.close(),b.close()]);await fs.rm(dir,{recursive:true,force:true});}
-});

package/src/upload-commands.test.mjs

@@ -1,140 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import http from 'node:http';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { spawn } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-
-const CLI = fileURLToPath(new URL('../bin/opencrawl.mjs', import.meta.url));
-const jobId = 'job_' + 'a'.repeat(32);
-async function fixture(t) {
-  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-commands-'));
-  const receipts = new Map(), calls = [], state = { jobStatus: 'completed', losePrepare: false, loseContent: false, failLookup: 0, holdJob: false };
-  const server = http.createServer(async (req, res) => {
-    const chunks = []; for await (const chunk of req) chunks.push(chunk);
-    const raw = Buffer.concat(chunks), body = req.headers['content-type']?.startsWith('application/json') ? JSON.parse(raw) : raw;
-    calls.push({ path: req.url, method: req.method, body });
-    const send = (status, value) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(value)); };
-    const job = () => ({ id: jobId, kind: 'upload', status: state.jobStatus, terminal: ['completed','failed','cancelled','discarded'].includes(state.jobStatus), indexed_documents: state.jobStatus === 'completed' ? 1 : 0, filename: 'test\u001b[2J.pdf', documents: [] });
-    if (req.url === '/api/auth/me') return send(200, { user: { id: 'usr_test' }, workspace: { id: 'ws_test' } });
-    if (req.url === '/api/v1/uploads/prepare') {
-      let receipt = receipts.get(body.request_key);
-      if (receipt && JSON.stringify(receipt.descriptor) !== JSON.stringify(body)) return send(409, { detail: { code: 'descriptor_mismatch', message: 'raw body must not escape' } });
-      if (!receipt) { receipt = { id: 'receipt_' + body.request_key.replaceAll('-', ''), request_key: body.request_key, state: 'prepared', operation: body.operation, job: null, descriptor: body }; receipt.upload_url = `/api/v1/uploads/receipts/${receipt.id}/content`; receipts.set(body.request_key, receipt); }
-      if (receipt.state === 'accepted') receipt.job = job();
-      if (state.losePrepare) { state.losePrepare = false; res.destroy(); return; }
-      return send(200, receipt);
-    }
-    if (req.url.endsWith('/content')) {
-      const receipt = [...receipts.values()].find(item => item.upload_url === req.url);
-      assert(receipt); assert(raw.includes(Buffer.from('synthetic fixture'))); receipt.state = 'accepted'; receipt.job = job();
-      if (state.loseContent) { state.loseContent = false; res.destroy(); return; }
-      return send(200, receipt);
-    }
-    if (req.url.startsWith('/api/v1/uploads/receipts/by-key/')) {
-      if (state.failLookup-- > 0) { res.destroy(); return; }
-      const receipt = receipts.get(req.url.split('/').at(-1)); if (!receipt) return send(404, { detail: 'Unavailable' });
-      if (receipt.state === 'accepted') receipt.job = job(); return send(200, receipt);
-    }
-    if (req.url === `/api/v1/jobs/${jobId}`) { if (state.holdJob) return; return send(200, job()); }
-    if (req.url === '/api/v1/search') return send(200, { results: [{ title: 'bad\u001b[2J', snippet: 'keep\nline\u001b]52;x\u0007', url: 'a\rb' }] });
-    if (req.url === '/api/v1/documents') return send(200, { id: 'doc_' + 'c'.repeat(32), title: 'synchronous' });
-    return send(404, { detail: 'Unknown route' });
-  });
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
-  t.after(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); await fs.rm(dir, { recursive: true, force: true }); });
-  const env = { PATH: process.env.PATH, HOME: dir, OPENCRAWL_CONFIG_DIR: path.join(dir, 'config'), OPENCRAWL_API_KEY: 'oc_synthetic_test_only', OPENCRAWL_URL: `http://127.0.0.1:${server.address().port}` };
-  const run = (args, onSpawn) => new Promise((resolve, reject) => {
-    const child = spawn(process.execPath, [CLI, ...args], { env, cwd: dir }); let stdout = '', stderr = '';
-    child.stdout.on('data', data => { stdout += data; }); child.stderr.on('data', data => { stderr += data; });
-    const timer = setTimeout(() => { child.kill('SIGKILL'); reject(Error('CLI test timeout')); }, 10000);
-    child.on('error', reject); child.on('exit', (code, signal) => { clearTimeout(timer); resolve({ code, signal, stdout, stderr }); });
-    onSpawn?.(child);
-  });
-  const file = path.join(dir, 'fixture.pdf'); await fs.writeFile(file, 'synthetic fixture');
-  return { dir, file, run, calls, state, receipts, env };
-}
-test('real CLI waits finitely, preserves job receipt on timeout/failure, and can later inspect completion', async t => {
-  const f = await fixture(t); f.state.jobStatus = 'queued';
-  const timed = await f.run(['ingest', f.file, '--wait', '--timeout', '1', '--json']);
-  assert.equal(timed.code, 1); const receipt = JSON.parse(timed.stdout); assert.equal(receipt.job_id, jobId); assert.equal(receipt.wait.outcome, 'timeout'); assert(timed.stderr.includes(receipt.request_key));
-  f.state.jobStatus = 'completed';
-  const completed = await f.run(['jobs', 'wait', jobId, '--timeout', '2', '--json']);
-  assert.equal(completed.code, 0); assert.equal(JSON.parse(completed.stdout).wait.outcome, 'completed');
-  f.state.jobStatus = 'failed'; const failed = await f.run(['jobs', 'wait', jobId, '--json']);
-  assert.equal(failed.code, 1); assert.equal(JSON.parse(failed.stdout).job_id, jobId); assert.equal(JSON.parse(failed.stdout).wait.outcome, 'failed');
-  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
-});
-test('real dropped acceptance is looked up, explicit reuse never uploads twice, changed scope conflicts', async t => {
-  const f = await fixture(t); f.state.loseContent = true;
-  const first = await f.run(['ingest', f.file, '--json']); assert.equal(first.code, 0); const result = JSON.parse(first.stdout); assert.equal(result.job_id, jobId);
-  const repeated = await f.run(['ingest', f.file, '--request-key', result.request_key, '--json']); assert.equal(repeated.code, 0); assert.equal(JSON.parse(repeated.stdout).job_id, jobId);
-  const conflict = await f.run(['ingest', f.file, '--request-key', result.request_key, '--visibility', 'workspace', '--json']);
-  assert.equal(conflict.code, 1); assert.equal(JSON.parse(conflict.stdout).error.status, 409); assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
-});
-test('real lost prepare response requires explicit same-key retry and never auto-transfers', async t => {
-  const f = await fixture(t); f.state.losePrepare = true;
-  const first = await f.run(['ingest', f.file, '--json']); assert.equal(first.code, 1); const result = JSON.parse(first.stdout); assert.equal(result.outcome, 'prepared'); assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 0);
-  const retried = await f.run(['ingest', f.file, '--request-key', result.request_key, '--wait', '--json']); assert.equal(retried.code, 0); assert.equal(JSON.parse(retried.stdout).wait.outcome, 'completed');
-});
-test('watch persists unknown acceptance across process restart and key rotation without resending pending bytes', async t => {
-  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); await fs.writeFile(path.join(folder, 'chat.json'), JSON.stringify({ messages: [{ content: 'synthetic fixture' }] }));
-  f.state.loseContent = true; f.state.failLookup = 1;
-  const first = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(first.code, 1); assert.equal(JSON.parse(first.stdout).outcomes[0].outcome, 'unconfirmed');
-  const stateFile = (await fs.readdir(path.join(f.dir, 'config'))).find(name => /^watch-.*\.json$/.test(name));
-  const saved = JSON.parse(await fs.readFile(path.join(f.dir, 'config', stateFile), 'utf8')); assert(Object.values(saved.entries)[0].pending.request_key);
-  f.env.OPENCRAWL_API_KEY = 'oc_synthetic_rotated_same_account';
-  const second = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(second.code, 0); assert.equal(JSON.parse(second.stdout).outcomes[0].outcome, 'accepted');
-  const third = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']); assert.equal(third.code, 0); assert.equal(JSON.parse(third.stdout).outcomes.length, 0);
-  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1); assert.equal(f.calls.filter(call => call.path.endsWith('/prepare')).length, 1);
-});
-test('interrupt aborts an in-flight job request locally and keeps exact ID', async t => {
-  const f = await fixture(t); f.state.holdJob = true;
-  const result = await f.run(['jobs', 'wait', jobId, '--json'], child => {
-    const poll = setInterval(() => { if (f.calls.some(call => call.path === `/api/v1/jobs/${jobId}`)) { clearInterval(poll); child.kill('SIGINT'); } }, 10);
-    child.on('exit', () => clearInterval(poll));
-  });
-  assert.equal(result.code, 1); assert.equal(result.signal, null); assert.equal(JSON.parse(result.stdout).job_id, jobId); assert.equal(JSON.parse(result.stdout).wait.outcome, 'interrupted'); assert.equal(f.calls.length, 1);
-});
-test('strict flags fail with usable JSON and no network; human search escapes controls while JSON preserves bytes', async t => {
-  const f = await fixture(t);
-  for (const args of [['ingest', f.file, '--timeout', '1'], ['ingest', f.file, '--wait=false'], ['ingest', f.file, '--wait', '--wait'], ['jobs', 'wait', jobId, '--retry'], ['ingest', f.file, '--surprise', 'x']]) {
-    const result = await f.run([...args, '--json']); assert.equal(result.code, 1); assert(JSON.parse(result.stdout).error);
-  }
-  assert.equal(f.calls.length, 0);
-  const human = await f.run(['search', 'test']); assert.equal(human.code, 0); assert(!human.stdout.includes('\u001b')); assert(human.stdout.includes('\\u001b')); assert(human.stdout.includes('keep\nline'));
-  const json = await f.run(['search', 'test', '--json']); assert.equal(JSON.parse(json.stdout).results[0].title, 'bad\u001b[2J');
-  const text = path.join(f.dir, 'note.txt'); await fs.writeFile(text, 'synchronous note');
-  const inline = await f.run(['import', text, '--wait', '--json']); assert.equal(inline.code, 0); assert.equal(JSON.parse(inline.stdout).id, 'doc_' + 'c'.repeat(32)); assert(!JSON.parse(inline.stdout).job_id);
-});
-
-
-test('watch treats changed explicit source or visibility as a separate intent and preserves old pending scope', async t => {
-  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); await fs.writeFile(path.join(folder, 'chat.json'), '{"content":"synthetic fixture"}');
-  f.state.losePrepare = true;
-  const first = await f.run(['watch', folder, '--provider', 'generic', '--source', 'src_private', '--once', '--json']);
-  assert.equal(first.code, 1);
-  const next = await f.run(['watch', folder, '--provider', 'generic', '--source', 'src_shared', '--visibility', 'workspace', '--once', '--json']);
-  assert.equal(next.code, 0);
-  const states = await Promise.all((await fs.readdir(path.join(f.dir, 'config'))).filter(name => name.endsWith('.json')).map(name => fs.readFile(path.join(f.dir, 'config', name), 'utf8').then(JSON.parse)));
-  assert.equal(states.length, 2);
-  assert(Object.values(states.find(value => value.scope.source_id === 'src_private').entries)[0].pending);
-  assert(Object.values(states.find(value => value.scope.source_id === 'src_shared').entries)[0].accepted);
-  assert.equal(f.calls.filter(call => call.path.endsWith('/prepare')).length, 2);
-  assert.equal(f.calls.filter(call => call.path.endsWith('/content')).length, 1);
-});
-
-test('unscoped legacy watcher state is preserved and blocks automatic migration or new uploads', async t => {
-  const f = await fixture(t); const folder = path.join(f.dir, 'exports'); await fs.mkdir(folder); const file = path.join(folder, 'chat.json'); await fs.writeFile(file, '{"content":"synthetic fixture"}');
-  const root = await fs.realpath(folder), identity = crypto.createHash('sha256').update(f.env.OPENCRAWL_API_KEY).digest('hex').slice(0,16);
-  const name = crypto.createHash('sha256').update(`${f.env.OPENCRAWL_URL}:${identity}:${root}:generic`).digest('hex').slice(0,24);
-  const config = path.join(f.dir, 'config'); await fs.mkdir(config);
-  const legacy = path.join(config, `watch-${name}.json`), original = JSON.stringify({ [file]: '25:1' }); await fs.writeFile(legacy, original);
-  const result = await f.run(['watch', folder, '--provider', 'generic', '--once', '--json']);
-  assert.equal(result.code, 1); assert(JSON.parse(result.stdout).error.message.includes('Older watcher state'));
-  assert.equal(await fs.readFile(legacy, 'utf8'), original);
-  assert(!f.calls.some(call => call.method === 'POST')); assert.equal((await fs.readdir(config)).length, 1);
-});

package/src/uploads.mjs

@@ -1,141 +0,0 @@
-import fs from 'node:fs/promises';
-import { createReadStream, openAsBlob } from 'node:fs';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { allowedOptions, identifier, requestFailure, waitForJob, validateJob, withInterrupt, waitOptions, renderJob } from './jobs.mjs';
-import { displayLine } from './display.mjs';
-
-export const UPLOADS_HELP = `OpenCrawl · recover an upload receipt
-
-  opencrawl uploads show RECEIPT_ID [--json]
-  opencrawl uploads lookup REQUEST_KEY [--json]
-  opencrawl ingest FILE [--wait] [--timeout 300] [--request-key UUID]
-  opencrawl import FILE [--provider P] [--wait] [--timeout 300] [--request-key UUID]
-
-Every queued upload prints its request key before sending. Keep that key if the
-connection drops. Lookup is read-only and never resends a file. To explicitly
-retry a prepared/retryable receipt, rerun the original command with the same
-file, source, visibility, provider, and --request-key. Changed details are rejected.
-Accepted receipts return their existing job; receiving receipts are only checked.
-Expired receipts require an explicit new upload without the old request key.
---wait follows acceptance and job completion for at most 1–3600 seconds (default
-300). Ctrl+C stops local waiting. No automatic retry, reupload, or cancellation.
-Upload options: --source ID --visibility private|workspace (default private)
-Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
-`;
-export const PROVIDERS = ['auto', 'chatgpt', 'claude', 'claude-code', 'codex', 'generic'];
-const STATES = new Set(['prepared', 'receiving', 'retryable', 'accepted', 'expired']);
-export function requestKey(value) {
-  if (typeof value !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error('--request-key must be the UUID printed before the original upload.');
-  return value.toLowerCase();
-}
-export function validateReceipt(value, { key, id } = {}) {
-  if (!value || typeof value.id !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(value.id) || !STATES.has(value.state) || id && value.id !== id || key && value.request_key !== key || !['upload', 'chat_import'].includes(value.operation)) throw new Error('Server returned an invalid upload receipt. No file was resent.');
-  requestKey(value.request_key);
-  if (value.state === 'accepted') validateJob(value.job);
-  return value;
-}
-export async function lookupReceipt(client, key, options = {}) {
-  key = requestKey(key);
-  return validateReceipt(await client.request(`/api/v1/uploads/receipts/by-key/${key}`, { maxResponseBytes: 512 * 1024, ...options }), { key });
-}
-export async function fileDescriptor(file, { chat = false, provider = 'auto', source, visibility = 'private' } = {}) {
-  if (!['private', 'workspace'].includes(visibility)) throw new Error('--visibility must be private or workspace.');
-  if (!PROVIDERS.includes(provider)) throw new Error('Choose a supported --provider from import --help.');
-  if (source !== undefined) identifier(source, 'source');
-  const before = await fs.stat(file); const filename = path.basename(file);
-  if (!before.isFile() || !before.size || before.size > 200 * 1024 * 1024) throw new Error('Upload a nonempty regular file of at most 200 MB.');
-  if (/[\\\u0000-\u001f\u007f]/u.test(filename)) throw new Error('Use a filename without backslashes or control characters.');
-  if (Array.from(filename).length > 200) throw new Error('The upload filename must be at most 200 characters.');
-  const hash = crypto.createHash('sha256');
-  let size = 0;
-  for await (const chunk of createReadStream(file)) {
-    size += chunk.length;
-    if (size > before.size) throw new Error('The file grew while its checksum was being read. Wait for it to finish writing, then explicitly import it again.');
-    hash.update(chunk);
-  }
-  const after = await fs.stat(file);
-  if (['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs'].some(key => before[key] !== after[key])) throw new Error('The file changed while its checksum was being read. Wait for it to finish writing, then explicitly import it again.');
-  return { operation: chat ? 'chat_import' : 'upload', filename, size_bytes: before.size, sha256: hash.digest('hex'), source_id: source || null, visibility, ...(chat ? { provider } : {}) };
-}
-const snapshot = (key, receipt, error) => ({ request_key: key, receipt_id: receipt?.id || null, receipt: receipt || null,
-  job_id: receipt?.job?.id || null, job: receipt?.job || null, outcome: receipt?.state || 'unconfirmed', ...(error ? { error } : {}) });
-
-/** Sends file bytes at most once. A failed response is reconciled only by GET. */
-export async function submitUpload(file, client, options = {}, { onIntent = async () => {}, onReceipt = async () => {}, stderr = text => process.stderr.write(text), signal } = {}) {
-  const descriptor = await fileDescriptor(file, options);
-  const key = options['request-key'] === undefined ? crypto.randomUUID() : requestKey(options['request-key']);
-  await onIntent(key, descriptor);
-  stderr(`Upload request key: ${key}\n`);
-  let receipt;
-  const recover = async error => {
-    let confirmed = false;
-    try { receipt = await lookupReceipt(client, key, { timeout: 10000, signal }); confirmed = true; await onReceipt(receipt); }
-    catch { /* Acceptance remains unknown; keep the key rather than retransmitting. */ }
-    return { ...snapshot(key, receipt, requestFailure(error)), ...(!confirmed && receipt?.state !== 'accepted' ? { outcome: 'unconfirmed' } : {}) };
-  };
-  try {
-    // Idempotent prepare validates the entire descriptor even for an existing key.
-    receipt = validateReceipt(await client.request('/api/v1/uploads/prepare', { method: 'POST', body: { request_key: key, ...descriptor }, timeout: 30000, signal, maxResponseBytes: 512 * 1024 }), { key });
-    await onReceipt(receipt);
-  } catch (error) {
-    // A definite descriptor conflict must not be disguised as successful reuse.
-    if (Number.isInteger(error.status) && error.status >= 400 && error.status < 500) return snapshot(key, null, requestFailure(error));
-    return recover(error);
-  }
-  if (receipt.state === 'accepted' || receipt.state === 'receiving' || receipt.state === 'expired') return snapshot(key, receipt);
-  const route = `/api/v1/uploads/receipts/${encodeURIComponent(receipt.id)}/content`;
-  if (receipt.upload_url !== route) return snapshot(key, receipt, { code: 'invalid_upload_route', message: 'The receipt returned an unexpected upload route. No file was sent.' });
-  try {
-    const form = new FormData();
-    const type = /\.zip$/i.test(file) ? 'application/zip' : /\.json$/i.test(file) ? 'application/json' : /\.jsonl$/i.test(file) ? 'application/x-ndjson' : 'application/octet-stream';
-    form.set('file', await openAsBlob(file, { type }), descriptor.filename);
-    receipt = validateReceipt(await client.request(route, { method: 'POST', body: form, timeout: 300000, signal, maxResponseBytes: 512 * 1024 }), { key, id: receipt.id });
-    await onReceipt(receipt);
-    return snapshot(key, receipt);
-  } catch (error) { return recover(error); }
-}
-
-export async function followUpload(result, client, { timeout = 300, signal, now = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) {
-  const deadline = now() + timeout * 1000;
-  while (!result.job_id && result.receipt?.state === 'receiving') {
-    if (signal?.aborted || now() >= deadline) return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : 'timeout' } };
-    const delay = Number.isInteger(result.receipt.retry_after_seconds) ? Math.max(1, Math.min(10, result.receipt.retry_after_seconds)) * 1000 : 2000;
-    const next = Math.min(deadline, now() + delay);
-    while (!signal?.aborted && now() < next) await sleep(Math.min(100, next - now()));
-    if (signal?.aborted || now() >= deadline) continue;
-    try { result = snapshot(result.request_key, await lookupReceipt(client, result.request_key, { timeout: Math.max(1, Math.min(30000, Math.ceil(deadline - now()))), signal })); }
-    catch (error) { return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : now() >= deadline ? 'timeout' : 'unavailable', error: requestFailure(error) } }; }
-  }
-  if (!result.job_id) return { ...result, wait: { outcome: signal?.aborted ? 'interrupted' : 'not_accepted' } };
-  const waited = await waitForJob(client, result.job_id, { timeout, signal, now, sleep, deadline, initial: result.job });
-  return { ...result, ...waited };
-}
-export function renderUpload(result) {
-  let text = `Upload request key: ${displayLine(result.request_key)}\n`;
-  if (result.receipt_id) text += `Receipt: ${displayLine(result.receipt_id)}\n`;
-  text += `Upload: ${displayLine(result.outcome)}\n`;
-  if (result.job) text += renderJob(result.job);
-  if (result.wait) text += `Wait: ${displayLine(result.wait.outcome)}. No job was cancelled or resubmitted.\n`;
-  if (result.receipt?.error?.message) text += `Receipt issue: ${displayLine(result.receipt.error.message)}\n`;
-  if (result.error || result.wait?.error) text += displayLine((result.wait?.error || result.error).message) + '\n';
-  if (result.outcome !== 'accepted') text += `Inspect with: opencrawl uploads lookup ${displayLine(result.request_key)}\n`;
-  if (['prepared', 'retryable'].includes(result.outcome)) text += 'Retry only explicitly with the original file and scope plus --request-key.\n';
-  if (result.outcome === 'receiving') text += 'The server is still receiving or reconciling this request. Lookup only; do not upload it again.\n';
-  if (result.outcome === 'expired') text += 'This receipt expired. Start a new upload explicitly if it is still needed.\n';
-  return text;
-}
-export function uploadExitCode(result, wait) { return wait ? result.wait?.outcome === 'completed' ? 0 : 1 : result.outcome === 'accepted' ? 0 : 1; }
-export async function finishUpload(result, options, client, { stdout = text => process.stdout.write(text) } = {}) {
-  if (options.wait) result = await withInterrupt(signal => followUpload(result, client, { timeout: waitOptions(options), signal }));
-  stdout(options.json ? JSON.stringify(result, null, 2) + '\n' : renderUpload(result));
-  return uploadExitCode(result, options.wait);
-}
-export async function uploadsCommand(rest, options, client, { stdout = text => process.stdout.write(text) } = {}) {
-  const [command, id, ...extra] = rest;
-  allowedOptions(options, [], 'uploads');
-  if (!['show', 'lookup'].includes(command) || extra.length) throw new Error('Choose uploads show RECEIPT_ID or uploads lookup REQUEST_KEY.');
-  const receipt = command === 'lookup' ? await lookupReceipt(client, id) : validateReceipt(await client.request(`/api/v1/uploads/receipts/${encodeURIComponent(identifier(id, 'receipt'))}`, { maxResponseBytes: 512 * 1024 }), { id });
-  stdout(options.json ? JSON.stringify(receipt, null, 2) + '\n' : renderUpload(snapshot(receipt.request_key, receipt)));
-  return 0;
-}

package/src/uploads.test.mjs

@@ -1,64 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import crypto from 'node:crypto';
-import { submitUpload, fileDescriptor, followUpload, uploadExitCode, requestKey } from './uploads.mjs';
-
-const key = 'abcd1234-1234-4234-8234-abcd12345678', id = 'receipt_' + 'b'.repeat(32), jobId = 'job_' + 'a'.repeat(32);
-const receipt = (state, overrides = {}) => ({ id, request_key: key, state, operation: 'upload', upload_url: `/api/v1/uploads/receipts/${id}/content`, job: state === 'accepted' ? { id: jobId, kind: 'upload', status: 'queued' } : null, ...overrides });
-async function fixture(t) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'opencrawl-upload-')); t.after(() => fs.rm(dir, { recursive: true, force: true })); const file = path.join(dir, 'binary.pdf'); await fs.writeFile(file, Buffer.from([1,2,0,255,127])); return file; }
-const quiet = { stderr: () => {} };
-test('upload hashes exact bytes then sends one multipart file and no duplicated scope fields', async t => {
-  const file = await fixture(t); const events = [];
-  const client = { request: async (route, options) => {
-    events.push(route);
-    if (route.endsWith('/prepare')) { assert.equal(options.body.sha256, crypto.createHash('sha256').update(await fs.readFile(file)).digest('hex')); assert.equal(options.body.visibility, 'workspace'); assert.equal(options.body.source_id, 'src_test'); return receipt('prepared'); }
-    assert.deepEqual([...options.body.keys()], ['file']); assert.deepEqual(Buffer.from(await options.body.get('file').arrayBuffer()), await fs.readFile(file)); return receipt('accepted');
-  } };
-  const result = await submitUpload(file, client, { 'request-key': key, source: 'src_test', visibility: 'workspace' }, { ...quiet, onIntent: async () => { events.push('persist-intent'); } });
-  assert.equal(events[0], 'persist-intent'); assert.equal(events.length, 3); assert.equal(result.job_id, jobId); assert.equal(uploadExitCode(result, false), 0);
-});
-test('accepted, receiving, and expired prepare receipts never resend file bytes', async t => {
-  const file = await fixture(t);
-  for (const state of ['accepted','receiving','expired']) {
-    const calls = [];
-    const result = await submitUpload(file, { request: async route => { calls.push(route); return receipt(state); } }, { 'request-key': key }, quiet);
-    assert.deepEqual(calls, ['/api/v1/uploads/prepare']); assert.equal(result.outcome, state);
-  }
-});
-test('lost prepare response only looks up, even when lookup says prepared', async t => {
-  const file = await fixture(t); const calls = [];
-  const result = await submitUpload(file, { request: async route => { calls.push(route); if (calls.length === 1) throw Error('dropped'); return receipt('prepared'); } }, { 'request-key': key }, quiet);
-  assert.deepEqual(calls, ['/api/v1/uploads/prepare', `/api/v1/uploads/receipts/by-key/${key}`]); assert.equal(result.outcome, 'prepared'); assert.equal(uploadExitCode(result, false), 1);
-});
-test('lost acceptance resolves to original job by lookup without resending', async t => {
-  const file = await fixture(t); const calls = [];
-  const result = await submitUpload(file, { request: async route => { calls.push(route); if (route.endsWith('/prepare')) return receipt('prepared'); if (route.endsWith('/content')) throw Error('dropped acceptance'); return receipt('accepted'); } }, { 'request-key': key }, quiet);
-  assert.equal(calls.length, 3); assert.equal(result.job_id, jobId); assert.equal(result.outcome, 'accepted'); assert.equal(calls.filter(route => route.endsWith('/content')).length, 1);
-});
-test('descriptor conflicts do not treat the old accepted job as a new successful upload', async t => {
-  const file = await fixture(t); let calls = 0;
-  const result = await submitUpload(file, { request: async () => { calls++; const error = Error('private server body'); error.status = 409; throw error; } }, { 'request-key': key }, quiet);
-  assert.equal(calls, 1); assert.equal(result.request_key, key); assert.equal(result.job_id, null); assert.equal(result.error.status, 409); assert(!JSON.stringify(result).includes('private server'));
-});
-test('unknown outcomes retain request key and receipt ID; foreign upload routes never receive bytes', async t => {
-  const file = await fixture(t); const result = await submitUpload(file, { request: async route => { if (route.endsWith('/prepare')) return receipt('prepared'); throw Error('network'); } }, { 'request-key': key }, quiet);
-  assert.equal(result.request_key, key); assert.equal(result.receipt_id, id); assert.equal(result.job_id, null); assert.equal(result.outcome, 'unconfirmed'); assert.equal(result.receipt.state, 'prepared');
-  let calls = 0; const foreign = await submitUpload(file, { request: async () => { calls++; return receipt('prepared', { upload_url: 'https://other.example/upload' }); } }, { 'request-key': key }, quiet);
-  assert.equal(calls, 1); assert.equal(foreign.error.code, 'invalid_upload_route');
-});
-test('receiving wait checks receipt then existing job with one bounded deadline', async () => {
-  let time = 0; const calls = [];
-  const result = await followUpload({ request_key: key, receipt_id: id, receipt: receipt('receiving'), job_id: null, outcome: 'receiving' }, { request: async route => { calls.push(route); return route.includes('/receipts/') ? receipt('accepted') : { id: jobId, status: 'completed', kind: 'upload' }; } }, { timeout: 3, now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(result.wait.outcome, 'completed'); assert.deepEqual(calls, [`/api/v1/uploads/receipts/by-key/${key}`, `/api/v1/jobs/${jobId}`]);
-  const timeout = await followUpload({ request_key: key, receipt: receipt('receiving'), outcome: 'receiving' }, { request: async () => { throw Error('not reached'); } }, { timeout: 1, now: () => time, sleep: async ms => { time += ms; } });
-  assert.equal(timeout.wait.outcome, 'timeout'); assert.equal(timeout.request_key, key);
-});
-test('file validation and failed durable intent precede all network activity', async t => {
-  const file = await fixture(t); let calls = 0; const client = { request: async () => { calls++; } };
-  await assert.rejects(submitUpload(file, client, {}, { ...quiet, onIntent: async () => { throw Error('disk full'); } }), /disk full/);
-  await assert.rejects(fileDescriptor(file, { visibility: 'public' }), /visibility/);
-  assert.throws(() => requestKey('invalid'), /UUID/); await fs.writeFile(file, ''); await assert.rejects(fileDescriptor(file), /nonempty/); assert.equal(calls, 0);
-});