@getopencrawl/cli 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +75 -298
  2. package/bin/opencrawl.cjs +168 -0
  3. package/package.json +20 -14
  4. package/bin/opencrawl.mjs +0 -553
  5. package/browser.mjs +0 -166
  6. package/src/accounts.test.mjs +0 -223
  7. package/src/agent.mjs +0 -95
  8. package/src/agent.test.mjs +0 -58
  9. package/src/answer-contract.mjs +0 -112
  10. package/src/answer-view.mjs +0 -118
  11. package/src/browser.test.mjs +0 -72
  12. package/src/capture.mjs +0 -273
  13. package/src/capture.test.mjs +0 -171
  14. package/src/client.mjs +0 -183
  15. package/src/client.test.mjs +0 -34
  16. package/src/connect.mjs +0 -230
  17. package/src/connect.test.mjs +0 -131
  18. package/src/context.mjs +0 -34
  19. package/src/context.test.mjs +0 -32
  20. package/src/display.mjs +0 -4
  21. package/src/fixtures/letter-connections.json +0 -59
  22. package/src/fixtures/product-monitor-contract.json +0 -312
  23. package/src/fixtures/research-completion-hosted.json +0 -502
  24. package/src/fixtures/research-completion-v1.json +0 -58
  25. package/src/fixtures/research-completion-v2.json +0 -33
  26. package/src/fixtures/research-presentation-hosted.json +0 -720
  27. package/src/jobs.mjs +0 -117
  28. package/src/jobs.test.mjs +0 -67
  29. package/src/mcp-conversations.test.mjs +0 -80
  30. package/src/monitors.mjs +0 -146
  31. package/src/monitors.test.mjs +0 -62
  32. package/src/product-letter-capabilities.mjs +0 -71
  33. package/src/product-letter-read.mjs +0 -178
  34. package/src/product-monitor-login.test.mjs +0 -140
  35. package/src/product-monitors.mjs +0 -321
  36. package/src/product-monitors.test.mjs +0 -69
  37. package/src/research-agent-errors.test.mjs +0 -110
  38. package/src/research-agent-feedback.test.mjs +0 -28
  39. package/src/research-agent.mjs +0 -164
  40. package/src/research-agent.test.mjs +0 -231
  41. package/src/research-answer-contract.mjs +0 -3
  42. package/src/research-answer-contract.test.mjs +0 -14
  43. package/src/research-answer-view.mjs +0 -7
  44. package/src/research-answer-view.test.mjs +0 -27
  45. package/src/research-completion.mjs +0 -94
  46. package/src/research-completion.test.mjs +0 -110
  47. package/src/research-event.mjs +0 -39
  48. package/src/research-event.test.mjs +0 -57
  49. package/src/research-presentation.test.mjs +0 -56
  50. package/src/research.mjs +0 -141
  51. package/src/research.test.mjs +0 -97
  52. package/src/sessions.mjs +0 -165
  53. package/src/sessions.test.mjs +0 -151
  54. package/src/tasks.mjs +0 -115
  55. package/src/tasks.test.mjs +0 -83
  56. package/src/upload-commands.test.mjs +0 -140
  57. package/src/uploads.mjs +0 -141
  58. package/src/uploads.test.mjs +0 -64
@@ -1,97 +0,0 @@
1
- import test from 'node:test';
2
- import assert from 'node:assert/strict';
3
- import fs from 'node:fs/promises';
4
- import os from 'node:os';
5
- import path from 'node:path';
6
- import http from 'node:http';
7
- import {spawn} from 'node:child_process';
8
- import {fileURLToPath} from 'node:url';
9
- import {researchRequest,researchCommand,waitResearch,validateResearchJob,validateResearchSettings,validateConfigureReceipt} from './research.mjs';
10
- import {loggedInConfig} from './client.mjs';
11
- const collection={id:'rcol_'+ 'a'.repeat(32),source_id:'src_'+'a'.repeat(32),name:'Synthetic research',brief:'Public product pages',status:'active',can_manage:true,seed_urls:['https://example.com/'],allowed_origins:['https://example.com'],sitemap_urls:[],page_limit:100,schedule_minutes:0,include_media:false,revision:1,created_at:'2026-10-23T00:00:00Z',updated_at:'2026-10-23T00:00:00Z',last_checked_at:null,stats:{indexed:1,discovered:2,failed:1,bytes:123}};
12
- const job={id:'job_'+'b'.repeat(32),collection_id:collection.id,status:'partial',phase:'finished',created_at:'2026-10-23T00:00:00Z',started_at:null,finished_at:null,progress:{discovered:2,fetched:2,indexed:1,failed:1,bytes:123},coverage:{site_complete:false},error:null};
13
- const search={query:'product',results:[{document_id:'doc_test',title:'Product evidence',source_id:collection.source_id}],context:'[S1] product evidence\né🌲\n',evidence:{citations:[{document_id:'doc_test'}]},freshness:{status:'indexed',refresh_jobs:[]}};
14
- const key='research-test-key-0001';
15
- async function execute(rest,options,response){let stdout='',stderr='';const calls=[];const code=await researchCommand(rest,options,{request:async(route,params)=>{calls.push({route,params});if(response instanceof Error)throw response;return response}},{stdout:value=>stdout+=value,stderr:value=>stderr+=value});return {stdout,stderr,code,calls};}
16
- test('research commands require exact scope, revision and stable mutation key',()=>{assert.deepEqual(researchRequest(['search','release','notes'],{collection:collection.id}).body,{query:'release notes',collection_ids:[collection.id],freshness:'indexed',limit:10,budget:6000});assert.deepEqual(researchRequest(['pause',collection.id],{revision:'3'}).body,{status:'paused',expected_revision:3});assert.equal(researchRequest(['refresh',collection.id],{'request-key':key}).key,key);for(const [rest,options] of [[['search','x'],{}],[['refresh',collection.id],{}],[['pause',collection.id],{}],[['show','../secret'],{}],[['list','extra'],{}],[['events',collection.id],{limit:'100'}],[['search','x'],{collection:collection.id,freshness:'refresh'}],[['job',job.id],{timeout:'1'}]])assert.throws(()=>researchRequest(rest,options));});
17
- test('opaque event cursor is encoded as data on the selected collection route',()=>{const value=researchRequest(['events',collection.id],{cursor:'opaque&url=https://foreign.example'});const url=new URL(value.route,'https://example.com');assert.equal(url.searchParams.get('cursor'),'opaque&url=https://foreign.example');assert.equal(url.origin,'https://example.com');});
18
- test('query output is exact evidence and empty responses never broaden or refresh',async()=>{const result=await execute(['search','product'],{collection:collection.id},search);assert.equal(result.stdout,search.context);assert.equal(result.calls.length,1);assert.deepEqual(result.calls[0].params.body.collection_ids,[collection.id]);const empty=await execute(['search','none'],{collection:collection.id},{...search,query:'none',results:[],context:'',evidence:{citations:[]},freshness:{status:'empty',refresh_jobs:[]}});assert.equal(empty.stdout,'');assert.match(empty.stderr,/No matching indexed evidence/);});
19
- test('terminal partial job is distinct from success and remains recoverable',async()=>{const result=await execute(['job',job.id],{json:true},job);assert.equal(result.code,2);assert.equal(JSON.parse(result.stdout).job.status,'partial');assert.throws(()=>validateResearchJob({...job,status:'done'}));});
20
- test('bounded waits stop locally and never replay a mutation',async()=>{let time=0,calls=0;const client={request:async()=>{calls++;return {...job,status:'running',phase:'fetching'}}};const result=await waitResearch(client,{...job,status:'queued',phase:'discovery'},{timeout:1,now:()=>time,sleep:async ms=>{time+=ms}});assert.equal(result.wait.outcome,'timeout');assert.equal(result.job.id,job.id);assert.equal(calls,1);const controller=new AbortController();controller.abort();const stopped=await waitResearch(client,{...job,status:'queued'},{signal:controller.signal});assert.equal(stopped.wait.outcome,'interrupted');assert.equal(calls,1);});
21
- test('unknown refresh outcome keeps key with fixed safe errors and no retry',async()=>{for(const error of [new Error('RAW SECRET'),Object.assign(new Error('RAW SECRET'),{status:409}),Object.assign(new Error('RAW SECRET'),{status:401})]){const result=await execute(['refresh',collection.id],{'request-key':key,json:true},error);assert.equal(result.code,1);assert.equal(result.calls.length,1);assert.equal(result.calls[0].params.idempotencyKey,key);assert.equal(JSON.parse(result.stdout).error.request_key,key);assert.ok(!result.stdout.includes('SECRET'));}});
22
- test('observation display preserves uncertainty and escapes terminal controls',async()=>{const event={id:'event',collection_id:collection.id,url:'https://example.com/\u001b[2J',kind:'changed',observed_at:'2026-10-23',previous_observed_at:'2026-10-22',before:{title:'Old',excerpt:'Old\u001btext'},after:{title:'New',excerpt:'New'},gap:true,change_summary:{added:[],removed:[],truncated:true}};const result=await execute(['events',collection.id],{}, {items:[event],next_cursor:null});assert.ok(!result.stdout.includes('\u001b'));assert.match(result.stdout,/\\u001b/);assert.match(result.stdout,/^event\tchanged/);assert.match(result.stderr,/exact change time is unknown/);assert.match(result.stderr,/not new business events/);await assert.rejects(execute(['events',collection.id],{}, {items:[{...event,collection_id:'foreign'}],next_cursor:null}));});
23
- const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
24
- function run(args,env,cwd){return new Promise((resolve,reject)=>{const child=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>child.kill('SIGKILL'),10000);child.stdout.on('data',value=>stdout+=value);child.stderr.on('data',value=>stderr+=value);child.on('error',reject);child.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr})})});}
25
- test('actual CLI keeps account-origin binding and keyed recovery headers',async()=>{const folder=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-research-'));const calls=[];const server=http.createServer(async(req,res)=>{let raw='';for await(const part of req)raw+=part;const body=raw?JSON.parse(raw):null;calls.push({url:req.url,authorization:req.headers.authorization,key:req.headers['idempotency-key'],body,method:req.method});res.setHeader('Content-Type','application/json');res.end(JSON.stringify(req.url==='/api/research/collections'?{collection,job,replayed:true}:req.url.startsWith('/api/research/collections?')?{items:[collection],next_cursor:null}:req.url.endsWith('/refresh')?{job,replayed:true}:req.url==='/api/research/search'?search:collection));});await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const url='http://127.0.0.1:'+server.address().port;const env={...process.env};for(const name of Object.keys(env))if(name.startsWith('OPENCRAWL_'))delete env[name];env.OPENCRAWL_CONFIG_DIR=folder;const config=loggedInConfig({},url,'synthetic-pinned-key',{user:{id:'usr_'+'c'.repeat(32)},workspace:{id:'ws_'+'d'.repeat(32)}});await fs.writeFile(path.join(folder,'config.json'),JSON.stringify(config),{mode:0o600});const input=path.join(folder,'collection.json');await fs.writeFile(input,JSON.stringify({name:'Synthetic research',seed_urls:['https://example.com/'],schedule_minutes:0}));try{const help=await run(['research','--help'],env,folder);assert.equal(help.code,0);assert.equal(calls.length,0);const created=await run(['research','create','--file',input,'--request-key',key,'--json','--account',config.activeProfile],env,folder);assert.equal(created.code,0,created.stderr);assert.equal(JSON.parse(created.stdout).job.id,job.id);assert.equal(calls.at(-1).key,key);assert.equal(calls.at(-1).authorization,'Bearer synthetic-pinned-key');const before=calls.length;for(const extra of [['--request-key',key,'--request-key','different-key-000'],['--json=false'],['--source','other']]){const value=await run(['research','refresh',collection.id,...extra,'--account',config.activeProfile],env,folder);assert.equal(value.code,1);}assert.equal(calls.length,before);const wrong=await run(['research','list','--account',config.activeProfile,'--server','https://foreign.example'],env,folder);assert.equal(wrong.code,1);assert.equal(calls.length,before);const paused=await run(['research','pause',collection.id,'--revision','1','--json'],env,folder);assert.equal(paused.code,0,paused.stderr);assert.equal(calls.at(-1).method,'PATCH');assert.deepEqual(calls.at(-1).body,{status:'paused',expected_revision:1});assert.equal(calls.at(-1).key,undefined);}finally{server.closeAllConnections();await new Promise(resolve=>server.close(resolve));await fs.rm(folder,{recursive:true,force:true});}});
26
-
27
- test('search context enforces exact normalized query, UTF8 budget and real context citations',async()=>{for(const response of [{...search,query:'another question'},{...search,context:'é'.repeat(4000)},{...search,evidence:{citations:[]}},{...search,evidence:{citations:[{document_id:'foreign'}]}},{...search,results:[{...search.results[0],research_observation:{collection_id:'foreign'}}]}]){let stdout='';await assert.rejects(researchCommand(['search','product'],{collection:collection.id},{request:async()=>response},{stdout:v=>stdout+=v,stderr:()=>{}}));assert.equal(stdout,'');}const result=await execute(['search','product'],{collection:collection.id,budget:'0'},{...search,context:'',evidence:{citations:[]}});assert.equal(result.stdout,'');assert.match(result.stderr,/0 context citations from 1 matching results/);});
28
- test('invalid JSON descriptor never echoes private file snippets',async()=>{const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-research-invalid-')),file=path.join(dir,'input.json');try{await fs.writeFile(file,'{ PRIVATE_EXAMPLE_SECRET');await assert.rejects(researchCommand(['create'],{file,'request-key':key},{request:async()=>{throw new Error('must not send')} }),error=>error.message==='Invalid collection JSON. Supply a valid UTF-8 collection descriptor.');}finally{await fs.rm(dir,{recursive:true,force:true});}});
29
-
30
- const settings={expected_revision:1,page_limit:20,schedule_minutes:60,include_media:false};
31
- const configured={collection:{...collection,...settings,revision:2},next_scheduled_at:'2026-10-24T02:00:00.123456Z',replayed:false};
32
- delete configured.collection.expected_revision;
33
- async function withSettings(value,fn){const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-settings-'));const file=path.join(dir,'settings.json');try{await fs.writeFile(file,typeof value==='string'?value:JSON.stringify(value));return await fn(file);}finally{await fs.rm(dir,{recursive:true,force:true});}}
34
- test('configure requires four strict reviewed settings and disallows job-following flags',()=>{
35
- assert.deepEqual(validateResearchSettings(settings),settings);
36
- for(const field of Object.keys(settings)){const value={...settings};delete value[field];assert.throws(()=>validateResearchSettings(value));}
37
- for(const change of [{page_limit:0},{page_limit:5001},{page_limit:true},{page_limit:'20'},{expected_revision:0},{expected_revision:1.5},{expected_revision:true},{schedule_minutes:1},{schedule_minutes:10081},{schedule_minutes:'60'},{include_media:'false'},{seed_urls:[]},{status:'active'},{name:'Changed'}])assert.throws(()=>validateResearchSettings({...settings,...change}));
38
- for(const cadence of [0,5,10080])assert.equal(validateResearchSettings({...settings,schedule_minutes:cadence}).schedule_minutes,cadence);
39
- for(const option of [{wait:true},{timeout:'5'},{revision:'1'},{source:'foreign'}])assert.throws(()=>researchRequest(['configure',collection.id],{file:'settings.json','request-key':key,...option}));
40
- assert.throws(()=>researchRequest(['configure',collection.id],{file:'settings.json'}));
41
- const request=researchRequest(['configure',collection.id],{file:'settings.json','request-key':key});
42
- assert.equal(request.method,'POST');assert.equal(request.route,`/api/research/collections/${collection.id}/configure`);
43
- });
44
- test('configure file validation finishes before any network request and does not echo content',async()=>{
45
- for(const value of [{...settings,include_media:'true'},{...settings,allowed_origins:['https://foreign.example']},'{ PRIVATE_SECRET',Buffer.from([0xff]).toString()])await withSettings(value,async file=>{
46
- let count=0;await assert.rejects(researchCommand(['configure',collection.id],{file,'request-key':key,json:true},{request:async()=>{count++;return configured}}),error=>!error.message.includes('PRIVATE_SECRET'));assert.equal(count,0);
47
- });
48
- });
49
- test('configure validates changed and no-op scheduling receipts without inventing a job',()=>{
50
- const request={id:collection.id,body:settings};assert.equal(validateConfigureReceipt(configured,request),configured);
51
- assert.equal(validateConfigureReceipt({...configured,replayed:true},request).replayed,true);
52
- const noop={...configured,collection:{...configured.collection,revision:1},next_scheduled_at:null};assert.equal(validateConfigureReceipt(noop,request).next_scheduled_at,null);
53
- const paused={...configured,collection:{...configured.collection,status:'paused'},next_scheduled_at:null};assert.equal(validateConfigureReceipt(paused,request).collection.status,'paused');
54
- assert.equal(validateConfigureReceipt({...configured,collection:{...configured.collection,schedule_minutes:0},next_scheduled_at:null},{id:collection.id,body:{...settings,schedule_minutes:0}}).next_scheduled_at,null);
55
- for(const value of [
56
- {...configured,collection:{...configured.collection,id:'rcol_foreign'}},
57
- {...configured,collection:{...configured.collection,page_limit:21}},
58
- {...configured,collection:{...configured.collection,include_media:true}},
59
- {...configured,collection:{...configured.collection,revision:3}},
60
- {...configured,collection:{...configured.collection,seed_urls:[]}},
61
- {...configured,collection:{...configured.collection,status:'paused'}},
62
- {...configured,next_scheduled_at:null},{...configured,next_scheduled_at:'2026-10-24T02:00:00+00:00'},
63
- {...configured,next_scheduled_at:'2026-02-30T02:00:00Z'}, {...configured,replayed:'true'},
64
- {...configured,job}, {...configured,next_scheduled_at:undefined},
65
- ])assert.throws(()=>validateConfigureReceipt(value,request));
66
- });
67
- test('configure sends one keyed POST and preserves unknown or invalid acceptance without retry',async()=>{
68
- await withSettings(settings,async file=>{
69
- const result=await execute(['configure',collection.id],{file,'request-key':key,json:true},configured);
70
- assert.equal(result.code,0);assert.deepEqual(JSON.parse(result.stdout),configured);assert.equal(result.calls.length,1);
71
- assert.deepEqual(result.calls[0].params.body,settings);assert.equal(result.calls[0].params.idempotencyKey,key);
72
- for(const response of [new Error('RAW SECRET'),Object.assign(new Error('RAW SECRET'),{status:409}),Object.assign(new Error('RAW SECRET'),{status:401}),{...configured,job}]){
73
- const failed=await execute(['configure',collection.id],{file,'request-key':key,json:true},response);
74
- assert.equal(failed.code,1);assert.equal(failed.calls.length,1);const error=JSON.parse(failed.stdout).error;
75
- assert.equal(error.request_key,key);assert.equal(error.id,collection.id);assert.ok(!failed.stdout.includes('SECRET'));
76
- }
77
- const human=await execute(['configure',collection.id],{file,'request-key':key},{...configured,replayed:true});
78
- assert.equal(human.code,0);assert.match(human.stderr,/No immediate crawl/);assert.match(human.stderr,/Recovered the original receipt/);assert.match(human.stdout,/Schedule: 60 minutes/);
79
- });
80
- });
81
- test('installed CLI configure preserves pinned credentials and exact settings on its sole request',async()=>{
82
- const folder=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-configure-cli-')),calls=[];
83
- const server=http.createServer(async(req,res)=>{let raw='';for await(const chunk of req)raw+=chunk;calls.push({url:req.url,method:req.method,key:req.headers['idempotency-key'],authorization:req.headers.authorization,body:JSON.parse(raw)});res.setHeader('content-type','application/json');res.end(JSON.stringify(configured));});
84
- await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const origin='http://127.0.0.1:'+server.address().port;
85
- const env={...process.env};for(const name of Object.keys(env))if(name.startsWith('OPENCRAWL_'))delete env[name];env.OPENCRAWL_CONFIG_DIR=folder;
86
- const config=loggedInConfig({},origin,'synthetic-configure-key',{user:{id:'usr_'+'c'.repeat(32)},workspace:{id:'ws_'+'d'.repeat(32)}});
87
- await fs.writeFile(path.join(folder,'config.json'),JSON.stringify(config),{mode:0o600});const file=path.join(folder,'settings.json');await fs.writeFile(file,JSON.stringify(settings));
88
- try{
89
- const args=['research','configure',collection.id,'--file',file,'--request-key',key,'--json','--account',config.activeProfile];
90
- const result=await run(args,env,folder);assert.equal(result.code,0,result.stderr);assert.deepEqual(JSON.parse(result.stdout),configured);
91
- assert.deepEqual(calls,[{url:`/api/research/collections/${collection.id}/configure`,method:'POST',key,authorization:'Bearer synthetic-configure-key',body:settings}]);
92
- const mismatch=await run([...args,'--server','https://foreign.example'],env,folder);assert.equal(mismatch.code,1);assert.equal(calls.length,1);
93
- await fs.writeFile(file,JSON.stringify({...settings,include_media:1}));const invalid=await run(args,env,folder);assert.equal(invalid.code,1);assert.equal(calls.length,1);
94
- assert.deepEqual(JSON.parse(await fs.readFile(path.join(folder,'config.json'),'utf8')),config);
95
- }finally{server.closeAllConnections();await new Promise(resolve=>server.close(resolve));await fs.rm(folder,{recursive:true,force:true});}
96
- });
97
- test('events can omit volatile-only changes and print changed lines as escaped data',async()=>{const value=researchRequest(['events',collection.id],{'content-only':true});assert.equal(new URL(value.route,'https://example.com').searchParams.get('include_volatile'),'false');assert.equal(new URL(researchRequest(['events',collection.id],{}).route,'https://example.com').searchParams.get('include_volatile'),null);const event={id:'event',collection_id:collection.id,url:'https://example.com/pricing',kind:'changed',observed_at:'2026-10-23',previous_observed_at:'2026-10-22',before:null,after:null,gap:false,change_summary:{added:[],removed:[],truncated:false,changes:[{removed:['Pro $49'],added:['Pro $59\u001b[2J'],volatile:false}],changes_truncated:true,substance:'content'}};const result=await execute(['events',collection.id],{}, {items:[event],next_cursor:null});assert.match(result.stdout,/^- Pro \$49$/m);assert.match(result.stdout,/^\+ Pro \$59\\u001b\[2J$/m);assert.match(result.stderr,/Changed lines are shortened/);const clock=await execute(['events',collection.id],{}, {items:[{...event,change_summary:{...event.change_summary,changes:[],changes_truncated:false,substance:'volatile_only'}}],next_cursor:null});assert.match(clock.stdout,/Only dates, clock times, relative ages or whitespace changed/);});
package/src/sessions.mjs DELETED
@@ -1,165 +0,0 @@
1
- import { displayText, displayLine as line } from './display.mjs';
2
- /** Explicit, version-bound reading of imported conversations. No local history state. */
3
- export const SESSIONS_HELP = `OpenCrawl · read imported conversations
4
-
5
- opencrawl sessions list [--provider P] [--limit 20] [--offset 0]
6
- opencrawl sessions read ID --branch selected|all [--cursor TOKEN] [--limit 30]
7
- opencrawl sessions export ID --revision TOKEN --branch selected|all
8
- --selection tN:START:END [--selection tN:START:END ...] --budget BYTES
9
-
10
- Read prints the current revision and exact selectors for returned fragments.
11
- Copy selectors from read; START is inclusive and END exclusive, as returned by
12
- OpenCrawl. They are document character positions, not bytes or turn numbers.
13
- Each export requires an explicit revision, branch, selection and byte budget.
14
- It exports up to 20 selected ranges, without fetching unseen pages or guessing
15
- whole turns. A changed conversation requires reading it again before export.
16
-
17
- --branch selected follows the exported selected branch when available.
18
- --branch all includes alternate exported branches. Unknown branch information
19
- remains labeled unknown; the server does not infer a hosted conversation state.
20
- --cursor resumes a read page in the same revision and branch; it is opaque.
21
- --budget is 256–32000 UTF-8 bytes for the complete returned context block.
22
-
23
- Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
24
- List providers: auto (all), chatgpt, claude, claude-code, codex, generic
25
- Export stdout contains only the returned context, without an added newline.
26
- Notices and omitted selections go to stderr. --json preserves the full response.
27
- These commands read imported data; they do not resume or post to a hosted chat.
28
- `;
29
-
30
- const PROVIDERS = new Set(['auto', 'chatgpt', 'claude', 'claude-code', 'codex', 'generic']);
31
- const GLOBAL = ['account', 'server', 'config-dir', 'json'];
32
- function number(value, name, minimum, maximum, fallback) {
33
- if (value === undefined && fallback !== undefined) return fallback;
34
- if (typeof value !== 'string' || !/^(0|[1-9]\d*)$/.test(value))
35
- throw new Error(`--${name} must be a decimal integer between ${minimum} and ${maximum}.`);
36
- const result = Number(value);
37
- if (!Number.isSafeInteger(result) || result < minimum || result > maximum)
38
- throw new Error(`--${name} must be between ${minimum} and ${maximum}.`);
39
- return result;
40
- }
41
- function token(value, name, maximum = 256) {
42
- if (typeof value !== 'string' || !value || value.length > maximum || /[\s\u0000-\u001f\u007f]/u.test(value))
43
- throw new Error(`Supply the exact ${name} returned by sessions read.`);
44
- return value;
45
- }
46
- function identifier(value) {
47
- if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,200}$/.test(value))
48
- throw new Error('Supply a conversation document ID from sessions list.');
49
- return value;
50
- }
51
- function branch(value) {
52
- if (!['selected', 'all'].includes(value)) throw new Error('Choose --branch selected or --branch all explicitly.');
53
- return value;
54
- }
55
- export function selections(values) {
56
- if (!Array.isArray(values) || values.length < 1 || values.length > 20)
57
- throw new Error('Supply 1–20 explicit --selection tN:START:END ranges copied from sessions read.');
58
- const ranges = values.map(value => {
59
- const match = typeof value === 'string' && /^(t(?:0|[1-9]\d{0,4})):(0|[1-9]\d*):(0|[1-9]\d*)$/.exec(value);
60
- if (!match || match[1].length > 20) throw new Error('Each --selection must be tN:START:END, exactly as displayed by sessions read.');
61
- const start = number(match[2], 'selection start', 0, 20 * 1024 * 1024);
62
- const end = number(match[3], 'selection end', 1, 20 * 1024 * 1024);
63
- if (end <= start) throw new Error('Selection END must be greater than START. Empty or reversed ranges cannot be exported.');
64
- return { ref: match[1], start_char: start, end_char: end };
65
- });
66
- for (let index = 0; index < ranges.length; index++) {
67
- for (let previous = 0; previous < index; previous++) {
68
- const a = ranges[index], b = ranges[previous];
69
- if (a.start_char < b.end_char && b.start_char < a.end_char)
70
- throw new Error('Selections must not overlap or repeat. Copy distinct ranges from sessions read.');
71
- }
72
- }
73
- return ranges;
74
- }
75
-
76
- export function sessionRequest(rest, options) {
77
- const [command, id, ...extra] = rest;
78
- const local = command === 'list' ? ['provider', 'limit', 'offset'] : command === 'read' ? ['branch', 'cursor', 'limit'] : command === 'export' ? ['branch', 'revision', 'selection', 'budget'] : [];
79
- if (!['list', 'read', 'export'].includes(command)) throw new Error('Choose sessions list, sessions read, or sessions export. Run opencrawl sessions --help.');
80
- for (const key of Object.keys(options)) if (![...GLOBAL, ...local].includes(key)) throw new Error(`Unsupported sessions ${command} option: --${key}. Run opencrawl sessions --help.`);
81
- if (extra.length || command === 'list' && id !== undefined) throw new Error(`Unexpected argument for sessions ${command}. Run opencrawl sessions --help.`);
82
- if (command === 'list') {
83
- if (options.provider !== undefined && !PROVIDERS.has(options.provider)) throw new Error('Choose a supported --provider from sessions --help.');
84
- const query = new URLSearchParams({ limit: number(options.limit, 'limit', 1, 200, 20), offset: number(options.offset, 'offset', 0, 1000000000, 0) });
85
- if (options.provider && options.provider !== 'auto') query.set('provider', options.provider);
86
- return { command, route: '/api/v1/sessions?' + query, options: { maxResponseBytes: 2 * 1024 * 1024 } };
87
- }
88
- const route = `/api/v1/sessions/${encodeURIComponent(identifier(id))}`;
89
- const requestedBranch = branch(options.branch);
90
- if (command === 'read') {
91
- const query = new URLSearchParams({ branch: requestedBranch, limit: number(options.limit, 'limit', 1, 50, 30) });
92
- if (options.cursor !== undefined) query.set('cursor', token(options.cursor, 'cursor'));
93
- return { command, route: route + '/turns?' + query, options: { maxResponseBytes: 2 * 1024 * 1024 } };
94
- }
95
- if (typeof options.revision !== 'string' || !/^[a-f0-9]{64}$/.test(options.revision)) throw new Error('Supply the exact 64-character revision returned by sessions read.');
96
- return { command, route: route + '/context', options: { method: 'POST', maxResponseBytes: 1024 * 1024, body: {
97
- version: token(options.revision, 'revision', 256), branch: requestedBranch,
98
- selections: selections(options.selection), budget: number(options.budget, 'budget', 256, 32000),
99
- } } };
100
- }
101
-
102
- function notice(value, stderr) { if (typeof value === 'string' && value.trim()) stderr(`OpenCrawl: ${line(value)}\n`); }
103
- export function validateExport(data, body) {
104
- if (!data || typeof data.context !== 'string' || data.version !== body.version || data.budget_unit !== 'utf8_bytes' || data.budget !== body.budget ||
105
- !Number.isSafeInteger(data.context_bytes) || data.context_bytes !== Buffer.byteLength(data.context, 'utf8') || data.context_bytes > body.budget)
106
- throw new Error('Server returned an invalid or oversized conversation context. Nothing was exported.');
107
- const key=range=>`${range?.ref}:${range?.start_char}:${range?.end_char}`;
108
- const wanted=new Set(body.selections.map(key));
109
- if (!Array.isArray(data.selected) || data.selected.length !== wanted.size || new Set(data.selected.map(key)).size !== wanted.size || data.selected.some(range=>!wanted.has(key(range)))) throw new Error('Server returned a different conversation selection. Nothing was exported.');
110
- if (!Array.isArray(data.included) || !Array.isArray(data.omitted)) throw new Error('Server returned an invalid conversation context receipt. Nothing was exported.');
111
- const pending=new Map(body.selections.map(range=>[key(range),range]));
112
- const citations=new Set();
113
- const invalid=()=>{throw new Error('Server returned an inconsistent conversation range receipt. Nothing was exported.');};
114
- for (const item of data.included) {
115
- const entry=[...pending].find(([,selected])=>item?.ref===selected.ref && item.start_char===selected.start_char && Number.isSafeInteger(item.end_char) && item.end_char>item.start_char && item.end_char<=selected.end_char);
116
- if (!entry || typeof item.excerpt!=='string' || Array.from(item.excerpt).length!==item.end_char-item.start_char || item.truncated!==(item.end_char!==entry[1].end_char) || typeof item.citation!=='string' || !/^C[1-9]\d{0,2}$/.test(item.citation) || citations.has(item.citation)) invalid();
117
- citations.add(item.citation);pending.delete(entry[0]);
118
- }
119
- for (const item of data.omitted) {
120
- if (!pending.has(key(item)) || typeof item.reason!=='string') invalid();
121
- pending.delete(key(item));
122
- }
123
- if (pending.size || Boolean(data.context)!==Boolean(data.included.length)) invalid();
124
- }
125
-
126
- export async function sessionsCommand(rest, options, client, { stdout = text => process.stdout.write(text), stderr = text => process.stderr.write(text) } = {}) {
127
- const request = sessionRequest(rest, options);
128
- let data;
129
- try { data = await client.request(request.route, request.options); }
130
- catch (error) {
131
- const guidance = { 401: 'Sign in with a valid key for the intended workspace account.', 403: 'This account cannot read this conversation. Select an authorized account with --account.',
132
- 404: 'The conversation is unavailable to this account. Check sessions list and your selected workspace.',
133
- 409: 'The conversation revision or page cursor changed. Run sessions read again, then select ranges from its current revision.',
134
- 422: 'The branch, cursor, revision or selected range is invalid. Copy the current values from sessions read.' }[error.status];
135
- throw new Error(guidance ? `OpenCrawl ${error.status}: ${guidance}` : error.message);
136
- }
137
- if (request.command === 'export') validateExport(data, request.options.body);
138
- if (options.json) { stdout(JSON.stringify(data, null, 2) + '\n');return; }
139
- if (request.command === 'list') {
140
- if (!Array.isArray(data?.items)) throw new Error('Server returned an invalid conversation list.');
141
- if (!data.items.length) { stdout('No imported conversations on this page.\n');return; }
142
- for (const item of data.items) stdout(`${line(item.id)}\t${line(item.metadata?.provider || item.provider || '')}\t${line(item.title)}\n`);
143
- if (data.has_more) stderr(`More conversations are available; use sessions list --offset ${data.offset + data.items.length}.\n`);
144
- return;
145
- }
146
- if (request.command === 'read') {
147
- if (typeof data?.version !== 'string' || !Array.isArray(data.fragments)) throw new Error('Server returned an invalid conversation page.');
148
- stdout(`${line(data.document?.title || 'Conversation')}\nRevision: ${line(data.version)}\nBranch: ${line(data.branch?.requested)} (${line(data.branch?.status)})\n`);
149
- notice(data.branch?.notice, stderr);notice(data.notice, stderr);
150
- for (const fragment of data.fragments) {
151
- if (typeof fragment.text !== 'string' || !Number.isSafeInteger(fragment.start_char) || !Number.isSafeInteger(fragment.end_char)) throw new Error('Server returned an invalid conversation fragment.');
152
- stdout(`\nSelection: ${line(fragment.ref)}:${fragment.start_char}:${fragment.end_char}\n${line(fragment.role)} · ${line(fragment.branch)}${fragment.timestamp ? ' · ' + line(fragment.timestamp) : ''}\n${displayText(fragment.text)}\n`);
153
- if (fragment.truncated) stderr(`Fragment ${line(fragment.ref)} contains only the displayed character range. Check pagination before selecting additional text.\n`);
154
- }
155
- if (data.mode === 'plain') {
156
- stderr('Structured turn selection is unavailable for this imported record. Plain text can be read, but sessions export cannot guess turn ranges.\n');
157
- if (typeof data.plain?.text === 'string') stdout(`\n${displayText(data.plain.text)}\n`);
158
- }
159
- if (data.next_cursor) stderr(`Next cursor: ${line(data.next_cursor)}\nRead the same conversation and branch with --cursor to continue.\n`);
160
- return;
161
- }
162
- for (const item of data.omitted) stderr(`Omitted ${line(item.ref)}:${item.start_char}:${item.end_char}: ${line(item.reason)}. Try fewer ranges or a larger explicit --budget.\n`);
163
- for (const item of data.included) if (item.truncated) stderr(`Included ${line(item.ref)} only through character ${item.end_char}; the full selected range did not fit the byte budget.\n`);
164
- stdout(data.context);
165
- }
@@ -1,151 +0,0 @@
1
- import { test } from 'node:test';
2
- import assert from 'node:assert/strict';
3
- import fs from 'node:fs/promises';
4
- import os from 'node:os';
5
- import path from 'node:path';
6
- import http from 'node:http';
7
- import { spawn } from 'node:child_process';
8
- import { fileURLToPath } from 'node:url';
9
- import { sessionRequest, selections, sessionsCommand } from './sessions.mjs';
10
- import { loggedInConfig, createClient } from './client.mjs';
11
-
12
- const revision='a'.repeat(64);
13
- const options=()=>({revision,branch:'selected',selection:['t0:0:100'],budget:'256'});
14
- const receipt=(context='é'.repeat(128))=>({version:revision,context,context_bytes:Buffer.byteLength(context),budget:256,budget_unit:'utf8_bytes',selected:[{ref:'t0',start_char:0,end_char:100}],included:[{ref:'t0',start_char:0,end_char:Math.min(100,Array.from(context).length),role:'user',branch:'selected',excerpt:Array.from(context).slice(0,100).join(''),truncated:Array.from(context).length<100,citation:'C1'}],omitted:[]});
15
-
16
- test('export requires explicit revision branch selections and byte budget',()=>{
17
- for (const key of ['revision','branch','selection','budget']) { const value=options();delete value[key];assert.throws(()=>sessionRequest(['export','doc_fixture'],value)); }
18
- const request=sessionRequest(['export','doc_fixture'],options());
19
- assert.deepEqual(request.options.body,{version:revision,branch:'selected',selections:[{ref:'t0',start_char:0,end_char:100}],budget:256});
20
- assert.equal(request.route,'/api/v1/sessions/doc_fixture/context');assert.equal(request.options.method,'POST');
21
- assert.equal(request.route.includes(revision),false);
22
- });
23
-
24
- test('malformed, duplicate and overlapping ranges never reach the server',()=>{
25
- for (const value of [['t0:10:10'],['t0:10:9'],['t0:-1:10'],['t0:1e2:200'],['t0:0x10:20'],['t01:0:10'],['t0:0:01'],['t0:0:NaN'],['p0:0:5'],['t100000:0:5'],['t0:0:20971521'],['t0:0:99999999999999999'],['t0:0:10','t0:0:10'],['t0:0:10','t1:9:20'],Array(21).fill('t0:0:1')]) assert.throws(()=>selections(value));
26
- assert.deepEqual(selections(['t0:0:10','t1:10:20']),[{ref:'t0',start_char:0,end_char:10},{ref:'t1',start_char:10,end_char:20}]);
27
- for (const value of ['255','32001','1e3','+256',' 256','0x100']) assert.throws(()=>sessionRequest(['export','doc_fixture'],{...options(),budget:value}));
28
- });
29
-
30
- test('list and read pagination stay explicit and scoped',()=>{
31
- assert.equal(new URL(sessionRequest(['list'],{provider:'auto'}).route,'https://example.test').searchParams.has('provider'),false);
32
- assert.throws(()=>sessionRequest(['export','doc_fixture'],{...options(),revision:'not-a-revision'}));
33
- assert.throws(()=>sessionRequest(['read','doc_fixture'],{branch:'all',cursor:'x'.repeat(257)}));
34
- assert.match(sessionRequest(['list'],{provider:'claude',limit:'20',offset:'40'}).route,/limit=20&offset=40&provider=claude$/);
35
- const request=sessionRequest(['read','doc_fixture'],{branch:'all',cursor:'opaque_value-123',limit:'2'});
36
- const url=new URL(request.route,'https://example.test');assert.equal(url.searchParams.get('cursor'),'opaque_value-123');assert.equal(url.searchParams.get('branch'),'all');
37
- for (const [rest,values] of [[['read','doc_fixture'],{branch:'all',offset:'0'}],[['read','doc_fixture'],{branch:'active'}],[['read','../secret'],{branch:'all'}],[['list','extra'],{}],[['export','doc_fixture'],{...options(),from:'0'}]]) assert.throws(()=>sessionRequest(rest,values));
38
- });
39
-
40
- test('context stdout is exact UTF8 bytes while JSON preserves full receipts',async()=>{
41
- for (const json of [false,true]) {
42
- const output=[],errors=[],data=receipt();
43
- await sessionsCommand(['export','doc_fixture'],{...options(),...(json?{json:true}:{})},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
44
- const text=output.join('');assert.deepEqual(errors,[]);
45
- if (json) assert.deepEqual(JSON.parse(text),data);
46
- else { assert.equal(Buffer.byteLength(text),256);assert.equal(text,data.context);assert.equal(text.endsWith('\n'),false); }
47
- }
48
- });
49
-
50
- test('oversized wrong-revision or changed-selection responses fail before stdout',async()=>{
51
- for (const change of [{context:'é'.repeat(129),context_bytes:258},{context_bytes:1},{version:'different'},{selected:[]},{selected:[{ref:'t1',start_char:0,end_char:100}]},{budget_unit:'tokens'}]) {
52
- const output=[];await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>({...receipt(),...change})},{stdout:value=>output.push(value),stderr:()=>{}}));assert.deepEqual(output,[]);
53
- }
54
- });
55
-
56
- test('canonical server ordering is accepted without changing selected ranges',async()=>{
57
- const selected=[{ref:'t1',start_char:100,end_char:200},{ref:'t0',start_char:0,end_char:100}];
58
- const output=[];await sessionsCommand(['export','doc_fixture'],{...options(),selection:['t1:100:200','t0:0:100']},{request:async()=>({...receipt(),selected:selected.toReversed(),omitted:[{ref:'t1',start_char:100,end_char:200,reason:'Budget'}]})},{stdout:value=>output.push(value),stderr:()=>{}});
59
- assert.equal(output.join(''),receipt().context);
60
- });
61
-
62
- test('partial context reports exact omission and truncation on stderr only',async()=>{
63
- const data={...receipt('p'.repeat(40)),omitted:[{ref:'t1',start_char:100,end_char:200,reason:'budget'}]};
64
- data.selected.push({ref:'t1',start_char:100,end_char:200});const output=[],errors=[];
65
- await sessionsCommand(['export','doc_fixture'],{...options(),selection:['t0:0:100','t1:100:200']},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
66
- assert.equal(output.join(''),'p'.repeat(40));assert.match(errors.join(''),/Omitted t1:100:200/);assert.match(errors.join(''),/through character 40/);
67
- });
68
-
69
- test('permission and revision conflicts are actionable without automatic retry',async()=>{
70
- for (const [status,pattern] of [[401,/Sign in/],[403,/authorized account/],[404,/sessions list/],[409,/read again/],[422,/current values/]]) {
71
- let calls=0;const output=[];
72
- await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>{calls++;throw Object.assign(new Error('private raw detail'),{status});}},{stdout:value=>output.push(value),stderr:()=>{}}),pattern);
73
- assert.equal(calls,1);assert.deepEqual(output,[]);
74
- }
75
- });
76
-
77
- test('plain fallback reads but never invents selectable turns',async()=>{
78
- const output=[],errors=[];const data={version:revision,mode:'plain',document:{title:'Legacy'},branch:{requested:'selected',status:'unknown'},fragments:[],plain:{text:'Unstructured import'},next_cursor:null};
79
- await sessionsCommand(['read','doc_fixture'],{branch:'selected'},{request:async()=>data},{stdout:value=>output.push(value),stderr:value=>errors.push(value)});
80
- assert.match(output.join(''),/Unstructured import/);assert.equal(output.join('').includes('Selection:'),false);assert.match(errors.join(''),/selection is unavailable/);
81
- });
82
-
83
- const script=fileURLToPath(new URL('../bin/opencrawl.mjs',import.meta.url));
84
- function run(args,env,cwd) { return new Promise((resolve,reject)=>{
85
- const child=spawn(process.execPath,[script,...args],{env,cwd,stdio:['ignore','pipe','pipe']});let stdout='',stderr='';const timer=setTimeout(()=>child.kill('SIGKILL'),10000);
86
- child.stdout.on('data',part=>stdout+=part);child.stderr.on('data',part=>stderr+=part);child.on('error',reject);child.on('close',code=>{clearTimeout(timer);resolve({code,stdout,stderr});});
87
- }); }
88
- async function server(digit){
89
- const calls=[];const instance=http.createServer(async(request,response)=>{
90
- let raw='';for await(const part of request)raw+=part;
91
- calls.push({path:request.url,authorization:request.headers.authorization,body:raw?JSON.parse(raw):null});
92
- let data={items:[],has_more:false,offset:0};
93
- if(request.url.endsWith('/context')) data=receipt();
94
- else if(request.url.includes('/turns?')) data={version:revision,mode:'structured',document:{title:'Plan'},branch:{requested:'selected',status:'selected'},fragments:[{ref:'t0',role:'user',branch:'selected',text:'Atlas task',start_char:0,end_char:100,truncated:false}],next_cursor:'next-page'};
95
- response.writeHead(200,{'content-type':'application/json'});response.end(JSON.stringify(data));
96
- });await new Promise(resolve=>instance.listen(0,'127.0.0.1',resolve));
97
- return {calls,url:'http://127.0.0.1:'+instance.address().port,key:'oc_synthetic_'+digit,close:()=>new Promise(resolve=>instance.close(resolve))};
98
- }
99
-
100
- test('real sessions CLI supports help, repeated selections, exact output and pinned accounts',async()=>{
101
- const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-sessions-'));const a=await server('a'),b=await server('b');
102
- const env={...process.env};for(const key of Object.keys(env))if(key.startsWith('OPENCRAWL_'))delete env[key];env.OPENCRAWL_CONFIG_DIR=dir;
103
- try{
104
- const identity=digit=>({user:{id:'usr_'+digit.repeat(32)},workspace:{id:'ws_'+digit.repeat(32)}});
105
- let config=loggedInConfig({},a.url,a.key,identity('a'));const account=config.activeProfile;config=loggedInConfig(config,b.url,b.key,identity('b'));
106
- await fs.writeFile(path.join(dir,'config.json'),JSON.stringify(config),{mode:0o600});
107
- const selected={...env,OPENCRAWL_URL:b.url,OPENCRAWL_API_KEY:b.key};
108
- const help=await run(['sessions','export','--help'],selected,dir);assert.equal(help.code,0);assert.match(help.stdout,/--selection/);assert.equal(a.calls.length+b.calls.length,0);
109
- const read=await run(['sessions','read','doc_fixture','--branch','selected','--account',account],selected,dir);assert.equal(read.code,0,read.stderr);assert.match(read.stdout,/Selection: t0:0:100/);assert.match(read.stderr,/next-page/);
110
- const exported=await run(['sessions','export','doc_fixture','--branch','selected','--revision',revision,'--selection','t0:0:100','--budget','256','--account',account],selected,dir);
111
- assert.equal(exported.code,0,exported.stderr);assert.equal(exported.stdout,receipt().context);assert.equal(exported.stderr,'');
112
- assert.equal(b.calls.length,0);assert.ok(a.calls.every(call=>call.authorization==='Bearer '+a.key));
113
- const globalFirst=await run(['--account',account,'sessions','export','doc_fixture','--branch','selected','--revision',revision,'--selection','t0:0:100','--budget','256'],selected,dir);assert.equal(globalFirst.code,0,globalFirst.stderr);assert.equal(globalFirst.stdout,receipt().context);
114
- const before=a.calls.length;
115
- for(const extra of [['--selection','t0:0:100','--selection','t0:0:100'],['--budget','256','--budget','500'],['--json=false']]) {
116
- const result=await run(['sessions','export','doc_fixture','--branch','selected','--revision',revision,'--account',account,...extra],selected,dir);assert.equal(result.code,1);assert.equal(result.stdout,'');
117
- }
118
- assert.equal(a.calls.length,before);
119
- const conflict=await run(['sessions','read','doc_fixture','--branch','selected','--account',account,'--server',b.url],selected,dir);assert.equal(conflict.code,1);assert.equal(b.calls.length,0);
120
- }finally{await Promise.all([a.close(),b.close()]);await fs.rm(dir,{recursive:true,force:true});}
121
- });
122
-
123
- test('bounded HTTP reading refuses excessive bytes including streamed responses',async t=>{
124
- const dir=await fs.mkdtemp(path.join(os.tmpdir(),'opencrawl-response-'));const previous=process.env.OPENCRAWL_CONFIG_DIR;process.env.OPENCRAWL_CONFIG_DIR=dir;
125
- try{
126
- t.mock.method(globalThis,'fetch',async()=>new Response(new ReadableStream({start(controller){controller.enqueue(new TextEncoder().encode('x'.repeat(33)));controller.close();}})));
127
- const client=await createClient({server:'https://example.test',token:'oc_fixture'});
128
- await assert.rejects(client.request('/api/v1/sessions',{maxResponseBytes:32}),/byte limit/);
129
- }finally{if(previous===undefined)delete process.env.OPENCRAWL_CONFIG_DIR;else process.env.OPENCRAWL_CONFIG_DIR=previous;await fs.rm(dir,{recursive:true,force:true});}
130
- });
131
-
132
-
133
- test('human reader displays control sequences visibly without changing JSON or export',async()=>{
134
- const text='safe\n\tline\u001b[2J\rhidden\u009b31m\u202eend';
135
- const data={version:revision,mode:'structured',document:{title:'Plan'},branch:{requested:'selected',status:'selected'},fragments:[{ref:'t0',role:'user',branch:'selected',text,start_char:0,end_char:text.length,truncated:false}]};
136
- const output=[];await sessionsCommand(['read','doc_fixture'],{branch:'selected'},{request:async()=>data},{stdout:value=>output.push(value),stderr:()=>{}});
137
- assert.match(output.join(''),/safe\n\tline\\u001b\[2J\\u000dhidden\\u009b31m\\u202eend/);
138
- assert.equal(output.join('').includes('\u001b'),false);
139
- const json=[];await sessionsCommand(['read','doc_fixture'],{branch:'selected',json:true},{request:async()=>data},{stdout:value=>json.push(value),stderr:()=>{}});
140
- assert.equal(JSON.parse(json.join('')).fragments[0].text,text);
141
- const exact=[];await sessionsCommand(['export','doc_fixture'],options(),{request:async()=>receipt(text)},{stdout:value=>exact.push(value),stderr:()=>{}});
142
- assert.equal(exact.join(''),text);
143
- });
144
-
145
-
146
- test('included and omitted receipts must partition the requested ranges exactly',async()=>{
147
- const base=receipt();
148
- for (const change of [{included:[],omitted:[]},{included:[{...base.included[0],ref:'t1'}]},{included:[{...base.included[0],end_char:101}]},{included:[{...base.included[0],start_char:1}]},{included:[{...base.included[0],truncated:true}]},{included:[{...base.included[0],excerpt:'different'}]},{omitted:[{ref:'t0',start_char:0,end_char:100,reason:'duplicate'}]}]) {
149
- const output=[];await assert.rejects(sessionsCommand(['export','doc_fixture'],options(),{request:async()=>({...base,...change})},{stdout:value=>output.push(value),stderr:()=>{}}),/receipt/);assert.deepEqual(output,[]);
150
- }
151
- });
package/src/tasks.mjs DELETED
@@ -1,115 +0,0 @@
1
- import { allowedOptions, decimal } from './jobs.mjs';
2
- import { displayLine, displayText } from './display.mjs';
3
- import { coverageWarnings } from './context.mjs';
4
-
5
- export const TASKS_HELP = `OpenCrawl · reuse a saved question with current evidence
6
-
7
- opencrawl tasks list [--limit 20] [--offset 0] [--json]
8
- opencrawl tasks show TASK_ID [--json]
9
- opencrawl tasks run TASK_ID [--revision N] [--json]
10
-
11
- Create and edit private saved tasks in OpenCrawl. A run uses the saved question,
12
- source scope, result limit, and byte budget. It retrieves current accessible
13
- evidence; it does not sync sources or reuse a cached answer. A selected source
14
- becoming unavailable blocks the run instead of widening its scope.
15
- --revision pins a positive integer revision; omit it to run the current task.
16
- Run writes exact context bytes to stdout and diagnostics to stderr. --json
17
- returns the complete receipt. An empty successful run has no current evidence.
18
- Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
19
- `;
20
- const TASK_ID = /^task_[a-f0-9]{32}$/;
21
- const integer = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max;
22
- export function taskRequest(rest, options) {
23
- const [command, id, ...extra] = rest;
24
- if (!['list', 'show', 'run'].includes(command)) throw new Error('Choose tasks list, show, or run. Run opencrawl tasks --help.');
25
- allowedOptions(options, command === 'list' ? ['limit', 'offset'] : command === 'run' ? ['revision'] : [], 'tasks');
26
- if (extra.length || command === 'list' && id !== undefined) throw new Error('Unexpected tasks argument.');
27
- if (command === 'list') {
28
- const limit = decimal(options.limit, 'limit', 1, 20, 20), offset = decimal(options.offset, 'offset', 0, 1000000, 0);
29
- return {command, limit, offset, route: '/api/v1/tasks?' + new URLSearchParams({limit, offset}), options: {}};
30
- }
31
- if (typeof id !== 'string' || !TASK_ID.test(id)) throw new Error('Supply an exact saved task ID from tasks list.');
32
- const revision = options.revision === undefined ? undefined : decimal(options.revision, 'revision', 1, Number.MAX_SAFE_INTEGER);
33
- return {command, id, revision, route: '/api/v1/tasks/' + id + (command === 'run' ? '/run' : ''),
34
- options: command === 'run' ? {method: 'POST', body: revision === undefined ? {} : {expected_revision: revision}} : {}};
35
- }
36
- export function validateTask(task, id = task?.id) {
37
- const scope = task?.scope;
38
- if (!task || typeof task.id !== 'string' || !TASK_ID.test(task.id) || task.id !== id ||
39
- typeof task.title !== 'string' || !task.title || Array.from(task.title).length > 120 ||
40
- typeof task.query !== 'string' || !task.query.trim() || Array.from(task.query).length > 1000 ||
41
- !integer(task.revision, 1, Number.MAX_SAFE_INTEGER) || !integer(task.limit, 1, 20) || !integer(task.budget, 256, 16384) ||
42
- !['available', 'blocked'].includes(task.scope_status) || !scope ||
43
- !(scope.mode === 'all_accessible' && Object.keys(scope).length === 1 || scope.mode === 'selected' && Object.keys(scope).length === 2 && Array.isArray(scope.source_ids) &&
44
- scope.source_ids.length >= 1 && scope.source_ids.length <= 100 && new Set(scope.source_ids).size === scope.source_ids.length &&
45
- scope.source_ids.every(value => typeof value === 'string' && value.length > 0 && value.length <= 200)))
46
- throw new Error('Server returned an invalid saved task. No context was emitted.');
47
- return task;
48
- }
49
- export function validateTaskRun(data, request) {
50
- const task = validateTask(data?.task, request.id);
51
- if (task.scope_status !== 'available' || request.revision !== undefined && task.revision !== request.revision ||
52
- typeof data.context !== 'string' || data.budget_unit !== 'utf8_bytes' ||
53
- Buffer.byteLength(data.context) !== data.context_bytes || data.context_bytes > task.budget ||
54
- !Array.isArray(data.citations) || data.citations.length > task.limit || Boolean(data.context) !== Boolean(data.citations.length))
55
- throw new Error('Server returned an invalid task context receipt. No context was emitted; inspect the saved task before running again.');
56
- const labels = new Set(), chars = Array.from(data.context);
57
- let end = 0;
58
- for (const citation of data.citations) {
59
- const span = citation?.context_span, excerpt = citation?.excerpt;
60
- if (!citation || typeof citation.citation !== 'string' || !/^S[1-9]\d*$/.test(citation.citation) || labels.has(citation.citation) ||
61
- typeof citation.document_id !== 'string' || !citation.document_id ||
62
- !(typeof citation.source_id === 'string' && citation.source_id || task.scope.mode === 'all_accessible' && citation.source_id === null) ||
63
- task.scope.mode === 'selected' && !task.scope.source_ids.includes(citation.source_id) ||
64
- !span || !integer(span.start_char, end, chars.length) || !integer(span.end_char, span.start_char + 1, chars.length) ||
65
- !excerpt || !integer(excerpt.start_char, 0, Number.MAX_SAFE_INTEGER) || !integer(excerpt.end_char, excerpt.start_char + 1, Number.MAX_SAFE_INTEGER) ||
66
- span.end_char - span.start_char !== excerpt.end_char - excerpt.start_char)
67
- throw new Error('Server returned invalid or out-of-scope task citations. No context was emitted.');
68
- labels.add(citation.citation); end = span.end_char;
69
- }
70
- return data;
71
- }
72
- export function taskFailure(error) {
73
- const status = Number.isInteger(error?.status) ? error.status : undefined;
74
- if (status === 409 && error.code === 'task_changed') return {code: 'task_changed', status,
75
- message: 'The saved task changed. Run tasks show to review its current question and scope, then explicitly run that revision.'};
76
- if (status === 409 && error.code === 'task_scope_unavailable') return {code: 'task_scope_unavailable', status,
77
- message: 'A selected source is unavailable. Review the task in OpenCrawl and restore access or explicitly edit its scope. No broader search was run.'};
78
- if (status === 409 && error.code === 'task_evidence_changed') return {code: 'task_evidence_changed', status,
79
- message: 'Source evidence changed or is no longer accessible. Explicitly run this task again to retrieve current evidence. No automatic retry was made.'};
80
- const message = ({401: 'Sign in with a valid key for the intended workspace.',403: 'This account cannot access this saved task operation.',
81
- 404: 'This saved task is unavailable to this account. Check tasks list and the selected workspace.',
82
- 409: 'The saved task or its source access changed. Review tasks show before explicitly running again.',
83
- 422: 'The task request is invalid. Check its ID and positive integer revision.',429: 'OpenCrawl is busy or a workspace limit was reached. Try the task again later.'})[status] ||
84
- 'The task run could not be confirmed. Check the connection before explicitly running again. No automatic retry was made.';
85
- return {code: status ? `http_${status}` : 'request_unconfirmed', message, ...(status ? {status} : {})};
86
- }
87
- export async function tasksCommand(rest, options, client, {stdout = value => process.stdout.write(value), stderr = value => process.stderr.write(value)} = {}) {
88
- const request = taskRequest(rest, options);
89
- let data;
90
- try { data = await client.request(request.route, {...request.options, maxResponseBytes: 512 * 1024}); }
91
- catch (error) { const failure = taskFailure(error); if (options.json) stdout(JSON.stringify({error: failure}, null, 2) + '\n'); else stderr(failure.message + '\n'); return 1; }
92
- if (request.command === 'list') {
93
- if (!data || !Array.isArray(data.items) || data.items.length > request.limit || data.limit !== request.limit || data.offset !== request.offset)
94
- throw new Error('Server returned an invalid saved task page.');
95
- data.items.forEach(task => validateTask(task));
96
- if (options.json) stdout(JSON.stringify(data, null, 2) + '\n');
97
- else {
98
- stdout(data.items.map(task => `${task.id}\tr${task.revision}\t${task.scope_status}\t${displayLine(task.title)}\n`).join('') || 'No saved tasks on this page.\n');
99
- if (data.items.length === request.limit) stderr(`To check the next page, use --offset ${request.offset + data.items.length}.\n`);
100
- }
101
- } else if (request.command === 'show') {
102
- validateTask(data, request.id);
103
- stdout(options.json ? JSON.stringify(data, null, 2) + '\n' : `${displayLine(data.title)}\nTask: ${data.id}\nRevision: ${data.revision}\nQuestion: ${displayText(data.query)}\nScope: ${data.scope.mode === 'all_accessible' ? 'All currently accessible sources' : data.scope.source_ids.map(displayLine).join(', ')}\nSource access: ${data.scope_status}\nResult limit: ${data.limit}; context budget: ${data.budget} UTF-8 bytes\n`);
104
- } else {
105
- validateTaskRun(data, request);
106
- if (options.json) stdout(JSON.stringify(data, null, 2) + '\n');
107
- else {
108
- stderr(`Task revision ${data.task.revision}: ${data.citations.length} citations, ${data.context_bytes}/${data.task.budget} UTF-8 bytes.\n`);
109
- if (!data.context) stderr('No current evidence was returned for the saved question and scope.\n');
110
- for (const warning of coverageWarnings(data)) stderr(displayLine(warning) + '\n');
111
- stdout(data.context);
112
- }
113
- }
114
- return 0;
115
- }