@getopencrawl/cli 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +304 -2
- package/bin/opencrawl.mjs +528 -0
- package/browser.mjs +166 -0
- package/package.json +29 -4
- package/src/answer-contract.mjs +112 -0
- package/src/answer-view.mjs +118 -0
- package/src/capture.mjs +273 -0
- package/src/client.mjs +175 -0
- package/src/connect.mjs +230 -0
- package/src/context.mjs +34 -0
- package/src/display.mjs +4 -0
- package/src/fixtures/letter-connections.json +59 -0
- package/src/fixtures/product-monitor-contract.json +312 -0
- package/src/fixtures/research-completion-hosted.json +502 -0
- package/src/fixtures/research-completion-v1.json +58 -0
- package/src/fixtures/research-completion-v2.json +33 -0
- package/src/fixtures/research-presentation-hosted.json +720 -0
- package/src/jobs.mjs +108 -0
- package/src/monitors.mjs +146 -0
- package/src/product-letter-capabilities.mjs +71 -0
- package/src/product-letter-read.mjs +178 -0
- package/src/product-monitors.mjs +313 -0
- package/src/research-agent.mjs +164 -0
- package/src/research-answer-contract.mjs +3 -0
- package/src/research-answer-view.mjs +7 -0
- package/src/research-completion.mjs +94 -0
- package/src/research-event.mjs +39 -0
- package/src/research.mjs +133 -0
- package/src/sessions.mjs +165 -0
- package/src/tasks.mjs +115 -0
- package/src/uploads.mjs +141 -0
package/src/capture.mjs
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import { constants } from 'node:fs';
|
|
3
|
+
import crypto from 'node:crypto';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { configDir, createClient } from './client.mjs';
|
|
6
|
+
import { allowedOptions, decimal } from './jobs.mjs';
|
|
7
|
+
import { displayLine } from './display.mjs';
|
|
8
|
+
|
|
9
|
+
export const CAPTURE_HELP = `OpenCrawl · capture new project conversations
|
|
10
|
+
|
|
11
|
+
opencrawl install claude-code|codex --capture
|
|
12
|
+
opencrawl capture status --provider claude-code|codex [--json]
|
|
13
|
+
opencrawl capture flush --provider claude-code|codex [--timeout 10] [--json]
|
|
14
|
+
opencrawl capture pause|resume --provider claude-code|codex
|
|
15
|
+
|
|
16
|
+
The current project is the default; --project DIRECTORY selects an installed
|
|
17
|
+
project explicitly. New user prompts and final assistant messages are private.
|
|
18
|
+
No transcript files, old chats, tools, reasoning, or other projects are read.
|
|
19
|
+
Status reads saved receipts and current server readiness. Flush explicitly
|
|
20
|
+
replays only immutable queued events using server idempotence, for 1–60 seconds.
|
|
21
|
+
Unknown delivery is never called indexed. Conflicts/deleted sessions stay blocked.
|
|
22
|
+
Pause stops new local capture; resume captures future events, not missed history.
|
|
23
|
+
Persist the owner-only OpenCrawl configuration directory in disposable sandboxes.
|
|
24
|
+
Global options: --account PROFILE_ID --server URL --config-dir DIRECTORY --json
|
|
25
|
+
Environment-key install: --credential-env NAME (name only; key never in settings).
|
|
26
|
+
`;
|
|
27
|
+
export const AGENT_OPTIONS = ['credential-env','expect-user','expect-workspace'];
|
|
28
|
+
const PROVIDERS = ['claude-code','codex'];
|
|
29
|
+
const MAX_STORE = 8 * 1024 * 1024, MAX_CONTENT = 64 * 1024, MAX_SESSIONS = 100, MAX_EVENTS = 20000;
|
|
30
|
+
const hash = value => crypto.createHash('sha256').update(value).digest('hex');
|
|
31
|
+
const fail = message => { throw new Error(message); };
|
|
32
|
+
const identifier = value => typeof value === 'string' && value.length > 0 && value.length <= 200 && !/[\u0000-\u0020\u007f]/.test(value);
|
|
33
|
+
const user = value => /^usr_[a-f0-9]{32}$/.test(value || '');
|
|
34
|
+
const workspace = value => /^ws_[a-f0-9]{32}$/.test(value || '');
|
|
35
|
+
const same = (a,b) => JSON.stringify(a) === JSON.stringify(b);
|
|
36
|
+
|
|
37
|
+
/** Explicit environment credentials require a destination and durable identity pins. */
|
|
38
|
+
export async function agentClient(options, { installing = false } = {}) {
|
|
39
|
+
if (options['credential-env'] === undefined) {
|
|
40
|
+
if (options['expect-user'] !== undefined || options['expect-workspace'] !== undefined) fail('Identity pins require --credential-env.');
|
|
41
|
+
return createClient(options);
|
|
42
|
+
}
|
|
43
|
+
const name = options['credential-env'];
|
|
44
|
+
if (!/^[A-Z][A-Z0-9_]{1,79}$/.test(name) || !options.server || options.account) fail('Environment-key setup requires an explicit --server, a variable name, and no saved --account.');
|
|
45
|
+
if (!installing && (!user(options['expect-user']) || !workspace(options['expect-workspace']))) fail('Environment-key agent setup needs verified identity pins. Rerun the installer in this runtime.');
|
|
46
|
+
const token = process.env[name];
|
|
47
|
+
if (!token) fail('The configured credential environment variable is missing. No saved key was substituted.');
|
|
48
|
+
const client = await createClient({ server: options.server, token });
|
|
49
|
+
client.expectedIdentity = installing ? null : { user_id: options['expect-user'], workspace_id: options['expect-workspace'] };
|
|
50
|
+
return client;
|
|
51
|
+
}
|
|
52
|
+
export function bindingIdentity(client) {
|
|
53
|
+
const value = client.expectedIdentity || (client.profile && { user_id: client.profile.userId, workspace_id: client.profile.workspaceId });
|
|
54
|
+
if (!client.configured || !value || !user(value.user_id) || !workspace(value.workspace_id)) fail('Capture requires a verified saved account or an installed environment-key identity.');
|
|
55
|
+
return { server: client.baseUrl, ...value };
|
|
56
|
+
}
|
|
57
|
+
export async function verifyAgentIdentity(client, timeout = 2000) {
|
|
58
|
+
const result = await client.request('/api/auth/me', { timeout, maxResponseBytes: 128 * 1024 });
|
|
59
|
+
const actual = { user_id: result?.user?.id, workspace_id: result?.workspace?.id };
|
|
60
|
+
if (!user(actual.user_id) || !workspace(actual.workspace_id)) fail('The server did not return a valid account identity.');
|
|
61
|
+
const expected = client.expectedIdentity || (client.profile && { user_id: client.profile.userId, workspace_id: client.profile.workspaceId });
|
|
62
|
+
if (expected && !same(actual, expected)) fail('The credential does not match the installed user and workspace. Nothing was sent.');
|
|
63
|
+
return { server: client.baseUrl, ...actual };
|
|
64
|
+
}
|
|
65
|
+
async function privateDirectory(directory) {
|
|
66
|
+
await fs.mkdir(directory,{recursive:true,mode:0o700});
|
|
67
|
+
const stat=await fs.lstat(directory);
|
|
68
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || typeof process.getuid==='function' && stat.uid!==process.getuid()) fail('Capture state must be in a directory owned by this user, without symbolic links.');
|
|
69
|
+
await fs.chmod(directory,0o700);
|
|
70
|
+
}
|
|
71
|
+
async function projectScope(client, provider, project) {
|
|
72
|
+
if (!PROVIDERS.includes(provider)) fail('Choose --provider claude-code or codex.');
|
|
73
|
+
const root=await fs.realpath(project || process.cwd());
|
|
74
|
+
if (!(await fs.stat(root)).isDirectory()) fail('Capture project must be a directory.');
|
|
75
|
+
return { ...bindingIdentity(client), provider, project: root };
|
|
76
|
+
}
|
|
77
|
+
async function storage(scope, action, { initialize=false }={}) {
|
|
78
|
+
const root=path.resolve(configDir());await privateDirectory(root);
|
|
79
|
+
const directory=path.join(root,'capture');await privateDirectory(directory);
|
|
80
|
+
const name=hash(JSON.stringify(scope)),file=path.join(directory,name+'.json'),lockfile=path.join(directory,name+'.lock');
|
|
81
|
+
let lock;
|
|
82
|
+
const until=Date.now()+700;
|
|
83
|
+
while (!lock) {
|
|
84
|
+
try { lock=await fs.open(lockfile,constants.O_WRONLY|constants.O_CREAT|constants.O_EXCL|constants.O_NOFOLLOW,0o600);await lock.writeFile(String(process.pid)); }
|
|
85
|
+
catch (error) { if(error.code!=='EEXIST')throw error;if(Date.now()>=until)fail('Capture is busy or has a stale lock. Inspect capture status; remove its lock only after confirming the previous process stopped.');await new Promise(resolve=>setTimeout(resolve,25)); }
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
let state,handle;
|
|
89
|
+
try {
|
|
90
|
+
handle=await fs.open(file,constants.O_RDONLY|constants.O_NOFOLLOW);const stat=await handle.stat();
|
|
91
|
+
if(!stat.isFile()||stat.size>MAX_STORE||stat.mode&0o077||typeof process.getuid==='function'&&stat.uid!==process.getuid())fail('Capture state must be an owner-only bounded regular file.');
|
|
92
|
+
try { state=JSON.parse(await handle.readFile('utf8')); } catch { fail('Capture state is invalid. Preserve it before repair; no events were sent.'); }
|
|
93
|
+
} catch(error) {
|
|
94
|
+
if(error.code!=='ENOENT')throw error;
|
|
95
|
+
if(!initialize)fail('Capture is not configured for this project and account. Run install TARGET --capture here.');
|
|
96
|
+
if((await fs.readdir(directory)).filter(value=>/^[a-f0-9]{64}\.json$/.test(value)).length>=64)fail('Capture is limited to 64 project/account configurations. Preserve existing state before starting another.');
|
|
97
|
+
state={version:1,scope,paused:false,sessions:[],configured_at:new Date().toISOString()};
|
|
98
|
+
} finally { await handle?.close(); }
|
|
99
|
+
validateState(state,scope);
|
|
100
|
+
const save=async()=>{
|
|
101
|
+
const encoded=JSON.stringify(state);if(Buffer.byteLength(encoded)>MAX_STORE)fail('Capture spool reached its 8 MiB limit. Flush or review queued events; no content was silently dropped.');
|
|
102
|
+
const temp=file+'.'+crypto.randomUUID()+'.tmp';let output;
|
|
103
|
+
try { output=await fs.open(temp,'wx',0o600);await output.writeFile(encoded);await output.sync();await output.close();output=null;await fs.rename(temp,file);
|
|
104
|
+
if(process.platform!=='win32'){const dir=await fs.open(directory,constants.O_RDONLY|constants.O_DIRECTORY|constants.O_NOFOLLOW);try{await dir.sync();}finally{await dir.close();}} }
|
|
105
|
+
finally {await output?.close();await fs.unlink(temp).catch(error=>{if(error.code!=='ENOENT')throw error;});}
|
|
106
|
+
};
|
|
107
|
+
return await action(state,save);
|
|
108
|
+
} finally {await lock.close();await fs.unlink(lockfile);}
|
|
109
|
+
}
|
|
110
|
+
function validateState(state,scope) {
|
|
111
|
+
if(state?.version!==1||!same(state.scope,scope)||typeof state.paused!=='boolean'||!Array.isArray(state.sessions)||state.sessions.length>MAX_SESSIONS)fail('Capture binding or state is invalid. No events were sent.');
|
|
112
|
+
let count=0;
|
|
113
|
+
for(const session of state.sessions) {
|
|
114
|
+
if(!identifier(session.id)||!identifier(session.remote_id)||!Array.isArray(session.events)||!Number.isSafeInteger(session.next)||session.next<0||!Array.isArray(session.turns))fail('Capture session state is invalid.');
|
|
115
|
+
const ids=new Set(),sequences=new Set();
|
|
116
|
+
for(const event of session.events) {
|
|
117
|
+
if(!identifier(event.id)||ids.has(event.id)||!Number.isSafeInteger(event.sequence)||event.sequence<0||event.sequence>=session.next||sequences.has(event.sequence)||!['user','assistant'].includes(event.role)||!['queued','acknowledged'].includes(event.state)||!/^[a-f0-9]{64}$/.test(event.hash)||!Number.isInteger(event.attempts)||event.attempts<0)fail('Capture event state is invalid.');
|
|
118
|
+
if(event.state==='queued'&&(typeof event.content!=='string'||!event.content.trim()||Buffer.byteLength(event.content)>MAX_CONTENT||hash(event.content)!==event.hash))fail('Queued capture content is invalid.');
|
|
119
|
+
ids.add(event.id);sequences.add(event.sequence);count++;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if(count>MAX_EVENTS)fail('Capture state exceeds its event limit.');
|
|
123
|
+
}
|
|
124
|
+
export async function configureCapture(client,provider,project) {
|
|
125
|
+
const scope=await projectScope(client,provider,project);
|
|
126
|
+
await storage(scope,async(state,save)=>{await save();},{initialize:true});return scope;
|
|
127
|
+
}
|
|
128
|
+
function safeIssue(error) {
|
|
129
|
+
const status=error?.status;
|
|
130
|
+
return ({401:'authorization_required',403:'authorization_required',409:'event_conflict',410:'session_deleted',413:'capacity_exceeded',422:'invalid_event',429:'temporarily_unavailable',507:'storage_unavailable'})[status] || 'delivery_unconfirmed';
|
|
131
|
+
}
|
|
132
|
+
const permanent = issue => ['event_conflict','session_deleted','invalid_event','capacity_exceeded'].includes(issue);
|
|
133
|
+
export function validateCaptureReceipt(value,batchCount=null) {
|
|
134
|
+
if(!value||typeof value.document_id!=='string'||!/^doc_[a-f0-9]{32}$/.test(value.document_id)||value.lexical_ready!==true||typeof value.semantic_ready!=='boolean'||!Number.isSafeInteger(value.event_count)||value.event_count<1||!['ready','pending','unavailable'].includes(value.semantic?.status)||!Number.isSafeInteger(value.semantic?.indexed_chunks)||!Number.isSafeInteger(value.semantic?.total_chunks)||value.semantic.indexed_chunks<0||value.semantic.total_chunks<value.semantic.indexed_chunks)fail('Capture acknowledgement is invalid; queued events were preserved.');
|
|
135
|
+
if(batchCount!==null&&(!Number.isSafeInteger(value.accepted_events)||!Number.isSafeInteger(value.replayed_events)||value.accepted_events<0||value.replayed_events<0||value.accepted_events+value.replayed_events!==batchCount))fail('Capture acknowledgement did not account for this exact event batch.');
|
|
136
|
+
return {document_id:value.document_id,lexical_ready:true,semantic_ready:value.semantic_ready,semantic:{status:value.semantic.status,indexed_chunks:value.semantic.indexed_chunks,total_chunks:value.semantic.total_chunks},event_count:value.event_count,observed_at:new Date().toISOString()};
|
|
137
|
+
}
|
|
138
|
+
function enqueue(state,input) {
|
|
139
|
+
if(state.paused)return null;
|
|
140
|
+
const role=input.hook_event_name==='UserPromptSubmit'?'user':input.hook_event_name==='Stop'?'assistant':null;
|
|
141
|
+
if(!role||!identifier(input.session_id))fail('Unsupported capture hook or session identifier.');
|
|
142
|
+
const content=role==='user'?input.prompt:input.last_assistant_message;
|
|
143
|
+
if(typeof content!=='string'||!content.trim())return null;
|
|
144
|
+
if(Buffer.byteLength(content)>MAX_CONTENT)fail('This message exceeds the 64 KiB capture limit; it was not truncated or uploaded.');
|
|
145
|
+
if(Buffer.byteLength(JSON.stringify(content))>230*1024)fail('This message exceeds the encoded capture request limit; it was not truncated or uploaded.');
|
|
146
|
+
let session=state.sessions.find(value=>value.id===input.session_id);
|
|
147
|
+
if(!session) {
|
|
148
|
+
if(role==='assistant')return null; // Never attach an orphan Stop to a guessed turn.
|
|
149
|
+
if(state.sessions.length>=MAX_SESSIONS)fail('This capture scope reached 100 sessions. Preserve its state before starting another.');
|
|
150
|
+
session={id:input.session_id,remote_id:'occap_'+hash(state.scope.project+'\0'+input.session_id),events:[],turns:[],next:0};state.sessions.push(session);
|
|
151
|
+
}
|
|
152
|
+
if(permanent(session.issue))return session;
|
|
153
|
+
let turn;
|
|
154
|
+
if(state.scope.provider==='codex') {
|
|
155
|
+
if(!identifier(input.turn_id))fail('This Codex version did not provide a turn ID. Capture skipped; update the client and inspect /hooks.');
|
|
156
|
+
turn=session.turns.find(value=>value.id===input.turn_id);
|
|
157
|
+
if(!turn) {if(role==='assistant')return null;turn={id:input.turn_id,token:hash(input.turn_id)};session.turns.push(turn);}
|
|
158
|
+
} else if(role==='user') {
|
|
159
|
+
// prompt_id is optional in Claude versions. Without it, an unfinished
|
|
160
|
+
// identical prompt is treated as a hook replay; completed turns are new.
|
|
161
|
+
const last=session.turns.at(-1), native=identifier(input.prompt_id)?input.prompt_id:null;
|
|
162
|
+
if(native)turn=session.turns.find(value=>value.id===native);
|
|
163
|
+
else if(last&&!last.closed&&last.user_hash===hash(content))turn=last;
|
|
164
|
+
if(!turn){turn={id:native||crypto.randomUUID(),token:crypto.randomUUID(),user_hash:hash(content)};session.turns.push(turn);}
|
|
165
|
+
} else turn=session.turns.at(-1);
|
|
166
|
+
if(!turn)return null;
|
|
167
|
+
const id=hash(turn.token+'\0'+role),prior=session.events.find(value=>value.id===id);
|
|
168
|
+
if(prior) {
|
|
169
|
+
if(prior.hash!==hash(content)){session.issue='event_conflict';return session;}
|
|
170
|
+
return session;
|
|
171
|
+
}
|
|
172
|
+
if(state.sessions.reduce((count,item)=>count+item.events.length,0)>=MAX_EVENTS)fail('This capture scope reached 20,000 events. No queued event was discarded.');
|
|
173
|
+
session.events.push({id,sequence:session.next++,role,content,hash:hash(content),state:'queued',attempts:0,next_attempt_at:0});
|
|
174
|
+
if(role==='assistant')turn.closed=true;
|
|
175
|
+
return session;
|
|
176
|
+
}
|
|
177
|
+
async function flushOne(client,scope,sessionId,{explicit=false,timeout=1200}={}) {
|
|
178
|
+
const batch=await storage(scope,async(state,save)=>{
|
|
179
|
+
const session=state.sessions.find(value=>value.id===sessionId);
|
|
180
|
+
if(!session||state.paused||permanent(session.issue))return null;
|
|
181
|
+
let events=[];
|
|
182
|
+
for(const event of session.events.filter(event=>event.state==='queued')) {
|
|
183
|
+
if(!explicit&&(event.attempts>=3||event.next_attempt_at>Date.now()))break;
|
|
184
|
+
const next=[...events,{id:event.id,sequence:event.sequence,role:event.role,content:event.content}];
|
|
185
|
+
if(next.length>100||Buffer.byteLength(JSON.stringify({provider:scope.provider,session_id:session.remote_id,events:next}))>240*1024)break;
|
|
186
|
+
events=next;
|
|
187
|
+
}
|
|
188
|
+
if(!events.length)return null;
|
|
189
|
+
for(const item of events){const event=session.events.find(value=>value.id===item.id);event.attempts++;event.next_attempt_at=Date.now()+Math.min(60000,1000*2**Math.min(event.attempts,6));}
|
|
190
|
+
await save();
|
|
191
|
+
return {provider:scope.provider,session_id:session.remote_id,events};
|
|
192
|
+
});
|
|
193
|
+
if(!batch)return;
|
|
194
|
+
let receipt,issue;
|
|
195
|
+
try {receipt=validateCaptureReceipt(await client.request('/api/v1/sessions/events',{method:'POST',body:batch,timeout,maxResponseBytes:64*1024}),batch.events.length);}
|
|
196
|
+
catch(error){issue=safeIssue(error);}
|
|
197
|
+
await storage(scope,async(state,save)=>{
|
|
198
|
+
const session=state.sessions.find(value=>value.id===sessionId);
|
|
199
|
+
if(!session)return;
|
|
200
|
+
const events=batch.events.map(item=>session.events.find(value=>value.id===item.id&&value.sequence===item.sequence&&value.role===item.role&&value.hash===hash(item.content)));
|
|
201
|
+
if(events.some(event=>!event))fail('The queued capture changed while its request was in flight. Preserve the spool.');
|
|
202
|
+
if(receipt) {
|
|
203
|
+
if(!session.receipt||session.receipt.event_count<=receipt.event_count)session.receipt=receipt;
|
|
204
|
+
if(!permanent(session.issue))delete session.issue;
|
|
205
|
+
delete session.status_issue;session.status_verified_at=receipt.observed_at;
|
|
206
|
+
for(const event of events){event.state='acknowledged';delete event.content;}
|
|
207
|
+
} else if(events.some(event=>event.state==='queued')&&!permanent(session.issue))session.issue=issue;
|
|
208
|
+
await save();
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function summary(state) {
|
|
213
|
+
return {configured:true,paused:state.paused,provider:state.scope.provider,project:state.scope.project,server:state.scope.server,
|
|
214
|
+
sessions:state.sessions.map(session=>({session_id:session.id,state:permanent(session.issue)?'blocked':session.events.some(event=>event.state==='queued')?'queued':session.status_issue?'verification_unavailable':session.receipt?'lexical_ready':'configured',queued_events:session.events.filter(event=>event.state==='queued').length,acknowledged_events:session.events.filter(event=>event.state==='acknowledged').length,issue:session.issue||null,current_verification:session.status_issue?{state:'unavailable',issue:session.status_issue}:session.status_verified_at?{state:'verified',observed_at:session.status_verified_at}:{state:'not_checked'},...(session.receipt?{last_acknowledgement:session.receipt}:{})}))};
|
|
215
|
+
}
|
|
216
|
+
export async function captureHook(input,options,client) {
|
|
217
|
+
const scope=await assertCaptureHook(input,options,client);
|
|
218
|
+
const sessionId=await storage(scope,async(state,save)=>{
|
|
219
|
+
const session=enqueue(state,input);await save();return session?.id;
|
|
220
|
+
});
|
|
221
|
+
if(sessionId) {
|
|
222
|
+
try {await verifyAgentIdentity(client,1200);await flushOne(client,scope,sessionId,{timeout:1200});}
|
|
223
|
+
catch(error){await storage(scope,async(state,save)=>{const session=state.sessions.find(value=>value.id===sessionId);if(session&&!permanent(session.issue)){session.issue=safeIssue(error);await save();}});}
|
|
224
|
+
}
|
|
225
|
+
return storage(scope,async state=>summary(state));
|
|
226
|
+
}
|
|
227
|
+
export async function assertCaptureHook(input,options,client) {
|
|
228
|
+
const scope=await projectScope(client,options.provider,options.project);
|
|
229
|
+
// Both the host's event cwd and executing hook must belong to this exact
|
|
230
|
+
// installed project. Never infer scope from transcript paths or file contents.
|
|
231
|
+
if(typeof input?.cwd!=='string'||await fs.realpath(input.cwd)!==scope.project||await fs.realpath(process.cwd())!==scope.project)fail('Hook project does not match the installed capture scope.');
|
|
232
|
+
await storage(scope,async()=>{}); // Require the explicit installed capture registration.
|
|
233
|
+
return scope;
|
|
234
|
+
}
|
|
235
|
+
export async function captureCommand(rest,options,client,{stdout=text=>process.stdout.write(text)}={}) {
|
|
236
|
+
const [command,...extra]=rest;
|
|
237
|
+
if(extra.length||!['status','flush','pause','resume'].includes(command))fail('Choose capture status, flush, pause, or resume.');
|
|
238
|
+
allowedOptions(options,['provider','project','timeout',...AGENT_OPTIONS],'capture');
|
|
239
|
+
if(options.timeout!==undefined&&command!=='flush')fail('--timeout is only supported by capture flush.');
|
|
240
|
+
const timeout=decimal(options.timeout,'timeout',1,60,10),scope=await projectScope(client,options.provider,options.project),deadline=Date.now()+timeout*1000;
|
|
241
|
+
if(command==='pause'||command==='resume') {
|
|
242
|
+
await storage(scope,async(state,save)=>{state.paused=command==='pause';await save();});
|
|
243
|
+
} else {
|
|
244
|
+
const snapshot=await storage(scope,async state=>({paused:state.paused,sessions:state.sessions.map(session=>({id:session.id,remote_id:session.remote_id,pending:session.events.some(event=>event.state==='queued'),blocked:permanent(session.issue)}))}));
|
|
245
|
+
try {await verifyAgentIdentity(client,Math.min(2000,timeout*1000));}
|
|
246
|
+
catch(error){
|
|
247
|
+
await storage(scope,async(state,save)=>{for(const session of state.sessions)session.status_issue=safeIssue(error);await save();});
|
|
248
|
+
if(command==='flush')throw error;
|
|
249
|
+
const value=await storage(scope,async state=>summary(state));
|
|
250
|
+
stdout(options.json?JSON.stringify(value,null,2)+'\n':'Current capture verification is unavailable. Previous acknowledgements are historical; check account access.\n');return 1;
|
|
251
|
+
}
|
|
252
|
+
if(command==='flush') {
|
|
253
|
+
if(snapshot.paused)fail('Capture is paused. Resume explicitly before flushing queued messages.');
|
|
254
|
+
let operations=0;
|
|
255
|
+
for(const session of snapshot.sessions) {
|
|
256
|
+
if(Date.now()>=deadline||operations>=10)break;
|
|
257
|
+
if(session.pending&&!session.blocked){await flushOne(client,scope,session.id,{explicit:true,timeout:Math.max(1,Math.min(2000,deadline-Date.now()))});operations++;}
|
|
258
|
+
}
|
|
259
|
+
} else {
|
|
260
|
+
// Only query known captured sessions, with a shared finite deadline.
|
|
261
|
+
for(const session of snapshot.sessions.slice(-20)) {
|
|
262
|
+
if(Date.now()>=deadline)break;
|
|
263
|
+
let receipt,issue;
|
|
264
|
+
try {receipt=validateCaptureReceipt(await client.request('/api/v1/sessions/events/status?'+new URLSearchParams({provider:scope.provider,session_id:session.remote_id}),{timeout:Math.max(1,Math.min(1500,deadline-Date.now())),maxResponseBytes:64*1024}));}
|
|
265
|
+
catch(error){issue=error.status===410?'session_deleted':error.status===404?'not_received':safeIssue(error);}
|
|
266
|
+
await storage(scope,async(state,save)=>{const current=state.sessions.find(value=>value.id===session.id);if(current){if(receipt){if(!current.receipt||current.receipt.event_count<=receipt.event_count)current.receipt=receipt;current.status_verified_at=receipt.observed_at;delete current.status_issue;}else{current.status_issue=issue;if(issue==='session_deleted')current.issue=issue;}await save();}});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
const result=await storage(scope,async state=>summary(state));
|
|
271
|
+
stdout(options.json?JSON.stringify(result,null,2)+'\n':`Capture ${result.paused?'paused':'configured'}: ${displayLine(result.provider)} in ${displayLine(result.project)}\n`+result.sessions.map(session=>`${displayLine(session.session_id)}: ${session.state}; ${session.queued_events} queued, ${session.acknowledged_events} acknowledged${session.issue?' ('+session.issue+')':''}${session.current_verification.issue?' (current verification: '+session.current_verification.issue+')':''}\n`).join(''));
|
|
272
|
+
return result.sessions.some(session=>session.state==='blocked'||session.current_verification.state==='unavailable'||command==='flush'&&session.queued_events)?1:0;
|
|
273
|
+
}
|
package/src/client.mjs
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import crypto from 'node:crypto';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
|
|
6
|
+
export const configDir = () => process.env.OPENCRAWL_CONFIG_DIR || path.join(os.homedir(), '.config', 'opencrawl');
|
|
7
|
+
export async function readConfig() {
|
|
8
|
+
try {
|
|
9
|
+
const value = JSON.parse(await fs.readFile(path.join(configDir(), 'config.json'), 'utf8'));
|
|
10
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid configuration');
|
|
11
|
+
if (value.version !== undefined && value.version !== 2) throw new Error('Unsupported configuration version');
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
catch (error) { if (error.code === 'ENOENT') return {}; throw new Error('OpenCrawl config is invalid. Repair config.json or choose a separate --config-dir before signing in.'); }
|
|
15
|
+
}
|
|
16
|
+
export async function writeConfig(value) {
|
|
17
|
+
const dir = configDir();
|
|
18
|
+
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
19
|
+
const file = path.join(dir, 'config.json');
|
|
20
|
+
const temp = `${file}.${process.pid}.tmp`;
|
|
21
|
+
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 });
|
|
22
|
+
await fs.rename(temp, file);
|
|
23
|
+
await fs.chmod(file, 0o600);
|
|
24
|
+
}
|
|
25
|
+
export function validateBaseUrl(input) {
|
|
26
|
+
if (typeof input !== 'string' || !/^[A-Za-z][A-Za-z0-9+.-]*:\/\/[^/?#\\]+\/?$/.test(input.trim()))
|
|
27
|
+
throw new Error('Server must be an origin without credentials, a path, query parameters, or a fragment.');
|
|
28
|
+
let url;
|
|
29
|
+
try { url = new URL(input); } catch { throw new Error('Server must be a valid HTTP or HTTPS URL.'); }
|
|
30
|
+
if (url.username || url.password || url.search || url.hash || url.pathname !== '/') throw new Error('Server must be an origin without credentials, a path, query parameters, or a fragment.');
|
|
31
|
+
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) {
|
|
32
|
+
throw new Error('Use HTTPS, or HTTP on localhost.');
|
|
33
|
+
}
|
|
34
|
+
return url.origin;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const profileId = (server, user, workspace) => 'p_' + crypto.createHash('sha256').update(`${server}\0${user}\0${workspace}`).digest('hex').slice(0, 24);
|
|
38
|
+
function validToken(token) {
|
|
39
|
+
if (typeof token !== 'string' || !token.trim() || token.length > 8192 || /\s/.test(token.trim())) throw new Error('The explicitly selected API key is empty or invalid. Sign in again.');
|
|
40
|
+
return token.trim();
|
|
41
|
+
}
|
|
42
|
+
function selectedProfile(config, id) {
|
|
43
|
+
if (!id) return null;
|
|
44
|
+
if (config.version !== 2 || typeof id !== 'string' || !/^p_[a-f0-9]{24}$/.test(id) || !Object.hasOwn(config.profiles || {}, id))
|
|
45
|
+
throw new Error('Saved account profile not found. Run opencrawl profiles or sign in again.');
|
|
46
|
+
const value = config.profiles[id];
|
|
47
|
+
if (!value || typeof value !== 'object') throw new Error('Saved account profile is invalid. Sign in again.');
|
|
48
|
+
const server = validateBaseUrl(value.server);
|
|
49
|
+
if (value.origin !== server || !/^usr_[a-f0-9]{32}$/.test(value.userId || '') || !/^ws_[a-f0-9]{32}$/.test(value.workspaceId || '') || id !== profileId(server, value.userId, value.workspaceId))
|
|
50
|
+
throw new Error('Saved credential binding is invalid. Sign in again.');
|
|
51
|
+
return { ...value, server, id };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Pure selection: a saved credential never follows a changed destination. */
|
|
55
|
+
export function resolveCredentials(config, overrides = {}, environment = process.env) {
|
|
56
|
+
const pinned = overrides.account !== undefined;
|
|
57
|
+
const profile = pinned ? selectedProfile(config, overrides.account) : config.version === 2 ? selectedProfile(config, config.activeProfile) : null;
|
|
58
|
+
const baseUrl = validateBaseUrl(overrides.server ?? (pinned ? profile.server : environment.OPENCRAWL_URL ?? config.server ?? profile?.server ?? 'http://localhost:8000'));
|
|
59
|
+
if (pinned && baseUrl !== profile.server) throw new Error('The selected account is bound to another server. Choose that server’s account or sign in there first.');
|
|
60
|
+
// An explicit programmatic token is an intentional destination choice. A
|
|
61
|
+
// pinned account uses its own credential rather than ambient shell defaults.
|
|
62
|
+
const explicit = overrides.token !== undefined ? overrides.token : !pinned && environment.OPENCRAWL_API_KEY !== undefined ? environment.OPENCRAWL_API_KEY : undefined;
|
|
63
|
+
if (explicit !== undefined) return { baseUrl, token: validToken(explicit), credentialSource: 'explicit', profile: null };
|
|
64
|
+
if (profile?.apiKey && profile.origin === baseUrl) return { baseUrl, token: validToken(profile.apiKey), credentialSource: 'profile', profile };
|
|
65
|
+
const credentialError = config.apiKey ? 'This saved key predates server binding. Run opencrawl login with an explicitly supplied key before using it.'
|
|
66
|
+
: profile?.apiKey ? 'The saved key belongs to another server. Sign in to this server explicitly; no saved key was sent.'
|
|
67
|
+
: pinned ? 'This account has no saved key. Sign in to the same server and workspace again, or choose another account.'
|
|
68
|
+
: 'Sign in with opencrawl login, or explicitly set OPENCRAWL_API_KEY for the intended server.';
|
|
69
|
+
return { baseUrl, token: null, credentialSource: 'none', profile, credentialError };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function loggedInConfig(config, server, apiKey, identity) {
|
|
73
|
+
server = validateBaseUrl(server);
|
|
74
|
+
if (!/^usr_[a-f0-9]{32}$/.test(identity?.user?.id || '') || !/^ws_[a-f0-9]{32}$/.test(identity?.workspace?.id || ''))
|
|
75
|
+
throw new Error('The server did not return a valid workspace identity. No credential was saved.');
|
|
76
|
+
const id = profileId(server, identity.user.id, identity.workspace.id);
|
|
77
|
+
const next = { ...config, version: 2, server, activeProfile: id, profiles: { ...(config.version === 2 ? config.profiles : {}),
|
|
78
|
+
[id]: { server, origin: server, userId: identity.user.id, workspaceId: identity.workspace.id,
|
|
79
|
+
userName: String(identity.user.name || '').slice(0, 100), workspaceName: String(identity.workspace.name || '').slice(0, 100), apiKey: validToken(apiKey) } } };
|
|
80
|
+
delete next.apiKey;
|
|
81
|
+
return next;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function profileSummaries(config) {
|
|
85
|
+
return Object.keys(config.version === 2 ? config.profiles || {} : {}).map(id => {
|
|
86
|
+
const profile = selectedProfile(config, id);
|
|
87
|
+
return { id, server: profile.server, user: profile.userName, workspace: profile.workspaceName,
|
|
88
|
+
user_id: profile.userId, workspace_id: profile.workspaceId, configured: !!profile.apiKey, active: id === config.activeProfile };
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function loggedOutConfig(config, account) {
|
|
93
|
+
const next = { ...config, profiles: { ...(config.profiles || {}) } };
|
|
94
|
+
const profile = account !== undefined ? selectedProfile(config, account) : config.version === 2 ? selectedProfile(config, config.activeProfile) : null;
|
|
95
|
+
if (profile) { next.profiles[profile.id] = { ...config.profiles[profile.id] }; delete next.profiles[profile.id].apiKey; }
|
|
96
|
+
delete next.apiKey;
|
|
97
|
+
return next;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function createClient(overrides = {}) {
|
|
101
|
+
const config = await readConfig();
|
|
102
|
+
const { baseUrl, token, profile, credentialSource, credentialError } = resolveCredentials(config, overrides);
|
|
103
|
+
const publicProfile = profile ? Object.fromEntries(['id', 'server', 'userId', 'workspaceId', 'userName', 'workspaceName'].map(key => [key, profile[key]])) : null;
|
|
104
|
+
return { baseUrl, configured: Boolean(token), profile: publicProfile, credentialSource, credentialError, identity: token ? crypto.createHash('sha256').update(token).digest('hex').slice(0, 16) : 'anonymous', async request(route, { method = 'GET', body, authenticated = true, timeout = 30000, protocolVersion, channel, maxResponseBytes, signal, idempotencyKey, responseType } = {}) {
|
|
105
|
+
// The exported client must not let a caller turn an origin into URL userinfo
|
|
106
|
+
// or replace its port/host by supplying something other than a path.
|
|
107
|
+
if (typeof route !== 'string' || !/^\/(?!\/)/.test(route) || /[\\\u0000-\u0020\u007f]/.test(route))
|
|
108
|
+
throw new Error('Request routes must be absolute paths on the selected server.');
|
|
109
|
+
let destination;
|
|
110
|
+
try { destination = new URL(route, baseUrl); } catch { throw new Error('Request route is invalid.'); }
|
|
111
|
+
if (destination.origin !== baseUrl || destination.username || destination.password)
|
|
112
|
+
throw new Error('Request routes cannot change the selected server.');
|
|
113
|
+
if (authenticated && !token) throw new Error(credentialError);
|
|
114
|
+
if(responseType!==undefined&&responseType!=='png')throw new Error('Unsupported response type.');
|
|
115
|
+
const headers = { Accept: responseType==='png'?'image/png':route === '/mcp' ? 'application/json, text/event-stream' : 'application/json', 'X-OpenCrawl-Client': channel || (route === '/mcp' ? 'mcp' : 'cli') };
|
|
116
|
+
if (protocolVersion) headers['MCP-Protocol-Version'] = protocolVersion;
|
|
117
|
+
if (idempotencyKey !== undefined) {
|
|
118
|
+
if (!(method === 'POST' || method === 'GET' && destination.pathname === '/api/research/agents/receipts/by-key') || typeof idempotencyKey !== 'string' || !/^[A-Za-z0-9_-]{16,128}$/.test(idempotencyKey)) throw new Error('Invalid idempotency key for this request.');
|
|
119
|
+
headers['Idempotency-Key'] = idempotencyKey;
|
|
120
|
+
}
|
|
121
|
+
if (authenticated) headers.Authorization = `Bearer ${token}`;
|
|
122
|
+
const multipart = typeof FormData !== 'undefined' && body instanceof FormData;
|
|
123
|
+
if (body !== undefined && !multipart) headers['Content-Type'] = 'application/json';
|
|
124
|
+
const controller = new AbortController();
|
|
125
|
+
const abort = () => controller.abort(signal?.reason);
|
|
126
|
+
if (signal?.aborted) abort(); else signal?.addEventListener('abort', abort, { once: true });
|
|
127
|
+
const timer = setTimeout(() => controller.abort(new DOMException('Timed out', 'TimeoutError')), timeout);
|
|
128
|
+
timer.unref?.();
|
|
129
|
+
try {
|
|
130
|
+
let response;
|
|
131
|
+
try {
|
|
132
|
+
response = await fetch(destination.href, { method, headers, body: body === undefined ? undefined : multipart ? body : JSON.stringify(body),
|
|
133
|
+
signal: controller.signal, redirect: 'error' });
|
|
134
|
+
} catch (error) { throw new Error(error.name === 'TimeoutError' ? 'OpenCrawl request timed out.' : 'Cannot reach OpenCrawl. Check your server URL and connection.'); }
|
|
135
|
+
if(responseType==='png'&&response.ok){
|
|
136
|
+
const cap=2*1024*1024;if(response.headers.get('content-type')?.split(';')[0]?.trim()!=='image/png'||Number(response.headers.get('content-length'))>cap){await response.body?.cancel();throw new Error('Invalid bounded PNG response.');}
|
|
137
|
+
const reader=response.body?.getReader();if(!reader)throw new Error('Missing PNG response.');const bytes=Buffer.alloc(cap);let length=0;const cancel=()=>{void reader.cancel().catch(()=>{});};if(controller.signal.aborted)cancel();else controller.signal.addEventListener('abort',cancel,{once:true});
|
|
138
|
+
try{while(true){const part=await reader.read();if(part.done)break;if(length+part.value.byteLength>cap){await reader.cancel();throw new Error('PNG response exceeds its limit.');}bytes.set(part.value,length);length+=part.value.byteLength;}}finally{controller.signal.removeEventListener('abort',cancel);reader.releaseLock();}
|
|
139
|
+
if(controller.signal.aborted)throw new Error('PNG request cancelled.');
|
|
140
|
+
if(length<33||!bytes.subarray(0,8).equals(Buffer.from([137,80,78,71,13,10,26,10]))||bytes.subarray(12,16).toString()!=='IHDR'||bytes.readUInt32BE(16)<320||bytes.readUInt32BE(16)>1920||bytes.readUInt32BE(20)<240||bytes.readUInt32BE(20)>1080)throw new Error('Invalid PNG signature.');return bytes.subarray(0,length);
|
|
141
|
+
}
|
|
142
|
+
let text;
|
|
143
|
+
if (maxResponseBytes !== undefined) {
|
|
144
|
+
if (!Number.isSafeInteger(maxResponseBytes) || maxResponseBytes < 1 || maxResponseBytes > 4 * 1024 * 1024) throw new Error('Invalid response byte limit.');
|
|
145
|
+
const reader=response.body?.getReader();const bytes=Buffer.alloc(maxResponseBytes);let length=0;
|
|
146
|
+
if (reader) {
|
|
147
|
+
try {
|
|
148
|
+
while (true) { const part=await reader.read();if (part.done) break;length+=part.value.byteLength;
|
|
149
|
+
if (length>maxResponseBytes) { await reader.cancel();throw new Error('OpenCrawl response exceeds the allowed byte limit.'); }
|
|
150
|
+
bytes.set(part.value,length-part.value.byteLength);
|
|
151
|
+
}
|
|
152
|
+
} finally { reader.releaseLock(); }
|
|
153
|
+
}
|
|
154
|
+
text=bytes.subarray(0,length).toString('utf8');
|
|
155
|
+
} else text = await response.text();
|
|
156
|
+
let value;
|
|
157
|
+
try { value = text ? JSON.parse(text) : null; } catch { throw new Error(`Server returned an unexpected response (${response.status}).`); }
|
|
158
|
+
if (!response.ok) {
|
|
159
|
+
const detail = typeof value?.detail === 'string' ? value.detail : typeof value?.error === 'string' ? value.error : response.statusText;
|
|
160
|
+
const error=new Error(`OpenCrawl ${response.status}: ${detail}`);error.status=response.status;
|
|
161
|
+
if (typeof value?.code === 'string' && /^[a-z][a-z0-9_]{0,63}$/.test(value.code)) error.code=value.code;
|
|
162
|
+
throw error;
|
|
163
|
+
}
|
|
164
|
+
return value;
|
|
165
|
+
} finally { clearTimeout(timer); signal?.removeEventListener('abort', abort); }
|
|
166
|
+
}};
|
|
167
|
+
}
|
|
168
|
+
export async function search(query, options = {}) {
|
|
169
|
+
const client = await createClient(options);
|
|
170
|
+
return client.request('/api/v1/search', { method: 'POST', body: { query, limit: options.limit || 8, ...(options.source_ids === undefined ? {} : {source_ids: options.source_ids}) } });
|
|
171
|
+
}
|
|
172
|
+
export async function context(query, options = {}) {
|
|
173
|
+
const client = await createClient(options);
|
|
174
|
+
return client.request('/api/v1/context', { method: 'POST', body: { query, limit: options.limit || 8, budget: options.budget || 2000, ...(options.source_ids === undefined ? {} : {source_ids: options.source_ids}) } });
|
|
175
|
+
}
|