@causyne/agent 1.10.13 → 1.10.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/ARCHITECTURE.md +101 -17
- package/docs/SECURITY_REVIEW.md +13 -1
- package/docs/THIRD_PARTY_NOTICES.md +43 -10
- package/package.json +10 -4
- package/src/agentLoop.js +25 -3
- package/src/agentProfiles.js +135 -0
- package/src/agentProfiles.ts +143 -0
- package/src/approvalGrants.js +17 -35
- package/src/approvalGrants.ts +54 -0
- package/src/auditLog.d.ts +3 -0
- package/src/browser/camofoxRuntime.js +93 -0
- package/src/browser/camofoxRuntime.ts +112 -0
- package/src/browser/causyneBrowser.js +495 -0
- package/src/browser/causyneBrowser.ts +594 -0
- package/src/browser/searchProviders.js +156 -0
- package/src/browser/searchProviders.ts +156 -0
- package/src/browser/snapshotWindow.js +28 -0
- package/src/browser/snapshotWindow.ts +40 -0
- package/src/browser/urlPolicy.js +89 -0
- package/src/browser/urlPolicy.ts +89 -0
- package/src/checkpoints.js +19 -6
- package/src/commands.js +37 -8
- package/src/config.d.ts +2 -0
- package/src/hooks.js +121 -107
- package/src/hooks.ts +193 -0
- package/src/mcp/mcpClient.js +520 -132
- package/src/mcp/mcpClient.ts +584 -0
- package/src/mcp/mcpRegistry.js +399 -185
- package/src/mcp/mcpRegistry.ts +451 -0
- package/src/planCache.js +473 -450
- package/src/planCache.ts +840 -0
- package/src/plugins/pluginManager.js +40 -5
- package/src/plugins/pluginSkills.js +84 -0
- package/src/plugins/pluginSkills.ts +106 -0
- package/src/provisioning/capabilityProvisioner.d.ts +10 -0
- package/src/provisioning/capabilityProvisioner.js +5 -0
- package/src/provisioning/capabilityTools.js +2 -1
- package/src/sandbox.js +32 -2
- package/src/sessionJournal.js +471 -0
- package/src/sessionJournal.ts +496 -0
- package/src/sessionStore.js +3 -279
- package/src/skillsManager.js +18 -0
- package/src/subagentJournal.js +215 -0
- package/src/subagentJournal.ts +254 -0
- package/src/taskShape.js +179 -198
- package/src/taskShape.ts +259 -0
- package/src/tools/browserAutomator.js +45 -241
- package/src/tools/browserAutomator.ts +57 -0
- package/src/tools/manageLocalAiDependencies.js +1 -1
- package/src/tools/subagentOrchestrator.js +73 -12
- package/src/tools/webFetch.js +183 -68
- package/src/tools/webFetch.ts +185 -0
- package/src/tools/webSearch.js +50 -75
- package/src/tools/webSearch.ts +56 -0
- package/src/types/contracts.js +8 -0
- package/src/types/contracts.ts +263 -0
- package/src/utils/paths.d.ts +3 -0
- package/src/version.d.ts +1 -0
- package/src/mcp/jsonRpc.js +0 -165
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { TokenTracker } from '../tokenTracker.js';
|
|
2
|
+
import { listAgentProfiles, loadAgentProfile } from '../agentProfiles.js';
|
|
3
|
+
import {
|
|
4
|
+
completeSubagentJournal,
|
|
5
|
+
failSubagentJournal,
|
|
6
|
+
startSubagentJournal
|
|
7
|
+
} from '../subagentJournal.js';
|
|
2
8
|
|
|
3
9
|
// Subagent delegation.
|
|
4
10
|
//
|
|
@@ -67,6 +73,8 @@ async function executeSubagent(record, parentLoop) {
|
|
|
67
73
|
const child = new AgentLoop(transport, record.subagentId, record.prompt);
|
|
68
74
|
|
|
69
75
|
// A delegated task is a bounded errand, not a second full session.
|
|
76
|
+
child.maxStepsCeiling = record.maxSteps;
|
|
77
|
+
child.taskTokenLimit = record.maxTokens;
|
|
70
78
|
child.maxSteps = Math.min(child.maxSteps || 25, record.maxSteps);
|
|
71
79
|
child.maxCircuitBreaker = child.maxSteps;
|
|
72
80
|
child.messages = [];
|
|
@@ -78,8 +86,16 @@ async function executeSubagent(record, parentLoop) {
|
|
|
78
86
|
|
|
79
87
|
record.loop = child;
|
|
80
88
|
record.startedAtMs = Date.now();
|
|
89
|
+
record.journalStarted = startSubagentJournal(record);
|
|
81
90
|
|
|
82
|
-
|
|
91
|
+
try {
|
|
92
|
+
await child.run();
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (record.journalStarted) {
|
|
95
|
+
failSubagentJournal(record.subagentId, error instanceof Error ? error.message : String(error));
|
|
96
|
+
}
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
83
99
|
|
|
84
100
|
const finalAssistant = [...child.messages].reverse()
|
|
85
101
|
.find(m => m.role === 'assistant' && typeof m.content === 'string' && m.content.trim());
|
|
@@ -88,7 +104,7 @@ async function executeSubagent(record, parentLoop) {
|
|
|
88
104
|
try { return JSON.parse(m.content).success === false; } catch (e) { return false; }
|
|
89
105
|
});
|
|
90
106
|
|
|
91
|
-
|
|
107
|
+
const result = {
|
|
92
108
|
subagentId: record.subagentId,
|
|
93
109
|
name: record.name,
|
|
94
110
|
stepsCompleted: child.stepsCompleted,
|
|
@@ -100,6 +116,8 @@ async function executeSubagent(record, parentLoop) {
|
|
|
100
116
|
summary: finalAssistant ? finalAssistant.content : null,
|
|
101
117
|
stoppedEarly: child.stepsCompleted >= child.maxSteps
|
|
102
118
|
};
|
|
119
|
+
if (record.journalStarted) completeSubagentJournal(record.subagentId, child.messages, result);
|
|
120
|
+
return result;
|
|
103
121
|
}
|
|
104
122
|
|
|
105
123
|
export const subagentOrchestrator = {
|
|
@@ -111,15 +129,16 @@ export const subagentOrchestrator = {
|
|
|
111
129
|
properties: {
|
|
112
130
|
action: {
|
|
113
131
|
type: 'string',
|
|
114
|
-
enum: ['spawn', 'list', 'kill', 'delegate_and_wait'],
|
|
115
|
-
description: 'spawn (start in the background
|
|
132
|
+
enum: ['spawn', 'list', 'list_profiles', 'kill', 'delegate_and_wait'],
|
|
133
|
+
description: 'spawn (start in the background), list (show active subagents), list_profiles (show static user-defined agent profiles), kill (stop one), or delegate_and_wait (run and return the result).'
|
|
116
134
|
},
|
|
117
135
|
name: { type: 'string', description: 'A short name for the subagent, e.g. Audit_Worker.' },
|
|
118
136
|
prompt: { type: 'string', description: 'The complete instruction for the subagent. It has no memory of this conversation, so it must be self-contained.' },
|
|
137
|
+
profile: { type: 'string', description: 'Optional profile name from ~/.causyne/agents/<name>.json. Its tool and budget limits cannot be widened by the call.' },
|
|
119
138
|
subagentId: { type: 'string', description: 'Target subagent for the kill action.' },
|
|
120
139
|
toolScope: { type: 'array', items: { type: 'string' }, description: 'Catalogue tool names the subagent may use. Defaults to read-only inspection tools. It can never receive the delegation tool itself.' },
|
|
121
|
-
maxSteps: { type: 'integer', description: '
|
|
122
|
-
maxTokens: { type: 'integer', description: '
|
|
140
|
+
maxSteps: { type: 'integer', description: 'Hard step ceiling for the subagent. Defaults to 25.' },
|
|
141
|
+
maxTokens: { type: 'integer', description: 'Hard per-task token ceiling for the subagent. Defaults to 50000.' },
|
|
123
142
|
depth: { type: 'integer', description: 'Hierarchy depth. 0 = master, 1 = worker. Subagents cannot spawn subagents.' }
|
|
124
143
|
},
|
|
125
144
|
required: ['action']
|
|
@@ -153,6 +172,23 @@ export const subagentOrchestrator = {
|
|
|
153
172
|
};
|
|
154
173
|
}
|
|
155
174
|
|
|
175
|
+
if (action === 'list_profiles') {
|
|
176
|
+
const { profiles, problems } = listAgentProfiles();
|
|
177
|
+
return {
|
|
178
|
+
success: true,
|
|
179
|
+
output: JSON.stringify({
|
|
180
|
+
profiles: profiles.map(profile => ({
|
|
181
|
+
name: profile.name,
|
|
182
|
+
description: profile.description,
|
|
183
|
+
toolScope: profile.toolScope,
|
|
184
|
+
maxSteps: profile.maxSteps,
|
|
185
|
+
maxTokens: profile.maxTokens
|
|
186
|
+
})),
|
|
187
|
+
problems
|
|
188
|
+
}, null, 2)
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
156
192
|
if (action === 'kill') {
|
|
157
193
|
const id = args.subagentId || args.name;
|
|
158
194
|
if (!id) return { success: false, error: 'subagentId or name is required to stop a subagent.' };
|
|
@@ -173,7 +209,17 @@ export const subagentOrchestrator = {
|
|
|
173
209
|
return { success: false, error: `Unknown action: ${action}` };
|
|
174
210
|
}
|
|
175
211
|
|
|
176
|
-
|
|
212
|
+
let profile = null;
|
|
213
|
+
if (args.profile !== undefined) {
|
|
214
|
+
const loaded = loadAgentProfile(args.profile);
|
|
215
|
+
if (!loaded.ok) return { success: false, error: `Agent profile ${JSON.stringify(args.profile)}: ${loaded.error}` };
|
|
216
|
+
profile = loaded.profile;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const requestedPrompt = args.prompt;
|
|
220
|
+
const prompt = profile
|
|
221
|
+
? `${profile.instructions}\n\nDelegated task:\n${String(requestedPrompt || '')}`
|
|
222
|
+
: requestedPrompt;
|
|
177
223
|
if (!prompt) return { success: false, error: 'A prompt is required to delegate a task.' };
|
|
178
224
|
|
|
179
225
|
const depth = args.depth !== undefined ? args.depth : 1;
|
|
@@ -195,11 +241,19 @@ export const subagentOrchestrator = {
|
|
|
195
241
|
};
|
|
196
242
|
}
|
|
197
243
|
|
|
198
|
-
const name = args.name || `Worker_${Math.random().toString(36).slice(2, 7)}`;
|
|
244
|
+
const name = args.name || profile?.name || `Worker_${Math.random().toString(36).slice(2, 7)}`;
|
|
199
245
|
const subagentId = `sub_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
200
|
-
|
|
246
|
+
let requestedScope = Array.isArray(args.toolScope) && args.toolScope.length > 0
|
|
201
247
|
? args.toolScope
|
|
202
|
-
: DEFAULT_TOOL_SCOPE;
|
|
248
|
+
: profile?.toolScope || DEFAULT_TOOL_SCOPE;
|
|
249
|
+
if (profile) {
|
|
250
|
+
const ceiling = new Set(profile.toolScope);
|
|
251
|
+
requestedScope = requestedScope.filter(tool => ceiling.has(String(tool).replace(/\./g, '_')));
|
|
252
|
+
if (requestedScope.length === 0) {
|
|
253
|
+
return { success: false, error: `Agent profile ${profile.name} permits none of the requested tools.` };
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
requestedScope = [...new Set(requestedScope.map(tool => String(tool).replace(/\./g, '_')))];
|
|
203
257
|
|
|
204
258
|
// Reject an unknown tool name instead of silently accepting it. The old
|
|
205
259
|
// implementation accepted anything, which is easy to do when the scope is
|
|
@@ -211,14 +265,21 @@ export const subagentOrchestrator = {
|
|
|
211
265
|
return { success: false, error: `Unknown tools in toolScope: ${unknown.join(', ')}.` };
|
|
212
266
|
}
|
|
213
267
|
|
|
268
|
+
const requestedMaxSteps = Number.isInteger(args.maxSteps)
|
|
269
|
+
? Math.max(1, Math.min(args.maxSteps, 100))
|
|
270
|
+
: profile?.maxSteps || 25;
|
|
271
|
+
const requestedMaxTokens = Number.isInteger(args.maxTokens)
|
|
272
|
+
? Math.max(1_000, Math.min(args.maxTokens, 200_000))
|
|
273
|
+
: profile?.maxTokens || 50_000;
|
|
214
274
|
const record = {
|
|
215
275
|
subagentId,
|
|
216
276
|
name,
|
|
217
277
|
prompt,
|
|
218
278
|
toolScope: requestedScope,
|
|
219
279
|
depth,
|
|
220
|
-
maxSteps:
|
|
221
|
-
maxTokens:
|
|
280
|
+
maxSteps: Math.min(requestedMaxSteps, profile?.maxSteps || 100),
|
|
281
|
+
maxTokens: Math.min(requestedMaxTokens, profile?.maxTokens || 200_000),
|
|
282
|
+
profile: profile?.name || null,
|
|
222
283
|
status: 'running',
|
|
223
284
|
startTime: new Date().toISOString(),
|
|
224
285
|
summary: null,
|
package/src/tools/webFetch.js
CHANGED
|
@@ -1,74 +1,189 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { BrowserCapabilityError, causyneBrowser } from '../browser/causyneBrowser.js';
|
|
5
|
+
import { assertPublicWebUrl } from '../browser/urlPolicy.js';
|
|
6
|
+
const MAX_REDIRECTS = 5;
|
|
7
|
+
const MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
|
|
8
|
+
const MAX_OUTPUT_CHARS = 80_000;
|
|
9
|
+
const REQUEST_TIMEOUT_MS = 30_000;
|
|
10
|
+
function decodeEntities(text) {
|
|
11
|
+
const named = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
|
|
12
|
+
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (full, entity) => {
|
|
13
|
+
if (entity.startsWith('#x'))
|
|
14
|
+
return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
|
|
15
|
+
if (entity.startsWith('#'))
|
|
16
|
+
return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
|
|
17
|
+
return named[entity.toLowerCase()] ?? full;
|
|
18
|
+
});
|
|
16
19
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
20
|
+
export function htmlToReadableText(html) {
|
|
21
|
+
return decodeEntities(String(html ?? ''))
|
|
22
|
+
.replace(/<(script|style|svg|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
|
|
23
|
+
.replace(/<\s*br\s*\/?>/gi, '\n')
|
|
24
|
+
.replace(/<\/(p|div|article|section|li|h[1-6]|tr)>/gi, '\n')
|
|
25
|
+
.replace(/<[^>]+>/g, ' ')
|
|
26
|
+
.replace(/[ \t]+/g, ' ')
|
|
27
|
+
.replace(/ *\n */g, '\n')
|
|
28
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
29
|
+
.trim();
|
|
30
|
+
}
|
|
31
|
+
function isWithin(root, candidate) {
|
|
32
|
+
const relative = path.relative(root, candidate);
|
|
33
|
+
return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative));
|
|
34
|
+
}
|
|
35
|
+
function validatedHeaders(value) {
|
|
36
|
+
if (value === undefined)
|
|
37
|
+
return {};
|
|
38
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
39
|
+
throw new Error('headers must be an object');
|
|
40
|
+
const result = {};
|
|
41
|
+
for (const [key, item] of Object.entries(value)) {
|
|
42
|
+
if (typeof item !== 'string' || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(key) || /[\r\n]/.test(item)) {
|
|
43
|
+
throw new Error('headers contain an invalid name or value');
|
|
44
|
+
}
|
|
45
|
+
result[key] = item;
|
|
32
46
|
}
|
|
33
|
-
|
|
34
|
-
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
async function fetchFollowingSafeRedirects(urlValue, options) {
|
|
50
|
+
let url = await assertPublicWebUrl(urlValue);
|
|
51
|
+
let request = { ...options };
|
|
52
|
+
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect++) {
|
|
53
|
+
const response = await fetch(url, { ...request, redirect: 'manual' });
|
|
54
|
+
if (![301, 302, 303, 307, 308].includes(response.status))
|
|
55
|
+
return { response, finalUrl: url };
|
|
56
|
+
if (redirect === MAX_REDIRECTS)
|
|
57
|
+
throw new Error(`Too many redirects (maximum ${MAX_REDIRECTS})`);
|
|
58
|
+
const location = response.headers.get('location');
|
|
59
|
+
if (!location)
|
|
60
|
+
throw new Error(`HTTP ${response.status} redirect has no Location header`);
|
|
61
|
+
await response.body?.cancel().catch(() => { });
|
|
62
|
+
url = await assertPublicWebUrl(new URL(location, url).href);
|
|
63
|
+
if (response.status === 303 || ([301, 302].includes(response.status) && request.method === 'POST')) {
|
|
64
|
+
const redirectedHeaders = new Headers(request.headers);
|
|
65
|
+
redirectedHeaders.delete('content-length');
|
|
66
|
+
redirectedHeaders.delete('content-type');
|
|
67
|
+
const { body: _discardedBody, ...redirectedRequest } = request;
|
|
68
|
+
request = { ...redirectedRequest, method: 'GET', headers: redirectedHeaders };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
throw new Error('Redirect handling failed');
|
|
72
|
+
}
|
|
73
|
+
async function readBoundedBody(response) {
|
|
74
|
+
if (!response.body)
|
|
75
|
+
return Buffer.alloc(0);
|
|
76
|
+
const reader = response.body.getReader();
|
|
77
|
+
const chunks = [];
|
|
78
|
+
let size = 0;
|
|
35
79
|
try {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
80
|
+
while (true) {
|
|
81
|
+
const { done, value } = await reader.read();
|
|
82
|
+
if (done)
|
|
83
|
+
break;
|
|
84
|
+
size += value.byteLength;
|
|
85
|
+
if (size > MAX_RESPONSE_BYTES) {
|
|
86
|
+
await reader.cancel(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`).catch(() => { });
|
|
87
|
+
throw new Error(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`);
|
|
88
|
+
}
|
|
89
|
+
chunks.push(value);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
reader.releaseLock();
|
|
94
|
+
}
|
|
95
|
+
return Buffer.concat(chunks.map(chunk => Buffer.from(chunk)), size);
|
|
96
|
+
}
|
|
97
|
+
async function browserFallback(url) {
|
|
98
|
+
try {
|
|
99
|
+
const results = await causyneBrowser().run([{ type: 'goto', url }, { type: 'extract', selector: 'body' }]);
|
|
100
|
+
const output = results.find(result => result.action === 'extract')?.output;
|
|
101
|
+
return output ? { success: true, output } : { success: false, error: 'Browser loaded the page but extracted no readable content' };
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (error instanceof BrowserCapabilityError) {
|
|
105
|
+
return { success: false, error: error.message, requiresConsent: error.requiresConsent };
|
|
106
|
+
}
|
|
107
|
+
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
export const webFetch = {
|
|
111
|
+
schema: {
|
|
112
|
+
name: 'web.fetch',
|
|
113
|
+
description: 'Fetch a public HTTP(S) resource with redirect-by-redirect SSRF checks, bounded output, browser fallback for blocked HTML, and atomic workspace-scoped downloads.',
|
|
114
|
+
parameters: {
|
|
115
|
+
type: 'object',
|
|
116
|
+
properties: {
|
|
117
|
+
url: { type: 'string' },
|
|
118
|
+
method: { type: 'string', enum: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] },
|
|
119
|
+
headers: { type: 'object', additionalProperties: { type: 'string' } },
|
|
120
|
+
body: { type: 'string' },
|
|
121
|
+
outputFile: { type: 'string', description: 'File destination inside the selected workspace.' }
|
|
122
|
+
},
|
|
123
|
+
required: ['url'],
|
|
124
|
+
additionalProperties: false
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
async execute(args) {
|
|
128
|
+
const controller = new AbortController();
|
|
129
|
+
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
130
|
+
try {
|
|
131
|
+
if (!args.url)
|
|
132
|
+
return { success: false, error: 'url is required' };
|
|
133
|
+
const method = (args.method ?? 'GET').toUpperCase();
|
|
134
|
+
const headers = {
|
|
135
|
+
'User-Agent': 'Causyne-Agent/1.10 (+https://causyne.com)',
|
|
136
|
+
Accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*;q=0.8',
|
|
137
|
+
...validatedHeaders(args.headers)
|
|
138
|
+
};
|
|
139
|
+
const request = { method, headers, signal: controller.signal };
|
|
140
|
+
if (args.body !== undefined && method !== 'GET' && method !== 'HEAD')
|
|
141
|
+
request.body = args.body;
|
|
142
|
+
const { response, finalUrl } = await fetchFollowingSafeRedirects(args.url, request);
|
|
143
|
+
const contentType = response.headers.get('content-type')?.toLowerCase() ?? '';
|
|
144
|
+
if (!response.ok && !args.outputFile && contentType.includes('text/html') && [401, 403, 429, 503].includes(response.status)) {
|
|
145
|
+
const fallback = await browserFallback(finalUrl.href);
|
|
146
|
+
if (fallback.success)
|
|
147
|
+
return { ...fallback, status: response.status, finalUrl: finalUrl.href, via: 'camoufox' };
|
|
148
|
+
return { success: false, error: `HTTP ${response.status}; browser fallback failed: ${fallback.error}`, requiresConsent: fallback.requiresConsent };
|
|
149
|
+
}
|
|
150
|
+
if (!response.ok)
|
|
151
|
+
return { success: false, error: `HTTP ${response.status} ${response.statusText}`.trim(), status: response.status, finalUrl: finalUrl.href };
|
|
152
|
+
const declaredLength = Number(response.headers.get('content-length') ?? '0');
|
|
153
|
+
if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) {
|
|
154
|
+
return { success: false, error: `Response exceeds ${MAX_RESPONSE_BYTES} bytes` };
|
|
155
|
+
}
|
|
156
|
+
const bytes = await readBoundedBody(response);
|
|
157
|
+
if (args.outputFile) {
|
|
158
|
+
const workspace = path.resolve(process.cwd());
|
|
159
|
+
const target = path.resolve(workspace, args.outputFile);
|
|
160
|
+
if (!isWithin(workspace, target))
|
|
161
|
+
return { success: false, error: 'Download target must stay inside the selected workspace' };
|
|
162
|
+
const temporary = `${target}.part-${crypto.randomUUID()}`;
|
|
163
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
164
|
+
try {
|
|
165
|
+
await fs.writeFile(temporary, bytes, { mode: 0o600 });
|
|
166
|
+
await fs.rename(temporary, target);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
await fs.unlink(temporary).catch(() => { });
|
|
170
|
+
}
|
|
171
|
+
return { success: true, output: `Downloaded ${bytes.length} bytes to ${target}`, status: response.status, finalUrl: finalUrl.href };
|
|
172
|
+
}
|
|
173
|
+
const decoded = bytes.toString('utf8');
|
|
174
|
+
const readable = contentType.includes('text/html') ? htmlToReadableText(decoded) : decoded;
|
|
175
|
+
const output = readable.length > MAX_OUTPUT_CHARS
|
|
176
|
+
? `${readable.slice(0, MAX_OUTPUT_CHARS)}\n\n[Web output truncated at ${MAX_OUTPUT_CHARS} characters]`
|
|
177
|
+
: readable;
|
|
178
|
+
return { success: true, output, status: response.status, finalUrl: finalUrl.href, contentType };
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
182
|
+
const timedOut = error instanceof Error && (error.name === 'AbortError' || message === 'This operation was aborted');
|
|
183
|
+
return { success: false, error: timedOut ? `Request timed out after ${REQUEST_TIMEOUT_MS}ms` : message };
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
clearTimeout(timer);
|
|
41
187
|
}
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
if (args.body) options.body = args.body;
|
|
45
|
-
|
|
46
|
-
const response = await fetch(args.url, options);
|
|
47
|
-
|
|
48
|
-
if (args.outputFile) {
|
|
49
|
-
const target = args.outputFile.replace(/^~\//, os.homedir() + '/');
|
|
50
|
-
const buffer = await response.arrayBuffer();
|
|
51
|
-
await fs.writeFile(target, Buffer.from(buffer));
|
|
52
|
-
return { success: true, output: `Downloaded ${buffer.byteLength} bytes to ${args.outputFile}`, status: response.status };
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
const text = await response.text();
|
|
56
|
-
|
|
57
|
-
// Phase 3 AGI Upgrade: Clean HTML unless it's an API JSON response
|
|
58
|
-
let cleanOutput = text;
|
|
59
|
-
const contentType = response.headers.get('content-type') || '';
|
|
60
|
-
if (contentType.includes('text/html')) {
|
|
61
|
-
cleanOutput = htmlToText(text);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// Strict Context Paging Protection
|
|
65
|
-
if (cleanOutput.length > 50000) {
|
|
66
|
-
cleanOutput = cleanOutput.substring(0, 50000) + '\n\n[... OUTPUT TRUNCATED TO 50KB ...]';
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return { success: true, output: cleanOutput, status: response.status };
|
|
70
|
-
} catch (e) {
|
|
71
|
-
return { success: false, error: e.message };
|
|
72
188
|
}
|
|
73
|
-
}
|
|
74
189
|
};
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
import { BrowserCapabilityError, causyneBrowser } from '../browser/causyneBrowser.js';
|
|
6
|
+
import { assertPublicWebUrl } from '../browser/urlPolicy.js';
|
|
7
|
+
|
|
8
|
+
const MAX_REDIRECTS = 5;
|
|
9
|
+
const MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
|
|
10
|
+
const MAX_OUTPUT_CHARS = 80_000;
|
|
11
|
+
const REQUEST_TIMEOUT_MS = 30_000;
|
|
12
|
+
|
|
13
|
+
function decodeEntities(text: string): string {
|
|
14
|
+
const named: Record<string, string> = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
|
|
15
|
+
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (full, entity: string) => {
|
|
16
|
+
if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
|
|
17
|
+
if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
|
|
18
|
+
return named[entity.toLowerCase()] ?? full;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function htmlToReadableText(html: string): string {
|
|
23
|
+
return decodeEntities(String(html ?? ''))
|
|
24
|
+
.replace(/<(script|style|svg|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
|
|
25
|
+
.replace(/<\s*br\s*\/?>/gi, '\n')
|
|
26
|
+
.replace(/<\/(p|div|article|section|li|h[1-6]|tr)>/gi, '\n')
|
|
27
|
+
.replace(/<[^>]+>/g, ' ')
|
|
28
|
+
.replace(/[ \t]+/g, ' ')
|
|
29
|
+
.replace(/ *\n */g, '\n')
|
|
30
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
31
|
+
.trim();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function isWithin(root: string, candidate: string): boolean {
|
|
35
|
+
const relative = path.relative(root, candidate);
|
|
36
|
+
return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function validatedHeaders(value: unknown): Record<string, string> {
|
|
40
|
+
if (value === undefined) return {};
|
|
41
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('headers must be an object');
|
|
42
|
+
const result: Record<string, string> = {};
|
|
43
|
+
for (const [key, item] of Object.entries(value)) {
|
|
44
|
+
if (typeof item !== 'string' || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(key) || /[\r\n]/.test(item)) {
|
|
45
|
+
throw new Error('headers contain an invalid name or value');
|
|
46
|
+
}
|
|
47
|
+
result[key] = item;
|
|
48
|
+
}
|
|
49
|
+
return result;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function fetchFollowingSafeRedirects(urlValue: string, options: RequestInit): Promise<{ response: Response; finalUrl: URL }> {
|
|
53
|
+
let url = await assertPublicWebUrl(urlValue);
|
|
54
|
+
let request = { ...options };
|
|
55
|
+
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect++) {
|
|
56
|
+
const response = await fetch(url, { ...request, redirect: 'manual' });
|
|
57
|
+
if (![301, 302, 303, 307, 308].includes(response.status)) return { response, finalUrl: url };
|
|
58
|
+
if (redirect === MAX_REDIRECTS) throw new Error(`Too many redirects (maximum ${MAX_REDIRECTS})`);
|
|
59
|
+
const location = response.headers.get('location');
|
|
60
|
+
if (!location) throw new Error(`HTTP ${response.status} redirect has no Location header`);
|
|
61
|
+
await response.body?.cancel().catch(() => {});
|
|
62
|
+
url = await assertPublicWebUrl(new URL(location, url).href);
|
|
63
|
+
if (response.status === 303 || ([301, 302].includes(response.status) && request.method === 'POST')) {
|
|
64
|
+
const redirectedHeaders = new Headers(request.headers);
|
|
65
|
+
redirectedHeaders.delete('content-length');
|
|
66
|
+
redirectedHeaders.delete('content-type');
|
|
67
|
+
const { body: _discardedBody, ...redirectedRequest } = request;
|
|
68
|
+
request = { ...redirectedRequest, method: 'GET', headers: redirectedHeaders };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
throw new Error('Redirect handling failed');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function readBoundedBody(response: Response): Promise<Buffer> {
|
|
75
|
+
if (!response.body) return Buffer.alloc(0);
|
|
76
|
+
const reader = response.body.getReader();
|
|
77
|
+
const chunks: Uint8Array[] = [];
|
|
78
|
+
let size = 0;
|
|
79
|
+
try {
|
|
80
|
+
while (true) {
|
|
81
|
+
const { done, value } = await reader.read();
|
|
82
|
+
if (done) break;
|
|
83
|
+
size += value.byteLength;
|
|
84
|
+
if (size > MAX_RESPONSE_BYTES) {
|
|
85
|
+
await reader.cancel(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`).catch(() => {});
|
|
86
|
+
throw new Error(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`);
|
|
87
|
+
}
|
|
88
|
+
chunks.push(value);
|
|
89
|
+
}
|
|
90
|
+
} finally {
|
|
91
|
+
reader.releaseLock();
|
|
92
|
+
}
|
|
93
|
+
return Buffer.concat(chunks.map(chunk => Buffer.from(chunk)), size);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function browserFallback(url: string): Promise<{ success: boolean; output?: string; error?: string; requiresConsent?: boolean }> {
|
|
97
|
+
try {
|
|
98
|
+
const results = await causyneBrowser().run([{ type: 'goto', url }, { type: 'extract', selector: 'body' }]);
|
|
99
|
+
const output = results.find(result => result.action === 'extract')?.output;
|
|
100
|
+
return output ? { success: true, output } : { success: false, error: 'Browser loaded the page but extracted no readable content' };
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (error instanceof BrowserCapabilityError) {
|
|
103
|
+
return { success: false, error: error.message, requiresConsent: error.requiresConsent };
|
|
104
|
+
}
|
|
105
|
+
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export const webFetch = {
|
|
110
|
+
schema: {
|
|
111
|
+
name: 'web.fetch',
|
|
112
|
+
description: 'Fetch a public HTTP(S) resource with redirect-by-redirect SSRF checks, bounded output, browser fallback for blocked HTML, and atomic workspace-scoped downloads.',
|
|
113
|
+
parameters: {
|
|
114
|
+
type: 'object',
|
|
115
|
+
properties: {
|
|
116
|
+
url: { type: 'string' },
|
|
117
|
+
method: { type: 'string', enum: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] },
|
|
118
|
+
headers: { type: 'object', additionalProperties: { type: 'string' } },
|
|
119
|
+
body: { type: 'string' },
|
|
120
|
+
outputFile: { type: 'string', description: 'File destination inside the selected workspace.' }
|
|
121
|
+
},
|
|
122
|
+
required: ['url'],
|
|
123
|
+
additionalProperties: false
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
async execute(args: { url?: string; method?: string; headers?: unknown; body?: string; outputFile?: string }) {
|
|
128
|
+
const controller = new AbortController();
|
|
129
|
+
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
130
|
+
try {
|
|
131
|
+
if (!args.url) return { success: false, error: 'url is required' };
|
|
132
|
+
const method = (args.method ?? 'GET').toUpperCase();
|
|
133
|
+
const headers = {
|
|
134
|
+
'User-Agent': 'Causyne-Agent/1.10 (+https://causyne.com)',
|
|
135
|
+
Accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*;q=0.8',
|
|
136
|
+
...validatedHeaders(args.headers)
|
|
137
|
+
};
|
|
138
|
+
const request: RequestInit = { method, headers, signal: controller.signal };
|
|
139
|
+
if (args.body !== undefined && method !== 'GET' && method !== 'HEAD') request.body = args.body;
|
|
140
|
+
const { response, finalUrl } = await fetchFollowingSafeRedirects(args.url, request);
|
|
141
|
+
const contentType = response.headers.get('content-type')?.toLowerCase() ?? '';
|
|
142
|
+
|
|
143
|
+
if (!response.ok && !args.outputFile && contentType.includes('text/html') && [401, 403, 429, 503].includes(response.status)) {
|
|
144
|
+
const fallback = await browserFallback(finalUrl.href);
|
|
145
|
+
if (fallback.success) return { ...fallback, status: response.status, finalUrl: finalUrl.href, via: 'camoufox' };
|
|
146
|
+
return { success: false, error: `HTTP ${response.status}; browser fallback failed: ${fallback.error}`, requiresConsent: fallback.requiresConsent };
|
|
147
|
+
}
|
|
148
|
+
if (!response.ok) return { success: false, error: `HTTP ${response.status} ${response.statusText}`.trim(), status: response.status, finalUrl: finalUrl.href };
|
|
149
|
+
|
|
150
|
+
const declaredLength = Number(response.headers.get('content-length') ?? '0');
|
|
151
|
+
if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) {
|
|
152
|
+
return { success: false, error: `Response exceeds ${MAX_RESPONSE_BYTES} bytes` };
|
|
153
|
+
}
|
|
154
|
+
const bytes = await readBoundedBody(response);
|
|
155
|
+
|
|
156
|
+
if (args.outputFile) {
|
|
157
|
+
const workspace = path.resolve(process.cwd());
|
|
158
|
+
const target = path.resolve(workspace, args.outputFile);
|
|
159
|
+
if (!isWithin(workspace, target)) return { success: false, error: 'Download target must stay inside the selected workspace' };
|
|
160
|
+
const temporary = `${target}.part-${crypto.randomUUID()}`;
|
|
161
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
162
|
+
try {
|
|
163
|
+
await fs.writeFile(temporary, bytes, { mode: 0o600 });
|
|
164
|
+
await fs.rename(temporary, target);
|
|
165
|
+
} finally {
|
|
166
|
+
await fs.unlink(temporary).catch(() => {});
|
|
167
|
+
}
|
|
168
|
+
return { success: true, output: `Downloaded ${bytes.length} bytes to ${target}`, status: response.status, finalUrl: finalUrl.href };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const decoded = bytes.toString('utf8');
|
|
172
|
+
const readable = contentType.includes('text/html') ? htmlToReadableText(decoded) : decoded;
|
|
173
|
+
const output = readable.length > MAX_OUTPUT_CHARS
|
|
174
|
+
? `${readable.slice(0, MAX_OUTPUT_CHARS)}\n\n[Web output truncated at ${MAX_OUTPUT_CHARS} characters]`
|
|
175
|
+
: readable;
|
|
176
|
+
return { success: true, output, status: response.status, finalUrl: finalUrl.href, contentType };
|
|
177
|
+
} catch (error) {
|
|
178
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
179
|
+
const timedOut = error instanceof Error && (error.name === 'AbortError' || message === 'This operation was aborted');
|
|
180
|
+
return { success: false, error: timedOut ? `Request timed out after ${REQUEST_TIMEOUT_MS}ms` : message };
|
|
181
|
+
} finally {
|
|
182
|
+
clearTimeout(timer);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
};
|