@causyne/agent 1.10.13 → 1.10.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/docs/ARCHITECTURE.md +101 -17
  2. package/docs/SECURITY_REVIEW.md +13 -1
  3. package/docs/THIRD_PARTY_NOTICES.md +43 -10
  4. package/package.json +10 -4
  5. package/src/agentLoop.js +25 -3
  6. package/src/agentProfiles.js +135 -0
  7. package/src/agentProfiles.ts +143 -0
  8. package/src/approvalGrants.js +17 -35
  9. package/src/approvalGrants.ts +54 -0
  10. package/src/auditLog.d.ts +3 -0
  11. package/src/browser/camofoxRuntime.js +93 -0
  12. package/src/browser/camofoxRuntime.ts +112 -0
  13. package/src/browser/causyneBrowser.js +495 -0
  14. package/src/browser/causyneBrowser.ts +594 -0
  15. package/src/browser/searchProviders.js +156 -0
  16. package/src/browser/searchProviders.ts +156 -0
  17. package/src/browser/snapshotWindow.js +28 -0
  18. package/src/browser/snapshotWindow.ts +40 -0
  19. package/src/browser/urlPolicy.js +89 -0
  20. package/src/browser/urlPolicy.ts +89 -0
  21. package/src/checkpoints.js +19 -6
  22. package/src/commands.js +37 -8
  23. package/src/config.d.ts +2 -0
  24. package/src/hooks.js +121 -107
  25. package/src/hooks.ts +193 -0
  26. package/src/mcp/mcpClient.js +520 -132
  27. package/src/mcp/mcpClient.ts +584 -0
  28. package/src/mcp/mcpRegistry.js +399 -185
  29. package/src/mcp/mcpRegistry.ts +451 -0
  30. package/src/planCache.js +473 -450
  31. package/src/planCache.ts +840 -0
  32. package/src/plugins/pluginManager.js +40 -5
  33. package/src/plugins/pluginSkills.js +84 -0
  34. package/src/plugins/pluginSkills.ts +106 -0
  35. package/src/provisioning/capabilityProvisioner.d.ts +10 -0
  36. package/src/provisioning/capabilityProvisioner.js +5 -0
  37. package/src/provisioning/capabilityTools.js +2 -1
  38. package/src/sandbox.js +32 -2
  39. package/src/sessionJournal.js +471 -0
  40. package/src/sessionJournal.ts +496 -0
  41. package/src/sessionStore.js +3 -279
  42. package/src/skillsManager.js +18 -0
  43. package/src/subagentJournal.js +215 -0
  44. package/src/subagentJournal.ts +254 -0
  45. package/src/taskShape.js +179 -198
  46. package/src/taskShape.ts +259 -0
  47. package/src/tools/browserAutomator.js +45 -241
  48. package/src/tools/browserAutomator.ts +57 -0
  49. package/src/tools/manageLocalAiDependencies.js +1 -1
  50. package/src/tools/subagentOrchestrator.js +73 -12
  51. package/src/tools/webFetch.js +183 -68
  52. package/src/tools/webFetch.ts +185 -0
  53. package/src/tools/webSearch.js +50 -75
  54. package/src/tools/webSearch.ts +56 -0
  55. package/src/types/contracts.js +8 -0
  56. package/src/types/contracts.ts +263 -0
  57. package/src/utils/paths.d.ts +3 -0
  58. package/src/version.d.ts +1 -0
  59. package/src/mcp/jsonRpc.js +0 -165
@@ -1,4 +1,10 @@
1
1
  import { TokenTracker } from '../tokenTracker.js';
2
+ import { listAgentProfiles, loadAgentProfile } from '../agentProfiles.js';
3
+ import {
4
+ completeSubagentJournal,
5
+ failSubagentJournal,
6
+ startSubagentJournal
7
+ } from '../subagentJournal.js';
2
8
 
3
9
  // Subagent delegation.
4
10
  //
@@ -67,6 +73,8 @@ async function executeSubagent(record, parentLoop) {
67
73
  const child = new AgentLoop(transport, record.subagentId, record.prompt);
68
74
 
69
75
  // A delegated task is a bounded errand, not a second full session.
76
+ child.maxStepsCeiling = record.maxSteps;
77
+ child.taskTokenLimit = record.maxTokens;
70
78
  child.maxSteps = Math.min(child.maxSteps || 25, record.maxSteps);
71
79
  child.maxCircuitBreaker = child.maxSteps;
72
80
  child.messages = [];
@@ -78,8 +86,16 @@ async function executeSubagent(record, parentLoop) {
78
86
 
79
87
  record.loop = child;
80
88
  record.startedAtMs = Date.now();
89
+ record.journalStarted = startSubagentJournal(record);
81
90
 
82
- await child.run();
91
+ try {
92
+ await child.run();
93
+ } catch (error) {
94
+ if (record.journalStarted) {
95
+ failSubagentJournal(record.subagentId, error instanceof Error ? error.message : String(error));
96
+ }
97
+ throw error;
98
+ }
83
99
 
84
100
  const finalAssistant = [...child.messages].reverse()
85
101
  .find(m => m.role === 'assistant' && typeof m.content === 'string' && m.content.trim());
@@ -88,7 +104,7 @@ async function executeSubagent(record, parentLoop) {
88
104
  try { return JSON.parse(m.content).success === false; } catch (e) { return false; }
89
105
  });
90
106
 
91
- return {
107
+ const result = {
92
108
  subagentId: record.subagentId,
93
109
  name: record.name,
94
110
  stepsCompleted: child.stepsCompleted,
@@ -100,6 +116,8 @@ async function executeSubagent(record, parentLoop) {
100
116
  summary: finalAssistant ? finalAssistant.content : null,
101
117
  stoppedEarly: child.stepsCompleted >= child.maxSteps
102
118
  };
119
+ if (record.journalStarted) completeSubagentJournal(record.subagentId, child.messages, result);
120
+ return result;
103
121
  }
104
122
 
105
123
  export const subagentOrchestrator = {
@@ -111,15 +129,16 @@ export const subagentOrchestrator = {
111
129
  properties: {
112
130
  action: {
113
131
  type: 'string',
114
- enum: ['spawn', 'list', 'kill', 'delegate_and_wait'],
115
- description: 'spawn (start in the background and return immediately), list (show active subagents), kill (stop one), delegate_and_wait (run it and return the result).'
132
+ enum: ['spawn', 'list', 'list_profiles', 'kill', 'delegate_and_wait'],
133
+ description: 'spawn (start in the background), list (show active subagents), list_profiles (show static user-defined agent profiles), kill (stop one), or delegate_and_wait (run and return the result).'
116
134
  },
117
135
  name: { type: 'string', description: 'A short name for the subagent, e.g. Audit_Worker.' },
118
136
  prompt: { type: 'string', description: 'The complete instruction for the subagent. It has no memory of this conversation, so it must be self-contained.' },
137
+ profile: { type: 'string', description: 'Optional profile name from ~/.causyne/agents/<name>.json. Its tool and budget limits cannot be widened by the call.' },
119
138
  subagentId: { type: 'string', description: 'Target subagent for the kill action.' },
120
139
  toolScope: { type: 'array', items: { type: 'string' }, description: 'Catalogue tool names the subagent may use. Defaults to read-only inspection tools. It can never receive the delegation tool itself.' },
121
- maxSteps: { type: 'integer', description: 'Step ceiling for the subagent. Defaults to 25.' },
122
- maxTokens: { type: 'integer', description: 'Advisory token budget for this subagent. Defaults to 50000.' },
140
+ maxSteps: { type: 'integer', description: 'Hard step ceiling for the subagent. Defaults to 25.' },
141
+ maxTokens: { type: 'integer', description: 'Hard per-task token ceiling for the subagent. Defaults to 50000.' },
123
142
  depth: { type: 'integer', description: 'Hierarchy depth. 0 = master, 1 = worker. Subagents cannot spawn subagents.' }
124
143
  },
125
144
  required: ['action']
@@ -153,6 +172,23 @@ export const subagentOrchestrator = {
153
172
  };
154
173
  }
155
174
 
175
+ if (action === 'list_profiles') {
176
+ const { profiles, problems } = listAgentProfiles();
177
+ return {
178
+ success: true,
179
+ output: JSON.stringify({
180
+ profiles: profiles.map(profile => ({
181
+ name: profile.name,
182
+ description: profile.description,
183
+ toolScope: profile.toolScope,
184
+ maxSteps: profile.maxSteps,
185
+ maxTokens: profile.maxTokens
186
+ })),
187
+ problems
188
+ }, null, 2)
189
+ };
190
+ }
191
+
156
192
  if (action === 'kill') {
157
193
  const id = args.subagentId || args.name;
158
194
  if (!id) return { success: false, error: 'subagentId or name is required to stop a subagent.' };
@@ -173,7 +209,17 @@ export const subagentOrchestrator = {
173
209
  return { success: false, error: `Unknown action: ${action}` };
174
210
  }
175
211
 
176
- const prompt = args.prompt;
212
+ let profile = null;
213
+ if (args.profile !== undefined) {
214
+ const loaded = loadAgentProfile(args.profile);
215
+ if (!loaded.ok) return { success: false, error: `Agent profile ${JSON.stringify(args.profile)}: ${loaded.error}` };
216
+ profile = loaded.profile;
217
+ }
218
+
219
+ const requestedPrompt = args.prompt;
220
+ const prompt = profile
221
+ ? `${profile.instructions}\n\nDelegated task:\n${String(requestedPrompt || '')}`
222
+ : requestedPrompt;
177
223
  if (!prompt) return { success: false, error: 'A prompt is required to delegate a task.' };
178
224
 
179
225
  const depth = args.depth !== undefined ? args.depth : 1;
@@ -195,11 +241,19 @@ export const subagentOrchestrator = {
195
241
  };
196
242
  }
197
243
 
198
- const name = args.name || `Worker_${Math.random().toString(36).slice(2, 7)}`;
244
+ const name = args.name || profile?.name || `Worker_${Math.random().toString(36).slice(2, 7)}`;
199
245
  const subagentId = `sub_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
200
- const requestedScope = Array.isArray(args.toolScope) && args.toolScope.length > 0
246
+ let requestedScope = Array.isArray(args.toolScope) && args.toolScope.length > 0
201
247
  ? args.toolScope
202
- : DEFAULT_TOOL_SCOPE;
248
+ : profile?.toolScope || DEFAULT_TOOL_SCOPE;
249
+ if (profile) {
250
+ const ceiling = new Set(profile.toolScope);
251
+ requestedScope = requestedScope.filter(tool => ceiling.has(String(tool).replace(/\./g, '_')));
252
+ if (requestedScope.length === 0) {
253
+ return { success: false, error: `Agent profile ${profile.name} permits none of the requested tools.` };
254
+ }
255
+ }
256
+ requestedScope = [...new Set(requestedScope.map(tool => String(tool).replace(/\./g, '_')))];
203
257
 
204
258
  // Reject an unknown tool name instead of silently accepting it. The old
205
259
  // implementation accepted anything, which is easy to do when the scope is
@@ -211,14 +265,21 @@ export const subagentOrchestrator = {
211
265
  return { success: false, error: `Unknown tools in toolScope: ${unknown.join(', ')}.` };
212
266
  }
213
267
 
268
+ const requestedMaxSteps = Number.isInteger(args.maxSteps)
269
+ ? Math.max(1, Math.min(args.maxSteps, 100))
270
+ : profile?.maxSteps || 25;
271
+ const requestedMaxTokens = Number.isInteger(args.maxTokens)
272
+ ? Math.max(1_000, Math.min(args.maxTokens, 200_000))
273
+ : profile?.maxTokens || 50_000;
214
274
  const record = {
215
275
  subagentId,
216
276
  name,
217
277
  prompt,
218
278
  toolScope: requestedScope,
219
279
  depth,
220
- maxSteps: args.maxSteps || 25,
221
- maxTokens: args.maxTokens || 50000,
280
+ maxSteps: Math.min(requestedMaxSteps, profile?.maxSteps || 100),
281
+ maxTokens: Math.min(requestedMaxTokens, profile?.maxTokens || 200_000),
282
+ profile: profile?.name || null,
222
283
  status: 'running',
223
284
  startTime: new Date().toISOString(),
224
285
  summary: null,
@@ -1,74 +1,189 @@
1
- import fs from 'fs/promises';
2
- import path from 'path';
3
- import os from 'os';
4
-
5
- // Phase 3 AGI Upgrade: HTML Cleanup to Markdown
6
- function htmlToText(html) {
7
- // AGI Optimization: Aggressively strip scripts, styles, svg, and tags
8
- let text = html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
9
- text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
10
- text = text.replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, '');
11
- text = text.replace(/<[^>]+>/g, ' '); // Strip HTML tags
12
- text = text.replace(/&nbsp;/g, ' '); // Decode basic entities
13
- text = text.replace(/\s{2,}/g, ' '); // Normalize spaces
14
- text = text.replace(/\n\s*\n/g, '\n\n'); // Normalize line breaks
15
- return text.trim();
1
+ import crypto from 'node:crypto';
2
+ import fs from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import { BrowserCapabilityError, causyneBrowser } from '../browser/causyneBrowser.js';
5
+ import { assertPublicWebUrl } from '../browser/urlPolicy.js';
6
+ const MAX_REDIRECTS = 5;
7
+ const MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
8
+ const MAX_OUTPUT_CHARS = 80_000;
9
+ const REQUEST_TIMEOUT_MS = 30_000;
10
+ function decodeEntities(text) {
11
+ const named = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
12
+ return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (full, entity) => {
13
+ if (entity.startsWith('#x'))
14
+ return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
15
+ if (entity.startsWith('#'))
16
+ return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
17
+ return named[entity.toLowerCase()] ?? full;
18
+ });
16
19
  }
17
-
18
- export const webFetch = {
19
- schema: {
20
- name: 'web.fetch',
21
- description: 'Make HTTP requests. Automatically strips raw HTML into clean text to protect Context Window.',
22
- parameters: {
23
- type: 'object',
24
- properties: {
25
- url: { type: 'string' },
26
- method: { type: 'string' },
27
- headers: { type: 'object' },
28
- body: { type: 'string' },
29
- outputFile: { type: 'string', description: 'If provided, saves as a binary file instead of returning text' }
30
- },
31
- required: ['url']
20
+ export function htmlToReadableText(html) {
21
+ return decodeEntities(String(html ?? ''))
22
+ .replace(/<(script|style|svg|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
23
+ .replace(/<\s*br\s*\/?>/gi, '\n')
24
+ .replace(/<\/(p|div|article|section|li|h[1-6]|tr)>/gi, '\n')
25
+ .replace(/<[^>]+>/g, ' ')
26
+ .replace(/[ \t]+/g, ' ')
27
+ .replace(/ *\n */g, '\n')
28
+ .replace(/\n{3,}/g, '\n\n')
29
+ .trim();
30
+ }
31
+ function isWithin(root, candidate) {
32
+ const relative = path.relative(root, candidate);
33
+ return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative));
34
+ }
35
+ function validatedHeaders(value) {
36
+ if (value === undefined)
37
+ return {};
38
+ if (!value || typeof value !== 'object' || Array.isArray(value))
39
+ throw new Error('headers must be an object');
40
+ const result = {};
41
+ for (const [key, item] of Object.entries(value)) {
42
+ if (typeof item !== 'string' || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(key) || /[\r\n]/.test(item)) {
43
+ throw new Error('headers contain an invalid name or value');
44
+ }
45
+ result[key] = item;
32
46
  }
33
- },
34
- async execute(args) {
47
+ return result;
48
+ }
49
+ async function fetchFollowingSafeRedirects(urlValue, options) {
50
+ let url = await assertPublicWebUrl(urlValue);
51
+ let request = { ...options };
52
+ for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect++) {
53
+ const response = await fetch(url, { ...request, redirect: 'manual' });
54
+ if (![301, 302, 303, 307, 308].includes(response.status))
55
+ return { response, finalUrl: url };
56
+ if (redirect === MAX_REDIRECTS)
57
+ throw new Error(`Too many redirects (maximum ${MAX_REDIRECTS})`);
58
+ const location = response.headers.get('location');
59
+ if (!location)
60
+ throw new Error(`HTTP ${response.status} redirect has no Location header`);
61
+ await response.body?.cancel().catch(() => { });
62
+ url = await assertPublicWebUrl(new URL(location, url).href);
63
+ if (response.status === 303 || ([301, 302].includes(response.status) && request.method === 'POST')) {
64
+ const redirectedHeaders = new Headers(request.headers);
65
+ redirectedHeaders.delete('content-length');
66
+ redirectedHeaders.delete('content-type');
67
+ const { body: _discardedBody, ...redirectedRequest } = request;
68
+ request = { ...redirectedRequest, method: 'GET', headers: redirectedHeaders };
69
+ }
70
+ }
71
+ throw new Error('Redirect handling failed');
72
+ }
73
+ async function readBoundedBody(response) {
74
+ if (!response.body)
75
+ return Buffer.alloc(0);
76
+ const reader = response.body.getReader();
77
+ const chunks = [];
78
+ let size = 0;
35
79
  try {
36
- const options = {
37
- method: args.method || 'GET',
38
- headers: {
39
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0.0.0 Safari/537.36',
40
- ...args.headers
80
+ while (true) {
81
+ const { done, value } = await reader.read();
82
+ if (done)
83
+ break;
84
+ size += value.byteLength;
85
+ if (size > MAX_RESPONSE_BYTES) {
86
+ await reader.cancel(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`).catch(() => { });
87
+ throw new Error(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`);
88
+ }
89
+ chunks.push(value);
90
+ }
91
+ }
92
+ finally {
93
+ reader.releaseLock();
94
+ }
95
+ return Buffer.concat(chunks.map(chunk => Buffer.from(chunk)), size);
96
+ }
97
+ async function browserFallback(url) {
98
+ try {
99
+ const results = await causyneBrowser().run([{ type: 'goto', url }, { type: 'extract', selector: 'body' }]);
100
+ const output = results.find(result => result.action === 'extract')?.output;
101
+ return output ? { success: true, output } : { success: false, error: 'Browser loaded the page but extracted no readable content' };
102
+ }
103
+ catch (error) {
104
+ if (error instanceof BrowserCapabilityError) {
105
+ return { success: false, error: error.message, requiresConsent: error.requiresConsent };
106
+ }
107
+ return { success: false, error: error instanceof Error ? error.message : String(error) };
108
+ }
109
+ }
110
+ export const webFetch = {
111
+ schema: {
112
+ name: 'web.fetch',
113
+ description: 'Fetch a public HTTP(S) resource with redirect-by-redirect SSRF checks, bounded output, browser fallback for blocked HTML, and atomic workspace-scoped downloads.',
114
+ parameters: {
115
+ type: 'object',
116
+ properties: {
117
+ url: { type: 'string' },
118
+ method: { type: 'string', enum: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] },
119
+ headers: { type: 'object', additionalProperties: { type: 'string' } },
120
+ body: { type: 'string' },
121
+ outputFile: { type: 'string', description: 'File destination inside the selected workspace.' }
122
+ },
123
+ required: ['url'],
124
+ additionalProperties: false
125
+ }
126
+ },
127
+ async execute(args) {
128
+ const controller = new AbortController();
129
+ const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
130
+ try {
131
+ if (!args.url)
132
+ return { success: false, error: 'url is required' };
133
+ const method = (args.method ?? 'GET').toUpperCase();
134
+ const headers = {
135
+ 'User-Agent': 'Causyne-Agent/1.10 (+https://causyne.com)',
136
+ Accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*;q=0.8',
137
+ ...validatedHeaders(args.headers)
138
+ };
139
+ const request = { method, headers, signal: controller.signal };
140
+ if (args.body !== undefined && method !== 'GET' && method !== 'HEAD')
141
+ request.body = args.body;
142
+ const { response, finalUrl } = await fetchFollowingSafeRedirects(args.url, request);
143
+ const contentType = response.headers.get('content-type')?.toLowerCase() ?? '';
144
+ if (!response.ok && !args.outputFile && contentType.includes('text/html') && [401, 403, 429, 503].includes(response.status)) {
145
+ const fallback = await browserFallback(finalUrl.href);
146
+ if (fallback.success)
147
+ return { ...fallback, status: response.status, finalUrl: finalUrl.href, via: 'camoufox' };
148
+ return { success: false, error: `HTTP ${response.status}; browser fallback failed: ${fallback.error}`, requiresConsent: fallback.requiresConsent };
149
+ }
150
+ if (!response.ok)
151
+ return { success: false, error: `HTTP ${response.status} ${response.statusText}`.trim(), status: response.status, finalUrl: finalUrl.href };
152
+ const declaredLength = Number(response.headers.get('content-length') ?? '0');
153
+ if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) {
154
+ return { success: false, error: `Response exceeds ${MAX_RESPONSE_BYTES} bytes` };
155
+ }
156
+ const bytes = await readBoundedBody(response);
157
+ if (args.outputFile) {
158
+ const workspace = path.resolve(process.cwd());
159
+ const target = path.resolve(workspace, args.outputFile);
160
+ if (!isWithin(workspace, target))
161
+ return { success: false, error: 'Download target must stay inside the selected workspace' };
162
+ const temporary = `${target}.part-${crypto.randomUUID()}`;
163
+ await fs.mkdir(path.dirname(target), { recursive: true });
164
+ try {
165
+ await fs.writeFile(temporary, bytes, { mode: 0o600 });
166
+ await fs.rename(temporary, target);
167
+ }
168
+ finally {
169
+ await fs.unlink(temporary).catch(() => { });
170
+ }
171
+ return { success: true, output: `Downloaded ${bytes.length} bytes to ${target}`, status: response.status, finalUrl: finalUrl.href };
172
+ }
173
+ const decoded = bytes.toString('utf8');
174
+ const readable = contentType.includes('text/html') ? htmlToReadableText(decoded) : decoded;
175
+ const output = readable.length > MAX_OUTPUT_CHARS
176
+ ? `${readable.slice(0, MAX_OUTPUT_CHARS)}\n\n[Web output truncated at ${MAX_OUTPUT_CHARS} characters]`
177
+ : readable;
178
+ return { success: true, output, status: response.status, finalUrl: finalUrl.href, contentType };
179
+ }
180
+ catch (error) {
181
+ const message = error instanceof Error ? error.message : String(error);
182
+ const timedOut = error instanceof Error && (error.name === 'AbortError' || message === 'This operation was aborted');
183
+ return { success: false, error: timedOut ? `Request timed out after ${REQUEST_TIMEOUT_MS}ms` : message };
184
+ }
185
+ finally {
186
+ clearTimeout(timer);
41
187
  }
42
- };
43
-
44
- if (args.body) options.body = args.body;
45
-
46
- const response = await fetch(args.url, options);
47
-
48
- if (args.outputFile) {
49
- const target = args.outputFile.replace(/^~\//, os.homedir() + '/');
50
- const buffer = await response.arrayBuffer();
51
- await fs.writeFile(target, Buffer.from(buffer));
52
- return { success: true, output: `Downloaded ${buffer.byteLength} bytes to ${args.outputFile}`, status: response.status };
53
- }
54
-
55
- const text = await response.text();
56
-
57
- // Phase 3 AGI Upgrade: Clean HTML unless it's an API JSON response
58
- let cleanOutput = text;
59
- const contentType = response.headers.get('content-type') || '';
60
- if (contentType.includes('text/html')) {
61
- cleanOutput = htmlToText(text);
62
- }
63
-
64
- // Strict Context Paging Protection
65
- if (cleanOutput.length > 50000) {
66
- cleanOutput = cleanOutput.substring(0, 50000) + '\n\n[... OUTPUT TRUNCATED TO 50KB ...]';
67
- }
68
-
69
- return { success: true, output: cleanOutput, status: response.status };
70
- } catch (e) {
71
- return { success: false, error: e.message };
72
188
  }
73
- }
74
189
  };
@@ -0,0 +1,185 @@
1
+ import crypto from 'node:crypto';
2
+ import fs from 'node:fs/promises';
3
+ import path from 'node:path';
4
+
5
+ import { BrowserCapabilityError, causyneBrowser } from '../browser/causyneBrowser.js';
6
+ import { assertPublicWebUrl } from '../browser/urlPolicy.js';
7
+
8
+ const MAX_REDIRECTS = 5;
9
+ const MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
10
+ const MAX_OUTPUT_CHARS = 80_000;
11
+ const REQUEST_TIMEOUT_MS = 30_000;
12
+
13
+ function decodeEntities(text: string): string {
14
+ const named: Record<string, string> = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
15
+ return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (full, entity: string) => {
16
+ if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
17
+ if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
18
+ return named[entity.toLowerCase()] ?? full;
19
+ });
20
+ }
21
+
22
+ export function htmlToReadableText(html: string): string {
23
+ return decodeEntities(String(html ?? ''))
24
+ .replace(/<(script|style|svg|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
25
+ .replace(/<\s*br\s*\/?>/gi, '\n')
26
+ .replace(/<\/(p|div|article|section|li|h[1-6]|tr)>/gi, '\n')
27
+ .replace(/<[^>]+>/g, ' ')
28
+ .replace(/[ \t]+/g, ' ')
29
+ .replace(/ *\n */g, '\n')
30
+ .replace(/\n{3,}/g, '\n\n')
31
+ .trim();
32
+ }
33
+
34
+ function isWithin(root: string, candidate: string): boolean {
35
+ const relative = path.relative(root, candidate);
36
+ return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative));
37
+ }
38
+
39
+ function validatedHeaders(value: unknown): Record<string, string> {
40
+ if (value === undefined) return {};
41
+ if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('headers must be an object');
42
+ const result: Record<string, string> = {};
43
+ for (const [key, item] of Object.entries(value)) {
44
+ if (typeof item !== 'string' || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(key) || /[\r\n]/.test(item)) {
45
+ throw new Error('headers contain an invalid name or value');
46
+ }
47
+ result[key] = item;
48
+ }
49
+ return result;
50
+ }
51
+
52
+ async function fetchFollowingSafeRedirects(urlValue: string, options: RequestInit): Promise<{ response: Response; finalUrl: URL }> {
53
+ let url = await assertPublicWebUrl(urlValue);
54
+ let request = { ...options };
55
+ for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect++) {
56
+ const response = await fetch(url, { ...request, redirect: 'manual' });
57
+ if (![301, 302, 303, 307, 308].includes(response.status)) return { response, finalUrl: url };
58
+ if (redirect === MAX_REDIRECTS) throw new Error(`Too many redirects (maximum ${MAX_REDIRECTS})`);
59
+ const location = response.headers.get('location');
60
+ if (!location) throw new Error(`HTTP ${response.status} redirect has no Location header`);
61
+ await response.body?.cancel().catch(() => {});
62
+ url = await assertPublicWebUrl(new URL(location, url).href);
63
+ if (response.status === 303 || ([301, 302].includes(response.status) && request.method === 'POST')) {
64
+ const redirectedHeaders = new Headers(request.headers);
65
+ redirectedHeaders.delete('content-length');
66
+ redirectedHeaders.delete('content-type');
67
+ const { body: _discardedBody, ...redirectedRequest } = request;
68
+ request = { ...redirectedRequest, method: 'GET', headers: redirectedHeaders };
69
+ }
70
+ }
71
+ throw new Error('Redirect handling failed');
72
+ }
73
+
74
+ async function readBoundedBody(response: Response): Promise<Buffer> {
75
+ if (!response.body) return Buffer.alloc(0);
76
+ const reader = response.body.getReader();
77
+ const chunks: Uint8Array[] = [];
78
+ let size = 0;
79
+ try {
80
+ while (true) {
81
+ const { done, value } = await reader.read();
82
+ if (done) break;
83
+ size += value.byteLength;
84
+ if (size > MAX_RESPONSE_BYTES) {
85
+ await reader.cancel(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`).catch(() => {});
86
+ throw new Error(`Response exceeds ${MAX_RESPONSE_BYTES} bytes`);
87
+ }
88
+ chunks.push(value);
89
+ }
90
+ } finally {
91
+ reader.releaseLock();
92
+ }
93
+ return Buffer.concat(chunks.map(chunk => Buffer.from(chunk)), size);
94
+ }
95
+
96
+ async function browserFallback(url: string): Promise<{ success: boolean; output?: string; error?: string; requiresConsent?: boolean }> {
97
+ try {
98
+ const results = await causyneBrowser().run([{ type: 'goto', url }, { type: 'extract', selector: 'body' }]);
99
+ const output = results.find(result => result.action === 'extract')?.output;
100
+ return output ? { success: true, output } : { success: false, error: 'Browser loaded the page but extracted no readable content' };
101
+ } catch (error) {
102
+ if (error instanceof BrowserCapabilityError) {
103
+ return { success: false, error: error.message, requiresConsent: error.requiresConsent };
104
+ }
105
+ return { success: false, error: error instanceof Error ? error.message : String(error) };
106
+ }
107
+ }
108
+
109
+ export const webFetch = {
110
+ schema: {
111
+ name: 'web.fetch',
112
+ description: 'Fetch a public HTTP(S) resource with redirect-by-redirect SSRF checks, bounded output, browser fallback for blocked HTML, and atomic workspace-scoped downloads.',
113
+ parameters: {
114
+ type: 'object',
115
+ properties: {
116
+ url: { type: 'string' },
117
+ method: { type: 'string', enum: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'] },
118
+ headers: { type: 'object', additionalProperties: { type: 'string' } },
119
+ body: { type: 'string' },
120
+ outputFile: { type: 'string', description: 'File destination inside the selected workspace.' }
121
+ },
122
+ required: ['url'],
123
+ additionalProperties: false
124
+ }
125
+ },
126
+
127
+ async execute(args: { url?: string; method?: string; headers?: unknown; body?: string; outputFile?: string }) {
128
+ const controller = new AbortController();
129
+ const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
130
+ try {
131
+ if (!args.url) return { success: false, error: 'url is required' };
132
+ const method = (args.method ?? 'GET').toUpperCase();
133
+ const headers = {
134
+ 'User-Agent': 'Causyne-Agent/1.10 (+https://causyne.com)',
135
+ Accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*;q=0.8',
136
+ ...validatedHeaders(args.headers)
137
+ };
138
+ const request: RequestInit = { method, headers, signal: controller.signal };
139
+ if (args.body !== undefined && method !== 'GET' && method !== 'HEAD') request.body = args.body;
140
+ const { response, finalUrl } = await fetchFollowingSafeRedirects(args.url, request);
141
+ const contentType = response.headers.get('content-type')?.toLowerCase() ?? '';
142
+
143
+ if (!response.ok && !args.outputFile && contentType.includes('text/html') && [401, 403, 429, 503].includes(response.status)) {
144
+ const fallback = await browserFallback(finalUrl.href);
145
+ if (fallback.success) return { ...fallback, status: response.status, finalUrl: finalUrl.href, via: 'camoufox' };
146
+ return { success: false, error: `HTTP ${response.status}; browser fallback failed: ${fallback.error}`, requiresConsent: fallback.requiresConsent };
147
+ }
148
+ if (!response.ok) return { success: false, error: `HTTP ${response.status} ${response.statusText}`.trim(), status: response.status, finalUrl: finalUrl.href };
149
+
150
+ const declaredLength = Number(response.headers.get('content-length') ?? '0');
151
+ if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) {
152
+ return { success: false, error: `Response exceeds ${MAX_RESPONSE_BYTES} bytes` };
153
+ }
154
+ const bytes = await readBoundedBody(response);
155
+
156
+ if (args.outputFile) {
157
+ const workspace = path.resolve(process.cwd());
158
+ const target = path.resolve(workspace, args.outputFile);
159
+ if (!isWithin(workspace, target)) return { success: false, error: 'Download target must stay inside the selected workspace' };
160
+ const temporary = `${target}.part-${crypto.randomUUID()}`;
161
+ await fs.mkdir(path.dirname(target), { recursive: true });
162
+ try {
163
+ await fs.writeFile(temporary, bytes, { mode: 0o600 });
164
+ await fs.rename(temporary, target);
165
+ } finally {
166
+ await fs.unlink(temporary).catch(() => {});
167
+ }
168
+ return { success: true, output: `Downloaded ${bytes.length} bytes to ${target}`, status: response.status, finalUrl: finalUrl.href };
169
+ }
170
+
171
+ const decoded = bytes.toString('utf8');
172
+ const readable = contentType.includes('text/html') ? htmlToReadableText(decoded) : decoded;
173
+ const output = readable.length > MAX_OUTPUT_CHARS
174
+ ? `${readable.slice(0, MAX_OUTPUT_CHARS)}\n\n[Web output truncated at ${MAX_OUTPUT_CHARS} characters]`
175
+ : readable;
176
+ return { success: true, output, status: response.status, finalUrl: finalUrl.href, contentType };
177
+ } catch (error) {
178
+ const message = error instanceof Error ? error.message : String(error);
179
+ const timedOut = error instanceof Error && (error.name === 'AbortError' || message === 'This operation was aborted');
180
+ return { success: false, error: timedOut ? `Request timed out after ${REQUEST_TIMEOUT_MS}ms` : message };
181
+ } finally {
182
+ clearTimeout(timer);
183
+ }
184
+ }
185
+ };