@timmeck/brain-core 2.36.80 → 2.36.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/command-center.html +35 -0
- package/dist/browser/browser-agent.d.ts +188 -0
- package/dist/browser/browser-agent.js +787 -0
- package/dist/browser/browser-agent.js.map +1 -0
- package/dist/chat/brain-bot.d.ts +75 -0
- package/dist/chat/brain-bot.js +246 -0
- package/dist/chat/brain-bot.js.map +1 -0
- package/dist/chat/chat-engine.d.ts +3 -0
- package/dist/chat/chat-engine.js +17 -0
- package/dist/chat/chat-engine.js.map +1 -1
- package/dist/chat/index.d.ts +2 -0
- package/dist/chat/index.js +1 -0
- package/dist/chat/index.js.map +1 -1
- package/dist/hypothesis/engine.d.ts +34 -0
- package/dist/hypothesis/engine.js +82 -0
- package/dist/hypothesis/engine.js.map +1 -1
- package/dist/index.d.ts +9 -3
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/memory/conversation-memory.d.ts +141 -0
- package/dist/memory/conversation-memory.js +470 -0
- package/dist/memory/conversation-memory.js.map +1 -0
- package/dist/research/autonomous-research-loop.d.ts +104 -0
- package/dist/research/autonomous-research-loop.js +246 -0
- package/dist/research/autonomous-research-loop.js.map +1 -0
- package/dist/research/research-orchestrator.d.ts +3 -0
- package/dist/research/research-orchestrator.js +28 -0
- package/dist/research/research-orchestrator.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,787 @@
|
|
|
1
|
+
// ── Browser Agent ──────────────────────────────────────────
|
|
2
|
+
//
|
|
3
|
+
// Autonomer Browser-Agent inspiriert von OpenBrowser (ntegrals).
|
|
4
|
+
//
|
|
5
|
+
// Kern-Pattern: LLM-gesteuerter Feedback-Loop
|
|
6
|
+
// 1. analyzePage() → Page-State (DOM, Links, Forms, Text)
|
|
7
|
+
// 2. LLM entscheidet nächste Action(s) basierend auf State + Task
|
|
8
|
+
// 3. executeAction() → Ergebnis
|
|
9
|
+
// 4. Observe → Stall Detection → Loop oder Done
|
|
10
|
+
//
|
|
11
|
+
// Sicherheit: Domain-Whitelist/Blacklist, Step-Limit, Failure-Threshold,
|
|
12
|
+
// Stall Detection, Timeout, Max-Pages.
|
|
13
|
+
import { getLogger } from '../utils/logger.js';
|
|
14
|
+
// ── Stall Detector ────────────────────────────────────────
|
|
15
|
+
export class StallDetector {
|
|
16
|
+
maxRepeats;
|
|
17
|
+
urlHistory = [];
|
|
18
|
+
actionHistory = [];
|
|
19
|
+
constructor(maxRepeats = 3) {
|
|
20
|
+
this.maxRepeats = maxRepeats;
|
|
21
|
+
}
|
|
22
|
+
record(url, actions) {
|
|
23
|
+
this.urlHistory.push(url);
|
|
24
|
+
this.actionHistory.push(actions.join(','));
|
|
25
|
+
}
|
|
26
|
+
isStalled() {
|
|
27
|
+
// Check URL repetition
|
|
28
|
+
if (this.urlHistory.length >= this.maxRepeats) {
|
|
29
|
+
const recent = this.urlHistory.slice(-this.maxRepeats);
|
|
30
|
+
if (recent.every(u => u === recent[0])) {
|
|
31
|
+
// Same URL N times — check if actions also repeat
|
|
32
|
+
const recentActions = this.actionHistory.slice(-this.maxRepeats);
|
|
33
|
+
if (recentActions.every(a => a === recentActions[0])) {
|
|
34
|
+
return true; // Same URL + same actions = stalled
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
// Check action pattern repetition (ABAB or ABCABC)
|
|
39
|
+
if (this.actionHistory.length >= 4) {
|
|
40
|
+
const last4 = this.actionHistory.slice(-4);
|
|
41
|
+
if (last4[0] === last4[2] && last4[1] === last4[3]) {
|
|
42
|
+
return true; // ABAB pattern
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
reset() {
|
|
48
|
+
this.urlHistory = [];
|
|
49
|
+
this.actionHistory = [];
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
// ── Migration ─────────────────────────────────────────────
|
|
53
|
+
export function runBrowserAgentMigration(db) {
|
|
54
|
+
db.exec(`
|
|
55
|
+
CREATE TABLE IF NOT EXISTS browser_agent_log (
|
|
56
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
57
|
+
task_id TEXT NOT NULL,
|
|
58
|
+
step INTEGER NOT NULL DEFAULT 0,
|
|
59
|
+
action_type TEXT NOT NULL,
|
|
60
|
+
url TEXT,
|
|
61
|
+
selector TEXT,
|
|
62
|
+
success INTEGER NOT NULL DEFAULT 1,
|
|
63
|
+
error TEXT,
|
|
64
|
+
duration_ms INTEGER,
|
|
65
|
+
tokens_used INTEGER DEFAULT 0,
|
|
66
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
67
|
+
);
|
|
68
|
+
CREATE INDEX IF NOT EXISTS idx_browser_log_task ON browser_agent_log(task_id);
|
|
69
|
+
`);
|
|
70
|
+
}
|
|
71
|
+
// ── Default blocked domains ───────────────────────────────
|
|
72
|
+
const DEFAULT_BLOCKED_DOMAINS = [
|
|
73
|
+
'doubleclick.net', 'googlesyndication.com', 'googleadservices.com',
|
|
74
|
+
'facebook.com/tr', 'analytics.google.com', 'pixel.facebook.com',
|
|
75
|
+
];
|
|
76
|
+
// ── Default LLM Planner (JSON-based) ─────────────────────
|
|
77
|
+
// Parses LLM response as JSON array of actions.
|
|
78
|
+
// Can be replaced with a real LLM planner via setPlanner().
|
|
79
|
+
export function parseLLMActions(text) {
|
|
80
|
+
// Try to extract JSON array from LLM response
|
|
81
|
+
const jsonMatch = text.match(/\[[\s\S]*\]/);
|
|
82
|
+
if (!jsonMatch) {
|
|
83
|
+
// Single action fallback
|
|
84
|
+
const objMatch = text.match(/\{[\s\S]*\}/);
|
|
85
|
+
if (objMatch) {
|
|
86
|
+
try {
|
|
87
|
+
return [JSON.parse(objMatch[0])];
|
|
88
|
+
}
|
|
89
|
+
catch { /* parse failed */ }
|
|
90
|
+
}
|
|
91
|
+
return [{ type: 'fail', message: 'Could not parse LLM response as actions' }];
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
const actions = JSON.parse(jsonMatch[0]);
|
|
95
|
+
if (!Array.isArray(actions))
|
|
96
|
+
return [{ type: 'fail', message: 'Expected array of actions' }];
|
|
97
|
+
return actions;
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return [{ type: 'fail', message: 'Invalid JSON in LLM response' }];
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/** Build the system prompt for the browser agent LLM. */
|
|
104
|
+
export function buildBrowserSystemPrompt() {
|
|
105
|
+
return `You are an autonomous browser agent. You navigate web pages to complete tasks.
|
|
106
|
+
|
|
107
|
+
AVAILABLE ACTIONS (return as JSON array):
|
|
108
|
+
- {"type":"navigate","url":"https://..."} — Go to URL
|
|
109
|
+
- {"type":"click","selector":"CSS selector"} — Click element
|
|
110
|
+
- {"type":"fill","selector":"CSS selector","value":"text"} — Type into input
|
|
111
|
+
- {"type":"select","selector":"CSS selector","value":"option"} — Select dropdown
|
|
112
|
+
- {"type":"scroll_down"} — Scroll down one viewport
|
|
113
|
+
- {"type":"scroll_up"} — Scroll up one viewport
|
|
114
|
+
- {"type":"back"} — Go back one page
|
|
115
|
+
- {"type":"wait","value":"1000"} — Wait ms
|
|
116
|
+
- {"type":"extract","selector":"CSS selector","extractKey":"name"} — Extract text content
|
|
117
|
+
- {"type":"screenshot"} — Take screenshot
|
|
118
|
+
- {"type":"evaluate","script":"JS code"} — Run JavaScript
|
|
119
|
+
- {"type":"done","message":"result summary"} — Task completed successfully
|
|
120
|
+
- {"type":"fail","message":"reason"} — Task cannot be completed
|
|
121
|
+
|
|
122
|
+
RULES:
|
|
123
|
+
1. Return a JSON array of 1-5 actions to execute in sequence
|
|
124
|
+
2. Use CSS selectors that are specific (prefer #id, [name=...], [aria-label=...])
|
|
125
|
+
3. After extracting data, use "done" with a summary
|
|
126
|
+
4. If stuck after 3+ attempts, use "fail" with explanation
|
|
127
|
+
5. Never navigate to domains not in the current task context
|
|
128
|
+
6. Prefer efficient paths — minimize unnecessary clicks
|
|
129
|
+
|
|
130
|
+
EXAMPLE RESPONSE:
|
|
131
|
+
[{"type":"navigate","url":"https://example.com"},{"type":"click","selector":"#search-input"},{"type":"fill","selector":"#search-input","value":"test query"}]`;
|
|
132
|
+
}
|
|
133
|
+
/** Build the user prompt for each step. */
|
|
134
|
+
export function buildStepPrompt(context) {
|
|
135
|
+
const parts = [];
|
|
136
|
+
parts.push(`TASK: ${context.task}`);
|
|
137
|
+
parts.push(`STEP: ${context.currentStep}/${context.maxSteps}`);
|
|
138
|
+
parts.push(`URL: ${context.pageState.url}`);
|
|
139
|
+
parts.push(`TITLE: ${context.pageState.title}`);
|
|
140
|
+
if (context.consecutiveFailures > 0) {
|
|
141
|
+
parts.push(`WARNING: ${context.consecutiveFailures} consecutive failures. Try a different approach.`);
|
|
142
|
+
}
|
|
143
|
+
// Headings
|
|
144
|
+
if (context.pageState.headings.length > 0) {
|
|
145
|
+
parts.push('\nHEADINGS:');
|
|
146
|
+
for (const h of context.pageState.headings.slice(0, 10)) {
|
|
147
|
+
parts.push(` ${'#'.repeat(h.level)} ${h.text}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// Interactive elements
|
|
151
|
+
if (context.pageState.interactiveElements.length > 0) {
|
|
152
|
+
parts.push('\nINTERACTIVE ELEMENTS:');
|
|
153
|
+
for (const el of context.pageState.interactiveElements.slice(0, 30)) {
|
|
154
|
+
const selector = el.id ? `#${el.id}` : el.name ? `[name="${el.name}"]` : el.ariaLabel ? `[aria-label="${el.ariaLabel}"]` : `${el.tag}${el.classes ? '.' + el.classes[0] : ''}`;
|
|
155
|
+
const label = el.text || el.placeholder || el.ariaLabel || el.type || '';
|
|
156
|
+
parts.push(` [${el.tag}] ${selector} — "${label.slice(0, 60)}"`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// Links
|
|
160
|
+
if (context.pageState.links.length > 0) {
|
|
161
|
+
parts.push('\nLINKS:');
|
|
162
|
+
for (const link of context.pageState.links.slice(0, 15)) {
|
|
163
|
+
parts.push(` "${(link.text ?? '').slice(0, 50)}" → ${link.href}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Forms
|
|
167
|
+
if (context.pageState.forms.length > 0) {
|
|
168
|
+
parts.push('\nFORMS:');
|
|
169
|
+
for (const form of context.pageState.forms) {
|
|
170
|
+
const fieldNames = form.fields.map(f => f.name || f.id || f.type || 'unknown').join(', ');
|
|
171
|
+
parts.push(` <form action="${form.action ?? ''}"> fields: ${fieldNames}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
// Text content (truncated)
|
|
175
|
+
if (context.pageState.textContent.length > 0) {
|
|
176
|
+
parts.push(`\nPAGE TEXT (truncated):\n${context.pageState.textContent.slice(0, 2000)}`);
|
|
177
|
+
}
|
|
178
|
+
// Previous steps summary
|
|
179
|
+
if (context.previousSteps.length > 0) {
|
|
180
|
+
parts.push('\nPREVIOUS STEPS:');
|
|
181
|
+
for (const s of context.previousSteps.slice(-5)) {
|
|
182
|
+
parts.push(` Step ${s.step}: ${s.actions.join(' → ')} [${s.results.join(', ')}] @ ${s.url}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Extracted data so far
|
|
186
|
+
const extractedKeys = Object.keys(context.extractedData);
|
|
187
|
+
if (extractedKeys.length > 0) {
|
|
188
|
+
parts.push('\nEXTRACTED DATA:');
|
|
189
|
+
for (const key of extractedKeys) {
|
|
190
|
+
parts.push(` ${key}: ${context.extractedData[key].slice(0, 200)}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
parts.push('\nReturn your next action(s) as a JSON array:');
|
|
194
|
+
return parts.join('\n');
|
|
195
|
+
}
|
|
196
|
+
// ── Agent ─────────────────────────────────────────────────
|
|
197
|
+
export class BrowserAgent {
|
|
198
|
+
db;
|
|
199
|
+
config;
|
|
200
|
+
log = getLogger();
|
|
201
|
+
browser = null;
|
|
202
|
+
planner = null;
|
|
203
|
+
activeTasks = 0;
|
|
204
|
+
completedTasks = 0;
|
|
205
|
+
stalledTasks = 0;
|
|
206
|
+
totalSteps = 0;
|
|
207
|
+
totalTokensUsed = 0;
|
|
208
|
+
openPages = 0;
|
|
209
|
+
// Prepared statements
|
|
210
|
+
stmtLogAction;
|
|
211
|
+
constructor(db, config = {}) {
|
|
212
|
+
this.db = db;
|
|
213
|
+
this.config = {
|
|
214
|
+
maxSteps: config.maxSteps ?? 25,
|
|
215
|
+
failureThreshold: config.failureThreshold ?? 5,
|
|
216
|
+
pageTimeoutMs: config.pageTimeoutMs ?? 30_000,
|
|
217
|
+
allowedDomains: config.allowedDomains ?? [],
|
|
218
|
+
blockedDomains: config.blockedDomains ?? DEFAULT_BLOCKED_DOMAINS,
|
|
219
|
+
maxPages: config.maxPages ?? 3,
|
|
220
|
+
screenshotEachStep: config.screenshotEachStep ?? false,
|
|
221
|
+
actionsPerStep: config.actionsPerStep ?? 5,
|
|
222
|
+
maxUrlRepeats: config.maxUrlRepeats ?? 3,
|
|
223
|
+
};
|
|
224
|
+
runBrowserAgentMigration(db);
|
|
225
|
+
this.stmtLogAction = db.prepare('INSERT INTO browser_agent_log (task_id, step, action_type, url, selector, success, error, duration_ms, tokens_used) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
226
|
+
}
|
|
227
|
+
// ── Setters ────────────────────────────────────────────
|
|
228
|
+
/** Set the LLM planner for autonomous action selection. */
|
|
229
|
+
setPlanner(planner) {
|
|
230
|
+
this.planner = planner;
|
|
231
|
+
}
|
|
232
|
+
// ── Domain Safety ────────────────────────────────────────
|
|
233
|
+
/** Check if a URL is allowed by domain whitelist/blacklist. */
|
|
234
|
+
isDomainAllowed(url) {
|
|
235
|
+
try {
|
|
236
|
+
const parsed = new URL(url);
|
|
237
|
+
const hostname = parsed.hostname.toLowerCase();
|
|
238
|
+
// Block check first
|
|
239
|
+
if (this.config.blockedDomains.some(d => hostname.includes(d))) {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
// If whitelist is set, enforce it
|
|
243
|
+
if (this.config.allowedDomains.length > 0) {
|
|
244
|
+
return this.config.allowedDomains.some(d => hostname.includes(d));
|
|
245
|
+
}
|
|
246
|
+
return true;
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
return false;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
// ── Browser Lifecycle ──────────────────────────────────────
|
|
253
|
+
/** Get or launch the browser instance (lazy init). */
|
|
254
|
+
async getBrowser() {
|
|
255
|
+
if (this.browser) {
|
|
256
|
+
try {
|
|
257
|
+
const b = this.browser;
|
|
258
|
+
if (b.isConnected && !b.isConnected()) {
|
|
259
|
+
this.browser = null;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
this.browser = null;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (!this.browser) {
|
|
267
|
+
try {
|
|
268
|
+
const pwPath = 'playwright';
|
|
269
|
+
const pw = await import(/* webpackIgnore: true */ pwPath);
|
|
270
|
+
this.browser = await pw.chromium.launch({
|
|
271
|
+
headless: true,
|
|
272
|
+
args: ['--no-sandbox', '--disable-dev-shm-usage'],
|
|
273
|
+
});
|
|
274
|
+
this.log.info('[browser-agent] Browser launched');
|
|
275
|
+
}
|
|
276
|
+
catch (err) {
|
|
277
|
+
this.log.error(`[browser-agent] Failed to launch browser: ${err.message}`);
|
|
278
|
+
throw new Error('Playwright not available. Install: npx playwright install chromium');
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return this.browser;
|
|
282
|
+
}
|
|
283
|
+
/** Shutdown the browser. */
|
|
284
|
+
async shutdown() {
|
|
285
|
+
if (this.browser) {
|
|
286
|
+
try {
|
|
287
|
+
await this.browser.close();
|
|
288
|
+
}
|
|
289
|
+
catch { /* best effort */ }
|
|
290
|
+
this.browser = null;
|
|
291
|
+
this.openPages = 0;
|
|
292
|
+
this.log.info('[browser-agent] Browser shut down');
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
// ── DOM Analysis ──────────────────────────────────────────
|
|
296
|
+
/** Analyze the current page DOM to understand interactive elements. */
|
|
297
|
+
async analyzePage(page) {
|
|
298
|
+
const p = page;
|
|
299
|
+
const url = p.url();
|
|
300
|
+
const title = await p.title();
|
|
301
|
+
const analysis = await p.evaluate(() => {
|
|
302
|
+
const result = {
|
|
303
|
+
interactiveElements: [],
|
|
304
|
+
links: [],
|
|
305
|
+
forms: [],
|
|
306
|
+
textContent: '',
|
|
307
|
+
headings: [],
|
|
308
|
+
};
|
|
309
|
+
// Interactive elements (buttons, inputs, selects)
|
|
310
|
+
const interactive = document.querySelectorAll('button, input, select, textarea, [role="button"], [onclick]');
|
|
311
|
+
for (const el of Array.from(interactive).slice(0, 50)) {
|
|
312
|
+
const rect = el.getBoundingClientRect();
|
|
313
|
+
if (rect.width === 0 && rect.height === 0)
|
|
314
|
+
continue;
|
|
315
|
+
result.interactiveElements.push({
|
|
316
|
+
tag: el.tagName.toLowerCase(),
|
|
317
|
+
id: el.id || undefined,
|
|
318
|
+
classes: el.className ? el.className.split(/\s+/).slice(0, 5) : undefined,
|
|
319
|
+
text: (el.textContent ?? '').trim().slice(0, 100) || undefined,
|
|
320
|
+
type: el.type || undefined,
|
|
321
|
+
name: el.name || undefined,
|
|
322
|
+
placeholder: el.placeholder || undefined,
|
|
323
|
+
ariaLabel: el.getAttribute('aria-label') || undefined,
|
|
324
|
+
rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
// Links
|
|
328
|
+
const links = document.querySelectorAll('a[href]');
|
|
329
|
+
for (const el of Array.from(links).slice(0, 30)) {
|
|
330
|
+
const rect = el.getBoundingClientRect();
|
|
331
|
+
if (rect.width === 0 && rect.height === 0)
|
|
332
|
+
continue;
|
|
333
|
+
result.links.push({
|
|
334
|
+
tag: 'a',
|
|
335
|
+
text: (el.textContent ?? '').trim().slice(0, 100) || undefined,
|
|
336
|
+
href: el.href || undefined,
|
|
337
|
+
ariaLabel: el.getAttribute('aria-label') || undefined,
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
// Forms
|
|
341
|
+
const forms = document.querySelectorAll('form');
|
|
342
|
+
for (const form of Array.from(forms).slice(0, 5)) {
|
|
343
|
+
const fields = [];
|
|
344
|
+
for (const field of Array.from(form.querySelectorAll('input, select, textarea')).slice(0, 20)) {
|
|
345
|
+
fields.push({
|
|
346
|
+
tag: field.tagName.toLowerCase(),
|
|
347
|
+
id: field.id || undefined,
|
|
348
|
+
name: field.name || undefined,
|
|
349
|
+
type: field.type || undefined,
|
|
350
|
+
placeholder: field.placeholder || undefined,
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
result.forms.push({
|
|
354
|
+
action: form.action || undefined,
|
|
355
|
+
method: form.method || undefined,
|
|
356
|
+
fields,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
// Text content (cleaned)
|
|
360
|
+
const clone = document.body.cloneNode(true);
|
|
361
|
+
clone.querySelectorAll('script, style, nav, footer, header, aside, [role="navigation"]')
|
|
362
|
+
.forEach(el => el.remove());
|
|
363
|
+
result.textContent = (clone.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, 5000);
|
|
364
|
+
// Headings
|
|
365
|
+
const headings = document.querySelectorAll('h1, h2, h3, h4');
|
|
366
|
+
for (const h of Array.from(headings).slice(0, 20)) {
|
|
367
|
+
const level = parseInt(h.tagName[1], 10);
|
|
368
|
+
result.headings.push({ level, text: (h.textContent ?? '').trim().slice(0, 200) });
|
|
369
|
+
}
|
|
370
|
+
return result;
|
|
371
|
+
});
|
|
372
|
+
return { url, title, ...analysis };
|
|
373
|
+
}
|
|
374
|
+
// ── Action Execution ─────────────────────────────────────
|
|
375
|
+
/** Execute a single browser action. */
|
|
376
|
+
async executeAction(page, action) {
|
|
377
|
+
const start = Date.now();
|
|
378
|
+
const p = page;
|
|
379
|
+
try {
|
|
380
|
+
switch (action.type) {
|
|
381
|
+
case 'navigate': {
|
|
382
|
+
if (!action.url)
|
|
383
|
+
throw new Error('navigate requires url');
|
|
384
|
+
if (!this.isDomainAllowed(action.url)) {
|
|
385
|
+
throw new Error(`Domain not allowed: ${action.url}`);
|
|
386
|
+
}
|
|
387
|
+
await p.goto(action.url, { timeout: this.config.pageTimeoutMs, waitUntil: 'domcontentloaded' });
|
|
388
|
+
break;
|
|
389
|
+
}
|
|
390
|
+
case 'click': {
|
|
391
|
+
if (!action.selector)
|
|
392
|
+
throw new Error('click requires selector');
|
|
393
|
+
await p.waitForSelector(action.selector, { timeout: 5000 });
|
|
394
|
+
await p.click(action.selector, { timeout: 5000 });
|
|
395
|
+
break;
|
|
396
|
+
}
|
|
397
|
+
case 'fill': {
|
|
398
|
+
if (!action.selector || action.value === undefined)
|
|
399
|
+
throw new Error('fill requires selector + value');
|
|
400
|
+
await p.fill(action.selector, action.value);
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
403
|
+
case 'select': {
|
|
404
|
+
if (!action.selector || !action.value)
|
|
405
|
+
throw new Error('select requires selector + value');
|
|
406
|
+
await p.selectOption(action.selector, action.value);
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
case 'scroll_down': {
|
|
410
|
+
await p.evaluate(() => window.scrollBy(0, window.innerHeight));
|
|
411
|
+
break;
|
|
412
|
+
}
|
|
413
|
+
case 'scroll_up': {
|
|
414
|
+
await p.evaluate(() => window.scrollBy(0, -window.innerHeight));
|
|
415
|
+
break;
|
|
416
|
+
}
|
|
417
|
+
case 'wait': {
|
|
418
|
+
const ms = parseInt(action.value ?? '1000', 10);
|
|
419
|
+
await p.waitForTimeout(Math.min(ms, 10_000));
|
|
420
|
+
break;
|
|
421
|
+
}
|
|
422
|
+
case 'screenshot': {
|
|
423
|
+
const buf = await p.screenshot({ fullPage: action.value === 'full', type: 'png' });
|
|
424
|
+
return {
|
|
425
|
+
action, success: true, url: p.url(),
|
|
426
|
+
screenshot: buf.toString('base64'),
|
|
427
|
+
durationMs: Date.now() - start,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
case 'extract': {
|
|
431
|
+
if (!action.selector)
|
|
432
|
+
throw new Error('extract requires selector');
|
|
433
|
+
await p.waitForSelector(action.selector, { timeout: 5000 });
|
|
434
|
+
const text = await p.textContent(action.selector);
|
|
435
|
+
return {
|
|
436
|
+
action, success: true, url: p.url(),
|
|
437
|
+
extractedText: (text ?? '').trim().slice(0, 5000),
|
|
438
|
+
durationMs: Date.now() - start,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
case 'back': {
|
|
442
|
+
await p.goBack();
|
|
443
|
+
break;
|
|
444
|
+
}
|
|
445
|
+
case 'evaluate': {
|
|
446
|
+
if (!action.script)
|
|
447
|
+
throw new Error('evaluate requires script');
|
|
448
|
+
await p.evaluate(action.script);
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
451
|
+
case 'done':
|
|
452
|
+
case 'fail': {
|
|
453
|
+
// Terminal actions — handled by loop, not executed
|
|
454
|
+
return { action, success: true, url: p.url(), durationMs: Date.now() - start };
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
let screenshot;
|
|
458
|
+
if (this.config.screenshotEachStep) {
|
|
459
|
+
const buf = await p.screenshot({ type: 'png' });
|
|
460
|
+
screenshot = buf.toString('base64');
|
|
461
|
+
}
|
|
462
|
+
return { action, success: true, url: p.url(), screenshot, durationMs: Date.now() - start };
|
|
463
|
+
}
|
|
464
|
+
catch (err) {
|
|
465
|
+
return {
|
|
466
|
+
action, success: false, url: p.url?.() ?? undefined,
|
|
467
|
+
error: err.message,
|
|
468
|
+
durationMs: Date.now() - start,
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
// ── Scripted Task (pre-defined actions) ───────────────────
|
|
473
|
+
/** Execute a pre-defined sequence of browser actions. No LLM needed. */
|
|
474
|
+
async executeTask(taskId, actions) {
|
|
475
|
+
const start = Date.now();
|
|
476
|
+
const steps = [];
|
|
477
|
+
const screenshots = [];
|
|
478
|
+
const extractedData = {};
|
|
479
|
+
let consecutiveFailures = 0;
|
|
480
|
+
if (actions.length > this.config.maxSteps) {
|
|
481
|
+
return {
|
|
482
|
+
taskId, status: 'aborted', steps, screenshots, extractedData,
|
|
483
|
+
totalTokensUsed: 0, consecutiveFailures: 0,
|
|
484
|
+
error: `Too many steps: ${actions.length} > ${this.config.maxSteps}`,
|
|
485
|
+
durationMs: Date.now() - start,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
if (this.openPages >= this.config.maxPages) {
|
|
489
|
+
return {
|
|
490
|
+
taskId, status: 'aborted', steps, screenshots, extractedData,
|
|
491
|
+
totalTokensUsed: 0, consecutiveFailures: 0,
|
|
492
|
+
error: `Max concurrent pages reached (${this.config.maxPages})`,
|
|
493
|
+
durationMs: Date.now() - start,
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
this.activeTasks++;
|
|
497
|
+
let page = null;
|
|
498
|
+
try {
|
|
499
|
+
const browser = await this.getBrowser();
|
|
500
|
+
const b = browser;
|
|
501
|
+
page = await b.newPage();
|
|
502
|
+
this.openPages++;
|
|
503
|
+
const stepStart = Date.now();
|
|
504
|
+
const actionResults = [];
|
|
505
|
+
for (const action of actions) {
|
|
506
|
+
const result = await this.executeAction(page, action);
|
|
507
|
+
actionResults.push(result);
|
|
508
|
+
this.totalSteps++;
|
|
509
|
+
this.stmtLogAction.run(taskId, 0, action.type, result.url ?? null, action.selector ?? null, result.success ? 1 : 0, result.error ?? null, result.durationMs, 0);
|
|
510
|
+
if (result.screenshot)
|
|
511
|
+
screenshots.push(result.screenshot);
|
|
512
|
+
if (result.extractedText && action.extractKey) {
|
|
513
|
+
extractedData[action.extractKey] = result.extractedText;
|
|
514
|
+
}
|
|
515
|
+
if (!result.success) {
|
|
516
|
+
consecutiveFailures++;
|
|
517
|
+
this.log.warn(`[browser-agent] Step failed: ${action.type} — ${result.error}`);
|
|
518
|
+
if (action.type === 'navigate' || consecutiveFailures >= this.config.failureThreshold) {
|
|
519
|
+
return {
|
|
520
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
521
|
+
totalTokensUsed: 0, consecutiveFailures,
|
|
522
|
+
error: result.error, durationMs: Date.now() - start,
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
else {
|
|
527
|
+
consecutiveFailures = 0;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
steps.push({
|
|
531
|
+
step: 0, actions, results: actionResults,
|
|
532
|
+
stallDetected: false, durationMs: Date.now() - stepStart,
|
|
533
|
+
});
|
|
534
|
+
this.completedTasks++;
|
|
535
|
+
return {
|
|
536
|
+
taskId, status: 'completed', steps, screenshots, extractedData,
|
|
537
|
+
totalTokensUsed: 0, consecutiveFailures,
|
|
538
|
+
durationMs: Date.now() - start,
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
catch (err) {
|
|
542
|
+
return {
|
|
543
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
544
|
+
totalTokensUsed: 0, consecutiveFailures,
|
|
545
|
+
error: err.message, durationMs: Date.now() - start,
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
finally {
|
|
549
|
+
this.activeTasks--;
|
|
550
|
+
if (page) {
|
|
551
|
+
try {
|
|
552
|
+
await page.close();
|
|
553
|
+
}
|
|
554
|
+
catch { /* */ }
|
|
555
|
+
this.openPages--;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
// ── Autonomous Task (LLM-driven loop) ────────────────────
|
|
560
|
+
/**
|
|
561
|
+
* Execute a task autonomously using the LLM feedback loop.
|
|
562
|
+
*
|
|
563
|
+
* Loop:
|
|
564
|
+
* 1. analyzePage() → Page State
|
|
565
|
+
* 2. planner.planNextActions(context) → Actions
|
|
566
|
+
* 3. Execute actions → Results
|
|
567
|
+
* 4. Check for 'done'/'fail'/stall → End or loop
|
|
568
|
+
*/
|
|
569
|
+
async runAutonomous(taskId, task) {
|
|
570
|
+
const start = Date.now();
|
|
571
|
+
const steps = [];
|
|
572
|
+
const screenshots = [];
|
|
573
|
+
const extractedData = {};
|
|
574
|
+
let totalTokensUsed = 0;
|
|
575
|
+
let consecutiveFailures = 0;
|
|
576
|
+
if (!this.planner) {
|
|
577
|
+
return {
|
|
578
|
+
taskId, status: 'aborted', steps, screenshots, extractedData,
|
|
579
|
+
totalTokensUsed: 0, consecutiveFailures: 0,
|
|
580
|
+
error: 'No planner set. Call setPlanner() with a BrowserActionPlanner.',
|
|
581
|
+
durationMs: Date.now() - start,
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
if (this.openPages >= this.config.maxPages) {
|
|
585
|
+
return {
|
|
586
|
+
taskId, status: 'aborted', steps, screenshots, extractedData,
|
|
587
|
+
totalTokensUsed: 0, consecutiveFailures: 0,
|
|
588
|
+
error: `Max concurrent pages reached (${this.config.maxPages})`,
|
|
589
|
+
durationMs: Date.now() - start,
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
this.activeTasks++;
|
|
593
|
+
const stall = new StallDetector(this.config.maxUrlRepeats);
|
|
594
|
+
let page = null;
|
|
595
|
+
try {
|
|
596
|
+
const browser = await this.getBrowser();
|
|
597
|
+
const b = browser;
|
|
598
|
+
page = await b.newPage();
|
|
599
|
+
this.openPages++;
|
|
600
|
+
const previousSteps = [];
|
|
601
|
+
for (let step = 1; step <= this.config.maxSteps; step++) {
|
|
602
|
+
const stepStart = Date.now();
|
|
603
|
+
// 1. Analyze page
|
|
604
|
+
let pageState;
|
|
605
|
+
try {
|
|
606
|
+
pageState = await this.analyzePage(page);
|
|
607
|
+
}
|
|
608
|
+
catch {
|
|
609
|
+
pageState = {
|
|
610
|
+
url: 'about:blank', title: '', interactiveElements: [],
|
|
611
|
+
links: [], forms: [], textContent: '', headings: [],
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
// 2. Ask LLM for next actions
|
|
615
|
+
const plannerContext = {
|
|
616
|
+
task,
|
|
617
|
+
currentStep: step,
|
|
618
|
+
maxSteps: this.config.maxSteps,
|
|
619
|
+
pageState,
|
|
620
|
+
previousSteps,
|
|
621
|
+
extractedData,
|
|
622
|
+
consecutiveFailures,
|
|
623
|
+
};
|
|
624
|
+
let planResult;
|
|
625
|
+
try {
|
|
626
|
+
planResult = await this.planner.planNextActions(plannerContext);
|
|
627
|
+
}
|
|
628
|
+
catch (err) {
|
|
629
|
+
this.log.error(`[browser-agent] Planner error: ${err.message}`);
|
|
630
|
+
consecutiveFailures++;
|
|
631
|
+
if (consecutiveFailures >= this.config.failureThreshold) {
|
|
632
|
+
return {
|
|
633
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
634
|
+
totalTokensUsed, consecutiveFailures,
|
|
635
|
+
error: `Planner failed ${consecutiveFailures} times consecutively`,
|
|
636
|
+
durationMs: Date.now() - start,
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
if (planResult.tokensUsed) {
|
|
642
|
+
totalTokensUsed += planResult.tokensUsed;
|
|
643
|
+
}
|
|
644
|
+
// Limit actions per step
|
|
645
|
+
const actions = planResult.actions.slice(0, this.config.actionsPerStep);
|
|
646
|
+
// 3. Check for terminal actions
|
|
647
|
+
const doneAction = actions.find(a => a.type === 'done');
|
|
648
|
+
if (doneAction) {
|
|
649
|
+
steps.push({
|
|
650
|
+
step, actions, results: [],
|
|
651
|
+
pageState, stallDetected: false,
|
|
652
|
+
durationMs: Date.now() - stepStart,
|
|
653
|
+
});
|
|
654
|
+
this.completedTasks++;
|
|
655
|
+
return {
|
|
656
|
+
taskId, status: 'completed', steps, screenshots, extractedData,
|
|
657
|
+
totalTokensUsed, consecutiveFailures,
|
|
658
|
+
finalMessage: doneAction.message,
|
|
659
|
+
durationMs: Date.now() - start,
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
const failAction = actions.find(a => a.type === 'fail');
|
|
663
|
+
if (failAction) {
|
|
664
|
+
steps.push({
|
|
665
|
+
step, actions, results: [],
|
|
666
|
+
pageState, stallDetected: false,
|
|
667
|
+
durationMs: Date.now() - stepStart,
|
|
668
|
+
});
|
|
669
|
+
return {
|
|
670
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
671
|
+
totalTokensUsed, consecutiveFailures,
|
|
672
|
+
error: failAction.message ?? 'Agent decided task cannot be completed',
|
|
673
|
+
durationMs: Date.now() - start,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
// 4. Execute actions
|
|
677
|
+
const results = [];
|
|
678
|
+
let stepFailed = false;
|
|
679
|
+
for (const action of actions) {
|
|
680
|
+
const result = await this.executeAction(page, action);
|
|
681
|
+
results.push(result);
|
|
682
|
+
this.totalSteps++;
|
|
683
|
+
this.stmtLogAction.run(taskId, step, action.type, result.url ?? null, action.selector ?? null, result.success ? 1 : 0, result.error ?? null, result.durationMs, planResult.tokensUsed ?? 0);
|
|
684
|
+
if (result.screenshot)
|
|
685
|
+
screenshots.push(result.screenshot);
|
|
686
|
+
if (result.extractedText && action.extractKey) {
|
|
687
|
+
extractedData[action.extractKey] = result.extractedText;
|
|
688
|
+
}
|
|
689
|
+
if (!result.success) {
|
|
690
|
+
stepFailed = true;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
// Track failures
|
|
694
|
+
if (stepFailed) {
|
|
695
|
+
consecutiveFailures++;
|
|
696
|
+
}
|
|
697
|
+
else {
|
|
698
|
+
consecutiveFailures = 0;
|
|
699
|
+
}
|
|
700
|
+
// 5. Stall detection
|
|
701
|
+
const currentUrl = page.url();
|
|
702
|
+
stall.record(currentUrl, actions.map(a => `${a.type}:${a.selector ?? a.url ?? ''}`));
|
|
703
|
+
const isStalled = stall.isStalled();
|
|
704
|
+
steps.push({
|
|
705
|
+
step, actions, results, pageState,
|
|
706
|
+
stallDetected: isStalled,
|
|
707
|
+
durationMs: Date.now() - stepStart,
|
|
708
|
+
});
|
|
709
|
+
// Record for next planner call
|
|
710
|
+
previousSteps.push({
|
|
711
|
+
step,
|
|
712
|
+
actions: actions.map(a => `${a.type}${a.selector ? '(' + a.selector + ')' : ''}${a.url ? '(' + a.url + ')' : ''}`),
|
|
713
|
+
results: results.map(r => r.success ? 'ok' : `fail:${r.error?.slice(0, 50)}`),
|
|
714
|
+
url: currentUrl,
|
|
715
|
+
});
|
|
716
|
+
if (isStalled) {
|
|
717
|
+
this.stalledTasks++;
|
|
718
|
+
this.log.warn(`[browser-agent] Stall detected at step ${step} — aborting task "${taskId}"`);
|
|
719
|
+
return {
|
|
720
|
+
taskId, status: 'stalled', steps, screenshots, extractedData,
|
|
721
|
+
totalTokensUsed, consecutiveFailures,
|
|
722
|
+
error: 'Agent is stuck in a loop (same URL + same actions repeated)',
|
|
723
|
+
durationMs: Date.now() - start,
|
|
724
|
+
};
|
|
725
|
+
}
|
|
726
|
+
// Failure threshold
|
|
727
|
+
if (consecutiveFailures >= this.config.failureThreshold) {
|
|
728
|
+
this.log.warn(`[browser-agent] Too many failures (${consecutiveFailures}) — aborting task "${taskId}"`);
|
|
729
|
+
return {
|
|
730
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
731
|
+
totalTokensUsed, consecutiveFailures,
|
|
732
|
+
error: `${consecutiveFailures} consecutive step failures`,
|
|
733
|
+
durationMs: Date.now() - start,
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
// Step limit reached
|
|
738
|
+
return {
|
|
739
|
+
taskId, status: 'aborted', steps, screenshots, extractedData,
|
|
740
|
+
totalTokensUsed, consecutiveFailures,
|
|
741
|
+
error: `Step limit reached (${this.config.maxSteps})`,
|
|
742
|
+
durationMs: Date.now() - start,
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
catch (err) {
|
|
746
|
+
return {
|
|
747
|
+
taskId, status: 'failed', steps, screenshots, extractedData,
|
|
748
|
+
totalTokensUsed, consecutiveFailures,
|
|
749
|
+
error: err.message, durationMs: Date.now() - start,
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
finally {
|
|
753
|
+
this.activeTasks--;
|
|
754
|
+
if (page) {
|
|
755
|
+
try {
|
|
756
|
+
await page.close();
|
|
757
|
+
}
|
|
758
|
+
catch { /* */ }
|
|
759
|
+
this.openPages--;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
// ── Status ────────────────────────────────────────────────
|
|
764
|
+
getStatus() {
|
|
765
|
+
let connected = false;
|
|
766
|
+
if (this.browser) {
|
|
767
|
+
try {
|
|
768
|
+
const b = this.browser;
|
|
769
|
+
connected = b.isConnected?.() ?? true;
|
|
770
|
+
}
|
|
771
|
+
catch { /* not connected */ }
|
|
772
|
+
}
|
|
773
|
+
return {
|
|
774
|
+
activeTasks: this.activeTasks,
|
|
775
|
+
completedTasks: this.completedTasks,
|
|
776
|
+
stalledTasks: this.stalledTasks,
|
|
777
|
+
totalSteps: this.totalSteps,
|
|
778
|
+
totalTokensUsed: this.totalTokensUsed,
|
|
779
|
+
pagesOpen: this.openPages,
|
|
780
|
+
browserConnected: connected,
|
|
781
|
+
};
|
|
782
|
+
}
|
|
783
|
+
getConfig() {
|
|
784
|
+
return { ...this.config };
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
//# sourceMappingURL=browser-agent.js.map
|