@bubblelab/bubble-core 0.1.150 → 0.1.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bubble-bundle.d.ts +6 -6
- package/dist/bubbles/service-bubble/browserbase/browserbase.d.ts +37 -0
- package/dist/bubbles/service-bubble/browserbase/browserbase.d.ts.map +1 -1
- package/dist/bubbles/service-bubble/browserbase/browserbase.js +80 -0
- package/dist/bubbles/service-bubble/browserbase/browserbase.js.map +1 -1
- package/dist/bubbles/service-bubble/browserbase/browserbase.schema.d.ts +33 -0
- package/dist/bubbles/service-bubble/browserbase/browserbase.schema.d.ts.map +1 -1
- package/dist/bubbles/service-bubble/browserbase/browserbase.schema.js +30 -0
- package/dist/bubbles/service-bubble/browserbase/browserbase.schema.js.map +1 -1
- package/dist/bubbles/service-bubble/google-drive.d.ts +1 -1
- package/dist/bubbles/service-bubble/google-drive.d.ts.map +1 -1
- package/dist/bubbles/service-bubble/google-drive.js +37 -1
- package/dist/bubbles/service-bubble/google-drive.js.map +1 -1
- package/dist/bubbles/service-bubble/slack/slack.utils.d.ts.map +1 -1
- package/dist/bubbles/service-bubble/slack/slack.utils.js +33 -4
- package/dist/bubbles/service-bubble/slack/slack.utils.js.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.d.ts +66 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.d.ts.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.js +484 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.js.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.d.ts +85 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.d.ts.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.js +2 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.js.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-fallback-step.d.ts +27 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-fallback-step.d.ts.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-fallback-step.js +77 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/ai-fallback-step.js.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/index.d.ts +4 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/index.d.ts.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/index.js +3 -0
- package/dist/bubbles/tool-bubble/browser-tools/_shared/ai/index.js.map +1 -0
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-accept-invitations/tool.d.ts.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-accept-invitations/tool.js +254 -195
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-accept-invitations/tool.js.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-connection/tool.d.ts +2 -0
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-connection/tool.d.ts.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-connection/tool.js +346 -229
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-connection/tool.js.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-received-invitations/tool.d.ts +34 -33
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-received-invitations/tool.d.ts.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-received-invitations/tool.js +212 -151
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-received-invitations/tool.js.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-sent-invitations/tool.d.ts +33 -32
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-sent-invitations/tool.d.ts.map +1 -1
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-sent-invitations/tool.js +188 -127
- package/dist/bubbles/tool-bubble/browser-tools/linkedin-sent-invitations/tool.js.map +1 -1
- package/dist/bubbles.json +73 -2
- package/package.json +2 -2
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/index.d.ts +0 -3
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/index.d.ts.map +0 -1
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/index.js +0 -3
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/index.js.map +0 -1
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.d.ts +0 -160
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.d.ts.map +0 -1
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.js +0 -706
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.js.map +0 -1
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.schema.d.ts +0 -93
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.schema.d.ts.map +0 -1
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.schema.js +0 -50
- package/dist/bubbles/tool-bubble/linkedin-connection-tool/linkedin-connection-tool.schema.js.map +0 -1
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
import { AIAgentBubble } from '../../../../service-bubble/ai-agent.js';
|
|
2
|
+
import { BrowserBaseBubble } from '../../../../service-bubble/browserbase/index.js';
|
|
3
|
+
import { CredentialType } from '@bubblelab/shared-schemas';
|
|
4
|
+
/**
|
|
5
|
+
* Script to extract interactive elements from the page.
|
|
6
|
+
* Finds buttons, links, inputs, and other interactive elements.
|
|
7
|
+
* Limited to 100 elements to avoid overwhelming the AI.
|
|
8
|
+
* Only includes elements that are truly visible (not in closed dropdowns, etc.)
|
|
9
|
+
*/
|
|
10
|
+
const EXTRACT_ELEMENTS_SCRIPT = `
|
|
11
|
+
(() => {
|
|
12
|
+
const selectors = 'button, a, input, select, textarea, [role="button"], [role="link"], [onclick]';
|
|
13
|
+
const elements = document.querySelectorAll(selectors);
|
|
14
|
+
|
|
15
|
+
// Check if element is truly visible (not just has dimensions)
|
|
16
|
+
function isElementVisible(el) {
|
|
17
|
+
const rect = el.getBoundingClientRect();
|
|
18
|
+
if (rect.width === 0 || rect.height === 0) return false;
|
|
19
|
+
|
|
20
|
+
// Check if element is within viewport
|
|
21
|
+
if (rect.bottom < 0 || rect.top > window.innerHeight) return false;
|
|
22
|
+
if (rect.right < 0 || rect.left > window.innerWidth) return false;
|
|
23
|
+
|
|
24
|
+
// Check computed styles
|
|
25
|
+
const style = window.getComputedStyle(el);
|
|
26
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
|
|
27
|
+
|
|
28
|
+
// Check if any parent hides it (dropdown items in closed dropdowns)
|
|
29
|
+
let parent = el.parentElement;
|
|
30
|
+
while (parent) {
|
|
31
|
+
const parentStyle = window.getComputedStyle(parent);
|
|
32
|
+
if (parentStyle.display === 'none' || parentStyle.visibility === 'hidden') return false;
|
|
33
|
+
// Check for closed dropdown containers (common patterns)
|
|
34
|
+
if (parent.classList.contains('artdeco-dropdown__content') ||
|
|
35
|
+
parent.classList.contains('dropdown-menu')) {
|
|
36
|
+
if (!parent.classList.contains('artdeco-dropdown__content--is-open') &&
|
|
37
|
+
!parent.classList.contains('show') &&
|
|
38
|
+
!parent.classList.contains('open')) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
parent = parent.parentElement;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return [...elements].filter(isElementVisible).slice(0, 100).map(el => {
|
|
49
|
+
const rect = el.getBoundingClientRect();
|
|
50
|
+
let selector = el.tagName.toLowerCase();
|
|
51
|
+
if (el.id) {
|
|
52
|
+
selector = '#' + el.id;
|
|
53
|
+
} else if (el.className && typeof el.className === 'string') {
|
|
54
|
+
const classes = el.className.trim().split(/\\s+/).slice(0, 3).join('.');
|
|
55
|
+
if (classes) selector = el.tagName.toLowerCase() + '.' + classes;
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
tagName: el.tagName.toLowerCase(),
|
|
59
|
+
role: el.getAttribute('role') || undefined,
|
|
60
|
+
name: el.getAttribute('aria-label') || el.textContent?.trim().slice(0, 50) || undefined,
|
|
61
|
+
id: el.id || undefined,
|
|
62
|
+
selector,
|
|
63
|
+
boundingBox: {
|
|
64
|
+
x: Math.round(rect.x),
|
|
65
|
+
y: Math.round(rect.y),
|
|
66
|
+
width: Math.round(rect.width),
|
|
67
|
+
height: Math.round(rect.height)
|
|
68
|
+
},
|
|
69
|
+
isVisible: true,
|
|
70
|
+
isEnabled: !el.disabled
|
|
71
|
+
};
|
|
72
|
+
});
|
|
73
|
+
})()
|
|
74
|
+
`;
|
|
75
|
+
/**
|
|
76
|
+
* System prompt for the AI browser agent when suggesting recovery actions.
|
|
77
|
+
*/
|
|
78
|
+
const RECOVERY_SYSTEM_PROMPT = `You are an AI browser automation assistant. You analyze screenshots and page context to help recover from failed browser automation steps.
|
|
79
|
+
|
|
80
|
+
When a step fails, you receive:
|
|
81
|
+
1. A screenshot of the current page state
|
|
82
|
+
2. The task that was being attempted
|
|
83
|
+
3. The error that occurred
|
|
84
|
+
4. A list of interactive elements on the page
|
|
85
|
+
|
|
86
|
+
Your job is to suggest ONE action to help recover from the error. Choose from:
|
|
87
|
+
|
|
88
|
+
- click: Click an element by CSS selector
|
|
89
|
+
- click_coordinates: Click at specific x,y coordinates (use when selector is unreliable)
|
|
90
|
+
- type: Type text into an input field
|
|
91
|
+
- scroll: Scroll the page up or down
|
|
92
|
+
- wait: Wait for content to load
|
|
93
|
+
- none: If you cannot help, explain why
|
|
94
|
+
|
|
95
|
+
Respond ONLY with valid JSON in this exact format:
|
|
96
|
+
{"action": "click", "selector": "#submit-btn"}
|
|
97
|
+
{"action": "click_coordinates", "coordinates": [500, 300]}
|
|
98
|
+
{"action": "type", "selector": "#email-input", "value": "test@example.com"}
|
|
99
|
+
{"action": "scroll", "direction": "down", "amount": 500}
|
|
100
|
+
{"action": "wait", "milliseconds": 2000}
|
|
101
|
+
{"action": "none", "reason": "explanation here"}
|
|
102
|
+
|
|
103
|
+
Be concise and precise. Analyze the screenshot carefully to find the right element.`;
|
|
104
|
+
/**
|
|
105
|
+
* System prompt for data extraction.
|
|
106
|
+
*/
|
|
107
|
+
const EXTRACTION_SYSTEM_PROMPT = `You are an AI data extraction assistant. You analyze screenshots to extract structured data.
|
|
108
|
+
|
|
109
|
+
You will receive:
|
|
110
|
+
1. A screenshot of the current page
|
|
111
|
+
2. A description of what data to extract
|
|
112
|
+
3. The expected data structure (field names and types)
|
|
113
|
+
|
|
114
|
+
Your job is to extract the requested data from the screenshot and return it as valid JSON.
|
|
115
|
+
|
|
116
|
+
Rules:
|
|
117
|
+
- Only extract information that is clearly visible in the screenshot
|
|
118
|
+
- Use null for fields that cannot be determined
|
|
119
|
+
- Keep string values concise and accurate
|
|
120
|
+
- Follow the exact structure requested
|
|
121
|
+
|
|
122
|
+
Respond ONLY with valid JSON matching the requested structure.`;
|
|
123
|
+
/**
|
|
124
|
+
* AI Browser Agent for error recovery and data extraction.
|
|
125
|
+
*
|
|
126
|
+
* Uses AIAgentBubble from @bubblelab/bubble-core to analyze screenshots
|
|
127
|
+
* and suggest recovery actions or extract structured data.
|
|
128
|
+
*/
|
|
129
|
+
export class AIBrowserAgent {
|
|
130
|
+
sessionId;
|
|
131
|
+
context;
|
|
132
|
+
credentials;
|
|
133
|
+
constructor(config) {
|
|
134
|
+
this.sessionId = config.sessionId;
|
|
135
|
+
this.context = config.context;
|
|
136
|
+
this.credentials = config.credentials;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Suggest a recovery action for a failed step.
|
|
140
|
+
*
|
|
141
|
+
* @param task - Description of what the step was trying to accomplish
|
|
142
|
+
* @param error - The error message from the failed step
|
|
143
|
+
* @returns The suggested recovery action
|
|
144
|
+
*/
|
|
145
|
+
async suggestRecoveryAction(task, error) {
|
|
146
|
+
try {
|
|
147
|
+
// Capture current page state
|
|
148
|
+
const screenshot = await this.captureScreenshot();
|
|
149
|
+
if (!screenshot) {
|
|
150
|
+
return { action: 'none', reason: 'Could not capture screenshot' };
|
|
151
|
+
}
|
|
152
|
+
const currentUrl = await this.getCurrentUrl();
|
|
153
|
+
const elements = await this.extractInteractiveElements();
|
|
154
|
+
// Log elements for debugging
|
|
155
|
+
console.log(`[AIBrowserAgent] Found ${elements.length} interactive elements:`);
|
|
156
|
+
elements.slice(0, 20).forEach((el, i) => {
|
|
157
|
+
console.log(` ${i + 1}. ${el.tagName}${el.id ? '#' + el.id : ''} - "${el.name?.slice(0, 40) || 'no name'}" at (${el.boundingBox.x},${el.boundingBox.y})`);
|
|
158
|
+
});
|
|
159
|
+
if (elements.length > 20) {
|
|
160
|
+
console.log(` ... and ${elements.length - 20} more elements`);
|
|
161
|
+
}
|
|
162
|
+
// Build the user message
|
|
163
|
+
const userMessage = this.buildRecoveryPrompt(task, error, currentUrl, elements);
|
|
164
|
+
// Call AI
|
|
165
|
+
const response = await this.callAI(userMessage, screenshot, RECOVERY_SYSTEM_PROMPT);
|
|
166
|
+
// Log raw AI response
|
|
167
|
+
console.log(`[AIBrowserAgent] AI raw response: ${response}`);
|
|
168
|
+
// Parse and validate response
|
|
169
|
+
return this.parseRecoveryAction(response);
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
console.error('[AIBrowserAgent] Error suggesting recovery action:', err);
|
|
173
|
+
return {
|
|
174
|
+
action: 'none',
|
|
175
|
+
reason: err instanceof Error ? err.message : 'Unknown error',
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Extract structured data from the page using AI vision.
|
|
181
|
+
*
|
|
182
|
+
* @param schema - Zod schema defining the expected data structure
|
|
183
|
+
* @param task - Description of what data to extract
|
|
184
|
+
* @returns The extracted data matching the schema, or null if extraction fails
|
|
185
|
+
*/
|
|
186
|
+
async extractData(schema, task) {
|
|
187
|
+
try {
|
|
188
|
+
// Capture screenshot
|
|
189
|
+
const screenshot = await this.captureScreenshot();
|
|
190
|
+
if (!screenshot) {
|
|
191
|
+
console.error('[AIBrowserAgent] Could not capture screenshot for extraction');
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
// Generate schema description for the AI
|
|
195
|
+
const schemaDescription = this.describeZodSchema(schema);
|
|
196
|
+
// Build user message
|
|
197
|
+
const userMessage = `Task: ${task}
|
|
198
|
+
|
|
199
|
+
Extract the following data from the screenshot:
|
|
200
|
+
${schemaDescription}
|
|
201
|
+
|
|
202
|
+
Respond with ONLY valid JSON matching this structure.`;
|
|
203
|
+
// Call AI
|
|
204
|
+
const response = await this.callAI(userMessage, screenshot, EXTRACTION_SYSTEM_PROMPT);
|
|
205
|
+
// Parse and validate with Zod
|
|
206
|
+
const parsed = JSON.parse(response);
|
|
207
|
+
const validated = schema.safeParse(parsed);
|
|
208
|
+
if (validated.success) {
|
|
209
|
+
return validated.data;
|
|
210
|
+
}
|
|
211
|
+
console.error('[AIBrowserAgent] Extracted data did not match schema:', validated.error);
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
catch (err) {
|
|
215
|
+
console.error('[AIBrowserAgent] Error extracting data:', err);
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Execute the suggested recovery action on the page.
|
|
221
|
+
*
|
|
222
|
+
* @param action - The action to execute
|
|
223
|
+
* @returns true if the action was executed successfully
|
|
224
|
+
*/
|
|
225
|
+
async executeAction(action) {
|
|
226
|
+
try {
|
|
227
|
+
const session = BrowserBaseBubble.getSession(this.sessionId);
|
|
228
|
+
if (!session) {
|
|
229
|
+
console.error('[AIBrowserAgent] No active session');
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
const page = session.page;
|
|
233
|
+
switch (action.action) {
|
|
234
|
+
case 'click': {
|
|
235
|
+
await page.waitForSelector(action.selector, { timeout: 5000 });
|
|
236
|
+
await page.click(action.selector);
|
|
237
|
+
return true;
|
|
238
|
+
}
|
|
239
|
+
case 'click_coordinates': {
|
|
240
|
+
const [x, y] = action.coordinates;
|
|
241
|
+
await page.mouse.click(x, y);
|
|
242
|
+
return true;
|
|
243
|
+
}
|
|
244
|
+
case 'type': {
|
|
245
|
+
// Clear existing content first
|
|
246
|
+
await page.waitForSelector(action.selector, { timeout: 5000 });
|
|
247
|
+
await page.click(action.selector);
|
|
248
|
+
// Clear via evaluate with string to avoid type issues
|
|
249
|
+
const clearScript = `
|
|
250
|
+
(() => {
|
|
251
|
+
const el = document.querySelector(${JSON.stringify(action.selector)});
|
|
252
|
+
if (el && 'value' in el) el.value = '';
|
|
253
|
+
})()
|
|
254
|
+
`;
|
|
255
|
+
await page.evaluate(clearScript);
|
|
256
|
+
await page.type(action.selector, action.value, { delay: 50 });
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
case 'scroll': {
|
|
260
|
+
const amount = action.direction === 'down' ? action.amount : -action.amount;
|
|
261
|
+
const scrollScript = `window.scrollBy({ top: ${amount}, behavior: 'smooth' })`;
|
|
262
|
+
await page.evaluate(scrollScript);
|
|
263
|
+
// Wait for scroll to complete
|
|
264
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
265
|
+
return true;
|
|
266
|
+
}
|
|
267
|
+
case 'wait': {
|
|
268
|
+
await new Promise((r) => setTimeout(r, action.milliseconds));
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
case 'extract':
|
|
272
|
+
case 'none':
|
|
273
|
+
// These don't require page interaction
|
|
274
|
+
return action.action === 'extract';
|
|
275
|
+
default:
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
catch (err) {
|
|
280
|
+
console.error('[AIBrowserAgent] Error executing action:', err);
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// ==================== Private Methods ====================
|
|
285
|
+
/**
|
|
286
|
+
* Capture a screenshot from the current browser session.
|
|
287
|
+
*/
|
|
288
|
+
async captureScreenshot() {
|
|
289
|
+
const session = BrowserBaseBubble.getSession(this.sessionId);
|
|
290
|
+
if (!session)
|
|
291
|
+
return null;
|
|
292
|
+
try {
|
|
293
|
+
return (await session.page.screenshot({
|
|
294
|
+
encoding: 'base64',
|
|
295
|
+
type: 'png',
|
|
296
|
+
}));
|
|
297
|
+
}
|
|
298
|
+
catch (err) {
|
|
299
|
+
console.error('[AIBrowserAgent] Screenshot capture failed:', err);
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Get the current page URL.
|
|
305
|
+
*/
|
|
306
|
+
async getCurrentUrl() {
|
|
307
|
+
const session = BrowserBaseBubble.getSession(this.sessionId);
|
|
308
|
+
if (!session)
|
|
309
|
+
return '';
|
|
310
|
+
try {
|
|
311
|
+
return session.page.url();
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
return '';
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Extract interactive elements from the page.
|
|
319
|
+
*/
|
|
320
|
+
async extractInteractiveElements() {
|
|
321
|
+
const session = BrowserBaseBubble.getSession(this.sessionId);
|
|
322
|
+
if (!session)
|
|
323
|
+
return [];
|
|
324
|
+
try {
|
|
325
|
+
const elements = await session.page.evaluate(EXTRACT_ELEMENTS_SCRIPT);
|
|
326
|
+
return elements;
|
|
327
|
+
}
|
|
328
|
+
catch (err) {
|
|
329
|
+
console.error('[AIBrowserAgent] Element extraction failed:', err);
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Call the AI agent with a message and optional screenshot.
|
|
335
|
+
*/
|
|
336
|
+
async callAI(message, screenshotBase64, systemPrompt) {
|
|
337
|
+
const geminiKey = this.credentials?.[CredentialType.GOOGLE_GEMINI_CRED];
|
|
338
|
+
if (!geminiKey) {
|
|
339
|
+
throw new Error('No Google Gemini credentials provided for AI fallback');
|
|
340
|
+
}
|
|
341
|
+
const agent = new AIAgentBubble({
|
|
342
|
+
name: 'Browser Recovery Agent',
|
|
343
|
+
message,
|
|
344
|
+
systemPrompt,
|
|
345
|
+
model: {
|
|
346
|
+
model: 'google/gemini-3-flash-preview',
|
|
347
|
+
temperature: 0.1,
|
|
348
|
+
jsonMode: true,
|
|
349
|
+
},
|
|
350
|
+
images: [
|
|
351
|
+
{
|
|
352
|
+
type: 'base64',
|
|
353
|
+
data: screenshotBase64,
|
|
354
|
+
mimeType: 'image/png',
|
|
355
|
+
},
|
|
356
|
+
],
|
|
357
|
+
credentials: {
|
|
358
|
+
[CredentialType.GOOGLE_GEMINI_CRED]: geminiKey,
|
|
359
|
+
},
|
|
360
|
+
// Disable tools for simple JSON responses
|
|
361
|
+
tools: [],
|
|
362
|
+
}, this.context);
|
|
363
|
+
const result = await agent.action();
|
|
364
|
+
if (!result.data?.response) {
|
|
365
|
+
throw new Error('No response from AI agent');
|
|
366
|
+
}
|
|
367
|
+
return result.data.response;
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Build the prompt for recovery action suggestion.
|
|
371
|
+
*/
|
|
372
|
+
buildRecoveryPrompt(task, error, currentUrl, elements) {
|
|
373
|
+
// Summarize elements for context (limit to key info)
|
|
374
|
+
const elementSummary = elements
|
|
375
|
+
.slice(0, 30)
|
|
376
|
+
.map((el) => {
|
|
377
|
+
const parts = [el.tagName];
|
|
378
|
+
if (el.name)
|
|
379
|
+
parts.push(`"${el.name}"`);
|
|
380
|
+
if (el.id)
|
|
381
|
+
parts.push(`#${el.id}`);
|
|
382
|
+
if (el.role)
|
|
383
|
+
parts.push(`[role="${el.role}"]`);
|
|
384
|
+
parts.push(`at (${el.boundingBox.x},${el.boundingBox.y})`);
|
|
385
|
+
return parts.join(' ');
|
|
386
|
+
})
|
|
387
|
+
.join('\n');
|
|
388
|
+
return `Task: ${task}
|
|
389
|
+
|
|
390
|
+
Error: ${error}
|
|
391
|
+
|
|
392
|
+
Current URL: ${currentUrl}
|
|
393
|
+
|
|
394
|
+
Interactive elements on page:
|
|
395
|
+
${elementSummary || 'No interactive elements found'}
|
|
396
|
+
|
|
397
|
+
Analyze the screenshot and suggest ONE action to help recover from this error.`;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Parse the AI response into a typed action.
|
|
401
|
+
*/
|
|
402
|
+
parseRecoveryAction(response) {
|
|
403
|
+
try {
|
|
404
|
+
const parsed = JSON.parse(response);
|
|
405
|
+
// Validate action type
|
|
406
|
+
switch (parsed.action) {
|
|
407
|
+
case 'click':
|
|
408
|
+
if (typeof parsed.selector === 'string') {
|
|
409
|
+
return { action: 'click', selector: parsed.selector };
|
|
410
|
+
}
|
|
411
|
+
break;
|
|
412
|
+
case 'click_coordinates':
|
|
413
|
+
if (Array.isArray(parsed.coordinates) &&
|
|
414
|
+
parsed.coordinates.length === 2 &&
|
|
415
|
+
typeof parsed.coordinates[0] === 'number' &&
|
|
416
|
+
typeof parsed.coordinates[1] === 'number') {
|
|
417
|
+
return {
|
|
418
|
+
action: 'click_coordinates',
|
|
419
|
+
coordinates: parsed.coordinates,
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
break;
|
|
423
|
+
case 'type':
|
|
424
|
+
if (typeof parsed.selector === 'string' &&
|
|
425
|
+
typeof parsed.value === 'string') {
|
|
426
|
+
return {
|
|
427
|
+
action: 'type',
|
|
428
|
+
selector: parsed.selector,
|
|
429
|
+
value: parsed.value,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
break;
|
|
433
|
+
case 'scroll':
|
|
434
|
+
if ((parsed.direction === 'up' || parsed.direction === 'down') &&
|
|
435
|
+
typeof parsed.amount === 'number') {
|
|
436
|
+
return {
|
|
437
|
+
action: 'scroll',
|
|
438
|
+
direction: parsed.direction,
|
|
439
|
+
amount: parsed.amount,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
break;
|
|
443
|
+
case 'wait':
|
|
444
|
+
if (typeof parsed.milliseconds === 'number') {
|
|
445
|
+
return { action: 'wait', milliseconds: parsed.milliseconds };
|
|
446
|
+
}
|
|
447
|
+
break;
|
|
448
|
+
case 'none':
|
|
449
|
+
return {
|
|
450
|
+
action: 'none',
|
|
451
|
+
reason: typeof parsed.reason === 'string'
|
|
452
|
+
? parsed.reason
|
|
453
|
+
: 'Unknown reason',
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
return { action: 'none', reason: 'Invalid action format from AI' };
|
|
457
|
+
}
|
|
458
|
+
catch {
|
|
459
|
+
return { action: 'none', reason: 'Failed to parse AI response' };
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Generate a human-readable description of a Zod schema.
|
|
464
|
+
*/
|
|
465
|
+
describeZodSchema(schema) {
|
|
466
|
+
// Get the shape if it's an object schema
|
|
467
|
+
const def = schema._def;
|
|
468
|
+
if (def.typeName === 'ZodObject' && def.shape) {
|
|
469
|
+
const shape = def.shape();
|
|
470
|
+
const fields = Object.entries(shape).map(([key, fieldSchema]) => {
|
|
471
|
+
const fieldDef = fieldSchema._def;
|
|
472
|
+
const type = fieldDef.typeName?.replace('Zod', '').toLowerCase() || 'unknown';
|
|
473
|
+
const optional = fieldDef.typeName === 'ZodOptional' ? ' (optional)' : '';
|
|
474
|
+
const desc = fieldDef.description ? ` - ${fieldDef.description}` : '';
|
|
475
|
+
return ` "${key}": ${type}${optional}${desc}`;
|
|
476
|
+
});
|
|
477
|
+
return `{
|
|
478
|
+
${fields.join(',\n')}
|
|
479
|
+
}`;
|
|
480
|
+
}
|
|
481
|
+
return 'JSON object';
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
//# sourceMappingURL=ai-browser-agent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-browser-agent.js","sourceRoot":"","sources":["../../../../../../src/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iDAAiD,CAAC;AAEpF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAO3D;;;;;GAKG;AACH,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgE/B,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;oFAyBqD,CAAC;AAErF;;GAEG;AACH,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;+DAe8B,CAAC;AAEhE;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IACjB,SAAS,CAAS;IAClB,OAAO,CAAiB;IACxB,WAAW,CAA0B;IAE7C,YAAY,MAA4B;QACtC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAoC,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACxC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,qBAAqB,CACzB,IAAY,EACZ,KAAa;QAEb,IAAI,CAAC;YACH,6BAA6B;YAC7B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;YACpE,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAEzD,6BAA6B;YAC7B,OAAO,CAAC,GAAG,CACT,0BAA0B,QAAQ,CAAC,MAAM,wBAAwB,CAClE,CAAC;YACF,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBACtC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,SAAS,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,GAAG,CAC9I,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,GAAG,EAAE,gBAAgB,CAAC,CAAC;YACjE,CAAC;YAED,yBAAyB;YACzB,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAC1C,IAAI,EACJ,KAAK,EACL,UAAU,EACV,QAAQ,CACT,CAAC;YAEF,UAAU;YACV,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAChC,WAAW,EACX,UAAU,EACV,sBAAsB,CACvB,CAAC;YAEF,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;YAE7D,8BAA8B;YAC9B,OAAO,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,oDAAoD,EAAE,GAAG,CAAC,CAAC;YACzE,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAC7D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,WAAW,CAAI,MAAoB,EAAE,IAAY;QACrD,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,CACX,8DAA8D,CAC/D,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;YAED,yCAAyC;YACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;YAEzD,qBAAqB;YACrB,MAAM,WAAW,GAAG,SAAS,IAAI;;;EAGrC,iBAAiB;;sDAEmC,CAAC;YAEjD,UAAU;YACV,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAChC,WAAW,EACX,UAAU,EACV,wBAAwB,CACzB,CAAC;YAEF,8BAA8B;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE3C,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAC,IAAI,CAAC;YACxB,CAAC;YAED,OAAO,CAAC,KAAK,CACX,uDAAuD,EACvD,SAAS,CAAC,KAAK,CAChB,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,yCAAyC,EAAE,GAAG,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CAAC,MAAuB;QACzC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACpD,OAAO,KAAK,CAAC;YACf,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YAE1B,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAClC,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,KAAK,mBAAmB,CAAC,CAAC,CAAC;oBACzB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;oBAClC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC7B,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,+BAA+B;oBAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAClC,sDAAsD;oBACtD,MAAM,WAAW,GAAG;;kDAEoB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;;;WAGtE,CAAC;oBACF,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACd,MAAM,MAAM,GACV,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;oBAC/D,MAAM,YAAY,GAAG,0BAA0B,MAAM,yBAAyB,CAAC;oBAC/E,MAAM,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;oBAClC,8BAA8B;oBAC9B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC7C,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;oBAC7D,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,KAAK,SAAS,CAAC;gBACf,KAAK,MAAM;oBACT,uCAAuC;oBACvC,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;gBAErC;oBACE,OAAO,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,GAAG,CAAC,CAAC;YAC/D,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,4DAA4D;IAE5D;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC7B,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,IAAI,CAAC;YACH,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;gBACpC,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,KAAK;aACZ,CAAC,CAAW,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa;QACzB,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,0BAA0B;QACtC,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;YACtE,OAAO,QAAgC,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,MAAM,CAClB,OAAe,EACf,gBAAwB,EACxB,YAAoB;QAEpB,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,cAAc,CAAC,kBAAkB,CAAC,CAAC;QACxE,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,aAAa,CAC7B;YACE,IAAI,EAAE,wBAAwB;YAC9B,OAAO;YACP,YAAY;YACZ,KAAK,EAAE;gBACL,KAAK,EAAE,+BAA+B;gBACtC,WAAW,EAAE,GAAG;gBAChB,QAAQ,EAAE,IAAI;aACf;YACD,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACtB;aACF;YACD,WAAW,EAAE;gBACX,CAAC,cAAc,CAAC,kBAAkB,CAAC,EAAE,SAAS;aAC/C;YACD,0CAA0C;YAC1C,KAAK,EAAE,EAAE;SACV,EACD,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QAEpC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,IAAY,EACZ,KAAa,EACb,UAAkB,EAClB,QAA8B;QAE9B,qDAAqD;QACrD,MAAM,cAAc,GAAG,QAAQ;aAC5B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACV,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAC3B,IAAI,EAAE,CAAC,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC;YACxC,IAAI,EAAE,CAAC,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC;YAC3D,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,SAAS,IAAI;;SAEf,KAAK;;eAEC,UAAU;;;EAGvB,cAAc,IAAI,+BAA+B;;+EAE4B,CAAC;IAC9E,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,QAAgB;QAC1C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAEpC,uBAAuB;YACvB,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,KAAK,OAAO;oBACV,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;wBACxC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACxD,CAAC;oBACD,MAAM;gBAER,KAAK,mBAAmB;oBACtB,IACE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;wBACjC,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;wBAC/B,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,QAAQ;wBACzC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,QAAQ,EACzC,CAAC;wBACD,OAAO;4BACL,MAAM,EAAE,mBAAmB;4BAC3B,WAAW,EAAE,MAAM,CAAC,WAA+B;yBACpD,CAAC;oBACJ,CAAC;oBACD,MAAM;gBAER,KAAK,MAAM;oBACT,IACE,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;wBACnC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAChC,CAAC;wBACD,OAAO;4BACL,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,MAAM,CAAC,QAAQ;4BACzB,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC;oBACJ,CAAC;oBACD,MAAM;gBAER,KAAK,QAAQ;oBACX,IACE,CAAC,MAAM,CAAC,SAAS,KAAK,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC;wBAC1D,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EACjC,CAAC;wBACD,OAAO;4BACL,MAAM,EAAE,QAAQ;4BAChB,SAAS,EAAE,MAAM,CAAC,SAAS;4BAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;yBACtB,CAAC;oBACJ,CAAC;oBACD,MAAM;gBAER,KAAK,MAAM;oBACT,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;wBAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;oBAC/D,CAAC;oBACD,MAAM;gBAER,KAAK,MAAM;oBACT,OAAO;wBACL,MAAM,EAAE,MAAM;wBACd,MAAM,EACJ,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;4BAC/B,CAAC,CAAC,MAAM,CAAC,MAAM;4BACf,CAAC,CAAC,gBAAgB;qBACvB,CAAC;YACN,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAC;QACnE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,MAA0B;QAClD,yCAAyC;QACzC,MAAM,GAAG,GAAG,MAAM,CAAC,IAGlB,CAAC;QAEF,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,EAAE;gBAC9D,MAAM,QAAQ,GAAG,WAAW,CAAC,IAG5B,CAAC;gBACF,MAAM,IAAI,GACR,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,SAAS,CAAC;gBACnE,MAAM,QAAQ,GACZ,QAAQ,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,OAAO,MAAM,GAAG,MAAM,IAAI,GAAG,QAAQ,GAAG,IAAI,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,OAAO;EACX,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;EAClB,CAAC;QACC,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Represents an interactive element extracted from the page.
|
|
4
|
+
* Used to provide context to the AI for suggesting recovery actions.
|
|
5
|
+
*/
|
|
6
|
+
export interface InteractiveElement {
|
|
7
|
+
tagName: string;
|
|
8
|
+
role?: string;
|
|
9
|
+
name?: string;
|
|
10
|
+
id?: string;
|
|
11
|
+
selector: string;
|
|
12
|
+
boundingBox: {
|
|
13
|
+
x: number;
|
|
14
|
+
y: number;
|
|
15
|
+
width: number;
|
|
16
|
+
height: number;
|
|
17
|
+
};
|
|
18
|
+
isVisible: boolean;
|
|
19
|
+
isEnabled: boolean;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Possible actions the AI can suggest for error recovery.
|
|
23
|
+
*/
|
|
24
|
+
export type AIBrowserAction = {
|
|
25
|
+
action: 'click';
|
|
26
|
+
selector: string;
|
|
27
|
+
} | {
|
|
28
|
+
action: 'click_coordinates';
|
|
29
|
+
coordinates: [number, number];
|
|
30
|
+
} | {
|
|
31
|
+
action: 'type';
|
|
32
|
+
selector: string;
|
|
33
|
+
value: string;
|
|
34
|
+
} | {
|
|
35
|
+
action: 'scroll';
|
|
36
|
+
direction: 'up' | 'down';
|
|
37
|
+
amount: number;
|
|
38
|
+
} | {
|
|
39
|
+
action: 'wait';
|
|
40
|
+
milliseconds: number;
|
|
41
|
+
} | {
|
|
42
|
+
action: 'extract';
|
|
43
|
+
data: unknown;
|
|
44
|
+
} | {
|
|
45
|
+
action: 'none';
|
|
46
|
+
reason: string;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Context provided to the AI for recovery decision making.
|
|
50
|
+
*/
|
|
51
|
+
export interface AIRecoveryContext {
|
|
52
|
+
taskDescription: string;
|
|
53
|
+
errorMessage: string;
|
|
54
|
+
currentUrl: string;
|
|
55
|
+
screenshotBase64: string;
|
|
56
|
+
interactiveElements: InteractiveElement[];
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Configuration for the AI browser agent.
|
|
60
|
+
*/
|
|
61
|
+
export interface AIBrowserAgentConfig {
|
|
62
|
+
sessionId: string;
|
|
63
|
+
context?: unknown;
|
|
64
|
+
credentials?: Record<string, string>;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Result of an AI recovery attempt.
|
|
68
|
+
*/
|
|
69
|
+
export interface AIRecoveryResult {
|
|
70
|
+
success: boolean;
|
|
71
|
+
action: AIBrowserAction;
|
|
72
|
+
error?: string;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Extended options for RecordableStep with AI fallback support.
|
|
76
|
+
*/
|
|
77
|
+
export interface AIFallbackOptions {
|
|
78
|
+
/** Enable AI fallback for error recovery */
|
|
79
|
+
aiFallback?: boolean;
|
|
80
|
+
/** Additional context for the AI about what this step is trying to accomplish */
|
|
81
|
+
taskDescription?: string;
|
|
82
|
+
/** Zod schema for data extraction - when provided, AI extracts data matching the schema */
|
|
83
|
+
extractionSchema?: z.ZodType<unknown>;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=ai-browser-agent.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-browser-agent.types.d.ts","sourceRoot":"","sources":["../../../../../../src/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACrE,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,MAAM,EAAE,mBAAmB,CAAC;IAAC,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,SAAS,EAAE,IAAI,GAAG,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACxC;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GACpC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4CAA4C;IAC5C,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,2FAA2F;IAC3F,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-browser-agent.types.js","sourceRoot":"","sources":["../../../../../../src/bubbles/tool-bubble/browser-tools/_shared/ai/ai-browser-agent.types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Options for the AI fallback decorator.
|
|
4
|
+
*/
|
|
5
|
+
interface AIFallbackStepOptions {
|
|
6
|
+
taskDescription?: string;
|
|
7
|
+
extractionSchema?: z.ZodType<unknown>;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Lightweight decorator that wraps a method with AI fallback error recovery.
|
|
11
|
+
* This is the OSS-compatible version of @RecordableStep — no recording,
|
|
12
|
+
* just AI-powered recovery when selectors/actions fail.
|
|
13
|
+
*
|
|
14
|
+
* When the decorated method throws, the decorator:
|
|
15
|
+
* 1. Creates an AIBrowserAgent with the active session
|
|
16
|
+
* 2. If extractionSchema is provided: uses AI vision to extract data
|
|
17
|
+
* 3. Otherwise: asks AI to suggest a recovery action (click, type, scroll, etc.)
|
|
18
|
+
* 4. Executes the suggested action
|
|
19
|
+
* 5. For wait/scroll: retries the original method
|
|
20
|
+
* 6. For click/type/click_coordinates: returns true (action completed)
|
|
21
|
+
*
|
|
22
|
+
* @param stepName - Human-readable name for logging
|
|
23
|
+
* @param options - Task description and optional extraction schema
|
|
24
|
+
*/
|
|
25
|
+
export declare function AIFallbackStep(stepName: string, options?: AIFallbackStepOptions): <This, Args extends unknown[], Return>(originalMethod: (this: This, ...args: Args) => Promise<Return>, _context: ClassMethodDecoratorContext<This, (this: This, ...args: Args) => Promise<Return>>) => (this: This, ...args: Args) => Promise<Return>;
|
|
26
|
+
export {};
|
|
27
|
+
//# sourceMappingURL=ai-fallback-step.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-fallback-step.d.ts","sourceRoot":"","sources":["../../../../../../src/bubbles/tool-bubble/browser-tools/_shared/ai/ai-fallback-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAG7B;;GAEG;AACH,UAAU,qBAAqB;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,qBAA0B,IAElB,IAAI,EAAE,IAAI,SAAS,OAAO,EAAE,EAAE,MAAM,EACnD,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,EAC9D,UAAU,2BAA2B,CACnC,IAAI,EACJ,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,CAC/C,MAEsB,MAAM,IAAI,EAAE,GAAG,MAAM,IAAI,KAAG,OAAO,CAAC,MAAM,CAAC,CAoErE"}
|