@agent-relay/browser-primitive 4.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DESIGN.md +778 -0
- package/examples/browser-workflow.ts +138 -0
- package/package.json +57 -0
package/DESIGN.md
ADDED
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
# Browser Workflow Primitive
|
|
2
|
+
|
|
3
|
+
A workflow primitive that enables agents to perform browser automation using Playwright, designed to complement the existing GitHub and other integration primitives.
|
|
4
|
+
|
|
5
|
+
## Package Structure
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
packages/browser-primitive/
|
|
9
|
+
├── DESIGN.md # This design document
|
|
10
|
+
├── package.json # Package manifest
|
|
11
|
+
├── src/
|
|
12
|
+
│ ├── index.ts # Main exports
|
|
13
|
+
│ ├── types.ts # TypeScript interfaces
|
|
14
|
+
│ ├── executor.ts # Browser action executor
|
|
15
|
+
│ ├── session.ts # Browser session management
|
|
16
|
+
│ ├── actions/ # Browser action implementations
|
|
17
|
+
│ │ ├── navigation.ts # navigate, reload, back, forward
|
|
18
|
+
│ │ ├── interaction.ts # click, fill, submit, hover
|
|
19
|
+
│ │ ├── extraction.ts # getText, getHTML, getAttribute
|
|
20
|
+
│ │ ├── screenshot.ts # screenshot, elementScreenshot
|
|
21
|
+
│ │ ├── javascript.ts # evaluate, addScript
|
|
22
|
+
│ │ └── iframe.ts # iframe handling
|
|
23
|
+
│ ├── utils/
|
|
24
|
+
│ │ ├── selector.ts # Selector validation and enhancement
|
|
25
|
+
│ │ ├── wait.ts # Waiting strategies
|
|
26
|
+
│ │ └── console.ts # Console log capture
|
|
27
|
+
│ └── __tests__/
|
|
28
|
+
│ ├── actions/ # Action unit tests
|
|
29
|
+
│ ├── integration/ # Integration tests
|
|
30
|
+
│ └── fixtures/ # Test HTML files
|
|
31
|
+
├── templates/ # Workflow templates
|
|
32
|
+
│ ├── web-scraping.yaml # Data extraction workflow
|
|
33
|
+
│ ├── form-filling.yaml # Form automation workflow
|
|
34
|
+
│ └── e2e-testing.yaml # End-to-end testing workflow
|
|
35
|
+
├── docs/
|
|
36
|
+
│ ├── getting-started.md # Usage guide
|
|
37
|
+
│ ├── actions.md # Action reference
|
|
38
|
+
│ └── examples.md # Example workflows
|
|
39
|
+
└── README.md # Package overview
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## TypeScript Interfaces
|
|
43
|
+
|
|
44
|
+
### Core Action Types
|
|
45
|
+
|
|
46
|
+
```typescript
|
|
47
|
+
// Browser action types that map to workflow step actions
|
|
48
|
+
export type BrowserAction =
|
|
49
|
+
| 'navigate'
|
|
50
|
+
| 'click'
|
|
51
|
+
| 'fill'
|
|
52
|
+
| 'submit'
|
|
53
|
+
| 'waitForElement'
|
|
54
|
+
| 'waitForNavigation'
|
|
55
|
+
| 'screenshot'
|
|
56
|
+
| 'elementScreenshot'
|
|
57
|
+
| 'getText'
|
|
58
|
+
| 'getHTML'
|
|
59
|
+
| 'getAttribute'
|
|
60
|
+
| 'evaluate'
|
|
61
|
+
| 'addScript'
|
|
62
|
+
| 'reload'
|
|
63
|
+
| 'back'
|
|
64
|
+
| 'forward'
|
|
65
|
+
| 'hover'
|
|
66
|
+
| 'select'
|
|
67
|
+
| 'upload'
|
|
68
|
+
| 'switchFrame'
|
|
69
|
+
| 'clearCookies'
|
|
70
|
+
| 'setCookie'
|
|
71
|
+
| 'setHeaders';
|
|
72
|
+
|
|
73
|
+
// Browser configuration for session setup
|
|
74
|
+
export interface BrowserConfig {
|
|
75
|
+
/** Browser engine to use */
|
|
76
|
+
browser?: 'chromium' | 'firefox' | 'webkit';
|
|
77
|
+
/** Run in headless mode (default: true) */
|
|
78
|
+
headless?: boolean;
|
|
79
|
+
/** Viewport dimensions */
|
|
80
|
+
viewport?: { width: number; height: number };
|
|
81
|
+
/** Navigation timeout in ms (default: 30000) */
|
|
82
|
+
timeout?: number;
|
|
83
|
+
/** Custom user agent string */
|
|
84
|
+
userAgent?: string;
|
|
85
|
+
/** Extra HTTP headers to send with requests */
|
|
86
|
+
extraHTTPHeaders?: Record<string, string>;
|
|
87
|
+
/** Browser launch arguments */
|
|
88
|
+
args?: string[];
|
|
89
|
+
/** Enable browser console log capture (default: true) */
|
|
90
|
+
captureConsole?: boolean;
|
|
91
|
+
/** Enable network request logging (default: false) */
|
|
92
|
+
captureNetwork?: boolean;
|
|
93
|
+
/** Session persistence between actions */
|
|
94
|
+
persistSession?: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Action parameter interfaces
|
|
98
|
+
export interface NavigateParams {
|
|
99
|
+
url: string;
|
|
100
|
+
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle';
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface ClickParams {
|
|
104
|
+
selector: string;
|
|
105
|
+
/** Wait for element to exist before clicking */
|
|
106
|
+
waitFor?: boolean;
|
|
107
|
+
/** Force click even if element is not visible */
|
|
108
|
+
force?: boolean;
|
|
109
|
+
/** Click position relative to element */
|
|
110
|
+
position?: { x: number; y: number };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export interface FillParams {
|
|
114
|
+
selector: string;
|
|
115
|
+
value: string;
|
|
116
|
+
/** Clear existing value before filling */
|
|
117
|
+
clear?: boolean;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface SubmitParams {
|
|
121
|
+
/** Form selector, if not provided submits the first form */
|
|
122
|
+
selector?: string;
|
|
123
|
+
/** Wait for navigation after submit */
|
|
124
|
+
waitForNavigation?: boolean;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export interface WaitForElementParams {
|
|
128
|
+
selector: string;
|
|
129
|
+
/** Wait condition */
|
|
130
|
+
state?: 'attached' | 'detached' | 'visible' | 'hidden';
|
|
131
|
+
/** Timeout in ms */
|
|
132
|
+
timeout?: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface ScreenshotParams {
|
|
136
|
+
/** Element selector for partial screenshot */
|
|
137
|
+
selector?: string;
|
|
138
|
+
/** Output file path (relative to workflow working directory) */
|
|
139
|
+
path?: string;
|
|
140
|
+
/** Full page screenshot */
|
|
141
|
+
fullPage?: boolean;
|
|
142
|
+
/** Screenshot format */
|
|
143
|
+
type?: 'png' | 'jpeg';
|
|
144
|
+
/** JPEG quality (0-100) */
|
|
145
|
+
quality?: number;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export interface GetTextParams {
|
|
149
|
+
selector: string;
|
|
150
|
+
/** Get inner text vs text content */
|
|
151
|
+
innerText?: boolean;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export interface GetHTMLParams {
|
|
155
|
+
selector?: string;
|
|
156
|
+
/** Get outer HTML vs inner HTML */
|
|
157
|
+
outerHTML?: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export interface EvaluateParams {
|
|
161
|
+
/** JavaScript code to execute */
|
|
162
|
+
script: string;
|
|
163
|
+
/** Arguments to pass to the script */
|
|
164
|
+
args?: unknown[];
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export interface SetCookieParams {
|
|
168
|
+
name: string;
|
|
169
|
+
value: string;
|
|
170
|
+
domain?: string;
|
|
171
|
+
path?: string;
|
|
172
|
+
expires?: number;
|
|
173
|
+
httpOnly?: boolean;
|
|
174
|
+
secure?: boolean;
|
|
175
|
+
sameSite?: 'Strict' | 'Lax' | 'None';
|
|
176
|
+
}
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### Session and State Management
|
|
180
|
+
|
|
181
|
+
```typescript
|
|
182
|
+
// Browser session state
|
|
183
|
+
export interface BrowserSession {
|
|
184
|
+
/** Session ID for tracking */
|
|
185
|
+
id: string;
|
|
186
|
+
/** Browser instance configuration */
|
|
187
|
+
config: BrowserConfig;
|
|
188
|
+
/** Current page URL */
|
|
189
|
+
currentUrl?: string;
|
|
190
|
+
/** Session cookies */
|
|
191
|
+
cookies: Cookie[];
|
|
192
|
+
/** Console logs from this session */
|
|
193
|
+
consoleLogs: ConsoleMessage[];
|
|
194
|
+
/** Network requests (if enabled) */
|
|
195
|
+
networkLogs?: NetworkRequest[];
|
|
196
|
+
/** Session start time */
|
|
197
|
+
startTime: Date;
|
|
198
|
+
/** Whether session is currently active */
|
|
199
|
+
active: boolean;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export interface ConsoleMessage {
|
|
203
|
+
type: 'log' | 'error' | 'warn' | 'info' | 'debug';
|
|
204
|
+
text: string;
|
|
205
|
+
timestamp: Date;
|
|
206
|
+
location?: string;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export interface NetworkRequest {
|
|
210
|
+
url: string;
|
|
211
|
+
method: string;
|
|
212
|
+
status: number;
|
|
213
|
+
responseTime: number;
|
|
214
|
+
timestamp: Date;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// Action execution result
|
|
218
|
+
export interface BrowserActionResult {
|
|
219
|
+
/** Whether action succeeded */
|
|
220
|
+
success: boolean;
|
|
221
|
+
/** Action output (text, HTML, screenshot path, etc.) */
|
|
222
|
+
output: string;
|
|
223
|
+
/** Error message if action failed */
|
|
224
|
+
error?: string;
|
|
225
|
+
/** Additional metadata */
|
|
226
|
+
metadata?: {
|
|
227
|
+
/** Current page URL after action */
|
|
228
|
+
currentUrl?: string;
|
|
229
|
+
/** Screenshot path if taken automatically on error */
|
|
230
|
+
errorScreenshot?: string;
|
|
231
|
+
/** Console logs during action */
|
|
232
|
+
consoleLogs?: ConsoleMessage[];
|
|
233
|
+
/** Network activity during action */
|
|
234
|
+
networkActivity?: NetworkRequest[];
|
|
235
|
+
/** Action execution time in ms */
|
|
236
|
+
executionTime?: number;
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## Step Configuration Schema
|
|
242
|
+
|
|
243
|
+
Browser steps integrate into workflows using the existing integration step pattern:
|
|
244
|
+
|
|
245
|
+
```yaml
|
|
246
|
+
steps:
|
|
247
|
+
- name: login-to-app
|
|
248
|
+
type: integration
|
|
249
|
+
integration: browser
|
|
250
|
+
action: navigate
|
|
251
|
+
params:
|
|
252
|
+
url: 'https://app.example.com/login'
|
|
253
|
+
waitUntil: 'networkidle'
|
|
254
|
+
|
|
255
|
+
- name: fill-credentials
|
|
256
|
+
type: integration
|
|
257
|
+
integration: browser
|
|
258
|
+
action: fill
|
|
259
|
+
params:
|
|
260
|
+
selector: 'input[name="email"]'
|
|
261
|
+
value: '{{steps.get-credentials.output.email}}'
|
|
262
|
+
|
|
263
|
+
- name: submit-login
|
|
264
|
+
type: integration
|
|
265
|
+
integration: browser
|
|
266
|
+
action: submit
|
|
267
|
+
params:
|
|
268
|
+
selector: 'form#login-form'
|
|
269
|
+
waitForNavigation: 'true'
|
|
270
|
+
|
|
271
|
+
- name: capture-dashboard
|
|
272
|
+
type: integration
|
|
273
|
+
integration: browser
|
|
274
|
+
action: screenshot
|
|
275
|
+
params:
|
|
276
|
+
path: 'dashboard-screenshot.png'
|
|
277
|
+
fullPage: 'true'
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
### Global Browser Configuration
|
|
281
|
+
|
|
282
|
+
Browser configuration can be set at the workflow level:
|
|
283
|
+
|
|
284
|
+
```yaml
|
|
285
|
+
# Global browser configuration
|
|
286
|
+
browserConfig:
|
|
287
|
+
headless: false
|
|
288
|
+
viewport:
|
|
289
|
+
width: 1920
|
|
290
|
+
height: 1080
|
|
291
|
+
timeout: 30000
|
|
292
|
+
captureConsole: true
|
|
293
|
+
persistSession: true
|
|
294
|
+
|
|
295
|
+
steps:
|
|
296
|
+
# Browser steps inherit global config
|
|
297
|
+
- name: navigate-home
|
|
298
|
+
type: integration
|
|
299
|
+
integration: browser
|
|
300
|
+
action: navigate
|
|
301
|
+
params:
|
|
302
|
+
url: 'https://example.com'
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
### Step-Level Configuration Override
|
|
306
|
+
|
|
307
|
+
Individual steps can override global browser config:
|
|
308
|
+
|
|
309
|
+
```yaml
|
|
310
|
+
steps:
|
|
311
|
+
- name: mobile-test
|
|
312
|
+
type: integration
|
|
313
|
+
integration: browser
|
|
314
|
+
action: navigate
|
|
315
|
+
params:
|
|
316
|
+
url: 'https://example.com'
|
|
317
|
+
# Step-specific browser config
|
|
318
|
+
browserConfig:
|
|
319
|
+
viewport:
|
|
320
|
+
width: 375
|
|
321
|
+
height: 667
|
|
322
|
+
userAgent: 'Mobile Safari'
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
## Example Workflow Usage
|
|
326
|
+
|
|
327
|
+
### 1. Data Extraction Workflow
|
|
328
|
+
|
|
329
|
+
```yaml
|
|
330
|
+
version: '1.0'
|
|
331
|
+
name: extract-product-data
|
|
332
|
+
description: Extract product information from e-commerce site
|
|
333
|
+
|
|
334
|
+
browserConfig:
|
|
335
|
+
headless: true
|
|
336
|
+
captureConsole: false
|
|
337
|
+
persistSession: true
|
|
338
|
+
|
|
339
|
+
steps:
|
|
340
|
+
- name: navigate-to-products
|
|
341
|
+
type: integration
|
|
342
|
+
integration: browser
|
|
343
|
+
action: navigate
|
|
344
|
+
params:
|
|
345
|
+
url: 'https://store.example.com/products'
|
|
346
|
+
waitUntil: 'networkidle'
|
|
347
|
+
|
|
348
|
+
- name: search-for-item
|
|
349
|
+
type: integration
|
|
350
|
+
integration: browser
|
|
351
|
+
action: fill
|
|
352
|
+
params:
|
|
353
|
+
selector: 'input[name="search"]'
|
|
354
|
+
value: '{{workflow.searchTerm}}'
|
|
355
|
+
|
|
356
|
+
- name: submit-search
|
|
357
|
+
type: integration
|
|
358
|
+
integration: browser
|
|
359
|
+
action: submit
|
|
360
|
+
params:
|
|
361
|
+
selector: 'form.search-form'
|
|
362
|
+
waitForNavigation: true
|
|
363
|
+
|
|
364
|
+
- name: extract-product-titles
|
|
365
|
+
type: integration
|
|
366
|
+
integration: browser
|
|
367
|
+
action: evaluate
|
|
368
|
+
params:
|
|
369
|
+
script: |
|
|
370
|
+
Array.from(document.querySelectorAll('.product-title'))
|
|
371
|
+
.map(el => el.textContent.trim())
|
|
372
|
+
|
|
373
|
+
- name: capture-results-page
|
|
374
|
+
type: integration
|
|
375
|
+
integration: browser
|
|
376
|
+
action: screenshot
|
|
377
|
+
params:
|
|
378
|
+
path: 'search-results-{{workflow.searchTerm}}.png'
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
### 2. Form Automation Workflow
|
|
382
|
+
|
|
383
|
+
```yaml
|
|
384
|
+
version: '1.0'
|
|
385
|
+
name: submit-application
|
|
386
|
+
description: Automate job application form submission
|
|
387
|
+
|
|
388
|
+
browserConfig:
|
|
389
|
+
headless: false
|
|
390
|
+
timeout: 45000
|
|
391
|
+
persistSession: true
|
|
392
|
+
|
|
393
|
+
steps:
|
|
394
|
+
- name: navigate-to-application
|
|
395
|
+
type: integration
|
|
396
|
+
integration: browser
|
|
397
|
+
action: navigate
|
|
398
|
+
params:
|
|
399
|
+
url: '{{steps.get-job-url.output}}'
|
|
400
|
+
|
|
401
|
+
- name: fill-personal-info
|
|
402
|
+
type: integration
|
|
403
|
+
integration: browser
|
|
404
|
+
action: fill
|
|
405
|
+
params:
|
|
406
|
+
selector: 'input[name="fullName"]'
|
|
407
|
+
value: '{{steps.get-applicant-data.output.name}}'
|
|
408
|
+
|
|
409
|
+
- name: fill-email
|
|
410
|
+
type: integration
|
|
411
|
+
integration: browser
|
|
412
|
+
action: fill
|
|
413
|
+
params:
|
|
414
|
+
selector: 'input[name="email"]'
|
|
415
|
+
value: '{{steps.get-applicant-data.output.email}}'
|
|
416
|
+
|
|
417
|
+
- name: upload-resume
|
|
418
|
+
type: integration
|
|
419
|
+
integration: browser
|
|
420
|
+
action: upload
|
|
421
|
+
params:
|
|
422
|
+
selector: 'input[type="file"]'
|
|
423
|
+
filePath: '{{steps.generate-resume.output.filePath}}'
|
|
424
|
+
|
|
425
|
+
- name: submit-application
|
|
426
|
+
type: integration
|
|
427
|
+
integration: browser
|
|
428
|
+
action: submit
|
|
429
|
+
params:
|
|
430
|
+
selector: 'form.application-form'
|
|
431
|
+
waitForNavigation: true
|
|
432
|
+
|
|
433
|
+
- name: capture-confirmation
|
|
434
|
+
type: integration
|
|
435
|
+
integration: browser
|
|
436
|
+
action: screenshot
|
|
437
|
+
params:
|
|
438
|
+
path: 'application-confirmation.png'
|
|
439
|
+
|
|
440
|
+
- name: get-confirmation-text
|
|
441
|
+
type: integration
|
|
442
|
+
integration: browser
|
|
443
|
+
action: getText
|
|
444
|
+
params:
|
|
445
|
+
selector: '.confirmation-message'
|
|
446
|
+
```
|
|
447
|
+
|
|
448
|
+
### 3. Multi-Step Testing Workflow
|
|
449
|
+
|
|
450
|
+
```yaml
|
|
451
|
+
version: '1.0'
|
|
452
|
+
name: e2e-user-journey
|
|
453
|
+
description: End-to-end test of user signup and onboarding
|
|
454
|
+
|
|
455
|
+
browserConfig:
|
|
456
|
+
headless: true
|
|
457
|
+
captureConsole: true
|
|
458
|
+
captureNetwork: true
|
|
459
|
+
|
|
460
|
+
steps:
|
|
461
|
+
- name: navigate-home
|
|
462
|
+
type: integration
|
|
463
|
+
integration: browser
|
|
464
|
+
action: navigate
|
|
465
|
+
params:
|
|
466
|
+
url: 'https://app.example.com'
|
|
467
|
+
|
|
468
|
+
- name: click-signup
|
|
469
|
+
type: integration
|
|
470
|
+
integration: browser
|
|
471
|
+
action: click
|
|
472
|
+
params:
|
|
473
|
+
selector: 'a[href="/signup"]'
|
|
474
|
+
waitFor: true
|
|
475
|
+
|
|
476
|
+
- name: wait-for-signup-form
|
|
477
|
+
type: integration
|
|
478
|
+
integration: browser
|
|
479
|
+
action: waitForElement
|
|
480
|
+
params:
|
|
481
|
+
selector: 'form#signup-form'
|
|
482
|
+
state: 'visible'
|
|
483
|
+
timeout: 10000
|
|
484
|
+
|
|
485
|
+
- name: fill-signup-form
|
|
486
|
+
type: integration
|
|
487
|
+
integration: browser
|
|
488
|
+
action: evaluate
|
|
489
|
+
params:
|
|
490
|
+
script: |
|
|
491
|
+
document.querySelector('input[name="email"]').value = '{{workflow.testEmail}}';
|
|
492
|
+
document.querySelector('input[name="password"]').value = '{{workflow.testPassword}}';
|
|
493
|
+
document.querySelector('input[name="confirmPassword"]').value = '{{workflow.testPassword}}';
|
|
494
|
+
|
|
495
|
+
- name: submit-signup
|
|
496
|
+
type: integration
|
|
497
|
+
integration: browser
|
|
498
|
+
action: submit
|
|
499
|
+
params:
|
|
500
|
+
selector: 'form#signup-form'
|
|
501
|
+
waitForNavigation: true
|
|
502
|
+
|
|
503
|
+
- name: verify-welcome-message
|
|
504
|
+
type: integration
|
|
505
|
+
integration: browser
|
|
506
|
+
action: waitForElement
|
|
507
|
+
params:
|
|
508
|
+
selector: '.welcome-message'
|
|
509
|
+
state: 'visible'
|
|
510
|
+
|
|
511
|
+
- name: capture-onboarding-screen
|
|
512
|
+
type: integration
|
|
513
|
+
integration: browser
|
|
514
|
+
action: screenshot
|
|
515
|
+
params:
|
|
516
|
+
path: 'onboarding-welcome.png'
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
## Integration with Existing Workflow System
|
|
520
|
+
|
|
521
|
+
### Executor Interface Implementation
|
|
522
|
+
|
|
523
|
+
The browser primitive implements the `WorkflowExecutor.executeIntegrationStep` interface:
|
|
524
|
+
|
|
525
|
+
```typescript
|
|
526
|
+
export class BrowserExecutor implements WorkflowExecutor {
|
|
527
|
+
async executeIntegrationStep(
|
|
528
|
+
step: WorkflowStep,
|
|
529
|
+
resolvedParams: Record<string, string>,
|
|
530
|
+
context: { workspaceId?: string }
|
|
531
|
+
): Promise<{ output: string; success: boolean }> {
|
|
532
|
+
if (step.integration !== 'browser') {
|
|
533
|
+
throw new Error(`BrowserExecutor only handles browser integration steps`);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
try {
|
|
537
|
+
const result = await this.executeBrowserAction(step.action as BrowserAction, resolvedParams, context);
|
|
538
|
+
|
|
539
|
+
return {
|
|
540
|
+
output: result.output,
|
|
541
|
+
success: result.success,
|
|
542
|
+
};
|
|
543
|
+
} catch (error) {
|
|
544
|
+
return {
|
|
545
|
+
output: error instanceof Error ? error.message : String(error),
|
|
546
|
+
success: false,
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
private async executeBrowserAction(
|
|
552
|
+
action: BrowserAction,
|
|
553
|
+
params: Record<string, string>,
|
|
554
|
+
context: { workspaceId?: string }
|
|
555
|
+
): Promise<BrowserActionResult> {
|
|
556
|
+
// Implementation details...
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
```
|
|
560
|
+
|
|
561
|
+
### Session Management
|
|
562
|
+
|
|
563
|
+
Sessions persist across steps when `persistSession: true`:
|
|
564
|
+
|
|
565
|
+
```typescript
|
|
566
|
+
export class BrowserSessionManager {
|
|
567
|
+
private sessions = new Map<string, BrowserSession>();
|
|
568
|
+
|
|
569
|
+
async getOrCreateSession(workspaceId: string, config: BrowserConfig): Promise<BrowserSession> {
|
|
570
|
+
const sessionKey = `${workspaceId}:${JSON.stringify(config)}`;
|
|
571
|
+
|
|
572
|
+
if (this.sessions.has(sessionKey)) {
|
|
573
|
+
return this.sessions.get(sessionKey)!;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
const session = await this.createSession(config);
|
|
577
|
+
this.sessions.set(sessionKey, session);
|
|
578
|
+
return session;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
async cleanupSession(sessionId: string): Promise<void> {
|
|
582
|
+
// Cleanup browser resources
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
```
|
|
586
|
+
|
|
587
|
+
### Error Handling and Recovery
|
|
588
|
+
|
|
589
|
+
The browser executor provides robust error handling:
|
|
590
|
+
|
|
591
|
+
1. **Automatic Screenshots**: Captures screenshot on action failure for debugging
|
|
592
|
+
2. **Retry Logic**: Configurable retry attempts for transient failures
|
|
593
|
+
3. **Timeout Management**: Respects workflow-level and action-level timeouts
|
|
594
|
+
4. **Graceful Degradation**: Falls back to alternative selectors when primary fails
|
|
595
|
+
|
|
596
|
+
### Output Chaining
|
|
597
|
+
|
|
598
|
+
Browser actions output results that can be chained to subsequent steps:
|
|
599
|
+
|
|
600
|
+
```yaml
|
|
601
|
+
steps:
|
|
602
|
+
- name: extract-product-price
|
|
603
|
+
type: integration
|
|
604
|
+
integration: browser
|
|
605
|
+
action: getText
|
|
606
|
+
params:
|
|
607
|
+
selector: '.price'
|
|
608
|
+
|
|
609
|
+
- name: compare-price
|
|
610
|
+
type: agent
|
|
611
|
+
agent: price-analyst
|
|
612
|
+
task: 'Analyze if price {{steps.extract-product-price.output}} is competitive'
|
|
613
|
+
```
|
|
614
|
+
|
|
615
|
+
## Agent Interaction Features
|
|
616
|
+
|
|
617
|
+
### Real-time Console Capture
|
|
618
|
+
|
|
619
|
+
When `captureConsole: true`, all browser console messages are captured and can be accessed:
|
|
620
|
+
|
|
621
|
+
```typescript
|
|
622
|
+
// Console logs are included in action metadata
|
|
623
|
+
{
|
|
624
|
+
success: true,
|
|
625
|
+
output: "Login successful",
|
|
626
|
+
metadata: {
|
|
627
|
+
consoleLogs: [
|
|
628
|
+
{ type: 'log', text: 'User authenticated', timestamp: new Date() },
|
|
629
|
+
{ type: 'error', text: 'Analytics script failed', timestamp: new Date() }
|
|
630
|
+
]
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
```
|
|
634
|
+
|
|
635
|
+
### Network Request Logging
|
|
636
|
+
|
|
637
|
+
When `captureNetwork: true`, HTTP requests are logged:
|
|
638
|
+
|
|
639
|
+
```typescript
|
|
640
|
+
{
|
|
641
|
+
success: true,
|
|
642
|
+
output: "Page loaded",
|
|
643
|
+
metadata: {
|
|
644
|
+
networkActivity: [
|
|
645
|
+
{ url: '/api/user', method: 'GET', status: 200, responseTime: 150 },
|
|
646
|
+
{ url: '/api/preferences', method: 'GET', status: 200, responseTime: 89 }
|
|
647
|
+
]
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
```
|
|
651
|
+
|
|
652
|
+
### Error Capture and Reporting
|
|
653
|
+
|
|
654
|
+
Detailed error information helps agents understand failures:
|
|
655
|
+
|
|
656
|
+
```typescript
|
|
657
|
+
{
|
|
658
|
+
success: false,
|
|
659
|
+
output: "",
|
|
660
|
+
error: "Element not found: .submit-button",
|
|
661
|
+
metadata: {
|
|
662
|
+
currentUrl: "https://example.com/form",
|
|
663
|
+
errorScreenshot: "error-step-submit-20241210-143022.png",
|
|
664
|
+
consoleLogs: [
|
|
665
|
+
{ type: 'error', text: 'Submit button removed by JS', timestamp: new Date() }
|
|
666
|
+
]
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
```
|
|
670
|
+
|
|
671
|
+
## Integration Examples
|
|
672
|
+
|
|
673
|
+
### With GitHub Primitive
|
|
674
|
+
|
|
675
|
+
```yaml
|
|
676
|
+
steps:
|
|
677
|
+
- name: test-deployment
|
|
678
|
+
type: integration
|
|
679
|
+
integration: browser
|
|
680
|
+
action: navigate
|
|
681
|
+
params:
|
|
682
|
+
url: '{{steps.deploy-to-staging.output.url}}'
|
|
683
|
+
|
|
684
|
+
- name: run-smoke-tests
|
|
685
|
+
type: integration
|
|
686
|
+
integration: browser
|
|
687
|
+
action: evaluate
|
|
688
|
+
params:
|
|
689
|
+
script: |
|
|
690
|
+
// Run basic functionality tests
|
|
691
|
+
const results = [];
|
|
692
|
+
// ... test implementation
|
|
693
|
+
return JSON.stringify(results);
|
|
694
|
+
|
|
695
|
+
- name: create-test-report
|
|
696
|
+
type: integration
|
|
697
|
+
integration: github
|
|
698
|
+
action: create-issue
|
|
699
|
+
params:
|
|
700
|
+
title: 'Smoke Test Results'
|
|
701
|
+
body: |
|
|
702
|
+
Deployment URL: {{steps.test-deployment.output}}
|
|
703
|
+
Test Results: {{steps.run-smoke-tests.output}}
|
|
704
|
+
```
|
|
705
|
+
|
|
706
|
+
### With Slack Integration
|
|
707
|
+
|
|
708
|
+
```yaml
|
|
709
|
+
steps:
|
|
710
|
+
- name: monitor-checkout-flow
|
|
711
|
+
type: integration
|
|
712
|
+
integration: browser
|
|
713
|
+
action: navigate
|
|
714
|
+
params:
|
|
715
|
+
url: 'https://store.example.com/checkout'
|
|
716
|
+
|
|
717
|
+
- name: capture-checkout-error
|
|
718
|
+
type: integration
|
|
719
|
+
integration: browser
|
|
720
|
+
action: screenshot
|
|
721
|
+
params:
|
|
722
|
+
path: 'checkout-error.png'
|
|
723
|
+
selector: '.error-message'
|
|
724
|
+
|
|
725
|
+
- name: alert-team
|
|
726
|
+
type: integration
|
|
727
|
+
integration: slack
|
|
728
|
+
action: post-message
|
|
729
|
+
params:
|
|
730
|
+
channel: '#alerts'
|
|
731
|
+
text: |
|
|
732
|
+
🚨 Checkout flow error detected
|
|
733
|
+
Screenshot: {{steps.capture-checkout-error.output}}
|
|
734
|
+
```
|
|
735
|
+
|
|
736
|
+
## Security Considerations
|
|
737
|
+
|
|
738
|
+
1. **Sandboxing**: Browser instances run in isolated containers
|
|
739
|
+
2. **URL Validation**: Configurable allowlist/denylist for navigation targets
|
|
740
|
+
3. **File Access**: Upload/download operations respect workflow file permissions
|
|
741
|
+
4. **Credential Management**: Secure handling of authentication data
|
|
742
|
+
5. **Network Isolation**: Optional network access restrictions
|
|
743
|
+
|
|
744
|
+
## Implementation Priorities
|
|
745
|
+
|
|
746
|
+
### Phase 1: Core Actions (Week 1-2)
|
|
747
|
+
|
|
748
|
+
- [ ] Basic navigation (navigate, reload, back, forward)
|
|
749
|
+
- [ ] Element interaction (click, fill, submit)
|
|
750
|
+
- [ ] Content extraction (getText, getHTML)
|
|
751
|
+
- [ ] Screenshot capture
|
|
752
|
+
- [ ] Session management
|
|
753
|
+
|
|
754
|
+
### Phase 2: Advanced Features (Week 3-4)
|
|
755
|
+
|
|
756
|
+
- [ ] JavaScript execution (evaluate, addScript)
|
|
757
|
+
- [ ] Iframe handling
|
|
758
|
+
- [ ] File upload/download
|
|
759
|
+
- [ ] Cookie and header management
|
|
760
|
+
- [ ] Network request logging
|
|
761
|
+
|
|
762
|
+
### Phase 3: Workflow Integration (Week 5-6)
|
|
763
|
+
|
|
764
|
+
- [ ] Executor implementation
|
|
765
|
+
- [ ] Error handling and recovery
|
|
766
|
+
- [ ] Output chaining support
|
|
767
|
+
- [ ] Template workflows
|
|
768
|
+
- [ ] Documentation and examples
|
|
769
|
+
|
|
770
|
+
### Phase 4: Production Readiness (Week 7-8)
|
|
771
|
+
|
|
772
|
+
- [ ] Comprehensive test suite
|
|
773
|
+
- [ ] Performance optimization
|
|
774
|
+
- [ ] Security hardening
|
|
775
|
+
- [ ] Monitoring and observability
|
|
776
|
+
- [ ] CI/CD integration
|
|
777
|
+
|
|
778
|
+
This design provides a comprehensive browser automation primitive that integrates seamlessly with the existing relay workflow system while offering powerful capabilities for web interaction, testing, and data extraction workflows.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { WorkflowRunner, type RelayYamlConfig } from '@agent-relay/sdk/workflows';
|
|
2
|
+
|
|
3
|
+
import { BrowserStepExecutor, createBrowserStep } from '../src/workflow-step.js';
|
|
4
|
+
|
|
5
|
+
const browserExecutor = new BrowserStepExecutor();
|
|
6
|
+
|
|
7
|
+
const config: RelayYamlConfig = {
|
|
8
|
+
version: '1.0',
|
|
9
|
+
name: 'browser-primitive-workflow',
|
|
10
|
+
description: 'Browser primitive workflow with chained actions and captured output.',
|
|
11
|
+
swarm: {
|
|
12
|
+
pattern: 'pipeline',
|
|
13
|
+
},
|
|
14
|
+
agents: [],
|
|
15
|
+
workflows: [
|
|
16
|
+
{
|
|
17
|
+
name: 'browser-primitive-workflow',
|
|
18
|
+
steps: [
|
|
19
|
+
createBrowserStep({
|
|
20
|
+
name: 'inspect-example-page',
|
|
21
|
+
sessionId: 'example-page-session',
|
|
22
|
+
config: {
|
|
23
|
+
browser: 'chromium',
|
|
24
|
+
headless: true,
|
|
25
|
+
viewport: { width: 1280, height: 720 },
|
|
26
|
+
captureConsole: true,
|
|
27
|
+
persistSession: true,
|
|
28
|
+
},
|
|
29
|
+
actions: [
|
|
30
|
+
{
|
|
31
|
+
action: 'goto',
|
|
32
|
+
params: {
|
|
33
|
+
url: 'https://example.com',
|
|
34
|
+
waitUntil: 'domcontentloaded',
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
action: 'text',
|
|
39
|
+
id: 'heading',
|
|
40
|
+
params: {
|
|
41
|
+
selector: 'h1',
|
|
42
|
+
innerText: true,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
output: {
|
|
47
|
+
mode: 'last',
|
|
48
|
+
format: 'text',
|
|
49
|
+
},
|
|
50
|
+
}),
|
|
51
|
+
createBrowserStep({
|
|
52
|
+
name: 'use-captured-heading',
|
|
53
|
+
dependsOn: ['inspect-example-page'],
|
|
54
|
+
sessionId: 'example-page-session',
|
|
55
|
+
config: {
|
|
56
|
+
browser: 'chromium',
|
|
57
|
+
headless: true,
|
|
58
|
+
persistSession: true,
|
|
59
|
+
},
|
|
60
|
+
actions: [
|
|
61
|
+
{
|
|
62
|
+
action: 'evaluate',
|
|
63
|
+
params: {
|
|
64
|
+
script: '() => `Current title from the persisted session: ${document.title}`',
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
],
|
|
68
|
+
output: {
|
|
69
|
+
mode: 'last',
|
|
70
|
+
format: 'text',
|
|
71
|
+
},
|
|
72
|
+
}),
|
|
73
|
+
createBrowserStep({
|
|
74
|
+
name: 'capture-page-report',
|
|
75
|
+
dependsOn: ['use-captured-heading'],
|
|
76
|
+
sessionId: 'example-page-session',
|
|
77
|
+
config: {
|
|
78
|
+
browser: 'chromium',
|
|
79
|
+
headless: true,
|
|
80
|
+
persistSession: true,
|
|
81
|
+
},
|
|
82
|
+
actions: [
|
|
83
|
+
{
|
|
84
|
+
action: 'evaluate',
|
|
85
|
+
id: 'pageFacts',
|
|
86
|
+
outputKey: 'pageFacts',
|
|
87
|
+
capture: true,
|
|
88
|
+
params: {
|
|
89
|
+
script: '() => ({ title: document.title, links: document.links.length })',
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
action: 'screenshot',
|
|
94
|
+
id: 'screenshot',
|
|
95
|
+
outputKey: 'screenshot',
|
|
96
|
+
capture: true,
|
|
97
|
+
params: {
|
|
98
|
+
path: 'artifacts/example-page.png',
|
|
99
|
+
fullPage: true,
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
],
|
|
103
|
+
output: {
|
|
104
|
+
mode: 'captures',
|
|
105
|
+
includeMetadata: true,
|
|
106
|
+
includeSession: true,
|
|
107
|
+
pretty: true,
|
|
108
|
+
},
|
|
109
|
+
closeSession: true,
|
|
110
|
+
}),
|
|
111
|
+
],
|
|
112
|
+
},
|
|
113
|
+
],
|
|
114
|
+
errorHandling: {
|
|
115
|
+
strategy: 'fail-fast',
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
async function main(): Promise<void> {
|
|
120
|
+
const runner = new WorkflowRunner({
|
|
121
|
+
cwd: process.cwd(),
|
|
122
|
+
executor: browserExecutor,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const result = await runner.execute(config);
|
|
126
|
+
console.log(`Browser workflow completed: ${result.status}`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
|
130
|
+
main()
|
|
131
|
+
.catch((error) => {
|
|
132
|
+
console.error(error instanceof Error ? error.stack : error);
|
|
133
|
+
process.exitCode = 1;
|
|
134
|
+
})
|
|
135
|
+
.finally(async () => {
|
|
136
|
+
await browserExecutor.closeAll();
|
|
137
|
+
});
|
|
138
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@agent-relay/browser-primitive",
|
|
3
|
+
"version": "4.0.9",
|
|
4
|
+
"description": "Browser automation workflow primitive for Agent Relay",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./workflow-step": {
|
|
15
|
+
"types": "./dist/workflow-step.d.ts",
|
|
16
|
+
"import": "./dist/workflow-step.js",
|
|
17
|
+
"default": "./dist/workflow-step.js"
|
|
18
|
+
},
|
|
19
|
+
"./mcp-server": {
|
|
20
|
+
"types": "./dist/mcp-server.d.ts",
|
|
21
|
+
"import": "./dist/mcp-server.js",
|
|
22
|
+
"default": "./dist/mcp-server.js"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"bin": {
|
|
26
|
+
"agent-relay-browser-mcp": "./dist/mcp-server.js"
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"dist",
|
|
30
|
+
"examples",
|
|
31
|
+
"DESIGN.md",
|
|
32
|
+
"README.md"
|
|
33
|
+
],
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsc",
|
|
36
|
+
"clean": "rm -rf dist",
|
|
37
|
+
"test": "vitest run",
|
|
38
|
+
"test:watch": "vitest"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"@agent-relay/sdk": "4.0.9",
|
|
42
|
+
"playwright": "^1.51.1"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@types/node": "^22.19.3",
|
|
46
|
+
"typescript": "^5.9.3",
|
|
47
|
+
"vitest": "^3.2.4"
|
|
48
|
+
},
|
|
49
|
+
"publishConfig": {
|
|
50
|
+
"access": "public"
|
|
51
|
+
},
|
|
52
|
+
"repository": {
|
|
53
|
+
"type": "git",
|
|
54
|
+
"url": "git+https://github.com/AgentWorkforce/relay.git",
|
|
55
|
+
"directory": "packages/browser-primitive"
|
|
56
|
+
}
|
|
57
|
+
}
|