opencode-browser-kit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +95 -2
- package/dist/plugin.js +12805 -0
- package/package.json +28 -4
- package/src/plugin.ts +548 -0
package/package.json
CHANGED
|
@@ -1,6 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-browser-kit",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Headless browser + visual element picker for OpenCode. Tools: browser_navigate, browser_snapshop, browser_click, browser_type, browser_screenshot, browser_close and browser_inspect (opens a visible browser so you click an element and type what to change).",
|
|
5
|
+
"main": "dist/plugin.js",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"files": [
|
|
8
|
+
"dist",
|
|
9
|
+
"src",
|
|
10
|
+
"README.md"
|
|
11
|
+
],
|
|
12
|
+
"scripts": {
|
|
13
|
+
"build": "bun build src/plugin.ts --outdir dist --target node --external playwright",
|
|
14
|
+
"prepublishOnly": "bun run build"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"opencode",
|
|
18
|
+
"opencode-plugin",
|
|
19
|
+
"browser",
|
|
20
|
+
"playwright",
|
|
21
|
+
"automation",
|
|
22
|
+
"screenshots",
|
|
23
|
+
"element-picker"
|
|
24
|
+
],
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@opencode-ai/plugin": "^1.19.0",
|
|
28
|
+
"playwright": "^1.64.0"
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/plugin.ts
ADDED
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
import { type Plugin, tool } from "@opencode-ai/plugin"
|
|
2
|
+
import type { Browser, BrowserContext, Page } from "playwright"
|
|
3
|
+
import fs from "fs"
|
|
4
|
+
import os from "os"
|
|
5
|
+
import path from "path"
|
|
6
|
+
import { spawn } from "node:child_process"
|
|
7
|
+
import { createRequire } from "node:module"
|
|
8
|
+
import { fileURLToPath } from "url"
|
|
9
|
+
|
|
10
|
+
type PlaywrightModule = typeof import("playwright")
|
|
11
|
+
|
|
12
|
+
function pluginRoot(): string {
|
|
13
|
+
// dist/plugin.js -> package root; src/plugin.ts -> package root too
|
|
14
|
+
const here = path.dirname(fileURLToPath(import.meta.url))
|
|
15
|
+
return path.basename(here) === "dist" ? path.join(here, "..") : here
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let playwrightModule: PlaywrightModule | null = null
|
|
19
|
+
|
|
20
|
+
function hasPlaywrightOnDisk(): boolean {
|
|
21
|
+
const root = pluginRoot()
|
|
22
|
+
return (
|
|
23
|
+
fs.existsSync(path.join(root, "node_modules", "playwright", "package.json")) ||
|
|
24
|
+
fs.existsSync(path.join(root, "..", "node_modules", "playwright", "package.json"))
|
|
25
|
+
)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function selfHealInstall(): Promise<void> {
|
|
29
|
+
const root = pluginRoot()
|
|
30
|
+
const cacheDir = path.join(os.tmpdir(), "opencode-bun-cache")
|
|
31
|
+
fs.mkdirSync(cacheDir, { recursive: true })
|
|
32
|
+
const envPatch = { BUN_INSTALL_CACHE_DIR: cacheDir, CI: "1" } as Record<string, string>
|
|
33
|
+
const install = (command: string, ms: number) =>
|
|
34
|
+
new Promise<boolean>((resolve) => {
|
|
35
|
+
const child = spawn(command, { cwd: root, shell: true, stdio: "ignore", env: { ...process.env, ...envPatch } })
|
|
36
|
+
let done = false
|
|
37
|
+
const finish = (ok: boolean) => {
|
|
38
|
+
if (done) return
|
|
39
|
+
done = true
|
|
40
|
+
clearTimeout(timer)
|
|
41
|
+
resolve(ok)
|
|
42
|
+
}
|
|
43
|
+
const timer = setTimeout(() => {
|
|
44
|
+
try { child.kill("SIGKILL") } catch {}
|
|
45
|
+
finish(false)
|
|
46
|
+
}, ms)
|
|
47
|
+
child.on("exit", (code) => finish(code === 0))
|
|
48
|
+
child.on("error", () => finish(false))
|
|
49
|
+
})
|
|
50
|
+
if (!hasPlaywrightOnDisk()) await install(`bun add playwright@^1.64.0`, 45000)
|
|
51
|
+
if (!hasPlaywrightOnDisk())
|
|
52
|
+
await install(`npm install playwright@^1.64.0 --no-audit --no-fund --loglevel=error`, 120000)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function loadPlaywright(): Promise<PlaywrightModule> {
|
|
56
|
+
if (playwrightModule) return playwrightModule
|
|
57
|
+
// IMPORTANT: never `import("playwright")` speculatively. If the package is
|
|
58
|
+
// missing, bun's runtime auto-install kicks in and can hang for minutes on
|
|
59
|
+
// slow package caches; and after a fresh install, bun's async ESM loader
|
|
60
|
+
// can stall on the new files. require() via createRequire is instant and
|
|
61
|
+
// deterministic, resolved from this plugin's own directory.
|
|
62
|
+
if (!hasPlaywrightOnDisk()) await selfHealInstall()
|
|
63
|
+
if (!hasPlaywrightOnDisk()) {
|
|
64
|
+
throw new Error(
|
|
65
|
+
"opencode-browser-kit could not install playwright. Run manually in the plugin's folder: `bun add playwright && bunx playwright install chromium` (or use system Chrome/Edge).",
|
|
66
|
+
)
|
|
67
|
+
}
|
|
68
|
+
playwrightModule = createRequire(import.meta.url)("playwright") as PlaywrightModule
|
|
69
|
+
return playwrightModule
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let browser: Browser | null = null
|
|
73
|
+
let context: BrowserContext | null = null
|
|
74
|
+
let page: Page | null = null
|
|
75
|
+
|
|
76
|
+
const SCREENSHOT_DIR = path.join(os.tmpdir(), "opencode-browser-screenshots")
|
|
77
|
+
const VIEWPORT = { width: 1280, height: 800 }
|
|
78
|
+
|
|
79
|
+
async function launchBrowser(headless = true): Promise<Browser> {
|
|
80
|
+
const { chromium } = await loadPlaywright()
|
|
81
|
+
const attempts: { channel?: string }[] = [
|
|
82
|
+
{},
|
|
83
|
+
{ channel: "chrome" },
|
|
84
|
+
{ channel: "msedge" },
|
|
85
|
+
]
|
|
86
|
+
let lastError: unknown
|
|
87
|
+
for (const opts of attempts) {
|
|
88
|
+
try {
|
|
89
|
+
return await chromium.launch({ headless, ...opts })
|
|
90
|
+
} catch (e) {
|
|
91
|
+
lastError = e
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
throw new Error(
|
|
95
|
+
`Could not launch any Chromium. Tried default install and system Chrome/Edge. ${String(lastError)}`,
|
|
96
|
+
)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function ensurePage(): Promise<Page> {
|
|
100
|
+
if (!browser || !browser.isConnected()) {
|
|
101
|
+
browser = await launchBrowser()
|
|
102
|
+
context = await browser.newContext({
|
|
103
|
+
viewport: VIEWPORT,
|
|
104
|
+
deviceScaleFactor: 2,
|
|
105
|
+
})
|
|
106
|
+
context.setDefaultTimeout(20000)
|
|
107
|
+
page = await context.newPage()
|
|
108
|
+
}
|
|
109
|
+
if (!context || !page || page.isClosed()) {
|
|
110
|
+
page = await context!.newPage()
|
|
111
|
+
}
|
|
112
|
+
return page!
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function resolveSelector(target: string): string {
|
|
116
|
+
const t = target.trim()
|
|
117
|
+
if (
|
|
118
|
+
/^(text=|css=|xpath=|role=|label=|id=|\/{1,2}|\.\.)/i.test(t) ||
|
|
119
|
+
/^[.#*[\]]/.test(t)
|
|
120
|
+
) {
|
|
121
|
+
return t
|
|
122
|
+
}
|
|
123
|
+
const looksCss =
|
|
124
|
+
/^(html|head|body|div|span|a|button|input|textarea|select|form|img|svg|main|nav|header|footer|section|article|ul|ol|li|table|tr|td|th|h[1-6]|p|label|iframe|dialog)([.#\[:\s>]|$)/i.test(
|
|
125
|
+
t,
|
|
126
|
+
) || /^[\w-]+$/i.test(t)
|
|
127
|
+
if (looksCss) return t
|
|
128
|
+
return `text=${t}`
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function saveScreenshot(
|
|
132
|
+
p: Page,
|
|
133
|
+
label: string,
|
|
134
|
+
fullPage = false,
|
|
135
|
+
): Promise<string> {
|
|
136
|
+
fs.mkdirSync(SCREENSHOT_DIR, { recursive: true })
|
|
137
|
+
const file = path.join(SCREENSHOT_DIR, `${label}-${Date.now()}.png`)
|
|
138
|
+
await p.screenshot({ path: file, fullPage })
|
|
139
|
+
return file
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function formatSnap(file: string): string {
|
|
143
|
+
return `\n\nScreenshot saved: ${file}\n(Use the read tool on that path to view it.)`
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const PICKER_JS = `(function () {
|
|
147
|
+
if (window.__ocPickerActive) return
|
|
148
|
+
window.__ocPickerActive = true
|
|
149
|
+
window.__ocPicked = undefined
|
|
150
|
+
var CSS_TXT =
|
|
151
|
+
'.oc-ph { outline: 2px solid #e11d48 !important; outline-offset: -2px !important; cursor: crosshair !important; }' +
|
|
152
|
+
'.oc-panel { position: fixed; z-index: 2147483647; right: 16px; top: 16px; width: 340px; box-sizing: border-box; font: 13px/1.45 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #fff; color: #111; border-radius: 10px; box-shadow: 0 8px 30px rgba(0,0,0,.28); padding: 12px 14px; }' +
|
|
153
|
+
'.oc-panel .oc-tag { font-weight: 600; font-size: 13px; }' +
|
|
154
|
+
'.oc-panel .oc-sel { color: #888; font: 11px ui-monospace, SFMono-Regular, Menlo, monospace; word-break: break-all; margin-top: 2px; }' +
|
|
155
|
+
'.oc-panel .oc-txt { color: #444; font-size: 12px; margin-top: 4px; max-height: 48px; overflow: hidden; }' +
|
|
156
|
+
'.oc-panel textarea { width: 100%; height: 70px; box-sizing: border-box; margin: 10px 0 8px; padding: 8px; border: 1px solid #ddd; border-radius: 6px; font: inherit; resize: vertical; }' +
|
|
157
|
+
'.oc-btns { display: flex; gap: 8px; }' +
|
|
158
|
+
'.oc-btns button { flex: 1; padding: 7px 10px; border-radius: 6px; border: 1px solid #ddd; background: #fff; color: #111; font: inherit; cursor: pointer; }' +
|
|
159
|
+
'.oc-btns button[data-a="apply"] { background: #111; color: #fff; border-color: #111; }'
|
|
160
|
+
var style = document.createElement('style')
|
|
161
|
+
style.textContent = CSS_TXT
|
|
162
|
+
style.setAttribute('data-oc', '1')
|
|
163
|
+
document.head.appendChild(style)
|
|
164
|
+
|
|
165
|
+
var banner = document.createElement('div')
|
|
166
|
+
banner.className = 'oc-panel'
|
|
167
|
+
banner.textContent = 'Click any element to select it · Esc to cancel'
|
|
168
|
+
document.documentElement.appendChild(banner)
|
|
169
|
+
|
|
170
|
+
var picking = true
|
|
171
|
+
var last = null
|
|
172
|
+
|
|
173
|
+
function hover(e) {
|
|
174
|
+
if (!picking) return
|
|
175
|
+
if (last) last.classList.remove('oc-ph')
|
|
176
|
+
last = null
|
|
177
|
+
var el = document.elementFromPoint(e.clientX, e.clientY)
|
|
178
|
+
if (el && el !== document.documentElement && el !== document.body && !(el.closest && el.closest('.oc-panel'))) {
|
|
179
|
+
last = el
|
|
180
|
+
el.classList.add('oc-ph')
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function selFor(el) {
|
|
185
|
+
if (el.dataset && el.dataset.testid) return '[data-testid="' + el.dataset.testid + '"]'
|
|
186
|
+
var parts = []
|
|
187
|
+
var node = el
|
|
188
|
+
while (node && node.nodeType === 1 && node !== document.body) {
|
|
189
|
+
var p = node.nodeName.toLowerCase()
|
|
190
|
+
if (node.id) { parts.unshift('#' + CSS.escape(node.id)); break }
|
|
191
|
+
var cl = Array.prototype.filter
|
|
192
|
+
.call(node.classList, function (c) { return !c.startsWith('oc-') })
|
|
193
|
+
.slice(0, 2)
|
|
194
|
+
.map(function (c) { return '.' + CSS.escape(c) })
|
|
195
|
+
.join('')
|
|
196
|
+
p += cl
|
|
197
|
+
if (node.parentElement) {
|
|
198
|
+
var sib = Array.prototype.filter.call(node.parentElement.children, function (c) { return c.nodeName === node.nodeName })
|
|
199
|
+
if (sib.length > 1) p += ':nth-of-type(' + (sib.indexOf(node) + 1) + ')'
|
|
200
|
+
}
|
|
201
|
+
parts.unshift(p)
|
|
202
|
+
node = node.parentElement
|
|
203
|
+
}
|
|
204
|
+
return parts.join(' > ') || 'body'
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function reactInfo(el) {
|
|
208
|
+
try {
|
|
209
|
+
for (var k in el) {
|
|
210
|
+
if (k.indexOf('__reactFiber$') === 0) {
|
|
211
|
+
var f = el[k]
|
|
212
|
+
var name = null
|
|
213
|
+
while (f) {
|
|
214
|
+
if (f._debugSource) {
|
|
215
|
+
return { file: f._debugSource.fileName, line: f._debugSource.lineNumber, comp: name || '' }
|
|
216
|
+
}
|
|
217
|
+
var t = f.type || f.elementType
|
|
218
|
+
if (typeof t === 'function' && !name) name = t.displayName || t.name
|
|
219
|
+
f = f.return
|
|
220
|
+
}
|
|
221
|
+
return { comp: name || '' }
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
} catch (err) {}
|
|
225
|
+
return null
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function collect(el) {
|
|
229
|
+
var cs = getComputedStyle(el)
|
|
230
|
+
var r = el.getBoundingClientRect()
|
|
231
|
+
return {
|
|
232
|
+
selector: selFor(el),
|
|
233
|
+
tag: el.tagName.toLowerCase(),
|
|
234
|
+
id: el.id || '',
|
|
235
|
+
classes: Array.prototype.filter.call(el.classList, function (c) { return !c.startsWith('oc-') }).join(' '),
|
|
236
|
+
text: (el.innerText || '')
|
|
237
|
+
.split('\\n')
|
|
238
|
+
.map(function (t) { return t.trim() })
|
|
239
|
+
.filter(Boolean)
|
|
240
|
+
.slice(0, 3)
|
|
241
|
+
.join(' | ')
|
|
242
|
+
.slice(0, 150),
|
|
243
|
+
html: el.outerHTML.slice(0, 400),
|
|
244
|
+
rect: { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) },
|
|
245
|
+
styles: {
|
|
246
|
+
fontSize: cs.fontSize,
|
|
247
|
+
fontWeight: cs.fontWeight,
|
|
248
|
+
color: cs.color,
|
|
249
|
+
backgroundColor: cs.backgroundColor,
|
|
250
|
+
borderRadius: cs.borderRadius,
|
|
251
|
+
fontFamily: cs.fontFamily,
|
|
252
|
+
},
|
|
253
|
+
react: reactInfo(el),
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function mk(a, label) {
|
|
258
|
+
var b = document.createElement('button')
|
|
259
|
+
b.setAttribute('data-a', a)
|
|
260
|
+
b.textContent = label
|
|
261
|
+
return b
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function openPanel(el, info) {
|
|
265
|
+
if (last) last.classList.remove('oc-ph')
|
|
266
|
+
last = el
|
|
267
|
+
el.classList.add('oc-ph')
|
|
268
|
+
banner.remove()
|
|
269
|
+
var panel = document.createElement('div')
|
|
270
|
+
panel.className = 'oc-panel'
|
|
271
|
+
var tag = document.createElement('div')
|
|
272
|
+
tag.className = 'oc-tag'
|
|
273
|
+
tag.textContent = '<' + info.tag + '>' + (info.id ? ' #' + info.id : info.classes ? ' ' + info.classes : '')
|
|
274
|
+
var path = document.createElement('div')
|
|
275
|
+
path.className = 'oc-sel'
|
|
276
|
+
path.textContent = info.selector
|
|
277
|
+
var txt = document.createElement('div')
|
|
278
|
+
txt.className = 'oc-txt'
|
|
279
|
+
txt.textContent = info.text || ''
|
|
280
|
+
var ta = document.createElement('textarea')
|
|
281
|
+
ta.placeholder = 'Describe what you want changed…'
|
|
282
|
+
ta.setAttribute('data-oc-ta', '1')
|
|
283
|
+
var btns = document.createElement('div')
|
|
284
|
+
btns.className = 'oc-btns'
|
|
285
|
+
btns.append(mk('apply', 'Apply'), mk('reselect', 'Reselect'), mk('cancel', 'Cancel'))
|
|
286
|
+
panel.append(tag, path, txt, ta, btns)
|
|
287
|
+
document.documentElement.appendChild(panel)
|
|
288
|
+
ta.focus()
|
|
289
|
+
panel.addEventListener('mousedown', function (e) { e.stopPropagation() })
|
|
290
|
+
panel.addEventListener('click', function (e) {
|
|
291
|
+
var a = e.target.getAttribute && e.target.getAttribute('data-a')
|
|
292
|
+
if (a === 'apply') finish(Object.assign({}, info, { note: ta.value.trim() }))
|
|
293
|
+
else if (a === 'cancel') finish(null)
|
|
294
|
+
else if (a === 'reselect') {
|
|
295
|
+
panel.remove()
|
|
296
|
+
picking = true
|
|
297
|
+
document.documentElement.appendChild(banner)
|
|
298
|
+
document.addEventListener('mousemove', hover, true)
|
|
299
|
+
}
|
|
300
|
+
})
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function down(e) {
|
|
304
|
+
if (!picking || e.button !== 0) return
|
|
305
|
+
var el = document.elementFromPoint(e.clientX, e.clientY)
|
|
306
|
+
if (!el || (el.closest && el.closest('.oc-panel'))) return
|
|
307
|
+
e.preventDefault()
|
|
308
|
+
e.stopPropagation()
|
|
309
|
+
picking = false
|
|
310
|
+
document.removeEventListener('mousemove', hover, true)
|
|
311
|
+
openPanel(el, collect(el))
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function esc(e) {
|
|
315
|
+
if (e.key === 'Escape') { e.preventDefault(); finish(null) }
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function blockClick(e) {
|
|
319
|
+
if (e.target.closest && e.target.closest('.oc-panel')) return
|
|
320
|
+
e.preventDefault()
|
|
321
|
+
e.stopPropagation()
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function finish(val) { cleanup(); window.__ocPicked = val }
|
|
325
|
+
|
|
326
|
+
function cleanup() {
|
|
327
|
+
picking = false
|
|
328
|
+
document.removeEventListener('mousemove', hover, true)
|
|
329
|
+
document.removeEventListener('mousedown', down, true)
|
|
330
|
+
document.removeEventListener('keydown', esc, true)
|
|
331
|
+
document.removeEventListener('click', blockClick, true)
|
|
332
|
+
Array.prototype.forEach.call(document.querySelectorAll('.oc-ph'), function (n) { n.classList.remove('oc-ph') })
|
|
333
|
+
Array.prototype.forEach.call(document.querySelectorAll('.oc-panel'), function (n) { n.remove() })
|
|
334
|
+
var st = document.querySelector('style[data-oc]')
|
|
335
|
+
if (st) st.remove()
|
|
336
|
+
window.__ocPickerActive = false
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
document.addEventListener('mousemove', hover, true)
|
|
340
|
+
document.addEventListener('mousedown', down, true)
|
|
341
|
+
document.addEventListener('keydown', esc, true)
|
|
342
|
+
document.addEventListener('click', blockClick, true)
|
|
343
|
+
})()`
|
|
344
|
+
|
|
345
|
+
export const BrowserPlugin: Plugin = async (ctx) => {
|
|
346
|
+
return {
|
|
347
|
+
tool: {
|
|
348
|
+
browser_navigate: tool({
|
|
349
|
+
description:
|
|
350
|
+
"Navigate the embedded headless browser to a URL. Returns page title plus a screenshot path you can view with the read tool. The browser session persists across tool calls.",
|
|
351
|
+
args: {
|
|
352
|
+
url: tool.schema
|
|
353
|
+
.string()
|
|
354
|
+
.describe("Full URL, e.g. https://example.com or http://localhost:3000"),
|
|
355
|
+
},
|
|
356
|
+
async execute(args) {
|
|
357
|
+
const p = await ensurePage()
|
|
358
|
+
try {
|
|
359
|
+
await p.goto(args.url, { waitUntil: "networkidle", timeout: 20000 })
|
|
360
|
+
} catch {
|
|
361
|
+
await p.goto(args.url, { waitUntil: "domcontentloaded", timeout: 20000 })
|
|
362
|
+
}
|
|
363
|
+
const title = await p.title()
|
|
364
|
+
const file = await saveScreenshot(p, "navigate")
|
|
365
|
+
return `Navigated to ${p.url()}\nTitle: ${title}${formatSnap(file)}`
|
|
366
|
+
},
|
|
367
|
+
}),
|
|
368
|
+
|
|
369
|
+
browser_snapshot: tool({
|
|
370
|
+
description:
|
|
371
|
+
"Read the currently open page's content as text (title, url, visible text). Optionally scope to a CSS/text selector.",
|
|
372
|
+
args: {
|
|
373
|
+
selector: tool.schema
|
|
374
|
+
.string()
|
|
375
|
+
.optional()
|
|
376
|
+
.describe(
|
|
377
|
+
"Optional CSS or text= selector to scope the snapshot to a region, e.g. main or text=Pricing",
|
|
378
|
+
),
|
|
379
|
+
maxLength: tool.schema
|
|
380
|
+
.number()
|
|
381
|
+
.optional()
|
|
382
|
+
.describe("Max characters of text to return (default 8000)"),
|
|
383
|
+
},
|
|
384
|
+
async execute(args) {
|
|
385
|
+
const p = await ensurePage()
|
|
386
|
+
const scope = args.selector ? resolveSelector(args.selector) : "body"
|
|
387
|
+
const text = await p.evaluate(
|
|
388
|
+
({ sel }) => {
|
|
389
|
+
const el = document.querySelector(sel)
|
|
390
|
+
if (!el) return null
|
|
391
|
+
return (el as HTMLElement).innerText
|
|
392
|
+
},
|
|
393
|
+
{ sel: scope },
|
|
394
|
+
)
|
|
395
|
+
if (text === null) {
|
|
396
|
+
return `No element matched "${scope}" on ${p.url()}`
|
|
397
|
+
}
|
|
398
|
+
const max = args.maxLength ?? 8000
|
|
399
|
+
const trimmed =
|
|
400
|
+
text.length > max ? text.slice(0, max) + "\n…[truncated]" : text
|
|
401
|
+
return `URL: ${p.url()}\nTitle: ${await p.title()}\n\n---\n${trimmed}`
|
|
402
|
+
},
|
|
403
|
+
}),
|
|
404
|
+
|
|
405
|
+
browser_click: tool({
|
|
406
|
+
description:
|
|
407
|
+
"Click an element in the browser. Accepts a CSS selector (#id, .class, button), xpath (//button[2]), or plain text (wrapped as text= automatically). A screenshot is saved after clicking.",
|
|
408
|
+
args: {
|
|
409
|
+
target: tool.schema
|
|
410
|
+
.string()
|
|
411
|
+
.describe('e.g. "#submit", "a", //button[2], "Sign in"'),
|
|
412
|
+
},
|
|
413
|
+
async execute(args) {
|
|
414
|
+
const p = await ensurePage()
|
|
415
|
+
const selector = resolveSelector(args.target)
|
|
416
|
+
await p.click(selector, { timeout: 15000 })
|
|
417
|
+
await p.waitForLoadState("domcontentloaded").catch(() => {})
|
|
418
|
+
const file = await saveScreenshot(p, "click")
|
|
419
|
+
return `Clicked "${selector}" on ${p.url()}${formatSnap(file)}`
|
|
420
|
+
},
|
|
421
|
+
}),
|
|
422
|
+
|
|
423
|
+
browser_type: tool({
|
|
424
|
+
description:
|
|
425
|
+
"Type text into a form field. Clears existing value, types the text, and optionally presses Enter to submit.",
|
|
426
|
+
args: {
|
|
427
|
+
target: tool.schema
|
|
428
|
+
.string()
|
|
429
|
+
.describe('Field selector, e.g. "input[name=q]" or "Search"'),
|
|
430
|
+
text: tool.schema.string().describe("Text to type"),
|
|
431
|
+
submit: tool.schema
|
|
432
|
+
.boolean()
|
|
433
|
+
.optional()
|
|
434
|
+
.describe("Press Enter after typing (default false)"),
|
|
435
|
+
},
|
|
436
|
+
async execute(args) {
|
|
437
|
+
const p = await ensurePage()
|
|
438
|
+
const selector = resolveSelector(args.target)
|
|
439
|
+
await p.fill(selector, args.text, { timeout: 15000 })
|
|
440
|
+
if (args.submit) {
|
|
441
|
+
await p.press(selector, "Enter")
|
|
442
|
+
await p.waitForLoadState("domcontentloaded").catch(() => {})
|
|
443
|
+
}
|
|
444
|
+
const file = await saveScreenshot(p, "type")
|
|
445
|
+
return `Typed ${JSON.stringify(args.text)} into "${selector}" (submit=${!!args.submit})${formatSnap(file)}`
|
|
446
|
+
},
|
|
447
|
+
}),
|
|
448
|
+
|
|
449
|
+
browser_screenshot: tool({
|
|
450
|
+
description:
|
|
451
|
+
"Take a screenshot of the current browser page. Returns the file path; use the read tool to view it.",
|
|
452
|
+
args: {
|
|
453
|
+
fullPage: tool.schema
|
|
454
|
+
.boolean()
|
|
455
|
+
.optional()
|
|
456
|
+
.describe("Capture the full scrollable page (default: viewport only)"),
|
|
457
|
+
},
|
|
458
|
+
async execute(args) {
|
|
459
|
+
const p = await ensurePage()
|
|
460
|
+
const file = await saveScreenshot(p, "shot", !!args.fullPage)
|
|
461
|
+
return `Screenshot of ${p.url()} saved: ${file}`
|
|
462
|
+
},
|
|
463
|
+
}),
|
|
464
|
+
|
|
465
|
+
browser_close: tool({
|
|
466
|
+
description: "Close the embedded browser and release its resources.",
|
|
467
|
+
args: {},
|
|
468
|
+
async execute() {
|
|
469
|
+
if (browser) {
|
|
470
|
+
await browser.close().catch(() => {})
|
|
471
|
+
browser = null
|
|
472
|
+
context = null
|
|
473
|
+
page = null
|
|
474
|
+
}
|
|
475
|
+
return "Browser closed."
|
|
476
|
+
},
|
|
477
|
+
}),
|
|
478
|
+
|
|
479
|
+
browser_inspect: tool({
|
|
480
|
+
description:
|
|
481
|
+
"Interactive element picker for design changes. Opens a VISIBLE browser window on the URL, waits for the user to hover and CLICK an element, then the user TYPES what they want changed and clicks Apply. Returns the element's unique selector, text, HTML, styles, React component source (when available) and the user's change note. Use whenever the user wants to point at something on the page to modify it.",
|
|
482
|
+
args: {
|
|
483
|
+
url: tool.schema
|
|
484
|
+
.string()
|
|
485
|
+
.describe("URL to open, e.g. http://localhost:3000"),
|
|
486
|
+
timeoutSeconds: tool.schema
|
|
487
|
+
.number()
|
|
488
|
+
.optional()
|
|
489
|
+
.describe("How long to wait for the user's selection (default 300)"),
|
|
490
|
+
},
|
|
491
|
+
async execute(args) {
|
|
492
|
+
let inspectBrowser: Browser | null = null
|
|
493
|
+
try {
|
|
494
|
+
const { chromium } = await loadPlaywright()
|
|
495
|
+
inspectBrowser = await launchBrowser(false)
|
|
496
|
+
const pageCtx = await inspectBrowser.newContext({
|
|
497
|
+
viewport: VIEWPORT,
|
|
498
|
+
deviceScaleFactor: 1,
|
|
499
|
+
})
|
|
500
|
+
pageCtx.setDefaultTimeout(30000)
|
|
501
|
+
const p = await pageCtx.newPage()
|
|
502
|
+
await p.goto(args.url, { waitUntil: "domcontentloaded", timeout: 30000 })
|
|
503
|
+
await p.evaluate(PICKER_JS)
|
|
504
|
+
try {
|
|
505
|
+
await p.waitForFunction("window.__ocPicked !== undefined", null, {
|
|
506
|
+
timeout: (args.timeoutSeconds ?? 300) * 1000,
|
|
507
|
+
polling: 200,
|
|
508
|
+
})
|
|
509
|
+
} catch {
|
|
510
|
+
return "No selection made within the timeout — visible browser closed."
|
|
511
|
+
}
|
|
512
|
+
const outcome = (await p.evaluate("window.__ocPicked")) as
|
|
513
|
+
| null
|
|
514
|
+
| {
|
|
515
|
+
note: string
|
|
516
|
+
selector: string
|
|
517
|
+
tag: string
|
|
518
|
+
id: string
|
|
519
|
+
classes: string
|
|
520
|
+
text: string
|
|
521
|
+
html: string
|
|
522
|
+
rect: { x: number; y: number; width: number; height: number }
|
|
523
|
+
styles: Record<string, string>
|
|
524
|
+
react: { file?: string; line?: number; comp?: string } | null
|
|
525
|
+
}
|
|
526
|
+
if (outcome === null) return "Selection cancelled by user."
|
|
527
|
+
const lines = [
|
|
528
|
+
`User picked an element on ${p.url()}`,
|
|
529
|
+
`Selector: ${outcome.selector}`,
|
|
530
|
+
`Tag: ${outcome.tag}${outcome.id ? ` #${outcome.id}` : ""}${outcome.classes ? ` (${outcome.classes})` : ""}`,
|
|
531
|
+
`Text: ${outcome.text || "(none)"}`,
|
|
532
|
+
`Box: ${outcome.rect.width}x${outcome.rect.height} at (${outcome.rect.x}, ${outcome.rect.y})`,
|
|
533
|
+
`Styles: fontSize=${outcome.styles.fontSize} fontWeight=${outcome.styles.fontWeight} color=${outcome.styles.color} background=${outcome.styles.backgroundColor} radius=${outcome.styles.borderRadius} font=${(outcome.styles.fontFamily || "").split(",")[0]}`,
|
|
534
|
+
`HTML: ${outcome.html}`,
|
|
535
|
+
outcome.react?.file
|
|
536
|
+
? `React source: ${outcome.react.file}${outcome.react.line ? `:${outcome.react.line}` : ""}${outcome.react.comp ? ` (${outcome.react.comp})` : ""}`
|
|
537
|
+
: null,
|
|
538
|
+
`User note: ${outcome.note || "(nothing typed — ask the user what to change)"}`,
|
|
539
|
+
].filter(Boolean)
|
|
540
|
+
return lines.join("\n")
|
|
541
|
+
} finally {
|
|
542
|
+
if (inspectBrowser) await inspectBrowser.close().catch(() => {})
|
|
543
|
+
}
|
|
544
|
+
},
|
|
545
|
+
}),
|
|
546
|
+
},
|
|
547
|
+
}
|
|
548
|
+
}
|