@thoughtflow/browser 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/tools/workspace/browser-session-pool.ts +376 -0
- package/src/tools/workspace/browser.tool.ts +362 -97
- package/src/tools/workspace/vision-browser.tool.ts +158 -4
- package/tsconfig.json +4 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +0 -1
- package/dist/tools/workspace/browser.tool.d.ts +0 -23
- package/dist/tools/workspace/browser.tool.d.ts.map +0 -1
- package/dist/tools/workspace/browser.tool.js +0 -170
- package/dist/tools/workspace/browser.tool.js.map +0 -1
- package/dist/tools/workspace/vision-browser.tool.d.ts +0 -30
- package/dist/tools/workspace/vision-browser.tool.d.ts.map +0 -1
- package/dist/tools/workspace/vision-browser.tool.js +0 -269
- package/dist/tools/workspace/vision-browser.tool.js.map +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod"
|
|
2
|
-
import { ToolBase, ToolContext, ToolExecutionResult, LlmAdapter
|
|
2
|
+
import { ToolBase, ToolContext, ToolExecutionResult, LlmAdapter } from "@thoughtflow/core"
|
|
3
|
+
import { buildUserVisionMessage } from "@thoughtflow/core"
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* VisionBrowserTool — navigates to a URL, optionally interacts with the page,
|
|
@@ -23,7 +24,11 @@ export class VisionBrowserTool extends ToolBase {
|
|
|
23
24
|
"Analyzes it according to the provided prompt using a vision-capable model. " +
|
|
24
25
|
"Use this tool when you need to visually inspect a rendered web page — checking layout, " +
|
|
25
26
|
"colors, visibility of elements, UI correctness, or any visual property that cannot " +
|
|
26
|
-
"be determined from DOM text alone.
|
|
27
|
+
"be determined from DOM text alone. " +
|
|
28
|
+
"Optionally captures browser console logs (captureConsole: true) and network " +
|
|
29
|
+
"requests with full request/response details (captureNetwork: true). " +
|
|
30
|
+
"Network defaults to summary + full detail for failed requests — use " +
|
|
31
|
+
"\"all\" for every request body/headers, or \"failed\" for only errors."
|
|
27
32
|
readonly namespace = "web"
|
|
28
33
|
readonly tags = ["read", "analyze"]
|
|
29
34
|
readonly readonly = true
|
|
@@ -70,6 +75,14 @@ export class VisionBrowserTool extends ToolBase {
|
|
|
70
75
|
selector: z.string().optional(),
|
|
71
76
|
value: z.string().optional(),
|
|
72
77
|
})).optional().describe("Interactions to perform before screenshot"),
|
|
78
|
+
captureConsole: z.union([z.boolean(), z.enum(["error", "all"])]).optional().default(false)
|
|
79
|
+
.describe("Capture browser console output: true for last 50 entries, " +
|
|
80
|
+
"\"error\" for only errors/warnings, \"all\" for everything"),
|
|
81
|
+
captureNetwork: z.union([z.boolean(), z.enum(["failed", "all"])]).optional().default(false)
|
|
82
|
+
.describe("Capture network requests: true for summary + full detail on errors, " +
|
|
83
|
+
"\"failed\" for only errors with full detail, \"all\" for every request/response with headers and body"),
|
|
84
|
+
consoleMaxEntries: z.number().optional().default(50).describe("Max console entries (ignored when \"all\")"),
|
|
85
|
+
networkMaxEntries: z.number().optional().default(50).describe("Max network entries (ignored when \"all\")"),
|
|
73
86
|
})
|
|
74
87
|
)
|
|
75
88
|
}
|
|
@@ -142,10 +155,86 @@ export class VisionBrowserTool extends ToolBase {
|
|
|
142
155
|
if (sources.length > 1) return { success: false, error: "Provide exactly one of: url, html, or htmlFile" }
|
|
143
156
|
if (!prompt) return { success: false, error: "prompt is required" }
|
|
144
157
|
|
|
158
|
+
const captureConsole = input["captureConsole"] as boolean | "error" | "all" | undefined
|
|
159
|
+
const captureNetwork = input["captureNetwork"] as boolean | "failed" | "all" | undefined
|
|
160
|
+
const consoleMax = (input["consoleMaxEntries"] as number) ?? 50
|
|
161
|
+
const networkMax = (input["networkMaxEntries"] as number) ?? 50
|
|
162
|
+
|
|
163
|
+
// ── Console & network collectors ─────────────────────────────────
|
|
164
|
+
interface ConsoleEntry { type: string; text: string; location: string }
|
|
165
|
+
interface NetworkEntry {
|
|
166
|
+
idx: number; method: string; url: string; status?: number; statusText?: string
|
|
167
|
+
duration?: number; error?: string
|
|
168
|
+
reqHeaders?: Record<string, string>; reqBody?: string
|
|
169
|
+
resHeaders?: Record<string, string>; resBody?: string
|
|
170
|
+
}
|
|
171
|
+
const consoleEntries: ConsoleEntry[] = []
|
|
172
|
+
const networkEntries: NetworkEntry[] = []
|
|
173
|
+
let netIdx = 0
|
|
174
|
+
|
|
145
175
|
try {
|
|
146
176
|
const page = await this.getPage()
|
|
147
177
|
await page.setViewportSize({ width, height })
|
|
148
178
|
|
|
179
|
+
// ── Register console & network listeners (before navigation) ──
|
|
180
|
+
if (captureConsole) {
|
|
181
|
+
page.on("console", (msg) => {
|
|
182
|
+
const type = msg.type()
|
|
183
|
+
if (captureConsole === "error" && type !== "error" && type !== "warning") return
|
|
184
|
+
consoleEntries.push({
|
|
185
|
+
type,
|
|
186
|
+
text: msg.text(),
|
|
187
|
+
location: msg.location().url ? `${msg.location().url}:${msg.location().lineNumber}` : "",
|
|
188
|
+
})
|
|
189
|
+
})
|
|
190
|
+
}
|
|
191
|
+
if (captureNetwork) {
|
|
192
|
+
const timings = new Map<string, number>()
|
|
193
|
+
page.on("request", (req) => {
|
|
194
|
+
if (captureNetwork === "failed") return // only collect on response/failure
|
|
195
|
+
const idx = ++netIdx
|
|
196
|
+
timings.set(req.url() + req.method(), Date.now())
|
|
197
|
+
// Capture request body for POST/PUT/PATCH
|
|
198
|
+
let reqBody: string | undefined
|
|
199
|
+
try { reqBody = req.postData() ?? undefined } catch {}
|
|
200
|
+
networkEntries.push({
|
|
201
|
+
idx, method: req.method(), url: req.url(),
|
|
202
|
+
reqHeaders: req.headers() as Record<string, string>,
|
|
203
|
+
reqBody,
|
|
204
|
+
})
|
|
205
|
+
})
|
|
206
|
+
page.on("response", async (resp) => {
|
|
207
|
+
const key = resp.url() + resp.request().method()
|
|
208
|
+
const start = timings.get(key)
|
|
209
|
+
const duration = start ? Date.now() - start : undefined
|
|
210
|
+
// Find existing entry or create
|
|
211
|
+
let entry = networkEntries.find(e => e.url === resp.url() && e.method === resp.request().method())
|
|
212
|
+
if (!entry) {
|
|
213
|
+
entry = { idx: ++netIdx, method: resp.request().method(), url: resp.url() }
|
|
214
|
+
networkEntries.push(entry)
|
|
215
|
+
}
|
|
216
|
+
entry.status = resp.status()
|
|
217
|
+
entry.statusText = resp.statusText()
|
|
218
|
+
entry.duration = duration
|
|
219
|
+
// Capture response headers & body
|
|
220
|
+
entry.resHeaders = resp.headers() as Record<string, string>
|
|
221
|
+
try {
|
|
222
|
+
const ct = resp.headers()["content-type"] ?? ""
|
|
223
|
+
if (ct.includes("json") || ct.includes("text") || ct.includes("xml") || ct.includes("javascript")) {
|
|
224
|
+
entry.resBody = await resp.text().catch(() => undefined)
|
|
225
|
+
}
|
|
226
|
+
} catch {}
|
|
227
|
+
})
|
|
228
|
+
page.on("requestfailed", (req) => {
|
|
229
|
+
const idx = ++netIdx
|
|
230
|
+
networkEntries.push({
|
|
231
|
+
idx, method: req.method(), url: req.url(),
|
|
232
|
+
error: req.failure()?.errorText ?? "Unknown error",
|
|
233
|
+
reqHeaders: req.headers() as Record<string, string>,
|
|
234
|
+
})
|
|
235
|
+
})
|
|
236
|
+
}
|
|
237
|
+
|
|
149
238
|
// Set custom headers if provided (for auth, etc.)
|
|
150
239
|
if (headers && url) {
|
|
151
240
|
await page.setExtraHTTPHeaders(headers)
|
|
@@ -274,9 +363,74 @@ export class VisionBrowserTool extends ToolBase {
|
|
|
274
363
|
tools: [],
|
|
275
364
|
})
|
|
276
365
|
|
|
277
|
-
|
|
366
|
+
// ── Build final output with optional console & network ──────────
|
|
367
|
+
let output = response.message.content ?? ""
|
|
368
|
+
|
|
369
|
+
if (captureConsole && consoleEntries.length > 0) {
|
|
370
|
+
const entries = captureConsole === "all" ? consoleEntries : consoleEntries.slice(-consoleMax)
|
|
371
|
+
output += "\n\n── Console"
|
|
372
|
+
if (captureConsole !== "all" && consoleEntries.length > consoleMax) {
|
|
373
|
+
output += ` (${consoleEntries.length} total, showing last ${consoleMax})`
|
|
374
|
+
}
|
|
375
|
+
output += " ──\n"
|
|
376
|
+
for (const e of entries) {
|
|
377
|
+
const tag = e.type === "error" ? "error" : e.type === "warning" ? "warn " : e.type
|
|
378
|
+
const loc = e.location ? ` — ${e.location}` : ""
|
|
379
|
+
output += ` [${tag}] ${e.text.slice(0, 500)}${loc}\n`
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
if (captureNetwork && networkEntries.length > 0) {
|
|
384
|
+
const sorted = [...networkEntries].sort((a, b) => a.idx - b.idx)
|
|
385
|
+
const failed = sorted.filter(e => e.error || (e.status && e.status >= 400))
|
|
386
|
+
const isAll = captureNetwork === "all"
|
|
387
|
+
const isFailed = captureNetwork === "failed"
|
|
388
|
+
const toShow = isAll ? sorted : sorted.slice(-networkMax)
|
|
389
|
+
|
|
390
|
+
output += `\n\n── Network (${sorted.length} requests`
|
|
391
|
+
if (failed.length > 0) output += `, ${failed.length} failed`
|
|
392
|
+
output += ") ──\n"
|
|
393
|
+
for (const e of toShow) {
|
|
394
|
+
const status = e.error ? "FAILED" : `${e.status} ${e.statusText ?? ""}`
|
|
395
|
+
const dur = e.duration != null ? ` (${e.duration}ms)` : ""
|
|
396
|
+
const icon = e.error || (e.status && e.status >= 400) ? "✗" : "✓"
|
|
397
|
+
output += ` ${icon} ${e.method.padEnd(6)} ${e.url.slice(0, 120)} → ${status}${dur}\n`
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Detail section: failed by default, all if requested
|
|
401
|
+
const detailEntries = isAll ? toShow : isFailed ? failed : failed
|
|
402
|
+
if (detailEntries.length > 0) {
|
|
403
|
+
output += `\n── Request Detail ──\n`
|
|
404
|
+
for (const e of detailEntries) {
|
|
405
|
+
output += `\n### ${e.method} ${e.url}`
|
|
406
|
+
if (e.error) {
|
|
407
|
+
output += ` → FAILED\nError: ${e.error}\n`
|
|
408
|
+
} else {
|
|
409
|
+
output += ` → ${e.status} ${e.statusText ?? ""}`
|
|
410
|
+
if (e.duration != null) output += ` (${e.duration}ms)`
|
|
411
|
+
output += "\n"
|
|
412
|
+
}
|
|
413
|
+
if (e.reqHeaders && Object.keys(e.reqHeaders).length > 0) {
|
|
414
|
+
output += `Request headers:\n`
|
|
415
|
+
for (const [k, v] of Object.entries(e.reqHeaders)) {
|
|
416
|
+
output += ` ${k}: ${v}\n`
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
if (e.reqBody) output += `Request body:\n ${e.reqBody.slice(0, 2000)}\n`
|
|
420
|
+
if (e.resHeaders && Object.keys(e.resHeaders).length > 0) {
|
|
421
|
+
output += `Response headers:\n`
|
|
422
|
+
for (const [k, v] of Object.entries(e.resHeaders)) {
|
|
423
|
+
output += ` ${k}: ${v}\n`
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
if (e.resBody) output += `Response body:\n ${e.resBody.slice(0, 3000)}\n`
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return { success: true, output }
|
|
278
432
|
} catch (err) {
|
|
279
433
|
return { success: false, error: String(err instanceof Error ? err.message : err) }
|
|
280
434
|
}
|
|
281
435
|
}
|
|
282
|
-
}
|
|
436
|
+
}
|
package/tsconfig.json
CHANGED
package/dist/index.d.ts
DELETED
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC"}
|
package/dist/index.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.VisionBrowserTool = exports.BrowserTool = void 0;
|
|
4
|
-
var browser_tool_1 = require("./tools/workspace/browser.tool");
|
|
5
|
-
Object.defineProperty(exports, "BrowserTool", { enumerable: true, get: function () { return browser_tool_1.BrowserTool; } });
|
|
6
|
-
var vision_browser_tool_1 = require("./tools/workspace/vision-browser.tool");
|
|
7
|
-
Object.defineProperty(exports, "VisionBrowserTool", { enumerable: true, get: function () { return vision_browser_tool_1.VisionBrowserTool; } });
|
|
8
|
-
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+DAA6D;AAApD,2GAAA,WAAW,OAAA;AACpB,6EAA0E;AAAjE,wHAAA,iBAAiB,OAAA"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { ToolBase, ToolContext, ToolExecutionResult } from "@thoughtflow/core";
|
|
2
|
-
/**
|
|
3
|
-
* BrowserTool — Playwright-backed browser automation (Faza 8 stub).
|
|
4
|
-
*
|
|
5
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
6
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
7
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
8
|
-
*
|
|
9
|
-
* Note: Full implementation planned in Faza 8. Currently returns an
|
|
10
|
-
* informative "not yet implemented" error for all actions.
|
|
11
|
-
*/
|
|
12
|
-
export declare class BrowserTool extends ToolBase {
|
|
13
|
-
readonly name = "browser";
|
|
14
|
-
readonly description: string;
|
|
15
|
-
/** Read-only actions: navigate/get_text/screenshot/get_dom/extract */
|
|
16
|
-
isReadOnly(input: Record<string, unknown>): boolean;
|
|
17
|
-
constructor();
|
|
18
|
-
private browser;
|
|
19
|
-
private page;
|
|
20
|
-
private getPage;
|
|
21
|
-
run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult>;
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=browser.tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/workspace/browser.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAwB/E;;;;;;;;;GASG;AACH,qBAAa,WAAY,SAAQ,QAAQ;IACvC,QAAQ,CAAC,IAAI,aAAa;IAC1B,QAAQ,CAAC,WAAW,SAIwD;IAE5E,sEAAsE;IAC7D,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO;;IAqB5D,OAAO,CAAC,OAAO,CAA6C;IAC5D,OAAO,CAAC,IAAI,CAA0C;YAExC,OAAO;IAYf,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAsG3F"}
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BrowserTool = void 0;
|
|
4
|
-
const zod_1 = require("zod");
|
|
5
|
-
const core_1 = require("@thoughtflow/core");
|
|
6
|
-
const BROWSER_ACTIONS = [
|
|
7
|
-
"navigate",
|
|
8
|
-
"get_text",
|
|
9
|
-
"screenshot",
|
|
10
|
-
"get_dom",
|
|
11
|
-
"extract",
|
|
12
|
-
"click",
|
|
13
|
-
"type",
|
|
14
|
-
"select",
|
|
15
|
-
"close",
|
|
16
|
-
];
|
|
17
|
-
const READ_ONLY_BROWSER_ACTIONS = new Set([
|
|
18
|
-
"navigate",
|
|
19
|
-
"get_text",
|
|
20
|
-
"screenshot",
|
|
21
|
-
"get_dom",
|
|
22
|
-
"extract",
|
|
23
|
-
]);
|
|
24
|
-
/**
|
|
25
|
-
* BrowserTool — Playwright-backed browser automation (Faza 8 stub).
|
|
26
|
-
*
|
|
27
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
28
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
29
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
30
|
-
*
|
|
31
|
-
* Note: Full implementation planned in Faza 8. Currently returns an
|
|
32
|
-
* informative "not yet implemented" error for all actions.
|
|
33
|
-
*/
|
|
34
|
-
class BrowserTool extends core_1.ToolBase {
|
|
35
|
-
name = "browser";
|
|
36
|
-
description = "Headless browser automation with Playwright (Chromium). " +
|
|
37
|
-
"Read: navigate, get_text, screenshot (vision only), get_dom, extract. " +
|
|
38
|
-
"Write: click, type, select, close. " +
|
|
39
|
-
"Playwright is loaded on first use — no install required at import time.";
|
|
40
|
-
/** Read-only actions: navigate/get_text/screenshot/get_dom/extract */
|
|
41
|
-
isReadOnly(input) {
|
|
42
|
-
return READ_ONLY_BROWSER_ACTIONS.has(input["action"] ?? "navigate");
|
|
43
|
-
}
|
|
44
|
-
constructor() {
|
|
45
|
-
super(zod_1.z.object({
|
|
46
|
-
action: zod_1.z
|
|
47
|
-
.enum(BROWSER_ACTIONS)
|
|
48
|
-
.describe("Browser action to perform"),
|
|
49
|
-
url: zod_1.z.string().optional().describe("URL to navigate to (for navigate action)"),
|
|
50
|
-
selector: zod_1.z.string().optional().describe("CSS selector (for click/type/select/extract/get_dom)"),
|
|
51
|
-
text: zod_1.z.string().optional().describe("Text to type (for type action)"),
|
|
52
|
-
value: zod_1.z.string().optional().describe("Option value to select (for select action)"),
|
|
53
|
-
fields: zod_1.z.array(zod_1.z.string()).optional().describe("Fields to extract from rows (for extract action)"),
|
|
54
|
-
waitFor: zod_1.z.number().optional().describe("Max ms to wait for page load (default: 5000)"),
|
|
55
|
-
}));
|
|
56
|
-
}
|
|
57
|
-
// Shared browser instance (reused across calls)
|
|
58
|
-
browser = null;
|
|
59
|
-
page = null;
|
|
60
|
-
async getPage() {
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
62
|
-
const { chromium } = require("playwright");
|
|
63
|
-
if (!this.browser || !this.browser.isConnected()) {
|
|
64
|
-
this.browser = await chromium.launch({ headless: true });
|
|
65
|
-
}
|
|
66
|
-
if (!this.page || this.page.isClosed()) {
|
|
67
|
-
this.page = await this.browser.newPage();
|
|
68
|
-
}
|
|
69
|
-
return this.page;
|
|
70
|
-
}
|
|
71
|
-
async run(input, _ctx) {
|
|
72
|
-
// Lazy-load Playwright on first use (optional peer dependency)
|
|
73
|
-
try {
|
|
74
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
75
|
-
require("playwright");
|
|
76
|
-
}
|
|
77
|
-
catch {
|
|
78
|
-
return {
|
|
79
|
-
success: false,
|
|
80
|
-
error: "Playwright is not installed. Run: npm install playwright && npx playwright install chromium",
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
const action = input["action"] ?? "navigate";
|
|
84
|
-
const waitFor = input["waitFor"] ?? 5000;
|
|
85
|
-
try {
|
|
86
|
-
const page = await this.getPage();
|
|
87
|
-
switch (action) {
|
|
88
|
-
case "navigate": {
|
|
89
|
-
const url = input["url"];
|
|
90
|
-
if (!url)
|
|
91
|
-
return { success: false, error: "url is required for navigate" };
|
|
92
|
-
await page.goto(url, { waitUntil: "domcontentloaded", timeout: waitFor });
|
|
93
|
-
const title = await page.title();
|
|
94
|
-
return { success: true, output: JSON.stringify({ title, url: page.url() }) };
|
|
95
|
-
}
|
|
96
|
-
case "get_text": {
|
|
97
|
-
const selector = input["selector"] ?? "body";
|
|
98
|
-
const text = await page.locator(selector).first().innerText({ timeout: waitFor });
|
|
99
|
-
return { success: true, output: text.trim() };
|
|
100
|
-
}
|
|
101
|
-
case "get_dom": {
|
|
102
|
-
const selector = input["selector"] ?? "body";
|
|
103
|
-
const html = await page.locator(selector).first().innerHTML({ timeout: waitFor });
|
|
104
|
-
return { success: true, output: html };
|
|
105
|
-
}
|
|
106
|
-
case "extract": {
|
|
107
|
-
const selector = input["selector"];
|
|
108
|
-
const fields = input["fields"] ?? [];
|
|
109
|
-
if (!selector)
|
|
110
|
-
return { success: false, error: "selector is required for extract" };
|
|
111
|
-
const rows = await page.locator(selector).all();
|
|
112
|
-
const results = [];
|
|
113
|
-
for (const row of rows) {
|
|
114
|
-
if (fields.length === 0) {
|
|
115
|
-
results.push({ text: (await row.innerText()).trim() });
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
const obj = {};
|
|
119
|
-
for (const field of fields) {
|
|
120
|
-
const el = row.locator(`[class*="${field}"], [data-${field}], .${field}`).first();
|
|
121
|
-
obj[field] = (await el.innerText().catch(() => "")).trim();
|
|
122
|
-
}
|
|
123
|
-
results.push(obj);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
return { success: true, output: JSON.stringify(results) };
|
|
127
|
-
}
|
|
128
|
-
case "click": {
|
|
129
|
-
const selector = input["selector"];
|
|
130
|
-
if (!selector)
|
|
131
|
-
return { success: false, error: "selector is required for click" };
|
|
132
|
-
await page.locator(selector).first().click({ timeout: waitFor });
|
|
133
|
-
return { success: true, output: `Clicked: ${selector}` };
|
|
134
|
-
}
|
|
135
|
-
case "type": {
|
|
136
|
-
const selector = input["selector"];
|
|
137
|
-
const text = input["text"];
|
|
138
|
-
if (!selector || text === undefined)
|
|
139
|
-
return { success: false, error: "selector and text required for type" };
|
|
140
|
-
await page.locator(selector).first().fill(text, { timeout: waitFor });
|
|
141
|
-
return { success: true, output: `Typed into ${selector}` };
|
|
142
|
-
}
|
|
143
|
-
case "select": {
|
|
144
|
-
const selector = input["selector"];
|
|
145
|
-
const value = input["value"];
|
|
146
|
-
if (!selector || !value)
|
|
147
|
-
return { success: false, error: "selector and value required for select" };
|
|
148
|
-
await page.locator(selector).first().selectOption(value, { timeout: waitFor });
|
|
149
|
-
return { success: true, output: `Selected ${value} in ${selector}` };
|
|
150
|
-
}
|
|
151
|
-
case "screenshot": {
|
|
152
|
-
const buf = await page.screenshot({ type: "png" });
|
|
153
|
-
return { success: true, output: `screenshot:base64:${buf.toString("base64")}` };
|
|
154
|
-
}
|
|
155
|
-
case "close": {
|
|
156
|
-
await this.page?.close();
|
|
157
|
-
this.page = null;
|
|
158
|
-
return { success: true, output: "Browser page closed" };
|
|
159
|
-
}
|
|
160
|
-
default:
|
|
161
|
-
return { success: false, error: `Unknown action: ${action}` };
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
catch (err) {
|
|
165
|
-
return { success: false, error: String(err instanceof Error ? err.message : err) };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
exports.BrowserTool = BrowserTool;
|
|
170
|
-
//# sourceMappingURL=browser.tool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.tool.js","sourceRoot":"","sources":["../../../src/tools/workspace/browser.tool.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AACxB,4CAA+E;AAE/E,MAAM,eAAe,GAAG;IACtB,UAAU;IACV,UAAU;IACV,YAAY;IACZ,SAAS;IACT,SAAS;IACT,OAAO;IACP,MAAM;IACN,QAAQ;IACR,OAAO;CACC,CAAC;AAIX,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAgB;IACvD,UAAU;IACV,UAAU;IACV,YAAY;IACZ,SAAS;IACT,SAAS;CACV,CAAC,CAAC;AAEH;;;;;;;;;GASG;AACH,MAAa,WAAY,SAAQ,eAAQ;IAC9B,IAAI,GAAG,SAAS,CAAC;IACjB,WAAW,GAClB,0DAA0D;QAC1D,wEAAwE;QACxE,qCAAqC;QACrC,yEAAyE,CAAC;IAE5E,sEAAsE;IAC7D,UAAU,CAAC,KAA8B;QAChD,OAAO,yBAAyB,CAAC,GAAG,CAAE,KAAK,CAAC,QAAQ,CAAmB,IAAI,UAAU,CAAC,CAAC;IACzF,CAAC;IAED;QACE,KAAK,CACH,OAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,OAAC;iBACN,IAAI,CAAC,eAAe,CAAC;iBACrB,QAAQ,CAAC,2BAA2B,CAAC;YACxC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;YAC/E,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;YAChG,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;YACtE,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;YACnF,MAAM,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACnG,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;SACxF,CAAC,CACH,CAAC;IACJ,CAAC;IAED,gDAAgD;IACxC,OAAO,GAAwC,IAAI,CAAC;IACpD,IAAI,GAAqC,IAAI,CAAC;IAE9C,KAAK,CAAC,OAAO;QACnB,iEAAiE;QACjE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,YAAY,CAAgC,CAAC;QAC1E,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAA8B,EAAE,IAAiB;QACzD,+DAA+D;QAC/D,IAAI,CAAC;YACH,iEAAiE;YACjE,OAAO,CAAC,YAAY,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,6FAA6F;aAChG,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAI,KAAK,CAAC,QAAQ,CAAmB,IAAI,UAAU,CAAC;QAChE,MAAM,OAAO,GAAI,KAAK,CAAC,SAAS,CAAwB,IAAI,IAAI,CAAC;QAEjE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAW,CAAC;oBACnC,IAAI,CAAC,GAAG;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC;oBAC3E,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC/E,CAAC;gBAED,KAAK,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,QAAQ,GAAI,KAAK,CAAC,UAAU,CAAwB,IAAI,MAAM,CAAC;oBACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAClF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChD,CAAC;gBAED,KAAK,SAAS,CAAC,CAAC,CAAC;oBACf,MAAM,QAAQ,GAAI,KAAK,CAAC,UAAU,CAAwB,IAAI,MAAM,CAAC;oBACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAClF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;gBACzC,CAAC;gBAED,KAAK,SAAS,CAAC,CAAC,CAAC;oBACf,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,MAAM,GAAI,KAAK,CAAC,QAAQ,CAA0B,IAAI,EAAE,CAAC;oBAC/D,IAAI,CAAC,QAAQ;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC;oBACpF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC;oBAChD,MAAM,OAAO,GAA6B,EAAE,CAAC;oBAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBACzD,CAAC;6BAAM,CAAC;4BACN,MAAM,GAAG,GAA2B,EAAE,CAAC;4BACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gCAC3B,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,aAAa,KAAK,OAAO,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gCAClF,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC7D,CAAC;4BACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACpB,CAAC;oBACH,CAAC;oBACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5D,CAAC;gBAED,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,IAAI,CAAC,QAAQ;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;oBAClF,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,QAAQ,EAAE,EAAE,CAAC;gBAC3D,CAAC;gBAED,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAW,CAAC;oBACrC,IAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;oBAC7G,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,QAAQ,EAAE,EAAE,CAAC;gBAC7D,CAAC;gBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACd,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAW,CAAC;oBACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;oBACpG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC/E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,OAAO,QAAQ,EAAE,EAAE,CAAC;gBACvE,CAAC;gBAED,KAAK,YAAY,CAAC,CAAC,CAAC;oBAClB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClF,CAAC;gBAED,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;oBACjB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;gBAC1D,CAAC;gBAED;oBACE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,MAAM,EAAE,EAAE,CAAC;YAClE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACrF,CAAC;IACH,CAAC;CACF;AAnJD,kCAmJC"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { ToolBase, ToolContext, ToolExecutionResult, LlmAdapter } from "@thoughtflow/core";
|
|
2
|
-
/**
|
|
3
|
-
* VisionBrowserTool — navigates to a URL, optionally interacts with the page,
|
|
4
|
-
* takes a screenshot, and analyzes it using a vision-capable LLM.
|
|
5
|
-
*
|
|
6
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
7
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
8
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
9
|
-
*
|
|
10
|
-
* The tool is self-contained: it calls the vision LLM directly rather than
|
|
11
|
-
* routing images through the conversation pipeline.
|
|
12
|
-
*/
|
|
13
|
-
export declare class VisionBrowserTool extends ToolBase {
|
|
14
|
-
private readonly visionAdapter;
|
|
15
|
-
private readonly defaultModel;
|
|
16
|
-
private readonly rootDir?;
|
|
17
|
-
readonly name = "visionBrowser";
|
|
18
|
-
readonly description: string;
|
|
19
|
-
readonly namespace = "web";
|
|
20
|
-
readonly tags: string[];
|
|
21
|
-
readonly readonly = true;
|
|
22
|
-
constructor(visionAdapter: LlmAdapter, defaultModel?: string, rootDir?: string | undefined);
|
|
23
|
-
private browser;
|
|
24
|
-
private page;
|
|
25
|
-
private getPage;
|
|
26
|
-
private closePage;
|
|
27
|
-
closeBrowser(): Promise<void>;
|
|
28
|
-
run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult>;
|
|
29
|
-
}
|
|
30
|
-
//# sourceMappingURL=vision-browser.tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"vision-browser.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/workspace/vision-browser.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,EAA0B,MAAM,mBAAmB,CAAA;AAElH;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,SAAQ,QAAQ;IAiB3C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAlB3B,QAAQ,CAAC,IAAI,mBAAkB;IAC/B,QAAQ,CAAC,WAAW,SASqD;IACzE,QAAQ,CAAC,SAAS,SAAQ;IAC1B,QAAQ,CAAC,IAAI,WAAsB;IACnC,QAAQ,CAAC,QAAQ,QAAO;gBAGL,aAAa,EAAE,UAAU,EACzB,YAAY,GAAE,MAAqB,EACnC,OAAO,CAAC,EAAE,MAAM,YAAA;IA2CnC,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,IAAI,CAAyC;YAEvC,OAAO;YAYP,SAAS;IAKjB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAM7B,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAmL3F"}
|