@thoughtflow/browser 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/tsconfig.json +4 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +0 -1
- package/dist/tools/workspace/browser.tool.d.ts +0 -23
- package/dist/tools/workspace/browser.tool.d.ts.map +0 -1
- package/dist/tools/workspace/browser.tool.js +0 -170
- package/dist/tools/workspace/browser.tool.js.map +0 -1
- package/dist/tools/workspace/vision-browser.tool.d.ts +0 -30
- package/dist/tools/workspace/vision-browser.tool.d.ts.map +0 -1
- package/dist/tools/workspace/vision-browser.tool.js +0 -269
- package/dist/tools/workspace/vision-browser.tool.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thoughtflow/browser",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
5
|
"types": "dist/index.d.ts",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"build": "tsc -p tsconfig.build.json"
|
|
8
8
|
},
|
|
9
9
|
"dependencies": {
|
|
10
|
-
"@thoughtflow/core": "
|
|
11
|
-
"playwright": "^1.59.1"
|
|
10
|
+
"@thoughtflow/core": "*",
|
|
11
|
+
"playwright": "^1.59.1",
|
|
12
|
+
"zod": "^3.24.3"
|
|
12
13
|
}
|
|
13
14
|
}
|
package/tsconfig.json
CHANGED
package/dist/index.d.ts
DELETED
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC"}
|
package/dist/index.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.VisionBrowserTool = exports.BrowserTool = void 0;
|
|
4
|
-
var browser_tool_1 = require("./tools/workspace/browser.tool");
|
|
5
|
-
Object.defineProperty(exports, "BrowserTool", { enumerable: true, get: function () { return browser_tool_1.BrowserTool; } });
|
|
6
|
-
var vision_browser_tool_1 = require("./tools/workspace/vision-browser.tool");
|
|
7
|
-
Object.defineProperty(exports, "VisionBrowserTool", { enumerable: true, get: function () { return vision_browser_tool_1.VisionBrowserTool; } });
|
|
8
|
-
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+DAA6D;AAApD,2GAAA,WAAW,OAAA;AACpB,6EAA0E;AAAjE,wHAAA,iBAAiB,OAAA"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { ToolBase, ToolContext, ToolExecutionResult } from "@thoughtflow/core";
|
|
2
|
-
/**
|
|
3
|
-
* BrowserTool — Playwright-backed browser automation (Faza 8 stub).
|
|
4
|
-
*
|
|
5
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
6
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
7
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
8
|
-
*
|
|
9
|
-
* Note: Full implementation planned in Faza 8. Currently returns an
|
|
10
|
-
* informative "not yet implemented" error for all actions.
|
|
11
|
-
*/
|
|
12
|
-
export declare class BrowserTool extends ToolBase {
|
|
13
|
-
readonly name = "browser";
|
|
14
|
-
readonly description: string;
|
|
15
|
-
/** Read-only actions: navigate/get_text/screenshot/get_dom/extract */
|
|
16
|
-
isReadOnly(input: Record<string, unknown>): boolean;
|
|
17
|
-
constructor();
|
|
18
|
-
private browser;
|
|
19
|
-
private page;
|
|
20
|
-
private getPage;
|
|
21
|
-
run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult>;
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=browser.tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/workspace/browser.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAwB/E;;;;;;;;;GASG;AACH,qBAAa,WAAY,SAAQ,QAAQ;IACvC,QAAQ,CAAC,IAAI,aAAa;IAC1B,QAAQ,CAAC,WAAW,SAIwD;IAE5E,sEAAsE;IAC7D,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO;;IAqB5D,OAAO,CAAC,OAAO,CAA6C;IAC5D,OAAO,CAAC,IAAI,CAA0C;YAExC,OAAO;IAYf,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAsG3F"}
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BrowserTool = void 0;
|
|
4
|
-
const zod_1 = require("zod");
|
|
5
|
-
const core_1 = require("@thoughtflow/core");
|
|
6
|
-
const BROWSER_ACTIONS = [
|
|
7
|
-
"navigate",
|
|
8
|
-
"get_text",
|
|
9
|
-
"screenshot",
|
|
10
|
-
"get_dom",
|
|
11
|
-
"extract",
|
|
12
|
-
"click",
|
|
13
|
-
"type",
|
|
14
|
-
"select",
|
|
15
|
-
"close",
|
|
16
|
-
];
|
|
17
|
-
const READ_ONLY_BROWSER_ACTIONS = new Set([
|
|
18
|
-
"navigate",
|
|
19
|
-
"get_text",
|
|
20
|
-
"screenshot",
|
|
21
|
-
"get_dom",
|
|
22
|
-
"extract",
|
|
23
|
-
]);
|
|
24
|
-
/**
|
|
25
|
-
* BrowserTool — Playwright-backed browser automation (Faza 8 stub).
|
|
26
|
-
*
|
|
27
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
28
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
29
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
30
|
-
*
|
|
31
|
-
* Note: Full implementation planned in Faza 8. Currently returns an
|
|
32
|
-
* informative "not yet implemented" error for all actions.
|
|
33
|
-
*/
|
|
34
|
-
class BrowserTool extends core_1.ToolBase {
|
|
35
|
-
name = "browser";
|
|
36
|
-
description = "Headless browser automation with Playwright (Chromium). " +
|
|
37
|
-
"Read: navigate, get_text, screenshot (vision only), get_dom, extract. " +
|
|
38
|
-
"Write: click, type, select, close. " +
|
|
39
|
-
"Playwright is loaded on first use — no install required at import time.";
|
|
40
|
-
/** Read-only actions: navigate/get_text/screenshot/get_dom/extract */
|
|
41
|
-
isReadOnly(input) {
|
|
42
|
-
return READ_ONLY_BROWSER_ACTIONS.has(input["action"] ?? "navigate");
|
|
43
|
-
}
|
|
44
|
-
constructor() {
|
|
45
|
-
super(zod_1.z.object({
|
|
46
|
-
action: zod_1.z
|
|
47
|
-
.enum(BROWSER_ACTIONS)
|
|
48
|
-
.describe("Browser action to perform"),
|
|
49
|
-
url: zod_1.z.string().optional().describe("URL to navigate to (for navigate action)"),
|
|
50
|
-
selector: zod_1.z.string().optional().describe("CSS selector (for click/type/select/extract/get_dom)"),
|
|
51
|
-
text: zod_1.z.string().optional().describe("Text to type (for type action)"),
|
|
52
|
-
value: zod_1.z.string().optional().describe("Option value to select (for select action)"),
|
|
53
|
-
fields: zod_1.z.array(zod_1.z.string()).optional().describe("Fields to extract from rows (for extract action)"),
|
|
54
|
-
waitFor: zod_1.z.number().optional().describe("Max ms to wait for page load (default: 5000)"),
|
|
55
|
-
}));
|
|
56
|
-
}
|
|
57
|
-
// Shared browser instance (reused across calls)
|
|
58
|
-
browser = null;
|
|
59
|
-
page = null;
|
|
60
|
-
async getPage() {
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
62
|
-
const { chromium } = require("playwright");
|
|
63
|
-
if (!this.browser || !this.browser.isConnected()) {
|
|
64
|
-
this.browser = await chromium.launch({ headless: true });
|
|
65
|
-
}
|
|
66
|
-
if (!this.page || this.page.isClosed()) {
|
|
67
|
-
this.page = await this.browser.newPage();
|
|
68
|
-
}
|
|
69
|
-
return this.page;
|
|
70
|
-
}
|
|
71
|
-
async run(input, _ctx) {
|
|
72
|
-
// Lazy-load Playwright on first use (optional peer dependency)
|
|
73
|
-
try {
|
|
74
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
75
|
-
require("playwright");
|
|
76
|
-
}
|
|
77
|
-
catch {
|
|
78
|
-
return {
|
|
79
|
-
success: false,
|
|
80
|
-
error: "Playwright is not installed. Run: npm install playwright && npx playwright install chromium",
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
const action = input["action"] ?? "navigate";
|
|
84
|
-
const waitFor = input["waitFor"] ?? 5000;
|
|
85
|
-
try {
|
|
86
|
-
const page = await this.getPage();
|
|
87
|
-
switch (action) {
|
|
88
|
-
case "navigate": {
|
|
89
|
-
const url = input["url"];
|
|
90
|
-
if (!url)
|
|
91
|
-
return { success: false, error: "url is required for navigate" };
|
|
92
|
-
await page.goto(url, { waitUntil: "domcontentloaded", timeout: waitFor });
|
|
93
|
-
const title = await page.title();
|
|
94
|
-
return { success: true, output: JSON.stringify({ title, url: page.url() }) };
|
|
95
|
-
}
|
|
96
|
-
case "get_text": {
|
|
97
|
-
const selector = input["selector"] ?? "body";
|
|
98
|
-
const text = await page.locator(selector).first().innerText({ timeout: waitFor });
|
|
99
|
-
return { success: true, output: text.trim() };
|
|
100
|
-
}
|
|
101
|
-
case "get_dom": {
|
|
102
|
-
const selector = input["selector"] ?? "body";
|
|
103
|
-
const html = await page.locator(selector).first().innerHTML({ timeout: waitFor });
|
|
104
|
-
return { success: true, output: html };
|
|
105
|
-
}
|
|
106
|
-
case "extract": {
|
|
107
|
-
const selector = input["selector"];
|
|
108
|
-
const fields = input["fields"] ?? [];
|
|
109
|
-
if (!selector)
|
|
110
|
-
return { success: false, error: "selector is required for extract" };
|
|
111
|
-
const rows = await page.locator(selector).all();
|
|
112
|
-
const results = [];
|
|
113
|
-
for (const row of rows) {
|
|
114
|
-
if (fields.length === 0) {
|
|
115
|
-
results.push({ text: (await row.innerText()).trim() });
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
const obj = {};
|
|
119
|
-
for (const field of fields) {
|
|
120
|
-
const el = row.locator(`[class*="${field}"], [data-${field}], .${field}`).first();
|
|
121
|
-
obj[field] = (await el.innerText().catch(() => "")).trim();
|
|
122
|
-
}
|
|
123
|
-
results.push(obj);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
return { success: true, output: JSON.stringify(results) };
|
|
127
|
-
}
|
|
128
|
-
case "click": {
|
|
129
|
-
const selector = input["selector"];
|
|
130
|
-
if (!selector)
|
|
131
|
-
return { success: false, error: "selector is required for click" };
|
|
132
|
-
await page.locator(selector).first().click({ timeout: waitFor });
|
|
133
|
-
return { success: true, output: `Clicked: ${selector}` };
|
|
134
|
-
}
|
|
135
|
-
case "type": {
|
|
136
|
-
const selector = input["selector"];
|
|
137
|
-
const text = input["text"];
|
|
138
|
-
if (!selector || text === undefined)
|
|
139
|
-
return { success: false, error: "selector and text required for type" };
|
|
140
|
-
await page.locator(selector).first().fill(text, { timeout: waitFor });
|
|
141
|
-
return { success: true, output: `Typed into ${selector}` };
|
|
142
|
-
}
|
|
143
|
-
case "select": {
|
|
144
|
-
const selector = input["selector"];
|
|
145
|
-
const value = input["value"];
|
|
146
|
-
if (!selector || !value)
|
|
147
|
-
return { success: false, error: "selector and value required for select" };
|
|
148
|
-
await page.locator(selector).first().selectOption(value, { timeout: waitFor });
|
|
149
|
-
return { success: true, output: `Selected ${value} in ${selector}` };
|
|
150
|
-
}
|
|
151
|
-
case "screenshot": {
|
|
152
|
-
const buf = await page.screenshot({ type: "png" });
|
|
153
|
-
return { success: true, output: `screenshot:base64:${buf.toString("base64")}` };
|
|
154
|
-
}
|
|
155
|
-
case "close": {
|
|
156
|
-
await this.page?.close();
|
|
157
|
-
this.page = null;
|
|
158
|
-
return { success: true, output: "Browser page closed" };
|
|
159
|
-
}
|
|
160
|
-
default:
|
|
161
|
-
return { success: false, error: `Unknown action: ${action}` };
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
catch (err) {
|
|
165
|
-
return { success: false, error: String(err instanceof Error ? err.message : err) };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
exports.BrowserTool = BrowserTool;
|
|
170
|
-
//# sourceMappingURL=browser.tool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.tool.js","sourceRoot":"","sources":["../../../src/tools/workspace/browser.tool.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AACxB,4CAA+E;AAE/E,MAAM,eAAe,GAAG;IACtB,UAAU;IACV,UAAU;IACV,YAAY;IACZ,SAAS;IACT,SAAS;IACT,OAAO;IACP,MAAM;IACN,QAAQ;IACR,OAAO;CACC,CAAC;AAIX,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAgB;IACvD,UAAU;IACV,UAAU;IACV,YAAY;IACZ,SAAS;IACT,SAAS;CACV,CAAC,CAAC;AAEH;;;;;;;;;GASG;AACH,MAAa,WAAY,SAAQ,eAAQ;IAC9B,IAAI,GAAG,SAAS,CAAC;IACjB,WAAW,GAClB,0DAA0D;QAC1D,wEAAwE;QACxE,qCAAqC;QACrC,yEAAyE,CAAC;IAE5E,sEAAsE;IAC7D,UAAU,CAAC,KAA8B;QAChD,OAAO,yBAAyB,CAAC,GAAG,CAAE,KAAK,CAAC,QAAQ,CAAmB,IAAI,UAAU,CAAC,CAAC;IACzF,CAAC;IAED;QACE,KAAK,CACH,OAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,OAAC;iBACN,IAAI,CAAC,eAAe,CAAC;iBACrB,QAAQ,CAAC,2BAA2B,CAAC;YACxC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;YAC/E,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;YAChG,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;YACtE,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;YACnF,MAAM,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;YACnG,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;SACxF,CAAC,CACH,CAAC;IACJ,CAAC;IAED,gDAAgD;IACxC,OAAO,GAAwC,IAAI,CAAC;IACpD,IAAI,GAAqC,IAAI,CAAC;IAE9C,KAAK,CAAC,OAAO;QACnB,iEAAiE;QACjE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,YAAY,CAAgC,CAAC;QAC1E,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAA8B,EAAE,IAAiB;QACzD,+DAA+D;QAC/D,IAAI,CAAC;YACH,iEAAiE;YACjE,OAAO,CAAC,YAAY,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,6FAA6F;aAChG,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAI,KAAK,CAAC,QAAQ,CAAmB,IAAI,UAAU,CAAC;QAChE,MAAM,OAAO,GAAI,KAAK,CAAC,SAAS,CAAwB,IAAI,IAAI,CAAC;QAEjE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAW,CAAC;oBACnC,IAAI,CAAC,GAAG;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC;oBAC3E,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC/E,CAAC;gBAED,KAAK,UAAU,CAAC,CAAC,CAAC;oBAChB,MAAM,QAAQ,GAAI,KAAK,CAAC,UAAU,CAAwB,IAAI,MAAM,CAAC;oBACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAClF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChD,CAAC;gBAED,KAAK,SAAS,CAAC,CAAC,CAAC;oBACf,MAAM,QAAQ,GAAI,KAAK,CAAC,UAAU,CAAwB,IAAI,MAAM,CAAC;oBACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAClF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;gBACzC,CAAC;gBAED,KAAK,SAAS,CAAC,CAAC,CAAC;oBACf,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,MAAM,GAAI,KAAK,CAAC,QAAQ,CAA0B,IAAI,EAAE,CAAC;oBAC/D,IAAI,CAAC,QAAQ;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC;oBACpF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC;oBAChD,MAAM,OAAO,GAA6B,EAAE,CAAC;oBAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBACzD,CAAC;6BAAM,CAAC;4BACN,MAAM,GAAG,GAA2B,EAAE,CAAC;4BACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gCAC3B,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,aAAa,KAAK,OAAO,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gCAClF,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC7D,CAAC;4BACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACpB,CAAC;oBACH,CAAC;oBACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5D,CAAC;gBAED,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,IAAI,CAAC,QAAQ;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;oBAClF,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,QAAQ,EAAE,EAAE,CAAC;gBAC3D,CAAC;gBAED,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAW,CAAC;oBACrC,IAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;oBAC7G,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,QAAQ,EAAE,EAAE,CAAC;gBAC7D,CAAC;gBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACd,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAW,CAAC;oBAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAW,CAAC;oBACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK;wBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;oBACpG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC/E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,OAAO,QAAQ,EAAE,EAAE,CAAC;gBACvE,CAAC;gBAED,KAAK,YAAY,CAAC,CAAC,CAAC;oBAClB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClF,CAAC;gBAED,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;oBACjB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;gBAC1D,CAAC;gBAED;oBACE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,MAAM,EAAE,EAAE,CAAC;YAClE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACrF,CAAC;IACH,CAAC;CACF;AAnJD,kCAmJC"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { ToolBase, ToolContext, ToolExecutionResult, LlmAdapter } from "@thoughtflow/core";
|
|
2
|
-
/**
|
|
3
|
-
* VisionBrowserTool — navigates to a URL, optionally interacts with the page,
|
|
4
|
-
* takes a screenshot, and analyzes it using a vision-capable LLM.
|
|
5
|
-
*
|
|
6
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
7
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
8
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
9
|
-
*
|
|
10
|
-
* The tool is self-contained: it calls the vision LLM directly rather than
|
|
11
|
-
* routing images through the conversation pipeline.
|
|
12
|
-
*/
|
|
13
|
-
export declare class VisionBrowserTool extends ToolBase {
|
|
14
|
-
private readonly visionAdapter;
|
|
15
|
-
private readonly defaultModel;
|
|
16
|
-
private readonly rootDir?;
|
|
17
|
-
readonly name = "visionBrowser";
|
|
18
|
-
readonly description: string;
|
|
19
|
-
readonly namespace = "web";
|
|
20
|
-
readonly tags: string[];
|
|
21
|
-
readonly readonly = true;
|
|
22
|
-
constructor(visionAdapter: LlmAdapter, defaultModel?: string, rootDir?: string | undefined);
|
|
23
|
-
private browser;
|
|
24
|
-
private page;
|
|
25
|
-
private getPage;
|
|
26
|
-
private closePage;
|
|
27
|
-
closeBrowser(): Promise<void>;
|
|
28
|
-
run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult>;
|
|
29
|
-
}
|
|
30
|
-
//# sourceMappingURL=vision-browser.tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"vision-browser.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/workspace/vision-browser.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,EAA0B,MAAM,mBAAmB,CAAA;AAElH;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,SAAQ,QAAQ;IAiB3C,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAlB3B,QAAQ,CAAC,IAAI,mBAAkB;IAC/B,QAAQ,CAAC,WAAW,SASqD;IACzE,QAAQ,CAAC,SAAS,SAAQ;IAC1B,QAAQ,CAAC,IAAI,WAAsB;IACnC,QAAQ,CAAC,QAAQ,QAAO;gBAGL,aAAa,EAAE,UAAU,EACzB,YAAY,GAAE,MAAqB,EACnC,OAAO,CAAC,EAAE,MAAM,YAAA;IA2CnC,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,IAAI,CAAyC;YAEvC,OAAO;YAYP,SAAS;IAKjB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAM7B,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAmL3F"}
|
|
@@ -1,269 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.VisionBrowserTool = void 0;
|
|
4
|
-
const zod_1 = require("zod");
|
|
5
|
-
const core_1 = require("@thoughtflow/core");
|
|
6
|
-
/**
|
|
7
|
-
* VisionBrowserTool — navigates to a URL, optionally interacts with the page,
|
|
8
|
-
* takes a screenshot, and analyzes it using a vision-capable LLM.
|
|
9
|
-
*
|
|
10
|
-
* Playwright is loaded lazily on the first run() call so that registering
|
|
11
|
-
* this tool does NOT require Playwright to be installed. If Playwright is
|
|
12
|
-
* missing, run() returns `success: false` with a helpful install message.
|
|
13
|
-
*
|
|
14
|
-
* The tool is self-contained: it calls the vision LLM directly rather than
|
|
15
|
-
* routing images through the conversation pipeline.
|
|
16
|
-
*/
|
|
17
|
-
class VisionBrowserTool extends core_1.ToolBase {
|
|
18
|
-
visionAdapter;
|
|
19
|
-
defaultModel;
|
|
20
|
-
rootDir;
|
|
21
|
-
name = "visionBrowser";
|
|
22
|
-
description = "Visual web page analysis using headless browser screenshots and a vision LLM. " +
|
|
23
|
-
"Provide exactly one of: url (navigate), html (inline HTML string), or htmlFile (path to .html file). " +
|
|
24
|
-
"Optionally performs interactions (click, type, scroll, wait) before capturing the screenshot. " +
|
|
25
|
-
"Supports custom HTTP headers for auth/access, cookies for session state, " +
|
|
26
|
-
"waitForSelector for slow-loading pages, and waitForResponse for API-dependent pages. " +
|
|
27
|
-
"Analyzes it according to the provided prompt using a vision-capable model. " +
|
|
28
|
-
"Use this tool when you need to visually inspect a rendered web page — checking layout, " +
|
|
29
|
-
"colors, visibility of elements, UI correctness, or any visual property that cannot " +
|
|
30
|
-
"be determined from DOM text alone. Playwright is loaded on first use.";
|
|
31
|
-
namespace = "web";
|
|
32
|
-
tags = ["read", "analyze"];
|
|
33
|
-
readonly = true;
|
|
34
|
-
constructor(visionAdapter, defaultModel = "gemma3:12b", rootDir) {
|
|
35
|
-
super(zod_1.z.object({
|
|
36
|
-
url: zod_1.z.string().optional().describe("URL of the page to analyze"),
|
|
37
|
-
html: zod_1.z.string().optional().describe("Raw HTML string to render"),
|
|
38
|
-
htmlFile: zod_1.z.string().optional().describe("Path to a .html file to render"),
|
|
39
|
-
prompt: zod_1.z.string().describe("What to check/analyze on the rendered page"),
|
|
40
|
-
model: zod_1.z.string().optional().describe("Vision model name override"),
|
|
41
|
-
width: zod_1.z.number().optional().default(1280),
|
|
42
|
-
height: zod_1.z.number().optional().default(800),
|
|
43
|
-
fullPage: zod_1.z.boolean().optional().default(false),
|
|
44
|
-
waitFor: zod_1.z.number().optional().default(3000).describe("ms to wait after load"),
|
|
45
|
-
// ── New: auth & slow-page resilience ──
|
|
46
|
-
headers: zod_1.z.record(zod_1.z.string(), zod_1.z.string()).optional()
|
|
47
|
-
.describe("Custom HTTP headers (e.g. Authorization, Cookie, X-Auth-Token) for access-protected pages. Only applies with url navigation."),
|
|
48
|
-
cookies: zod_1.z.array(zod_1.z.object({
|
|
49
|
-
name: zod_1.z.string(),
|
|
50
|
-
value: zod_1.z.string(),
|
|
51
|
-
domain: zod_1.z.string().optional(),
|
|
52
|
-
path: zod_1.z.string().optional().default("/"),
|
|
53
|
-
httpOnly: zod_1.z.boolean().optional(),
|
|
54
|
-
secure: zod_1.z.boolean().optional(),
|
|
55
|
-
sameSite: zod_1.z.enum(["Strict", "Lax", "None"]).optional(),
|
|
56
|
-
})).optional().describe("Cookies to set before navigation (for session/auth state). Only applies with url navigation."),
|
|
57
|
-
waitForSelector: zod_1.z.string().optional()
|
|
58
|
-
.describe("CSS selector to wait for before taking screenshot (for slow-loading pages). " +
|
|
59
|
-
"If the element does not appear within maxWaitMs, proceeds anyway."),
|
|
60
|
-
waitForResponse: zod_1.z.string().optional()
|
|
61
|
-
.describe("URL pattern (glob substring or regex) to wait for a matching network response. " +
|
|
62
|
-
"Useful for pages that depend on API calls completing. Only applies with url navigation."),
|
|
63
|
-
maxWaitMs: zod_1.z.number().optional().default(30000)
|
|
64
|
-
.describe("Maximum ms to wait for waitForSelector or waitForResponse (default 30s)"),
|
|
65
|
-
selector: zod_1.z.string().optional().describe("CSS selector - screenshot only this element"),
|
|
66
|
-
actions: zod_1.z.array(zod_1.z.object({
|
|
67
|
-
action: zod_1.z.enum(["click", "type", "scroll", "wait"]),
|
|
68
|
-
selector: zod_1.z.string().optional(),
|
|
69
|
-
value: zod_1.z.string().optional(),
|
|
70
|
-
})).optional().describe("Interactions to perform before screenshot"),
|
|
71
|
-
}));
|
|
72
|
-
this.visionAdapter = visionAdapter;
|
|
73
|
-
this.defaultModel = defaultModel;
|
|
74
|
-
this.rootDir = rootDir;
|
|
75
|
-
}
|
|
76
|
-
browser = null;
|
|
77
|
-
page = null;
|
|
78
|
-
async getPage() {
|
|
79
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
80
|
-
const { chromium } = require("playwright");
|
|
81
|
-
if (!this.browser || !this.browser.isConnected()) {
|
|
82
|
-
this.browser = await chromium.launch({ headless: true });
|
|
83
|
-
}
|
|
84
|
-
if (!this.page || this.page.isClosed()) {
|
|
85
|
-
this.page = await this.browser.newPage();
|
|
86
|
-
}
|
|
87
|
-
return this.page;
|
|
88
|
-
}
|
|
89
|
-
async closePage() {
|
|
90
|
-
await this.page?.close();
|
|
91
|
-
this.page = null;
|
|
92
|
-
}
|
|
93
|
-
async closeBrowser() {
|
|
94
|
-
await this.closePage();
|
|
95
|
-
await this.browser?.close();
|
|
96
|
-
this.browser = null;
|
|
97
|
-
}
|
|
98
|
-
async run(input, _ctx) {
|
|
99
|
-
// Lazy-load Playwright on first use (optional peer dependency)
|
|
100
|
-
try {
|
|
101
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
102
|
-
require("playwright");
|
|
103
|
-
}
|
|
104
|
-
catch {
|
|
105
|
-
return {
|
|
106
|
-
success: false,
|
|
107
|
-
error: "Playwright is not installed. Run: npm install playwright && npx playwright install chromium",
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
const url = input["url"];
|
|
111
|
-
const html = input["html"];
|
|
112
|
-
const htmlFile = input["htmlFile"];
|
|
113
|
-
const prompt = input["prompt"];
|
|
114
|
-
const model = input["model"];
|
|
115
|
-
const width = input["width"] ?? 1280;
|
|
116
|
-
const height = input["height"] ?? 800;
|
|
117
|
-
const fullPage = input["fullPage"] ?? false;
|
|
118
|
-
const waitFor = input["waitFor"] ?? 3000;
|
|
119
|
-
const selector = input["selector"];
|
|
120
|
-
const headers = input["headers"];
|
|
121
|
-
const cookies = input["cookies"];
|
|
122
|
-
const waitForSelector = input["waitForSelector"];
|
|
123
|
-
const waitForResponse = input["waitForResponse"];
|
|
124
|
-
const maxWaitMs = input["maxWaitMs"] ?? 30000;
|
|
125
|
-
const actions = input["actions"];
|
|
126
|
-
const sources = [url, html, htmlFile].filter(Boolean);
|
|
127
|
-
if (sources.length === 0)
|
|
128
|
-
return { success: false, error: "One of url, html, or htmlFile is required" };
|
|
129
|
-
if (sources.length > 1)
|
|
130
|
-
return { success: false, error: "Provide exactly one of: url, html, or htmlFile" };
|
|
131
|
-
if (!prompt)
|
|
132
|
-
return { success: false, error: "prompt is required" };
|
|
133
|
-
try {
|
|
134
|
-
const page = await this.getPage();
|
|
135
|
-
await page.setViewportSize({ width, height });
|
|
136
|
-
// Set custom headers if provided (for auth, etc.)
|
|
137
|
-
if (headers && url) {
|
|
138
|
-
await page.setExtraHTTPHeaders(headers);
|
|
139
|
-
}
|
|
140
|
-
// Set cookies if provided
|
|
141
|
-
if (cookies && cookies.length > 0) {
|
|
142
|
-
const cookieParams = cookies.map((c) => ({
|
|
143
|
-
name: c.name,
|
|
144
|
-
value: c.value,
|
|
145
|
-
domain: c.domain,
|
|
146
|
-
path: c.path ?? "/",
|
|
147
|
-
httpOnly: c.httpOnly ?? false,
|
|
148
|
-
secure: c.secure ?? false,
|
|
149
|
-
sameSite: c.sameSite,
|
|
150
|
-
}));
|
|
151
|
-
// If url-based navigation, set cookies before navigating
|
|
152
|
-
if (url) {
|
|
153
|
-
try {
|
|
154
|
-
await page.context().addCookies(cookieParams);
|
|
155
|
-
}
|
|
156
|
-
catch {
|
|
157
|
-
// Some cookie params may be rejected by browser — non-fatal
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
// Set up response waiting before navigation (if requested)
|
|
162
|
-
let responsePromise;
|
|
163
|
-
if (waitForResponse && url) {
|
|
164
|
-
let responsePattern;
|
|
165
|
-
try {
|
|
166
|
-
responsePattern = new RegExp(waitForResponse);
|
|
167
|
-
}
|
|
168
|
-
catch {
|
|
169
|
-
return { success: false, error: `Invalid waitForResponse pattern: ${waitForResponse}. Must be a valid regex or simple string.` };
|
|
170
|
-
}
|
|
171
|
-
responsePromise = page.waitForResponse((resp) => responsePattern.test(resp.url()), { timeout: maxWaitMs }).then(() => { }).catch(() => {
|
|
172
|
-
// Timeout on response wait is non-fatal
|
|
173
|
-
});
|
|
174
|
-
}
|
|
175
|
-
if (html) {
|
|
176
|
-
await page.setContent(html, { waitUntil: "networkidle" });
|
|
177
|
-
}
|
|
178
|
-
else if (htmlFile) {
|
|
179
|
-
const { resolve, isAbsolute } = require("path");
|
|
180
|
-
const fs = require("fs");
|
|
181
|
-
const resolvedPath = isAbsolute(htmlFile) || !this.rootDir
|
|
182
|
-
? htmlFile
|
|
183
|
-
: resolve(this.rootDir, htmlFile);
|
|
184
|
-
if (!fs.existsSync(resolvedPath))
|
|
185
|
-
return { success: false, error: `File not found: ${resolvedPath}` };
|
|
186
|
-
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
|
|
187
|
-
await page.setContent(fileContent, { waitUntil: "networkidle" });
|
|
188
|
-
}
|
|
189
|
-
else if (url) {
|
|
190
|
-
await page.goto(url, { waitUntil: "networkidle" });
|
|
191
|
-
}
|
|
192
|
-
// Wait for specific selector if provided (slow-page resilience)
|
|
193
|
-
if (waitForSelector) {
|
|
194
|
-
try {
|
|
195
|
-
await page.waitForSelector(waitForSelector, { timeout: maxWaitMs, state: "visible" });
|
|
196
|
-
}
|
|
197
|
-
catch {
|
|
198
|
-
// Selector didn't appear — non-fatal, proceed with screenshot anyway
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
// Wait for response if we set it up
|
|
202
|
-
if (responsePromise) {
|
|
203
|
-
await responsePromise;
|
|
204
|
-
}
|
|
205
|
-
// Standard wait
|
|
206
|
-
await page.waitForTimeout(waitFor);
|
|
207
|
-
// Perform interactions
|
|
208
|
-
if (actions) {
|
|
209
|
-
for (const act of actions) {
|
|
210
|
-
switch (act.action) {
|
|
211
|
-
case "click": {
|
|
212
|
-
if (!act.selector)
|
|
213
|
-
break;
|
|
214
|
-
try {
|
|
215
|
-
await page.click(act.selector, { timeout: 5000 });
|
|
216
|
-
}
|
|
217
|
-
catch {
|
|
218
|
-
// Click failure is non-fatal
|
|
219
|
-
}
|
|
220
|
-
break;
|
|
221
|
-
}
|
|
222
|
-
case "type": {
|
|
223
|
-
if (!act.selector || act.value === undefined)
|
|
224
|
-
break;
|
|
225
|
-
try {
|
|
226
|
-
await page.type(act.selector, act.value);
|
|
227
|
-
}
|
|
228
|
-
catch {
|
|
229
|
-
// Type failure is non-fatal
|
|
230
|
-
}
|
|
231
|
-
break;
|
|
232
|
-
}
|
|
233
|
-
case "scroll": {
|
|
234
|
-
const amount = parseInt(act.value ?? "") || 500;
|
|
235
|
-
await page.evaluate((px) => window.scrollBy(0, px), amount);
|
|
236
|
-
break;
|
|
237
|
-
}
|
|
238
|
-
case "wait": {
|
|
239
|
-
const ms = parseInt(act.value ?? "") || 1000;
|
|
240
|
-
await page.waitForTimeout(ms);
|
|
241
|
-
break;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
let buf;
|
|
247
|
-
if (selector) {
|
|
248
|
-
const el = page.locator(selector).first();
|
|
249
|
-
buf = await el.screenshot({ type: "png" });
|
|
250
|
-
}
|
|
251
|
-
else {
|
|
252
|
-
buf = await page.screenshot({ type: "png", fullPage });
|
|
253
|
-
}
|
|
254
|
-
const base64Data = buf.toString("base64");
|
|
255
|
-
const visionMessage = (0, core_1.buildUserVisionMessage)(prompt, base64Data);
|
|
256
|
-
const response = await this.visionAdapter.chat({
|
|
257
|
-
model: model || this.defaultModel,
|
|
258
|
-
messages: [visionMessage],
|
|
259
|
-
tools: [],
|
|
260
|
-
});
|
|
261
|
-
return { success: true, output: response.message.content };
|
|
262
|
-
}
|
|
263
|
-
catch (err) {
|
|
264
|
-
return { success: false, error: String(err instanceof Error ? err.message : err) };
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
exports.VisionBrowserTool = VisionBrowserTool;
|
|
269
|
-
//# sourceMappingURL=vision-browser.tool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"vision-browser.tool.js","sourceRoot":"","sources":["../../../src/tools/workspace/vision-browser.tool.ts"],"names":[],"mappings":";;;AAAA,6BAAuB;AACvB,4CAAkH;AAElH;;;;;;;;;;GAUG;AACH,MAAa,iBAAkB,SAAQ,eAAQ;IAiB1B;IACA;IACA;IAlBV,IAAI,GAAG,eAAe,CAAA;IACtB,WAAW,GAClB,gFAAgF;QAChF,uGAAuG;QACvG,gGAAgG;QAChG,2EAA2E;QAC3E,uFAAuF;QACvF,6EAA6E;QAC7E,yFAAyF;QACzF,qFAAqF;QACrF,uEAAuE,CAAA;IAChE,SAAS,GAAG,KAAK,CAAA;IACjB,IAAI,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAC1B,QAAQ,GAAG,IAAI,CAAA;IAExB,YACmB,aAAyB,EACzB,eAAuB,YAAY,EACnC,OAAgB;QAEjC,KAAK,CACH,OAAC,CAAC,MAAM,CAAC;YACP,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;YACjE,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;YACjE,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;YAC1E,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;YACzE,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;YACnE,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1C,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;YAC1C,QAAQ,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;YAC/C,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9E,yCAAyC;YACzC,OAAO,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;iBACjD,QAAQ,CAAC,8HAA8H,CAAC;YAC3I,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,CAAC;gBACxB,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;gBAChB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE;gBACjB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;gBAC7B,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;gBACxC,QAAQ,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;gBAChC,MAAM,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;gBAC9B,QAAQ,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;aACvD,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8FAA8F,CAAC;YACvH,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;iBACnC,QAAQ,CAAC,8EAA8E;gBACtF,mEAAmE,CAAC;YACxE,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;iBACnC,QAAQ,CAAC,iFAAiF;gBACzF,yFAAyF,CAAC;YAC9F,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC5C,QAAQ,CAAC,yEAAyE,CAAC;YACtF,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;YACvF,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,CAAC;gBACxB,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnD,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;gBAC/B,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;aAC7B,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;SACrE,CAAC,CACH,CAAA;QA1CgB,kBAAa,GAAb,aAAa,CAAY;QACzB,iBAAY,GAAZ,YAAY,CAAuB;QACnC,YAAO,GAAP,OAAO,CAAS;IAyCnC,CAAC;IAEO,OAAO,GAAwC,IAAI,CAAA;IACnD,IAAI,GAAqC,IAAI,CAAA;IAE7C,KAAK,CAAC,OAAO;QACnB,iEAAiE;QACjE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,YAAY,CAAgC,CAAA;QACzE,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;QAC1D,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAA;IAClB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAA;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;QACtB,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAA8B,EAAE,IAAiB;QACzD,+DAA+D;QAC/D,IAAI,CAAC;YACH,iEAAiE;YACjE,OAAO,CAAC,YAAY,CAAC,CAAA;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,6FAA6F;aAChG,CAAA;QACH,CAAC;QAED,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAuB,CAAA;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAuB,CAAA;QAChD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAuB,CAAA;QACxD,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAW,CAAA;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAuB,CAAA;QAClD,MAAM,KAAK,GAAI,KAAK,CAAC,OAAO,CAAY,IAAI,IAAI,CAAA;QAChD,MAAM,MAAM,GAAI,KAAK,CAAC,QAAQ,CAAY,IAAI,GAAG,CAAA;QACjD,MAAM,QAAQ,GAAI,KAAK,CAAC,UAAU,CAAa,IAAI,KAAK,CAAA;QACxD,MAAM,OAAO,GAAI,KAAK,CAAC,SAAS,CAAY,IAAI,IAAI,CAAA;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAuB,CAAA;QACxD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAuC,CAAA;QACtE,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAGjB,CAAA;QACd,MAAM,eAAe,GAAG,KAAK,CAAC,iBAAiB,CAAuB,CAAA;QACtE,MAAM,eAAe,GAAG,KAAK,CAAC,iBAAiB,CAAuB,CAAA;QACtE,MAAM,SAAS,GAAI,KAAK,CAAC,WAAW,CAAY,IAAI,KAAK,CAAA;QACzD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAIjB,CAAA;QAEd,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2CAA2C,EAAE,CAAA;QACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gDAAgD,EAAE,CAAA;QAC1G,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAA;QAEnE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;YACjC,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YAE7C,kDAAkD;YAClD,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC;gBACnB,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC;YAED,0BAA0B;YAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG;oBACnB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,KAAK;oBAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,KAAK;oBACzB,QAAQ,EAAE,CAAC,CAAC,QAAiD;iBAC9D,CAAC,CAAC,CAAA;gBACH,yDAAyD;gBACzD,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,YAAmB,CAAC,CAAA;oBACtD,CAAC;oBAAC,MAAM,CAAC;wBACP,4DAA4D;oBAC9D,CAAC;gBACH,CAAC;YACH,CAAC;YAED,2DAA2D;YAC3D,IAAI,eAA0C,CAAA;YAC9C,IAAI,eAAe,IAAI,GAAG,EAAE,CAAC;gBAC3B,IAAI,eAAuB,CAAC;gBAC5B,IAAI,CAAC;oBACH,eAAe,GAAG,IAAI,MAAM,CAAC,eAAe,CAAC,CAAC;gBAChD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oCAAoC,eAAe,2CAA2C,EAAE,CAAC;gBACnI,CAAC;gBACD,eAAe,GAAG,IAAI,CAAC,eAAe,CACpC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAC1C,EAAE,OAAO,EAAE,SAAS,EAAE,CACvB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC1B,wCAAwC;gBAC1C,CAAC,CAAC,CAAA;YACJ,CAAC;YAED,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;YAC3D,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;gBACzE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;gBACzB,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO;oBACxD,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;oBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,YAAY,EAAE,EAAE,CAAA;gBACrG,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;gBAC1D,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;YAClE,CAAC;iBAAM,IAAI,GAAG,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;YACpD,CAAC;YAED,gEAAgE;YAChE,IAAI,eAAe,EAAE,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;gBACvF,CAAC;gBAAC,MAAM,CAAC;oBACP,qEAAqE;gBACvE,CAAC;YACH,CAAC;YAED,oCAAoC;YACpC,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,eAAe,CAAA;YACvB,CAAC;YAED,gBAAgB;YAChB,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YAElC,uBAAuB;YACvB,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC1B,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC;wBACnB,KAAK,OAAO,CAAC,CAAC,CAAC;4BACb,IAAI,CAAC,GAAG,CAAC,QAAQ;gCAAE,MAAK;4BACxB,IAAI,CAAC;gCACH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;4BACnD,CAAC;4BAAC,MAAM,CAAC;gCACP,6BAA6B;4BAC/B,CAAC;4BACD,MAAK;wBACP,CAAC;wBACD,KAAK,MAAM,CAAC,CAAC,CAAC;4BACZ,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS;gCAAE,MAAK;4BACnD,IAAI,CAAC;gCACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAA;4BAC1C,CAAC;4BAAC,MAAM,CAAC;gCACP,4BAA4B;4BAC9B,CAAC;4BACD,MAAK;wBACP,CAAC;wBACD,KAAK,QAAQ,CAAC,CAAC,CAAC;4BACd,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,GAAG,CAAA;4BAC/C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;4BAC3D,MAAK;wBACP,CAAC;wBACD,KAAK,MAAM,CAAC,CAAC,CAAC;4BACZ,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,IAAI,CAAA;4BAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;4BAC7B,MAAK;wBACP,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,GAAW,CAAA;YACf,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAA;gBACzC,GAAG,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACN,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;YACxD,CAAC;YAED,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACzC,MAAM,aAAa,GAAG,IAAA,6BAAsB,EAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YAEhE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;gBAC7C,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY;gBACjC,QAAQ,EAAE,CAAC,aAAa,CAAC;gBACzB,KAAK,EAAE,EAAE;aACV,CAAC,CAAA;YAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QAC5D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAA;QACpF,CAAC;IACH,CAAC;CACF;AA3QD,8CA2QC"}
|