nodebench-mcp 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -0
- package/dist/__tests__/tools.test.d.ts +1 -0
- package/dist/__tests__/tools.test.js +402 -0
- package/dist/__tests__/tools.test.js.map +1 -0
- package/dist/db.d.ts +4 -0
- package/dist/db.js +198 -0
- package/dist/db.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +237 -0
- package/dist/index.js.map +1 -0
- package/dist/tools/documentTools.d.ts +5 -0
- package/dist/tools/documentTools.js +524 -0
- package/dist/tools/documentTools.js.map +1 -0
- package/dist/tools/documentationTools.d.ts +12 -0
- package/dist/tools/documentationTools.js +647 -0
- package/dist/tools/documentationTools.js.map +1 -0
- package/dist/tools/evalTools.d.ts +6 -0
- package/dist/tools/evalTools.js +335 -0
- package/dist/tools/evalTools.js.map +1 -0
- package/dist/tools/financialTools.d.ts +10 -0
- package/dist/tools/financialTools.js +403 -0
- package/dist/tools/financialTools.js.map +1 -0
- package/dist/tools/flywheelTools.d.ts +6 -0
- package/dist/tools/flywheelTools.js +366 -0
- package/dist/tools/flywheelTools.js.map +1 -0
- package/dist/tools/githubTools.d.ts +12 -0
- package/dist/tools/githubTools.js +432 -0
- package/dist/tools/githubTools.js.map +1 -0
- package/dist/tools/learningTools.d.ts +6 -0
- package/dist/tools/learningTools.js +199 -0
- package/dist/tools/learningTools.js.map +1 -0
- package/dist/tools/memoryTools.d.ts +5 -0
- package/dist/tools/memoryTools.js +137 -0
- package/dist/tools/memoryTools.js.map +1 -0
- package/dist/tools/metaTools.d.ts +7 -0
- package/dist/tools/metaTools.js +837 -0
- package/dist/tools/metaTools.js.map +1 -0
- package/dist/tools/planningTools.d.ts +5 -0
- package/dist/tools/planningTools.js +147 -0
- package/dist/tools/planningTools.js.map +1 -0
- package/dist/tools/qualityGateTools.d.ts +6 -0
- package/dist/tools/qualityGateTools.js +347 -0
- package/dist/tools/qualityGateTools.js.map +1 -0
- package/dist/tools/reconTools.d.ts +8 -0
- package/dist/tools/reconTools.js +729 -0
- package/dist/tools/reconTools.js.map +1 -0
- package/dist/tools/searchTools.d.ts +5 -0
- package/dist/tools/searchTools.js +145 -0
- package/dist/tools/searchTools.js.map +1 -0
- package/dist/tools/uiCaptureTools.d.ts +8 -0
- package/dist/tools/uiCaptureTools.js +339 -0
- package/dist/tools/uiCaptureTools.js.map +1 -0
- package/dist/tools/verificationTools.d.ts +6 -0
- package/dist/tools/verificationTools.js +472 -0
- package/dist/tools/verificationTools.js.map +1 -0
- package/dist/tools/visionTools.d.ts +12 -0
- package/dist/tools/visionTools.js +553 -0
- package/dist/tools/visionTools.js.map +1 -0
- package/dist/tools/webTools.d.ts +12 -0
- package/dist/tools/webTools.js +443 -0
- package/dist/tools/webTools.js.map +1 -0
- package/dist/types.d.ts +16 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +66 -0
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision tools — AI-powered visual analysis, environment discovery, and image manipulation.
|
|
3
|
+
* Enables the agentic vision loop: Capture → Analyze → Manipulate → Iterate → Gate.
|
|
4
|
+
*
|
|
5
|
+
* - discover_vision_env: Scans for available API keys and SDKs
|
|
6
|
+
* - analyze_screenshot: Sends screenshots to vision models for analysis
|
|
7
|
+
* - manipulate_screenshot: Crop, resize, annotate images using sharp
|
|
8
|
+
*
|
|
9
|
+
* All AI SDKs and sharp are optional dependencies — tools fail gracefully if missing.
|
|
10
|
+
*/
|
|
11
|
+
import { join } from "path";
|
|
12
|
+
import { homedir } from "os";
|
|
13
|
+
import { mkdirSync, existsSync, writeFileSync } from "fs";
|
|
14
|
+
const CAPTURE_DIR = join(homedir(), ".nodebench", "captures");
|
|
15
|
+
function ensureCaptureDir() {
|
|
16
|
+
if (!existsSync(CAPTURE_DIR)) {
|
|
17
|
+
mkdirSync(CAPTURE_DIR, { recursive: true });
|
|
18
|
+
}
|
|
19
|
+
return CAPTURE_DIR;
|
|
20
|
+
}
|
|
21
|
+
// ─── Dynamic import helpers ───────────────────────────────────────────────────
|
|
22
|
+
async function canImport(pkg) {
|
|
23
|
+
try {
|
|
24
|
+
await import(pkg);
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function getSharp() {
|
|
32
|
+
try {
|
|
33
|
+
const mod = await import("sharp");
|
|
34
|
+
return mod.default ?? mod;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function escapeXml(s) {
|
|
41
|
+
return s
|
|
42
|
+
.replace(/&/g, "&")
|
|
43
|
+
.replace(/</g, "<")
|
|
44
|
+
.replace(/>/g, ">")
|
|
45
|
+
.replace(/"/g, """)
|
|
46
|
+
.replace(/'/g, "'");
|
|
47
|
+
}
|
|
48
|
+
// ─── Default analysis prompt ──────────────────────────────────────────────────
|
|
49
|
+
const DEFAULT_ANALYSIS_PROMPT = `Analyze this UI screenshot for quality issues. Evaluate:
|
|
50
|
+
|
|
51
|
+
1. LAYOUT: Is the layout balanced? Any overlapping elements, broken grids, or misaligned components?
|
|
52
|
+
2. SPACING: Is whitespace consistent? Any cramped or overly sparse areas?
|
|
53
|
+
3. TYPOGRAPHY: Are font sizes readable? Is there clear visual hierarchy (headings, body, captions)?
|
|
54
|
+
4. COLOR & CONTRAST: Are text/background combinations readable? Does it follow WCAG 2.1 AA contrast ratios?
|
|
55
|
+
5. RESPONSIVENESS: Does the layout look appropriate for its viewport width?
|
|
56
|
+
6. COMPONENT STATES: Are there visible loading spinners, error states, or empty states that look broken?
|
|
57
|
+
7. VISUAL CONSISTENCY: Do colors, borders, shadows, and rounding match a consistent design system?
|
|
58
|
+
8. ACCESSIBILITY: Are interactive elements visually distinct? Are focus indicators visible?
|
|
59
|
+
|
|
60
|
+
For each issue found, describe:
|
|
61
|
+
- What the issue is
|
|
62
|
+
- Where it is (describe the location in the screenshot)
|
|
63
|
+
- Severity: CRITICAL (broken/unusable), HIGH (visually wrong), MEDIUM (suboptimal), LOW (nitpick)
|
|
64
|
+
- Suggested fix
|
|
65
|
+
|
|
66
|
+
End with a summary: total issues by severity, overall quality score (1-10), and top 3 action items.`;
|
|
67
|
+
// ─── Provider implementations ─────────────────────────────────────────────────
|
|
68
|
+
async function analyzeWithGemini(imageBase64, prompt) {
|
|
69
|
+
const { GoogleGenAI } = await import("@google/genai");
|
|
70
|
+
const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || "";
|
|
71
|
+
const ai = new GoogleGenAI({ apiKey });
|
|
72
|
+
const response = await ai.models.generateContent({
|
|
73
|
+
model: "gemini-2.5-flash",
|
|
74
|
+
contents: [
|
|
75
|
+
{
|
|
76
|
+
role: "user",
|
|
77
|
+
parts: [
|
|
78
|
+
{ text: prompt },
|
|
79
|
+
{ inlineData: { mimeType: "image/png", data: imageBase64 } },
|
|
80
|
+
],
|
|
81
|
+
},
|
|
82
|
+
],
|
|
83
|
+
config: {
|
|
84
|
+
tools: [{ codeExecution: {} }],
|
|
85
|
+
maxOutputTokens: 8192,
|
|
86
|
+
temperature: 0.2,
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
// Extract text and any generated images from code execution
|
|
90
|
+
const parts = response?.candidates?.[0]?.content?.parts ?? [];
|
|
91
|
+
const textParts = [];
|
|
92
|
+
const images = [];
|
|
93
|
+
for (const part of parts) {
|
|
94
|
+
if (part.text)
|
|
95
|
+
textParts.push(part.text);
|
|
96
|
+
if (part.executableCode)
|
|
97
|
+
textParts.push(`\`\`\`python\n${part.executableCode.code}\n\`\`\``);
|
|
98
|
+
if (part.codeExecutionResult)
|
|
99
|
+
textParts.push(`Output: ${part.codeExecutionResult.output}`);
|
|
100
|
+
if (part.inlineData?.data)
|
|
101
|
+
images.push(part.inlineData.data);
|
|
102
|
+
}
|
|
103
|
+
return { text: textParts.join("\n\n"), images };
|
|
104
|
+
}
|
|
105
|
+
async function analyzeWithOpenAI(imageBase64, prompt) {
|
|
106
|
+
const OpenAI = (await import("openai")).default;
|
|
107
|
+
const client = new OpenAI();
|
|
108
|
+
const response = await client.chat.completions.create({
|
|
109
|
+
model: "gpt-4o",
|
|
110
|
+
messages: [
|
|
111
|
+
{
|
|
112
|
+
role: "user",
|
|
113
|
+
content: [
|
|
114
|
+
{ type: "text", text: prompt },
|
|
115
|
+
{
|
|
116
|
+
type: "image_url",
|
|
117
|
+
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
118
|
+
},
|
|
119
|
+
],
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
max_tokens: 4096,
|
|
123
|
+
});
|
|
124
|
+
return { text: response.choices[0]?.message?.content ?? "" };
|
|
125
|
+
}
|
|
126
|
+
async function analyzeWithAnthropic(imageBase64, prompt) {
|
|
127
|
+
const Anthropic = (await import("@anthropic-ai/sdk")).default;
|
|
128
|
+
const client = new Anthropic();
|
|
129
|
+
const response = await client.messages.create({
|
|
130
|
+
model: "claude-sonnet-4-20250514",
|
|
131
|
+
max_tokens: 4096,
|
|
132
|
+
messages: [
|
|
133
|
+
{
|
|
134
|
+
role: "user",
|
|
135
|
+
content: [
|
|
136
|
+
{
|
|
137
|
+
type: "image",
|
|
138
|
+
source: {
|
|
139
|
+
type: "base64",
|
|
140
|
+
media_type: "image/png",
|
|
141
|
+
data: imageBase64,
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
{ type: "text", text: prompt },
|
|
145
|
+
],
|
|
146
|
+
},
|
|
147
|
+
],
|
|
148
|
+
});
|
|
149
|
+
const text = response.content
|
|
150
|
+
.filter((b) => b.type === "text")
|
|
151
|
+
.map((b) => b.text)
|
|
152
|
+
.join("\n");
|
|
153
|
+
return { text };
|
|
154
|
+
}
|
|
155
|
+
async function analyzeWithOpenRouter(imageBase64, prompt) {
|
|
156
|
+
const OpenAI = (await import("openai")).default;
|
|
157
|
+
const client = new OpenAI({
|
|
158
|
+
apiKey: process.env.OPENROUTER_API_KEY,
|
|
159
|
+
baseURL: process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1",
|
|
160
|
+
});
|
|
161
|
+
const response = await client.chat.completions.create({
|
|
162
|
+
model: "google/gemini-2.5-flash",
|
|
163
|
+
messages: [
|
|
164
|
+
{
|
|
165
|
+
role: "user",
|
|
166
|
+
content: [
|
|
167
|
+
{ type: "text", text: prompt },
|
|
168
|
+
{
|
|
169
|
+
type: "image_url",
|
|
170
|
+
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
171
|
+
},
|
|
172
|
+
],
|
|
173
|
+
},
|
|
174
|
+
],
|
|
175
|
+
max_tokens: 4096,
|
|
176
|
+
});
|
|
177
|
+
return { text: response.choices[0]?.message?.content ?? "" };
|
|
178
|
+
}
|
|
179
|
+
// ─── Tools ────────────────────────────────────────────────────────────────────
|
|
180
|
+
export const visionTools = [
|
|
181
|
+
{
|
|
182
|
+
name: "discover_vision_env",
|
|
183
|
+
description: "Discover available vision-capable AI SDKs and API keys in the current environment. Returns which providers can be used for screenshot analysis. Call this before analyze_screenshot to know what's available. Checks: GEMINI_API_KEY (agentic vision with code execution), OPENAI_API_KEY (GPT-4o vision), ANTHROPIC_API_KEY (Claude vision), OPENROUTER_API_KEY. Also checks for sharp (image manipulation) and playwright (screenshot capture).",
|
|
184
|
+
inputSchema: {
|
|
185
|
+
type: "object",
|
|
186
|
+
properties: {},
|
|
187
|
+
},
|
|
188
|
+
handler: async () => {
|
|
189
|
+
const keys = {
|
|
190
|
+
GEMINI_API_KEY: !!process.env.GEMINI_API_KEY,
|
|
191
|
+
GOOGLE_AI_API_KEY: !!process.env.GOOGLE_AI_API_KEY,
|
|
192
|
+
OPENAI_API_KEY: !!process.env.OPENAI_API_KEY,
|
|
193
|
+
ANTHROPIC_API_KEY: !!process.env.ANTHROPIC_API_KEY,
|
|
194
|
+
OPENROUTER_API_KEY: !!process.env.OPENROUTER_API_KEY,
|
|
195
|
+
};
|
|
196
|
+
const sdks = {
|
|
197
|
+
googleGenai: await canImport("@google/genai"),
|
|
198
|
+
openai: await canImport("openai"),
|
|
199
|
+
anthropic: await canImport("@anthropic-ai/sdk"),
|
|
200
|
+
sharp: await canImport("sharp"),
|
|
201
|
+
playwright: await canImport("playwright"),
|
|
202
|
+
};
|
|
203
|
+
const providers = [];
|
|
204
|
+
if ((keys.GEMINI_API_KEY || keys.GOOGLE_AI_API_KEY) && sdks.googleGenai) {
|
|
205
|
+
providers.push({
|
|
206
|
+
name: "gemini",
|
|
207
|
+
model: "gemini-2.5-flash",
|
|
208
|
+
priority: 1,
|
|
209
|
+
features: ["vision", "code_execution", "agentic_vision"],
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (keys.OPENAI_API_KEY && sdks.openai) {
|
|
213
|
+
providers.push({
|
|
214
|
+
name: "openai",
|
|
215
|
+
model: "gpt-4o",
|
|
216
|
+
priority: 2,
|
|
217
|
+
features: ["vision"],
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
if (keys.ANTHROPIC_API_KEY && sdks.anthropic) {
|
|
221
|
+
providers.push({
|
|
222
|
+
name: "anthropic",
|
|
223
|
+
model: "claude-sonnet-4-20250514",
|
|
224
|
+
priority: 3,
|
|
225
|
+
features: ["vision"],
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
if (keys.OPENROUTER_API_KEY && sdks.openai) {
|
|
229
|
+
providers.push({
|
|
230
|
+
name: "openrouter",
|
|
231
|
+
model: "google/gemini-2.5-flash",
|
|
232
|
+
priority: 4,
|
|
233
|
+
features: ["vision"],
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
providers.sort((a, b) => a.priority - b.priority);
|
|
237
|
+
return {
|
|
238
|
+
apiKeys: keys,
|
|
239
|
+
sdks,
|
|
240
|
+
providers,
|
|
241
|
+
bestProvider: providers[0] ?? null,
|
|
242
|
+
canAnalyze: providers.length > 0,
|
|
243
|
+
canManipulate: sdks.sharp,
|
|
244
|
+
canCapture: sdks.playwright,
|
|
245
|
+
recommendation: providers.length === 0
|
|
246
|
+
? "No vision providers available. Set an API key and install the SDK. Recommended: npm install @google/genai && set GEMINI_API_KEY for agentic vision with code execution."
|
|
247
|
+
: `Best provider: ${providers[0].name} (${providers[0].model}). ${providers[0].features.includes("code_execution") ? "Supports agentic vision with code execution (zoom, crop, compute)." : "Standard vision analysis."}`,
|
|
248
|
+
};
|
|
249
|
+
},
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
name: "analyze_screenshot",
|
|
253
|
+
description: "Send a screenshot to a vision-capable AI model for analysis. Accepts base64 image data (from capture_ui_screenshot) and returns the model's analysis of layout, spacing, typography, accessibility, and visual quality. Auto-selects the best available provider — Gemini with code execution (agentic vision: zoom, crop, annotate, compute) > OpenAI GPT-4o > Anthropic Claude > OpenRouter. Call discover_vision_env first to check availability.",
|
|
254
|
+
rawContent: true,
|
|
255
|
+
inputSchema: {
|
|
256
|
+
type: "object",
|
|
257
|
+
properties: {
|
|
258
|
+
imageBase64: {
|
|
259
|
+
type: "string",
|
|
260
|
+
description: "Base64-encoded PNG image data (from capture_ui_screenshot output)",
|
|
261
|
+
},
|
|
262
|
+
prompt: {
|
|
263
|
+
type: "string",
|
|
264
|
+
description: "Custom analysis prompt. Default: comprehensive UI/UX review covering layout, spacing, typography, color, accessibility, responsiveness.",
|
|
265
|
+
},
|
|
266
|
+
provider: {
|
|
267
|
+
type: "string",
|
|
268
|
+
enum: ["auto", "gemini", "openai", "anthropic", "openrouter"],
|
|
269
|
+
description: "Which vision provider to use. Default: 'auto' (selects best available).",
|
|
270
|
+
},
|
|
271
|
+
context: {
|
|
272
|
+
type: "string",
|
|
273
|
+
description: "Additional context about what was captured (component name, expected behavior, known issues).",
|
|
274
|
+
},
|
|
275
|
+
},
|
|
276
|
+
required: ["imageBase64"],
|
|
277
|
+
},
|
|
278
|
+
handler: async (args) => {
|
|
279
|
+
const providerChoice = args.provider ?? "auto";
|
|
280
|
+
const analysisPrompt = args.context
|
|
281
|
+
? `Context: ${args.context}\n\n${args.prompt ?? DEFAULT_ANALYSIS_PROMPT}`
|
|
282
|
+
: args.prompt ?? DEFAULT_ANALYSIS_PROMPT;
|
|
283
|
+
let selectedProvider = null;
|
|
284
|
+
if (providerChoice !== "auto") {
|
|
285
|
+
selectedProvider = providerChoice;
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
// Auto-select: Gemini > OpenAI > Anthropic > OpenRouter
|
|
289
|
+
if ((process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY) &&
|
|
290
|
+
(await canImport("@google/genai"))) {
|
|
291
|
+
selectedProvider = "gemini";
|
|
292
|
+
}
|
|
293
|
+
else if (process.env.OPENAI_API_KEY &&
|
|
294
|
+
(await canImport("openai"))) {
|
|
295
|
+
selectedProvider = "openai";
|
|
296
|
+
}
|
|
297
|
+
else if (process.env.ANTHROPIC_API_KEY &&
|
|
298
|
+
(await canImport("@anthropic-ai/sdk"))) {
|
|
299
|
+
selectedProvider = "anthropic";
|
|
300
|
+
}
|
|
301
|
+
else if (process.env.OPENROUTER_API_KEY &&
|
|
302
|
+
(await canImport("openai"))) {
|
|
303
|
+
selectedProvider = "openrouter";
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (!selectedProvider) {
|
|
307
|
+
return [
|
|
308
|
+
{
|
|
309
|
+
type: "text",
|
|
310
|
+
text: JSON.stringify({
|
|
311
|
+
error: true,
|
|
312
|
+
message: "No vision provider available. Call discover_vision_env to see what's needed.",
|
|
313
|
+
suggestion: "Set one of: GEMINI_API_KEY (recommended), OPENAI_API_KEY, ANTHROPIC_API_KEY, or OPENROUTER_API_KEY. " +
|
|
314
|
+
"Also install the corresponding SDK: @google/genai, openai, or @anthropic-ai/sdk.",
|
|
315
|
+
}),
|
|
316
|
+
},
|
|
317
|
+
];
|
|
318
|
+
}
|
|
319
|
+
try {
|
|
320
|
+
let result;
|
|
321
|
+
switch (selectedProvider) {
|
|
322
|
+
case "gemini":
|
|
323
|
+
result = await analyzeWithGemini(args.imageBase64, analysisPrompt);
|
|
324
|
+
break;
|
|
325
|
+
case "openai":
|
|
326
|
+
result = await analyzeWithOpenAI(args.imageBase64, analysisPrompt);
|
|
327
|
+
break;
|
|
328
|
+
case "anthropic":
|
|
329
|
+
result = await analyzeWithAnthropic(args.imageBase64, analysisPrompt);
|
|
330
|
+
break;
|
|
331
|
+
case "openrouter":
|
|
332
|
+
result = await analyzeWithOpenRouter(args.imageBase64, analysisPrompt);
|
|
333
|
+
break;
|
|
334
|
+
}
|
|
335
|
+
const content = [
|
|
336
|
+
{
|
|
337
|
+
type: "text",
|
|
338
|
+
text: JSON.stringify({
|
|
339
|
+
provider: selectedProvider,
|
|
340
|
+
model: selectedProvider === "gemini"
|
|
341
|
+
? "gemini-2.5-flash"
|
|
342
|
+
: selectedProvider === "openai"
|
|
343
|
+
? "gpt-4o"
|
|
344
|
+
: selectedProvider === "anthropic"
|
|
345
|
+
? "claude-sonnet-4-20250514"
|
|
346
|
+
: "google/gemini-2.5-flash",
|
|
347
|
+
agenticVision: selectedProvider === "gemini"
|
|
348
|
+
? "enabled (code execution active)"
|
|
349
|
+
: "not available (standard vision only)",
|
|
350
|
+
annotatedImagesCount: result.images?.length ?? 0,
|
|
351
|
+
}),
|
|
352
|
+
},
|
|
353
|
+
{ type: "text", text: result.text },
|
|
354
|
+
];
|
|
355
|
+
// Append any annotated images from Gemini code execution
|
|
356
|
+
if (result.images && result.images.length > 0) {
|
|
357
|
+
for (const img of result.images) {
|
|
358
|
+
content.push({
|
|
359
|
+
type: "image",
|
|
360
|
+
data: img,
|
|
361
|
+
mimeType: "image/png",
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
return content;
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
return [
|
|
369
|
+
{
|
|
370
|
+
type: "text",
|
|
371
|
+
text: JSON.stringify({
|
|
372
|
+
error: true,
|
|
373
|
+
provider: selectedProvider,
|
|
374
|
+
message: `Vision analysis failed: ${err.message}`,
|
|
375
|
+
suggestion: "Check that the API key is valid and the SDK is installed. Try a different provider with provider='openai' or provider='anthropic'.",
|
|
376
|
+
}),
|
|
377
|
+
},
|
|
378
|
+
];
|
|
379
|
+
}
|
|
380
|
+
},
|
|
381
|
+
},
|
|
382
|
+
{
|
|
383
|
+
name: "manipulate_screenshot",
|
|
384
|
+
description: "Manipulate a screenshot using sharp (image processing). Supports crop (extract a region), resize, and annotate (draw colored rectangles and labels to highlight areas). Use after analyze_screenshot identifies regions of interest, or to prepare focused crops for deeper analysis. Returns the processed image as base64. Requires sharp to be installed.",
|
|
385
|
+
rawContent: true,
|
|
386
|
+
inputSchema: {
|
|
387
|
+
type: "object",
|
|
388
|
+
properties: {
|
|
389
|
+
imageBase64: {
|
|
390
|
+
type: "string",
|
|
391
|
+
description: "Base64-encoded PNG image to manipulate",
|
|
392
|
+
},
|
|
393
|
+
operation: {
|
|
394
|
+
type: "string",
|
|
395
|
+
enum: ["crop", "resize", "annotate"],
|
|
396
|
+
description: "Operation: crop (extract region), resize (change dimensions), annotate (draw rectangles/labels)",
|
|
397
|
+
},
|
|
398
|
+
x: { type: "number", description: "Crop: left offset in pixels" },
|
|
399
|
+
y: { type: "number", description: "Crop: top offset in pixels" },
|
|
400
|
+
cropWidth: {
|
|
401
|
+
type: "number",
|
|
402
|
+
description: "Crop: width of region in pixels",
|
|
403
|
+
},
|
|
404
|
+
cropHeight: {
|
|
405
|
+
type: "number",
|
|
406
|
+
description: "Crop: height of region in pixels",
|
|
407
|
+
},
|
|
408
|
+
width: { type: "number", description: "Resize: target width" },
|
|
409
|
+
height: { type: "number", description: "Resize: target height" },
|
|
410
|
+
annotations: {
|
|
411
|
+
type: "array",
|
|
412
|
+
description: "Annotate: rectangles and labels to draw on the image",
|
|
413
|
+
items: {
|
|
414
|
+
type: "object",
|
|
415
|
+
properties: {
|
|
416
|
+
x: { type: "number", description: "Left offset" },
|
|
417
|
+
y: { type: "number", description: "Top offset" },
|
|
418
|
+
width: { type: "number", description: "Rectangle width" },
|
|
419
|
+
height: { type: "number", description: "Rectangle height" },
|
|
420
|
+
color: {
|
|
421
|
+
type: "string",
|
|
422
|
+
description: "CSS color for the rectangle (default: red)",
|
|
423
|
+
},
|
|
424
|
+
label: {
|
|
425
|
+
type: "string",
|
|
426
|
+
description: "Text label above the rectangle",
|
|
427
|
+
},
|
|
428
|
+
},
|
|
429
|
+
required: ["x", "y", "width", "height"],
|
|
430
|
+
},
|
|
431
|
+
},
|
|
432
|
+
label: {
|
|
433
|
+
type: "string",
|
|
434
|
+
description: "Label for the output (used in saved filename, e.g. 'cropped-nav-mobile')",
|
|
435
|
+
},
|
|
436
|
+
},
|
|
437
|
+
required: ["imageBase64", "operation"],
|
|
438
|
+
},
|
|
439
|
+
handler: async (args) => {
|
|
440
|
+
const sharp = await getSharp();
|
|
441
|
+
if (!sharp) {
|
|
442
|
+
return [
|
|
443
|
+
{
|
|
444
|
+
type: "text",
|
|
445
|
+
text: JSON.stringify({
|
|
446
|
+
error: true,
|
|
447
|
+
message: "sharp is not installed. Install it with: npm install sharp",
|
|
448
|
+
suggestion: "The manipulate_screenshot tool requires sharp for image processing.",
|
|
449
|
+
}),
|
|
450
|
+
},
|
|
451
|
+
];
|
|
452
|
+
}
|
|
453
|
+
const inputBuffer = Buffer.from(args.imageBase64, "base64");
|
|
454
|
+
try {
|
|
455
|
+
let outputBuffer;
|
|
456
|
+
switch (args.operation) {
|
|
457
|
+
case "crop": {
|
|
458
|
+
if (args.x == null ||
|
|
459
|
+
args.y == null ||
|
|
460
|
+
!args.cropWidth ||
|
|
461
|
+
!args.cropHeight) {
|
|
462
|
+
throw new Error("Crop requires x, y, cropWidth, and cropHeight parameters");
|
|
463
|
+
}
|
|
464
|
+
outputBuffer = await sharp(inputBuffer)
|
|
465
|
+
.extract({
|
|
466
|
+
left: Math.round(args.x),
|
|
467
|
+
top: Math.round(args.y),
|
|
468
|
+
width: Math.round(args.cropWidth),
|
|
469
|
+
height: Math.round(args.cropHeight),
|
|
470
|
+
})
|
|
471
|
+
.png()
|
|
472
|
+
.toBuffer();
|
|
473
|
+
break;
|
|
474
|
+
}
|
|
475
|
+
case "resize": {
|
|
476
|
+
if (!args.width && !args.height) {
|
|
477
|
+
throw new Error("Resize requires at least one of width or height");
|
|
478
|
+
}
|
|
479
|
+
outputBuffer = await sharp(inputBuffer)
|
|
480
|
+
.resize(args.width ?? null, args.height ?? null, {
|
|
481
|
+
fit: "inside",
|
|
482
|
+
})
|
|
483
|
+
.png()
|
|
484
|
+
.toBuffer();
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
case "annotate": {
|
|
488
|
+
if (!args.annotations || args.annotations.length === 0) {
|
|
489
|
+
throw new Error("Annotate requires at least one annotation in the annotations array");
|
|
490
|
+
}
|
|
491
|
+
const metadata = await sharp(inputBuffer).metadata();
|
|
492
|
+
const w = metadata.width ?? 1280;
|
|
493
|
+
const h = metadata.height ?? 800;
|
|
494
|
+
const svgParts = args.annotations.map((a) => {
|
|
495
|
+
const color = a.color || "red";
|
|
496
|
+
const labelSvg = a.label
|
|
497
|
+
? `<text x="${a.x + 2}" y="${a.y - 6}" fill="${color}" font-size="14" font-family="Arial, sans-serif" font-weight="bold">${escapeXml(a.label)}</text>`
|
|
498
|
+
: "";
|
|
499
|
+
return `${labelSvg}<rect x="${a.x}" y="${a.y}" width="${a.width}" height="${a.height}" fill="none" stroke="${color}" stroke-width="3"/>`;
|
|
500
|
+
});
|
|
501
|
+
const svgOverlay = Buffer.from(`<svg width="${w}" height="${h}" xmlns="http://www.w3.org/2000/svg">${svgParts.join("")}</svg>`);
|
|
502
|
+
outputBuffer = await sharp(inputBuffer)
|
|
503
|
+
.composite([{ input: svgOverlay, top: 0, left: 0 }])
|
|
504
|
+
.png()
|
|
505
|
+
.toBuffer();
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
default:
|
|
509
|
+
throw new Error(`Unknown operation: ${args.operation}. Use crop, resize, or annotate.`);
|
|
510
|
+
}
|
|
511
|
+
const outputBase64 = outputBuffer.toString("base64");
|
|
512
|
+
// Save to captures dir
|
|
513
|
+
const captureDir = ensureCaptureDir();
|
|
514
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
515
|
+
const labelSlug = args.label
|
|
516
|
+
? args.label.replace(/[^a-zA-Z0-9-_]/g, "-").toLowerCase()
|
|
517
|
+
: args.operation;
|
|
518
|
+
const filename = `${labelSlug}_${timestamp}.png`;
|
|
519
|
+
const filepath = join(captureDir, filename);
|
|
520
|
+
writeFileSync(filepath, outputBuffer);
|
|
521
|
+
return [
|
|
522
|
+
{
|
|
523
|
+
type: "text",
|
|
524
|
+
text: JSON.stringify({
|
|
525
|
+
operation: args.operation,
|
|
526
|
+
label: args.label ?? args.operation,
|
|
527
|
+
filepath,
|
|
528
|
+
outputSizeBytes: outputBuffer.length,
|
|
529
|
+
}),
|
|
530
|
+
},
|
|
531
|
+
{
|
|
532
|
+
type: "image",
|
|
533
|
+
data: outputBase64,
|
|
534
|
+
mimeType: "image/png",
|
|
535
|
+
},
|
|
536
|
+
];
|
|
537
|
+
}
|
|
538
|
+
catch (err) {
|
|
539
|
+
return [
|
|
540
|
+
{
|
|
541
|
+
type: "text",
|
|
542
|
+
text: JSON.stringify({
|
|
543
|
+
error: true,
|
|
544
|
+
operation: args.operation,
|
|
545
|
+
message: `Image manipulation failed: ${err.message}`,
|
|
546
|
+
}),
|
|
547
|
+
},
|
|
548
|
+
];
|
|
549
|
+
}
|
|
550
|
+
},
|
|
551
|
+
},
|
|
552
|
+
];
|
|
553
|
+
//# sourceMappingURL=visionTools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"visionTools.js","sourceRoot":"","sources":["../../src/tools/visionTools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAG1D,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;AAE9D,SAAS,gBAAgB;IACvB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,SAAS,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ;IACrB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC;SACL,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,iFAAiF;AAEjF,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;oGAiBoE,CAAC;AAErG,iFAAiF;AAEjF,KAAK,UAAU,iBAAiB,CAC9B,WAAmB,EACnB,MAAc;IAEd,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IACtD,MAAM,MAAM,GACV,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;IACpE,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;QAC/C,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAe;gBACrB,KAAK,EAAE;oBACL,EAAE,IAAI,EAAE,MAAM,EAAE;oBAChB,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE;iBAC7D;aACF;SACF;QACD,MAAM,EAAE;YACN,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;YAC9B,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,GAAG;SACjB;KACF,CAAC,CAAC;IAEH,4DAA4D;IAC5D,MAAM,KAAK,GAAI,QAAgB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACvE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,cAAc;YACrB,SAAS,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,cAAc,CAAC,IAAI,UAAU,CAAC,CAAC;QACtE,IAAI,IAAI,CAAC,mBAAmB;YAC1B,SAAS,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI;YAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,WAAmB,EACnB,MAAc;IAEd,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAE5B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QACpD,KAAK,EAAE,QAAQ;QACf,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;oBAC9B;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,WAAW,EAAE,EAAE;qBAC3D;iBACF;aACF;SACF;QACD,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;AAC/D,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,WAAmB,EACnB,MAAc;IAEd,MAAM,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC;IAC9D,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;IAE/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,KAAK,EAAE,0BAA0B;QACjC,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE,WAAW;4BACvB,IAAI,EAAE,WAAW;yBAClB;qBACF;oBACD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;iBAC/B;aACF;SACF;KACF,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO;SAC1B,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SACrC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,EAAE,IAAI,EAAE,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,WAAmB,EACnB,MAAc;IAEd,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACxB,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,8BAA8B;KAC3E,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QACpD,KAAK,EAAE,yBAAyB;QAChC,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;oBAC9B;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,WAAW,EAAE,EAAE;qBAC3D;iBACF;aACF;SACF;QACD,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;AAC/D,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,MAAM,WAAW,GAAc;IACpC;QACE,IAAI,EAAE,qBAAqB;QAC3B,WAAW,EACT,mbAAmb;QACrb,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE;SACf;QACD,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,MAAM,IAAI,GAAG;gBACX,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC5C,iBAAiB,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB;gBAClD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC5C,iBAAiB,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB;gBAClD,kBAAkB,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB;aACrD,CAAC;YAEF,MAAM,IAAI,GAAG;gBACX,WAAW,EAAE,MAAM,SAAS,CAAC,eAAe,CAAC;gBAC7C,MAAM,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,MAAM,SAAS,CAAC,mBAAmB,CAAC;gBAC/C,KAAK,EAAE,MAAM,SAAS,CAAC,OAAO,CAAC;gBAC/B,UAAU,EAAE,MAAM,SAAS,CAAC,YAAY,CAAC;aAC1C,CAAC;YAEF,MAAM,SAAS,GAKV,EAAE,CAAC;YAER,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxE,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,kBAAkB;oBACzB,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,QAAQ,EAAE,gBAAgB,EAAE,gBAAgB,CAAC;iBACzD,CAAC,CAAC;YACL,CAAC;YACD,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvC,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,QAAQ;oBACf,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,QAAQ,CAAC;iBACrB,CAAC,CAAC;YACL,CAAC;YACD,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7C,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,0BAA0B;oBACjC,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,QAAQ,CAAC;iBACrB,CAAC,CAAC;YACL,CAAC;YACD,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3C,SAAS,CAAC,IAAI,CAAC;oBACb,IAAI,EAAE,YAAY;oBAClB,KAAK,EAAE,yBAAyB;oBAChC,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC,QAAQ,CAAC;iBACrB,CAAC,CAAC;YACL,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAElD,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI;gBACJ,SAAS;gBACT,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI;gBAClC,UAAU,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC;gBAChC,aAAa,EAAE,IAAI,CAAC,KAAK;gBACzB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,cAAc,EACZ,SAAS,CAAC,MAAM,KAAK,CAAC;oBACpB,CAAC,CAAC,yKAAyK;oBAC3K,CAAC,CAAC,kBAAkB,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,oEAAoE,CAAC,CAAC,CAAC,2BAA2B,EAAE;aAC9N,CAAC;QACJ,CAAC;KACF;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EACT,sbAAsb;QACxb,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,mEAAmE;iBACtE;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,yIAAyI;iBAC5I;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,CAAC;oBAC7D,WAAW,EACT,yEAAyE;iBAC5E;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,+FAA+F;iBAClG;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,CAAC;SAC1B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAA2B,EAAE;YAC/C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC;YAC/C,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO;gBACjC,CAAC,CAAC,YAAY,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,MAAM,IAAI,uBAAuB,EAAE;gBACzE,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,uBAAuB,CAAC;YAI3C,IAAI,gBAAgB,GAAwB,IAAI,CAAC;YAEjD,IAAI,cAAc,KAAK,MAAM,EAAE,CAAC;gBAC9B,gBAAgB,GAAG,cAA8B,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,IACE,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;oBAC7D,CAAC,MAAM,SAAS,CAAC,eAAe,CAAC,CAAC,EAClC,CAAC;oBACD,gBAAgB,GAAG,QAAQ,CAAC;gBAC9B,CAAC;qBAAM,IACL,OAAO,CAAC,GAAG,CAAC,cAAc;oBAC1B,CAAC,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,EAC3B,CAAC;oBACD,gBAAgB,GAAG,QAAQ,CAAC;gBAC9B,CAAC;qBAAM,IACL,OAAO,CAAC,GAAG,CAAC,iBAAiB;oBAC7B,CAAC,MAAM,SAAS,CAAC,mBAAmB,CAAC,CAAC,EACtC,CAAC;oBACD,gBAAgB,GAAG,WAAW,CAAC;gBACjC,CAAC;qBAAM,IACL,OAAO,CAAC,GAAG,CAAC,kBAAkB;oBAC9B,CAAC,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,EAC3B,CAAC;oBACD,gBAAgB,GAAG,YAAY,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,KAAK,EAAE,IAAI;4BACX,OAAO,EACL,8EAA8E;4BAChF,UAAU,EACR,sGAAsG;gCACtG,kFAAkF;yBACrF,CAAC;qBACH;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,MAA2C,CAAC;gBAEhD,QAAQ,gBAAgB,EAAE,CAAC;oBACzB,KAAK,QAAQ;wBACX,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;wBACnE,MAAM;oBACR,KAAK,QAAQ;wBACX,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;wBACnE,MAAM;oBACR,KAAK,WAAW;wBACd,MAAM,GAAG,MAAM,oBAAoB,CACjC,IAAI,CAAC,WAAW,EAChB,cAAc,CACf,CAAC;wBACF,MAAM;oBACR,KAAK,YAAY;wBACf,MAAM,GAAG,MAAM,qBAAqB,CAClC,IAAI,CAAC,WAAW,EAChB,cAAc,CACf,CAAC;wBACF,MAAM;gBACV,CAAC;gBAED,MAAM,OAAO,GAAmB;oBAC9B;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,QAAQ,EAAE,gBAAgB;4BAC1B,KAAK,EACH,gBAAgB,KAAK,QAAQ;gCAC3B,CAAC,CAAC,kBAAkB;gCACpB,CAAC,CAAC,gBAAgB,KAAK,QAAQ;oCAC7B,CAAC,CAAC,QAAQ;oCACV,CAAC,CAAC,gBAAgB,KAAK,WAAW;wCAChC,CAAC,CAAC,0BAA0B;wCAC5B,CAAC,CAAC,yBAAyB;4BACnC,aAAa,EACX,gBAAgB,KAAK,QAAQ;gCAC3B,CAAC,CAAC,iCAAiC;gCACnC,CAAC,CAAC,sCAAsC;4BAC5C,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;yBACjD,CAAC;qBACH;oBACD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;iBACpC,CAAC;gBAEF,yDAAyD;gBACzD,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChC,OAAO,CAAC,IAAI,CAAC;4BACX,IAAI,EAAE,OAAO;4BACb,IAAI,EAAE,GAAG;4BACT,QAAQ,EAAE,WAAW;yBACtB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,KAAK,EAAE,IAAI;4BACX,QAAQ,EAAE,gBAAgB;4BAC1B,OAAO,EAAE,2BAA2B,GAAG,CAAC,OAAO,EAAE;4BACjD,UAAU,EACR,oIAAoI;yBACvI,CAAC;qBACH;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;KACF;IACD;QACE,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EACT,8VAA8V;QAChW,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,wCAAwC;iBACtD;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC;oBACpC,WAAW,EACT,iGAAiG;iBACpG;gBACD,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE;gBACjE,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4BAA4B,EAAE;gBAChE,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iCAAiC;iBAC/C;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kCAAkC;iBAChD;gBACD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE;gBAC9D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,EAAE;gBAChE,WAAW,EAAE;oBACX,IAAI,EAAE,OAAO;oBACb,WAAW,EAAE,sDAAsD;oBACnE,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE;4BACjD,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE;4BAChD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE;4BACzD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE;4BAC3D,KAAK,EAAE;gCACL,IAAI,EAAE,QAAQ;gCACd,WAAW,EAAE,4CAA4C;6BAC1D;4BACD,KAAK,EAAE;gCACL,IAAI,EAAE,QAAQ;gCACd,WAAW,EAAE,gCAAgC;6BAC9C;yBACF;wBACD,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC;qBACxC;iBACF;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,0EAA0E;iBAC7E;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,EAAE,WAAW,CAAC;SACvC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAA2B,EAAE;YAC/C,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,KAAK,EAAE,IAAI;4BACX,OAAO,EACL,4DAA4D;4BAC9D,UAAU,EACR,qEAAqE;yBACxE,CAAC;qBACH;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAE5D,IAAI,CAAC;gBACH,IAAI,YAAoB,CAAC;gBAEzB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,KAAK,MAAM,CAAC,CAAC,CAAC;wBACZ,IACE,IAAI,CAAC,CAAC,IAAI,IAAI;4BACd,IAAI,CAAC,CAAC,IAAI,IAAI;4BACd,CAAC,IAAI,CAAC,SAAS;4BACf,CAAC,IAAI,CAAC,UAAU,EAChB,CAAC;4BACD,MAAM,IAAI,KAAK,CACb,0DAA0D,CAC3D,CAAC;wBACJ,CAAC;wBACD,YAAY,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC;6BACpC,OAAO,CAAC;4BACP,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;4BACxB,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;4BACvB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;4BACjC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;yBACpC,CAAC;6BACD,GAAG,EAAE;6BACL,QAAQ,EAAE,CAAC;wBACd,MAAM;oBACR,CAAC;oBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;wBACd,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;4BAChC,MAAM,IAAI,KAAK,CACb,iDAAiD,CAClD,CAAC;wBACJ,CAAC;wBACD,YAAY,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC;6BACpC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE;4BAC/C,GAAG,EAAE,QAAQ;yBACd,CAAC;6BACD,GAAG,EAAE;6BACL,QAAQ,EAAE,CAAC;wBACd,MAAM;oBACR,CAAC;oBAED,KAAK,UAAU,CAAC,CAAC,CAAC;wBAChB,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACvD,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAC;wBACJ,CAAC;wBACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;wBACrD,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC;wBACjC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC;wBAEjC,MAAM,QAAQ,GAAI,IAAI,CAAC,WAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;4BACrD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC;4BAC/B,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK;gCACtB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,KAAK,uEAAuE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS;gCACtJ,CAAC,CAAC,EAAE,CAAC;4BACP,OAAO,GAAG,QAAQ,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,aAAa,CAAC,CAAC,MAAM,yBAAyB,KAAK,sBAAsB,CAAC;wBAC3I,CAAC,CAAC,CAAC;wBAEH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAC5B,eAAe,CAAC,aAAa,CAAC,wCAAwC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChG,CAAC;wBAEF,YAAY,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC;6BACpC,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;6BACnD,GAAG,EAAE;6BACL,QAAQ,EAAE,CAAC;wBACd,MAAM;oBACR,CAAC;oBAED;wBACE,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,kCAAkC,CACvE,CAAC;gBACN,CAAC;gBAED,MAAM,YAAY,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAErD,uBAAuB;gBACvB,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACjE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK;oBAC1B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE;oBAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;gBACnB,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,SAAS,MAAM,CAAC;gBACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBAC5C,aAAa,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;gBAEtC,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;4BACnC,QAAQ;4BACR,eAAe,EAAE,YAAY,CAAC,MAAM;yBACrC,CAAC;qBACH;oBACD;wBACE,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,YAAY;wBAClB,QAAQ,EAAE,WAAW;qBACtB;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,KAAK,EAAE,IAAI;4BACX,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,OAAO,EAAE,8BAA8B,GAAG,CAAC,OAAO,EAAE;yBACrD,CAAC;qBACH;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;KACF;CACF,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web tools — URL fetching and web search capabilities.
|
|
3
|
+
* Enables agents to gather information from the web for research, ideation, and discovery.
|
|
4
|
+
*
|
|
5
|
+
* - web_search: Searches the web using AI providers with search grounding
|
|
6
|
+
* - fetch_url: Fetches a URL and extracts content as markdown/text/html
|
|
7
|
+
*
|
|
8
|
+
* Uses Gemini grounding (preferred), OpenAI web search, or Perplexity as providers.
|
|
9
|
+
* Cheerio is optional for HTML parsing.
|
|
10
|
+
*/
|
|
11
|
+
import type { McpTool } from "../types.js";
|
|
12
|
+
export declare const webTools: McpTool[];
|