@mindstudio-ai/remy 0.1.255 → 0.1.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/headless.js +168 -230
- package/dist/index.js +177 -249
- package/dist/subagents/browserAutomation/prompt.md +2 -1
- package/package.json +1 -1
package/dist/headless.js
CHANGED
|
@@ -2764,6 +2764,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2764
2764
|
throw new Error("Sidecar not available");
|
|
2765
2765
|
}
|
|
2766
2766
|
const url = `${baseUrl}${endpoint}`;
|
|
2767
|
+
let data;
|
|
2767
2768
|
try {
|
|
2768
2769
|
const res = await fetch(url, {
|
|
2769
2770
|
method: "POST",
|
|
@@ -2775,12 +2776,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2775
2776
|
log4.error("Sidecar error", { endpoint, status: res.status });
|
|
2776
2777
|
throw new Error(`Sidecar error: ${res.status}`);
|
|
2777
2778
|
}
|
|
2778
|
-
|
|
2779
|
-
if (data?.success === false) {
|
|
2780
|
-
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
2781
|
-
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
2782
|
-
}
|
|
2783
|
-
return data;
|
|
2779
|
+
data = await res.json();
|
|
2784
2780
|
} catch (err) {
|
|
2785
2781
|
if (err.message.startsWith("Sidecar error")) {
|
|
2786
2782
|
throw err;
|
|
@@ -2788,6 +2784,16 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2788
2784
|
log4.error("Sidecar connection error", { endpoint, error: err.message });
|
|
2789
2785
|
throw new Error(`Sidecar connection error: ${err.message}`);
|
|
2790
2786
|
}
|
|
2787
|
+
if (data?.success === false) {
|
|
2788
|
+
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
2789
|
+
log4.error("Sidecar command failed", {
|
|
2790
|
+
endpoint,
|
|
2791
|
+
error: data.error,
|
|
2792
|
+
errorCode: data.errorCode
|
|
2793
|
+
});
|
|
2794
|
+
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
2795
|
+
}
|
|
2796
|
+
return data;
|
|
2791
2797
|
}
|
|
2792
2798
|
|
|
2793
2799
|
// src/tools/_helpers/lsp.ts
|
|
@@ -2973,6 +2979,8 @@ async function analyzeImage(params) {
|
|
|
2973
2979
|
}
|
|
2974
2980
|
|
|
2975
2981
|
// src/tools/_helpers/screenshot.ts
|
|
2982
|
+
var VIEWPORT_CAPTURE_TIMEOUT_MS = 45e3;
|
|
2983
|
+
var FULLPAGE_CAPTURE_TIMEOUT_MS = 135e3;
|
|
2976
2984
|
var SCREENSHOT_ANALYSIS_PROMPT = `Describe everything visible on screen from top to bottom \u2014 every element, its position, its size relative to the viewport, its colors, its content. Be comprehensive, thorough, and spatial. After the inventory, note anything that looks visually broken (overlapping elements, clipped text, misaligned components).`;
|
|
2977
2985
|
var ANALYSIS_RESPONSE_FORMAT = `Respond only with your analysis as Markdown and absolutely no other text. Do not use emojis - use unicode if you need symbols.`;
|
|
2978
2986
|
function buildScreenshotAnalysisPrompt(opts) {
|
|
@@ -3048,7 +3056,9 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3048
3056
|
...height != null ? { height } : {},
|
|
3049
3057
|
...format ? { format } : {}
|
|
3050
3058
|
},
|
|
3051
|
-
{
|
|
3059
|
+
{
|
|
3060
|
+
timeout: fullPage ? FULLPAGE_CAPTURE_TIMEOUT_MS : VIEWPORT_CAPTURE_TIMEOUT_MS
|
|
3061
|
+
}
|
|
3052
3062
|
);
|
|
3053
3063
|
url = ssResult?.url || ssResult?.screenshotUrl;
|
|
3054
3064
|
if (!url) {
|
|
@@ -3250,6 +3260,9 @@ function findLastSummaryCheckpoint(messages, name) {
|
|
|
3250
3260
|
}
|
|
3251
3261
|
return -1;
|
|
3252
3262
|
}
|
|
3263
|
+
function portableToolCallId(id) {
|
|
3264
|
+
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
3265
|
+
}
|
|
3253
3266
|
function fixOrphanedToolCalls(messages) {
|
|
3254
3267
|
const toolResultIds = /* @__PURE__ */ new Set();
|
|
3255
3268
|
for (const msg of messages) {
|
|
@@ -3341,14 +3354,30 @@ ${summaryBlock.text}
|
|
|
3341
3354
|
|
|
3342
3355
|
${content}` : attachmentHeader;
|
|
3343
3356
|
}
|
|
3344
|
-
return {
|
|
3357
|
+
return {
|
|
3358
|
+
...rest,
|
|
3359
|
+
content,
|
|
3360
|
+
...msg.toolCallId && {
|
|
3361
|
+
toolCallId: portableToolCallId(msg.toolCallId)
|
|
3362
|
+
}
|
|
3363
|
+
};
|
|
3345
3364
|
}
|
|
3346
3365
|
if (!Array.isArray(msg.content)) {
|
|
3347
3366
|
return msg;
|
|
3348
3367
|
}
|
|
3349
3368
|
const blocks = msg.content;
|
|
3350
3369
|
const text = blocks.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
3351
|
-
const
|
|
3370
|
+
const toolBlocks = blocks.filter(
|
|
3371
|
+
(b) => b.type === "tool"
|
|
3372
|
+
);
|
|
3373
|
+
const toolCalls = toolBlocks.map((b) => ({
|
|
3374
|
+
id: portableToolCallId(b.id),
|
|
3375
|
+
name: b.name,
|
|
3376
|
+
input: b.input
|
|
3377
|
+
}));
|
|
3378
|
+
const rewroteToolIds = toolBlocks.some(
|
|
3379
|
+
(b) => portableToolCallId(b.id) !== b.id
|
|
3380
|
+
);
|
|
3352
3381
|
const cleaned2 = {
|
|
3353
3382
|
role: msg.role,
|
|
3354
3383
|
content: text
|
|
@@ -3356,7 +3385,7 @@ ${content}` : attachmentHeader;
|
|
|
3356
3385
|
if (toolCalls.length > 0) {
|
|
3357
3386
|
cleaned2.toolCalls = toolCalls;
|
|
3358
3387
|
}
|
|
3359
|
-
if (msg.providerMetadata) {
|
|
3388
|
+
if (msg.providerMetadata && !rewroteToolIds) {
|
|
3360
3389
|
cleaned2.providerMetadata = msg.providerMetadata;
|
|
3361
3390
|
}
|
|
3362
3391
|
if (msg.hidden) {
|
|
@@ -3874,7 +3903,7 @@ var BROWSER_TOOLS = [
|
|
|
3874
3903
|
"screenshotViewport",
|
|
3875
3904
|
"setViewport"
|
|
3876
3905
|
],
|
|
3877
|
-
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage:
|
|
3906
|
+
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: screenshot of the whole page top-to-bottom (returns a CDN url with dimensions and a written analysis). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
|
|
3878
3907
|
},
|
|
3879
3908
|
ref: {
|
|
3880
3909
|
type: "string",
|
|
@@ -3942,20 +3971,11 @@ var BROWSER_TOOLS = [
|
|
|
3942
3971
|
required: ["steps"]
|
|
3943
3972
|
}
|
|
3944
3973
|
},
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
type: "object",
|
|
3951
|
-
properties: {
|
|
3952
|
-
path: {
|
|
3953
|
-
type: "string",
|
|
3954
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
3955
|
-
}
|
|
3956
|
-
}
|
|
3957
|
-
}
|
|
3958
|
-
},
|
|
3974
|
+
// Captures are `browserCommand` steps only — there is deliberately no
|
|
3975
|
+
// standalone screenshot tool here. Both used to exist for full-page, with
|
|
3976
|
+
// different budgets, different result plumbing, and analysis on only one of
|
|
3977
|
+
// them, so which door you picked changed what you got back.
|
|
3978
|
+
//
|
|
3959
3979
|
// Read tools so the QA agent can pull full spec detail on demand — the spec
|
|
3960
3980
|
// context in its prompt is a lightweight index (see prompt.ts) that points
|
|
3961
3981
|
// here. Routed to the global executeTool in index.ts, mirroring specSync.
|
|
@@ -4149,6 +4169,7 @@ function getBrowserAutomationPrompt() {
|
|
|
4149
4169
|
|
|
4150
4170
|
// src/subagents/browserAutomation/index.ts
|
|
4151
4171
|
var log7 = createLogger("browser-automation");
|
|
4172
|
+
var CAPTURE_COMMANDS = /* @__PURE__ */ new Set(["screenshotViewport", "screenshotFullPage"]);
|
|
4152
4173
|
async function runBrowserAutomation(task, context, opts) {
|
|
4153
4174
|
const release = await acquireBrowserLock();
|
|
4154
4175
|
try {
|
|
@@ -4160,7 +4181,7 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4160
4181
|
);
|
|
4161
4182
|
} catch {
|
|
4162
4183
|
}
|
|
4163
|
-
let
|
|
4184
|
+
let lastCapture = {};
|
|
4164
4185
|
const result = await runSubAgent({
|
|
4165
4186
|
system: getBrowserAutomationPrompt(),
|
|
4166
4187
|
task,
|
|
@@ -4182,22 +4203,6 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4182
4203
|
return `Error setting up browser: ${err.message}`;
|
|
4183
4204
|
}
|
|
4184
4205
|
}
|
|
4185
|
-
if (name === "screenshotFullPage") {
|
|
4186
|
-
try {
|
|
4187
|
-
return await captureAndAnalyzeScreenshot({
|
|
4188
|
-
path: _input.path,
|
|
4189
|
-
fullPage: true,
|
|
4190
|
-
onLog,
|
|
4191
|
-
model: resolveModel(
|
|
4192
|
-
"imageAnalysis",
|
|
4193
|
-
context.models,
|
|
4194
|
-
context.model
|
|
4195
|
-
)
|
|
4196
|
-
});
|
|
4197
|
-
} catch (err) {
|
|
4198
|
-
return `Error taking screenshot: ${err.message}`;
|
|
4199
|
-
}
|
|
4200
|
-
}
|
|
4201
4206
|
if (COMMON_READ_TOOL_NAMES.has(name) || name === readSpecTool.definition.name) {
|
|
4202
4207
|
return executeTool(name, _input, context);
|
|
4203
4208
|
}
|
|
@@ -4219,14 +4224,16 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4219
4224
|
try {
|
|
4220
4225
|
const parsed = JSON.parse(result2);
|
|
4221
4226
|
const screenshotSteps = (parsed.steps || []).filter(
|
|
4222
|
-
(s) => s.command
|
|
4227
|
+
(s) => CAPTURE_COMMANDS.has(s.command) && s.result?.url
|
|
4223
4228
|
);
|
|
4224
4229
|
if (screenshotSteps.length > 0) {
|
|
4225
|
-
const
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
+
for (const step of screenshotSteps) {
|
|
4231
|
+
const kind = step.command === "screenshotFullPage" ? "fullPage" : "viewport";
|
|
4232
|
+
lastCapture[kind] = {
|
|
4233
|
+
url: step.result.url,
|
|
4234
|
+
styleMap: step.result.styleMap
|
|
4235
|
+
};
|
|
4236
|
+
}
|
|
4230
4237
|
const visionOverride = {
|
|
4231
4238
|
model: resolveModel(
|
|
4232
4239
|
"imageAnalysis",
|
|
@@ -4250,13 +4257,12 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4250
4257
|
);
|
|
4251
4258
|
try {
|
|
4252
4259
|
const analyses = JSON.parse(batchResult);
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
|
|
4257
|
-
ai++;
|
|
4260
|
+
screenshotSteps.forEach((step, i) => {
|
|
4261
|
+
if (i >= analyses.length) {
|
|
4262
|
+
return;
|
|
4258
4263
|
}
|
|
4259
|
-
|
|
4264
|
+
step.result.analysis = analyses[i]?.output?.analysis || analyses[i]?.output || "";
|
|
4265
|
+
});
|
|
4260
4266
|
} catch {
|
|
4261
4267
|
log7.debug("Failed to parse batch analysis result", {
|
|
4262
4268
|
batchResult
|
|
@@ -4269,13 +4275,10 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4269
4275
|
}
|
|
4270
4276
|
return result2;
|
|
4271
4277
|
},
|
|
4272
|
-
toolRegistry: context.toolRegistry
|
|
4273
|
-
captureArtifacts: ["screenshotFullPage"]
|
|
4278
|
+
toolRegistry: context.toolRegistry
|
|
4274
4279
|
});
|
|
4275
4280
|
context.subAgentMessages?.set(context.toolCallId, result.messages);
|
|
4276
|
-
const
|
|
4277
|
-
const viewport = lastBrowserCommandViewport;
|
|
4278
|
-
const preferred = opts?.capture === "viewport" ? viewport ?? fullPage : fullPage ?? viewport;
|
|
4281
|
+
const preferred = opts?.capture === "viewport" ? lastCapture.viewport ?? lastCapture.fullPage : lastCapture.fullPage ?? lastCapture.viewport;
|
|
4279
4282
|
return {
|
|
4280
4283
|
text: result.text,
|
|
4281
4284
|
...preferred?.url ? { screenshot: { url: preferred.url, styleMap: preferred.styleMap } } : {}
|
|
@@ -4315,98 +4318,102 @@ var browserAutomationTool = {
|
|
|
4315
4318
|
};
|
|
4316
4319
|
|
|
4317
4320
|
// src/tools/code/screenshot.ts
|
|
4318
|
-
var
|
|
4321
|
+
var screenshotDefinition = {
|
|
4319
4322
|
clearable: true,
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4329
|
-
},
|
|
4330
|
-
prompt: {
|
|
4331
|
-
type: "string",
|
|
4332
|
-
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
4333
|
-
},
|
|
4334
|
-
imageUrl: {
|
|
4335
|
-
type: "string",
|
|
4336
|
-
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
4337
|
-
},
|
|
4338
|
-
path: {
|
|
4339
|
-
type: "string",
|
|
4340
|
-
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
4341
|
-
},
|
|
4342
|
-
width: {
|
|
4343
|
-
type: "number",
|
|
4344
|
-
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
4345
|
-
},
|
|
4346
|
-
height: {
|
|
4347
|
-
type: "number",
|
|
4348
|
-
description: "Exact capture height in pixels. Set together with `width`."
|
|
4349
|
-
},
|
|
4350
|
-
format: {
|
|
4351
|
-
type: "string",
|
|
4352
|
-
enum: ["png", "jpeg"],
|
|
4353
|
-
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
4354
|
-
},
|
|
4355
|
-
instructions: {
|
|
4356
|
-
type: "string",
|
|
4357
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
4358
|
-
}
|
|
4323
|
+
name: "screenshot",
|
|
4324
|
+
description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. The analysis is not precise about every detail \u2014 for example it cannot reliably identify specific fonts by name, only describe what the letterforms look like. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
|
|
4325
|
+
inputSchema: {
|
|
4326
|
+
type: "object",
|
|
4327
|
+
properties: {
|
|
4328
|
+
fullPage: {
|
|
4329
|
+
type: "boolean",
|
|
4330
|
+
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4359
4331
|
},
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4389
|
-
});
|
|
4332
|
+
prompt: {
|
|
4333
|
+
type: "string",
|
|
4334
|
+
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
4335
|
+
},
|
|
4336
|
+
imageUrl: {
|
|
4337
|
+
type: "string",
|
|
4338
|
+
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
4339
|
+
},
|
|
4340
|
+
path: {
|
|
4341
|
+
type: "string",
|
|
4342
|
+
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
4343
|
+
},
|
|
4344
|
+
width: {
|
|
4345
|
+
type: "number",
|
|
4346
|
+
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
4347
|
+
},
|
|
4348
|
+
height: {
|
|
4349
|
+
type: "number",
|
|
4350
|
+
description: "Exact capture height in pixels. Set together with `width`."
|
|
4351
|
+
},
|
|
4352
|
+
format: {
|
|
4353
|
+
type: "string",
|
|
4354
|
+
enum: ["png", "jpeg"],
|
|
4355
|
+
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
4356
|
+
},
|
|
4357
|
+
instructions: {
|
|
4358
|
+
type: "string",
|
|
4359
|
+
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
4390
4360
|
}
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4361
|
+
},
|
|
4362
|
+
required: ["fullPage"]
|
|
4363
|
+
}
|
|
4364
|
+
};
|
|
4365
|
+
async function executeScreenshot(input, onLog, context) {
|
|
4366
|
+
const fullPage = input.fullPage === true;
|
|
4367
|
+
const model = resolveModel("imageAnalysis", context?.models, context?.model);
|
|
4368
|
+
try {
|
|
4369
|
+
if (input.imageUrl) {
|
|
4370
|
+
return await captureAndAnalyzeScreenshot({
|
|
4371
|
+
prompt: input.prompt,
|
|
4372
|
+
imageUrl: input.imageUrl,
|
|
4373
|
+
onLog,
|
|
4374
|
+
model
|
|
4375
|
+
});
|
|
4376
|
+
}
|
|
4377
|
+
if (input.instructions && context) {
|
|
4378
|
+
const shotKind = fullPage ? "full-page" : "viewport";
|
|
4379
|
+
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
4380
|
+
const result = await runBrowserAutomation(task, context, {
|
|
4381
|
+
capture: fullPage ? "fullPage" : "viewport"
|
|
4382
|
+
});
|
|
4383
|
+
if (!result.screenshot) {
|
|
4384
|
+
return result.text;
|
|
4405
4385
|
}
|
|
4406
|
-
|
|
4407
|
-
|
|
4386
|
+
return await streamScreenshotAnalysis({
|
|
4387
|
+
url: result.screenshot.url,
|
|
4388
|
+
prompt: input.prompt,
|
|
4389
|
+
styleMap: result.screenshot.styleMap,
|
|
4390
|
+
onLog,
|
|
4391
|
+
model
|
|
4392
|
+
});
|
|
4408
4393
|
}
|
|
4394
|
+
const release = await acquireBrowserLock();
|
|
4395
|
+
try {
|
|
4396
|
+
return await captureAndAnalyzeScreenshot({
|
|
4397
|
+
prompt: input.prompt,
|
|
4398
|
+
path: input.path,
|
|
4399
|
+
fullPage,
|
|
4400
|
+
width: input.width,
|
|
4401
|
+
height: input.height,
|
|
4402
|
+
format: input.format,
|
|
4403
|
+
onLog,
|
|
4404
|
+
model
|
|
4405
|
+
});
|
|
4406
|
+
} finally {
|
|
4407
|
+
release();
|
|
4408
|
+
}
|
|
4409
|
+
} catch (err) {
|
|
4410
|
+
return `Error taking screenshot: ${err.message}`;
|
|
4409
4411
|
}
|
|
4412
|
+
}
|
|
4413
|
+
var screenshotTool = {
|
|
4414
|
+
clearable: true,
|
|
4415
|
+
definition: screenshotDefinition,
|
|
4416
|
+
execute: (input, context) => executeScreenshot(input, context?.onLog, context)
|
|
4410
4417
|
};
|
|
4411
4418
|
|
|
4412
4419
|
// src/subagents/designExpert/tools/searchGoogle.ts
|
|
@@ -4603,83 +4610,11 @@ async function execute4(input, onLog, context) {
|
|
|
4603
4610
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4604
4611
|
}
|
|
4605
4612
|
|
|
4606
|
-
// src/subagents/designExpert/tools/screenshot.ts
|
|
4607
|
-
var screenshot_exports = {};
|
|
4608
|
-
__export(screenshot_exports, {
|
|
4609
|
-
definition: () => definition5,
|
|
4610
|
-
execute: () => execute5
|
|
4611
|
-
});
|
|
4612
|
-
var definition5 = {
|
|
4613
|
-
clearable: true,
|
|
4614
|
-
name: "screenshot",
|
|
4615
|
-
description: "Capture a screenshot of the current app preview and get it back with visual analysis. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 use it to review a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 use it to review overall composition or a layout you can't see in one screen). Use to review the current state of the UI being built. Remember, the screenshot analysis is not overly precise - for example, it cannot reliably identify specific fonts by name \u2014 it can only describe what letterforms look like.",
|
|
4616
|
-
inputSchema: {
|
|
4617
|
-
type: "object",
|
|
4618
|
-
properties: {
|
|
4619
|
-
fullPage: {
|
|
4620
|
-
type: "boolean",
|
|
4621
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4622
|
-
},
|
|
4623
|
-
prompt: {
|
|
4624
|
-
type: "string",
|
|
4625
|
-
description: "Optional specific question about the screenshot. Use a bulleted list to ask many questions at once."
|
|
4626
|
-
},
|
|
4627
|
-
path: {
|
|
4628
|
-
type: "string",
|
|
4629
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
4630
|
-
},
|
|
4631
|
-
instructions: {
|
|
4632
|
-
type: "string",
|
|
4633
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, scrolling to a specific section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing."
|
|
4634
|
-
}
|
|
4635
|
-
},
|
|
4636
|
-
required: ["fullPage"]
|
|
4637
|
-
}
|
|
4638
|
-
};
|
|
4639
|
-
async function execute5(input, onLog, context) {
|
|
4640
|
-
const fullPage = input.fullPage === true;
|
|
4641
|
-
const shotKind = fullPage ? "full-page" : "viewport";
|
|
4642
|
-
if (input.instructions && context) {
|
|
4643
|
-
try {
|
|
4644
|
-
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
4645
|
-
const result = await runBrowserAutomation(task, context, {
|
|
4646
|
-
capture: fullPage ? "fullPage" : "viewport"
|
|
4647
|
-
});
|
|
4648
|
-
if (!result.screenshot) {
|
|
4649
|
-
return result.text;
|
|
4650
|
-
}
|
|
4651
|
-
return await streamScreenshotAnalysis({
|
|
4652
|
-
url: result.screenshot.url,
|
|
4653
|
-
prompt: input.prompt,
|
|
4654
|
-
styleMap: result.screenshot.styleMap,
|
|
4655
|
-
onLog,
|
|
4656
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4657
|
-
});
|
|
4658
|
-
} catch (err) {
|
|
4659
|
-
return `Error taking interactive screenshot: ${err.message}`;
|
|
4660
|
-
}
|
|
4661
|
-
}
|
|
4662
|
-
const release = await acquireBrowserLock();
|
|
4663
|
-
try {
|
|
4664
|
-
return await captureAndAnalyzeScreenshot({
|
|
4665
|
-
prompt: input.prompt,
|
|
4666
|
-
path: input.path,
|
|
4667
|
-
fullPage,
|
|
4668
|
-
onLog,
|
|
4669
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4670
|
-
});
|
|
4671
|
-
} catch (err) {
|
|
4672
|
-
return `Error taking screenshot: ${err.message}`;
|
|
4673
|
-
} finally {
|
|
4674
|
-
release();
|
|
4675
|
-
}
|
|
4676
|
-
}
|
|
4677
|
-
|
|
4678
4613
|
// src/subagents/designExpert/tools/images/generateImages.ts
|
|
4679
4614
|
var generateImages_exports = {};
|
|
4680
4615
|
__export(generateImages_exports, {
|
|
4681
|
-
definition: () =>
|
|
4682
|
-
execute: () =>
|
|
4616
|
+
definition: () => definition5,
|
|
4617
|
+
execute: () => execute5
|
|
4683
4618
|
});
|
|
4684
4619
|
|
|
4685
4620
|
// src/subagents/designExpert/tools/images/enhancePrompt.ts
|
|
@@ -4872,7 +4807,7 @@ async function generateImageAssets(opts) {
|
|
|
4872
4807
|
}
|
|
4873
4808
|
|
|
4874
4809
|
// src/subagents/designExpert/tools/images/generateImages.ts
|
|
4875
|
-
var
|
|
4810
|
+
var definition5 = {
|
|
4876
4811
|
clearable: false,
|
|
4877
4812
|
name: "generateImages",
|
|
4878
4813
|
description: "Generate images. Returns CDN URLs with a quality analysis for each image. Produces high-quality results for everything from photorealistic images and abstract/creative visuals. Pass multiple prompts to generate in parallel. No need to analyze images separately after generating \u2014 the analysis is included.",
|
|
@@ -4906,7 +4841,7 @@ var definition6 = {
|
|
|
4906
4841
|
required: ["prompts"]
|
|
4907
4842
|
}
|
|
4908
4843
|
};
|
|
4909
|
-
async function
|
|
4844
|
+
async function execute5(input, onLog, context) {
|
|
4910
4845
|
return generateImageAssets({
|
|
4911
4846
|
prompts: input.prompts,
|
|
4912
4847
|
width: input.width,
|
|
@@ -4936,10 +4871,10 @@ async function execute6(input, onLog, context) {
|
|
|
4936
4871
|
// src/subagents/designExpert/tools/images/editImages.ts
|
|
4937
4872
|
var editImages_exports = {};
|
|
4938
4873
|
__export(editImages_exports, {
|
|
4939
|
-
definition: () =>
|
|
4940
|
-
execute: () =>
|
|
4874
|
+
definition: () => definition6,
|
|
4875
|
+
execute: () => execute6
|
|
4941
4876
|
});
|
|
4942
|
-
var
|
|
4877
|
+
var definition6 = {
|
|
4943
4878
|
clearable: false,
|
|
4944
4879
|
name: "editImages",
|
|
4945
4880
|
description: "Edit or transform existing images. Provide one or more source image URLs as reference and a prompt describing the desired edit. Use for compositing, style transfer, subject transformation, blending multiple references, or incorporating one or more references into something new. Returns CDN URLs with analysis.",
|
|
@@ -4976,7 +4911,7 @@ var definition7 = {
|
|
|
4976
4911
|
required: ["prompts", "sourceImages"]
|
|
4977
4912
|
}
|
|
4978
4913
|
};
|
|
4979
|
-
async function
|
|
4914
|
+
async function execute6(input, onLog, context) {
|
|
4980
4915
|
return generateImageAssets({
|
|
4981
4916
|
prompts: input.prompts,
|
|
4982
4917
|
sourceImages: input.sourceImages,
|
|
@@ -5006,8 +4941,8 @@ async function execute7(input, onLog, context) {
|
|
|
5006
4941
|
// src/subagents/designExpert/tools/polishCopy.ts
|
|
5007
4942
|
var polishCopy_exports = {};
|
|
5008
4943
|
__export(polishCopy_exports, {
|
|
5009
|
-
definition: () =>
|
|
5010
|
-
execute: () =>
|
|
4944
|
+
definition: () => definition7,
|
|
4945
|
+
execute: () => execute7
|
|
5011
4946
|
});
|
|
5012
4947
|
|
|
5013
4948
|
// src/subagents/copyEditor/tools.ts
|
|
@@ -5064,7 +4999,7 @@ var copyEditorTool = {
|
|
|
5064
4999
|
};
|
|
5065
5000
|
|
|
5066
5001
|
// src/subagents/designExpert/tools/polishCopy.ts
|
|
5067
|
-
var
|
|
5002
|
+
var definition7 = {
|
|
5068
5003
|
clearable: false,
|
|
5069
5004
|
name: "polishCopy",
|
|
5070
5005
|
description: "Hand off any user-facing copy you've written \u2014 headlines, captions, labels, body text \u2014 and get back a sharper version: better built for its audience and free of the fingerprints that make writing read as AI. It elevates how the copy communicates without inventing facts or claims you didn't give it. Give it the text plus what it's for (where it appears, the audience).",
|
|
@@ -5079,7 +5014,7 @@ var definition8 = {
|
|
|
5079
5014
|
required: ["task"]
|
|
5080
5015
|
}
|
|
5081
5016
|
};
|
|
5082
|
-
async function
|
|
5017
|
+
async function execute7(input, _onLog, context) {
|
|
5083
5018
|
return copyEditorTool.execute(input, context);
|
|
5084
5019
|
}
|
|
5085
5020
|
|
|
@@ -5089,7 +5024,10 @@ var tools = {
|
|
|
5089
5024
|
scrapeWebUrl: scrapeWebUrl_exports,
|
|
5090
5025
|
analyzeDesign: analyzeDesign_exports,
|
|
5091
5026
|
analyzeImage: analyzeImage_exports,
|
|
5092
|
-
|
|
5027
|
+
// Same tool the main agent offers, imported rather than reimplemented — the
|
|
5028
|
+
// two used to be near-identical copies and had already drifted apart. Its core
|
|
5029
|
+
// already takes (input, onLog, context), which is this registry's convention.
|
|
5030
|
+
screenshot: { definition: screenshotDefinition, execute: executeScreenshot },
|
|
5093
5031
|
generateImages: generateImages_exports,
|
|
5094
5032
|
editImages: editImages_exports,
|
|
5095
5033
|
polishCopy: polishCopy_exports
|
package/dist/index.js
CHANGED
|
@@ -2238,6 +2238,9 @@ function findLastSummaryCheckpoint(messages, name) {
|
|
|
2238
2238
|
}
|
|
2239
2239
|
return -1;
|
|
2240
2240
|
}
|
|
2241
|
+
function portableToolCallId(id) {
|
|
2242
|
+
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2243
|
+
}
|
|
2241
2244
|
function fixOrphanedToolCalls(messages) {
|
|
2242
2245
|
const toolResultIds = /* @__PURE__ */ new Set();
|
|
2243
2246
|
for (const msg of messages) {
|
|
@@ -2329,14 +2332,30 @@ ${summaryBlock.text}
|
|
|
2329
2332
|
|
|
2330
2333
|
${content}` : attachmentHeader;
|
|
2331
2334
|
}
|
|
2332
|
-
return {
|
|
2335
|
+
return {
|
|
2336
|
+
...rest,
|
|
2337
|
+
content,
|
|
2338
|
+
...msg.toolCallId && {
|
|
2339
|
+
toolCallId: portableToolCallId(msg.toolCallId)
|
|
2340
|
+
}
|
|
2341
|
+
};
|
|
2333
2342
|
}
|
|
2334
2343
|
if (!Array.isArray(msg.content)) {
|
|
2335
2344
|
return msg;
|
|
2336
2345
|
}
|
|
2337
2346
|
const blocks = msg.content;
|
|
2338
2347
|
const text = blocks.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
2339
|
-
const
|
|
2348
|
+
const toolBlocks = blocks.filter(
|
|
2349
|
+
(b) => b.type === "tool"
|
|
2350
|
+
);
|
|
2351
|
+
const toolCalls = toolBlocks.map((b) => ({
|
|
2352
|
+
id: portableToolCallId(b.id),
|
|
2353
|
+
name: b.name,
|
|
2354
|
+
input: b.input
|
|
2355
|
+
}));
|
|
2356
|
+
const rewroteToolIds = toolBlocks.some(
|
|
2357
|
+
(b) => portableToolCallId(b.id) !== b.id
|
|
2358
|
+
);
|
|
2340
2359
|
const cleaned2 = {
|
|
2341
2360
|
role: msg.role,
|
|
2342
2361
|
content: text
|
|
@@ -2344,7 +2363,7 @@ ${content}` : attachmentHeader;
|
|
|
2344
2363
|
if (toolCalls.length > 0) {
|
|
2345
2364
|
cleaned2.toolCalls = toolCalls;
|
|
2346
2365
|
}
|
|
2347
|
-
if (msg.providerMetadata) {
|
|
2366
|
+
if (msg.providerMetadata && !rewroteToolIds) {
|
|
2348
2367
|
cleaned2.providerMetadata = msg.providerMetadata;
|
|
2349
2368
|
}
|
|
2350
2369
|
if (msg.hidden) {
|
|
@@ -3599,6 +3618,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
3599
3618
|
throw new Error("Sidecar not available");
|
|
3600
3619
|
}
|
|
3601
3620
|
const url = `${baseUrl}${endpoint}`;
|
|
3621
|
+
let data;
|
|
3602
3622
|
try {
|
|
3603
3623
|
const res = await fetch(url, {
|
|
3604
3624
|
method: "POST",
|
|
@@ -3610,12 +3630,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
3610
3630
|
log5.error("Sidecar error", { endpoint, status: res.status });
|
|
3611
3631
|
throw new Error(`Sidecar error: ${res.status}`);
|
|
3612
3632
|
}
|
|
3613
|
-
|
|
3614
|
-
if (data?.success === false) {
|
|
3615
|
-
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
3616
|
-
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
3617
|
-
}
|
|
3618
|
-
return data;
|
|
3633
|
+
data = await res.json();
|
|
3619
3634
|
} catch (err) {
|
|
3620
3635
|
if (err.message.startsWith("Sidecar error")) {
|
|
3621
3636
|
throw err;
|
|
@@ -3623,6 +3638,16 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
3623
3638
|
log5.error("Sidecar connection error", { endpoint, error: err.message });
|
|
3624
3639
|
throw new Error(`Sidecar connection error: ${err.message}`);
|
|
3625
3640
|
}
|
|
3641
|
+
if (data?.success === false) {
|
|
3642
|
+
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
3643
|
+
log5.error("Sidecar command failed", {
|
|
3644
|
+
endpoint,
|
|
3645
|
+
error: data.error,
|
|
3646
|
+
errorCode: data.errorCode
|
|
3647
|
+
});
|
|
3648
|
+
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
3649
|
+
}
|
|
3650
|
+
return data;
|
|
3626
3651
|
}
|
|
3627
3652
|
var log5, baseUrl;
|
|
3628
3653
|
var init_sidecar = __esm({
|
|
@@ -3935,7 +3960,9 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3935
3960
|
...height != null ? { height } : {},
|
|
3936
3961
|
...format ? { format } : {}
|
|
3937
3962
|
},
|
|
3938
|
-
{
|
|
3963
|
+
{
|
|
3964
|
+
timeout: fullPage ? FULLPAGE_CAPTURE_TIMEOUT_MS : VIEWPORT_CAPTURE_TIMEOUT_MS
|
|
3965
|
+
}
|
|
3939
3966
|
);
|
|
3940
3967
|
url = ssResult?.url || ssResult?.screenshotUrl;
|
|
3941
3968
|
if (!url) {
|
|
@@ -3961,12 +3988,14 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3961
3988
|
model
|
|
3962
3989
|
});
|
|
3963
3990
|
}
|
|
3964
|
-
var SCREENSHOT_ANALYSIS_PROMPT, ANALYSIS_RESPONSE_FORMAT;
|
|
3991
|
+
var VIEWPORT_CAPTURE_TIMEOUT_MS, FULLPAGE_CAPTURE_TIMEOUT_MS, SCREENSHOT_ANALYSIS_PROMPT, ANALYSIS_RESPONSE_FORMAT;
|
|
3965
3992
|
var init_screenshot = __esm({
|
|
3966
3993
|
"src/tools/_helpers/screenshot.ts"() {
|
|
3967
3994
|
"use strict";
|
|
3968
3995
|
init_sidecar();
|
|
3969
3996
|
init_analyzeImage();
|
|
3997
|
+
VIEWPORT_CAPTURE_TIMEOUT_MS = 45e3;
|
|
3998
|
+
FULLPAGE_CAPTURE_TIMEOUT_MS = 135e3;
|
|
3970
3999
|
SCREENSHOT_ANALYSIS_PROMPT = `Describe everything visible on screen from top to bottom \u2014 every element, its position, its size relative to the viewport, its colors, its content. Be comprehensive, thorough, and spatial. After the inventory, note anything that looks visually broken (overlapping elements, clipped text, misaligned components).`;
|
|
3971
4000
|
ANALYSIS_RESPONSE_FORMAT = `Respond only with your analysis as Markdown and absolutely no other text. Do not use emojis - use unicode if you need symbols.`;
|
|
3972
4001
|
}
|
|
@@ -4673,7 +4702,7 @@ var init_tools2 = __esm({
|
|
|
4673
4702
|
"screenshotViewport",
|
|
4674
4703
|
"setViewport"
|
|
4675
4704
|
],
|
|
4676
|
-
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage:
|
|
4705
|
+
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: screenshot of the whole page top-to-bottom (returns a CDN url with dimensions and a written analysis). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
|
|
4677
4706
|
},
|
|
4678
4707
|
ref: {
|
|
4679
4708
|
type: "string",
|
|
@@ -4741,20 +4770,11 @@ var init_tools2 = __esm({
|
|
|
4741
4770
|
required: ["steps"]
|
|
4742
4771
|
}
|
|
4743
4772
|
},
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
type: "object",
|
|
4750
|
-
properties: {
|
|
4751
|
-
path: {
|
|
4752
|
-
type: "string",
|
|
4753
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
4754
|
-
}
|
|
4755
|
-
}
|
|
4756
|
-
}
|
|
4757
|
-
},
|
|
4773
|
+
// Captures are `browserCommand` steps only — there is deliberately no
|
|
4774
|
+
// standalone screenshot tool here. Both used to exist for full-page, with
|
|
4775
|
+
// different budgets, different result plumbing, and analysis on only one of
|
|
4776
|
+
// them, so which door you picked changed what you got back.
|
|
4777
|
+
//
|
|
4758
4778
|
// Read tools so the QA agent can pull full spec detail on demand — the spec
|
|
4759
4779
|
// context in its prompt is a lightweight index (see prompt.ts) that points
|
|
4760
4780
|
// here. Routed to the global executeTool in index.ts, mirroring specSync.
|
|
@@ -4973,7 +4993,7 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4973
4993
|
);
|
|
4974
4994
|
} catch {
|
|
4975
4995
|
}
|
|
4976
|
-
let
|
|
4996
|
+
let lastCapture = {};
|
|
4977
4997
|
const result = await runSubAgent({
|
|
4978
4998
|
system: getBrowserAutomationPrompt(),
|
|
4979
4999
|
task,
|
|
@@ -4995,22 +5015,6 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4995
5015
|
return `Error setting up browser: ${err.message}`;
|
|
4996
5016
|
}
|
|
4997
5017
|
}
|
|
4998
|
-
if (name === "screenshotFullPage") {
|
|
4999
|
-
try {
|
|
5000
|
-
return await captureAndAnalyzeScreenshot({
|
|
5001
|
-
path: _input.path,
|
|
5002
|
-
fullPage: true,
|
|
5003
|
-
onLog,
|
|
5004
|
-
model: resolveModel(
|
|
5005
|
-
"imageAnalysis",
|
|
5006
|
-
context.models,
|
|
5007
|
-
context.model
|
|
5008
|
-
)
|
|
5009
|
-
});
|
|
5010
|
-
} catch (err) {
|
|
5011
|
-
return `Error taking screenshot: ${err.message}`;
|
|
5012
|
-
}
|
|
5013
|
-
}
|
|
5014
5018
|
if (COMMON_READ_TOOL_NAMES.has(name) || name === readSpecTool.definition.name) {
|
|
5015
5019
|
return executeTool(name, _input, context);
|
|
5016
5020
|
}
|
|
@@ -5032,14 +5036,16 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
5032
5036
|
try {
|
|
5033
5037
|
const parsed = JSON.parse(result2);
|
|
5034
5038
|
const screenshotSteps = (parsed.steps || []).filter(
|
|
5035
|
-
(s) => s.command
|
|
5039
|
+
(s) => CAPTURE_COMMANDS.has(s.command) && s.result?.url
|
|
5036
5040
|
);
|
|
5037
5041
|
if (screenshotSteps.length > 0) {
|
|
5038
|
-
const
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
5042
|
-
|
|
5042
|
+
for (const step of screenshotSteps) {
|
|
5043
|
+
const kind = step.command === "screenshotFullPage" ? "fullPage" : "viewport";
|
|
5044
|
+
lastCapture[kind] = {
|
|
5045
|
+
url: step.result.url,
|
|
5046
|
+
styleMap: step.result.styleMap
|
|
5047
|
+
};
|
|
5048
|
+
}
|
|
5043
5049
|
const visionOverride = {
|
|
5044
5050
|
model: resolveModel(
|
|
5045
5051
|
"imageAnalysis",
|
|
@@ -5063,13 +5069,12 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
5063
5069
|
);
|
|
5064
5070
|
try {
|
|
5065
5071
|
const analyses = JSON.parse(batchResult);
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5069
|
-
step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
|
|
5070
|
-
ai++;
|
|
5072
|
+
screenshotSteps.forEach((step, i) => {
|
|
5073
|
+
if (i >= analyses.length) {
|
|
5074
|
+
return;
|
|
5071
5075
|
}
|
|
5072
|
-
|
|
5076
|
+
step.result.analysis = analyses[i]?.output?.analysis || analyses[i]?.output || "";
|
|
5077
|
+
});
|
|
5073
5078
|
} catch {
|
|
5074
5079
|
log8.debug("Failed to parse batch analysis result", {
|
|
5075
5080
|
batchResult
|
|
@@ -5082,13 +5087,10 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
5082
5087
|
}
|
|
5083
5088
|
return result2;
|
|
5084
5089
|
},
|
|
5085
|
-
toolRegistry: context.toolRegistry
|
|
5086
|
-
captureArtifacts: ["screenshotFullPage"]
|
|
5090
|
+
toolRegistry: context.toolRegistry
|
|
5087
5091
|
});
|
|
5088
5092
|
context.subAgentMessages?.set(context.toolCallId, result.messages);
|
|
5089
|
-
const
|
|
5090
|
-
const viewport = lastBrowserCommandViewport;
|
|
5091
|
-
const preferred = opts?.capture === "viewport" ? viewport ?? fullPage : fullPage ?? viewport;
|
|
5093
|
+
const preferred = opts?.capture === "viewport" ? lastCapture.viewport ?? lastCapture.fullPage : lastCapture.fullPage ?? lastCapture.viewport;
|
|
5092
5094
|
return {
|
|
5093
5095
|
text: result.text,
|
|
5094
5096
|
...preferred?.url ? { screenshot: { url: preferred.url, styleMap: preferred.styleMap } } : {}
|
|
@@ -5097,7 +5099,7 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
5097
5099
|
release();
|
|
5098
5100
|
}
|
|
5099
5101
|
}
|
|
5100
|
-
var log8, browserAutomationTool;
|
|
5102
|
+
var log8, CAPTURE_COMMANDS, browserAutomationTool;
|
|
5101
5103
|
var init_browserAutomation = __esm({
|
|
5102
5104
|
"src/subagents/browserAutomation/index.ts"() {
|
|
5103
5105
|
"use strict";
|
|
@@ -5114,6 +5116,7 @@ var init_browserAutomation = __esm({
|
|
|
5114
5116
|
init_surfaces();
|
|
5115
5117
|
init_logger();
|
|
5116
5118
|
log8 = createLogger("browser-automation");
|
|
5119
|
+
CAPTURE_COMMANDS = /* @__PURE__ */ new Set(["screenshotViewport", "screenshotFullPage"]);
|
|
5117
5120
|
browserAutomationTool = {
|
|
5118
5121
|
clearable: true,
|
|
5119
5122
|
definition: {
|
|
@@ -5147,7 +5150,55 @@ var init_browserAutomation = __esm({
|
|
|
5147
5150
|
});
|
|
5148
5151
|
|
|
5149
5152
|
// src/tools/code/screenshot.ts
|
|
5150
|
-
|
|
5153
|
+
async function executeScreenshot(input, onLog, context) {
|
|
5154
|
+
const fullPage = input.fullPage === true;
|
|
5155
|
+
const model = resolveModel("imageAnalysis", context?.models, context?.model);
|
|
5156
|
+
try {
|
|
5157
|
+
if (input.imageUrl) {
|
|
5158
|
+
return await captureAndAnalyzeScreenshot({
|
|
5159
|
+
prompt: input.prompt,
|
|
5160
|
+
imageUrl: input.imageUrl,
|
|
5161
|
+
onLog,
|
|
5162
|
+
model
|
|
5163
|
+
});
|
|
5164
|
+
}
|
|
5165
|
+
if (input.instructions && context) {
|
|
5166
|
+
const shotKind = fullPage ? "full-page" : "viewport";
|
|
5167
|
+
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
5168
|
+
const result = await runBrowserAutomation(task, context, {
|
|
5169
|
+
capture: fullPage ? "fullPage" : "viewport"
|
|
5170
|
+
});
|
|
5171
|
+
if (!result.screenshot) {
|
|
5172
|
+
return result.text;
|
|
5173
|
+
}
|
|
5174
|
+
return await streamScreenshotAnalysis({
|
|
5175
|
+
url: result.screenshot.url,
|
|
5176
|
+
prompt: input.prompt,
|
|
5177
|
+
styleMap: result.screenshot.styleMap,
|
|
5178
|
+
onLog,
|
|
5179
|
+
model
|
|
5180
|
+
});
|
|
5181
|
+
}
|
|
5182
|
+
const release = await acquireBrowserLock();
|
|
5183
|
+
try {
|
|
5184
|
+
return await captureAndAnalyzeScreenshot({
|
|
5185
|
+
prompt: input.prompt,
|
|
5186
|
+
path: input.path,
|
|
5187
|
+
fullPage,
|
|
5188
|
+
width: input.width,
|
|
5189
|
+
height: input.height,
|
|
5190
|
+
format: input.format,
|
|
5191
|
+
onLog,
|
|
5192
|
+
model
|
|
5193
|
+
});
|
|
5194
|
+
} finally {
|
|
5195
|
+
release();
|
|
5196
|
+
}
|
|
5197
|
+
} catch (err) {
|
|
5198
|
+
return `Error taking screenshot: ${err.message}`;
|
|
5199
|
+
}
|
|
5200
|
+
}
|
|
5201
|
+
var screenshotDefinition, screenshotTool;
|
|
5151
5202
|
var init_screenshot2 = __esm({
|
|
5152
5203
|
"src/tools/code/screenshot.ts"() {
|
|
5153
5204
|
"use strict";
|
|
@@ -5155,99 +5206,55 @@ var init_screenshot2 = __esm({
|
|
|
5155
5206
|
init_browserLock();
|
|
5156
5207
|
init_browserAutomation();
|
|
5157
5208
|
init_surfaces();
|
|
5158
|
-
|
|
5209
|
+
screenshotDefinition = {
|
|
5159
5210
|
clearable: true,
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
5169
|
-
},
|
|
5170
|
-
prompt: {
|
|
5171
|
-
type: "string",
|
|
5172
|
-
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
5173
|
-
},
|
|
5174
|
-
imageUrl: {
|
|
5175
|
-
type: "string",
|
|
5176
|
-
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
5177
|
-
},
|
|
5178
|
-
path: {
|
|
5179
|
-
type: "string",
|
|
5180
|
-
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
5181
|
-
},
|
|
5182
|
-
width: {
|
|
5183
|
-
type: "number",
|
|
5184
|
-
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
5185
|
-
},
|
|
5186
|
-
height: {
|
|
5187
|
-
type: "number",
|
|
5188
|
-
description: "Exact capture height in pixels. Set together with `width`."
|
|
5189
|
-
},
|
|
5190
|
-
format: {
|
|
5191
|
-
type: "string",
|
|
5192
|
-
enum: ["png", "jpeg"],
|
|
5193
|
-
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
5194
|
-
},
|
|
5195
|
-
instructions: {
|
|
5196
|
-
type: "string",
|
|
5197
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
5198
|
-
}
|
|
5211
|
+
name: "screenshot",
|
|
5212
|
+
description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. The analysis is not precise about every detail \u2014 for example it cannot reliably identify specific fonts by name, only describe what the letterforms look like. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
|
|
5213
|
+
inputSchema: {
|
|
5214
|
+
type: "object",
|
|
5215
|
+
properties: {
|
|
5216
|
+
fullPage: {
|
|
5217
|
+
type: "boolean",
|
|
5218
|
+
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
5199
5219
|
},
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
5229
|
-
});
|
|
5230
|
-
}
|
|
5231
|
-
const release = await acquireBrowserLock();
|
|
5232
|
-
try {
|
|
5233
|
-
return await captureAndAnalyzeScreenshot({
|
|
5234
|
-
prompt: input.prompt,
|
|
5235
|
-
path: input.path,
|
|
5236
|
-
fullPage,
|
|
5237
|
-
width: input.width,
|
|
5238
|
-
height: input.height,
|
|
5239
|
-
format: input.format,
|
|
5240
|
-
onLog: context?.onLog,
|
|
5241
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
5242
|
-
});
|
|
5243
|
-
} finally {
|
|
5244
|
-
release();
|
|
5220
|
+
prompt: {
|
|
5221
|
+
type: "string",
|
|
5222
|
+
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
5223
|
+
},
|
|
5224
|
+
imageUrl: {
|
|
5225
|
+
type: "string",
|
|
5226
|
+
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
5227
|
+
},
|
|
5228
|
+
path: {
|
|
5229
|
+
type: "string",
|
|
5230
|
+
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
5231
|
+
},
|
|
5232
|
+
width: {
|
|
5233
|
+
type: "number",
|
|
5234
|
+
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
5235
|
+
},
|
|
5236
|
+
height: {
|
|
5237
|
+
type: "number",
|
|
5238
|
+
description: "Exact capture height in pixels. Set together with `width`."
|
|
5239
|
+
},
|
|
5240
|
+
format: {
|
|
5241
|
+
type: "string",
|
|
5242
|
+
enum: ["png", "jpeg"],
|
|
5243
|
+
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
5244
|
+
},
|
|
5245
|
+
instructions: {
|
|
5246
|
+
type: "string",
|
|
5247
|
+
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
5245
5248
|
}
|
|
5246
|
-
}
|
|
5247
|
-
|
|
5248
|
-
}
|
|
5249
|
+
},
|
|
5250
|
+
required: ["fullPage"]
|
|
5249
5251
|
}
|
|
5250
5252
|
};
|
|
5253
|
+
screenshotTool = {
|
|
5254
|
+
clearable: true,
|
|
5255
|
+
definition: screenshotDefinition,
|
|
5256
|
+
execute: (input, context) => executeScreenshot(input, context?.onLog, context)
|
|
5257
|
+
};
|
|
5251
5258
|
}
|
|
5252
5259
|
});
|
|
5253
5260
|
|
|
@@ -5479,88 +5486,6 @@ var init_analyzeImage2 = __esm({
|
|
|
5479
5486
|
}
|
|
5480
5487
|
});
|
|
5481
5488
|
|
|
5482
|
-
// src/subagents/designExpert/tools/screenshot.ts
|
|
5483
|
-
var screenshot_exports = {};
|
|
5484
|
-
__export(screenshot_exports, {
|
|
5485
|
-
definition: () => definition5,
|
|
5486
|
-
execute: () => execute5
|
|
5487
|
-
});
|
|
5488
|
-
async function execute5(input, onLog, context) {
|
|
5489
|
-
const fullPage = input.fullPage === true;
|
|
5490
|
-
const shotKind = fullPage ? "full-page" : "viewport";
|
|
5491
|
-
if (input.instructions && context) {
|
|
5492
|
-
try {
|
|
5493
|
-
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
5494
|
-
const result = await runBrowserAutomation(task, context, {
|
|
5495
|
-
capture: fullPage ? "fullPage" : "viewport"
|
|
5496
|
-
});
|
|
5497
|
-
if (!result.screenshot) {
|
|
5498
|
-
return result.text;
|
|
5499
|
-
}
|
|
5500
|
-
return await streamScreenshotAnalysis({
|
|
5501
|
-
url: result.screenshot.url,
|
|
5502
|
-
prompt: input.prompt,
|
|
5503
|
-
styleMap: result.screenshot.styleMap,
|
|
5504
|
-
onLog,
|
|
5505
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
5506
|
-
});
|
|
5507
|
-
} catch (err) {
|
|
5508
|
-
return `Error taking interactive screenshot: ${err.message}`;
|
|
5509
|
-
}
|
|
5510
|
-
}
|
|
5511
|
-
const release = await acquireBrowserLock();
|
|
5512
|
-
try {
|
|
5513
|
-
return await captureAndAnalyzeScreenshot({
|
|
5514
|
-
prompt: input.prompt,
|
|
5515
|
-
path: input.path,
|
|
5516
|
-
fullPage,
|
|
5517
|
-
onLog,
|
|
5518
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
5519
|
-
});
|
|
5520
|
-
} catch (err) {
|
|
5521
|
-
return `Error taking screenshot: ${err.message}`;
|
|
5522
|
-
} finally {
|
|
5523
|
-
release();
|
|
5524
|
-
}
|
|
5525
|
-
}
|
|
5526
|
-
var definition5;
|
|
5527
|
-
var init_screenshot3 = __esm({
|
|
5528
|
-
"src/subagents/designExpert/tools/screenshot.ts"() {
|
|
5529
|
-
"use strict";
|
|
5530
|
-
init_screenshot();
|
|
5531
|
-
init_browserLock();
|
|
5532
|
-
init_browserAutomation();
|
|
5533
|
-
init_surfaces();
|
|
5534
|
-
definition5 = {
|
|
5535
|
-
clearable: true,
|
|
5536
|
-
name: "screenshot",
|
|
5537
|
-
description: "Capture a screenshot of the current app preview and get it back with visual analysis. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 use it to review a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 use it to review overall composition or a layout you can't see in one screen). Use to review the current state of the UI being built. Remember, the screenshot analysis is not overly precise - for example, it cannot reliably identify specific fonts by name \u2014 it can only describe what letterforms look like.",
|
|
5538
|
-
inputSchema: {
|
|
5539
|
-
type: "object",
|
|
5540
|
-
properties: {
|
|
5541
|
-
fullPage: {
|
|
5542
|
-
type: "boolean",
|
|
5543
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
5544
|
-
},
|
|
5545
|
-
prompt: {
|
|
5546
|
-
type: "string",
|
|
5547
|
-
description: "Optional specific question about the screenshot. Use a bulleted list to ask many questions at once."
|
|
5548
|
-
},
|
|
5549
|
-
path: {
|
|
5550
|
-
type: "string",
|
|
5551
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
5552
|
-
},
|
|
5553
|
-
instructions: {
|
|
5554
|
-
type: "string",
|
|
5555
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, scrolling to a specific section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing."
|
|
5556
|
-
}
|
|
5557
|
-
},
|
|
5558
|
-
required: ["fullPage"]
|
|
5559
|
-
}
|
|
5560
|
-
};
|
|
5561
|
-
}
|
|
5562
|
-
});
|
|
5563
|
-
|
|
5564
5489
|
// src/subagents/designExpert/tools/images/enhancePrompt.ts
|
|
5565
5490
|
async function enhanceImagePrompt(params) {
|
|
5566
5491
|
const {
|
|
@@ -5770,10 +5695,10 @@ var init_imageGenerator = __esm({
|
|
|
5770
5695
|
// src/subagents/designExpert/tools/images/generateImages.ts
|
|
5771
5696
|
var generateImages_exports = {};
|
|
5772
5697
|
__export(generateImages_exports, {
|
|
5773
|
-
definition: () =>
|
|
5774
|
-
execute: () =>
|
|
5698
|
+
definition: () => definition5,
|
|
5699
|
+
execute: () => execute5
|
|
5775
5700
|
});
|
|
5776
|
-
async function
|
|
5701
|
+
async function execute5(input, onLog, context) {
|
|
5777
5702
|
return generateImageAssets({
|
|
5778
5703
|
prompts: input.prompts,
|
|
5779
5704
|
width: input.width,
|
|
@@ -5799,13 +5724,13 @@ async function execute6(input, onLog, context) {
|
|
|
5799
5724
|
)
|
|
5800
5725
|
});
|
|
5801
5726
|
}
|
|
5802
|
-
var
|
|
5727
|
+
var definition5;
|
|
5803
5728
|
var init_generateImages = __esm({
|
|
5804
5729
|
"src/subagents/designExpert/tools/images/generateImages.ts"() {
|
|
5805
5730
|
"use strict";
|
|
5806
5731
|
init_imageGenerator();
|
|
5807
5732
|
init_surfaces();
|
|
5808
|
-
|
|
5733
|
+
definition5 = {
|
|
5809
5734
|
clearable: false,
|
|
5810
5735
|
name: "generateImages",
|
|
5811
5736
|
description: "Generate images. Returns CDN URLs with a quality analysis for each image. Produces high-quality results for everything from photorealistic images and abstract/creative visuals. Pass multiple prompts to generate in parallel. No need to analyze images separately after generating \u2014 the analysis is included.",
|
|
@@ -5845,10 +5770,10 @@ var init_generateImages = __esm({
|
|
|
5845
5770
|
// src/subagents/designExpert/tools/images/editImages.ts
|
|
5846
5771
|
var editImages_exports = {};
|
|
5847
5772
|
__export(editImages_exports, {
|
|
5848
|
-
definition: () =>
|
|
5849
|
-
execute: () =>
|
|
5773
|
+
definition: () => definition6,
|
|
5774
|
+
execute: () => execute6
|
|
5850
5775
|
});
|
|
5851
|
-
async function
|
|
5776
|
+
async function execute6(input, onLog, context) {
|
|
5852
5777
|
return generateImageAssets({
|
|
5853
5778
|
prompts: input.prompts,
|
|
5854
5779
|
sourceImages: input.sourceImages,
|
|
@@ -5874,13 +5799,13 @@ async function execute7(input, onLog, context) {
|
|
|
5874
5799
|
)
|
|
5875
5800
|
});
|
|
5876
5801
|
}
|
|
5877
|
-
var
|
|
5802
|
+
var definition6;
|
|
5878
5803
|
var init_editImages = __esm({
|
|
5879
5804
|
"src/subagents/designExpert/tools/images/editImages.ts"() {
|
|
5880
5805
|
"use strict";
|
|
5881
5806
|
init_imageGenerator();
|
|
5882
5807
|
init_surfaces();
|
|
5883
|
-
|
|
5808
|
+
definition6 = {
|
|
5884
5809
|
clearable: false,
|
|
5885
5810
|
name: "editImages",
|
|
5886
5811
|
description: "Edit or transform existing images. Provide one or more source image URLs as reference and a prompt describing the desired edit. Use for compositing, style transfer, subject transformation, blending multiple references, or incorporating one or more references into something new. Returns CDN URLs with analysis.",
|
|
@@ -5995,18 +5920,18 @@ var init_copyEditor = __esm({
|
|
|
5995
5920
|
// src/subagents/designExpert/tools/polishCopy.ts
|
|
5996
5921
|
var polishCopy_exports = {};
|
|
5997
5922
|
__export(polishCopy_exports, {
|
|
5998
|
-
definition: () =>
|
|
5999
|
-
execute: () =>
|
|
5923
|
+
definition: () => definition7,
|
|
5924
|
+
execute: () => execute7
|
|
6000
5925
|
});
|
|
6001
|
-
async function
|
|
5926
|
+
async function execute7(input, _onLog, context) {
|
|
6002
5927
|
return copyEditorTool.execute(input, context);
|
|
6003
5928
|
}
|
|
6004
|
-
var
|
|
5929
|
+
var definition7;
|
|
6005
5930
|
var init_polishCopy = __esm({
|
|
6006
5931
|
"src/subagents/designExpert/tools/polishCopy.ts"() {
|
|
6007
5932
|
"use strict";
|
|
6008
5933
|
init_copyEditor();
|
|
6009
|
-
|
|
5934
|
+
definition7 = {
|
|
6010
5935
|
clearable: false,
|
|
6011
5936
|
name: "polishCopy",
|
|
6012
5937
|
description: "Hand off any user-facing copy you've written \u2014 headlines, captions, labels, body text \u2014 and get back a sharper version: better built for its audience and free of the fingerprints that make writing read as AI. It elevates how the copy communicates without inventing facts or claims you didn't give it. Give it the text plus what it's for (where it appears, the audience).",
|
|
@@ -6043,16 +5968,19 @@ var init_tools4 = __esm({
|
|
|
6043
5968
|
init_scrapeWebUrl();
|
|
6044
5969
|
init_analyzeDesign();
|
|
6045
5970
|
init_analyzeImage2();
|
|
6046
|
-
init_screenshot3();
|
|
6047
5971
|
init_generateImages();
|
|
6048
5972
|
init_editImages();
|
|
6049
5973
|
init_polishCopy();
|
|
5974
|
+
init_screenshot2();
|
|
6050
5975
|
tools = {
|
|
6051
5976
|
searchGoogle: searchGoogle_exports,
|
|
6052
5977
|
scrapeWebUrl: scrapeWebUrl_exports,
|
|
6053
5978
|
analyzeDesign: analyzeDesign_exports,
|
|
6054
5979
|
analyzeImage: analyzeImage_exports,
|
|
6055
|
-
|
|
5980
|
+
// Same tool the main agent offers, imported rather than reimplemented — the
|
|
5981
|
+
// two used to be near-identical copies and had already drifted apart. Its core
|
|
5982
|
+
// already takes (input, onLog, context), which is this registry's convention.
|
|
5983
|
+
screenshot: { definition: screenshotDefinition, execute: executeScreenshot },
|
|
6056
5984
|
generateImages: generateImages_exports,
|
|
6057
5985
|
editImages: editImages_exports,
|
|
6058
5986
|
polishCopy: polishCopy_exports
|
|
@@ -43,6 +43,7 @@ Note: the snapshot concatenates inline text and strips whitespace. If you need t
|
|
|
43
43
|
- `navigate`: Navigate to a new URL within the app. Waits for the new page to load before continuing with subsequent steps. Use this instead of evaluate with `window.location.href` when you need to navigate and then continue interacting with the new page. Steps after navigate execute on the new page automatically.
|
|
44
44
|
- `evaluate`: Run arbitrary JavaScript in the page and return the result.
|
|
45
45
|
- `styles`: Read computed CSS styles from page elements. Pass a `properties` array with camelCase CSS property names (e.g., `["backgroundColor", "borderRadius", "fontSize"]`). Omit `properties` for a default set covering colors, typography, spacing, borders, shadows, dimensions, and layout. Uses the same targeting as click/type (ref, text, role, label, selector). Omit the target to get styles for all elements from the last snapshot.
|
|
46
|
+
- `screenshotFullPage`: Take a screenshot of the whole page, top to bottom. Returns CDN url with full text analysis and dimensions. Use for overall composition or content past the fold.
|
|
46
47
|
- `screenshotViewport`: Take a screenshot of the visible viewport. Returns CDN url with full text analysis and dimensions. To capture a specific section, set `scrollToSelector` (a CSS selector) — or `scrollY` (an absolute offset) — on this same step; it scrolls the target into view and captures it atomically, so you do NOT need a separate scroll step. Do not use if you can get what you need with other tools - only use when you need to visually see the viewport.
|
|
47
48
|
- `setViewport`: Switch the browser between desktop and mobile rendering. Set `mode` to `"desktop"` or `"mobile"`. Mobile emulates a phone (390-wide, touch, device pixel ratio 2); desktop is the standard wide viewport. This reloads the page so media queries, responsive layouts, and `matchMedia` re-evaluate — the reload clears in-page state, so switch before you set up the state you want to inspect. The mode persists across navigations within a run. Each run starts in the app's default mode, so only use this when you need to check the other one.
|
|
48
49
|
|
|
@@ -161,7 +162,7 @@ Check a count with evaluate:
|
|
|
161
162
|
|
|
162
163
|
### Final Screenshot
|
|
163
164
|
How you take the final screenshot depends on what the task asked for:
|
|
164
|
-
- **Whole page** → use
|
|
165
|
+
- **Whole page** → use a `browserCommand` batch ending in a `screenshotFullPage` step. Returns the URL plus a full-text description.
|
|
165
166
|
- **A specific section / viewport** → use a `browserCommand` batch ending in a `screenshotViewport` step with `scrollToSelector` set to the section (e.g. `{ "command": "screenshotViewport", "scrollToSelector": "#pricing" }`). This scrolls the section into view and captures it in one atomic step. Do this rather than a separate scroll step followed by a capture — capturing the viewport is only reliable when the scroll and the shot are in the same step.
|
|
166
167
|
|
|
167
168
|
<rules>
|