@mindstudio-ai/remy 0.1.254 → 0.1.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/headless.js +189 -239
- package/dist/index.js +199 -259
- package/dist/subagents/browserAutomation/prompt.md +2 -1
- package/package.json +1 -1
package/dist/headless.js
CHANGED
|
@@ -2764,6 +2764,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2764
2764
|
throw new Error("Sidecar not available");
|
|
2765
2765
|
}
|
|
2766
2766
|
const url = `${baseUrl}${endpoint}`;
|
|
2767
|
+
let data;
|
|
2767
2768
|
try {
|
|
2768
2769
|
const res = await fetch(url, {
|
|
2769
2770
|
method: "POST",
|
|
@@ -2775,12 +2776,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2775
2776
|
log4.error("Sidecar error", { endpoint, status: res.status });
|
|
2776
2777
|
throw new Error(`Sidecar error: ${res.status}`);
|
|
2777
2778
|
}
|
|
2778
|
-
|
|
2779
|
-
if (data?.success === false) {
|
|
2780
|
-
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
2781
|
-
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
2782
|
-
}
|
|
2783
|
-
return data;
|
|
2779
|
+
data = await res.json();
|
|
2784
2780
|
} catch (err) {
|
|
2785
2781
|
if (err.message.startsWith("Sidecar error")) {
|
|
2786
2782
|
throw err;
|
|
@@ -2788,6 +2784,16 @@ async function sidecarRequest(endpoint, body = {}, options) {
|
|
|
2788
2784
|
log4.error("Sidecar connection error", { endpoint, error: err.message });
|
|
2789
2785
|
throw new Error(`Sidecar connection error: ${err.message}`);
|
|
2790
2786
|
}
|
|
2787
|
+
if (data?.success === false) {
|
|
2788
|
+
const code = data.errorCode ? ` [${data.errorCode}]` : "";
|
|
2789
|
+
log4.error("Sidecar command failed", {
|
|
2790
|
+
endpoint,
|
|
2791
|
+
error: data.error,
|
|
2792
|
+
errorCode: data.errorCode
|
|
2793
|
+
});
|
|
2794
|
+
throw new Error(`${data.error || "Unknown error"}${code}`);
|
|
2795
|
+
}
|
|
2796
|
+
return data;
|
|
2791
2797
|
}
|
|
2792
2798
|
|
|
2793
2799
|
// src/tools/_helpers/lsp.ts
|
|
@@ -2973,6 +2979,8 @@ async function analyzeImage(params) {
|
|
|
2973
2979
|
}
|
|
2974
2980
|
|
|
2975
2981
|
// src/tools/_helpers/screenshot.ts
|
|
2982
|
+
var VIEWPORT_CAPTURE_TIMEOUT_MS = 45e3;
|
|
2983
|
+
var FULLPAGE_CAPTURE_TIMEOUT_MS = 135e3;
|
|
2976
2984
|
var SCREENSHOT_ANALYSIS_PROMPT = `Describe everything visible on screen from top to bottom \u2014 every element, its position, its size relative to the viewport, its colors, its content. Be comprehensive, thorough, and spatial. After the inventory, note anything that looks visually broken (overlapping elements, clipped text, misaligned components).`;
|
|
2977
2985
|
var ANALYSIS_RESPONSE_FORMAT = `Respond only with your analysis as Markdown and absolutely no other text. Do not use emojis - use unicode if you need symbols.`;
|
|
2978
2986
|
function buildScreenshotAnalysisPrompt(opts) {
|
|
@@ -3048,7 +3056,9 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3048
3056
|
...height != null ? { height } : {},
|
|
3049
3057
|
...format ? { format } : {}
|
|
3050
3058
|
},
|
|
3051
|
-
{
|
|
3059
|
+
{
|
|
3060
|
+
timeout: fullPage ? FULLPAGE_CAPTURE_TIMEOUT_MS : VIEWPORT_CAPTURE_TIMEOUT_MS
|
|
3061
|
+
}
|
|
3052
3062
|
);
|
|
3053
3063
|
url = ssResult?.url || ssResult?.screenshotUrl;
|
|
3054
3064
|
if (!url) {
|
|
@@ -3250,6 +3260,9 @@ function findLastSummaryCheckpoint(messages, name) {
|
|
|
3250
3260
|
}
|
|
3251
3261
|
return -1;
|
|
3252
3262
|
}
|
|
3263
|
+
function portableToolCallId(id) {
|
|
3264
|
+
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
3265
|
+
}
|
|
3253
3266
|
function fixOrphanedToolCalls(messages) {
|
|
3254
3267
|
const toolResultIds = /* @__PURE__ */ new Set();
|
|
3255
3268
|
for (const msg of messages) {
|
|
@@ -3341,14 +3354,30 @@ ${summaryBlock.text}
|
|
|
3341
3354
|
|
|
3342
3355
|
${content}` : attachmentHeader;
|
|
3343
3356
|
}
|
|
3344
|
-
return {
|
|
3357
|
+
return {
|
|
3358
|
+
...rest,
|
|
3359
|
+
content,
|
|
3360
|
+
...msg.toolCallId && {
|
|
3361
|
+
toolCallId: portableToolCallId(msg.toolCallId)
|
|
3362
|
+
}
|
|
3363
|
+
};
|
|
3345
3364
|
}
|
|
3346
3365
|
if (!Array.isArray(msg.content)) {
|
|
3347
3366
|
return msg;
|
|
3348
3367
|
}
|
|
3349
3368
|
const blocks = msg.content;
|
|
3350
3369
|
const text = blocks.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
3351
|
-
const
|
|
3370
|
+
const toolBlocks = blocks.filter(
|
|
3371
|
+
(b) => b.type === "tool"
|
|
3372
|
+
);
|
|
3373
|
+
const toolCalls = toolBlocks.map((b) => ({
|
|
3374
|
+
id: portableToolCallId(b.id),
|
|
3375
|
+
name: b.name,
|
|
3376
|
+
input: b.input
|
|
3377
|
+
}));
|
|
3378
|
+
const rewroteToolIds = toolBlocks.some(
|
|
3379
|
+
(b) => portableToolCallId(b.id) !== b.id
|
|
3380
|
+
);
|
|
3352
3381
|
const cleaned2 = {
|
|
3353
3382
|
role: msg.role,
|
|
3354
3383
|
content: text
|
|
@@ -3356,7 +3385,7 @@ ${content}` : attachmentHeader;
|
|
|
3356
3385
|
if (toolCalls.length > 0) {
|
|
3357
3386
|
cleaned2.toolCalls = toolCalls;
|
|
3358
3387
|
}
|
|
3359
|
-
if (msg.providerMetadata) {
|
|
3388
|
+
if (msg.providerMetadata && !rewroteToolIds) {
|
|
3360
3389
|
cleaned2.providerMetadata = msg.providerMetadata;
|
|
3361
3390
|
}
|
|
3362
3391
|
if (msg.hidden) {
|
|
@@ -3874,7 +3903,7 @@ var BROWSER_TOOLS = [
|
|
|
3874
3903
|
"screenshotViewport",
|
|
3875
3904
|
"setViewport"
|
|
3876
3905
|
],
|
|
3877
|
-
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage:
|
|
3906
|
+
description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: screenshot of the whole page top-to-bottom (returns a CDN url with dimensions and a written analysis). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
|
|
3878
3907
|
},
|
|
3879
3908
|
ref: {
|
|
3880
3909
|
type: "string",
|
|
@@ -3942,20 +3971,11 @@ var BROWSER_TOOLS = [
|
|
|
3942
3971
|
required: ["steps"]
|
|
3943
3972
|
}
|
|
3944
3973
|
},
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
type: "object",
|
|
3951
|
-
properties: {
|
|
3952
|
-
path: {
|
|
3953
|
-
type: "string",
|
|
3954
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
3955
|
-
}
|
|
3956
|
-
}
|
|
3957
|
-
}
|
|
3958
|
-
},
|
|
3974
|
+
// Captures are `browserCommand` steps only — there is deliberately no
|
|
3975
|
+
// standalone screenshot tool here. Both used to exist for full-page, with
|
|
3976
|
+
// different budgets, different result plumbing, and analysis on only one of
|
|
3977
|
+
// them, so which door you picked changed what you got back.
|
|
3978
|
+
//
|
|
3959
3979
|
// Read tools so the QA agent can pull full spec detail on demand — the spec
|
|
3960
3980
|
// context in its prompt is a lightweight index (see prompt.ts) that points
|
|
3961
3981
|
// here. Routed to the global executeTool in index.ts, mirroring specSync.
|
|
@@ -4149,6 +4169,7 @@ function getBrowserAutomationPrompt() {
|
|
|
4149
4169
|
|
|
4150
4170
|
// src/subagents/browserAutomation/index.ts
|
|
4151
4171
|
var log7 = createLogger("browser-automation");
|
|
4172
|
+
var CAPTURE_COMMANDS = /* @__PURE__ */ new Set(["screenshotViewport", "screenshotFullPage"]);
|
|
4152
4173
|
async function runBrowserAutomation(task, context, opts) {
|
|
4153
4174
|
const release = await acquireBrowserLock();
|
|
4154
4175
|
try {
|
|
@@ -4160,7 +4181,7 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4160
4181
|
);
|
|
4161
4182
|
} catch {
|
|
4162
4183
|
}
|
|
4163
|
-
let
|
|
4184
|
+
let lastCapture = {};
|
|
4164
4185
|
const result = await runSubAgent({
|
|
4165
4186
|
system: getBrowserAutomationPrompt(),
|
|
4166
4187
|
task,
|
|
@@ -4182,22 +4203,6 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4182
4203
|
return `Error setting up browser: ${err.message}`;
|
|
4183
4204
|
}
|
|
4184
4205
|
}
|
|
4185
|
-
if (name === "screenshotFullPage") {
|
|
4186
|
-
try {
|
|
4187
|
-
return await captureAndAnalyzeScreenshot({
|
|
4188
|
-
path: _input.path,
|
|
4189
|
-
fullPage: true,
|
|
4190
|
-
onLog,
|
|
4191
|
-
model: resolveModel(
|
|
4192
|
-
"imageAnalysis",
|
|
4193
|
-
context.models,
|
|
4194
|
-
context.model
|
|
4195
|
-
)
|
|
4196
|
-
});
|
|
4197
|
-
} catch (err) {
|
|
4198
|
-
return `Error taking screenshot: ${err.message}`;
|
|
4199
|
-
}
|
|
4200
|
-
}
|
|
4201
4206
|
if (COMMON_READ_TOOL_NAMES.has(name) || name === readSpecTool.definition.name) {
|
|
4202
4207
|
return executeTool(name, _input, context);
|
|
4203
4208
|
}
|
|
@@ -4219,14 +4224,16 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4219
4224
|
try {
|
|
4220
4225
|
const parsed = JSON.parse(result2);
|
|
4221
4226
|
const screenshotSteps = (parsed.steps || []).filter(
|
|
4222
|
-
(s) => s.command
|
|
4227
|
+
(s) => CAPTURE_COMMANDS.has(s.command) && s.result?.url
|
|
4223
4228
|
);
|
|
4224
4229
|
if (screenshotSteps.length > 0) {
|
|
4225
|
-
const
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
+
for (const step of screenshotSteps) {
|
|
4231
|
+
const kind = step.command === "screenshotFullPage" ? "fullPage" : "viewport";
|
|
4232
|
+
lastCapture[kind] = {
|
|
4233
|
+
url: step.result.url,
|
|
4234
|
+
styleMap: step.result.styleMap
|
|
4235
|
+
};
|
|
4236
|
+
}
|
|
4230
4237
|
const visionOverride = {
|
|
4231
4238
|
model: resolveModel(
|
|
4232
4239
|
"imageAnalysis",
|
|
@@ -4250,13 +4257,12 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4250
4257
|
);
|
|
4251
4258
|
try {
|
|
4252
4259
|
const analyses = JSON.parse(batchResult);
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
|
|
4257
|
-
ai++;
|
|
4260
|
+
screenshotSteps.forEach((step, i) => {
|
|
4261
|
+
if (i >= analyses.length) {
|
|
4262
|
+
return;
|
|
4258
4263
|
}
|
|
4259
|
-
|
|
4264
|
+
step.result.analysis = analyses[i]?.output?.analysis || analyses[i]?.output || "";
|
|
4265
|
+
});
|
|
4260
4266
|
} catch {
|
|
4261
4267
|
log7.debug("Failed to parse batch analysis result", {
|
|
4262
4268
|
batchResult
|
|
@@ -4269,13 +4275,10 @@ async function runBrowserAutomation(task, context, opts) {
|
|
|
4269
4275
|
}
|
|
4270
4276
|
return result2;
|
|
4271
4277
|
},
|
|
4272
|
-
toolRegistry: context.toolRegistry
|
|
4273
|
-
captureArtifacts: ["screenshotFullPage"]
|
|
4278
|
+
toolRegistry: context.toolRegistry
|
|
4274
4279
|
});
|
|
4275
4280
|
context.subAgentMessages?.set(context.toolCallId, result.messages);
|
|
4276
|
-
const
|
|
4277
|
-
const viewport = lastBrowserCommandViewport;
|
|
4278
|
-
const preferred = opts?.capture === "viewport" ? viewport ?? fullPage : fullPage ?? viewport;
|
|
4281
|
+
const preferred = opts?.capture === "viewport" ? lastCapture.viewport ?? lastCapture.fullPage : lastCapture.fullPage ?? lastCapture.viewport;
|
|
4279
4282
|
return {
|
|
4280
4283
|
text: result.text,
|
|
4281
4284
|
...preferred?.url ? { screenshot: { url: preferred.url, styleMap: preferred.styleMap } } : {}
|
|
@@ -4315,98 +4318,102 @@ var browserAutomationTool = {
|
|
|
4315
4318
|
};
|
|
4316
4319
|
|
|
4317
4320
|
// src/tools/code/screenshot.ts
|
|
4318
|
-
var
|
|
4321
|
+
var screenshotDefinition = {
|
|
4319
4322
|
clearable: true,
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4329
|
-
},
|
|
4330
|
-
prompt: {
|
|
4331
|
-
type: "string",
|
|
4332
|
-
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
4333
|
-
},
|
|
4334
|
-
imageUrl: {
|
|
4335
|
-
type: "string",
|
|
4336
|
-
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
4337
|
-
},
|
|
4338
|
-
path: {
|
|
4339
|
-
type: "string",
|
|
4340
|
-
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
4341
|
-
},
|
|
4342
|
-
width: {
|
|
4343
|
-
type: "number",
|
|
4344
|
-
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
4345
|
-
},
|
|
4346
|
-
height: {
|
|
4347
|
-
type: "number",
|
|
4348
|
-
description: "Exact capture height in pixels. Set together with `width`."
|
|
4349
|
-
},
|
|
4350
|
-
format: {
|
|
4351
|
-
type: "string",
|
|
4352
|
-
enum: ["png", "jpeg"],
|
|
4353
|
-
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
4354
|
-
},
|
|
4355
|
-
instructions: {
|
|
4356
|
-
type: "string",
|
|
4357
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
4358
|
-
}
|
|
4323
|
+
name: "screenshot",
|
|
4324
|
+
description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. The analysis is not precise about every detail \u2014 for example it cannot reliably identify specific fonts by name, only describe what the letterforms look like. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
|
|
4325
|
+
inputSchema: {
|
|
4326
|
+
type: "object",
|
|
4327
|
+
properties: {
|
|
4328
|
+
fullPage: {
|
|
4329
|
+
type: "boolean",
|
|
4330
|
+
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4359
4331
|
},
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4389
|
-
});
|
|
4332
|
+
prompt: {
|
|
4333
|
+
type: "string",
|
|
4334
|
+
description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
|
|
4335
|
+
},
|
|
4336
|
+
imageUrl: {
|
|
4337
|
+
type: "string",
|
|
4338
|
+
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
4339
|
+
},
|
|
4340
|
+
path: {
|
|
4341
|
+
type: "string",
|
|
4342
|
+
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
4343
|
+
},
|
|
4344
|
+
width: {
|
|
4345
|
+
type: "number",
|
|
4346
|
+
description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
|
|
4347
|
+
},
|
|
4348
|
+
height: {
|
|
4349
|
+
type: "number",
|
|
4350
|
+
description: "Exact capture height in pixels. Set together with `width`."
|
|
4351
|
+
},
|
|
4352
|
+
format: {
|
|
4353
|
+
type: "string",
|
|
4354
|
+
enum: ["png", "jpeg"],
|
|
4355
|
+
description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
|
|
4356
|
+
},
|
|
4357
|
+
instructions: {
|
|
4358
|
+
type: "string",
|
|
4359
|
+
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
|
|
4390
4360
|
}
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4361
|
+
},
|
|
4362
|
+
required: ["fullPage"]
|
|
4363
|
+
}
|
|
4364
|
+
};
|
|
4365
|
+
async function executeScreenshot(input, onLog, context) {
|
|
4366
|
+
const fullPage = input.fullPage === true;
|
|
4367
|
+
const model = resolveModel("imageAnalysis", context?.models, context?.model);
|
|
4368
|
+
try {
|
|
4369
|
+
if (input.imageUrl) {
|
|
4370
|
+
return await captureAndAnalyzeScreenshot({
|
|
4371
|
+
prompt: input.prompt,
|
|
4372
|
+
imageUrl: input.imageUrl,
|
|
4373
|
+
onLog,
|
|
4374
|
+
model
|
|
4375
|
+
});
|
|
4376
|
+
}
|
|
4377
|
+
if (input.instructions && context) {
|
|
4378
|
+
const shotKind = fullPage ? "full-page" : "viewport";
|
|
4379
|
+
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
4380
|
+
const result = await runBrowserAutomation(task, context, {
|
|
4381
|
+
capture: fullPage ? "fullPage" : "viewport"
|
|
4382
|
+
});
|
|
4383
|
+
if (!result.screenshot) {
|
|
4384
|
+
return result.text;
|
|
4405
4385
|
}
|
|
4406
|
-
|
|
4407
|
-
|
|
4386
|
+
return await streamScreenshotAnalysis({
|
|
4387
|
+
url: result.screenshot.url,
|
|
4388
|
+
prompt: input.prompt,
|
|
4389
|
+
styleMap: result.screenshot.styleMap,
|
|
4390
|
+
onLog,
|
|
4391
|
+
model
|
|
4392
|
+
});
|
|
4408
4393
|
}
|
|
4394
|
+
const release = await acquireBrowserLock();
|
|
4395
|
+
try {
|
|
4396
|
+
return await captureAndAnalyzeScreenshot({
|
|
4397
|
+
prompt: input.prompt,
|
|
4398
|
+
path: input.path,
|
|
4399
|
+
fullPage,
|
|
4400
|
+
width: input.width,
|
|
4401
|
+
height: input.height,
|
|
4402
|
+
format: input.format,
|
|
4403
|
+
onLog,
|
|
4404
|
+
model
|
|
4405
|
+
});
|
|
4406
|
+
} finally {
|
|
4407
|
+
release();
|
|
4408
|
+
}
|
|
4409
|
+
} catch (err) {
|
|
4410
|
+
return `Error taking screenshot: ${err.message}`;
|
|
4409
4411
|
}
|
|
4412
|
+
}
|
|
4413
|
+
var screenshotTool = {
|
|
4414
|
+
clearable: true,
|
|
4415
|
+
definition: screenshotDefinition,
|
|
4416
|
+
execute: (input, context) => executeScreenshot(input, context?.onLog, context)
|
|
4410
4417
|
};
|
|
4411
4418
|
|
|
4412
4419
|
// src/subagents/designExpert/tools/searchGoogle.ts
|
|
@@ -4603,83 +4610,11 @@ async function execute4(input, onLog, context) {
|
|
|
4603
4610
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4604
4611
|
}
|
|
4605
4612
|
|
|
4606
|
-
// src/subagents/designExpert/tools/screenshot.ts
|
|
4607
|
-
var screenshot_exports = {};
|
|
4608
|
-
__export(screenshot_exports, {
|
|
4609
|
-
definition: () => definition5,
|
|
4610
|
-
execute: () => execute5
|
|
4611
|
-
});
|
|
4612
|
-
var definition5 = {
|
|
4613
|
-
clearable: true,
|
|
4614
|
-
name: "screenshot",
|
|
4615
|
-
description: "Capture a screenshot of the current app preview and get it back with visual analysis. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 use it to review a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 use it to review overall composition or a layout you can't see in one screen). Use to review the current state of the UI being built. Remember, the screenshot analysis is not overly precise - for example, it cannot reliably identify specific fonts by name \u2014 it can only describe what letterforms look like.",
|
|
4616
|
-
inputSchema: {
|
|
4617
|
-
type: "object",
|
|
4618
|
-
properties: {
|
|
4619
|
-
fullPage: {
|
|
4620
|
-
type: "boolean",
|
|
4621
|
-
description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
|
|
4622
|
-
},
|
|
4623
|
-
prompt: {
|
|
4624
|
-
type: "string",
|
|
4625
|
-
description: "Optional specific question about the screenshot. Use a bulleted list to ask many questions at once."
|
|
4626
|
-
},
|
|
4627
|
-
path: {
|
|
4628
|
-
type: "string",
|
|
4629
|
-
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
4630
|
-
},
|
|
4631
|
-
instructions: {
|
|
4632
|
-
type: "string",
|
|
4633
|
-
description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, scrolling to a specific section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing."
|
|
4634
|
-
}
|
|
4635
|
-
},
|
|
4636
|
-
required: ["fullPage"]
|
|
4637
|
-
}
|
|
4638
|
-
};
|
|
4639
|
-
async function execute5(input, onLog, context) {
|
|
4640
|
-
const fullPage = input.fullPage === true;
|
|
4641
|
-
const shotKind = fullPage ? "full-page" : "viewport";
|
|
4642
|
-
if (input.instructions && context) {
|
|
4643
|
-
try {
|
|
4644
|
-
const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
|
|
4645
|
-
const result = await runBrowserAutomation(task, context, {
|
|
4646
|
-
capture: fullPage ? "fullPage" : "viewport"
|
|
4647
|
-
});
|
|
4648
|
-
if (!result.screenshot) {
|
|
4649
|
-
return result.text;
|
|
4650
|
-
}
|
|
4651
|
-
return await streamScreenshotAnalysis({
|
|
4652
|
-
url: result.screenshot.url,
|
|
4653
|
-
prompt: input.prompt,
|
|
4654
|
-
styleMap: result.screenshot.styleMap,
|
|
4655
|
-
onLog,
|
|
4656
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4657
|
-
});
|
|
4658
|
-
} catch (err) {
|
|
4659
|
-
return `Error taking interactive screenshot: ${err.message}`;
|
|
4660
|
-
}
|
|
4661
|
-
}
|
|
4662
|
-
const release = await acquireBrowserLock();
|
|
4663
|
-
try {
|
|
4664
|
-
return await captureAndAnalyzeScreenshot({
|
|
4665
|
-
prompt: input.prompt,
|
|
4666
|
-
path: input.path,
|
|
4667
|
-
fullPage,
|
|
4668
|
-
onLog,
|
|
4669
|
-
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4670
|
-
});
|
|
4671
|
-
} catch (err) {
|
|
4672
|
-
return `Error taking screenshot: ${err.message}`;
|
|
4673
|
-
} finally {
|
|
4674
|
-
release();
|
|
4675
|
-
}
|
|
4676
|
-
}
|
|
4677
|
-
|
|
4678
4613
|
// src/subagents/designExpert/tools/images/generateImages.ts
|
|
4679
4614
|
var generateImages_exports = {};
|
|
4680
4615
|
__export(generateImages_exports, {
|
|
4681
|
-
definition: () =>
|
|
4682
|
-
execute: () =>
|
|
4616
|
+
definition: () => definition5,
|
|
4617
|
+
execute: () => execute5
|
|
4683
4618
|
});
|
|
4684
4619
|
|
|
4685
4620
|
// src/subagents/designExpert/tools/images/enhancePrompt.ts
|
|
@@ -4872,7 +4807,7 @@ async function generateImageAssets(opts) {
|
|
|
4872
4807
|
}
|
|
4873
4808
|
|
|
4874
4809
|
// src/subagents/designExpert/tools/images/generateImages.ts
|
|
4875
|
-
var
|
|
4810
|
+
var definition5 = {
|
|
4876
4811
|
clearable: false,
|
|
4877
4812
|
name: "generateImages",
|
|
4878
4813
|
description: "Generate images. Returns CDN URLs with a quality analysis for each image. Produces high-quality results for everything from photorealistic images and abstract/creative visuals. Pass multiple prompts to generate in parallel. No need to analyze images separately after generating \u2014 the analysis is included.",
|
|
@@ -4906,7 +4841,7 @@ var definition6 = {
|
|
|
4906
4841
|
required: ["prompts"]
|
|
4907
4842
|
}
|
|
4908
4843
|
};
|
|
4909
|
-
async function
|
|
4844
|
+
async function execute5(input, onLog, context) {
|
|
4910
4845
|
return generateImageAssets({
|
|
4911
4846
|
prompts: input.prompts,
|
|
4912
4847
|
width: input.width,
|
|
@@ -4936,10 +4871,10 @@ async function execute6(input, onLog, context) {
|
|
|
4936
4871
|
// src/subagents/designExpert/tools/images/editImages.ts
|
|
4937
4872
|
var editImages_exports = {};
|
|
4938
4873
|
__export(editImages_exports, {
|
|
4939
|
-
definition: () =>
|
|
4940
|
-
execute: () =>
|
|
4874
|
+
definition: () => definition6,
|
|
4875
|
+
execute: () => execute6
|
|
4941
4876
|
});
|
|
4942
|
-
var
|
|
4877
|
+
var definition6 = {
|
|
4943
4878
|
clearable: false,
|
|
4944
4879
|
name: "editImages",
|
|
4945
4880
|
description: "Edit or transform existing images. Provide one or more source image URLs as reference and a prompt describing the desired edit. Use for compositing, style transfer, subject transformation, blending multiple references, or incorporating one or more references into something new. Returns CDN URLs with analysis.",
|
|
@@ -4976,7 +4911,7 @@ var definition7 = {
|
|
|
4976
4911
|
required: ["prompts", "sourceImages"]
|
|
4977
4912
|
}
|
|
4978
4913
|
};
|
|
4979
|
-
async function
|
|
4914
|
+
async function execute6(input, onLog, context) {
|
|
4980
4915
|
return generateImageAssets({
|
|
4981
4916
|
prompts: input.prompts,
|
|
4982
4917
|
sourceImages: input.sourceImages,
|
|
@@ -5006,8 +4941,8 @@ async function execute7(input, onLog, context) {
|
|
|
5006
4941
|
// src/subagents/designExpert/tools/polishCopy.ts
|
|
5007
4942
|
var polishCopy_exports = {};
|
|
5008
4943
|
__export(polishCopy_exports, {
|
|
5009
|
-
definition: () =>
|
|
5010
|
-
execute: () =>
|
|
4944
|
+
definition: () => definition7,
|
|
4945
|
+
execute: () => execute7
|
|
5011
4946
|
});
|
|
5012
4947
|
|
|
5013
4948
|
// src/subagents/copyEditor/tools.ts
|
|
@@ -5064,7 +4999,7 @@ var copyEditorTool = {
|
|
|
5064
4999
|
};
|
|
5065
5000
|
|
|
5066
5001
|
// src/subagents/designExpert/tools/polishCopy.ts
|
|
5067
|
-
var
|
|
5002
|
+
var definition7 = {
|
|
5068
5003
|
clearable: false,
|
|
5069
5004
|
name: "polishCopy",
|
|
5070
5005
|
description: "Hand off any user-facing copy you've written \u2014 headlines, captions, labels, body text \u2014 and get back a sharper version: better built for its audience and free of the fingerprints that make writing read as AI. It elevates how the copy communicates without inventing facts or claims you didn't give it. Give it the text plus what it's for (where it appears, the audience).",
|
|
@@ -5079,7 +5014,7 @@ var definition8 = {
|
|
|
5079
5014
|
required: ["task"]
|
|
5080
5015
|
}
|
|
5081
5016
|
};
|
|
5082
|
-
async function
|
|
5017
|
+
async function execute7(input, _onLog, context) {
|
|
5083
5018
|
return copyEditorTool.execute(input, context);
|
|
5084
5019
|
}
|
|
5085
5020
|
|
|
@@ -5089,7 +5024,10 @@ var tools = {
|
|
|
5089
5024
|
scrapeWebUrl: scrapeWebUrl_exports,
|
|
5090
5025
|
analyzeDesign: analyzeDesign_exports,
|
|
5091
5026
|
analyzeImage: analyzeImage_exports,
|
|
5092
|
-
|
|
5027
|
+
// Same tool the main agent offers, imported rather than reimplemented — the
|
|
5028
|
+
// two used to be near-identical copies and had already drifted apart. Its core
|
|
5029
|
+
// already takes (input, onLog, context), which is this registry's convention.
|
|
5030
|
+
screenshot: { definition: screenshotDefinition, execute: executeScreenshot },
|
|
5093
5031
|
generateImages: generateImages_exports,
|
|
5094
5032
|
editImages: editImages_exports,
|
|
5095
5033
|
polishCopy: polishCopy_exports
|
|
@@ -5249,6 +5187,11 @@ var PROMPT_TEMPLATE = readAsset(SUBAGENT, "prompt.md").replace(/\{\{([^}]+)\}\}/
|
|
|
5249
5187
|
const k = key.trim();
|
|
5250
5188
|
return RUNTIME_PLACEHOLDERS.has(k) ? match : readAsset(SUBAGENT, k);
|
|
5251
5189
|
}).replace(/\n{3,}/g, "\n\n");
|
|
5190
|
+
var RENDER_TASK_BLOCK = `<render_task>
|
|
5191
|
+
This task is a render: you are the author of the artifact, not an advisor on it. Everything above describes your usual work \u2014 proposing a design to a developer who then builds it. Here there is no developer downstream. You write the file the brief names, using writeFile (or editFile when it already exists), and that file is the entire deliverable.
|
|
5192
|
+
|
|
5193
|
+
The guidance about specifying layouts in prose, writing implementation notes, and keeping wireframes small doesn't apply here: it exists to hand a design to someone else to build, and you are the one building it. Your reply is a receipt \u2014 one line naming what you wrote.
|
|
5194
|
+
</render_task>`;
|
|
5252
5195
|
function renderDesignSystemBlock(designSystem) {
|
|
5253
5196
|
if (!designSystem) {
|
|
5254
5197
|
return "";
|
|
@@ -5259,7 +5202,7 @@ function renderDesignSystemBlock(designSystem) {
|
|
|
5259
5202
|
"</design_system>"
|
|
5260
5203
|
].join("\n");
|
|
5261
5204
|
}
|
|
5262
|
-
function getDesignExpertPrompt(onboardingState) {
|
|
5205
|
+
function getDesignExpertPrompt(onboardingState, opts) {
|
|
5263
5206
|
const specContext = loadSpecIndex();
|
|
5264
5207
|
const indices = getSampleIndices(
|
|
5265
5208
|
{
|
|
@@ -5304,6 +5247,11 @@ ${designSystemBlock}`;
|
|
|
5304
5247
|
<project_phase>
|
|
5305
5248
|
This project is in the "${state}" phase. The codebase is a placeholder scaffold or is being generated for the first time.
|
|
5306
5249
|
</project_phase>`;
|
|
5250
|
+
}
|
|
5251
|
+
if (opts?.render) {
|
|
5252
|
+
prompt += `
|
|
5253
|
+
|
|
5254
|
+
${RENDER_TASK_BLOCK}`;
|
|
5307
5255
|
}
|
|
5308
5256
|
return prompt;
|
|
5309
5257
|
}
|
|
@@ -5365,17 +5313,19 @@ var DESIGN_EXPERT_RENDER_TOOLS = [
|
|
|
5365
5313
|
async function runDesignExpert(opts, context) {
|
|
5366
5314
|
const history = context.conversationMessages ? getSubAgentHistory(context.conversationMessages, "visualDesignExpert") : [];
|
|
5367
5315
|
return runSubAgent({
|
|
5368
|
-
system: getDesignExpertPrompt(context.onboardingState
|
|
5316
|
+
system: getDesignExpertPrompt(context.onboardingState, {
|
|
5317
|
+
render: opts.render
|
|
5318
|
+
}),
|
|
5369
5319
|
task: opts.task,
|
|
5370
5320
|
history: history.length > 0 ? history : void 0,
|
|
5371
|
-
tools: opts.
|
|
5321
|
+
tools: opts.render ? DESIGN_EXPERT_RENDER_TOOLS : DESIGN_EXPERT_TOOLS,
|
|
5372
5322
|
externalTools: /* @__PURE__ */ new Set(),
|
|
5373
5323
|
executeTool: (name, input, toolCallId, onLog, sams) => {
|
|
5374
5324
|
const childCtx = toolCallId ? { ...deriveContext(context, toolCallId), subAgentMessages: sams } : { ...context, subAgentMessages: sams };
|
|
5375
5325
|
if (COMMON_READ_TOOL_NAMES.has(name)) {
|
|
5376
5326
|
return executeTool(name, input, childCtx);
|
|
5377
5327
|
}
|
|
5378
|
-
if (opts.
|
|
5328
|
+
if (opts.render && RENDER_WRITE_TOOL_NAMES.has(name)) {
|
|
5379
5329
|
return executeTool(name, input, childCtx);
|
|
5380
5330
|
}
|
|
5381
5331
|
return executeDesignExpertTool(name, input, childCtx, toolCallId, onLog);
|
|
@@ -5436,7 +5386,7 @@ var designExpertTool = {
|
|
|
5436
5386
|
}
|
|
5437
5387
|
};
|
|
5438
5388
|
async function runDesignExpertRender(opts, context) {
|
|
5439
|
-
return runDesignExpert({ ...opts,
|
|
5389
|
+
return runDesignExpert({ ...opts, render: true }, context);
|
|
5440
5390
|
}
|
|
5441
5391
|
|
|
5442
5392
|
// src/subagents/productVision/tools.ts
|
|
@@ -5547,14 +5497,14 @@ ${unifiedDiff(filePath, oldContent, "")}`;
|
|
|
5547
5497
|
const delivery = exists ? `### Your deliverable
|
|
5548
5498
|
The pitch deck already exists at \`${filePath}\`. Read it, then update it for the new <pitch_content>, keeping the presentation scaffolding intact \u2014 change only what needs to change.
|
|
5549
5499
|
|
|
5550
|
-
|
|
5500
|
+
Then reply with a one-line summary of what you changed.` : `### Your deliverable
|
|
5551
5501
|
Write the complete pitch deck to \`${filePath}\`. It does not exist yet \u2014 start from this scaffold, keeping its progress bar, chevron navigation, keyboard navigation, and transition mechanics intact:
|
|
5552
5502
|
|
|
5553
5503
|
<pitch_deck_shell>
|
|
5554
5504
|
${PITCH_DECK_SHELL}
|
|
5555
5505
|
</pitch_deck_shell>
|
|
5556
5506
|
|
|
5557
|
-
|
|
5507
|
+
Then reply with a one-line summary of what you wrote.`;
|
|
5558
5508
|
const task = `
|
|
5559
5509
|
<pitch_content>${input.task}</pitch_content>
|
|
5560
5510
|
|
|
@@ -5959,13 +5909,13 @@ Write the complete Build Overview to \`${OVERVIEW_FILE}\`. The file does not exi
|
|
|
5959
5909
|
${OVERVIEW_SHELL}
|
|
5960
5910
|
</overview_shell>
|
|
5961
5911
|
|
|
5962
|
-
|
|
5912
|
+
Then reply with a one-line summary of what you wrote.`;
|
|
5963
5913
|
}
|
|
5964
5914
|
function refreshDelivery() {
|
|
5965
5915
|
return `### Your deliverable
|
|
5966
5916
|
The Build Overview already exists at \`${OVERVIEW_FILE}\`. Read it, then update it to reflect <overview_copy>, preserving its established skin \u2014 change only what the copy changed.
|
|
5967
5917
|
|
|
5968
|
-
|
|
5918
|
+
Then reply with a one-line summary of what you changed.`;
|
|
5969
5919
|
}
|
|
5970
5920
|
var buildOverviewTool = {
|
|
5971
5921
|
clearable: false,
|