@mindstudio-ai/remy 0.1.255 → 0.1.256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/headless.js CHANGED
@@ -2764,6 +2764,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
2764
2764
  throw new Error("Sidecar not available");
2765
2765
  }
2766
2766
  const url = `${baseUrl}${endpoint}`;
2767
+ let data;
2767
2768
  try {
2768
2769
  const res = await fetch(url, {
2769
2770
  method: "POST",
@@ -2775,12 +2776,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
2775
2776
  log4.error("Sidecar error", { endpoint, status: res.status });
2776
2777
  throw new Error(`Sidecar error: ${res.status}`);
2777
2778
  }
2778
- const data = await res.json();
2779
- if (data?.success === false) {
2780
- const code = data.errorCode ? ` [${data.errorCode}]` : "";
2781
- throw new Error(`${data.error || "Unknown error"}${code}`);
2782
- }
2783
- return data;
2779
+ data = await res.json();
2784
2780
  } catch (err) {
2785
2781
  if (err.message.startsWith("Sidecar error")) {
2786
2782
  throw err;
@@ -2788,6 +2784,16 @@ async function sidecarRequest(endpoint, body = {}, options) {
2788
2784
  log4.error("Sidecar connection error", { endpoint, error: err.message });
2789
2785
  throw new Error(`Sidecar connection error: ${err.message}`);
2790
2786
  }
2787
+ if (data?.success === false) {
2788
+ const code = data.errorCode ? ` [${data.errorCode}]` : "";
2789
+ log4.error("Sidecar command failed", {
2790
+ endpoint,
2791
+ error: data.error,
2792
+ errorCode: data.errorCode
2793
+ });
2794
+ throw new Error(`${data.error || "Unknown error"}${code}`);
2795
+ }
2796
+ return data;
2791
2797
  }
2792
2798
 
2793
2799
  // src/tools/_helpers/lsp.ts
@@ -2973,6 +2979,8 @@ async function analyzeImage(params) {
2973
2979
  }
2974
2980
 
2975
2981
  // src/tools/_helpers/screenshot.ts
2982
+ var VIEWPORT_CAPTURE_TIMEOUT_MS = 45e3;
2983
+ var FULLPAGE_CAPTURE_TIMEOUT_MS = 135e3;
2976
2984
  var SCREENSHOT_ANALYSIS_PROMPT = `Describe everything visible on screen from top to bottom \u2014 every element, its position, its size relative to the viewport, its colors, its content. Be comprehensive, thorough, and spatial. After the inventory, note anything that looks visually broken (overlapping elements, clipped text, misaligned components).`;
2977
2985
  var ANALYSIS_RESPONSE_FORMAT = `Respond only with your analysis as Markdown and absolutely no other text. Do not use emojis - use unicode if you need symbols.`;
2978
2986
  function buildScreenshotAnalysisPrompt(opts) {
@@ -3048,7 +3056,9 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
3048
3056
  ...height != null ? { height } : {},
3049
3057
  ...format ? { format } : {}
3050
3058
  },
3051
- { timeout: fullPage ? 12e4 : 3e4 }
3059
+ {
3060
+ timeout: fullPage ? FULLPAGE_CAPTURE_TIMEOUT_MS : VIEWPORT_CAPTURE_TIMEOUT_MS
3061
+ }
3052
3062
  );
3053
3063
  url = ssResult?.url || ssResult?.screenshotUrl;
3054
3064
  if (!url) {
@@ -3250,6 +3260,9 @@ function findLastSummaryCheckpoint(messages, name) {
3250
3260
  }
3251
3261
  return -1;
3252
3262
  }
3263
+ function portableToolCallId(id) {
3264
+ return id.replace(/[^a-zA-Z0-9_-]/g, "_");
3265
+ }
3253
3266
  function fixOrphanedToolCalls(messages) {
3254
3267
  const toolResultIds = /* @__PURE__ */ new Set();
3255
3268
  for (const msg of messages) {
@@ -3341,14 +3354,30 @@ ${summaryBlock.text}
3341
3354
 
3342
3355
  ${content}` : attachmentHeader;
3343
3356
  }
3344
- return { ...rest, content };
3357
+ return {
3358
+ ...rest,
3359
+ content,
3360
+ ...msg.toolCallId && {
3361
+ toolCallId: portableToolCallId(msg.toolCallId)
3362
+ }
3363
+ };
3345
3364
  }
3346
3365
  if (!Array.isArray(msg.content)) {
3347
3366
  return msg;
3348
3367
  }
3349
3368
  const blocks = msg.content;
3350
3369
  const text = blocks.filter((b) => b.type === "text").map((b) => b.text).join("");
3351
- const toolCalls = blocks.filter((b) => b.type === "tool").map((b) => ({ id: b.id, name: b.name, input: b.input }));
3370
+ const toolBlocks = blocks.filter(
3371
+ (b) => b.type === "tool"
3372
+ );
3373
+ const toolCalls = toolBlocks.map((b) => ({
3374
+ id: portableToolCallId(b.id),
3375
+ name: b.name,
3376
+ input: b.input
3377
+ }));
3378
+ const rewroteToolIds = toolBlocks.some(
3379
+ (b) => portableToolCallId(b.id) !== b.id
3380
+ );
3352
3381
  const cleaned2 = {
3353
3382
  role: msg.role,
3354
3383
  content: text
@@ -3356,7 +3385,7 @@ ${content}` : attachmentHeader;
3356
3385
  if (toolCalls.length > 0) {
3357
3386
  cleaned2.toolCalls = toolCalls;
3358
3387
  }
3359
- if (msg.providerMetadata) {
3388
+ if (msg.providerMetadata && !rewroteToolIds) {
3360
3389
  cleaned2.providerMetadata = msg.providerMetadata;
3361
3390
  }
3362
3391
  if (msg.hidden) {
@@ -3874,7 +3903,7 @@ var BROWSER_TOOLS = [
3874
3903
  "screenshotViewport",
3875
3904
  "setViewport"
3876
3905
  ],
3877
- description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: full-page viewport-stitched screenshot (returns CDN url with dimensions). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
3906
+ description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: screenshot of the whole page top-to-bottom (returns a CDN url with dimensions and a written analysis). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
3878
3907
  },
3879
3908
  ref: {
3880
3909
  type: "string",
@@ -3942,20 +3971,11 @@ var BROWSER_TOOLS = [
3942
3971
  required: ["steps"]
3943
3972
  }
3944
3973
  },
3945
- {
3946
- clearable: true,
3947
- name: "screenshotFullPage",
3948
- description: "Capture a full-height screenshot of the current page. Returns a CDN URL with full text analysis and description.",
3949
- inputSchema: {
3950
- type: "object",
3951
- properties: {
3952
- path: {
3953
- type: "string",
3954
- description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
3955
- }
3956
- }
3957
- }
3958
- },
3974
+ // Captures are `browserCommand` steps only — there is deliberately no
3975
+ // standalone screenshot tool here. Both used to exist for full-page, with
3976
+ // different budgets, different result plumbing, and analysis on only one of
3977
+ // them, so which door you picked changed what you got back.
3978
+ //
3959
3979
  // Read tools so the QA agent can pull full spec detail on demand — the spec
3960
3980
  // context in its prompt is a lightweight index (see prompt.ts) that points
3961
3981
  // here. Routed to the global executeTool in index.ts, mirroring specSync.
@@ -4149,6 +4169,7 @@ function getBrowserAutomationPrompt() {
4149
4169
 
4150
4170
  // src/subagents/browserAutomation/index.ts
4151
4171
  var log7 = createLogger("browser-automation");
4172
+ var CAPTURE_COMMANDS = /* @__PURE__ */ new Set(["screenshotViewport", "screenshotFullPage"]);
4152
4173
  async function runBrowserAutomation(task, context, opts) {
4153
4174
  const release = await acquireBrowserLock();
4154
4175
  try {
@@ -4160,7 +4181,7 @@ async function runBrowserAutomation(task, context, opts) {
4160
4181
  );
4161
4182
  } catch {
4162
4183
  }
4163
- let lastBrowserCommandViewport;
4184
+ let lastCapture = {};
4164
4185
  const result = await runSubAgent({
4165
4186
  system: getBrowserAutomationPrompt(),
4166
4187
  task,
@@ -4182,22 +4203,6 @@ async function runBrowserAutomation(task, context, opts) {
4182
4203
  return `Error setting up browser: ${err.message}`;
4183
4204
  }
4184
4205
  }
4185
- if (name === "screenshotFullPage") {
4186
- try {
4187
- return await captureAndAnalyzeScreenshot({
4188
- path: _input.path,
4189
- fullPage: true,
4190
- onLog,
4191
- model: resolveModel(
4192
- "imageAnalysis",
4193
- context.models,
4194
- context.model
4195
- )
4196
- });
4197
- } catch (err) {
4198
- return `Error taking screenshot: ${err.message}`;
4199
- }
4200
- }
4201
4206
  if (COMMON_READ_TOOL_NAMES.has(name) || name === readSpecTool.definition.name) {
4202
4207
  return executeTool(name, _input, context);
4203
4208
  }
@@ -4219,14 +4224,16 @@ async function runBrowserAutomation(task, context, opts) {
4219
4224
  try {
4220
4225
  const parsed = JSON.parse(result2);
4221
4226
  const screenshotSteps = (parsed.steps || []).filter(
4222
- (s) => s.command === "screenshotViewport" && s.result?.url
4227
+ (s) => CAPTURE_COMMANDS.has(s.command) && s.result?.url
4223
4228
  );
4224
4229
  if (screenshotSteps.length > 0) {
4225
- const lastStep = screenshotSteps[screenshotSteps.length - 1];
4226
- lastBrowserCommandViewport = {
4227
- url: lastStep.result.url,
4228
- styleMap: lastStep.result.styleMap
4229
- };
4230
+ for (const step of screenshotSteps) {
4231
+ const kind = step.command === "screenshotFullPage" ? "fullPage" : "viewport";
4232
+ lastCapture[kind] = {
4233
+ url: step.result.url,
4234
+ styleMap: step.result.styleMap
4235
+ };
4236
+ }
4230
4237
  const visionOverride = {
4231
4238
  model: resolveModel(
4232
4239
  "imageAnalysis",
@@ -4250,13 +4257,12 @@ async function runBrowserAutomation(task, context, opts) {
4250
4257
  );
4251
4258
  try {
4252
4259
  const analyses = JSON.parse(batchResult);
4253
- let ai = 0;
4254
- for (const step of parsed.steps) {
4255
- if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
4256
- step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
4257
- ai++;
4260
+ screenshotSteps.forEach((step, i) => {
4261
+ if (i >= analyses.length) {
4262
+ return;
4258
4263
  }
4259
- }
4264
+ step.result.analysis = analyses[i]?.output?.analysis || analyses[i]?.output || "";
4265
+ });
4260
4266
  } catch {
4261
4267
  log7.debug("Failed to parse batch analysis result", {
4262
4268
  batchResult
@@ -4269,13 +4275,10 @@ async function runBrowserAutomation(task, context, opts) {
4269
4275
  }
4270
4276
  return result2;
4271
4277
  },
4272
- toolRegistry: context.toolRegistry,
4273
- captureArtifacts: ["screenshotFullPage"]
4278
+ toolRegistry: context.toolRegistry
4274
4279
  });
4275
4280
  context.subAgentMessages?.set(context.toolCallId, result.messages);
4276
- const fullPage = result.artifacts?.screenshotFullPage;
4277
- const viewport = lastBrowserCommandViewport;
4278
- const preferred = opts?.capture === "viewport" ? viewport ?? fullPage : fullPage ?? viewport;
4281
+ const preferred = opts?.capture === "viewport" ? lastCapture.viewport ?? lastCapture.fullPage : lastCapture.fullPage ?? lastCapture.viewport;
4279
4282
  return {
4280
4283
  text: result.text,
4281
4284
  ...preferred?.url ? { screenshot: { url: preferred.url, styleMap: preferred.styleMap } } : {}
@@ -4315,98 +4318,102 @@ var browserAutomationTool = {
4315
4318
  };
4316
4319
 
4317
4320
  // src/tools/code/screenshot.ts
4318
- var screenshotTool = {
4321
+ var screenshotDefinition = {
4319
4322
  clearable: true,
4320
- definition: {
4321
- name: "screenshot",
4322
- description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
4323
- inputSchema: {
4324
- type: "object",
4325
- properties: {
4326
- fullPage: {
4327
- type: "boolean",
4328
- description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
4329
- },
4330
- prompt: {
4331
- type: "string",
4332
- description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
4333
- },
4334
- imageUrl: {
4335
- type: "string",
4336
- description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
4337
- },
4338
- path: {
4339
- type: "string",
4340
- description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
4341
- },
4342
- width: {
4343
- type: "number",
4344
- description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
4345
- },
4346
- height: {
4347
- type: "number",
4348
- description: "Exact capture height in pixels. Set together with `width`."
4349
- },
4350
- format: {
4351
- type: "string",
4352
- enum: ["png", "jpeg"],
4353
- description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
4354
- },
4355
- instructions: {
4356
- type: "string",
4357
- description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
4358
- }
4323
+ name: "screenshot",
4324
+ description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. The analysis is not precise about every detail \u2014 for example it cannot reliably identify specific fonts by name, only describe what the letterforms look like. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
4325
+ inputSchema: {
4326
+ type: "object",
4327
+ properties: {
4328
+ fullPage: {
4329
+ type: "boolean",
4330
+ description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
4359
4331
  },
4360
- required: ["fullPage"]
4361
- }
4362
- },
4363
- async execute(input, context) {
4364
- const fullPage = input.fullPage === true;
4365
- const shotKind = fullPage ? "full-page" : "viewport";
4366
- try {
4367
- if (input.imageUrl) {
4368
- return await captureAndAnalyzeScreenshot({
4369
- prompt: input.prompt,
4370
- imageUrl: input.imageUrl,
4371
- onLog: context?.onLog,
4372
- model: resolveModel("imageAnalysis", context?.models, context?.model)
4373
- });
4374
- }
4375
- if (input.instructions && context) {
4376
- const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
4377
- const result = await runBrowserAutomation(task, context, {
4378
- capture: fullPage ? "fullPage" : "viewport"
4379
- });
4380
- if (!result.screenshot) {
4381
- return result.text;
4382
- }
4383
- return await streamScreenshotAnalysis({
4384
- url: result.screenshot.url,
4385
- prompt: input.prompt,
4386
- styleMap: result.screenshot.styleMap,
4387
- onLog: context?.onLog,
4388
- model: resolveModel("imageAnalysis", context?.models, context?.model)
4389
- });
4332
+ prompt: {
4333
+ type: "string",
4334
+ description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
4335
+ },
4336
+ imageUrl: {
4337
+ type: "string",
4338
+ description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
4339
+ },
4340
+ path: {
4341
+ type: "string",
4342
+ description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
4343
+ },
4344
+ width: {
4345
+ type: "number",
4346
+ description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
4347
+ },
4348
+ height: {
4349
+ type: "number",
4350
+ description: "Exact capture height in pixels. Set together with `width`."
4351
+ },
4352
+ format: {
4353
+ type: "string",
4354
+ enum: ["png", "jpeg"],
4355
+ description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
4356
+ },
4357
+ instructions: {
4358
+ type: "string",
4359
+ description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
4390
4360
  }
4391
- const release = await acquireBrowserLock();
4392
- try {
4393
- return await captureAndAnalyzeScreenshot({
4394
- prompt: input.prompt,
4395
- path: input.path,
4396
- fullPage,
4397
- width: input.width,
4398
- height: input.height,
4399
- format: input.format,
4400
- onLog: context?.onLog,
4401
- model: resolveModel("imageAnalysis", context?.models, context?.model)
4402
- });
4403
- } finally {
4404
- release();
4361
+ },
4362
+ required: ["fullPage"]
4363
+ }
4364
+ };
4365
+ async function executeScreenshot(input, onLog, context) {
4366
+ const fullPage = input.fullPage === true;
4367
+ const model = resolveModel("imageAnalysis", context?.models, context?.model);
4368
+ try {
4369
+ if (input.imageUrl) {
4370
+ return await captureAndAnalyzeScreenshot({
4371
+ prompt: input.prompt,
4372
+ imageUrl: input.imageUrl,
4373
+ onLog,
4374
+ model
4375
+ });
4376
+ }
4377
+ if (input.instructions && context) {
4378
+ const shotKind = fullPage ? "full-page" : "viewport";
4379
+ const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
4380
+ const result = await runBrowserAutomation(task, context, {
4381
+ capture: fullPage ? "fullPage" : "viewport"
4382
+ });
4383
+ if (!result.screenshot) {
4384
+ return result.text;
4405
4385
  }
4406
- } catch (err) {
4407
- return `Error taking screenshot: ${err.message}`;
4386
+ return await streamScreenshotAnalysis({
4387
+ url: result.screenshot.url,
4388
+ prompt: input.prompt,
4389
+ styleMap: result.screenshot.styleMap,
4390
+ onLog,
4391
+ model
4392
+ });
4408
4393
  }
4394
+ const release = await acquireBrowserLock();
4395
+ try {
4396
+ return await captureAndAnalyzeScreenshot({
4397
+ prompt: input.prompt,
4398
+ path: input.path,
4399
+ fullPage,
4400
+ width: input.width,
4401
+ height: input.height,
4402
+ format: input.format,
4403
+ onLog,
4404
+ model
4405
+ });
4406
+ } finally {
4407
+ release();
4408
+ }
4409
+ } catch (err) {
4410
+ return `Error taking screenshot: ${err.message}`;
4409
4411
  }
4412
+ }
4413
+ var screenshotTool = {
4414
+ clearable: true,
4415
+ definition: screenshotDefinition,
4416
+ execute: (input, context) => executeScreenshot(input, context?.onLog, context)
4410
4417
  };
4411
4418
 
4412
4419
  // src/subagents/designExpert/tools/searchGoogle.ts
@@ -4603,83 +4610,11 @@ async function execute4(input, onLog, context) {
4603
4610
  return JSON.stringify({ url: imageUrl, analysis });
4604
4611
  }
4605
4612
 
4606
- // src/subagents/designExpert/tools/screenshot.ts
4607
- var screenshot_exports = {};
4608
- __export(screenshot_exports, {
4609
- definition: () => definition5,
4610
- execute: () => execute5
4611
- });
4612
- var definition5 = {
4613
- clearable: true,
4614
- name: "screenshot",
4615
- description: "Capture a screenshot of the current app preview and get it back with visual analysis. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 use it to review a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 use it to review overall composition or a layout you can't see in one screen). Use to review the current state of the UI being built. Remember, the screenshot analysis is not overly precise - for example, it cannot reliably identify specific fonts by name \u2014 it can only describe what letterforms look like.",
4616
- inputSchema: {
4617
- type: "object",
4618
- properties: {
4619
- fullPage: {
4620
- type: "boolean",
4621
- description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
4622
- },
4623
- prompt: {
4624
- type: "string",
4625
- description: "Optional specific question about the screenshot. Use a bulleted list to ask many questions at once."
4626
- },
4627
- path: {
4628
- type: "string",
4629
- description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
4630
- },
4631
- instructions: {
4632
- type: "string",
4633
- description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, scrolling to a specific section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing."
4634
- }
4635
- },
4636
- required: ["fullPage"]
4637
- }
4638
- };
4639
- async function execute5(input, onLog, context) {
4640
- const fullPage = input.fullPage === true;
4641
- const shotKind = fullPage ? "full-page" : "viewport";
4642
- if (input.instructions && context) {
4643
- try {
4644
- const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
4645
- const result = await runBrowserAutomation(task, context, {
4646
- capture: fullPage ? "fullPage" : "viewport"
4647
- });
4648
- if (!result.screenshot) {
4649
- return result.text;
4650
- }
4651
- return await streamScreenshotAnalysis({
4652
- url: result.screenshot.url,
4653
- prompt: input.prompt,
4654
- styleMap: result.screenshot.styleMap,
4655
- onLog,
4656
- model: resolveModel("imageAnalysis", context?.models, context?.model)
4657
- });
4658
- } catch (err) {
4659
- return `Error taking interactive screenshot: ${err.message}`;
4660
- }
4661
- }
4662
- const release = await acquireBrowserLock();
4663
- try {
4664
- return await captureAndAnalyzeScreenshot({
4665
- prompt: input.prompt,
4666
- path: input.path,
4667
- fullPage,
4668
- onLog,
4669
- model: resolveModel("imageAnalysis", context?.models, context?.model)
4670
- });
4671
- } catch (err) {
4672
- return `Error taking screenshot: ${err.message}`;
4673
- } finally {
4674
- release();
4675
- }
4676
- }
4677
-
4678
4613
  // src/subagents/designExpert/tools/images/generateImages.ts
4679
4614
  var generateImages_exports = {};
4680
4615
  __export(generateImages_exports, {
4681
- definition: () => definition6,
4682
- execute: () => execute6
4616
+ definition: () => definition5,
4617
+ execute: () => execute5
4683
4618
  });
4684
4619
 
4685
4620
  // src/subagents/designExpert/tools/images/enhancePrompt.ts
@@ -4872,7 +4807,7 @@ async function generateImageAssets(opts) {
4872
4807
  }
4873
4808
 
4874
4809
  // src/subagents/designExpert/tools/images/generateImages.ts
4875
- var definition6 = {
4810
+ var definition5 = {
4876
4811
  clearable: false,
4877
4812
  name: "generateImages",
4878
4813
  description: "Generate images. Returns CDN URLs with a quality analysis for each image. Produces high-quality results for everything from photorealistic images and abstract/creative visuals. Pass multiple prompts to generate in parallel. No need to analyze images separately after generating \u2014 the analysis is included.",
@@ -4906,7 +4841,7 @@ var definition6 = {
4906
4841
  required: ["prompts"]
4907
4842
  }
4908
4843
  };
4909
- async function execute6(input, onLog, context) {
4844
+ async function execute5(input, onLog, context) {
4910
4845
  return generateImageAssets({
4911
4846
  prompts: input.prompts,
4912
4847
  width: input.width,
@@ -4936,10 +4871,10 @@ async function execute6(input, onLog, context) {
4936
4871
  // src/subagents/designExpert/tools/images/editImages.ts
4937
4872
  var editImages_exports = {};
4938
4873
  __export(editImages_exports, {
4939
- definition: () => definition7,
4940
- execute: () => execute7
4874
+ definition: () => definition6,
4875
+ execute: () => execute6
4941
4876
  });
4942
- var definition7 = {
4877
+ var definition6 = {
4943
4878
  clearable: false,
4944
4879
  name: "editImages",
4945
4880
  description: "Edit or transform existing images. Provide one or more source image URLs as reference and a prompt describing the desired edit. Use for compositing, style transfer, subject transformation, blending multiple references, or incorporating one or more references into something new. Returns CDN URLs with analysis.",
@@ -4976,7 +4911,7 @@ var definition7 = {
4976
4911
  required: ["prompts", "sourceImages"]
4977
4912
  }
4978
4913
  };
4979
- async function execute7(input, onLog, context) {
4914
+ async function execute6(input, onLog, context) {
4980
4915
  return generateImageAssets({
4981
4916
  prompts: input.prompts,
4982
4917
  sourceImages: input.sourceImages,
@@ -5006,8 +4941,8 @@ async function execute7(input, onLog, context) {
5006
4941
  // src/subagents/designExpert/tools/polishCopy.ts
5007
4942
  var polishCopy_exports = {};
5008
4943
  __export(polishCopy_exports, {
5009
- definition: () => definition8,
5010
- execute: () => execute8
4944
+ definition: () => definition7,
4945
+ execute: () => execute7
5011
4946
  });
5012
4947
 
5013
4948
  // src/subagents/copyEditor/tools.ts
@@ -5064,7 +4999,7 @@ var copyEditorTool = {
5064
4999
  };
5065
5000
 
5066
5001
  // src/subagents/designExpert/tools/polishCopy.ts
5067
- var definition8 = {
5002
+ var definition7 = {
5068
5003
  clearable: false,
5069
5004
  name: "polishCopy",
5070
5005
  description: "Hand off any user-facing copy you've written \u2014 headlines, captions, labels, body text \u2014 and get back a sharper version: better built for its audience and free of the fingerprints that make writing read as AI. It elevates how the copy communicates without inventing facts or claims you didn't give it. Give it the text plus what it's for (where it appears, the audience).",
@@ -5079,7 +5014,7 @@ var definition8 = {
5079
5014
  required: ["task"]
5080
5015
  }
5081
5016
  };
5082
- async function execute8(input, _onLog, context) {
5017
+ async function execute7(input, _onLog, context) {
5083
5018
  return copyEditorTool.execute(input, context);
5084
5019
  }
5085
5020
 
@@ -5089,7 +5024,10 @@ var tools = {
5089
5024
  scrapeWebUrl: scrapeWebUrl_exports,
5090
5025
  analyzeDesign: analyzeDesign_exports,
5091
5026
  analyzeImage: analyzeImage_exports,
5092
- screenshot: screenshot_exports,
5027
+ // Same tool the main agent offers, imported rather than reimplemented — the
5028
+ // two used to be near-identical copies and had already drifted apart. Its core
5029
+ // already takes (input, onLog, context), which is this registry's convention.
5030
+ screenshot: { definition: screenshotDefinition, execute: executeScreenshot },
5093
5031
  generateImages: generateImages_exports,
5094
5032
  editImages: editImages_exports,
5095
5033
  polishCopy: polishCopy_exports
package/dist/index.js CHANGED
@@ -2238,6 +2238,9 @@ function findLastSummaryCheckpoint(messages, name) {
2238
2238
  }
2239
2239
  return -1;
2240
2240
  }
2241
+ function portableToolCallId(id) {
2242
+ return id.replace(/[^a-zA-Z0-9_-]/g, "_");
2243
+ }
2241
2244
  function fixOrphanedToolCalls(messages) {
2242
2245
  const toolResultIds = /* @__PURE__ */ new Set();
2243
2246
  for (const msg of messages) {
@@ -2329,14 +2332,30 @@ ${summaryBlock.text}
2329
2332
 
2330
2333
  ${content}` : attachmentHeader;
2331
2334
  }
2332
- return { ...rest, content };
2335
+ return {
2336
+ ...rest,
2337
+ content,
2338
+ ...msg.toolCallId && {
2339
+ toolCallId: portableToolCallId(msg.toolCallId)
2340
+ }
2341
+ };
2333
2342
  }
2334
2343
  if (!Array.isArray(msg.content)) {
2335
2344
  return msg;
2336
2345
  }
2337
2346
  const blocks = msg.content;
2338
2347
  const text = blocks.filter((b) => b.type === "text").map((b) => b.text).join("");
2339
- const toolCalls = blocks.filter((b) => b.type === "tool").map((b) => ({ id: b.id, name: b.name, input: b.input }));
2348
+ const toolBlocks = blocks.filter(
2349
+ (b) => b.type === "tool"
2350
+ );
2351
+ const toolCalls = toolBlocks.map((b) => ({
2352
+ id: portableToolCallId(b.id),
2353
+ name: b.name,
2354
+ input: b.input
2355
+ }));
2356
+ const rewroteToolIds = toolBlocks.some(
2357
+ (b) => portableToolCallId(b.id) !== b.id
2358
+ );
2340
2359
  const cleaned2 = {
2341
2360
  role: msg.role,
2342
2361
  content: text
@@ -2344,7 +2363,7 @@ ${content}` : attachmentHeader;
2344
2363
  if (toolCalls.length > 0) {
2345
2364
  cleaned2.toolCalls = toolCalls;
2346
2365
  }
2347
- if (msg.providerMetadata) {
2366
+ if (msg.providerMetadata && !rewroteToolIds) {
2348
2367
  cleaned2.providerMetadata = msg.providerMetadata;
2349
2368
  }
2350
2369
  if (msg.hidden) {
@@ -3599,6 +3618,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
3599
3618
  throw new Error("Sidecar not available");
3600
3619
  }
3601
3620
  const url = `${baseUrl}${endpoint}`;
3621
+ let data;
3602
3622
  try {
3603
3623
  const res = await fetch(url, {
3604
3624
  method: "POST",
@@ -3610,12 +3630,7 @@ async function sidecarRequest(endpoint, body = {}, options) {
3610
3630
  log5.error("Sidecar error", { endpoint, status: res.status });
3611
3631
  throw new Error(`Sidecar error: ${res.status}`);
3612
3632
  }
3613
- const data = await res.json();
3614
- if (data?.success === false) {
3615
- const code = data.errorCode ? ` [${data.errorCode}]` : "";
3616
- throw new Error(`${data.error || "Unknown error"}${code}`);
3617
- }
3618
- return data;
3633
+ data = await res.json();
3619
3634
  } catch (err) {
3620
3635
  if (err.message.startsWith("Sidecar error")) {
3621
3636
  throw err;
@@ -3623,6 +3638,16 @@ async function sidecarRequest(endpoint, body = {}, options) {
3623
3638
  log5.error("Sidecar connection error", { endpoint, error: err.message });
3624
3639
  throw new Error(`Sidecar connection error: ${err.message}`);
3625
3640
  }
3641
+ if (data?.success === false) {
3642
+ const code = data.errorCode ? ` [${data.errorCode}]` : "";
3643
+ log5.error("Sidecar command failed", {
3644
+ endpoint,
3645
+ error: data.error,
3646
+ errorCode: data.errorCode
3647
+ });
3648
+ throw new Error(`${data.error || "Unknown error"}${code}`);
3649
+ }
3650
+ return data;
3626
3651
  }
3627
3652
  var log5, baseUrl;
3628
3653
  var init_sidecar = __esm({
@@ -3935,7 +3960,9 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
3935
3960
  ...height != null ? { height } : {},
3936
3961
  ...format ? { format } : {}
3937
3962
  },
3938
- { timeout: fullPage ? 12e4 : 3e4 }
3963
+ {
3964
+ timeout: fullPage ? FULLPAGE_CAPTURE_TIMEOUT_MS : VIEWPORT_CAPTURE_TIMEOUT_MS
3965
+ }
3939
3966
  );
3940
3967
  url = ssResult?.url || ssResult?.screenshotUrl;
3941
3968
  if (!url) {
@@ -3961,12 +3988,14 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
3961
3988
  model
3962
3989
  });
3963
3990
  }
3964
- var SCREENSHOT_ANALYSIS_PROMPT, ANALYSIS_RESPONSE_FORMAT;
3991
+ var VIEWPORT_CAPTURE_TIMEOUT_MS, FULLPAGE_CAPTURE_TIMEOUT_MS, SCREENSHOT_ANALYSIS_PROMPT, ANALYSIS_RESPONSE_FORMAT;
3965
3992
  var init_screenshot = __esm({
3966
3993
  "src/tools/_helpers/screenshot.ts"() {
3967
3994
  "use strict";
3968
3995
  init_sidecar();
3969
3996
  init_analyzeImage();
3997
+ VIEWPORT_CAPTURE_TIMEOUT_MS = 45e3;
3998
+ FULLPAGE_CAPTURE_TIMEOUT_MS = 135e3;
3970
3999
  SCREENSHOT_ANALYSIS_PROMPT = `Describe everything visible on screen from top to bottom \u2014 every element, its position, its size relative to the viewport, its colors, its content. Be comprehensive, thorough, and spatial. After the inventory, note anything that looks visually broken (overlapping elements, clipped text, misaligned components).`;
3971
4000
  ANALYSIS_RESPONSE_FORMAT = `Respond only with your analysis as Markdown and absolutely no other text. Do not use emojis - use unicode if you need symbols.`;
3972
4001
  }
@@ -4673,7 +4702,7 @@ var init_tools2 = __esm({
4673
4702
  "screenshotViewport",
4674
4703
  "setViewport"
4675
4704
  ],
4676
- description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: full-page viewport-stitched screenshot (returns CDN url with dimensions). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
4705
+ description: 'snapshot: accessibility tree of the page (waits for network to settle). click: click an element (animated cursor, full event sequence). type: type text into input (one char at a time, works with React/Vue/Svelte). select: select a dropdown option by text. wait: wait for an element to appear (polls 100ms, waits for network). navigate: navigate to a URL within the app (waits for load, subsequent steps run on new page). evaluate: run JS in the page. styles: read computed CSS styles from elements (pass properties array with camelCase names, or omit for defaults). screenshotFullPage: screenshot of the whole page top-to-bottom (returns a CDN url with dimensions and a written analysis). screenshotViewport: screenshot of just the visible viewport \u2014 pass `scrollToSelector` (or `scrollY`) on this step to scroll a section into view and capture it in one atomic step (no separate scroll needed). setViewport: switch the browser between desktop and mobile rendering (pass `mode`: "desktop" or "mobile"). Reloads the page so responsive layouts, media queries, and matchMedia re-evaluate \u2014 use it to QA mobile/responsive views.'
4677
4706
  },
4678
4707
  ref: {
4679
4708
  type: "string",
@@ -4741,20 +4770,11 @@ var init_tools2 = __esm({
4741
4770
  required: ["steps"]
4742
4771
  }
4743
4772
  },
4744
- {
4745
- clearable: true,
4746
- name: "screenshotFullPage",
4747
- description: "Capture a full-height screenshot of the current page. Returns a CDN URL with full text analysis and description.",
4748
- inputSchema: {
4749
- type: "object",
4750
- properties: {
4751
- path: {
4752
- type: "string",
4753
- description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
4754
- }
4755
- }
4756
- }
4757
- },
4773
+ // Captures are `browserCommand` steps only — there is deliberately no
4774
+ // standalone screenshot tool here. Both used to exist for full-page, with
4775
+ // different budgets, different result plumbing, and analysis on only one of
4776
+ // them, so which door you picked changed what you got back.
4777
+ //
4758
4778
  // Read tools so the QA agent can pull full spec detail on demand — the spec
4759
4779
  // context in its prompt is a lightweight index (see prompt.ts) that points
4760
4780
  // here. Routed to the global executeTool in index.ts, mirroring specSync.
@@ -4973,7 +4993,7 @@ async function runBrowserAutomation(task, context, opts) {
4973
4993
  );
4974
4994
  } catch {
4975
4995
  }
4976
- let lastBrowserCommandViewport;
4996
+ let lastCapture = {};
4977
4997
  const result = await runSubAgent({
4978
4998
  system: getBrowserAutomationPrompt(),
4979
4999
  task,
@@ -4995,22 +5015,6 @@ async function runBrowserAutomation(task, context, opts) {
4995
5015
  return `Error setting up browser: ${err.message}`;
4996
5016
  }
4997
5017
  }
4998
- if (name === "screenshotFullPage") {
4999
- try {
5000
- return await captureAndAnalyzeScreenshot({
5001
- path: _input.path,
5002
- fullPage: true,
5003
- onLog,
5004
- model: resolveModel(
5005
- "imageAnalysis",
5006
- context.models,
5007
- context.model
5008
- )
5009
- });
5010
- } catch (err) {
5011
- return `Error taking screenshot: ${err.message}`;
5012
- }
5013
- }
5014
5018
  if (COMMON_READ_TOOL_NAMES.has(name) || name === readSpecTool.definition.name) {
5015
5019
  return executeTool(name, _input, context);
5016
5020
  }
@@ -5032,14 +5036,16 @@ async function runBrowserAutomation(task, context, opts) {
5032
5036
  try {
5033
5037
  const parsed = JSON.parse(result2);
5034
5038
  const screenshotSteps = (parsed.steps || []).filter(
5035
- (s) => s.command === "screenshotViewport" && s.result?.url
5039
+ (s) => CAPTURE_COMMANDS.has(s.command) && s.result?.url
5036
5040
  );
5037
5041
  if (screenshotSteps.length > 0) {
5038
- const lastStep = screenshotSteps[screenshotSteps.length - 1];
5039
- lastBrowserCommandViewport = {
5040
- url: lastStep.result.url,
5041
- styleMap: lastStep.result.styleMap
5042
- };
5042
+ for (const step of screenshotSteps) {
5043
+ const kind = step.command === "screenshotFullPage" ? "fullPage" : "viewport";
5044
+ lastCapture[kind] = {
5045
+ url: step.result.url,
5046
+ styleMap: step.result.styleMap
5047
+ };
5048
+ }
5043
5049
  const visionOverride = {
5044
5050
  model: resolveModel(
5045
5051
  "imageAnalysis",
@@ -5063,13 +5069,12 @@ async function runBrowserAutomation(task, context, opts) {
5063
5069
  );
5064
5070
  try {
5065
5071
  const analyses = JSON.parse(batchResult);
5066
- let ai = 0;
5067
- for (const step of parsed.steps) {
5068
- if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
5069
- step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
5070
- ai++;
5072
+ screenshotSteps.forEach((step, i) => {
5073
+ if (i >= analyses.length) {
5074
+ return;
5071
5075
  }
5072
- }
5076
+ step.result.analysis = analyses[i]?.output?.analysis || analyses[i]?.output || "";
5077
+ });
5073
5078
  } catch {
5074
5079
  log8.debug("Failed to parse batch analysis result", {
5075
5080
  batchResult
@@ -5082,13 +5087,10 @@ async function runBrowserAutomation(task, context, opts) {
5082
5087
  }
5083
5088
  return result2;
5084
5089
  },
5085
- toolRegistry: context.toolRegistry,
5086
- captureArtifacts: ["screenshotFullPage"]
5090
+ toolRegistry: context.toolRegistry
5087
5091
  });
5088
5092
  context.subAgentMessages?.set(context.toolCallId, result.messages);
5089
- const fullPage = result.artifacts?.screenshotFullPage;
5090
- const viewport = lastBrowserCommandViewport;
5091
- const preferred = opts?.capture === "viewport" ? viewport ?? fullPage : fullPage ?? viewport;
5093
+ const preferred = opts?.capture === "viewport" ? lastCapture.viewport ?? lastCapture.fullPage : lastCapture.fullPage ?? lastCapture.viewport;
5092
5094
  return {
5093
5095
  text: result.text,
5094
5096
  ...preferred?.url ? { screenshot: { url: preferred.url, styleMap: preferred.styleMap } } : {}
@@ -5097,7 +5099,7 @@ async function runBrowserAutomation(task, context, opts) {
5097
5099
  release();
5098
5100
  }
5099
5101
  }
5100
- var log8, browserAutomationTool;
5102
+ var log8, CAPTURE_COMMANDS, browserAutomationTool;
5101
5103
  var init_browserAutomation = __esm({
5102
5104
  "src/subagents/browserAutomation/index.ts"() {
5103
5105
  "use strict";
@@ -5114,6 +5116,7 @@ var init_browserAutomation = __esm({
5114
5116
  init_surfaces();
5115
5117
  init_logger();
5116
5118
  log8 = createLogger("browser-automation");
5119
+ CAPTURE_COMMANDS = /* @__PURE__ */ new Set(["screenshotViewport", "screenshotFullPage"]);
5117
5120
  browserAutomationTool = {
5118
5121
  clearable: true,
5119
5122
  definition: {
@@ -5147,7 +5150,55 @@ var init_browserAutomation = __esm({
5147
5150
  });
5148
5151
 
5149
5152
  // src/tools/code/screenshot.ts
5150
- var screenshotTool;
5153
+ async function executeScreenshot(input, onLog, context) {
5154
+ const fullPage = input.fullPage === true;
5155
+ const model = resolveModel("imageAnalysis", context?.models, context?.model);
5156
+ try {
5157
+ if (input.imageUrl) {
5158
+ return await captureAndAnalyzeScreenshot({
5159
+ prompt: input.prompt,
5160
+ imageUrl: input.imageUrl,
5161
+ onLog,
5162
+ model
5163
+ });
5164
+ }
5165
+ if (input.instructions && context) {
5166
+ const shotKind = fullPage ? "full-page" : "viewport";
5167
+ const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
5168
+ const result = await runBrowserAutomation(task, context, {
5169
+ capture: fullPage ? "fullPage" : "viewport"
5170
+ });
5171
+ if (!result.screenshot) {
5172
+ return result.text;
5173
+ }
5174
+ return await streamScreenshotAnalysis({
5175
+ url: result.screenshot.url,
5176
+ prompt: input.prompt,
5177
+ styleMap: result.screenshot.styleMap,
5178
+ onLog,
5179
+ model
5180
+ });
5181
+ }
5182
+ const release = await acquireBrowserLock();
5183
+ try {
5184
+ return await captureAndAnalyzeScreenshot({
5185
+ prompt: input.prompt,
5186
+ path: input.path,
5187
+ fullPage,
5188
+ width: input.width,
5189
+ height: input.height,
5190
+ format: input.format,
5191
+ onLog,
5192
+ model
5193
+ });
5194
+ } finally {
5195
+ release();
5196
+ }
5197
+ } catch (err) {
5198
+ return `Error taking screenshot: ${err.message}`;
5199
+ }
5200
+ }
5201
+ var screenshotDefinition, screenshotTool;
5151
5202
  var init_screenshot2 = __esm({
5152
5203
  "src/tools/code/screenshot.ts"() {
5153
5204
  "use strict";
@@ -5155,99 +5206,55 @@ var init_screenshot2 = __esm({
5155
5206
  init_browserLock();
5156
5207
  init_browserAutomation();
5157
5208
  init_surfaces();
5158
- screenshotTool = {
5209
+ screenshotDefinition = {
5159
5210
  clearable: true,
5160
- definition: {
5161
- name: "screenshot",
5162
- description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
5163
- inputSchema: {
5164
- type: "object",
5165
- properties: {
5166
- fullPage: {
5167
- type: "boolean",
5168
- description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
5169
- },
5170
- prompt: {
5171
- type: "string",
5172
- description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
5173
- },
5174
- imageUrl: {
5175
- type: "string",
5176
- description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
5177
- },
5178
- path: {
5179
- type: "string",
5180
- description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
5181
- },
5182
- width: {
5183
- type: "number",
5184
- description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
5185
- },
5186
- height: {
5187
- type: "number",
5188
- description: "Exact capture height in pixels. Set together with `width`."
5189
- },
5190
- format: {
5191
- type: "string",
5192
- enum: ["png", "jpeg"],
5193
- description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
5194
- },
5195
- instructions: {
5196
- type: "string",
5197
- description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
5198
- }
5211
+ name: "screenshot",
5212
+ description: "Capture a screenshot of the app preview and get a description of what's on screen. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 for a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 for overall composition or content past the fold). Captures the settled page state \u2014 it cannot catch animations, transitions, or transient state. The analysis is not precise about every detail \u2014 for example it cannot reliably identify specific fonts by name, only describe what the letterforms look like. Optionally provide specific questions about what you're looking for. Use a bulleted list to ask many questions at once. To ask additional questions about a screenshot you have already captured, pass its URL as imageUrl to skip recapture. If the screenshot requires interaction first (logging in, clicking a tab, dismissing a modal, scrolling to a section), use the instructions param to describe the steps. To render a fixed-size image such as an Open Graph share card, set `width` and `height` (e.g. 1200 \xD7 630) and `format: 'png'`: the tool navigates to `path`, clips to exactly those pixel dimensions, and returns the image URL.",
5213
+ inputSchema: {
5214
+ type: "object",
5215
+ properties: {
5216
+ fullPage: {
5217
+ type: "boolean",
5218
+ description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
5199
5219
  },
5200
- required: ["fullPage"]
5201
- }
5202
- },
5203
- async execute(input, context) {
5204
- const fullPage = input.fullPage === true;
5205
- const shotKind = fullPage ? "full-page" : "viewport";
5206
- try {
5207
- if (input.imageUrl) {
5208
- return await captureAndAnalyzeScreenshot({
5209
- prompt: input.prompt,
5210
- imageUrl: input.imageUrl,
5211
- onLog: context?.onLog,
5212
- model: resolveModel("imageAnalysis", context?.models, context?.model)
5213
- });
5214
- }
5215
- if (input.instructions && context) {
5216
- const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
5217
- const result = await runBrowserAutomation(task, context, {
5218
- capture: fullPage ? "fullPage" : "viewport"
5219
- });
5220
- if (!result.screenshot) {
5221
- return result.text;
5222
- }
5223
- return await streamScreenshotAnalysis({
5224
- url: result.screenshot.url,
5225
- prompt: input.prompt,
5226
- styleMap: result.screenshot.styleMap,
5227
- onLog: context?.onLog,
5228
- model: resolveModel("imageAnalysis", context?.models, context?.model)
5229
- });
5230
- }
5231
- const release = await acquireBrowserLock();
5232
- try {
5233
- return await captureAndAnalyzeScreenshot({
5234
- prompt: input.prompt,
5235
- path: input.path,
5236
- fullPage,
5237
- width: input.width,
5238
- height: input.height,
5239
- format: input.format,
5240
- onLog: context?.onLog,
5241
- model: resolveModel("imageAnalysis", context?.models, context?.model)
5242
- });
5243
- } finally {
5244
- release();
5220
+ prompt: {
5221
+ type: "string",
5222
+ description: "Optional question about the screenshot. If omitted, returns a general description of what's visible."
5223
+ },
5224
+ imageUrl: {
5225
+ type: "string",
5226
+ description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
5227
+ },
5228
+ path: {
5229
+ type: "string",
5230
+ description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
5231
+ },
5232
+ width: {
5233
+ type: "number",
5234
+ description: "Exact capture width in pixels. Set together with `height` to render a fixed-size image; clips to exactly this viewport instead of the default preview size."
5235
+ },
5236
+ height: {
5237
+ type: "number",
5238
+ description: "Exact capture height in pixels. Set together with `width`."
5239
+ },
5240
+ format: {
5241
+ type: "string",
5242
+ enum: ["png", "jpeg"],
5243
+ description: "Output image format. Defaults to 'jpeg'. Use 'png' for crisp flat graphics like share cards, where JPEG artifacts show on sharp type and edges."
5244
+ },
5245
+ instructions: {
5246
+ type: "string",
5247
+ description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, navigating a flow, scrolling to a section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start its test at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing. Use only when interaction is required to *reach* the state you want to capture \u2014 log in, dismiss a modal, switch a tab, follow a route, scroll to a section. If your steps are exercising the app's functionality across multiple states (running flows, asserting behavior under interaction, multi-step QA), use `runAutomatedBrowserTest` instead."
5245
5248
  }
5246
- } catch (err) {
5247
- return `Error taking screenshot: ${err.message}`;
5248
- }
5249
+ },
5250
+ required: ["fullPage"]
5249
5251
  }
5250
5252
  };
5253
+ screenshotTool = {
5254
+ clearable: true,
5255
+ definition: screenshotDefinition,
5256
+ execute: (input, context) => executeScreenshot(input, context?.onLog, context)
5257
+ };
5251
5258
  }
5252
5259
  });
5253
5260
 
@@ -5479,88 +5486,6 @@ var init_analyzeImage2 = __esm({
5479
5486
  }
5480
5487
  });
5481
5488
 
5482
- // src/subagents/designExpert/tools/screenshot.ts
5483
- var screenshot_exports = {};
5484
- __export(screenshot_exports, {
5485
- definition: () => definition5,
5486
- execute: () => execute5
5487
- });
5488
- async function execute5(input, onLog, context) {
5489
- const fullPage = input.fullPage === true;
5490
- const shotKind = fullPage ? "full-page" : "viewport";
5491
- if (input.instructions && context) {
5492
- try {
5493
- const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a ${shotKind} screenshot.` : `${input.instructions}. After completing these steps, take a ${shotKind} screenshot.`;
5494
- const result = await runBrowserAutomation(task, context, {
5495
- capture: fullPage ? "fullPage" : "viewport"
5496
- });
5497
- if (!result.screenshot) {
5498
- return result.text;
5499
- }
5500
- return await streamScreenshotAnalysis({
5501
- url: result.screenshot.url,
5502
- prompt: input.prompt,
5503
- styleMap: result.screenshot.styleMap,
5504
- onLog,
5505
- model: resolveModel("imageAnalysis", context?.models, context?.model)
5506
- });
5507
- } catch (err) {
5508
- return `Error taking interactive screenshot: ${err.message}`;
5509
- }
5510
- }
5511
- const release = await acquireBrowserLock();
5512
- try {
5513
- return await captureAndAnalyzeScreenshot({
5514
- prompt: input.prompt,
5515
- path: input.path,
5516
- fullPage,
5517
- onLog,
5518
- model: resolveModel("imageAnalysis", context?.models, context?.model)
5519
- });
5520
- } catch (err) {
5521
- return `Error taking screenshot: ${err.message}`;
5522
- } finally {
5523
- release();
5524
- }
5525
- }
5526
- var definition5;
5527
- var init_screenshot3 = __esm({
5528
- "src/subagents/designExpert/tools/screenshot.ts"() {
5529
- "use strict";
5530
- init_screenshot();
5531
- init_browserLock();
5532
- init_browserAutomation();
5533
- init_surfaces();
5534
- definition5 = {
5535
- clearable: true,
5536
- name: "screenshot",
5537
- description: "Capture a screenshot of the current app preview and get it back with visual analysis. Choose `fullPage`: `false` captures just the visible viewport (fast \u2014 use it to review a specific section the page is scrolled to), `true` captures the entire page top-to-bottom (slower \u2014 use it to review overall composition or a layout you can't see in one screen). Use to review the current state of the UI being built. Remember, the screenshot analysis is not overly precise - for example, it cannot reliably identify specific fonts by name \u2014 it can only describe what letterforms look like.",
5538
- inputSchema: {
5539
- type: "object",
5540
- properties: {
5541
- fullPage: {
5542
- type: "boolean",
5543
- description: "true = full-height capture of the entire page; false = just the visible viewport. Pick based on whether you need the whole page or a specific section."
5544
- },
5545
- prompt: {
5546
- type: "string",
5547
- description: "Optional specific question about the screenshot. Use a bulleted list to ask many questions at once."
5548
- },
5549
- path: {
5550
- type: "string",
5551
- description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
5552
- },
5553
- instructions: {
5554
- type: "string",
5555
- description: "If the screenshot you need requires interaction first (dismissing a modal, clicking a tab, filling out a form, scrolling to a specific section, getting through a login/auth checkpoint), describe the steps to get there. A browser automation agent will follow these instructions, then capture per your `fullPage` choice \u2014 so with `fullPage: false` you can scroll to a section and capture just that viewport. It can bypass auth and get right to where it needs to be if you tell it to authenticate as a test user and give it the path/screen to start at. Never describe what names or values to use when applying the instructions - the browser automation agent must use its own values for it to work properly. If a specific auth role is required to access the content, be sure to note that - it can automatically assume it for the purpose of testing."
5556
- }
5557
- },
5558
- required: ["fullPage"]
5559
- }
5560
- };
5561
- }
5562
- });
5563
-
5564
5489
  // src/subagents/designExpert/tools/images/enhancePrompt.ts
5565
5490
  async function enhanceImagePrompt(params) {
5566
5491
  const {
@@ -5770,10 +5695,10 @@ var init_imageGenerator = __esm({
5770
5695
  // src/subagents/designExpert/tools/images/generateImages.ts
5771
5696
  var generateImages_exports = {};
5772
5697
  __export(generateImages_exports, {
5773
- definition: () => definition6,
5774
- execute: () => execute6
5698
+ definition: () => definition5,
5699
+ execute: () => execute5
5775
5700
  });
5776
- async function execute6(input, onLog, context) {
5701
+ async function execute5(input, onLog, context) {
5777
5702
  return generateImageAssets({
5778
5703
  prompts: input.prompts,
5779
5704
  width: input.width,
@@ -5799,13 +5724,13 @@ async function execute6(input, onLog, context) {
5799
5724
  )
5800
5725
  });
5801
5726
  }
5802
- var definition6;
5727
+ var definition5;
5803
5728
  var init_generateImages = __esm({
5804
5729
  "src/subagents/designExpert/tools/images/generateImages.ts"() {
5805
5730
  "use strict";
5806
5731
  init_imageGenerator();
5807
5732
  init_surfaces();
5808
- definition6 = {
5733
+ definition5 = {
5809
5734
  clearable: false,
5810
5735
  name: "generateImages",
5811
5736
  description: "Generate images. Returns CDN URLs with a quality analysis for each image. Produces high-quality results for everything from photorealistic images and abstract/creative visuals. Pass multiple prompts to generate in parallel. No need to analyze images separately after generating \u2014 the analysis is included.",
@@ -5845,10 +5770,10 @@ var init_generateImages = __esm({
5845
5770
  // src/subagents/designExpert/tools/images/editImages.ts
5846
5771
  var editImages_exports = {};
5847
5772
  __export(editImages_exports, {
5848
- definition: () => definition7,
5849
- execute: () => execute7
5773
+ definition: () => definition6,
5774
+ execute: () => execute6
5850
5775
  });
5851
- async function execute7(input, onLog, context) {
5776
+ async function execute6(input, onLog, context) {
5852
5777
  return generateImageAssets({
5853
5778
  prompts: input.prompts,
5854
5779
  sourceImages: input.sourceImages,
@@ -5874,13 +5799,13 @@ async function execute7(input, onLog, context) {
5874
5799
  )
5875
5800
  });
5876
5801
  }
5877
- var definition7;
5802
+ var definition6;
5878
5803
  var init_editImages = __esm({
5879
5804
  "src/subagents/designExpert/tools/images/editImages.ts"() {
5880
5805
  "use strict";
5881
5806
  init_imageGenerator();
5882
5807
  init_surfaces();
5883
- definition7 = {
5808
+ definition6 = {
5884
5809
  clearable: false,
5885
5810
  name: "editImages",
5886
5811
  description: "Edit or transform existing images. Provide one or more source image URLs as reference and a prompt describing the desired edit. Use for compositing, style transfer, subject transformation, blending multiple references, or incorporating one or more references into something new. Returns CDN URLs with analysis.",
@@ -5995,18 +5920,18 @@ var init_copyEditor = __esm({
5995
5920
  // src/subagents/designExpert/tools/polishCopy.ts
5996
5921
  var polishCopy_exports = {};
5997
5922
  __export(polishCopy_exports, {
5998
- definition: () => definition8,
5999
- execute: () => execute8
5923
+ definition: () => definition7,
5924
+ execute: () => execute7
6000
5925
  });
6001
- async function execute8(input, _onLog, context) {
5926
+ async function execute7(input, _onLog, context) {
6002
5927
  return copyEditorTool.execute(input, context);
6003
5928
  }
6004
- var definition8;
5929
+ var definition7;
6005
5930
  var init_polishCopy = __esm({
6006
5931
  "src/subagents/designExpert/tools/polishCopy.ts"() {
6007
5932
  "use strict";
6008
5933
  init_copyEditor();
6009
- definition8 = {
5934
+ definition7 = {
6010
5935
  clearable: false,
6011
5936
  name: "polishCopy",
6012
5937
  description: "Hand off any user-facing copy you've written \u2014 headlines, captions, labels, body text \u2014 and get back a sharper version: better built for its audience and free of the fingerprints that make writing read as AI. It elevates how the copy communicates without inventing facts or claims you didn't give it. Give it the text plus what it's for (where it appears, the audience).",
@@ -6043,16 +5968,19 @@ var init_tools4 = __esm({
6043
5968
  init_scrapeWebUrl();
6044
5969
  init_analyzeDesign();
6045
5970
  init_analyzeImage2();
6046
- init_screenshot3();
6047
5971
  init_generateImages();
6048
5972
  init_editImages();
6049
5973
  init_polishCopy();
5974
+ init_screenshot2();
6050
5975
  tools = {
6051
5976
  searchGoogle: searchGoogle_exports,
6052
5977
  scrapeWebUrl: scrapeWebUrl_exports,
6053
5978
  analyzeDesign: analyzeDesign_exports,
6054
5979
  analyzeImage: analyzeImage_exports,
6055
- screenshot: screenshot_exports,
5980
+ // Same tool the main agent offers, imported rather than reimplemented — the
5981
+ // two used to be near-identical copies and had already drifted apart. Its core
5982
+ // already takes (input, onLog, context), which is this registry's convention.
5983
+ screenshot: { definition: screenshotDefinition, execute: executeScreenshot },
6056
5984
  generateImages: generateImages_exports,
6057
5985
  editImages: editImages_exports,
6058
5986
  polishCopy: polishCopy_exports
@@ -43,6 +43,7 @@ Note: the snapshot concatenates inline text and strips whitespace. If you need t
43
43
  - `navigate`: Navigate to a new URL within the app. Waits for the new page to load before continuing with subsequent steps. Use this instead of evaluate with `window.location.href` when you need to navigate and then continue interacting with the new page. Steps after navigate execute on the new page automatically.
44
44
  - `evaluate`: Run arbitrary JavaScript in the page and return the result.
45
45
  - `styles`: Read computed CSS styles from page elements. Pass a `properties` array with camelCase CSS property names (e.g., `["backgroundColor", "borderRadius", "fontSize"]`). Omit `properties` for a default set covering colors, typography, spacing, borders, shadows, dimensions, and layout. Uses the same targeting as click/type (ref, text, role, label, selector). Omit the target to get styles for all elements from the last snapshot.
46
+ - `screenshotFullPage`: Take a screenshot of the whole page, top to bottom. Returns CDN url with full text analysis and dimensions. Use for overall composition or content past the fold.
46
47
  - `screenshotViewport`: Take a screenshot of the visible viewport. Returns CDN url with full text analysis and dimensions. To capture a specific section, set `scrollToSelector` (a CSS selector) — or `scrollY` (an absolute offset) — on this same step; it scrolls the target into view and captures it atomically, so you do NOT need a separate scroll step. Do not use if you can get what you need with other tools - only use when you need to visually see the viewport.
47
48
  - `setViewport`: Switch the browser between desktop and mobile rendering. Set `mode` to `"desktop"` or `"mobile"`. Mobile emulates a phone (390-wide, touch, device pixel ratio 2); desktop is the standard wide viewport. This reloads the page so media queries, responsive layouts, and `matchMedia` re-evaluate — the reload clears in-page state, so switch before you set up the state you want to inspect. The mode persists across navigations within a run. Each run starts in the app's default mode, so only use this when you need to check the other one.
48
49
 
@@ -161,7 +162,7 @@ Check a count with evaluate:
161
162
 
162
163
  ### Final Screenshot
163
164
  How you take the final screenshot depends on what the task asked for:
164
- - **Whole page** → use the standalone `screenshotFullPage` tool. It takes a full-height screenshot of the current page and returns the URL plus a full-text description.
165
+ - **Whole page** → use a `browserCommand` batch ending in a `screenshotFullPage` step. Returns the URL plus a full-text description.
165
166
  - **A specific section / viewport** → use a `browserCommand` batch ending in a `screenshotViewport` step with `scrollToSelector` set to the section (e.g. `{ "command": "screenshotViewport", "scrollToSelector": "#pricing" }`). This scrolls the section into view and captures it in one atomic step. Do this rather than a separate scroll step followed by a capture — capturing the viewport is only reliable when the scroll and the shot are in the same step.
166
167
 
167
168
  <rules>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mindstudio-ai/remy",
3
- "version": "0.1.255",
3
+ "version": "0.1.256",
4
4
  "description": "Remy coding agent",
5
5
  "repository": {
6
6
  "type": "git",