@mindstudio-ai/remy 0.1.178 → 0.1.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/headless.js +202 -45
- package/dist/index.js +222 -48
- package/package.json +1 -1
package/dist/headless.js
CHANGED
|
@@ -404,7 +404,7 @@ async function* streamChat(params) {
|
|
|
404
404
|
const url = `${baseUrl2}/_internal/v2/agent/remy/chat`;
|
|
405
405
|
const startTime = Date.now();
|
|
406
406
|
const subAgentId = rest.subAgentId;
|
|
407
|
-
const requestBody = { ...rest,
|
|
407
|
+
const requestBody = { ...rest, modelId: model };
|
|
408
408
|
log2.info("API request", {
|
|
409
409
|
requestId,
|
|
410
410
|
...subAgentId && { subAgentId },
|
|
@@ -2787,13 +2787,8 @@ var queryDatabaseTool = {
|
|
|
2787
2787
|
};
|
|
2788
2788
|
|
|
2789
2789
|
// src/subagents/common/analyzeImage.ts
|
|
2790
|
-
var VISION_MODEL = "claude-4-6-sonnet";
|
|
2791
|
-
var VISION_MODEL_OVERRIDE = {
|
|
2792
|
-
model: VISION_MODEL,
|
|
2793
|
-
config: { thinkingBudget: "off" }
|
|
2794
|
-
};
|
|
2795
2790
|
async function analyzeImage(params) {
|
|
2796
|
-
const { prompt, imageUrl, timeout = 2e5, onLog } = params;
|
|
2791
|
+
const { prompt, imageUrl, model, timeout = 2e5, onLog } = params;
|
|
2797
2792
|
return runMindstudioCli(
|
|
2798
2793
|
[
|
|
2799
2794
|
"analyze-image",
|
|
@@ -2802,7 +2797,7 @@ async function analyzeImage(params) {
|
|
|
2802
2797
|
"--image-url",
|
|
2803
2798
|
imageUrl,
|
|
2804
2799
|
"--vision-model-override",
|
|
2805
|
-
JSON.stringify(
|
|
2800
|
+
JSON.stringify({ model })
|
|
2806
2801
|
],
|
|
2807
2802
|
{ outputKey: "analysis", timeout, onLog }
|
|
2808
2803
|
);
|
|
@@ -2830,13 +2825,14 @@ ${TEXT_WRAP_DISCLAIMER}`;
|
|
|
2830
2825
|
return p;
|
|
2831
2826
|
}
|
|
2832
2827
|
async function streamScreenshotAnalysis(opts) {
|
|
2833
|
-
const { url, prompt, styleMap, onLog } = opts;
|
|
2828
|
+
const { url, prompt, styleMap, onLog, model } = opts;
|
|
2834
2829
|
onLog?.(JSON.stringify({ url, analysis: null }));
|
|
2835
2830
|
const analysisPrompt = buildScreenshotAnalysisPrompt({ prompt, styleMap });
|
|
2836
2831
|
let accumulated = "";
|
|
2837
2832
|
const analysis = await analyzeImage({
|
|
2838
2833
|
prompt: analysisPrompt,
|
|
2839
2834
|
imageUrl: url,
|
|
2835
|
+
model,
|
|
2840
2836
|
onLog: (chunk) => {
|
|
2841
2837
|
accumulated += chunk;
|
|
2842
2838
|
onLog?.(JSON.stringify({ url, analysis: accumulated }));
|
|
@@ -2848,12 +2844,14 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2848
2844
|
let prompt;
|
|
2849
2845
|
let existingUrl;
|
|
2850
2846
|
let onLog;
|
|
2847
|
+
let model;
|
|
2851
2848
|
let path12;
|
|
2852
2849
|
if (typeof promptOrOptions === "object" && promptOrOptions !== null) {
|
|
2853
2850
|
prompt = promptOrOptions.prompt;
|
|
2854
2851
|
existingUrl = promptOrOptions.imageUrl;
|
|
2855
2852
|
path12 = promptOrOptions.path;
|
|
2856
2853
|
onLog = promptOrOptions.onLog;
|
|
2854
|
+
model = promptOrOptions.model;
|
|
2857
2855
|
} else {
|
|
2858
2856
|
prompt = promptOrOptions;
|
|
2859
2857
|
}
|
|
@@ -2878,11 +2876,17 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2878
2876
|
if (prompt === false) {
|
|
2879
2877
|
return url;
|
|
2880
2878
|
}
|
|
2879
|
+
if (!model) {
|
|
2880
|
+
throw new Error(
|
|
2881
|
+
"captureAndAnalyzeScreenshot: `model` is required when analysis is enabled"
|
|
2882
|
+
);
|
|
2883
|
+
}
|
|
2881
2884
|
return streamScreenshotAnalysis({
|
|
2882
2885
|
url,
|
|
2883
2886
|
prompt: prompt || void 0,
|
|
2884
2887
|
styleMap,
|
|
2885
|
-
onLog
|
|
2888
|
+
onLog,
|
|
2889
|
+
model
|
|
2886
2890
|
});
|
|
2887
2891
|
}
|
|
2888
2892
|
|
|
@@ -3715,6 +3719,98 @@ ${appSpec}
|
|
|
3715
3719
|
}
|
|
3716
3720
|
}
|
|
3717
3721
|
|
|
3722
|
+
// src/models/surfaces.ts
|
|
3723
|
+
var MODEL_SURFACES = {
|
|
3724
|
+
parent: {
|
|
3725
|
+
default: "claude-4-7-opus",
|
|
3726
|
+
label: "Remy",
|
|
3727
|
+
description: "The main Remy agent you chat with about your product. Writes code and manages delegation to other agents.",
|
|
3728
|
+
modelType: "text",
|
|
3729
|
+
userPickable: true
|
|
3730
|
+
},
|
|
3731
|
+
visualDesignExpert: {
|
|
3732
|
+
default: "claude-4-7-opus",
|
|
3733
|
+
label: "Design Agent",
|
|
3734
|
+
description: "Designs your product's interfaces, including components, layouts, typography, color, and visual identity.",
|
|
3735
|
+
modelType: "text",
|
|
3736
|
+
userPickable: true
|
|
3737
|
+
},
|
|
3738
|
+
productVision: {
|
|
3739
|
+
default: "claude-4-6-sonnet",
|
|
3740
|
+
label: "Roadmap Agent",
|
|
3741
|
+
description: "Owns your product's roadmap and pitch deck. Helps decide what to build next and how to frame the big picture.",
|
|
3742
|
+
modelType: "text",
|
|
3743
|
+
userPickable: true
|
|
3744
|
+
},
|
|
3745
|
+
browserAutomation: {
|
|
3746
|
+
default: "claude-4-6-sonnet",
|
|
3747
|
+
label: "QA Agent",
|
|
3748
|
+
description: "Tests features and UI flows in an automated browser to verify they work end to end.",
|
|
3749
|
+
modelType: "text",
|
|
3750
|
+
userPickable: true
|
|
3751
|
+
},
|
|
3752
|
+
codeSanityCheck: {
|
|
3753
|
+
default: "claude-4-6-sonnet",
|
|
3754
|
+
label: "Architecture Agent",
|
|
3755
|
+
description: "Reviews the architecture and structure of code changes to avoid technical debt.",
|
|
3756
|
+
modelType: "text",
|
|
3757
|
+
userPickable: true
|
|
3758
|
+
},
|
|
3759
|
+
imageGeneration: {
|
|
3760
|
+
default: "seedream-4.5",
|
|
3761
|
+
label: "Image Generation",
|
|
3762
|
+
description: "Creates images for your product \u2014 icons, illustrations, photos, and any other visual assets.",
|
|
3763
|
+
modelType: "image_generation",
|
|
3764
|
+
userPickable: true
|
|
3765
|
+
},
|
|
3766
|
+
imageAnalysis: {
|
|
3767
|
+
default: "claude-4-6-sonnet",
|
|
3768
|
+
label: "Image Analysis",
|
|
3769
|
+
description: "Reads screenshots taken by the QA agent during automated browser tests. Other agents use their own built-in image analysis when they need to read images.",
|
|
3770
|
+
modelType: "vision",
|
|
3771
|
+
userPickable: true
|
|
3772
|
+
},
|
|
3773
|
+
conversationSummarizer: {
|
|
3774
|
+
default: "claude-4-6-sonnet",
|
|
3775
|
+
label: "Compaction Utility",
|
|
3776
|
+
description: "Compresses long conversations into summaries to keep things responsive.",
|
|
3777
|
+
modelType: "text",
|
|
3778
|
+
userPickable: true
|
|
3779
|
+
},
|
|
3780
|
+
brandExtractor: {
|
|
3781
|
+
default: "claude-4-6-sonnet",
|
|
3782
|
+
label: "Brand Utility",
|
|
3783
|
+
description: "Extracts your product's name, colors, and fonts from your spec for use in branded documents.",
|
|
3784
|
+
modelType: "text",
|
|
3785
|
+
userPickable: true
|
|
3786
|
+
},
|
|
3787
|
+
// Internal surface — not user-pickable. Remy uses this to rewrite design
|
|
3788
|
+
// briefs into model-optimized image prompts before image generation.
|
|
3789
|
+
imagePromptEnhancer: {
|
|
3790
|
+
default: "claude-4-6-sonnet",
|
|
3791
|
+
label: "Image Prompt Enhancer",
|
|
3792
|
+
description: "Rewrites image briefs into model-optimized prompts before image generation.",
|
|
3793
|
+
modelType: "text",
|
|
3794
|
+
userPickable: false
|
|
3795
|
+
}
|
|
3796
|
+
};
|
|
3797
|
+
var ALLOWED_MODELS_BY_TYPE = {
|
|
3798
|
+
text: [
|
|
3799
|
+
"claude-4-7-opus",
|
|
3800
|
+
"claude-4-6-opus",
|
|
3801
|
+
"claude-4-6-sonnet",
|
|
3802
|
+
"gpt-5.5",
|
|
3803
|
+
"gemini-3-pro",
|
|
3804
|
+
"gemini-3.1-pro",
|
|
3805
|
+
"gemini-3-flash"
|
|
3806
|
+
]
|
|
3807
|
+
// vision: undefined — unconstrained
|
|
3808
|
+
// image_generation: undefined — unconstrained
|
|
3809
|
+
};
|
|
3810
|
+
function resolveModel(surfaceId, models, fallback) {
|
|
3811
|
+
return models?.[surfaceId] ?? fallback ?? MODEL_SURFACES[surfaceId].default;
|
|
3812
|
+
}
|
|
3813
|
+
|
|
3718
3814
|
// src/subagents/browserAutomation/index.ts
|
|
3719
3815
|
var log6 = createLogger("browser-automation");
|
|
3720
3816
|
var browserAutomationTool = {
|
|
@@ -3764,7 +3860,12 @@ var browserAutomationTool = {
|
|
|
3764
3860
|
try {
|
|
3765
3861
|
return await captureAndAnalyzeScreenshot({
|
|
3766
3862
|
path: _input.path,
|
|
3767
|
-
onLog
|
|
3863
|
+
onLog,
|
|
3864
|
+
model: resolveModel(
|
|
3865
|
+
"imageAnalysis",
|
|
3866
|
+
context.models,
|
|
3867
|
+
context.model
|
|
3868
|
+
)
|
|
3768
3869
|
});
|
|
3769
3870
|
} catch (err) {
|
|
3770
3871
|
return `Error taking screenshot: ${err.message}`;
|
|
@@ -3773,7 +3874,7 @@ var browserAutomationTool = {
|
|
|
3773
3874
|
return `Error: unknown local tool "${name}"`;
|
|
3774
3875
|
},
|
|
3775
3876
|
apiConfig: context.apiConfig,
|
|
3776
|
-
model: context.models
|
|
3877
|
+
model: resolveModel("browserAutomation", context.models, context.model),
|
|
3777
3878
|
subAgentId: "browserAutomation",
|
|
3778
3879
|
signal: context.signal,
|
|
3779
3880
|
parentToolId: context.toolCallId,
|
|
@@ -3791,6 +3892,13 @@ var browserAutomationTool = {
|
|
|
3791
3892
|
(s) => s.command === "screenshotViewport" && s.result?.url
|
|
3792
3893
|
);
|
|
3793
3894
|
if (screenshotSteps.length > 0) {
|
|
3895
|
+
const visionOverride = {
|
|
3896
|
+
model: resolveModel(
|
|
3897
|
+
"imageAnalysis",
|
|
3898
|
+
context.models,
|
|
3899
|
+
context.model
|
|
3900
|
+
)
|
|
3901
|
+
};
|
|
3794
3902
|
const batchInput = screenshotSteps.map((s) => ({
|
|
3795
3903
|
stepType: "analyzeImage",
|
|
3796
3904
|
step: {
|
|
@@ -3798,7 +3906,7 @@ var browserAutomationTool = {
|
|
|
3798
3906
|
prompt: buildScreenshotAnalysisPrompt({
|
|
3799
3907
|
styleMap: s.result.styleMap
|
|
3800
3908
|
}),
|
|
3801
|
-
visionModelOverride:
|
|
3909
|
+
visionModelOverride: visionOverride
|
|
3802
3910
|
}
|
|
3803
3911
|
}));
|
|
3804
3912
|
const batchResult = await runMindstudioCli(
|
|
@@ -3877,7 +3985,8 @@ var screenshotTool = {
|
|
|
3877
3985
|
return await captureAndAnalyzeScreenshot({
|
|
3878
3986
|
prompt: input.prompt,
|
|
3879
3987
|
imageUrl: input.imageUrl,
|
|
3880
|
-
onLog: context?.onLog
|
|
3988
|
+
onLog: context?.onLog,
|
|
3989
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
3881
3990
|
});
|
|
3882
3991
|
}
|
|
3883
3992
|
if (input.instructions && context) {
|
|
@@ -3899,7 +4008,8 @@ var screenshotTool = {
|
|
|
3899
4008
|
url,
|
|
3900
4009
|
prompt: input.prompt,
|
|
3901
4010
|
styleMap,
|
|
3902
|
-
onLog: context?.onLog
|
|
4011
|
+
onLog: context?.onLog,
|
|
4012
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
3903
4013
|
});
|
|
3904
4014
|
}
|
|
3905
4015
|
const release = await acquireBrowserLock();
|
|
@@ -3907,7 +4017,8 @@ var screenshotTool = {
|
|
|
3907
4017
|
return await captureAndAnalyzeScreenshot({
|
|
3908
4018
|
prompt: input.prompt,
|
|
3909
4019
|
path: input.path,
|
|
3910
|
-
onLog: context?.onLog
|
|
4020
|
+
onLog: context?.onLog,
|
|
4021
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
3911
4022
|
});
|
|
3912
4023
|
} finally {
|
|
3913
4024
|
release();
|
|
@@ -4118,7 +4229,7 @@ var definition3 = {
|
|
|
4118
4229
|
required: ["url"]
|
|
4119
4230
|
}
|
|
4120
4231
|
};
|
|
4121
|
-
async function execute3(input, onLog) {
|
|
4232
|
+
async function execute3(input, onLog, context) {
|
|
4122
4233
|
const url = input.url;
|
|
4123
4234
|
const analysisPrompt = input.prompt || DESIGN_REFERENCE_PROMPT;
|
|
4124
4235
|
const isImageUrl = /\.(png|jpe?g|webp|gif|svg|avif)(\?|$)/i.test(url);
|
|
@@ -4151,7 +4262,8 @@ async function execute3(input, onLog) {
|
|
|
4151
4262
|
const analysis = await analyzeImage({
|
|
4152
4263
|
prompt: analysisPrompt,
|
|
4153
4264
|
imageUrl,
|
|
4154
|
-
onLog
|
|
4265
|
+
onLog,
|
|
4266
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4155
4267
|
});
|
|
4156
4268
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4157
4269
|
}
|
|
@@ -4181,7 +4293,7 @@ var definition4 = {
|
|
|
4181
4293
|
required: ["imageUrl"]
|
|
4182
4294
|
}
|
|
4183
4295
|
};
|
|
4184
|
-
async function execute4(input, onLog) {
|
|
4296
|
+
async function execute4(input, onLog, context) {
|
|
4185
4297
|
const imageUrl = input.imageUrl;
|
|
4186
4298
|
const prompt = buildScreenshotAnalysisPrompt({
|
|
4187
4299
|
prompt: input.prompt
|
|
@@ -4189,7 +4301,8 @@ async function execute4(input, onLog) {
|
|
|
4189
4301
|
const analysis = await analyzeImage({
|
|
4190
4302
|
prompt,
|
|
4191
4303
|
imageUrl,
|
|
4192
|
-
onLog
|
|
4304
|
+
onLog,
|
|
4305
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4193
4306
|
});
|
|
4194
4307
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4195
4308
|
}
|
|
@@ -4243,7 +4356,8 @@ async function execute5(input, onLog, context) {
|
|
|
4243
4356
|
url,
|
|
4244
4357
|
prompt: input.prompt,
|
|
4245
4358
|
styleMap,
|
|
4246
|
-
onLog
|
|
4359
|
+
onLog,
|
|
4360
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4247
4361
|
});
|
|
4248
4362
|
} catch (err) {
|
|
4249
4363
|
return `Error taking interactive screenshot: ${err.message}`;
|
|
@@ -4254,7 +4368,8 @@ async function execute5(input, onLog, context) {
|
|
|
4254
4368
|
return await captureAndAnalyzeScreenshot({
|
|
4255
4369
|
prompt: input.prompt,
|
|
4256
4370
|
path: input.path,
|
|
4257
|
-
onLog
|
|
4371
|
+
onLog,
|
|
4372
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4258
4373
|
});
|
|
4259
4374
|
} catch (err) {
|
|
4260
4375
|
return `Error taking screenshot: ${err.message}`;
|
|
@@ -4271,16 +4386,11 @@ __export(generateImages_exports, {
|
|
|
4271
4386
|
});
|
|
4272
4387
|
|
|
4273
4388
|
// src/subagents/designExpert/tools/images/enhancePrompt.ts
|
|
4274
|
-
var ENHANCE_MODEL = "claude-4-6-sonnet";
|
|
4275
|
-
var MODEL_OVERRIDE = {
|
|
4276
|
-
model: ENHANCE_MODEL,
|
|
4277
|
-
config: { reasoning: "false" }
|
|
4278
|
-
};
|
|
4279
4389
|
var SYSTEM_PROMPT = readAsset(
|
|
4280
4390
|
"subagents/designExpert/tools/images/enhance-image-prompt.md"
|
|
4281
4391
|
);
|
|
4282
4392
|
async function enhanceImagePrompt(params) {
|
|
4283
|
-
const { brief, width, height, transparentBackground, onLog } = params;
|
|
4393
|
+
const { brief, width, height, transparentBackground, onLog, model } = params;
|
|
4284
4394
|
const contextParts = [
|
|
4285
4395
|
`Dimensions: ${width}x${height}${width > height ? " (landscape)" : width < height ? " (portrait)" : " (square)"}`
|
|
4286
4396
|
];
|
|
@@ -4305,7 +4415,7 @@ ${brief}
|
|
|
4305
4415
|
"--message",
|
|
4306
4416
|
message,
|
|
4307
4417
|
"--model-override",
|
|
4308
|
-
JSON.stringify(
|
|
4418
|
+
JSON.stringify({ model, config: { reasoning: "false" } })
|
|
4309
4419
|
],
|
|
4310
4420
|
{ outputKey: "content", timeout: 6e4, onLog, caller: "designExpert" }
|
|
4311
4421
|
);
|
|
@@ -4315,7 +4425,15 @@ ${brief}
|
|
|
4315
4425
|
// src/subagents/designExpert/tools/images/imageGenerator.ts
|
|
4316
4426
|
var ANALYZE_PROMPT = 'You are reviewing this image for a visual designer sourcing assets for a project. Describe: what the image depicts, the mood and color palette, how the lighting and composition work, any text present in the image, whether there are any issues (artifacts, distortions), and how it could be used in a layout for an app or website. Be concise and practical. Respond only with your analysis as Markdown (starting with the title "Asset Review") and absolutely no other text. Do not use emojis - use unicode if you need symbols.';
|
|
4317
4427
|
async function generateImageAssets(opts) {
|
|
4318
|
-
const {
|
|
4428
|
+
const {
|
|
4429
|
+
prompts,
|
|
4430
|
+
sourceImages,
|
|
4431
|
+
transparentBackground,
|
|
4432
|
+
onLog,
|
|
4433
|
+
imageGenerationModel: genModel,
|
|
4434
|
+
imageAnalysisModel,
|
|
4435
|
+
imagePromptEnhancerModel
|
|
4436
|
+
} = opts;
|
|
4319
4437
|
const width = opts.width || 2048;
|
|
4320
4438
|
const height = opts.height || 2048;
|
|
4321
4439
|
const config = { width, height };
|
|
@@ -4330,7 +4448,8 @@ async function generateImageAssets(opts) {
|
|
|
4330
4448
|
width,
|
|
4331
4449
|
height,
|
|
4332
4450
|
transparentBackground,
|
|
4333
|
-
onLog
|
|
4451
|
+
onLog,
|
|
4452
|
+
model: imagePromptEnhancerModel
|
|
4334
4453
|
})
|
|
4335
4454
|
)
|
|
4336
4455
|
);
|
|
@@ -4339,7 +4458,7 @@ async function generateImageAssets(opts) {
|
|
|
4339
4458
|
const step = JSON.stringify({
|
|
4340
4459
|
prompt: enhancedPrompts[0],
|
|
4341
4460
|
imageModelOverride: {
|
|
4342
|
-
model:
|
|
4461
|
+
model: genModel,
|
|
4343
4462
|
config
|
|
4344
4463
|
}
|
|
4345
4464
|
});
|
|
@@ -4358,7 +4477,7 @@ async function generateImageAssets(opts) {
|
|
|
4358
4477
|
step: {
|
|
4359
4478
|
prompt,
|
|
4360
4479
|
imageModelOverride: {
|
|
4361
|
-
model:
|
|
4480
|
+
model: genModel,
|
|
4362
4481
|
config
|
|
4363
4482
|
}
|
|
4364
4483
|
}
|
|
@@ -4410,7 +4529,8 @@ async function generateImageAssets(opts) {
|
|
|
4410
4529
|
const analysis = await analyzeImage({
|
|
4411
4530
|
prompt: ANALYZE_PROMPT,
|
|
4412
4531
|
imageUrl: url,
|
|
4413
|
-
onLog
|
|
4532
|
+
onLog,
|
|
4533
|
+
model: imageAnalysisModel
|
|
4414
4534
|
});
|
|
4415
4535
|
return {
|
|
4416
4536
|
url,
|
|
@@ -4456,13 +4576,28 @@ var definition6 = {
|
|
|
4456
4576
|
required: ["prompts"]
|
|
4457
4577
|
}
|
|
4458
4578
|
};
|
|
4459
|
-
async function execute6(input, onLog) {
|
|
4579
|
+
async function execute6(input, onLog, context) {
|
|
4460
4580
|
return generateImageAssets({
|
|
4461
4581
|
prompts: input.prompts,
|
|
4462
4582
|
width: input.width,
|
|
4463
4583
|
height: input.height,
|
|
4464
4584
|
transparentBackground: input.transparentBackground,
|
|
4465
|
-
onLog
|
|
4585
|
+
onLog,
|
|
4586
|
+
imageGenerationModel: resolveModel(
|
|
4587
|
+
"imageGeneration",
|
|
4588
|
+
context?.models,
|
|
4589
|
+
context?.model
|
|
4590
|
+
),
|
|
4591
|
+
imageAnalysisModel: resolveModel(
|
|
4592
|
+
"imageAnalysis",
|
|
4593
|
+
context?.models,
|
|
4594
|
+
context?.model
|
|
4595
|
+
),
|
|
4596
|
+
imagePromptEnhancerModel: resolveModel(
|
|
4597
|
+
"imagePromptEnhancer",
|
|
4598
|
+
context?.models,
|
|
4599
|
+
context?.model
|
|
4600
|
+
)
|
|
4466
4601
|
});
|
|
4467
4602
|
}
|
|
4468
4603
|
|
|
@@ -4509,14 +4644,29 @@ var definition7 = {
|
|
|
4509
4644
|
required: ["prompts", "sourceImages"]
|
|
4510
4645
|
}
|
|
4511
4646
|
};
|
|
4512
|
-
async function execute7(input, onLog) {
|
|
4647
|
+
async function execute7(input, onLog, context) {
|
|
4513
4648
|
return generateImageAssets({
|
|
4514
4649
|
prompts: input.prompts,
|
|
4515
4650
|
sourceImages: input.sourceImages,
|
|
4516
4651
|
width: input.width,
|
|
4517
4652
|
height: input.height,
|
|
4518
4653
|
transparentBackground: input.transparentBackground,
|
|
4519
|
-
onLog
|
|
4654
|
+
onLog,
|
|
4655
|
+
imageGenerationModel: resolveModel(
|
|
4656
|
+
"imageGeneration",
|
|
4657
|
+
context?.models,
|
|
4658
|
+
context?.model
|
|
4659
|
+
),
|
|
4660
|
+
imageAnalysisModel: resolveModel(
|
|
4661
|
+
"imageAnalysis",
|
|
4662
|
+
context?.models,
|
|
4663
|
+
context?.model
|
|
4664
|
+
),
|
|
4665
|
+
imagePromptEnhancerModel: resolveModel(
|
|
4666
|
+
"imagePromptEnhancer",
|
|
4667
|
+
context?.models,
|
|
4668
|
+
context?.model
|
|
4669
|
+
)
|
|
4520
4670
|
});
|
|
4521
4671
|
}
|
|
4522
4672
|
|
|
@@ -4989,7 +5139,7 @@ var designExpertTool = {
|
|
|
4989
5139
|
);
|
|
4990
5140
|
},
|
|
4991
5141
|
apiConfig: context.apiConfig,
|
|
4992
|
-
model: context.models
|
|
5142
|
+
model: resolveModel("visualDesignExpert", context.models, context.model),
|
|
4993
5143
|
subAgentId: "visualDesignExpert",
|
|
4994
5144
|
signal: context.signal,
|
|
4995
5145
|
parentToolId: context.toolCallId,
|
|
@@ -5207,7 +5357,7 @@ var productVisionTool = {
|
|
|
5207
5357
|
return executeVisionTool(name, input2, childCtx);
|
|
5208
5358
|
},
|
|
5209
5359
|
apiConfig: context.apiConfig,
|
|
5210
|
-
model: context.models
|
|
5360
|
+
model: resolveModel("productVision", context.models, context.model),
|
|
5211
5361
|
subAgentId: "productVision",
|
|
5212
5362
|
signal: context.signal,
|
|
5213
5363
|
parentToolId: context.toolCallId,
|
|
@@ -5315,7 +5465,7 @@ var codeSanityCheckTool = {
|
|
|
5315
5465
|
externalTools: /* @__PURE__ */ new Set(),
|
|
5316
5466
|
executeTool: (name, toolInput) => executeTool(name, toolInput, context),
|
|
5317
5467
|
apiConfig: context.apiConfig,
|
|
5318
|
-
model: context.models
|
|
5468
|
+
model: resolveModel("codeSanityCheck", context.models, context.model),
|
|
5319
5469
|
subAgentId: "codeSanityCheck",
|
|
5320
5470
|
signal: context.signal,
|
|
5321
5471
|
parentToolId: context.toolCallId,
|
|
@@ -5455,7 +5605,7 @@ function triggerCompaction(state, apiConfig, opts = {}) {
|
|
|
5455
5605
|
apiConfig,
|
|
5456
5606
|
system,
|
|
5457
5607
|
tools2,
|
|
5458
|
-
state.models
|
|
5608
|
+
resolveModel("conversationSummarizer", state.models, model)
|
|
5459
5609
|
).then((summaries) => {
|
|
5460
5610
|
pendingSummaries.push(...summaries);
|
|
5461
5611
|
listener?.({ type: "complete", requestId });
|
|
@@ -6252,7 +6402,7 @@ async function runTurn(params) {
|
|
|
6252
6402
|
onEvent({ type: "tool_input_delta", id, name, result: content });
|
|
6253
6403
|
}
|
|
6254
6404
|
}
|
|
6255
|
-
const parentModel = state.models
|
|
6405
|
+
const parentModel = resolveModel("parent", state.models, model);
|
|
6256
6406
|
try {
|
|
6257
6407
|
for await (const event of streamChatWithRetry(
|
|
6258
6408
|
{
|
|
@@ -6596,7 +6746,10 @@ async function runTurn(params) {
|
|
|
6596
6746
|
isError: r.isError
|
|
6597
6747
|
});
|
|
6598
6748
|
if (!r.isError && BRAND_TRIGGERING_TOOLS.has(tc.name)) {
|
|
6599
|
-
triggerBrandExtraction(
|
|
6749
|
+
triggerBrandExtraction(
|
|
6750
|
+
apiConfig,
|
|
6751
|
+
resolveModel("brandExtractor", state.models, model)
|
|
6752
|
+
);
|
|
6600
6753
|
}
|
|
6601
6754
|
return r;
|
|
6602
6755
|
})
|
|
@@ -7039,12 +7192,14 @@ var HeadlessSession = class {
|
|
|
7039
7192
|
this.emit("session_restored", {
|
|
7040
7193
|
messageCount: this.state.messages.length,
|
|
7041
7194
|
...this.state.models && { models: this.state.models },
|
|
7195
|
+
modelSurfaces: MODEL_SURFACES,
|
|
7196
|
+
allowedModelsByType: ALLOWED_MODELS_BY_TYPE,
|
|
7042
7197
|
...this.queueFields()
|
|
7043
7198
|
});
|
|
7044
7199
|
}
|
|
7045
7200
|
triggerBrandExtraction(
|
|
7046
7201
|
this.config,
|
|
7047
|
-
this.state.models
|
|
7202
|
+
resolveModel("brandExtractor", this.state.models, this.opts.model)
|
|
7048
7203
|
);
|
|
7049
7204
|
this.toolRegistry.onEvent = this.onEvent;
|
|
7050
7205
|
setCompactionListener((event) => {
|
|
@@ -7722,6 +7877,8 @@ var HeadlessSession = class {
|
|
|
7722
7877
|
running: this.running,
|
|
7723
7878
|
...this.running && this.currentRequestId ? { currentRequestId: this.currentRequestId } : {},
|
|
7724
7879
|
...this.state.models && { models: this.state.models },
|
|
7880
|
+
modelSurfaces: MODEL_SURFACES,
|
|
7881
|
+
allowedModelsByType: ALLOWED_MODELS_BY_TYPE,
|
|
7725
7882
|
...this.queueFields()
|
|
7726
7883
|
}));
|
|
7727
7884
|
return;
|
package/dist/index.js
CHANGED
|
@@ -90,7 +90,7 @@ async function* streamChat(params) {
|
|
|
90
90
|
const url = `${baseUrl2}/_internal/v2/agent/remy/chat`;
|
|
91
91
|
const startTime = Date.now();
|
|
92
92
|
const subAgentId = rest.subAgentId;
|
|
93
|
-
const requestBody = { ...rest,
|
|
93
|
+
const requestBody = { ...rest, modelId: model };
|
|
94
94
|
log.info("API request", {
|
|
95
95
|
requestId,
|
|
96
96
|
...subAgentId && { subAgentId },
|
|
@@ -2024,6 +2024,104 @@ var init_prompt = __esm({
|
|
|
2024
2024
|
}
|
|
2025
2025
|
});
|
|
2026
2026
|
|
|
2027
|
+
// src/models/surfaces.ts
|
|
2028
|
+
function resolveModel(surfaceId, models, fallback) {
|
|
2029
|
+
return models?.[surfaceId] ?? fallback ?? MODEL_SURFACES[surfaceId].default;
|
|
2030
|
+
}
|
|
2031
|
+
var MODEL_SURFACES, ALLOWED_MODELS_BY_TYPE;
|
|
2032
|
+
var init_surfaces = __esm({
|
|
2033
|
+
"src/models/surfaces.ts"() {
|
|
2034
|
+
"use strict";
|
|
2035
|
+
MODEL_SURFACES = {
|
|
2036
|
+
parent: {
|
|
2037
|
+
default: "claude-4-7-opus",
|
|
2038
|
+
label: "Remy",
|
|
2039
|
+
description: "The main Remy agent you chat with about your product. Writes code and manages delegation to other agents.",
|
|
2040
|
+
modelType: "text",
|
|
2041
|
+
userPickable: true
|
|
2042
|
+
},
|
|
2043
|
+
visualDesignExpert: {
|
|
2044
|
+
default: "claude-4-7-opus",
|
|
2045
|
+
label: "Design Agent",
|
|
2046
|
+
description: "Designs your product's interfaces, including components, layouts, typography, color, and visual identity.",
|
|
2047
|
+
modelType: "text",
|
|
2048
|
+
userPickable: true
|
|
2049
|
+
},
|
|
2050
|
+
productVision: {
|
|
2051
|
+
default: "claude-4-6-sonnet",
|
|
2052
|
+
label: "Roadmap Agent",
|
|
2053
|
+
description: "Owns your product's roadmap and pitch deck. Helps decide what to build next and how to frame the big picture.",
|
|
2054
|
+
modelType: "text",
|
|
2055
|
+
userPickable: true
|
|
2056
|
+
},
|
|
2057
|
+
browserAutomation: {
|
|
2058
|
+
default: "claude-4-6-sonnet",
|
|
2059
|
+
label: "QA Agent",
|
|
2060
|
+
description: "Tests features and UI flows in an automated browser to verify they work end to end.",
|
|
2061
|
+
modelType: "text",
|
|
2062
|
+
userPickable: true
|
|
2063
|
+
},
|
|
2064
|
+
codeSanityCheck: {
|
|
2065
|
+
default: "claude-4-6-sonnet",
|
|
2066
|
+
label: "Architecture Agent",
|
|
2067
|
+
description: "Reviews the architecture and structure of code changes to avoid technical debt.",
|
|
2068
|
+
modelType: "text",
|
|
2069
|
+
userPickable: true
|
|
2070
|
+
},
|
|
2071
|
+
imageGeneration: {
|
|
2072
|
+
default: "seedream-4.5",
|
|
2073
|
+
label: "Image Generation",
|
|
2074
|
+
description: "Creates images for your product \u2014 icons, illustrations, photos, and any other visual assets.",
|
|
2075
|
+
modelType: "image_generation",
|
|
2076
|
+
userPickable: true
|
|
2077
|
+
},
|
|
2078
|
+
imageAnalysis: {
|
|
2079
|
+
default: "claude-4-6-sonnet",
|
|
2080
|
+
label: "Image Analysis",
|
|
2081
|
+
description: "Reads screenshots taken by the QA agent during automated browser tests. Other agents use their own built-in image analysis when they need to read images.",
|
|
2082
|
+
modelType: "vision",
|
|
2083
|
+
userPickable: true
|
|
2084
|
+
},
|
|
2085
|
+
conversationSummarizer: {
|
|
2086
|
+
default: "claude-4-6-sonnet",
|
|
2087
|
+
label: "Compaction Utility",
|
|
2088
|
+
description: "Compresses long conversations into summaries to keep things responsive.",
|
|
2089
|
+
modelType: "text",
|
|
2090
|
+
userPickable: true
|
|
2091
|
+
},
|
|
2092
|
+
brandExtractor: {
|
|
2093
|
+
default: "claude-4-6-sonnet",
|
|
2094
|
+
label: "Brand Utility",
|
|
2095
|
+
description: "Extracts your product's name, colors, and fonts from your spec for use in branded documents.",
|
|
2096
|
+
modelType: "text",
|
|
2097
|
+
userPickable: true
|
|
2098
|
+
},
|
|
2099
|
+
// Internal surface — not user-pickable. Remy uses this to rewrite design
|
|
2100
|
+
// briefs into model-optimized image prompts before image generation.
|
|
2101
|
+
imagePromptEnhancer: {
|
|
2102
|
+
default: "claude-4-6-sonnet",
|
|
2103
|
+
label: "Image Prompt Enhancer",
|
|
2104
|
+
description: "Rewrites image briefs into model-optimized prompts before image generation.",
|
|
2105
|
+
modelType: "text",
|
|
2106
|
+
userPickable: false
|
|
2107
|
+
}
|
|
2108
|
+
};
|
|
2109
|
+
ALLOWED_MODELS_BY_TYPE = {
|
|
2110
|
+
text: [
|
|
2111
|
+
"claude-4-7-opus",
|
|
2112
|
+
"claude-4-6-opus",
|
|
2113
|
+
"claude-4-6-sonnet",
|
|
2114
|
+
"gpt-5.5",
|
|
2115
|
+
"gemini-3-pro",
|
|
2116
|
+
"gemini-3.1-pro",
|
|
2117
|
+
"gemini-3-flash"
|
|
2118
|
+
]
|
|
2119
|
+
// vision: undefined — unconstrained
|
|
2120
|
+
// image_generation: undefined — unconstrained
|
|
2121
|
+
};
|
|
2122
|
+
}
|
|
2123
|
+
});
|
|
2124
|
+
|
|
2027
2125
|
// src/compaction/trigger.ts
|
|
2028
2126
|
function getPendingSummaries() {
|
|
2029
2127
|
return pendingSummaries.splice(0);
|
|
@@ -2044,7 +2142,7 @@ function triggerCompaction(state, apiConfig, opts = {}) {
|
|
|
2044
2142
|
apiConfig,
|
|
2045
2143
|
system,
|
|
2046
2144
|
tools2,
|
|
2047
|
-
state.models
|
|
2145
|
+
resolveModel("conversationSummarizer", state.models, model)
|
|
2048
2146
|
).then((summaries) => {
|
|
2049
2147
|
pendingSummaries.push(...summaries);
|
|
2050
2148
|
listener?.({ type: "complete", requestId });
|
|
@@ -2067,6 +2165,7 @@ var init_trigger = __esm({
|
|
|
2067
2165
|
init_prompt();
|
|
2068
2166
|
init_tools6();
|
|
2069
2167
|
init_logger();
|
|
2168
|
+
init_surfaces();
|
|
2070
2169
|
log3 = createLogger("compaction:trigger");
|
|
2071
2170
|
pendingSummaries = [];
|
|
2072
2171
|
inflightCompaction = null;
|
|
@@ -3085,7 +3184,7 @@ var init_queryDatabase = __esm({
|
|
|
3085
3184
|
|
|
3086
3185
|
// src/subagents/common/analyzeImage.ts
|
|
3087
3186
|
async function analyzeImage(params) {
|
|
3088
|
-
const { prompt, imageUrl, timeout = 2e5, onLog } = params;
|
|
3187
|
+
const { prompt, imageUrl, model, timeout = 2e5, onLog } = params;
|
|
3089
3188
|
return runMindstudioCli(
|
|
3090
3189
|
[
|
|
3091
3190
|
"analyze-image",
|
|
@@ -3094,21 +3193,15 @@ async function analyzeImage(params) {
|
|
|
3094
3193
|
"--image-url",
|
|
3095
3194
|
imageUrl,
|
|
3096
3195
|
"--vision-model-override",
|
|
3097
|
-
JSON.stringify(
|
|
3196
|
+
JSON.stringify({ model })
|
|
3098
3197
|
],
|
|
3099
3198
|
{ outputKey: "analysis", timeout, onLog }
|
|
3100
3199
|
);
|
|
3101
3200
|
}
|
|
3102
|
-
var VISION_MODEL, VISION_MODEL_OVERRIDE;
|
|
3103
3201
|
var init_analyzeImage = __esm({
|
|
3104
3202
|
"src/subagents/common/analyzeImage.ts"() {
|
|
3105
3203
|
"use strict";
|
|
3106
3204
|
init_runMindstudioCli();
|
|
3107
|
-
VISION_MODEL = "claude-4-6-sonnet";
|
|
3108
|
-
VISION_MODEL_OVERRIDE = {
|
|
3109
|
-
model: VISION_MODEL,
|
|
3110
|
-
config: { thinkingBudget: "off" }
|
|
3111
|
-
};
|
|
3112
3205
|
}
|
|
3113
3206
|
});
|
|
3114
3207
|
|
|
@@ -3130,13 +3223,14 @@ ${TEXT_WRAP_DISCLAIMER}`;
|
|
|
3130
3223
|
return p;
|
|
3131
3224
|
}
|
|
3132
3225
|
async function streamScreenshotAnalysis(opts) {
|
|
3133
|
-
const { url, prompt, styleMap, onLog } = opts;
|
|
3226
|
+
const { url, prompt, styleMap, onLog, model } = opts;
|
|
3134
3227
|
onLog?.(JSON.stringify({ url, analysis: null }));
|
|
3135
3228
|
const analysisPrompt = buildScreenshotAnalysisPrompt({ prompt, styleMap });
|
|
3136
3229
|
let accumulated = "";
|
|
3137
3230
|
const analysis = await analyzeImage({
|
|
3138
3231
|
prompt: analysisPrompt,
|
|
3139
3232
|
imageUrl: url,
|
|
3233
|
+
model,
|
|
3140
3234
|
onLog: (chunk) => {
|
|
3141
3235
|
accumulated += chunk;
|
|
3142
3236
|
onLog?.(JSON.stringify({ url, analysis: accumulated }));
|
|
@@ -3148,12 +3242,14 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3148
3242
|
let prompt;
|
|
3149
3243
|
let existingUrl;
|
|
3150
3244
|
let onLog;
|
|
3245
|
+
let model;
|
|
3151
3246
|
let path13;
|
|
3152
3247
|
if (typeof promptOrOptions === "object" && promptOrOptions !== null) {
|
|
3153
3248
|
prompt = promptOrOptions.prompt;
|
|
3154
3249
|
existingUrl = promptOrOptions.imageUrl;
|
|
3155
3250
|
path13 = promptOrOptions.path;
|
|
3156
3251
|
onLog = promptOrOptions.onLog;
|
|
3252
|
+
model = promptOrOptions.model;
|
|
3157
3253
|
} else {
|
|
3158
3254
|
prompt = promptOrOptions;
|
|
3159
3255
|
}
|
|
@@ -3178,11 +3274,17 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
3178
3274
|
if (prompt === false) {
|
|
3179
3275
|
return url;
|
|
3180
3276
|
}
|
|
3277
|
+
if (!model) {
|
|
3278
|
+
throw new Error(
|
|
3279
|
+
"captureAndAnalyzeScreenshot: `model` is required when analysis is enabled"
|
|
3280
|
+
);
|
|
3281
|
+
}
|
|
3181
3282
|
return streamScreenshotAnalysis({
|
|
3182
3283
|
url,
|
|
3183
3284
|
prompt: prompt || void 0,
|
|
3184
3285
|
styleMap,
|
|
3185
|
-
onLog
|
|
3286
|
+
onLog,
|
|
3287
|
+
model
|
|
3186
3288
|
});
|
|
3187
3289
|
}
|
|
3188
3290
|
var SCREENSHOT_ANALYSIS_PROMPT, TEXT_WRAP_DISCLAIMER;
|
|
@@ -4085,7 +4187,7 @@ var init_browserAutomation = __esm({
|
|
|
4085
4187
|
init_browserLock();
|
|
4086
4188
|
init_screenshot();
|
|
4087
4189
|
init_runMindstudioCli();
|
|
4088
|
-
|
|
4190
|
+
init_surfaces();
|
|
4089
4191
|
init_logger();
|
|
4090
4192
|
log6 = createLogger("browser-automation");
|
|
4091
4193
|
browserAutomationTool = {
|
|
@@ -4135,7 +4237,12 @@ var init_browserAutomation = __esm({
|
|
|
4135
4237
|
try {
|
|
4136
4238
|
return await captureAndAnalyzeScreenshot({
|
|
4137
4239
|
path: _input.path,
|
|
4138
|
-
onLog
|
|
4240
|
+
onLog,
|
|
4241
|
+
model: resolveModel(
|
|
4242
|
+
"imageAnalysis",
|
|
4243
|
+
context.models,
|
|
4244
|
+
context.model
|
|
4245
|
+
)
|
|
4139
4246
|
});
|
|
4140
4247
|
} catch (err) {
|
|
4141
4248
|
return `Error taking screenshot: ${err.message}`;
|
|
@@ -4144,7 +4251,7 @@ var init_browserAutomation = __esm({
|
|
|
4144
4251
|
return `Error: unknown local tool "${name}"`;
|
|
4145
4252
|
},
|
|
4146
4253
|
apiConfig: context.apiConfig,
|
|
4147
|
-
model: context.models
|
|
4254
|
+
model: resolveModel("browserAutomation", context.models, context.model),
|
|
4148
4255
|
subAgentId: "browserAutomation",
|
|
4149
4256
|
signal: context.signal,
|
|
4150
4257
|
parentToolId: context.toolCallId,
|
|
@@ -4162,6 +4269,13 @@ var init_browserAutomation = __esm({
|
|
|
4162
4269
|
(s) => s.command === "screenshotViewport" && s.result?.url
|
|
4163
4270
|
);
|
|
4164
4271
|
if (screenshotSteps.length > 0) {
|
|
4272
|
+
const visionOverride = {
|
|
4273
|
+
model: resolveModel(
|
|
4274
|
+
"imageAnalysis",
|
|
4275
|
+
context.models,
|
|
4276
|
+
context.model
|
|
4277
|
+
)
|
|
4278
|
+
};
|
|
4165
4279
|
const batchInput = screenshotSteps.map((s) => ({
|
|
4166
4280
|
stepType: "analyzeImage",
|
|
4167
4281
|
step: {
|
|
@@ -4169,7 +4283,7 @@ var init_browserAutomation = __esm({
|
|
|
4169
4283
|
prompt: buildScreenshotAnalysisPrompt({
|
|
4170
4284
|
styleMap: s.result.styleMap
|
|
4171
4285
|
}),
|
|
4172
|
-
visionModelOverride:
|
|
4286
|
+
visionModelOverride: visionOverride
|
|
4173
4287
|
}
|
|
4174
4288
|
}));
|
|
4175
4289
|
const batchResult = await runMindstudioCli(
|
|
@@ -4224,6 +4338,7 @@ var init_screenshot2 = __esm({
|
|
|
4224
4338
|
init_screenshot();
|
|
4225
4339
|
init_browserLock();
|
|
4226
4340
|
init_browserAutomation();
|
|
4341
|
+
init_surfaces();
|
|
4227
4342
|
screenshotTool = {
|
|
4228
4343
|
clearable: true,
|
|
4229
4344
|
definition: {
|
|
@@ -4257,7 +4372,8 @@ var init_screenshot2 = __esm({
|
|
|
4257
4372
|
return await captureAndAnalyzeScreenshot({
|
|
4258
4373
|
prompt: input.prompt,
|
|
4259
4374
|
imageUrl: input.imageUrl,
|
|
4260
|
-
onLog: context?.onLog
|
|
4375
|
+
onLog: context?.onLog,
|
|
4376
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4261
4377
|
});
|
|
4262
4378
|
}
|
|
4263
4379
|
if (input.instructions && context) {
|
|
@@ -4279,7 +4395,8 @@ var init_screenshot2 = __esm({
|
|
|
4279
4395
|
url,
|
|
4280
4396
|
prompt: input.prompt,
|
|
4281
4397
|
styleMap,
|
|
4282
|
-
onLog: context?.onLog
|
|
4398
|
+
onLog: context?.onLog,
|
|
4399
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4283
4400
|
});
|
|
4284
4401
|
}
|
|
4285
4402
|
const release = await acquireBrowserLock();
|
|
@@ -4287,7 +4404,8 @@ var init_screenshot2 = __esm({
|
|
|
4287
4404
|
return await captureAndAnalyzeScreenshot({
|
|
4288
4405
|
prompt: input.prompt,
|
|
4289
4406
|
path: input.path,
|
|
4290
|
-
onLog: context?.onLog
|
|
4407
|
+
onLog: context?.onLog,
|
|
4408
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4291
4409
|
});
|
|
4292
4410
|
} finally {
|
|
4293
4411
|
release();
|
|
@@ -4478,7 +4596,7 @@ __export(analyzeDesign_exports, {
|
|
|
4478
4596
|
definition: () => definition3,
|
|
4479
4597
|
execute: () => execute3
|
|
4480
4598
|
});
|
|
4481
|
-
async function execute3(input, onLog) {
|
|
4599
|
+
async function execute3(input, onLog, context) {
|
|
4482
4600
|
const url = input.url;
|
|
4483
4601
|
const analysisPrompt = input.prompt || DESIGN_REFERENCE_PROMPT;
|
|
4484
4602
|
const isImageUrl = /\.(png|jpe?g|webp|gif|svg|avif)(\?|$)/i.test(url);
|
|
@@ -4511,7 +4629,8 @@ async function execute3(input, onLog) {
|
|
|
4511
4629
|
const analysis = await analyzeImage({
|
|
4512
4630
|
prompt: analysisPrompt,
|
|
4513
4631
|
imageUrl,
|
|
4514
|
-
onLog
|
|
4632
|
+
onLog,
|
|
4633
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4515
4634
|
});
|
|
4516
4635
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4517
4636
|
}
|
|
@@ -4521,6 +4640,7 @@ var init_analyzeDesign = __esm({
|
|
|
4521
4640
|
"use strict";
|
|
4522
4641
|
init_runMindstudioCli();
|
|
4523
4642
|
init_analyzeImage();
|
|
4643
|
+
init_surfaces();
|
|
4524
4644
|
DESIGN_REFERENCE_PROMPT = `
|
|
4525
4645
|
You are analyzing a screenshot of a real website or app for a designer's personal technique/inspiration reference notes.
|
|
4526
4646
|
|
|
@@ -4572,7 +4692,7 @@ __export(analyzeImage_exports, {
|
|
|
4572
4692
|
definition: () => definition4,
|
|
4573
4693
|
execute: () => execute4
|
|
4574
4694
|
});
|
|
4575
|
-
async function execute4(input, onLog) {
|
|
4695
|
+
async function execute4(input, onLog, context) {
|
|
4576
4696
|
const imageUrl = input.imageUrl;
|
|
4577
4697
|
const prompt = buildScreenshotAnalysisPrompt({
|
|
4578
4698
|
prompt: input.prompt
|
|
@@ -4580,7 +4700,8 @@ async function execute4(input, onLog) {
|
|
|
4580
4700
|
const analysis = await analyzeImage({
|
|
4581
4701
|
prompt,
|
|
4582
4702
|
imageUrl,
|
|
4583
|
-
onLog
|
|
4703
|
+
onLog,
|
|
4704
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4584
4705
|
});
|
|
4585
4706
|
return JSON.stringify({ url: imageUrl, analysis });
|
|
4586
4707
|
}
|
|
@@ -4590,6 +4711,7 @@ var init_analyzeImage2 = __esm({
|
|
|
4590
4711
|
"use strict";
|
|
4591
4712
|
init_analyzeImage();
|
|
4592
4713
|
init_screenshot();
|
|
4714
|
+
init_surfaces();
|
|
4593
4715
|
definition4 = {
|
|
4594
4716
|
clearable: true,
|
|
4595
4717
|
name: "analyzeImage",
|
|
@@ -4639,7 +4761,8 @@ async function execute5(input, onLog, context) {
|
|
|
4639
4761
|
url,
|
|
4640
4762
|
prompt: input.prompt,
|
|
4641
4763
|
styleMap,
|
|
4642
|
-
onLog
|
|
4764
|
+
onLog,
|
|
4765
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4643
4766
|
});
|
|
4644
4767
|
} catch (err) {
|
|
4645
4768
|
return `Error taking interactive screenshot: ${err.message}`;
|
|
@@ -4650,7 +4773,8 @@ async function execute5(input, onLog, context) {
|
|
|
4650
4773
|
return await captureAndAnalyzeScreenshot({
|
|
4651
4774
|
prompt: input.prompt,
|
|
4652
4775
|
path: input.path,
|
|
4653
|
-
onLog
|
|
4776
|
+
onLog,
|
|
4777
|
+
model: resolveModel("imageAnalysis", context?.models, context?.model)
|
|
4654
4778
|
});
|
|
4655
4779
|
} catch (err) {
|
|
4656
4780
|
return `Error taking screenshot: ${err.message}`;
|
|
@@ -4665,6 +4789,7 @@ var init_screenshot3 = __esm({
|
|
|
4665
4789
|
init_screenshot();
|
|
4666
4790
|
init_browserLock();
|
|
4667
4791
|
init_browserAutomation();
|
|
4792
|
+
init_surfaces();
|
|
4668
4793
|
definition5 = {
|
|
4669
4794
|
clearable: true,
|
|
4670
4795
|
name: "screenshot",
|
|
@@ -4692,7 +4817,7 @@ var init_screenshot3 = __esm({
|
|
|
4692
4817
|
|
|
4693
4818
|
// src/subagents/designExpert/tools/images/enhancePrompt.ts
|
|
4694
4819
|
async function enhanceImagePrompt(params) {
|
|
4695
|
-
const { brief, width, height, transparentBackground, onLog } = params;
|
|
4820
|
+
const { brief, width, height, transparentBackground, onLog, model } = params;
|
|
4696
4821
|
const contextParts = [
|
|
4697
4822
|
`Dimensions: ${width}x${height}${width > height ? " (landscape)" : width < height ? " (portrait)" : " (square)"}`
|
|
4698
4823
|
];
|
|
@@ -4717,23 +4842,18 @@ ${brief}
|
|
|
4717
4842
|
"--message",
|
|
4718
4843
|
message,
|
|
4719
4844
|
"--model-override",
|
|
4720
|
-
JSON.stringify(
|
|
4845
|
+
JSON.stringify({ model, config: { reasoning: "false" } })
|
|
4721
4846
|
],
|
|
4722
4847
|
{ outputKey: "content", timeout: 6e4, onLog, caller: "designExpert" }
|
|
4723
4848
|
);
|
|
4724
4849
|
return enhanced.trim();
|
|
4725
4850
|
}
|
|
4726
|
-
var
|
|
4851
|
+
var SYSTEM_PROMPT;
|
|
4727
4852
|
var init_enhancePrompt = __esm({
|
|
4728
4853
|
"src/subagents/designExpert/tools/images/enhancePrompt.ts"() {
|
|
4729
4854
|
"use strict";
|
|
4730
4855
|
init_runMindstudioCli();
|
|
4731
4856
|
init_assets();
|
|
4732
|
-
ENHANCE_MODEL = "claude-4-6-sonnet";
|
|
4733
|
-
MODEL_OVERRIDE = {
|
|
4734
|
-
model: ENHANCE_MODEL,
|
|
4735
|
-
config: { reasoning: "false" }
|
|
4736
|
-
};
|
|
4737
4857
|
SYSTEM_PROMPT = readAsset(
|
|
4738
4858
|
"subagents/designExpert/tools/images/enhance-image-prompt.md"
|
|
4739
4859
|
);
|
|
@@ -4742,7 +4862,15 @@ var init_enhancePrompt = __esm({
|
|
|
4742
4862
|
|
|
4743
4863
|
// src/subagents/designExpert/tools/images/imageGenerator.ts
|
|
4744
4864
|
async function generateImageAssets(opts) {
|
|
4745
|
-
const {
|
|
4865
|
+
const {
|
|
4866
|
+
prompts,
|
|
4867
|
+
sourceImages,
|
|
4868
|
+
transparentBackground,
|
|
4869
|
+
onLog,
|
|
4870
|
+
imageGenerationModel: genModel,
|
|
4871
|
+
imageAnalysisModel,
|
|
4872
|
+
imagePromptEnhancerModel
|
|
4873
|
+
} = opts;
|
|
4746
4874
|
const width = opts.width || 2048;
|
|
4747
4875
|
const height = opts.height || 2048;
|
|
4748
4876
|
const config = { width, height };
|
|
@@ -4757,7 +4885,8 @@ async function generateImageAssets(opts) {
|
|
|
4757
4885
|
width,
|
|
4758
4886
|
height,
|
|
4759
4887
|
transparentBackground,
|
|
4760
|
-
onLog
|
|
4888
|
+
onLog,
|
|
4889
|
+
model: imagePromptEnhancerModel
|
|
4761
4890
|
})
|
|
4762
4891
|
)
|
|
4763
4892
|
);
|
|
@@ -4766,7 +4895,7 @@ async function generateImageAssets(opts) {
|
|
|
4766
4895
|
const step = JSON.stringify({
|
|
4767
4896
|
prompt: enhancedPrompts[0],
|
|
4768
4897
|
imageModelOverride: {
|
|
4769
|
-
model:
|
|
4898
|
+
model: genModel,
|
|
4770
4899
|
config
|
|
4771
4900
|
}
|
|
4772
4901
|
});
|
|
@@ -4785,7 +4914,7 @@ async function generateImageAssets(opts) {
|
|
|
4785
4914
|
step: {
|
|
4786
4915
|
prompt,
|
|
4787
4916
|
imageModelOverride: {
|
|
4788
|
-
model:
|
|
4917
|
+
model: genModel,
|
|
4789
4918
|
config
|
|
4790
4919
|
}
|
|
4791
4920
|
}
|
|
@@ -4837,7 +4966,8 @@ async function generateImageAssets(opts) {
|
|
|
4837
4966
|
const analysis = await analyzeImage({
|
|
4838
4967
|
prompt: ANALYZE_PROMPT,
|
|
4839
4968
|
imageUrl: url,
|
|
4840
|
-
onLog
|
|
4969
|
+
onLog,
|
|
4970
|
+
model: imageAnalysisModel
|
|
4841
4971
|
});
|
|
4842
4972
|
return {
|
|
4843
4973
|
url,
|
|
@@ -4868,13 +4998,28 @@ __export(generateImages_exports, {
|
|
|
4868
4998
|
definition: () => definition6,
|
|
4869
4999
|
execute: () => execute6
|
|
4870
5000
|
});
|
|
4871
|
-
async function execute6(input, onLog) {
|
|
5001
|
+
async function execute6(input, onLog, context) {
|
|
4872
5002
|
return generateImageAssets({
|
|
4873
5003
|
prompts: input.prompts,
|
|
4874
5004
|
width: input.width,
|
|
4875
5005
|
height: input.height,
|
|
4876
5006
|
transparentBackground: input.transparentBackground,
|
|
4877
|
-
onLog
|
|
5007
|
+
onLog,
|
|
5008
|
+
imageGenerationModel: resolveModel(
|
|
5009
|
+
"imageGeneration",
|
|
5010
|
+
context?.models,
|
|
5011
|
+
context?.model
|
|
5012
|
+
),
|
|
5013
|
+
imageAnalysisModel: resolveModel(
|
|
5014
|
+
"imageAnalysis",
|
|
5015
|
+
context?.models,
|
|
5016
|
+
context?.model
|
|
5017
|
+
),
|
|
5018
|
+
imagePromptEnhancerModel: resolveModel(
|
|
5019
|
+
"imagePromptEnhancer",
|
|
5020
|
+
context?.models,
|
|
5021
|
+
context?.model
|
|
5022
|
+
)
|
|
4878
5023
|
});
|
|
4879
5024
|
}
|
|
4880
5025
|
var definition6;
|
|
@@ -4882,6 +5027,7 @@ var init_generateImages = __esm({
|
|
|
4882
5027
|
"src/subagents/designExpert/tools/images/generateImages.ts"() {
|
|
4883
5028
|
"use strict";
|
|
4884
5029
|
init_imageGenerator();
|
|
5030
|
+
init_surfaces();
|
|
4885
5031
|
definition6 = {
|
|
4886
5032
|
clearable: false,
|
|
4887
5033
|
name: "generateImages",
|
|
@@ -4921,14 +5067,29 @@ __export(editImages_exports, {
|
|
|
4921
5067
|
definition: () => definition7,
|
|
4922
5068
|
execute: () => execute7
|
|
4923
5069
|
});
|
|
4924
|
-
async function execute7(input, onLog) {
|
|
5070
|
+
async function execute7(input, onLog, context) {
|
|
4925
5071
|
return generateImageAssets({
|
|
4926
5072
|
prompts: input.prompts,
|
|
4927
5073
|
sourceImages: input.sourceImages,
|
|
4928
5074
|
width: input.width,
|
|
4929
5075
|
height: input.height,
|
|
4930
5076
|
transparentBackground: input.transparentBackground,
|
|
4931
|
-
onLog
|
|
5077
|
+
onLog,
|
|
5078
|
+
imageGenerationModel: resolveModel(
|
|
5079
|
+
"imageGeneration",
|
|
5080
|
+
context?.models,
|
|
5081
|
+
context?.model
|
|
5082
|
+
),
|
|
5083
|
+
imageAnalysisModel: resolveModel(
|
|
5084
|
+
"imageAnalysis",
|
|
5085
|
+
context?.models,
|
|
5086
|
+
context?.model
|
|
5087
|
+
),
|
|
5088
|
+
imagePromptEnhancerModel: resolveModel(
|
|
5089
|
+
"imagePromptEnhancer",
|
|
5090
|
+
context?.models,
|
|
5091
|
+
context?.model
|
|
5092
|
+
)
|
|
4932
5093
|
});
|
|
4933
5094
|
}
|
|
4934
5095
|
var definition7;
|
|
@@ -4936,6 +5097,7 @@ var init_editImages = __esm({
|
|
|
4936
5097
|
"src/subagents/designExpert/tools/images/editImages.ts"() {
|
|
4937
5098
|
"use strict";
|
|
4938
5099
|
init_imageGenerator();
|
|
5100
|
+
init_surfaces();
|
|
4939
5101
|
definition7 = {
|
|
4940
5102
|
clearable: false,
|
|
4941
5103
|
name: "editImages",
|
|
@@ -5474,6 +5636,7 @@ var init_designExpert = __esm({
|
|
|
5474
5636
|
init_tools2();
|
|
5475
5637
|
init_prompt3();
|
|
5476
5638
|
init_history();
|
|
5639
|
+
init_surfaces();
|
|
5477
5640
|
DESCRIPTION = `
|
|
5478
5641
|
Visual design expert. Describe the situation and what you need \u2014 the agent decides what to deliver. It reads the spec files automatically. Include relevant user requirements and context it can't get from the spec, but do not list specific deliverables or tell it how to do its job. Do not suggest implementation details or ideas - only relay what is needed.
|
|
5479
5642
|
`.trim();
|
|
@@ -5522,7 +5685,7 @@ Visual design expert. Describe the situation and what you need \u2014 the agent
|
|
|
5522
5685
|
);
|
|
5523
5686
|
},
|
|
5524
5687
|
apiConfig: context.apiConfig,
|
|
5525
|
-
model: context.models
|
|
5688
|
+
model: resolveModel("visualDesignExpert", context.models, context.model),
|
|
5526
5689
|
subAgentId: "visualDesignExpert",
|
|
5527
5690
|
signal: context.signal,
|
|
5528
5691
|
parentToolId: context.toolCallId,
|
|
@@ -5740,6 +5903,7 @@ var init_productVision = __esm({
|
|
|
5740
5903
|
init_executor();
|
|
5741
5904
|
init_prompt4();
|
|
5742
5905
|
init_history();
|
|
5906
|
+
init_surfaces();
|
|
5743
5907
|
productVisionTool = {
|
|
5744
5908
|
clearable: false,
|
|
5745
5909
|
definition: {
|
|
@@ -5779,7 +5943,7 @@ var init_productVision = __esm({
|
|
|
5779
5943
|
return executeVisionTool(name, input2, childCtx);
|
|
5780
5944
|
},
|
|
5781
5945
|
apiConfig: context.apiConfig,
|
|
5782
|
-
model: context.models
|
|
5946
|
+
model: resolveModel("productVision", context.models, context.model),
|
|
5783
5947
|
subAgentId: "productVision",
|
|
5784
5948
|
signal: context.signal,
|
|
5785
5949
|
parentToolId: context.toolCallId,
|
|
@@ -5870,6 +6034,7 @@ var init_codeSanityCheck = __esm({
|
|
|
5870
6034
|
init_context();
|
|
5871
6035
|
init_tools6();
|
|
5872
6036
|
init_tools5();
|
|
6037
|
+
init_surfaces();
|
|
5873
6038
|
BASE_PROMPT3 = readAsset("subagents/codeSanityCheck", "prompt.md");
|
|
5874
6039
|
codeSanityCheckTool = {
|
|
5875
6040
|
clearable: false,
|
|
@@ -5905,7 +6070,7 @@ var init_codeSanityCheck = __esm({
|
|
|
5905
6070
|
externalTools: /* @__PURE__ */ new Set(),
|
|
5906
6071
|
executeTool: (name, toolInput) => executeTool(name, toolInput, context),
|
|
5907
6072
|
apiConfig: context.apiConfig,
|
|
5908
|
-
model: context.models
|
|
6073
|
+
model: resolveModel("codeSanityCheck", context.models, context.model),
|
|
5909
6074
|
subAgentId: "codeSanityCheck",
|
|
5910
6075
|
signal: context.signal,
|
|
5911
6076
|
parentToolId: context.toolCallId,
|
|
@@ -6869,7 +7034,7 @@ async function runTurn(params) {
|
|
|
6869
7034
|
onEvent({ type: "tool_input_delta", id, name, result: content });
|
|
6870
7035
|
}
|
|
6871
7036
|
}
|
|
6872
|
-
const parentModel = state.models
|
|
7037
|
+
const parentModel = resolveModel("parent", state.models, model);
|
|
6873
7038
|
try {
|
|
6874
7039
|
for await (const event of streamChatWithRetry(
|
|
6875
7040
|
{
|
|
@@ -7213,7 +7378,10 @@ async function runTurn(params) {
|
|
|
7213
7378
|
isError: r.isError
|
|
7214
7379
|
});
|
|
7215
7380
|
if (!r.isError && BRAND_TRIGGERING_TOOLS.has(tc.name)) {
|
|
7216
|
-
triggerBrandExtraction(
|
|
7381
|
+
triggerBrandExtraction(
|
|
7382
|
+
apiConfig,
|
|
7383
|
+
resolveModel("brandExtractor", state.models, model)
|
|
7384
|
+
);
|
|
7217
7385
|
}
|
|
7218
7386
|
return r;
|
|
7219
7387
|
})
|
|
@@ -7270,6 +7438,7 @@ var init_agent = __esm({
|
|
|
7270
7438
|
init_tools6();
|
|
7271
7439
|
init_sentinel();
|
|
7272
7440
|
init_trigger2();
|
|
7441
|
+
init_surfaces();
|
|
7273
7442
|
log10 = createLogger("agent");
|
|
7274
7443
|
BRAND_TRIGGERING_TOOLS = /* @__PURE__ */ new Set(["writeSpec", "editSpec"]);
|
|
7275
7444
|
EXTERNAL_TOOLS = /* @__PURE__ */ new Set([
|
|
@@ -7731,6 +7900,7 @@ var init_headless = __esm({
|
|
|
7731
7900
|
init_lsp();
|
|
7732
7901
|
init_agent();
|
|
7733
7902
|
init_session();
|
|
7903
|
+
init_surfaces();
|
|
7734
7904
|
init_toolRegistry();
|
|
7735
7905
|
init_attachments();
|
|
7736
7906
|
init_planFile();
|
|
@@ -7811,12 +7981,14 @@ var init_headless = __esm({
|
|
|
7811
7981
|
this.emit("session_restored", {
|
|
7812
7982
|
messageCount: this.state.messages.length,
|
|
7813
7983
|
...this.state.models && { models: this.state.models },
|
|
7984
|
+
modelSurfaces: MODEL_SURFACES,
|
|
7985
|
+
allowedModelsByType: ALLOWED_MODELS_BY_TYPE,
|
|
7814
7986
|
...this.queueFields()
|
|
7815
7987
|
});
|
|
7816
7988
|
}
|
|
7817
7989
|
triggerBrandExtraction(
|
|
7818
7990
|
this.config,
|
|
7819
|
-
this.state.models
|
|
7991
|
+
resolveModel("brandExtractor", this.state.models, this.opts.model)
|
|
7820
7992
|
);
|
|
7821
7993
|
this.toolRegistry.onEvent = this.onEvent;
|
|
7822
7994
|
setCompactionListener((event) => {
|
|
@@ -8494,6 +8666,8 @@ var init_headless = __esm({
|
|
|
8494
8666
|
running: this.running,
|
|
8495
8667
|
...this.running && this.currentRequestId ? { currentRequestId: this.currentRequestId } : {},
|
|
8496
8668
|
...this.state.models && { models: this.state.models },
|
|
8669
|
+
modelSurfaces: MODEL_SURFACES,
|
|
8670
|
+
allowedModelsByType: ALLOWED_MODELS_BY_TYPE,
|
|
8497
8671
|
...this.queueFields()
|
|
8498
8672
|
}));
|
|
8499
8673
|
return;
|