@mindstudio-ai/remy 0.1.58 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/headless.js +30 -6
- package/dist/index.js +30 -6
- package/dist/prompt/compiled/interfaces.md +9 -5
- package/dist/prompt/static/team.md +3 -1
- package/package.json +1 -1
package/dist/headless.js
CHANGED
|
@@ -2350,9 +2350,11 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2350
2350
|
let prompt;
|
|
2351
2351
|
let existingUrl;
|
|
2352
2352
|
let onLog;
|
|
2353
|
+
let path9;
|
|
2353
2354
|
if (typeof promptOrOptions === "object" && promptOrOptions !== null) {
|
|
2354
2355
|
prompt = promptOrOptions.prompt;
|
|
2355
2356
|
existingUrl = promptOrOptions.imageUrl;
|
|
2357
|
+
path9 = promptOrOptions.path;
|
|
2356
2358
|
onLog = promptOrOptions.onLog;
|
|
2357
2359
|
} else {
|
|
2358
2360
|
prompt = promptOrOptions;
|
|
@@ -2361,9 +2363,11 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2361
2363
|
if (existingUrl) {
|
|
2362
2364
|
url = existingUrl;
|
|
2363
2365
|
} else {
|
|
2364
|
-
const ssResult = await sidecarRequest(
|
|
2365
|
-
|
|
2366
|
-
|
|
2366
|
+
const ssResult = await sidecarRequest(
|
|
2367
|
+
"/screenshot-full-page",
|
|
2368
|
+
path9 ? { path: path9 } : void 0,
|
|
2369
|
+
{ timeout: 12e4 }
|
|
2370
|
+
);
|
|
2367
2371
|
url = ssResult?.url || ssResult?.screenshotUrl;
|
|
2368
2372
|
if (!url) {
|
|
2369
2373
|
throw new Error(
|
|
@@ -2398,6 +2402,10 @@ var screenshotTool = {
|
|
|
2398
2402
|
imageUrl: {
|
|
2399
2403
|
type: "string",
|
|
2400
2404
|
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
2405
|
+
},
|
|
2406
|
+
path: {
|
|
2407
|
+
type: "string",
|
|
2408
|
+
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
2401
2409
|
}
|
|
2402
2410
|
}
|
|
2403
2411
|
}
|
|
@@ -2413,6 +2421,7 @@ var screenshotTool = {
|
|
|
2413
2421
|
}
|
|
2414
2422
|
return await captureAndAnalyzeScreenshot({
|
|
2415
2423
|
prompt: input.prompt,
|
|
2424
|
+
path: input.path,
|
|
2416
2425
|
onLog: context?.onLog
|
|
2417
2426
|
});
|
|
2418
2427
|
} catch (err) {
|
|
@@ -2425,6 +2434,7 @@ var screenshotTool = {
|
|
|
2425
2434
|
function startStatusWatcher(config) {
|
|
2426
2435
|
const { apiConfig, getContext, onStatus, interval = 3e3, signal } = config;
|
|
2427
2436
|
let lastLabel = "";
|
|
2437
|
+
let lastContext = "";
|
|
2428
2438
|
let inflight = false;
|
|
2429
2439
|
let stopped = false;
|
|
2430
2440
|
const url = `${apiConfig.baseUrl}/_internal/v2/agent/remy/generate-status`;
|
|
@@ -2435,9 +2445,10 @@ function startStatusWatcher(config) {
|
|
|
2435
2445
|
inflight = true;
|
|
2436
2446
|
try {
|
|
2437
2447
|
const context = getContext();
|
|
2438
|
-
if (!context) {
|
|
2448
|
+
if (!context || context === lastContext) {
|
|
2439
2449
|
return;
|
|
2440
2450
|
}
|
|
2451
|
+
lastContext = context;
|
|
2441
2452
|
const res = await fetch(url, {
|
|
2442
2453
|
method: "POST",
|
|
2443
2454
|
headers: {
|
|
@@ -2983,7 +2994,12 @@ var BROWSER_TOOLS = [
|
|
|
2983
2994
|
description: "Capture a full-height screenshot of the current page. Returns a CDN URL with full text analysis and description.",
|
|
2984
2995
|
inputSchema: {
|
|
2985
2996
|
type: "object",
|
|
2986
|
-
properties: {
|
|
2997
|
+
properties: {
|
|
2998
|
+
path: {
|
|
2999
|
+
type: "string",
|
|
3000
|
+
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
2987
3003
|
}
|
|
2988
3004
|
},
|
|
2989
3005
|
{
|
|
@@ -3056,7 +3072,10 @@ var browserAutomationTool = {
|
|
|
3056
3072
|
executeTool: async (name, _input, _toolCallId, onLog) => {
|
|
3057
3073
|
if (name === "screenshotFullPage") {
|
|
3058
3074
|
try {
|
|
3059
|
-
return await captureAndAnalyzeScreenshot({
|
|
3075
|
+
return await captureAndAnalyzeScreenshot({
|
|
3076
|
+
path: _input.path,
|
|
3077
|
+
onLog
|
|
3078
|
+
});
|
|
3060
3079
|
} catch (err) {
|
|
3061
3080
|
return `Error taking screenshot: ${err.message}`;
|
|
3062
3081
|
}
|
|
@@ -3308,6 +3327,10 @@ var definition5 = {
|
|
|
3308
3327
|
prompt: {
|
|
3309
3328
|
type: "string",
|
|
3310
3329
|
description: "Optional specific question about the screenshot."
|
|
3330
|
+
},
|
|
3331
|
+
path: {
|
|
3332
|
+
type: "string",
|
|
3333
|
+
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
3311
3334
|
}
|
|
3312
3335
|
}
|
|
3313
3336
|
}
|
|
@@ -3316,6 +3339,7 @@ async function execute5(input, onLog) {
|
|
|
3316
3339
|
try {
|
|
3317
3340
|
return await captureAndAnalyzeScreenshot({
|
|
3318
3341
|
prompt: input.prompt,
|
|
3342
|
+
path: input.path,
|
|
3319
3343
|
onLog
|
|
3320
3344
|
});
|
|
3321
3345
|
} catch (err) {
|
package/dist/index.js
CHANGED
|
@@ -2056,9 +2056,11 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2056
2056
|
let prompt;
|
|
2057
2057
|
let existingUrl;
|
|
2058
2058
|
let onLog;
|
|
2059
|
+
let path10;
|
|
2059
2060
|
if (typeof promptOrOptions === "object" && promptOrOptions !== null) {
|
|
2060
2061
|
prompt = promptOrOptions.prompt;
|
|
2061
2062
|
existingUrl = promptOrOptions.imageUrl;
|
|
2063
|
+
path10 = promptOrOptions.path;
|
|
2062
2064
|
onLog = promptOrOptions.onLog;
|
|
2063
2065
|
} else {
|
|
2064
2066
|
prompt = promptOrOptions;
|
|
@@ -2067,9 +2069,11 @@ async function captureAndAnalyzeScreenshot(promptOrOptions) {
|
|
|
2067
2069
|
if (existingUrl) {
|
|
2068
2070
|
url = existingUrl;
|
|
2069
2071
|
} else {
|
|
2070
|
-
const ssResult = await sidecarRequest(
|
|
2071
|
-
|
|
2072
|
-
|
|
2072
|
+
const ssResult = await sidecarRequest(
|
|
2073
|
+
"/screenshot-full-page",
|
|
2074
|
+
path10 ? { path: path10 } : void 0,
|
|
2075
|
+
{ timeout: 12e4 }
|
|
2076
|
+
);
|
|
2073
2077
|
url = ssResult?.url || ssResult?.screenshotUrl;
|
|
2074
2078
|
if (!url) {
|
|
2075
2079
|
throw new Error(
|
|
@@ -2118,6 +2122,10 @@ var init_screenshot2 = __esm({
|
|
|
2118
2122
|
imageUrl: {
|
|
2119
2123
|
type: "string",
|
|
2120
2124
|
description: "URL of an existing screenshot to analyze instead of capturing a new one. Use this for additional questions about a previous screenshot."
|
|
2125
|
+
},
|
|
2126
|
+
path: {
|
|
2127
|
+
type: "string",
|
|
2128
|
+
description: 'Navigate to this path before capturing (e.g. "/settings", "/dashboard"). If omitted, screenshots the current page.'
|
|
2121
2129
|
}
|
|
2122
2130
|
}
|
|
2123
2131
|
}
|
|
@@ -2133,6 +2141,7 @@ var init_screenshot2 = __esm({
|
|
|
2133
2141
|
}
|
|
2134
2142
|
return await captureAndAnalyzeScreenshot({
|
|
2135
2143
|
prompt: input.prompt,
|
|
2144
|
+
path: input.path,
|
|
2136
2145
|
onLog: context?.onLog
|
|
2137
2146
|
});
|
|
2138
2147
|
} catch (err) {
|
|
@@ -2147,6 +2156,7 @@ var init_screenshot2 = __esm({
|
|
|
2147
2156
|
function startStatusWatcher(config) {
|
|
2148
2157
|
const { apiConfig, getContext, onStatus, interval = 3e3, signal } = config;
|
|
2149
2158
|
let lastLabel = "";
|
|
2159
|
+
let lastContext = "";
|
|
2150
2160
|
let inflight = false;
|
|
2151
2161
|
let stopped = false;
|
|
2152
2162
|
const url = `${apiConfig.baseUrl}/_internal/v2/agent/remy/generate-status`;
|
|
@@ -2157,9 +2167,10 @@ function startStatusWatcher(config) {
|
|
|
2157
2167
|
inflight = true;
|
|
2158
2168
|
try {
|
|
2159
2169
|
const context = getContext();
|
|
2160
|
-
if (!context) {
|
|
2170
|
+
if (!context || context === lastContext) {
|
|
2161
2171
|
return;
|
|
2162
2172
|
}
|
|
2173
|
+
lastContext = context;
|
|
2163
2174
|
const res = await fetch(url, {
|
|
2164
2175
|
method: "POST",
|
|
2165
2176
|
headers: {
|
|
@@ -2729,7 +2740,12 @@ var init_tools = __esm({
|
|
|
2729
2740
|
description: "Capture a full-height screenshot of the current page. Returns a CDN URL with full text analysis and description.",
|
|
2730
2741
|
inputSchema: {
|
|
2731
2742
|
type: "object",
|
|
2732
|
-
properties: {
|
|
2743
|
+
properties: {
|
|
2744
|
+
path: {
|
|
2745
|
+
type: "string",
|
|
2746
|
+
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2733
2749
|
}
|
|
2734
2750
|
},
|
|
2735
2751
|
{
|
|
@@ -2865,7 +2881,10 @@ var init_browserAutomation = __esm({
|
|
|
2865
2881
|
executeTool: async (name, _input, _toolCallId, onLog) => {
|
|
2866
2882
|
if (name === "screenshotFullPage") {
|
|
2867
2883
|
try {
|
|
2868
|
-
return await captureAndAnalyzeScreenshot({
|
|
2884
|
+
return await captureAndAnalyzeScreenshot({
|
|
2885
|
+
path: _input.path,
|
|
2886
|
+
onLog
|
|
2887
|
+
});
|
|
2869
2888
|
} catch (err) {
|
|
2870
2889
|
return `Error taking screenshot: ${err.message}`;
|
|
2871
2890
|
}
|
|
@@ -3143,6 +3162,7 @@ async function execute5(input, onLog) {
|
|
|
3143
3162
|
try {
|
|
3144
3163
|
return await captureAndAnalyzeScreenshot({
|
|
3145
3164
|
prompt: input.prompt,
|
|
3165
|
+
path: input.path,
|
|
3146
3166
|
onLog
|
|
3147
3167
|
});
|
|
3148
3168
|
} catch (err) {
|
|
@@ -3163,6 +3183,10 @@ var init_screenshot3 = __esm({
|
|
|
3163
3183
|
prompt: {
|
|
3164
3184
|
type: "string",
|
|
3165
3185
|
description: "Optional specific question about the screenshot."
|
|
3186
|
+
},
|
|
3187
|
+
path: {
|
|
3188
|
+
type: "string",
|
|
3189
|
+
description: 'Navigate to this path before capturing (e.g. "/settings"). If omitted, screenshots the current page.'
|
|
3166
3190
|
}
|
|
3167
3191
|
}
|
|
3168
3192
|
}
|
|
@@ -26,12 +26,16 @@ dist/interfaces/web/
|
|
|
26
26
|
|
|
27
27
|
```json
|
|
28
28
|
{
|
|
29
|
-
"
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
"web": {
|
|
30
|
+
"devPort": 5173,
|
|
31
|
+
"devCommand": "npm run dev",
|
|
32
|
+
"defaultPreviewMode": "desktop"
|
|
33
|
+
}
|
|
32
34
|
}
|
|
33
35
|
```
|
|
34
36
|
|
|
37
|
+
All fields are nested under the `"web"` key.
|
|
38
|
+
|
|
35
39
|
| Field | Type | Default | Description |
|
|
36
40
|
|-------|------|---------|-------------|
|
|
37
41
|
| `devPort` | `number` | `5173` | Port for the dev server |
|
|
@@ -99,7 +103,7 @@ Omit the `methods` field (or the config entirely) to expose all methods.
|
|
|
99
103
|
### Usage
|
|
100
104
|
|
|
101
105
|
```bash
|
|
102
|
-
curl -X POST https://
|
|
106
|
+
curl -X POST https://{app-subdomain}.mindstudio.ai/_/methods/submit-vendor-request/invoke \
|
|
103
107
|
-H "Authorization: Bearer sk..." \
|
|
104
108
|
-H "Content-Type: application/json" \
|
|
105
109
|
-d '{ "input": { "name": "Acme" } }'
|
|
@@ -200,7 +204,7 @@ Inbound HTTP endpoints that invoke methods.
|
|
|
200
204
|
}
|
|
201
205
|
```
|
|
202
206
|
|
|
203
|
-
Endpoint URL: `https://
|
|
207
|
+
Endpoint URL: `https://{app-subdomain}.mindstudio.ai/_/webhook/{secret}`
|
|
204
208
|
|
|
205
209
|
Accepts any HTTP method. The method receives `{ method, headers, query, body }` as input.
|
|
206
210
|
|
|
@@ -38,7 +38,9 @@ Always consult the code sanity check before writing code in initialCodegen with
|
|
|
38
38
|
|
|
39
39
|
### QA (`runAutomatedBrowserTest`)
|
|
40
40
|
|
|
41
|
-
For verifying complex stateful interactions: multi-step form submissions, auth flows, real-time updates, flows that require specific data/role setup. This spins up a full chrome browser automation — it's heavyweight. Do not use it for basic rendering or navigation checks. If you can verify something with a screenshot or by reading the code, do that instead. Run a scenario first to seed test data and set user roles. The user is able to watch QA work on their screen via a live browser preview - the cursor will move, type, etc - so you can also use this to demo functionality to the user and help them understand how to use their app.
|
|
41
|
+
For verifying complex stateful interactions: multi-step form submissions, auth flows, real-time updates, flows that require specific data/role setup. This spins up a full chrome browser automation — it's heavyweight. Do not use it for basic rendering or navigation checks. If you can verify something with a screenshot or by reading the code, do that instead. Run a scenario first to seed test data and set user roles. The user is able to watch QA work on their screen via a live browser preview - the cursor will move, type, etc - so you can also use this to demo functionality to the user and help them understand how to use their app.
|
|
42
|
+
|
|
43
|
+
The QA agent can see the screen. Describe what to test, not how — it will figure out what to click, what to check, and what values to use.
|
|
42
44
|
|
|
43
45
|
### Background Execution
|
|
44
46
|
|