@mindstudio-ai/agent 0.1.171 → 0.1.173
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/cli.js +575 -12
- package/dist/index.d.ts +406 -10
- package/dist/index.js +569 -5
- package/dist/index.js.map +1 -1
- package/dist/postinstall.js +575 -12
- package/llms.txt +28 -2
- package/package.json +1 -1
package/dist/postinstall.js
CHANGED
|
@@ -271,9 +271,9 @@ var init_metadata = __esm({
|
|
|
271
271
|
"downloadVideo": {
|
|
272
272
|
stepType: "downloadVideo",
|
|
273
273
|
description: "Download a video file",
|
|
274
|
-
usageNotes: "- Works with YouTube, TikTok, etc., by using ytdlp behind the scenes\n- Can save as mp4 or mp3",
|
|
275
|
-
inputSchema: { "type": "object", "properties": { "videoUrl": { "type": "string", "description": "URL of the video to download (supports YouTube, TikTok, etc. via yt-dlp)" }, "format": { "enum": ["mp4", "mp3"], "type": "string", "description": "Output format for the downloaded file" } }, "required": ["videoUrl", "format"] },
|
|
276
|
-
outputSchema: { "type": "object", "properties": { "videoUrl": { "type": "string", "description": "URL of the downloaded and re-hosted video file" } }, "required": ["videoUrl"] }
|
|
274
|
+
usageNotes: "- Works with YouTube, TikTok, etc., by using ytdlp behind the scenes\n- Can save as mp4 or mp3\n- `item` picks one item of a URL that holds several: a carousel, a playlist, a post with several videos. A link to one slide of a carousel downloads that slide. `itemCount` in the output says how many there are, so download item 1 and loop for the rest\n- `start` and `duration` (seconds) download a clip. An mp4 clip is cut without re-encoding, so it can start a moment before `start` (for an exact cut, trim the result with `trimMedia`); an mp3 clip is exact\n- Videos and audio only: a photo (a photo post, a photo in a carousel) fails as `video_unavailable`, and so does an item the URL doesn't have",
|
|
275
|
+
inputSchema: { "type": "object", "properties": { "videoUrl": { "type": "string", "description": "URL of the video to download (supports YouTube, TikTok, etc. via yt-dlp)" }, "format": { "enum": ["mp4", "mp3"], "type": "string", "description": "Output format for the downloaded file" }, "item": { "type": "number", "description": "Which item to download from a URL that holds several (a carousel, a playlist, a post with several videos), counting from 1. Omit for the one the URL names, else the first" }, "start": { "type": "number", "description": "Start of a clip, in seconds. Omit to start at the beginning" }, "duration": { "type": "number", "description": "Length of a clip, in seconds. Omit to run to the end" } }, "required": ["videoUrl", "format"] },
|
|
276
|
+
outputSchema: { "type": "object", "properties": { "videoUrl": { "type": "string", "description": "URL of the downloaded and re-hosted video file" }, "title": { "type": "string", "description": "The video's title, as the site gives it" }, "durationSec": { "type": "number", "description": "How long the downloaded file runs, in seconds: a clip's own length" }, "itemCount": { "type": "number", "description": "How many items the URL holds: 1 for a single video. Absent for a playlist the site doesn't count up front (a channel, a long playlist)" } }, "required": ["videoUrl"] }
|
|
277
277
|
},
|
|
278
278
|
"enhanceImageGenerationPrompt": {
|
|
279
279
|
stepType: "enhanceImageGenerationPrompt",
|
|
@@ -5591,6 +5591,7 @@ function createVoice(call) {
|
|
|
5591
5591
|
async call(params) {
|
|
5592
5592
|
return await call("call", {
|
|
5593
5593
|
to: params.to,
|
|
5594
|
+
...params.from !== void 0 ? { from: params.from } : {},
|
|
5594
5595
|
assumeIdentity: params.assumeIdentity === true
|
|
5595
5596
|
});
|
|
5596
5597
|
}
|
|
@@ -5740,6 +5741,525 @@ var init_events = __esm({
|
|
|
5740
5741
|
}
|
|
5741
5742
|
});
|
|
5742
5743
|
|
|
5744
|
+
// src/browser/index.ts
|
|
5745
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
5746
|
+
function createBrowser(call, steps) {
|
|
5747
|
+
return {
|
|
5748
|
+
async open(options = {}) {
|
|
5749
|
+
const { session } = await call("open", {
|
|
5750
|
+
...options.viewport !== void 0 ? { viewport: options.viewport } : {},
|
|
5751
|
+
...options.idleTimeoutSeconds !== void 0 ? { idleTimeoutSeconds: options.idleTimeoutSeconds } : {}
|
|
5752
|
+
});
|
|
5753
|
+
let ready = session.ready;
|
|
5754
|
+
let failures = 0;
|
|
5755
|
+
while (!ready) {
|
|
5756
|
+
try {
|
|
5757
|
+
({ ready } = await call("ready", { sessionId: session.id }));
|
|
5758
|
+
failures = 0;
|
|
5759
|
+
} catch (err) {
|
|
5760
|
+
const answered = err instanceof MindStudioError && err.status < 500;
|
|
5761
|
+
if (answered || err.code === "browser_unavailable" || ++failures > 5) {
|
|
5762
|
+
throw err;
|
|
5763
|
+
}
|
|
5764
|
+
await new Promise((r) => setTimeout(r, 2e3));
|
|
5765
|
+
}
|
|
5766
|
+
}
|
|
5767
|
+
const opened = new BrowserSession(session, call, steps);
|
|
5768
|
+
if (options.url) {
|
|
5769
|
+
try {
|
|
5770
|
+
await opened.goto(options.url);
|
|
5771
|
+
} catch (err) {
|
|
5772
|
+
await opened.close().catch(() => {
|
|
5773
|
+
});
|
|
5774
|
+
throw err;
|
|
5775
|
+
}
|
|
5776
|
+
}
|
|
5777
|
+
return opened;
|
|
5778
|
+
},
|
|
5779
|
+
async session(id) {
|
|
5780
|
+
const { session } = await call("session", { sessionId: id });
|
|
5781
|
+
return new BrowserSession(session, call, steps);
|
|
5782
|
+
}
|
|
5783
|
+
};
|
|
5784
|
+
}
|
|
5785
|
+
var RETRYABLE_STATUSES, RETRY_BACKOFFS_MS, MAX_TOOL_STEPS, describeScreenPrompt, toWireTarget, BrowserSession;
|
|
5786
|
+
var init_browser = __esm({
|
|
5787
|
+
"src/browser/index.ts"() {
|
|
5788
|
+
"use strict";
|
|
5789
|
+
init_errors();
|
|
5790
|
+
RETRYABLE_STATUSES = /* @__PURE__ */ new Set([502, 503, 504]);
|
|
5791
|
+
RETRY_BACKOFFS_MS = [1e3, 2e3, 4e3, 8e3, 15e3, 15e3, 15e3];
|
|
5792
|
+
MAX_TOOL_STEPS = 20;
|
|
5793
|
+
describeScreenPrompt = (question) => [
|
|
5794
|
+
"This is a screenshot of a web page, taken by an agent working through the site.",
|
|
5795
|
+
question ? `First answer this question about it: ${question}
|
|
5796
|
+
|
|
5797
|
+
Then describe the page briefly: its layout, its content, and what state it is in.` : "Describe what a person looking at it would see, top to bottom: the layout, the content, what state the page is in (a form partly filled, an error, a dialog open), and anything that looks wrong.",
|
|
5798
|
+
"Be concise and concrete, and quote visible text exactly where it matters. Answer in plain Markdown with no preamble."
|
|
5799
|
+
].join("\n\n");
|
|
5800
|
+
toWireTarget = (target) => typeof target === "string" ? { ref: target } : target;
|
|
5801
|
+
BrowserSession = class {
|
|
5802
|
+
/** @internal Use `browser.open()`. */
|
|
5803
|
+
constructor(info, call, steps) {
|
|
5804
|
+
this.call = call;
|
|
5805
|
+
this.steps = steps;
|
|
5806
|
+
this.id = info.id;
|
|
5807
|
+
this.viewport = info.viewport;
|
|
5808
|
+
this.expiresAt = info.expiresAt;
|
|
5809
|
+
}
|
|
5810
|
+
call;
|
|
5811
|
+
steps;
|
|
5812
|
+
/** Pass to `browser.session(id)` from a later call to pick this session up again. */
|
|
5813
|
+
id;
|
|
5814
|
+
viewport;
|
|
5815
|
+
/** When the session closes, however busy it is. Milliseconds since epoch. */
|
|
5816
|
+
expiresAt;
|
|
5817
|
+
/** Every action so far, chained: see `run`. */
|
|
5818
|
+
queue = Promise.resolve();
|
|
5819
|
+
/**
|
|
5820
|
+
* One action at a time, in the order they were called. A session has one
|
|
5821
|
+
* page, so actions started together (a `Promise.all`, or a task agent's
|
|
5822
|
+
* tools from one turn, which run concurrently) would otherwise reach it in
|
|
5823
|
+
* whatever order their requests landed: a click on Submit before the field
|
|
5824
|
+
* it submits was typed.
|
|
5825
|
+
*/
|
|
5826
|
+
run(action) {
|
|
5827
|
+
const result = this.queue.then(() => this.send(action));
|
|
5828
|
+
this.queue = result.catch(() => {
|
|
5829
|
+
});
|
|
5830
|
+
return result;
|
|
5831
|
+
}
|
|
5832
|
+
/**
|
|
5833
|
+
* One action's request, asked again under the same id when the answer
|
|
5834
|
+
* didn't get back (a dropped connection, an api pod rolling). The platform
|
|
5835
|
+
* runs an id once, so asking again waits on the action already under way
|
|
5836
|
+
* rather than clicking twice.
|
|
5837
|
+
*/
|
|
5838
|
+
async send(action) {
|
|
5839
|
+
const commandId = randomUUID2();
|
|
5840
|
+
for (let attempt = 0; ; attempt++) {
|
|
5841
|
+
try {
|
|
5842
|
+
const { value } = await this.call("run", {
|
|
5843
|
+
sessionId: this.id,
|
|
5844
|
+
action,
|
|
5845
|
+
commandId
|
|
5846
|
+
});
|
|
5847
|
+
return value;
|
|
5848
|
+
} catch (err) {
|
|
5849
|
+
const cut = !(err instanceof MindStudioError) || RETRYABLE_STATUSES.has(err.status) && err.code !== "browser_unavailable";
|
|
5850
|
+
if (!cut || attempt >= RETRY_BACKOFFS_MS.length) {
|
|
5851
|
+
throw err;
|
|
5852
|
+
}
|
|
5853
|
+
await new Promise((r) => setTimeout(r, RETRY_BACKOFFS_MS[attempt]));
|
|
5854
|
+
}
|
|
5855
|
+
}
|
|
5856
|
+
}
|
|
5857
|
+
/** Open a URL and wait for it to load. */
|
|
5858
|
+
goto(url) {
|
|
5859
|
+
return this.run({ op: "goto", url });
|
|
5860
|
+
}
|
|
5861
|
+
/** The page as it is now. */
|
|
5862
|
+
snapshot() {
|
|
5863
|
+
return this.run({ op: "snapshot" });
|
|
5864
|
+
}
|
|
5865
|
+
/** Click an element (tap, on a mobile viewport), and wait for what it started to settle. */
|
|
5866
|
+
click(target) {
|
|
5867
|
+
return this.run({ op: "click", target: toWireTarget(target) });
|
|
5868
|
+
}
|
|
5869
|
+
/**
|
|
5870
|
+
* Click into a field and type. `clear` replaces what's there; `submit`
|
|
5871
|
+
* presses Enter after.
|
|
5872
|
+
*/
|
|
5873
|
+
type(target, text, options = {}) {
|
|
5874
|
+
return this.run({
|
|
5875
|
+
op: "type",
|
|
5876
|
+
target: toWireTarget(target),
|
|
5877
|
+
text,
|
|
5878
|
+
...options
|
|
5879
|
+
});
|
|
5880
|
+
}
|
|
5881
|
+
/** Choose an option of a `<select>`, by its text, value or label. */
|
|
5882
|
+
select(target, option) {
|
|
5883
|
+
return this.run({ op: "select", target: toWireTarget(target), option });
|
|
5884
|
+
}
|
|
5885
|
+
/**
|
|
5886
|
+
* Press a key on whatever has focus: `Enter`, `Tab`, `Escape`, `Backspace`,
|
|
5887
|
+
* `Delete`, the arrows, `PageUp`, `PageDown`, `Home`, `End`, `' '`, or any
|
|
5888
|
+
* single character.
|
|
5889
|
+
*/
|
|
5890
|
+
press(key) {
|
|
5891
|
+
return this.run({ op: "press", key });
|
|
5892
|
+
}
|
|
5893
|
+
/** Scroll an element into view, or the page by `by` pixels (most of a screen, by default). */
|
|
5894
|
+
scroll(options = {}) {
|
|
5895
|
+
return this.run({
|
|
5896
|
+
op: "scroll",
|
|
5897
|
+
...options.to !== void 0 ? { target: toWireTarget(options.to) } : {},
|
|
5898
|
+
...options.by !== void 0 ? { by: options.by } : {}
|
|
5899
|
+
});
|
|
5900
|
+
}
|
|
5901
|
+
/** Wait for text to appear on the page. Throws once `timeoutSeconds` (15 by default) pass without it. */
|
|
5902
|
+
waitFor(text, options = {}) {
|
|
5903
|
+
return this.run({
|
|
5904
|
+
op: "wait",
|
|
5905
|
+
text,
|
|
5906
|
+
...options.timeoutSeconds !== void 0 ? { timeoutMs: options.timeoutSeconds * 1e3 } : {}
|
|
5907
|
+
});
|
|
5908
|
+
}
|
|
5909
|
+
async screenshot(options = {}) {
|
|
5910
|
+
const { store, key, fullPage, cursor, boxes } = options;
|
|
5911
|
+
const asked = {
|
|
5912
|
+
op: "screenshot",
|
|
5913
|
+
fullPage: !!fullPage,
|
|
5914
|
+
cursor: !!cursor,
|
|
5915
|
+
...boxes?.length ? { boxes: boxes.map(toWireTarget) } : {}
|
|
5916
|
+
};
|
|
5917
|
+
const measured = (shot2) => ({
|
|
5918
|
+
width: shot2.width,
|
|
5919
|
+
height: shot2.height,
|
|
5920
|
+
deviceScaleFactor: shot2.deviceScaleFactor,
|
|
5921
|
+
...shot2.boxes ? { boxes: shot2.boxes } : {}
|
|
5922
|
+
});
|
|
5923
|
+
if (store) {
|
|
5924
|
+
const token = await store.createUploadToken({
|
|
5925
|
+
...key ? { key } : { filename: "screenshot.png" },
|
|
5926
|
+
contentType: "image/png"
|
|
5927
|
+
});
|
|
5928
|
+
const shot2 = await this.run({
|
|
5929
|
+
...asked,
|
|
5930
|
+
upload: { url: token.upload.url, fields: token.upload.fields }
|
|
5931
|
+
});
|
|
5932
|
+
return { ...await store.head(token.key), ...measured(shot2) };
|
|
5933
|
+
}
|
|
5934
|
+
const shot = await this.run(asked);
|
|
5935
|
+
return {
|
|
5936
|
+
data: Buffer.from(shot.data ?? "", "base64"),
|
|
5937
|
+
contentType: "image/png",
|
|
5938
|
+
...measured(shot)
|
|
5939
|
+
};
|
|
5940
|
+
}
|
|
5941
|
+
/**
|
|
5942
|
+
* Run a function in the page and get back what it returns (as JSON). The
|
|
5943
|
+
* function is sent as source, so it can't use variables from your code:
|
|
5944
|
+
* pass what it needs as `arg`.
|
|
5945
|
+
*
|
|
5946
|
+
* ```ts
|
|
5947
|
+
* const prices = await session.evaluate(
|
|
5948
|
+
* (selector) => [...document.querySelectorAll(selector)].map((el) => el.textContent),
|
|
5949
|
+
* '.price',
|
|
5950
|
+
* );
|
|
5951
|
+
* ```
|
|
5952
|
+
*/
|
|
5953
|
+
evaluate(fn, arg) {
|
|
5954
|
+
return this.run({
|
|
5955
|
+
op: "evaluate",
|
|
5956
|
+
expression: `(${fn.toString()})(${arg === void 0 ? "" : JSON.stringify(arg)})`
|
|
5957
|
+
});
|
|
5958
|
+
}
|
|
5959
|
+
/**
|
|
5960
|
+
* A tool for `mindstudio.runTask()` that drives this session, so a model can
|
|
5961
|
+
* browse. It takes a list of steps (open a URL, read the page, click, type,
|
|
5962
|
+
* choose an option, press a key, scroll, wait for text), runs them in order,
|
|
5963
|
+
* stops at the first that fails, and answers with each step's outcome and
|
|
5964
|
+
* the page after the last one. A model plans "fill the form and submit it"
|
|
5965
|
+
* as one call rather than one turn a field. Only the latest page stays in
|
|
5966
|
+
* the task's transcript (`supersedes`), so a long walk doesn't fill it.
|
|
5967
|
+
*
|
|
5968
|
+
* With `screenshots`, a second tool, `browser_screenshot`, lets the model
|
|
5969
|
+
* see the page: each shot goes into that store and the model reads a
|
|
5970
|
+
* description of it. Those answers all stay in the transcript, since each
|
|
5971
|
+
* names a kept file.
|
|
5972
|
+
*/
|
|
5973
|
+
tools(options = {}) {
|
|
5974
|
+
const secrets = options.secrets ?? {};
|
|
5975
|
+
const secretNames = Object.keys(secrets);
|
|
5976
|
+
const screenshots = options.screenshots;
|
|
5977
|
+
const tools = [
|
|
5978
|
+
{
|
|
5979
|
+
name: "browser",
|
|
5980
|
+
description: `Drive the browser. Run one or more steps in order: snapshot (read the page), navigate (open a URL), click, type, select (a dropdown option), press (a key), scroll, and wait (for text to appear). Start with a snapshot to see the page: everything you can act on has a ref ("e12"), good until the next page. Name an element by its ref, or by text, role, label or a CSS selector. Steps stop at the first that fails, and the answer says why. The answer has each step's outcome and the page after the last step: its URL, title and outline. Earlier pages leave the conversation when a newer one arrives, so plan from the latest. Batch the steps you can predict, such as filling a form and submitting it, into one call.` + (secretNames.length > 0 ? ` To type a secret you never see, give type a secret instead of text: ${secretNames.join(", ")}.` : "") + (screenshots ? " To see how the page looks, which the outline can only partly say, use browser_screenshot." : ""),
|
|
5981
|
+
inputSchema: {
|
|
5982
|
+
type: "object",
|
|
5983
|
+
properties: {
|
|
5984
|
+
steps: {
|
|
5985
|
+
type: "array",
|
|
5986
|
+
description: `One to ${MAX_TOOL_STEPS} steps, run in order.`,
|
|
5987
|
+
items: {
|
|
5988
|
+
type: "object",
|
|
5989
|
+
properties: {
|
|
5990
|
+
command: {
|
|
5991
|
+
type: "string",
|
|
5992
|
+
enum: [
|
|
5993
|
+
"snapshot",
|
|
5994
|
+
"navigate",
|
|
5995
|
+
"click",
|
|
5996
|
+
"type",
|
|
5997
|
+
"select",
|
|
5998
|
+
"press",
|
|
5999
|
+
"scroll",
|
|
6000
|
+
"wait"
|
|
6001
|
+
]
|
|
6002
|
+
},
|
|
6003
|
+
ref: {
|
|
6004
|
+
type: "string",
|
|
6005
|
+
description: "An element, by its ref from the latest page (most reliable)."
|
|
6006
|
+
},
|
|
6007
|
+
text: {
|
|
6008
|
+
type: "string",
|
|
6009
|
+
description: "For click and wait: the element's name or visible text, or the text to wait for. For type: the text to type."
|
|
6010
|
+
},
|
|
6011
|
+
role: {
|
|
6012
|
+
type: "string",
|
|
6013
|
+
description: "With text, only an element of this role (button, link, textbox\u2026)."
|
|
6014
|
+
},
|
|
6015
|
+
label: {
|
|
6016
|
+
type: "string",
|
|
6017
|
+
description: "A form field, by its label."
|
|
6018
|
+
},
|
|
6019
|
+
selector: {
|
|
6020
|
+
type: "string",
|
|
6021
|
+
description: "A CSS selector, the last resort."
|
|
6022
|
+
},
|
|
6023
|
+
url: {
|
|
6024
|
+
type: "string",
|
|
6025
|
+
description: "For navigate: an http or https URL."
|
|
6026
|
+
},
|
|
6027
|
+
...secretNames.length > 0 ? {
|
|
6028
|
+
secret: {
|
|
6029
|
+
type: "string",
|
|
6030
|
+
enum: secretNames,
|
|
6031
|
+
description: "For type: a secret to type instead of text."
|
|
6032
|
+
}
|
|
6033
|
+
} : {},
|
|
6034
|
+
clear: {
|
|
6035
|
+
type: "boolean",
|
|
6036
|
+
description: "For type: replace what the field holds."
|
|
6037
|
+
},
|
|
6038
|
+
submit: {
|
|
6039
|
+
type: "boolean",
|
|
6040
|
+
description: "For type: press Enter after."
|
|
6041
|
+
},
|
|
6042
|
+
option: {
|
|
6043
|
+
type: "string",
|
|
6044
|
+
description: "For select: the option's text or value."
|
|
6045
|
+
},
|
|
6046
|
+
key: {
|
|
6047
|
+
type: "string",
|
|
6048
|
+
description: "For press: Enter, Tab, Escape, Backspace, Delete, an arrow, PageUp, PageDown, Home, End, or one character."
|
|
6049
|
+
},
|
|
6050
|
+
by: {
|
|
6051
|
+
type: "number",
|
|
6052
|
+
description: "For scroll: pixels down (negative for up). With an element instead, scrolls it into view; with neither, most of a screen."
|
|
6053
|
+
},
|
|
6054
|
+
timeoutSeconds: {
|
|
6055
|
+
type: "number",
|
|
6056
|
+
description: "For wait: how long to wait (15 by default)."
|
|
6057
|
+
}
|
|
6058
|
+
},
|
|
6059
|
+
required: ["command"]
|
|
6060
|
+
}
|
|
6061
|
+
}
|
|
6062
|
+
},
|
|
6063
|
+
required: ["steps"]
|
|
6064
|
+
},
|
|
6065
|
+
supersedes: "browser-page",
|
|
6066
|
+
execute: async (input) => this.runSteps(input.steps, secrets)
|
|
6067
|
+
}
|
|
6068
|
+
];
|
|
6069
|
+
if (screenshots) {
|
|
6070
|
+
tools.push({
|
|
6071
|
+
name: "browser_screenshot",
|
|
6072
|
+
description: "See the page as a person would. Takes a screenshot of the browser, keeps it, and answers with a description of what it shows, and the answer to your question if you ask one. Use it when the page's outline can't tell you what you need: how the page looks, a chart or an image, whether something is visibly wrong. The answer's key names the kept screenshot.",
|
|
6073
|
+
inputSchema: {
|
|
6074
|
+
type: "object",
|
|
6075
|
+
properties: {
|
|
6076
|
+
question: {
|
|
6077
|
+
type: "string",
|
|
6078
|
+
description: "What to look for or answer from the page. Leave it out for a description of the whole page."
|
|
6079
|
+
},
|
|
6080
|
+
fullPage: {
|
|
6081
|
+
type: "boolean",
|
|
6082
|
+
description: "The whole page, top to bottom, not just what is on screen."
|
|
6083
|
+
},
|
|
6084
|
+
cursor: {
|
|
6085
|
+
type: "boolean",
|
|
6086
|
+
description: "Keep the cursor in the shot, where the last action landed."
|
|
6087
|
+
}
|
|
6088
|
+
}
|
|
6089
|
+
},
|
|
6090
|
+
execute: async (input) => this.describeScreen(screenshots, input)
|
|
6091
|
+
});
|
|
6092
|
+
}
|
|
6093
|
+
return tools;
|
|
6094
|
+
}
|
|
6095
|
+
/**
|
|
6096
|
+
* The screenshot tool: the shot into the store, then a vision model's
|
|
6097
|
+
* description of it. A description that fails still answers with the kept
|
|
6098
|
+
* shot's key, so the model can name it.
|
|
6099
|
+
*/
|
|
6100
|
+
async describeScreen(store, input) {
|
|
6101
|
+
const question = typeof input.question === "string" && input.question.trim() ? input.question.trim() : void 0;
|
|
6102
|
+
const shot = await this.screenshot({
|
|
6103
|
+
store,
|
|
6104
|
+
fullPage: input.fullPage === true,
|
|
6105
|
+
cursor: input.cursor === true
|
|
6106
|
+
});
|
|
6107
|
+
try {
|
|
6108
|
+
const { analysis } = await this.steps("analyzeImage", {
|
|
6109
|
+
imageUrl: shot.url,
|
|
6110
|
+
prompt: describeScreenPrompt(question)
|
|
6111
|
+
});
|
|
6112
|
+
return { key: shot.key, description: analysis };
|
|
6113
|
+
} catch (err) {
|
|
6114
|
+
return {
|
|
6115
|
+
key: shot.key,
|
|
6116
|
+
error: `The screenshot was kept, but describing it failed: ${err instanceof Error ? err.message : "unknown error"}`
|
|
6117
|
+
};
|
|
6118
|
+
}
|
|
6119
|
+
}
|
|
6120
|
+
/** The tool's steps, in order, stopping at the first that fails. */
|
|
6121
|
+
async runSteps(raw, secrets) {
|
|
6122
|
+
const steps = Array.isArray(raw) ? raw : [];
|
|
6123
|
+
if (steps.length === 0 || steps.length > MAX_TOOL_STEPS) {
|
|
6124
|
+
throw new MindStudioError(
|
|
6125
|
+
`Give between 1 and ${MAX_TOOL_STEPS} steps.`,
|
|
6126
|
+
"invalid_steps",
|
|
6127
|
+
400
|
|
6128
|
+
);
|
|
6129
|
+
}
|
|
6130
|
+
const outcomes = [];
|
|
6131
|
+
let last;
|
|
6132
|
+
let failure;
|
|
6133
|
+
for (const step of steps) {
|
|
6134
|
+
const command = typeof step.command === "string" ? step.command : "";
|
|
6135
|
+
try {
|
|
6136
|
+
const answer = await this.runStep(command, step, secrets);
|
|
6137
|
+
last = answer;
|
|
6138
|
+
outcomes.push({
|
|
6139
|
+
command,
|
|
6140
|
+
ok: true,
|
|
6141
|
+
..."matched" in answer ? { matched: answer.matched } : {}
|
|
6142
|
+
});
|
|
6143
|
+
} catch (err) {
|
|
6144
|
+
failure = err instanceof Error ? err : new Error(String(err));
|
|
6145
|
+
outcomes.push({ command, ok: false, error: failure.message });
|
|
6146
|
+
break;
|
|
6147
|
+
}
|
|
6148
|
+
}
|
|
6149
|
+
if (failure) {
|
|
6150
|
+
const stepError = failure;
|
|
6151
|
+
last = await this.snapshot().catch(() => {
|
|
6152
|
+
throw stepError;
|
|
6153
|
+
});
|
|
6154
|
+
}
|
|
6155
|
+
return {
|
|
6156
|
+
steps: outcomes,
|
|
6157
|
+
...outcomes.length < steps.length ? { notRun: steps.length - outcomes.length } : {},
|
|
6158
|
+
page: {
|
|
6159
|
+
url: last.url,
|
|
6160
|
+
title: last.title,
|
|
6161
|
+
snapshot: last.snapshot,
|
|
6162
|
+
...last.dialogs ? { dialogs: last.dialogs } : {}
|
|
6163
|
+
}
|
|
6164
|
+
};
|
|
6165
|
+
}
|
|
6166
|
+
runStep(command, step, secrets) {
|
|
6167
|
+
const str = (v) => typeof v === "string" ? v : void 0;
|
|
6168
|
+
const target = () => {
|
|
6169
|
+
if (str(step.ref)) {
|
|
6170
|
+
return str(step.ref);
|
|
6171
|
+
}
|
|
6172
|
+
const described = {
|
|
6173
|
+
...command !== "type" && str(step.text) ? { text: str(step.text) } : {},
|
|
6174
|
+
...str(step.role) ? { role: str(step.role) } : {},
|
|
6175
|
+
...str(step.label) ? { label: str(step.label) } : {},
|
|
6176
|
+
...str(step.selector) ? { selector: str(step.selector) } : {}
|
|
6177
|
+
};
|
|
6178
|
+
if (Object.keys(described).length === 0) {
|
|
6179
|
+
throw new MindStudioError(
|
|
6180
|
+
`${command} needs an element: a ref, text, role, label or selector.`,
|
|
6181
|
+
"invalid_steps",
|
|
6182
|
+
400
|
|
6183
|
+
);
|
|
6184
|
+
}
|
|
6185
|
+
return described;
|
|
6186
|
+
};
|
|
6187
|
+
switch (command) {
|
|
6188
|
+
case "snapshot":
|
|
6189
|
+
return this.snapshot();
|
|
6190
|
+
case "navigate":
|
|
6191
|
+
return this.goto(str(step.url) ?? "");
|
|
6192
|
+
case "click":
|
|
6193
|
+
return this.click(target());
|
|
6194
|
+
case "type": {
|
|
6195
|
+
const secret = str(step.secret);
|
|
6196
|
+
if (secret !== void 0 && !(secret in secrets)) {
|
|
6197
|
+
throw new MindStudioError(
|
|
6198
|
+
`No secret named "${secret}".`,
|
|
6199
|
+
"unknown_secret",
|
|
6200
|
+
400
|
|
6201
|
+
);
|
|
6202
|
+
}
|
|
6203
|
+
return this.type(
|
|
6204
|
+
target(),
|
|
6205
|
+
secret !== void 0 ? secrets[secret] : str(step.text) ?? "",
|
|
6206
|
+
{
|
|
6207
|
+
clear: step.clear === true || secret !== void 0,
|
|
6208
|
+
submit: step.submit === true
|
|
6209
|
+
}
|
|
6210
|
+
);
|
|
6211
|
+
}
|
|
6212
|
+
case "select":
|
|
6213
|
+
return this.select(target(), str(step.option) ?? "");
|
|
6214
|
+
case "press":
|
|
6215
|
+
return this.press(str(step.key) ?? "");
|
|
6216
|
+
case "scroll": {
|
|
6217
|
+
const hasTarget = str(step.ref) ?? str(step.label) ?? str(step.selector) ?? str(step.text);
|
|
6218
|
+
return this.scroll({
|
|
6219
|
+
...hasTarget ? { to: target() } : {},
|
|
6220
|
+
...typeof step.by === "number" ? { by: step.by } : {}
|
|
6221
|
+
});
|
|
6222
|
+
}
|
|
6223
|
+
case "wait":
|
|
6224
|
+
return this.waitFor(str(step.text) ?? "", {
|
|
6225
|
+
...typeof step.timeoutSeconds === "number" ? { timeoutSeconds: step.timeoutSeconds } : {}
|
|
6226
|
+
});
|
|
6227
|
+
default:
|
|
6228
|
+
throw new MindStudioError(
|
|
6229
|
+
`Unknown command "${command}".`,
|
|
6230
|
+
"invalid_steps",
|
|
6231
|
+
400
|
|
6232
|
+
);
|
|
6233
|
+
}
|
|
6234
|
+
}
|
|
6235
|
+
/**
|
|
6236
|
+
* A grant to watch this session live, for your frontend: return it from a
|
|
6237
|
+
* method and hand its `token` to `browser.view()` in
|
|
6238
|
+
* `@mindstudio-ai/interface`. Whoever holds it sees the page until it
|
|
6239
|
+
* expires, so check your own access first, as for `events.grant()`.
|
|
6240
|
+
* `ttlSeconds` is clamped to 60 through 3600 (900 by default); the frontend
|
|
6241
|
+
* asks again through your method when it runs out.
|
|
6242
|
+
*
|
|
6243
|
+
* ```ts
|
|
6244
|
+
* export async function watchRun({ runId }: { runId: string }) {
|
|
6245
|
+
* auth.requireRole('admin');
|
|
6246
|
+
* const run = await Runs.get(runId);
|
|
6247
|
+
* const session = await browser.session(run.browserSessionId);
|
|
6248
|
+
* return session.grantView();
|
|
6249
|
+
* }
|
|
6250
|
+
* ```
|
|
6251
|
+
*/
|
|
6252
|
+
grantView(options = {}) {
|
|
6253
|
+
return this.call("view", { sessionId: this.id, ...options });
|
|
6254
|
+
}
|
|
6255
|
+
/** Close the session and its browser. */
|
|
6256
|
+
async close() {
|
|
6257
|
+
await this.call("close", { sessionId: this.id });
|
|
6258
|
+
}
|
|
6259
|
+
};
|
|
6260
|
+
}
|
|
6261
|
+
});
|
|
6262
|
+
|
|
5743
6263
|
// src/task/index.ts
|
|
5744
6264
|
function resolveStepType(name) {
|
|
5745
6265
|
const meta = stepMetadata[name];
|
|
@@ -5947,7 +6467,7 @@ var init_schema = __esm({
|
|
|
5947
6467
|
});
|
|
5948
6468
|
|
|
5949
6469
|
// src/task/local.ts
|
|
5950
|
-
import { randomUUID as
|
|
6470
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
5951
6471
|
function isPlainObject(value) {
|
|
5952
6472
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
5953
6473
|
return false;
|
|
@@ -6166,9 +6686,14 @@ ${structuredOutputExample}
|
|
|
6166
6686
|
const toolKinds = /* @__PURE__ */ new Map();
|
|
6167
6687
|
const toolDefaults = /* @__PURE__ */ new Map();
|
|
6168
6688
|
const functionTools = /* @__PURE__ */ new Map();
|
|
6689
|
+
const supersedeKeys = /* @__PURE__ */ new Map();
|
|
6690
|
+
const latestByKey = /* @__PURE__ */ new Map();
|
|
6169
6691
|
for (const t of options.tools) {
|
|
6170
6692
|
if (typeof t === "object" && "execute" in t) {
|
|
6171
6693
|
functionTools.set(t.name, t.execute);
|
|
6694
|
+
if (t.supersedes) {
|
|
6695
|
+
supersedeKeys.set(t.name, t.supersedes);
|
|
6696
|
+
}
|
|
6172
6697
|
}
|
|
6173
6698
|
}
|
|
6174
6699
|
for (const t of wireTools) {
|
|
@@ -6191,7 +6716,7 @@ ${structuredOutputExample}
|
|
|
6191
6716
|
MAX_TURNS_LIMIT
|
|
6192
6717
|
);
|
|
6193
6718
|
const maxToolOutputChars = options.maxToolOutputChars ?? MAX_TOOL_OUTPUT_CHARS;
|
|
6194
|
-
const traceId =
|
|
6719
|
+
const traceId = randomUUID3();
|
|
6195
6720
|
const messages = [
|
|
6196
6721
|
{ role: "user", content: JSON.stringify(options.input) }
|
|
6197
6722
|
];
|
|
@@ -6370,6 +6895,17 @@ ${structuredOutputExample}
|
|
|
6370
6895
|
toolCallId: toolCall.id,
|
|
6371
6896
|
...isError && { isToolError: true }
|
|
6372
6897
|
});
|
|
6898
|
+
const key = supersedeKeys.get(toolCall.name);
|
|
6899
|
+
if (key !== void 0 && !isError) {
|
|
6900
|
+
const earlier = latestByKey.get(key);
|
|
6901
|
+
if (earlier !== void 0) {
|
|
6902
|
+
messages[earlier] = {
|
|
6903
|
+
...messages[earlier],
|
|
6904
|
+
content: SUPERSEDED_RESULT
|
|
6905
|
+
};
|
|
6906
|
+
}
|
|
6907
|
+
latestByKey.set(key, messages.length - 1);
|
|
6908
|
+
}
|
|
6373
6909
|
onEvent?.({
|
|
6374
6910
|
type: "tool_call_result",
|
|
6375
6911
|
id: toolCall.id,
|
|
@@ -6487,7 +7023,7 @@ ${formatValidationErrors(errors)}`
|
|
|
6487
7023
|
)
|
|
6488
7024
|
);
|
|
6489
7025
|
}
|
|
6490
|
-
var DEFAULT_MAX_TURNS, MAX_TURNS_LIMIT, MAX_SCHEMA_REPAIR_ATTEMPTS, MAX_TOOL_OUTPUT_CHARS, MAX_TURN_ATTEMPTS, INITIAL_BACKOFF_MS, STALL_TIMEOUT_MS, UNSAFE_KEYS, TurnError;
|
|
7026
|
+
var DEFAULT_MAX_TURNS, MAX_TURNS_LIMIT, MAX_SCHEMA_REPAIR_ATTEMPTS, MAX_TOOL_OUTPUT_CHARS, SUPERSEDED_RESULT, MAX_TURN_ATTEMPTS, INITIAL_BACKOFF_MS, STALL_TIMEOUT_MS, UNSAFE_KEYS, TurnError;
|
|
6491
7027
|
var init_local = __esm({
|
|
6492
7028
|
async "src/task/local.ts"() {
|
|
6493
7029
|
"use strict";
|
|
@@ -6498,6 +7034,9 @@ var init_local = __esm({
|
|
|
6498
7034
|
MAX_TURNS_LIMIT = 100;
|
|
6499
7035
|
MAX_SCHEMA_REPAIR_ATTEMPTS = 3;
|
|
6500
7036
|
MAX_TOOL_OUTPUT_CHARS = 5e4;
|
|
7037
|
+
SUPERSEDED_RESULT = JSON.stringify({
|
|
7038
|
+
superseded: "An earlier result, replaced by a newer one from the same tool further on. Work from the latest."
|
|
7039
|
+
});
|
|
6501
7040
|
MAX_TURN_ATTEMPTS = 5;
|
|
6502
7041
|
INITIAL_BACKOFF_MS = 1e3;
|
|
6503
7042
|
STALL_TIMEOUT_MS = 3e5;
|
|
@@ -7065,6 +7604,7 @@ var init_client = __esm({
|
|
|
7065
7604
|
init_analytics();
|
|
7066
7605
|
init_email();
|
|
7067
7606
|
init_events();
|
|
7607
|
+
init_browser();
|
|
7068
7608
|
await init_local();
|
|
7069
7609
|
init_steps();
|
|
7070
7610
|
init_metadata();
|
|
@@ -7121,6 +7661,7 @@ var init_client = __esm({
|
|
|
7121
7661
|
_analytics;
|
|
7122
7662
|
_email;
|
|
7123
7663
|
_events;
|
|
7664
|
+
_browser;
|
|
7124
7665
|
/** @internal Auth type — 'internal' for CALLBACK_TOKEN (managed mode), 'apiKey' otherwise. */
|
|
7125
7666
|
_authType;
|
|
7126
7667
|
/** @internal Usage source sent on step executions (from MINDSTUDIO_REQUEST_SOURCE).
|
|
@@ -8187,6 +8728,17 @@ var init_client = __esm({
|
|
|
8187
8728
|
get events() {
|
|
8188
8729
|
return this._events ??= createEvents(this._eventsRequest.bind(this));
|
|
8189
8730
|
}
|
|
8731
|
+
/**
|
|
8732
|
+
* The `browser` namespace — a real Chrome your app drives. `open()` starts
|
|
8733
|
+
* a session; every action answers with the page afterward. See
|
|
8734
|
+
* {@link Browser}.
|
|
8735
|
+
*/
|
|
8736
|
+
get browser() {
|
|
8737
|
+
return this._browser ??= createBrowser(
|
|
8738
|
+
this._browserRequest.bind(this),
|
|
8739
|
+
(stepType, step) => this.executeStep(stepType, step)
|
|
8740
|
+
);
|
|
8741
|
+
}
|
|
8190
8742
|
/**
|
|
8191
8743
|
* Outbound email: sending, the per-recipient delivery log, blast stats, and
|
|
8192
8744
|
* this app's unsubscribe list.
|
|
@@ -8384,6 +8936,17 @@ var init_client = __esm({
|
|
|
8384
8936
|
fallbackCode: "events_error"
|
|
8385
8937
|
});
|
|
8386
8938
|
}
|
|
8939
|
+
/**
|
|
8940
|
+
* @internal Transport for the `browser` namespace —
|
|
8941
|
+
* POST /_internal/v2/browser/<op> with the raw hook token. No 429 retry:
|
|
8942
|
+
* an action repeated is a second click.
|
|
8943
|
+
*/
|
|
8944
|
+
async _browserRequest(op, body) {
|
|
8945
|
+
return this._brokeredRequest("browser", op, body, {
|
|
8946
|
+
fallbackMessage: "Browser call failed",
|
|
8947
|
+
fallbackCode: "browser_error"
|
|
8948
|
+
});
|
|
8949
|
+
}
|
|
8387
8950
|
async _analyticsRequest(op, body) {
|
|
8388
8951
|
const call = () => this._brokeredRequest("analytics", op, body, {
|
|
8389
8952
|
fallbackMessage: "Analytics read failed",
|
|
@@ -9472,7 +10035,7 @@ async function startMcpServer(options) {
|
|
|
9472
10035
|
capabilities: { tools: {} },
|
|
9473
10036
|
serverInfo: {
|
|
9474
10037
|
name: "mindstudio-agent",
|
|
9475
|
-
version: "0.1.
|
|
10038
|
+
version: "0.1.173"
|
|
9476
10039
|
},
|
|
9477
10040
|
instructions: 'Welcome to MindStudio \u2014 a platform with 200+ AI models, 850+ third-party integrations, and pre-built agents.\n\nGetting started:\n1. Call `ask` with any question about the SDK \u2014 it knows every action, model, and connector and returns working code with real model IDs and config options. Examples: ask("generate an image with FLUX"), ask("what models support vision?"), ask("how do I send a Slack message?").\n2. Call `changeName` to set your display name \u2014 use your name or whatever your user calls you. This is how you\'ll appear in MindStudio request logs.\n3. If you have a profile picture, pass its public image URL to `changeProfilePicture`.\n4. For manual browsing, call `listActions` to discover all available actions.\n\nThen use the tools to generate text, images, video, audio, search the web, work with data sources, run agents, and more.\n\nImportant:\n- AI-powered actions (text generation, image generation, video, audio, etc.) cost money. Before running these, call `estimateActionCost` and confirm with the user before proceeding \u2014 unless they\'ve explicitly told you to go ahead.\n- Not all agents from `listAgents` are configured for API use. Do not try to run an agent just because it appears in the list \u2014 it will likely fail. Only run agents the user specifically asks you to run.'
|
|
9478
10041
|
});
|
|
@@ -10414,7 +10977,7 @@ function isNewerVersion(current, latest) {
|
|
|
10414
10977
|
return false;
|
|
10415
10978
|
}
|
|
10416
10979
|
async function checkForUpdate() {
|
|
10417
|
-
const currentVersion = "0.1.
|
|
10980
|
+
const currentVersion = "0.1.173";
|
|
10418
10981
|
if (!currentVersion || process.env.MINDSTUDIO_NO_UPDATE_CHECK) {
|
|
10419
10982
|
return null;
|
|
10420
10983
|
}
|
|
@@ -10449,7 +11012,7 @@ async function checkForUpdate() {
|
|
|
10449
11012
|
}
|
|
10450
11013
|
}
|
|
10451
11014
|
function printUpdateNotice(latestVersion) {
|
|
10452
|
-
const currentVersion = "0.1.
|
|
11015
|
+
const currentVersion = "0.1.173";
|
|
10453
11016
|
process.stderr.write(
|
|
10454
11017
|
`
|
|
10455
11018
|
${ansi2.cyanBright("Update available")} ${ansi2.gray(currentVersion + " \u2192")} ${ansi2.cyanBold(latestVersion)}
|
|
@@ -10462,7 +11025,7 @@ function isStandaloneBinary() {
|
|
|
10462
11025
|
return !argv1.includes("node_modules");
|
|
10463
11026
|
}
|
|
10464
11027
|
async function cmdUpdate() {
|
|
10465
|
-
const currentVersion = "0.1.
|
|
11028
|
+
const currentVersion = "0.1.173";
|
|
10466
11029
|
process.stderr.write(
|
|
10467
11030
|
` ${ansi2.gray("Current version:")} ${currentVersion}
|
|
10468
11031
|
`
|
|
@@ -10583,7 +11146,7 @@ async function cmdLogin(options) {
|
|
|
10583
11146
|
process.stderr.write("\n");
|
|
10584
11147
|
printLogo();
|
|
10585
11148
|
process.stderr.write("\n");
|
|
10586
|
-
const ver = "0.1.
|
|
11149
|
+
const ver = "0.1.173";
|
|
10587
11150
|
process.stderr.write(
|
|
10588
11151
|
` ${ansi2.bold("MindStudio Agent")} ${ver ? " " + ansi2.gray("v" + ver) : ""}
|
|
10589
11152
|
`
|
|
@@ -10895,7 +11458,7 @@ async function main() {
|
|
|
10895
11458
|
const updatePromise = command !== "mcp" && command !== "login" && command !== "update" ? checkForUpdate() : Promise.resolve(null);
|
|
10896
11459
|
try {
|
|
10897
11460
|
if (command === "version" || command === "-v") {
|
|
10898
|
-
process.stdout.write("0.1.
|
|
11461
|
+
process.stdout.write("0.1.173\n");
|
|
10899
11462
|
return;
|
|
10900
11463
|
}
|
|
10901
11464
|
if (command === "login") {
|