@mindstudio-ai/agent 0.1.170 → 0.1.172
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/cli.js +587 -21
- package/dist/index.d.ts +398 -14
- package/dist/index.js +580 -13
- package/dist/index.js.map +1 -1
- package/dist/postinstall.js +587 -21
- package/llms.txt +24 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -573,7 +573,8 @@ function serializeParam(val) {
|
|
|
573
573
|
}
|
|
574
574
|
function serializeColumnParam(val, columnName, columns) {
|
|
575
575
|
const col = columnsByName(columns).get(columnName);
|
|
576
|
-
|
|
576
|
+
const isUserColumn = col?.type === "user" || columnName === "last_updated_by";
|
|
577
|
+
if (isUserColumn && typeof val === "string") {
|
|
577
578
|
return val.startsWith(USER_PREFIX) ? val : `${USER_PREFIX}${val}`;
|
|
578
579
|
}
|
|
579
580
|
return serializeParam(val);
|
|
@@ -771,7 +772,7 @@ function buildExists(table, where, whereParams, negate) {
|
|
|
771
772
|
};
|
|
772
773
|
}
|
|
773
774
|
function buildInsert(table, data, columns) {
|
|
774
|
-
const filtered =
|
|
775
|
+
const filtered = stripPlatformColumns(data);
|
|
775
776
|
const keys = Object.keys(filtered);
|
|
776
777
|
const placeholders = keys.map(() => "?").join(", ");
|
|
777
778
|
const params = keys.map((k) => serializeColumnParam(filtered[k], k, columns));
|
|
@@ -781,7 +782,7 @@ function buildInsert(table, data, columns) {
|
|
|
781
782
|
};
|
|
782
783
|
}
|
|
783
784
|
function buildUpdate(table, id, data, columns) {
|
|
784
|
-
const filtered =
|
|
785
|
+
const filtered = stripPlatformColumns(data);
|
|
785
786
|
const keys = Object.keys(filtered);
|
|
786
787
|
const assignments = keys.map((k) => `${k} = ?`).join(", ");
|
|
787
788
|
const params = [
|
|
@@ -795,7 +796,7 @@ function buildUpdate(table, id, data, columns) {
|
|
|
795
796
|
};
|
|
796
797
|
}
|
|
797
798
|
function buildUpsert(table, data, conflictColumns, columns) {
|
|
798
|
-
const filtered =
|
|
799
|
+
const filtered = stripPlatformColumns(data);
|
|
799
800
|
const keys = Object.keys(filtered);
|
|
800
801
|
const placeholders = keys.map(() => "?").join(", ");
|
|
801
802
|
const params = keys.map((k) => serializeColumnParam(filtered[k], k, columns));
|
|
@@ -811,19 +812,16 @@ function buildDelete(table, where, whereParams) {
|
|
|
811
812
|
}
|
|
812
813
|
return { sql, params: whereParams?.length ? whereParams : void 0 };
|
|
813
814
|
}
|
|
814
|
-
var
|
|
815
|
+
var PLATFORM_COLUMNS = /* @__PURE__ */ new Set([
|
|
815
816
|
"id",
|
|
816
|
-
"created_at",
|
|
817
817
|
"createdAt",
|
|
818
|
-
"updated_at",
|
|
819
818
|
"updatedAt",
|
|
820
|
-
"last_updated_by",
|
|
821
819
|
"lastUpdatedBy"
|
|
822
820
|
]);
|
|
823
|
-
function
|
|
821
|
+
function stripPlatformColumns(data) {
|
|
824
822
|
const result = {};
|
|
825
823
|
for (const [key, value] of Object.entries(data)) {
|
|
826
|
-
if (!
|
|
824
|
+
if (!PLATFORM_COLUMNS.has(key)) {
|
|
827
825
|
result[key] = value;
|
|
828
826
|
}
|
|
829
827
|
}
|
|
@@ -2575,6 +2573,9 @@ var Table = class {
|
|
|
2575
2573
|
/**
|
|
2576
2574
|
* Update a row by ID. Only the provided fields are changed.
|
|
2577
2575
|
* Returns the updated row via `UPDATE ... RETURNING *`.
|
|
2576
|
+
*
|
|
2577
|
+
* The platform advances `updated_at` and stamps `last_updated_by` with the
|
|
2578
|
+
* signed-in user unless the data changes them itself.
|
|
2578
2579
|
*/
|
|
2579
2580
|
update(id, data) {
|
|
2580
2581
|
const build = () => {
|
|
@@ -2670,6 +2671,8 @@ var Table = class {
|
|
|
2670
2671
|
* @param conflictKey - Column name(s) that form the unique constraint.
|
|
2671
2672
|
* Pass a single string for single-column unique, or an array for compound.
|
|
2672
2673
|
* @param data - Row data to insert (or update on conflict). Defaults apply.
|
|
2674
|
+
* A `created_at`, `updated_at` or `last_updated_by` it gives is written
|
|
2675
|
+
* either way; whichever it leaves out the platform fills in.
|
|
2673
2676
|
*/
|
|
2674
2677
|
upsert(conflictKey, data) {
|
|
2675
2678
|
const conflictColumns = Array.isArray(conflictKey) ? conflictKey : [conflictKey];
|
|
@@ -4318,8 +4321,521 @@ function createEvents(call) {
|
|
|
4318
4321
|
};
|
|
4319
4322
|
}
|
|
4320
4323
|
|
|
4321
|
-
// src/
|
|
4324
|
+
// src/browser/index.ts
|
|
4322
4325
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
4326
|
+
var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([502, 503, 504]);
|
|
4327
|
+
var RETRY_BACKOFFS_MS = [1e3, 2e3, 4e3, 8e3, 15e3, 15e3, 15e3];
|
|
4328
|
+
var MAX_TOOL_STEPS = 20;
|
|
4329
|
+
var describeScreenPrompt = (question) => [
|
|
4330
|
+
"This is a screenshot of a web page, taken by an agent working through the site.",
|
|
4331
|
+
question ? `First answer this question about it: ${question}
|
|
4332
|
+
|
|
4333
|
+
Then describe the page briefly: its layout, its content, and what state it is in.` : "Describe what a person looking at it would see, top to bottom: the layout, the content, what state the page is in (a form partly filled, an error, a dialog open), and anything that looks wrong.",
|
|
4334
|
+
"Be concise and concrete, and quote visible text exactly where it matters. Answer in plain Markdown with no preamble."
|
|
4335
|
+
].join("\n\n");
|
|
4336
|
+
var toWireTarget = (target) => typeof target === "string" ? { ref: target } : target;
|
|
4337
|
+
var BrowserSession = class {
|
|
4338
|
+
/** @internal Use `browser.open()`. */
|
|
4339
|
+
constructor(info, call, steps) {
|
|
4340
|
+
this.call = call;
|
|
4341
|
+
this.steps = steps;
|
|
4342
|
+
this.id = info.id;
|
|
4343
|
+
this.viewport = info.viewport;
|
|
4344
|
+
this.expiresAt = info.expiresAt;
|
|
4345
|
+
}
|
|
4346
|
+
call;
|
|
4347
|
+
steps;
|
|
4348
|
+
/** Pass to `browser.session(id)` from a later call to pick this session up again. */
|
|
4349
|
+
id;
|
|
4350
|
+
viewport;
|
|
4351
|
+
/** When the session closes, however busy it is. Milliseconds since epoch. */
|
|
4352
|
+
expiresAt;
|
|
4353
|
+
/** Every action so far, chained: see `run`. */
|
|
4354
|
+
queue = Promise.resolve();
|
|
4355
|
+
/**
|
|
4356
|
+
* One action at a time, in the order they were called. A session has one
|
|
4357
|
+
* page, so actions started together (a `Promise.all`, or a task agent's
|
|
4358
|
+
* tools from one turn, which run concurrently) would otherwise reach it in
|
|
4359
|
+
* whatever order their requests landed: a click on Submit before the field
|
|
4360
|
+
* it submits was typed.
|
|
4361
|
+
*/
|
|
4362
|
+
run(action) {
|
|
4363
|
+
const result = this.queue.then(() => this.send(action));
|
|
4364
|
+
this.queue = result.catch(() => {
|
|
4365
|
+
});
|
|
4366
|
+
return result;
|
|
4367
|
+
}
|
|
4368
|
+
/**
|
|
4369
|
+
* One action's request, asked again under the same id when the answer
|
|
4370
|
+
* didn't get back (a dropped connection, an api pod rolling). The platform
|
|
4371
|
+
* runs an id once, so asking again waits on the action already under way
|
|
4372
|
+
* rather than clicking twice.
|
|
4373
|
+
*/
|
|
4374
|
+
async send(action) {
|
|
4375
|
+
const commandId = randomUUID2();
|
|
4376
|
+
for (let attempt = 0; ; attempt++) {
|
|
4377
|
+
try {
|
|
4378
|
+
const { value } = await this.call("run", {
|
|
4379
|
+
sessionId: this.id,
|
|
4380
|
+
action,
|
|
4381
|
+
commandId
|
|
4382
|
+
});
|
|
4383
|
+
return value;
|
|
4384
|
+
} catch (err) {
|
|
4385
|
+
const cut = !(err instanceof MindStudioError) || RETRYABLE_STATUSES.has(err.status) && err.code !== "browser_unavailable";
|
|
4386
|
+
if (!cut || attempt >= RETRY_BACKOFFS_MS.length) {
|
|
4387
|
+
throw err;
|
|
4388
|
+
}
|
|
4389
|
+
await new Promise((r) => setTimeout(r, RETRY_BACKOFFS_MS[attempt]));
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
4392
|
+
}
|
|
4393
|
+
/** Open a URL and wait for it to load. */
|
|
4394
|
+
goto(url) {
|
|
4395
|
+
return this.run({ op: "goto", url });
|
|
4396
|
+
}
|
|
4397
|
+
/** The page as it is now. */
|
|
4398
|
+
snapshot() {
|
|
4399
|
+
return this.run({ op: "snapshot" });
|
|
4400
|
+
}
|
|
4401
|
+
/** Click an element (tap, on a mobile viewport), and wait for what it started to settle. */
|
|
4402
|
+
click(target) {
|
|
4403
|
+
return this.run({ op: "click", target: toWireTarget(target) });
|
|
4404
|
+
}
|
|
4405
|
+
/**
|
|
4406
|
+
* Click into a field and type. `clear` replaces what's there; `submit`
|
|
4407
|
+
* presses Enter after.
|
|
4408
|
+
*/
|
|
4409
|
+
type(target, text, options = {}) {
|
|
4410
|
+
return this.run({
|
|
4411
|
+
op: "type",
|
|
4412
|
+
target: toWireTarget(target),
|
|
4413
|
+
text,
|
|
4414
|
+
...options
|
|
4415
|
+
});
|
|
4416
|
+
}
|
|
4417
|
+
/** Choose an option of a `<select>`, by its text, value or label. */
|
|
4418
|
+
select(target, option) {
|
|
4419
|
+
return this.run({ op: "select", target: toWireTarget(target), option });
|
|
4420
|
+
}
|
|
4421
|
+
/**
|
|
4422
|
+
* Press a key on whatever has focus: `Enter`, `Tab`, `Escape`, `Backspace`,
|
|
4423
|
+
* `Delete`, the arrows, `PageUp`, `PageDown`, `Home`, `End`, `' '`, or any
|
|
4424
|
+
* single character.
|
|
4425
|
+
*/
|
|
4426
|
+
press(key) {
|
|
4427
|
+
return this.run({ op: "press", key });
|
|
4428
|
+
}
|
|
4429
|
+
/** Scroll an element into view, or the page by `by` pixels (most of a screen, by default). */
|
|
4430
|
+
scroll(options = {}) {
|
|
4431
|
+
return this.run({
|
|
4432
|
+
op: "scroll",
|
|
4433
|
+
...options.to !== void 0 ? { target: toWireTarget(options.to) } : {},
|
|
4434
|
+
...options.by !== void 0 ? { by: options.by } : {}
|
|
4435
|
+
});
|
|
4436
|
+
}
|
|
4437
|
+
/** Wait for text to appear on the page. Throws once `timeoutSeconds` (15 by default) pass without it. */
|
|
4438
|
+
waitFor(text, options = {}) {
|
|
4439
|
+
return this.run({
|
|
4440
|
+
op: "wait",
|
|
4441
|
+
text,
|
|
4442
|
+
...options.timeoutSeconds !== void 0 ? { timeoutMs: options.timeoutSeconds * 1e3 } : {}
|
|
4443
|
+
});
|
|
4444
|
+
}
|
|
4445
|
+
async screenshot(options = {}) {
|
|
4446
|
+
const { store, key, fullPage, cursor } = options;
|
|
4447
|
+
if (store) {
|
|
4448
|
+
const token = await store.createUploadToken({
|
|
4449
|
+
...key ? { key } : { filename: "screenshot.png" },
|
|
4450
|
+
contentType: "image/png"
|
|
4451
|
+
});
|
|
4452
|
+
const shot2 = await this.run({
|
|
4453
|
+
op: "screenshot",
|
|
4454
|
+
fullPage: !!fullPage,
|
|
4455
|
+
cursor: !!cursor,
|
|
4456
|
+
upload: { url: token.upload.url, fields: token.upload.fields }
|
|
4457
|
+
});
|
|
4458
|
+
return {
|
|
4459
|
+
...await store.head(token.key),
|
|
4460
|
+
width: shot2.width,
|
|
4461
|
+
height: shot2.height,
|
|
4462
|
+
deviceScaleFactor: shot2.deviceScaleFactor
|
|
4463
|
+
};
|
|
4464
|
+
}
|
|
4465
|
+
const shot = await this.run({
|
|
4466
|
+
op: "screenshot",
|
|
4467
|
+
fullPage: !!fullPage,
|
|
4468
|
+
cursor: !!cursor
|
|
4469
|
+
});
|
|
4470
|
+
return {
|
|
4471
|
+
data: Buffer.from(shot.data ?? "", "base64"),
|
|
4472
|
+
contentType: "image/png",
|
|
4473
|
+
width: shot.width,
|
|
4474
|
+
height: shot.height,
|
|
4475
|
+
deviceScaleFactor: shot.deviceScaleFactor
|
|
4476
|
+
};
|
|
4477
|
+
}
|
|
4478
|
+
/**
|
|
4479
|
+
* Run a function in the page and get back what it returns (as JSON). The
|
|
4480
|
+
* function is sent as source, so it can't use variables from your code:
|
|
4481
|
+
* pass what it needs as `arg`.
|
|
4482
|
+
*
|
|
4483
|
+
* ```ts
|
|
4484
|
+
* const prices = await session.evaluate(
|
|
4485
|
+
* (selector) => [...document.querySelectorAll(selector)].map((el) => el.textContent),
|
|
4486
|
+
* '.price',
|
|
4487
|
+
* );
|
|
4488
|
+
* ```
|
|
4489
|
+
*/
|
|
4490
|
+
evaluate(fn, arg) {
|
|
4491
|
+
return this.run({
|
|
4492
|
+
op: "evaluate",
|
|
4493
|
+
expression: `(${fn.toString()})(${arg === void 0 ? "" : JSON.stringify(arg)})`
|
|
4494
|
+
});
|
|
4495
|
+
}
|
|
4496
|
+
/**
|
|
4497
|
+
* A tool for `mindstudio.runTask()` that drives this session, so a model can
|
|
4498
|
+
* browse. It takes a list of steps (open a URL, read the page, click, type,
|
|
4499
|
+
* choose an option, press a key, scroll, wait for text), runs them in order,
|
|
4500
|
+
* stops at the first that fails, and answers with each step's outcome and
|
|
4501
|
+
* the page after the last one. A model plans "fill the form and submit it"
|
|
4502
|
+
* as one call rather than one turn a field. Only the latest page stays in
|
|
4503
|
+
* the task's transcript (`supersedes`), so a long walk doesn't fill it.
|
|
4504
|
+
*
|
|
4505
|
+
* With `screenshots`, a second tool, `browser_screenshot`, lets the model
|
|
4506
|
+
* see the page: each shot goes into that store and the model reads a
|
|
4507
|
+
* description of it. Those answers all stay in the transcript, since each
|
|
4508
|
+
* names a kept file.
|
|
4509
|
+
*/
|
|
4510
|
+
tools(options = {}) {
|
|
4511
|
+
const secrets = options.secrets ?? {};
|
|
4512
|
+
const secretNames = Object.keys(secrets);
|
|
4513
|
+
const screenshots = options.screenshots;
|
|
4514
|
+
const tools = [
|
|
4515
|
+
{
|
|
4516
|
+
name: "browser",
|
|
4517
|
+
description: `Drive the browser. Run one or more steps in order: snapshot (read the page), navigate (open a URL), click, type, select (a dropdown option), press (a key), scroll, and wait (for text to appear). Start with a snapshot to see the page: everything you can act on has a ref ("e12"), good until the next page. Name an element by its ref, or by text, role, label or a CSS selector. Steps stop at the first that fails, and the answer says why. The answer has each step's outcome and the page after the last step: its URL, title and outline. Earlier pages leave the conversation when a newer one arrives, so plan from the latest. Batch the steps you can predict, such as filling a form and submitting it, into one call.` + (secretNames.length > 0 ? ` To type a secret you never see, give type a secret instead of text: ${secretNames.join(", ")}.` : "") + (screenshots ? " To see how the page looks, which the outline can only partly say, use browser_screenshot." : ""),
|
|
4518
|
+
inputSchema: {
|
|
4519
|
+
type: "object",
|
|
4520
|
+
properties: {
|
|
4521
|
+
steps: {
|
|
4522
|
+
type: "array",
|
|
4523
|
+
description: `One to ${MAX_TOOL_STEPS} steps, run in order.`,
|
|
4524
|
+
items: {
|
|
4525
|
+
type: "object",
|
|
4526
|
+
properties: {
|
|
4527
|
+
command: {
|
|
4528
|
+
type: "string",
|
|
4529
|
+
enum: [
|
|
4530
|
+
"snapshot",
|
|
4531
|
+
"navigate",
|
|
4532
|
+
"click",
|
|
4533
|
+
"type",
|
|
4534
|
+
"select",
|
|
4535
|
+
"press",
|
|
4536
|
+
"scroll",
|
|
4537
|
+
"wait"
|
|
4538
|
+
]
|
|
4539
|
+
},
|
|
4540
|
+
ref: {
|
|
4541
|
+
type: "string",
|
|
4542
|
+
description: "An element, by its ref from the latest page (most reliable)."
|
|
4543
|
+
},
|
|
4544
|
+
text: {
|
|
4545
|
+
type: "string",
|
|
4546
|
+
description: "For click and wait: the element's name or visible text, or the text to wait for. For type: the text to type."
|
|
4547
|
+
},
|
|
4548
|
+
role: {
|
|
4549
|
+
type: "string",
|
|
4550
|
+
description: "With text, only an element of this role (button, link, textbox\u2026)."
|
|
4551
|
+
},
|
|
4552
|
+
label: {
|
|
4553
|
+
type: "string",
|
|
4554
|
+
description: "A form field, by its label."
|
|
4555
|
+
},
|
|
4556
|
+
selector: {
|
|
4557
|
+
type: "string",
|
|
4558
|
+
description: "A CSS selector, the last resort."
|
|
4559
|
+
},
|
|
4560
|
+
url: {
|
|
4561
|
+
type: "string",
|
|
4562
|
+
description: "For navigate: an http or https URL."
|
|
4563
|
+
},
|
|
4564
|
+
...secretNames.length > 0 ? {
|
|
4565
|
+
secret: {
|
|
4566
|
+
type: "string",
|
|
4567
|
+
enum: secretNames,
|
|
4568
|
+
description: "For type: a secret to type instead of text."
|
|
4569
|
+
}
|
|
4570
|
+
} : {},
|
|
4571
|
+
clear: {
|
|
4572
|
+
type: "boolean",
|
|
4573
|
+
description: "For type: replace what the field holds."
|
|
4574
|
+
},
|
|
4575
|
+
submit: {
|
|
4576
|
+
type: "boolean",
|
|
4577
|
+
description: "For type: press Enter after."
|
|
4578
|
+
},
|
|
4579
|
+
option: {
|
|
4580
|
+
type: "string",
|
|
4581
|
+
description: "For select: the option's text or value."
|
|
4582
|
+
},
|
|
4583
|
+
key: {
|
|
4584
|
+
type: "string",
|
|
4585
|
+
description: "For press: Enter, Tab, Escape, Backspace, Delete, an arrow, PageUp, PageDown, Home, End, or one character."
|
|
4586
|
+
},
|
|
4587
|
+
by: {
|
|
4588
|
+
type: "number",
|
|
4589
|
+
description: "For scroll: pixels down (negative for up). With an element instead, scrolls it into view; with neither, most of a screen."
|
|
4590
|
+
},
|
|
4591
|
+
timeoutSeconds: {
|
|
4592
|
+
type: "number",
|
|
4593
|
+
description: "For wait: how long to wait (15 by default)."
|
|
4594
|
+
}
|
|
4595
|
+
},
|
|
4596
|
+
required: ["command"]
|
|
4597
|
+
}
|
|
4598
|
+
}
|
|
4599
|
+
},
|
|
4600
|
+
required: ["steps"]
|
|
4601
|
+
},
|
|
4602
|
+
supersedes: "browser-page",
|
|
4603
|
+
execute: async (input) => this.runSteps(input.steps, secrets)
|
|
4604
|
+
}
|
|
4605
|
+
];
|
|
4606
|
+
if (screenshots) {
|
|
4607
|
+
tools.push({
|
|
4608
|
+
name: "browser_screenshot",
|
|
4609
|
+
description: "See the page as a person would. Takes a screenshot of the browser, keeps it, and answers with a description of what it shows, and the answer to your question if you ask one. Use it when the page's outline can't tell you what you need: how the page looks, a chart or an image, whether something is visibly wrong. The answer's key names the kept screenshot.",
|
|
4610
|
+
inputSchema: {
|
|
4611
|
+
type: "object",
|
|
4612
|
+
properties: {
|
|
4613
|
+
question: {
|
|
4614
|
+
type: "string",
|
|
4615
|
+
description: "What to look for or answer from the page. Leave it out for a description of the whole page."
|
|
4616
|
+
},
|
|
4617
|
+
fullPage: {
|
|
4618
|
+
type: "boolean",
|
|
4619
|
+
description: "The whole page, top to bottom, not just what is on screen."
|
|
4620
|
+
},
|
|
4621
|
+
cursor: {
|
|
4622
|
+
type: "boolean",
|
|
4623
|
+
description: "Keep the cursor in the shot, where the last action landed."
|
|
4624
|
+
}
|
|
4625
|
+
}
|
|
4626
|
+
},
|
|
4627
|
+
execute: async (input) => this.describeScreen(screenshots, input)
|
|
4628
|
+
});
|
|
4629
|
+
}
|
|
4630
|
+
return tools;
|
|
4631
|
+
}
|
|
4632
|
+
/**
|
|
4633
|
+
* The screenshot tool: the shot into the store, then a vision model's
|
|
4634
|
+
* description of it. A description that fails still answers with the kept
|
|
4635
|
+
* shot's key, so the model can name it.
|
|
4636
|
+
*/
|
|
4637
|
+
async describeScreen(store, input) {
|
|
4638
|
+
const question = typeof input.question === "string" && input.question.trim() ? input.question.trim() : void 0;
|
|
4639
|
+
const shot = await this.screenshot({
|
|
4640
|
+
store,
|
|
4641
|
+
fullPage: input.fullPage === true,
|
|
4642
|
+
cursor: input.cursor === true
|
|
4643
|
+
});
|
|
4644
|
+
try {
|
|
4645
|
+
const { analysis } = await this.steps("analyzeImage", {
|
|
4646
|
+
imageUrl: shot.url,
|
|
4647
|
+
prompt: describeScreenPrompt(question)
|
|
4648
|
+
});
|
|
4649
|
+
return { key: shot.key, description: analysis };
|
|
4650
|
+
} catch (err) {
|
|
4651
|
+
return {
|
|
4652
|
+
key: shot.key,
|
|
4653
|
+
error: `The screenshot was kept, but describing it failed: ${err instanceof Error ? err.message : "unknown error"}`
|
|
4654
|
+
};
|
|
4655
|
+
}
|
|
4656
|
+
}
|
|
4657
|
+
/** The tool's steps, in order, stopping at the first that fails. */
|
|
4658
|
+
async runSteps(raw, secrets) {
|
|
4659
|
+
const steps = Array.isArray(raw) ? raw : [];
|
|
4660
|
+
if (steps.length === 0 || steps.length > MAX_TOOL_STEPS) {
|
|
4661
|
+
throw new MindStudioError(
|
|
4662
|
+
`Give between 1 and ${MAX_TOOL_STEPS} steps.`,
|
|
4663
|
+
"invalid_steps",
|
|
4664
|
+
400
|
|
4665
|
+
);
|
|
4666
|
+
}
|
|
4667
|
+
const outcomes = [];
|
|
4668
|
+
let last;
|
|
4669
|
+
let failure;
|
|
4670
|
+
for (const step of steps) {
|
|
4671
|
+
const command = typeof step.command === "string" ? step.command : "";
|
|
4672
|
+
try {
|
|
4673
|
+
const answer = await this.runStep(command, step, secrets);
|
|
4674
|
+
last = answer;
|
|
4675
|
+
outcomes.push({
|
|
4676
|
+
command,
|
|
4677
|
+
ok: true,
|
|
4678
|
+
..."matched" in answer ? { matched: answer.matched } : {}
|
|
4679
|
+
});
|
|
4680
|
+
} catch (err) {
|
|
4681
|
+
failure = err instanceof Error ? err : new Error(String(err));
|
|
4682
|
+
outcomes.push({ command, ok: false, error: failure.message });
|
|
4683
|
+
break;
|
|
4684
|
+
}
|
|
4685
|
+
}
|
|
4686
|
+
if (failure) {
|
|
4687
|
+
const stepError = failure;
|
|
4688
|
+
last = await this.snapshot().catch(() => {
|
|
4689
|
+
throw stepError;
|
|
4690
|
+
});
|
|
4691
|
+
}
|
|
4692
|
+
return {
|
|
4693
|
+
steps: outcomes,
|
|
4694
|
+
...outcomes.length < steps.length ? { notRun: steps.length - outcomes.length } : {},
|
|
4695
|
+
page: {
|
|
4696
|
+
url: last.url,
|
|
4697
|
+
title: last.title,
|
|
4698
|
+
snapshot: last.snapshot,
|
|
4699
|
+
...last.dialogs ? { dialogs: last.dialogs } : {}
|
|
4700
|
+
}
|
|
4701
|
+
};
|
|
4702
|
+
}
|
|
4703
|
+
runStep(command, step, secrets) {
|
|
4704
|
+
const str = (v) => typeof v === "string" ? v : void 0;
|
|
4705
|
+
const target = () => {
|
|
4706
|
+
if (str(step.ref)) {
|
|
4707
|
+
return str(step.ref);
|
|
4708
|
+
}
|
|
4709
|
+
const described = {
|
|
4710
|
+
...command !== "type" && str(step.text) ? { text: str(step.text) } : {},
|
|
4711
|
+
...str(step.role) ? { role: str(step.role) } : {},
|
|
4712
|
+
...str(step.label) ? { label: str(step.label) } : {},
|
|
4713
|
+
...str(step.selector) ? { selector: str(step.selector) } : {}
|
|
4714
|
+
};
|
|
4715
|
+
if (Object.keys(described).length === 0) {
|
|
4716
|
+
throw new MindStudioError(
|
|
4717
|
+
`${command} needs an element: a ref, text, role, label or selector.`,
|
|
4718
|
+
"invalid_steps",
|
|
4719
|
+
400
|
|
4720
|
+
);
|
|
4721
|
+
}
|
|
4722
|
+
return described;
|
|
4723
|
+
};
|
|
4724
|
+
switch (command) {
|
|
4725
|
+
case "snapshot":
|
|
4726
|
+
return this.snapshot();
|
|
4727
|
+
case "navigate":
|
|
4728
|
+
return this.goto(str(step.url) ?? "");
|
|
4729
|
+
case "click":
|
|
4730
|
+
return this.click(target());
|
|
4731
|
+
case "type": {
|
|
4732
|
+
const secret = str(step.secret);
|
|
4733
|
+
if (secret !== void 0 && !(secret in secrets)) {
|
|
4734
|
+
throw new MindStudioError(
|
|
4735
|
+
`No secret named "${secret}".`,
|
|
4736
|
+
"unknown_secret",
|
|
4737
|
+
400
|
|
4738
|
+
);
|
|
4739
|
+
}
|
|
4740
|
+
return this.type(
|
|
4741
|
+
target(),
|
|
4742
|
+
secret !== void 0 ? secrets[secret] : str(step.text) ?? "",
|
|
4743
|
+
{
|
|
4744
|
+
clear: step.clear === true || secret !== void 0,
|
|
4745
|
+
submit: step.submit === true
|
|
4746
|
+
}
|
|
4747
|
+
);
|
|
4748
|
+
}
|
|
4749
|
+
case "select":
|
|
4750
|
+
return this.select(target(), str(step.option) ?? "");
|
|
4751
|
+
case "press":
|
|
4752
|
+
return this.press(str(step.key) ?? "");
|
|
4753
|
+
case "scroll": {
|
|
4754
|
+
const hasTarget = str(step.ref) ?? str(step.label) ?? str(step.selector) ?? str(step.text);
|
|
4755
|
+
return this.scroll({
|
|
4756
|
+
...hasTarget ? { to: target() } : {},
|
|
4757
|
+
...typeof step.by === "number" ? { by: step.by } : {}
|
|
4758
|
+
});
|
|
4759
|
+
}
|
|
4760
|
+
case "wait":
|
|
4761
|
+
return this.waitFor(str(step.text) ?? "", {
|
|
4762
|
+
...typeof step.timeoutSeconds === "number" ? { timeoutSeconds: step.timeoutSeconds } : {}
|
|
4763
|
+
});
|
|
4764
|
+
default:
|
|
4765
|
+
throw new MindStudioError(
|
|
4766
|
+
`Unknown command "${command}".`,
|
|
4767
|
+
"invalid_steps",
|
|
4768
|
+
400
|
|
4769
|
+
);
|
|
4770
|
+
}
|
|
4771
|
+
}
|
|
4772
|
+
/**
|
|
4773
|
+
* A grant to watch this session live, for your frontend: return it from a
|
|
4774
|
+
* method and hand its `token` to `browser.view()` in
|
|
4775
|
+
* `@mindstudio-ai/interface`. Whoever holds it sees the page until it
|
|
4776
|
+
* expires, so check your own access first, as for `events.grant()`.
|
|
4777
|
+
* `ttlSeconds` is clamped to 60 through 3600 (900 by default); the frontend
|
|
4778
|
+
* asks again through your method when it runs out.
|
|
4779
|
+
*
|
|
4780
|
+
* ```ts
|
|
4781
|
+
* export async function watchRun({ runId }: { runId: string }) {
|
|
4782
|
+
* auth.requireRole('admin');
|
|
4783
|
+
* const run = await Runs.get(runId);
|
|
4784
|
+
* const session = await browser.session(run.browserSessionId);
|
|
4785
|
+
* return session.grantView();
|
|
4786
|
+
* }
|
|
4787
|
+
* ```
|
|
4788
|
+
*/
|
|
4789
|
+
grantView(options = {}) {
|
|
4790
|
+
return this.call("view", { sessionId: this.id, ...options });
|
|
4791
|
+
}
|
|
4792
|
+
/** Close the session and its browser. */
|
|
4793
|
+
async close() {
|
|
4794
|
+
await this.call("close", { sessionId: this.id });
|
|
4795
|
+
}
|
|
4796
|
+
};
|
|
4797
|
+
function createBrowser(call, steps) {
|
|
4798
|
+
return {
|
|
4799
|
+
async open(options = {}) {
|
|
4800
|
+
const { session: session2 } = await call("open", {
|
|
4801
|
+
...options.viewport !== void 0 ? { viewport: options.viewport } : {},
|
|
4802
|
+
...options.idleTimeoutSeconds !== void 0 ? { idleTimeoutSeconds: options.idleTimeoutSeconds } : {}
|
|
4803
|
+
});
|
|
4804
|
+
let ready = session2.ready;
|
|
4805
|
+
let failures = 0;
|
|
4806
|
+
while (!ready) {
|
|
4807
|
+
try {
|
|
4808
|
+
({ ready } = await call("ready", { sessionId: session2.id }));
|
|
4809
|
+
failures = 0;
|
|
4810
|
+
} catch (err) {
|
|
4811
|
+
const answered = err instanceof MindStudioError && err.status < 500;
|
|
4812
|
+
if (answered || err.code === "browser_unavailable" || ++failures > 5) {
|
|
4813
|
+
throw err;
|
|
4814
|
+
}
|
|
4815
|
+
await new Promise((r) => setTimeout(r, 2e3));
|
|
4816
|
+
}
|
|
4817
|
+
}
|
|
4818
|
+
const opened = new BrowserSession(session2, call, steps);
|
|
4819
|
+
if (options.url) {
|
|
4820
|
+
try {
|
|
4821
|
+
await opened.goto(options.url);
|
|
4822
|
+
} catch (err) {
|
|
4823
|
+
await opened.close().catch(() => {
|
|
4824
|
+
});
|
|
4825
|
+
throw err;
|
|
4826
|
+
}
|
|
4827
|
+
}
|
|
4828
|
+
return opened;
|
|
4829
|
+
},
|
|
4830
|
+
async session(id) {
|
|
4831
|
+
const { session: session2 } = await call("session", { sessionId: id });
|
|
4832
|
+
return new BrowserSession(session2, call, steps);
|
|
4833
|
+
}
|
|
4834
|
+
};
|
|
4835
|
+
}
|
|
4836
|
+
|
|
4837
|
+
// src/task/local.ts
|
|
4838
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
4323
4839
|
|
|
4324
4840
|
// src/generated/metadata.ts
|
|
4325
4841
|
var stepMetadata = {
|
|
@@ -4573,7 +5089,7 @@ var stepMetadata = {
|
|
|
4573
5089
|
"evaluateDecision": {
|
|
4574
5090
|
stepType: "evaluateDecision",
|
|
4575
5091
|
description: "Ask structured questions about text, JSON, or images and get calibrated answers from a decision model. The model returns a probability, a selected option, or a score. It does not write a reply. Each question is a noul (yes or no, as a probability from 0 to 1), a choice (one option from a set you define, with a probability for each), or a score (a rating against ordered levels). Put every question in the same call. The question id is yours: answers come back under the same ids. The model defaults to Jev (jev-latest).",
|
|
4576
|
-
usageNotes: "- state is the content to judge: a string, or a JSON object or array (a record, a chat log, application state).\n- questions is a map. The key is an id you choose. It is not shown to the model. The answer comes back under the same id.\n- A noul asks a yes/no question and returns noul, a probability from 0 (no) to 1 (yes). Optional criteria.true and criteria.false say what a yes and a no mean.\n- A choice picks one option. criteria is a map of option id to a short description (or null). At most 255 options. The answer includes the winning choice, a probability for every option, and confidence.\n- A score rates the state on an ordered rubric. criteria is 2 to 10 level descriptions, low to high. The answer's score can land between levels, and includes a probability per level plus confidence.\n- An answer of type refusal means the model declined that question (GPT-6 Luna Decisions can).\n- instructions may be a string, or an object that keeps the question in one field and reference data in the others. Refer to those fields by name in backticks.\n- Send every question you need in one call. The model reads the state once and answers them together.\n- Probabilities from different models are not comparable: tune a threshold against the model, and the version, it will run on.\n- decisionModelOverride.model picks the model:\n - jev-latest (the default) is TypeSafe's current stable release, text only, up to 64k tokens. jev-preview tracks TypeSafe's newest release. Pin a version such as jev-1.13.0 when a threshold was tuned against that version.\n - clef and clef-flash run Cloudflare Clef, which also reads up to 4 PNG, JPEG, or WebP images, each at most 4 MiB and 8 MiB together, and takes at most 64 questions, each id using only letters, digits, underscore, dot, and hyphen, up to 100 characters. clef is the larger model, clef-flash the faster one. @cf/cloudflare/clef and @cf/cloudflare/clef-flash name the same two.\n - mercury-decide runs Inception's Mercury Decide, which reads up to 64k tokens and up to 16 PNG, JPEG, WebP, or GIF images, each at most 10 MiB and 14 MiB together.\n - gpt-6-luna-decisions runs OpenAI's Decisions API (in beta), which reads up to 922k tokens and PNG, JPEG, WebP, or GIF images of up to 20 MB each. It takes up to 200 questions, and a choice needs at least 2 options. gpt-6-luna names it too.\n- images is for a model that reads them. Pass URLs or data URLs. A model that reads none refuses the call.\n- The response model field is the version that answered.\n- Billed per input token. Output tokens are free. Each model has its own rate, and GPT-6 Luna Decisions bills a prompt over 272k tokens at twice its rate.",
|
|
5092
|
+
usageNotes: "- state is the content to judge: a string, or a JSON object or array (a record, a chat log, application state).\n- questions is a map. The key is an id you choose. It is not shown to the model. The answer comes back under the same id.\n- A noul asks a yes/no question and returns noul, a probability from 0 (no) to 1 (yes). Optional criteria.true and criteria.false say what a yes and a no mean.\n- A choice picks one option. criteria is a map of option id to a short description (or null). At most 255 options. The answer includes the winning choice, a probability for every option, and confidence.\n- A score rates the state on an ordered rubric. criteria is 2 to 10 level descriptions, low to high. The answer's score can land between levels, and includes a probability per level plus confidence.\n- An answer of type refusal means the model declined that question (GPT-6 Luna Decisions can).\n- instructions may be a string, or an object that keeps the question in one field and reference data in the others. Refer to those fields by name in backticks.\n- Send every question you need in one call. The model reads the state once and answers them together.\n- Probabilities from different models are not comparable: tune a threshold against the model, and the version, it will run on.\n- decisionModelOverride.model picks the model:\n - jev-latest (the default) is TypeSafe's current stable release, text only, up to 64k tokens. jev-preview tracks TypeSafe's newest release. Pin a version such as jev-1.13.0 when a threshold was tuned against that version.\n - clef and clef-flash run Cloudflare Clef, which also reads up to 4 PNG, JPEG, or WebP images, each at most 4 MiB and 8 MiB together, and takes at most 64 questions, each id using only letters, digits, underscore, dot, and hyphen, up to 100 characters. clef is the larger model, clef-flash the faster one. @cf/cloudflare/clef and @cf/cloudflare/clef-flash name the same two.\n - mercury-decide runs Inception's Mercury Decide, which reads up to 64k tokens and up to 16 PNG, JPEG, WebP, or GIF images, each at most 10 MiB and 14 MiB together.\n - drex-max, drex-v2.0 and drex-v1.5 run Nace.AI's Drex, served by DeepInfra, and take at most 16 questions. drex-max reads up to 262k tokens and up to 4 PNG, JPEG, WebP, or GIF images, 9 MiB together. drex-v2.0 and drex-v1.5 read text only, up to 128k tokens. nace-ai/drex-max, nace-ai/drex-v2.0 and nace-ai/drex-v1.5 name the same three.\n - gpt-6-luna-decisions runs OpenAI's Decisions API (in beta), which reads up to 922k tokens and PNG, JPEG, WebP, or GIF images of up to 20 MB each. It takes up to 200 questions, and a choice needs at least 2 options. gpt-6-luna names it too.\n- images is for a model that reads them. Pass URLs or data URLs. A model that reads none refuses the call.\n- The response model field is the version that answered.\n- Billed per input token. Output tokens are free. Each model has its own rate, and GPT-6 Luna Decisions bills a prompt over 272k tokens at twice its rate.",
|
|
4577
5093
|
inputSchema: { "type": "object", "properties": { "state": { "anyOf": [{ "type": "string" }, { "type": "object", "properties": {}, "required": [] }, { "type": "array", "items": {} }] }, "questions": { "type": "object", "properties": {}, "required": [], "description": "Questions to ask about the state, keyed by an id you choose." }, "images": { "type": "array", "items": { "type": "string" }, "description": "Images for a model that reads them (Clef, Clef Flash, GPT-6 Luna Decisions). Each entry is a URL or a data URL. A model that reads none refuses the call." }, "decisionModelOverride": { "type": "object", "properties": { "model": { "type": "string", "description": "Decision model: a catalog id, its vendor's name for it (`@cf/cloudflare/clef`, `gpt-6-luna`), or a version it pins (`jev-1.13.0`)" }, "config": { "type": "object", "properties": {}, "required": [], "description": "Additional model-specific configuration" } }, "required": ["model"], "description": "Optional model override. Uses the workspace's default decision model (Jev) if not specified" } }, "required": ["state", "questions"] },
|
|
4578
5094
|
outputSchema: { "type": "object", "properties": { "model": { "type": "string", "description": "Versioned model id that answered, such as jev-1.13.0." }, "answers": { "type": "object", "properties": {}, "required": [], "description": "One answer per question, under the same ids." }, "usage": { "type": "object", "properties": { "input_tokens": { "type": "number" }, "output_tokens": { "type": "number" } }, "required": ["input_tokens", "output_tokens"], "description": "Token counts for the call. Input tokens are what the model bills." } }, "required": ["model", "answers", "usage"] }
|
|
4579
5095
|
},
|
|
@@ -5695,6 +6211,9 @@ var DEFAULT_MAX_TURNS = 20;
|
|
|
5695
6211
|
var MAX_TURNS_LIMIT = 100;
|
|
5696
6212
|
var MAX_SCHEMA_REPAIR_ATTEMPTS = 3;
|
|
5697
6213
|
var MAX_TOOL_OUTPUT_CHARS = 5e4;
|
|
6214
|
+
var SUPERSEDED_RESULT = JSON.stringify({
|
|
6215
|
+
superseded: "An earlier result, replaced by a newer one from the same tool further on. Work from the latest."
|
|
6216
|
+
});
|
|
5698
6217
|
var MAX_TURN_ATTEMPTS = 5;
|
|
5699
6218
|
var INITIAL_BACKOFF_MS = 1e3;
|
|
5700
6219
|
var STALL_TIMEOUT_MS = 3e5;
|
|
@@ -5930,9 +6449,14 @@ ${structuredOutputExample}
|
|
|
5930
6449
|
const toolKinds = /* @__PURE__ */ new Map();
|
|
5931
6450
|
const toolDefaults = /* @__PURE__ */ new Map();
|
|
5932
6451
|
const functionTools = /* @__PURE__ */ new Map();
|
|
6452
|
+
const supersedeKeys = /* @__PURE__ */ new Map();
|
|
6453
|
+
const latestByKey = /* @__PURE__ */ new Map();
|
|
5933
6454
|
for (const t of options.tools) {
|
|
5934
6455
|
if (typeof t === "object" && "execute" in t) {
|
|
5935
6456
|
functionTools.set(t.name, t.execute);
|
|
6457
|
+
if (t.supersedes) {
|
|
6458
|
+
supersedeKeys.set(t.name, t.supersedes);
|
|
6459
|
+
}
|
|
5936
6460
|
}
|
|
5937
6461
|
}
|
|
5938
6462
|
for (const t of wireTools) {
|
|
@@ -5955,7 +6479,7 @@ ${structuredOutputExample}
|
|
|
5955
6479
|
MAX_TURNS_LIMIT
|
|
5956
6480
|
);
|
|
5957
6481
|
const maxToolOutputChars = options.maxToolOutputChars ?? MAX_TOOL_OUTPUT_CHARS;
|
|
5958
|
-
const traceId =
|
|
6482
|
+
const traceId = randomUUID3();
|
|
5959
6483
|
const messages = [
|
|
5960
6484
|
{ role: "user", content: JSON.stringify(options.input) }
|
|
5961
6485
|
];
|
|
@@ -6134,6 +6658,17 @@ ${structuredOutputExample}
|
|
|
6134
6658
|
toolCallId: toolCall.id,
|
|
6135
6659
|
...isError && { isToolError: true }
|
|
6136
6660
|
});
|
|
6661
|
+
const key = supersedeKeys.get(toolCall.name);
|
|
6662
|
+
if (key !== void 0 && !isError) {
|
|
6663
|
+
const earlier = latestByKey.get(key);
|
|
6664
|
+
if (earlier !== void 0) {
|
|
6665
|
+
messages[earlier] = {
|
|
6666
|
+
...messages[earlier],
|
|
6667
|
+
content: SUPERSEDED_RESULT
|
|
6668
|
+
};
|
|
6669
|
+
}
|
|
6670
|
+
latestByKey.set(key, messages.length - 1);
|
|
6671
|
+
}
|
|
6137
6672
|
onEvent?.({
|
|
6138
6673
|
type: "tool_call_result",
|
|
6139
6674
|
id: toolCall.id,
|
|
@@ -6785,6 +7320,7 @@ var MindStudioAgent = class {
|
|
|
6785
7320
|
_analytics;
|
|
6786
7321
|
_email;
|
|
6787
7322
|
_events;
|
|
7323
|
+
_browser;
|
|
6788
7324
|
/** @internal Auth type — 'internal' for CALLBACK_TOKEN (managed mode), 'apiKey' otherwise. */
|
|
6789
7325
|
_authType;
|
|
6790
7326
|
/** @internal Usage source sent on step executions (from MINDSTUDIO_REQUEST_SOURCE).
|
|
@@ -7851,6 +8387,17 @@ var MindStudioAgent = class {
|
|
|
7851
8387
|
get events() {
|
|
7852
8388
|
return this._events ??= createEvents(this._eventsRequest.bind(this));
|
|
7853
8389
|
}
|
|
8390
|
+
/**
|
|
8391
|
+
* The `browser` namespace — a real Chrome your app drives. `open()` starts
|
|
8392
|
+
* a session; every action answers with the page afterward. See
|
|
8393
|
+
* {@link Browser}.
|
|
8394
|
+
*/
|
|
8395
|
+
get browser() {
|
|
8396
|
+
return this._browser ??= createBrowser(
|
|
8397
|
+
this._browserRequest.bind(this),
|
|
8398
|
+
(stepType, step) => this.executeStep(stepType, step)
|
|
8399
|
+
);
|
|
8400
|
+
}
|
|
7854
8401
|
/**
|
|
7855
8402
|
* Outbound email: sending, the per-recipient delivery log, blast stats, and
|
|
7856
8403
|
* this app's unsubscribe list.
|
|
@@ -8048,6 +8595,17 @@ var MindStudioAgent = class {
|
|
|
8048
8595
|
fallbackCode: "events_error"
|
|
8049
8596
|
});
|
|
8050
8597
|
}
|
|
8598
|
+
/**
|
|
8599
|
+
* @internal Transport for the `browser` namespace —
|
|
8600
|
+
* POST /_internal/v2/browser/<op> with the raw hook token. No 429 retry:
|
|
8601
|
+
* an action repeated is a second click.
|
|
8602
|
+
*/
|
|
8603
|
+
async _browserRequest(op, body) {
|
|
8604
|
+
return this._brokeredRequest("browser", op, body, {
|
|
8605
|
+
fallbackMessage: "Browser call failed",
|
|
8606
|
+
fallbackCode: "browser_error"
|
|
8607
|
+
});
|
|
8608
|
+
}
|
|
8051
8609
|
async _analyticsRequest(op, body) {
|
|
8052
8610
|
const call = () => this._brokeredRequest("analytics", op, body, {
|
|
8053
8611
|
fallbackMessage: "Analytics read failed",
|
|
@@ -9544,6 +10102,13 @@ var events = new Proxy({}, {
|
|
|
9544
10102
|
return typeof value === "function" ? value.bind(target) : value;
|
|
9545
10103
|
}
|
|
9546
10104
|
});
|
|
10105
|
+
var browser = new Proxy({}, {
|
|
10106
|
+
get(_, prop) {
|
|
10107
|
+
const target = mindstudio.browser;
|
|
10108
|
+
const value = Reflect.get(target, prop, target);
|
|
10109
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
10110
|
+
}
|
|
10111
|
+
});
|
|
9547
10112
|
var session = new Proxy(
|
|
9548
10113
|
{},
|
|
9549
10114
|
{
|
|
@@ -9562,6 +10127,7 @@ var prerender = {
|
|
|
9562
10127
|
};
|
|
9563
10128
|
export {
|
|
9564
10129
|
AuthContext,
|
|
10130
|
+
BrowserSession,
|
|
9565
10131
|
HttpResponse,
|
|
9566
10132
|
MAX_EVENT_PAYLOAD_CHARS,
|
|
9567
10133
|
MindStudioAgent2 as MindStudioAgent,
|
|
@@ -9571,6 +10137,7 @@ export {
|
|
|
9571
10137
|
analytics,
|
|
9572
10138
|
auth,
|
|
9573
10139
|
blockTypeAliases,
|
|
10140
|
+
browser,
|
|
9574
10141
|
dataSources,
|
|
9575
10142
|
db,
|
|
9576
10143
|
index_default as default,
|