@yoqa/runner 0.3.24 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1147 -246
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -16513,7 +16513,8 @@ var runTests = sqliteTable("run_tests", {
|
|
|
16513
16513
|
executionMode: text("execution_mode"),
|
|
16514
16514
|
error: text("error"),
|
|
16515
16515
|
startedAt: integer("started_at"),
|
|
16516
|
-
finishedAt: integer("finished_at")
|
|
16516
|
+
finishedAt: integer("finished_at"),
|
|
16517
|
+
currentCommand: text("current_command")
|
|
16517
16518
|
});
|
|
16518
16519
|
var runSteps = sqliteTable("run_steps", {
|
|
16519
16520
|
id: text("id").primaryKey(),
|
|
@@ -16524,6 +16525,7 @@ var runSteps = sqliteTable("run_steps", {
|
|
|
16524
16525
|
ok: integer("ok").notNull().default(0),
|
|
16525
16526
|
latencyMs: integer("latency_ms").notNull().default(0),
|
|
16526
16527
|
detail: text("detail"),
|
|
16528
|
+
command: text("command"),
|
|
16527
16529
|
createdAt: integer("created_at").notNull()
|
|
16528
16530
|
});
|
|
16529
16531
|
|
|
@@ -16638,7 +16640,8 @@ function ensureSchema(sqlite) {
|
|
|
16638
16640
|
execution_mode TEXT,
|
|
16639
16641
|
error TEXT,
|
|
16640
16642
|
started_at INTEGER,
|
|
16641
|
-
finished_at INTEGER
|
|
16643
|
+
finished_at INTEGER,
|
|
16644
|
+
current_command TEXT
|
|
16642
16645
|
);
|
|
16643
16646
|
|
|
16644
16647
|
CREATE TABLE IF NOT EXISTS run_steps (
|
|
@@ -16650,6 +16653,7 @@ function ensureSchema(sqlite) {
|
|
|
16650
16653
|
ok INTEGER NOT NULL DEFAULT 0,
|
|
16651
16654
|
latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
16652
16655
|
detail TEXT,
|
|
16656
|
+
command TEXT,
|
|
16653
16657
|
created_at INTEGER NOT NULL
|
|
16654
16658
|
);
|
|
16655
16659
|
|
|
@@ -16668,6 +16672,7 @@ function ensureSchema(sqlite) {
|
|
|
16668
16672
|
migrateAppsPrefix(sqlite);
|
|
16669
16673
|
migrateCaseScripts(sqlite);
|
|
16670
16674
|
migrateRunExecutionMode(sqlite);
|
|
16675
|
+
migrateRunCommands(sqlite);
|
|
16671
16676
|
}
|
|
16672
16677
|
function tableColumns(sqlite, table) {
|
|
16673
16678
|
const rows = sqlite.query(`PRAGMA table_info(${table})`).all();
|
|
@@ -16696,6 +16701,16 @@ function migrateRunExecutionMode(sqlite) {
|
|
|
16696
16701
|
addColumnIfMissing(sqlite, "run_tests", "execution_mode", "execution_mode TEXT", testCols);
|
|
16697
16702
|
}
|
|
16698
16703
|
}
|
|
16704
|
+
function migrateRunCommands(sqlite) {
|
|
16705
|
+
const testCols = tableColumns(sqlite, "run_tests");
|
|
16706
|
+
if (testCols.size > 0) {
|
|
16707
|
+
addColumnIfMissing(sqlite, "run_tests", "current_command", "current_command TEXT", testCols);
|
|
16708
|
+
}
|
|
16709
|
+
const stepCols = tableColumns(sqlite, "run_steps");
|
|
16710
|
+
if (stepCols.size > 0) {
|
|
16711
|
+
addColumnIfMissing(sqlite, "run_steps", "command", "command TEXT", stepCols);
|
|
16712
|
+
}
|
|
16713
|
+
}
|
|
16699
16714
|
function migrateProvidersTable(sqlite) {
|
|
16700
16715
|
const existing = tableColumns(sqlite, "providers");
|
|
16701
16716
|
if (existing.size === 0)
|
|
@@ -16801,7 +16816,7 @@ function getCatalogDb() {
|
|
|
16801
16816
|
// package.json
|
|
16802
16817
|
var package_default = {
|
|
16803
16818
|
name: "@yoqa/runner",
|
|
16804
|
-
version: "0.3.
|
|
16819
|
+
version: "0.3.26",
|
|
16805
16820
|
description: "Yoqa local runner \u2014 Appium + catalog HTTP API (Bun). Start with yoqa serve or the yoqa-runner bin.",
|
|
16806
16821
|
license: "UNLICENSED",
|
|
16807
16822
|
type: "module",
|
|
@@ -16884,12 +16899,14 @@ function trackMjpegProxy() {
|
|
|
16884
16899
|
return controller;
|
|
16885
16900
|
}
|
|
16886
16901
|
function abortAllMjpegProxies() {
|
|
16902
|
+
const had = proxies.size > 0;
|
|
16887
16903
|
for (const controller of [...proxies]) {
|
|
16888
16904
|
try {
|
|
16889
16905
|
controller.abort();
|
|
16890
16906
|
} catch {}
|
|
16891
16907
|
}
|
|
16892
16908
|
proxies.clear();
|
|
16909
|
+
return had;
|
|
16893
16910
|
}
|
|
16894
16911
|
|
|
16895
16912
|
// src/domains/devices/session.ts
|
|
@@ -17574,6 +17591,38 @@ async function listDevices(platform, options = {}) {
|
|
|
17574
17591
|
return listAndroidDevices(options);
|
|
17575
17592
|
}
|
|
17576
17593
|
|
|
17594
|
+
// src/domains/devices/keyboard.ts
|
|
17595
|
+
function isUnsupportedCommandError(error) {
|
|
17596
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17597
|
+
return /unknown command|unknown method|not (yet )?(been )?implemented|not (yet )?supported|does not support/i.test(message);
|
|
17598
|
+
}
|
|
17599
|
+
function w3cKeyActions(text2) {
|
|
17600
|
+
const actions = [];
|
|
17601
|
+
for (const ch of text2) {
|
|
17602
|
+
actions.push({ type: "keyDown", value: ch }, { type: "keyUp", value: ch });
|
|
17603
|
+
}
|
|
17604
|
+
return [{ type: "key", id: "keyboard", actions }];
|
|
17605
|
+
}
|
|
17606
|
+
async function w3cType(browser, text2) {
|
|
17607
|
+
await browser.performActions(w3cKeyActions(text2));
|
|
17608
|
+
try {
|
|
17609
|
+
await browser.releaseActions();
|
|
17610
|
+
} catch {}
|
|
17611
|
+
}
|
|
17612
|
+
async function typeText(browser, text2) {
|
|
17613
|
+
if (text2.length === 0)
|
|
17614
|
+
return "noop";
|
|
17615
|
+
try {
|
|
17616
|
+
await browser.execute("mobile: type", { text: text2 });
|
|
17617
|
+
return "mobile";
|
|
17618
|
+
} catch (error) {
|
|
17619
|
+
if (!isUnsupportedCommandError(error))
|
|
17620
|
+
throw error;
|
|
17621
|
+
}
|
|
17622
|
+
await w3cType(browser, text2);
|
|
17623
|
+
return "w3c";
|
|
17624
|
+
}
|
|
17625
|
+
|
|
17577
17626
|
// src/domains/devices/session.ts
|
|
17578
17627
|
var YOQA_ROOT4 = join7(homedir6(), ".yoqa");
|
|
17579
17628
|
var DEFAULT_MJPEG_PORT = Number(process.env.YOQA_MJPEG_PORT ?? "9100");
|
|
@@ -17597,6 +17646,7 @@ function isDeadSessionError(error) {
|
|
|
17597
17646
|
const message = error instanceof Error ? error.message : String(error);
|
|
17598
17647
|
return DEAD_SESSION_RE.test(message);
|
|
17599
17648
|
}
|
|
17649
|
+
var DEAD_SESSION_RETRY_MS = 200;
|
|
17600
17650
|
function mjpegSettingsForDevice(options) {
|
|
17601
17651
|
const physicalIos = options.platform === "ios" && looksLikePhysicalIosUdid(options.deviceId);
|
|
17602
17652
|
if (physicalIos) {
|
|
@@ -17981,10 +18031,17 @@ async function createDeviceSession(options) {
|
|
|
17981
18031
|
try {
|
|
17982
18032
|
return await fn();
|
|
17983
18033
|
} catch (error) {
|
|
17984
|
-
if (isDeadSessionError(error))
|
|
17985
|
-
|
|
18034
|
+
if (!isDeadSessionError(error))
|
|
18035
|
+
throw error;
|
|
18036
|
+
await Bun.sleep(DEAD_SESSION_RETRY_MS);
|
|
18037
|
+
try {
|
|
18038
|
+
return await fn();
|
|
18039
|
+
} catch (retryError) {
|
|
18040
|
+
if (isDeadSessionError(retryError)) {
|
|
18041
|
+
notifySessionDead();
|
|
18042
|
+
}
|
|
18043
|
+
throw retryError;
|
|
17986
18044
|
}
|
|
17987
|
-
throw error;
|
|
17988
18045
|
}
|
|
17989
18046
|
};
|
|
17990
18047
|
const getWindowSize = async () => guard(() => browser.getWindowSize());
|
|
@@ -18048,10 +18105,10 @@ async function createDeviceSession(options) {
|
|
|
18048
18105
|
});
|
|
18049
18106
|
});
|
|
18050
18107
|
};
|
|
18051
|
-
const swipe = async (x1, y1, x2, y2, durationMs = 400) => {
|
|
18108
|
+
const swipe = async (x1, y1, x2, y2, durationMs = 400, swipeOptions) => {
|
|
18052
18109
|
await gate.withLock(async () => {
|
|
18053
18110
|
await guard(async () => {
|
|
18054
|
-
const size = await getPointerSize();
|
|
18111
|
+
const size = swipeOptions?.coordSpace === "screenshot" ? await getScreenshotCoordSize() : await getPointerSize();
|
|
18055
18112
|
await injectSwipe(browser, toPx(x1, size.width), toPx(y1, size.height), toPx(x2, size.width), toPx(y2, size.height), durationMs);
|
|
18056
18113
|
});
|
|
18057
18114
|
});
|
|
@@ -18062,7 +18119,7 @@ async function createDeviceSession(options) {
|
|
|
18062
18119
|
const type = async (text2) => {
|
|
18063
18120
|
await gate.withLock(async () => {
|
|
18064
18121
|
await guard(async () => {
|
|
18065
|
-
await browser
|
|
18122
|
+
await typeText(browser, text2);
|
|
18066
18123
|
});
|
|
18067
18124
|
});
|
|
18068
18125
|
};
|
|
@@ -24263,6 +24320,8 @@ var caseScriptTapSchema = exports_external.object({
|
|
|
24263
24320
|
y: exports_external.number().min(0).max(1000).optional(),
|
|
24264
24321
|
label: exports_external.string().min(1).optional(),
|
|
24265
24322
|
id: exports_external.string().min(1).optional(),
|
|
24323
|
+
double: exports_external.boolean().optional(),
|
|
24324
|
+
durationMs: exports_external.number().min(50).max(5000).optional(),
|
|
24266
24325
|
reason: exports_external.string().optional()
|
|
24267
24326
|
}).superRefine((value, ctx) => {
|
|
24268
24327
|
const hasCoords = value.x != null && value.y != null;
|
|
@@ -24273,8 +24332,45 @@ var caseScriptTapSchema = exports_external.object({
|
|
|
24273
24332
|
});
|
|
24274
24333
|
}
|
|
24275
24334
|
});
|
|
24335
|
+
var caseScriptSwipeSchema = exports_external.object({
|
|
24336
|
+
type: exports_external.literal("swipe"),
|
|
24337
|
+
x: exports_external.number().min(0).max(1000),
|
|
24338
|
+
y: exports_external.number().min(0).max(1000),
|
|
24339
|
+
x2: exports_external.number().min(0).max(1000),
|
|
24340
|
+
y2: exports_external.number().min(0).max(1000),
|
|
24341
|
+
durationMs: exports_external.number().min(50).max(5000).optional(),
|
|
24342
|
+
reason: exports_external.string().optional()
|
|
24343
|
+
});
|
|
24344
|
+
var caseScriptDragSchema = exports_external.object({
|
|
24345
|
+
type: exports_external.literal("drag"),
|
|
24346
|
+
x: exports_external.number().min(0).max(1000),
|
|
24347
|
+
y: exports_external.number().min(0).max(1000),
|
|
24348
|
+
x2: exports_external.number().min(0).max(1000),
|
|
24349
|
+
y2: exports_external.number().min(0).max(1000),
|
|
24350
|
+
durationMs: exports_external.number().min(50).max(5000).optional(),
|
|
24351
|
+
reason: exports_external.string().optional()
|
|
24352
|
+
});
|
|
24353
|
+
var caseScriptAppIdSchema = exports_external.union([
|
|
24354
|
+
exports_external.object({
|
|
24355
|
+
type: exports_external.literal("activate-app"),
|
|
24356
|
+
appId: exports_external.string().min(1),
|
|
24357
|
+
reason: exports_external.string().optional()
|
|
24358
|
+
}),
|
|
24359
|
+
exports_external.object({
|
|
24360
|
+
type: exports_external.literal("terminate-app"),
|
|
24361
|
+
appId: exports_external.string().min(1),
|
|
24362
|
+
reason: exports_external.string().optional()
|
|
24363
|
+
}),
|
|
24364
|
+
exports_external.object({
|
|
24365
|
+
type: exports_external.literal("restart-app"),
|
|
24366
|
+
appId: exports_external.string().min(1),
|
|
24367
|
+
reason: exports_external.string().optional()
|
|
24368
|
+
})
|
|
24369
|
+
]);
|
|
24276
24370
|
var caseScriptActionSchema = exports_external.union([
|
|
24277
24371
|
caseScriptTapSchema,
|
|
24372
|
+
caseScriptSwipeSchema,
|
|
24373
|
+
caseScriptDragSchema,
|
|
24278
24374
|
exports_external.object({
|
|
24279
24375
|
type: exports_external.literal("type"),
|
|
24280
24376
|
text: exports_external.string(),
|
|
@@ -24296,6 +24392,17 @@ var caseScriptActionSchema = exports_external.union([
|
|
|
24296
24392
|
type: exports_external.literal("alert"),
|
|
24297
24393
|
alertAction: exports_external.union([exports_external.literal("accept"), exports_external.literal("dismiss")]).default("accept"),
|
|
24298
24394
|
reason: exports_external.string().optional()
|
|
24395
|
+
}),
|
|
24396
|
+
caseScriptAppIdSchema,
|
|
24397
|
+
exports_external.object({
|
|
24398
|
+
type: exports_external.literal("background-app"),
|
|
24399
|
+
seconds: exports_external.number().min(0).max(120).optional(),
|
|
24400
|
+
reason: exports_external.string().optional()
|
|
24401
|
+
}),
|
|
24402
|
+
exports_external.object({
|
|
24403
|
+
type: exports_external.literal("open-url"),
|
|
24404
|
+
url: exports_external.string().min(1),
|
|
24405
|
+
reason: exports_external.string().optional()
|
|
24299
24406
|
})
|
|
24300
24407
|
]);
|
|
24301
24408
|
var caseScriptSchema = exports_external.object({
|
|
@@ -24552,6 +24659,7 @@ var runStepSchema = exports_external.object({
|
|
|
24552
24659
|
ok: exports_external.boolean(),
|
|
24553
24660
|
latencyMs: exports_external.number().nonnegative(),
|
|
24554
24661
|
detail: exports_external.string().nullable(),
|
|
24662
|
+
command: exports_external.string().nullable().optional(),
|
|
24555
24663
|
createdAt: exports_external.number().int().nonnegative()
|
|
24556
24664
|
});
|
|
24557
24665
|
var runTestSchema = exports_external.object({
|
|
@@ -24563,6 +24671,7 @@ var runTestSchema = exports_external.object({
|
|
|
24563
24671
|
error: exports_external.string().nullable(),
|
|
24564
24672
|
startedAt: exports_external.number().int().nonnegative().nullable(),
|
|
24565
24673
|
finishedAt: exports_external.number().int().nonnegative().nullable(),
|
|
24674
|
+
currentCommand: exports_external.string().nullable().optional(),
|
|
24566
24675
|
steps: exports_external.array(runStepSchema).optional()
|
|
24567
24676
|
});
|
|
24568
24677
|
var runSchema = exports_external.object({
|
|
@@ -24736,6 +24845,59 @@ function decodeXmlEntities(value) {
|
|
|
24736
24845
|
}
|
|
24737
24846
|
|
|
24738
24847
|
// ../../packages/runner-client/src/shell-script.ts
|
|
24848
|
+
function shellSingleQuote(value) {
|
|
24849
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
24850
|
+
}
|
|
24851
|
+
function formatActionShellLine(action) {
|
|
24852
|
+
const parts = ["yoqa", "action", action.kind];
|
|
24853
|
+
if (action.id) {
|
|
24854
|
+
parts.push("--id", shellSingleQuote(action.id));
|
|
24855
|
+
}
|
|
24856
|
+
if (action.label) {
|
|
24857
|
+
parts.push("--label", shellSingleQuote(action.label));
|
|
24858
|
+
}
|
|
24859
|
+
if (action.description) {
|
|
24860
|
+
parts.push("-d", shellSingleQuote(action.description));
|
|
24861
|
+
}
|
|
24862
|
+
if (action.x != null)
|
|
24863
|
+
parts.push("--x", String(Math.round(action.x)));
|
|
24864
|
+
if (action.y != null)
|
|
24865
|
+
parts.push("--y", String(Math.round(action.y)));
|
|
24866
|
+
if (action.x2 != null)
|
|
24867
|
+
parts.push("--x2", String(Math.round(action.x2)));
|
|
24868
|
+
if (action.y2 != null)
|
|
24869
|
+
parts.push("--y2", String(Math.round(action.y2)));
|
|
24870
|
+
if (action.durationMs != null)
|
|
24871
|
+
parts.push("--duration", String(Math.round(action.durationMs)));
|
|
24872
|
+
if (action.double)
|
|
24873
|
+
parts.push("--double");
|
|
24874
|
+
if (action.text != null)
|
|
24875
|
+
parts.push("--text", shellSingleQuote(action.text));
|
|
24876
|
+
if (action.appId != null)
|
|
24877
|
+
parts.push("--app-id", shellSingleQuote(action.appId));
|
|
24878
|
+
if (action.url != null)
|
|
24879
|
+
parts.push("--url", shellSingleQuote(action.url));
|
|
24880
|
+
if (action.seconds != null)
|
|
24881
|
+
parts.push("--seconds", String(action.seconds));
|
|
24882
|
+
if (action.kind === "alert") {
|
|
24883
|
+
if (action.alertAction === "dismiss")
|
|
24884
|
+
parts.push("--dismiss");
|
|
24885
|
+
}
|
|
24886
|
+
return parts.join(" ");
|
|
24887
|
+
}
|
|
24888
|
+
function formatSleepShellLine(seconds) {
|
|
24889
|
+
const safe = Math.max(0, seconds);
|
|
24890
|
+
const rounded = Number.isInteger(safe) ? String(safe) : String(Number(safe.toFixed(3)));
|
|
24891
|
+
return `sleep ${rounded}`;
|
|
24892
|
+
}
|
|
24893
|
+
function formatAssertShellLine(input) {
|
|
24894
|
+
const parts = ["yoqa", "assert", input.assertion, "--text", shellSingleQuote(input.text.trim())];
|
|
24895
|
+
const timeout = input.timeoutSeconds ?? 5;
|
|
24896
|
+
if (timeout !== 5) {
|
|
24897
|
+
parts.push("--timeout", String(timeout));
|
|
24898
|
+
}
|
|
24899
|
+
return parts.join(" ");
|
|
24900
|
+
}
|
|
24739
24901
|
var DEFAULT_SHELL_SCRIPT_HEADER = [
|
|
24740
24902
|
"#!/usr/bin/env bash",
|
|
24741
24903
|
"# Yoqa inspector script",
|
|
@@ -25163,9 +25325,16 @@ function normalizeVisionJson(value) {
|
|
|
25163
25325
|
out.x = clampNormCoord(out.x);
|
|
25164
25326
|
if (typeof out.y === "number")
|
|
25165
25327
|
out.y = clampNormCoord(out.y);
|
|
25328
|
+
if (typeof out.x2 === "number")
|
|
25329
|
+
out.x2 = clampNormCoord(out.x2);
|
|
25330
|
+
if (typeof out.y2 === "number")
|
|
25331
|
+
out.y2 = clampNormCoord(out.y2);
|
|
25166
25332
|
if (typeof out.ms === "number" && Number.isFinite(out.ms)) {
|
|
25167
25333
|
out.ms = Math.min(1e4, Math.max(0, out.ms));
|
|
25168
25334
|
}
|
|
25335
|
+
if (typeof out.durationMs === "number" && Number.isFinite(out.durationMs)) {
|
|
25336
|
+
out.durationMs = Math.min(5000, Math.max(50, Math.round(out.durationMs)));
|
|
25337
|
+
}
|
|
25169
25338
|
const reason = typeof out.reason === "string" ? out.reason.trim() : "";
|
|
25170
25339
|
const thoughts = typeof out.thoughts === "string" ? out.thoughts.trim() : "";
|
|
25171
25340
|
if (reason)
|
|
@@ -96053,26 +96222,257 @@ async function completeVision(auth, input) {
|
|
|
96053
96222
|
}
|
|
96054
96223
|
|
|
96055
96224
|
// src/domains/runs/agent.ts
|
|
96225
|
+
var swipeDirectionSchema = exports_external.union([
|
|
96226
|
+
exports_external.literal("up"),
|
|
96227
|
+
exports_external.literal("down"),
|
|
96228
|
+
exports_external.literal("left"),
|
|
96229
|
+
exports_external.literal("right")
|
|
96230
|
+
]);
|
|
96056
96231
|
var agentDecisionSchema = exports_external.object({
|
|
96057
96232
|
type: exports_external.union([
|
|
96058
96233
|
exports_external.literal("tap"),
|
|
96234
|
+
exports_external.literal("swipe"),
|
|
96235
|
+
exports_external.literal("drag"),
|
|
96059
96236
|
exports_external.literal("type"),
|
|
96237
|
+
exports_external.literal("input"),
|
|
96060
96238
|
exports_external.literal("wait"),
|
|
96061
96239
|
exports_external.literal("alert"),
|
|
96240
|
+
exports_external.literal("activate-app"),
|
|
96241
|
+
exports_external.literal("terminate-app"),
|
|
96242
|
+
exports_external.literal("restart-app"),
|
|
96243
|
+
exports_external.literal("background-app"),
|
|
96244
|
+
exports_external.literal("open-url"),
|
|
96245
|
+
exports_external.literal("assert"),
|
|
96062
96246
|
exports_external.literal("verify"),
|
|
96063
96247
|
exports_external.literal("done"),
|
|
96064
96248
|
exports_external.literal("fail")
|
|
96065
96249
|
]),
|
|
96066
96250
|
x: exports_external.number().optional(),
|
|
96067
96251
|
y: exports_external.number().optional(),
|
|
96252
|
+
x2: exports_external.number().optional(),
|
|
96253
|
+
y2: exports_external.number().optional(),
|
|
96254
|
+
direction: swipeDirectionSchema.optional(),
|
|
96255
|
+
durationMs: exports_external.number().optional(),
|
|
96256
|
+
double: exports_external.boolean().optional(),
|
|
96068
96257
|
label: exports_external.string().min(1).optional(),
|
|
96069
96258
|
id: exports_external.string().min(1).optional(),
|
|
96259
|
+
description: exports_external.string().min(1).optional(),
|
|
96070
96260
|
text: exports_external.string().optional(),
|
|
96071
96261
|
ms: exports_external.number().optional(),
|
|
96072
96262
|
alertAction: exports_external.union([exports_external.literal("accept"), exports_external.literal("dismiss")]).optional(),
|
|
96263
|
+
appId: exports_external.string().min(1).optional(),
|
|
96264
|
+
url: exports_external.string().min(1).optional(),
|
|
96265
|
+
seconds: exports_external.number().optional(),
|
|
96266
|
+
assertion: exports_external.union([exports_external.literal("visible"), exports_external.literal("not-visible")]).optional(),
|
|
96267
|
+
timeoutMs: exports_external.number().optional(),
|
|
96073
96268
|
reason: exports_external.string().min(1),
|
|
96074
96269
|
thoughts: exports_external.string().min(1)
|
|
96270
|
+
}).superRefine((value, ctx) => {
|
|
96271
|
+
if (value.type === "swipe") {
|
|
96272
|
+
const hasCoords = value.x != null && value.y != null && value.x2 != null && value.y2 != null;
|
|
96273
|
+
if (!hasCoords && value.direction == null) {
|
|
96274
|
+
ctx.addIssue({
|
|
96275
|
+
code: exports_external.ZodIssueCode.custom,
|
|
96276
|
+
message: "swipe requires direction or x,y,x2,y2"
|
|
96277
|
+
});
|
|
96278
|
+
}
|
|
96279
|
+
}
|
|
96280
|
+
if (value.type === "drag") {
|
|
96281
|
+
if (value.x == null || value.y == null || value.x2 == null || value.y2 == null) {
|
|
96282
|
+
ctx.addIssue({
|
|
96283
|
+
code: exports_external.ZodIssueCode.custom,
|
|
96284
|
+
message: "drag requires x,y,x2,y2"
|
|
96285
|
+
});
|
|
96286
|
+
}
|
|
96287
|
+
}
|
|
96288
|
+
if ((value.type === "type" || value.type === "input") && !value.text?.trim()) {
|
|
96289
|
+
ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "type/input requires text" });
|
|
96290
|
+
}
|
|
96291
|
+
if (value.type === "open-url" && !value.url?.trim()) {
|
|
96292
|
+
ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "open-url requires url" });
|
|
96293
|
+
}
|
|
96294
|
+
if (value.type === "assert" && !value.text?.trim()) {
|
|
96295
|
+
ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "assert requires text" });
|
|
96296
|
+
}
|
|
96075
96297
|
});
|
|
96298
|
+
var FINGER_SWIPE_NORMS = {
|
|
96299
|
+
up: { x: 500, y: 800, x2: 500, y2: 200 },
|
|
96300
|
+
down: { x: 500, y: 200, x2: 500, y2: 800 },
|
|
96301
|
+
left: { x: 800, y: 500, x2: 200, y2: 500 },
|
|
96302
|
+
right: { x: 200, y: 500, x2: 800, y2: 500 }
|
|
96303
|
+
};
|
|
96304
|
+
function resolveSwipeNorm(decision) {
|
|
96305
|
+
if (decision.x != null && decision.y != null && decision.x2 != null && decision.y2 != null) {
|
|
96306
|
+
return { x: decision.x, y: decision.y, x2: decision.x2, y2: decision.y2 };
|
|
96307
|
+
}
|
|
96308
|
+
if (decision.direction)
|
|
96309
|
+
return FINGER_SWIPE_NORMS[decision.direction];
|
|
96310
|
+
return null;
|
|
96311
|
+
}
|
|
96312
|
+
var MAX_SCREEN_SNAPSHOT_ROWS = 80;
|
|
96313
|
+
function padNorm(n) {
|
|
96314
|
+
return String(Math.round(n)).padStart(4, " ");
|
|
96315
|
+
}
|
|
96316
|
+
function formatScreenSnapshot(elements) {
|
|
96317
|
+
if (elements == null)
|
|
96318
|
+
return "(screen tree unavailable)";
|
|
96319
|
+
if (elements.length === 0)
|
|
96320
|
+
return "(empty tree)";
|
|
96321
|
+
const ordered = [...elements].sort((a, b) => a.y - b.y || a.x - b.x);
|
|
96322
|
+
const shown = ordered.slice(0, MAX_SCREEN_SNAPSHOT_ROWS);
|
|
96323
|
+
const lines = shown.map((element) => {
|
|
96324
|
+
const id = element.id?.trim() ? ` id=${element.id.trim()}` : "";
|
|
96325
|
+
const label = element.label?.trim() || "(no label)";
|
|
96326
|
+
return `${padNorm(element.x)},${padNorm(element.y)} ${Math.round(element.width)}x${Math.round(element.height)}${id} ${label}`;
|
|
96327
|
+
});
|
|
96328
|
+
if (ordered.length > shown.length) {
|
|
96329
|
+
lines.push(`\u2026 ${ordered.length - shown.length} more elements omitted`);
|
|
96330
|
+
}
|
|
96331
|
+
return lines.join(`
|
|
96332
|
+
`);
|
|
96333
|
+
}
|
|
96334
|
+
function decisionToActionRequest(decision, options = {}) {
|
|
96335
|
+
const appId = decision.appId?.trim() || options.defaultAppId?.trim() || undefined;
|
|
96336
|
+
switch (decision.type) {
|
|
96337
|
+
case "tap": {
|
|
96338
|
+
const body = { kind: "tap" };
|
|
96339
|
+
if (decision.double)
|
|
96340
|
+
body.double = true;
|
|
96341
|
+
if (decision.durationMs != null)
|
|
96342
|
+
body.durationMs = decision.durationMs;
|
|
96343
|
+
if (decision.description)
|
|
96344
|
+
body.description = decision.description;
|
|
96345
|
+
if (prefersScreenshotTap(decision)) {
|
|
96346
|
+
body.x = decision.x;
|
|
96347
|
+
body.y = decision.y;
|
|
96348
|
+
} else if (decision.id) {
|
|
96349
|
+
body.id = decision.id;
|
|
96350
|
+
if (decision.label)
|
|
96351
|
+
body.label = decision.label;
|
|
96352
|
+
} else if (decision.label) {
|
|
96353
|
+
body.label = decision.label;
|
|
96354
|
+
} else if (decision.description) {} else {
|
|
96355
|
+
body.x = decision.x ?? 500;
|
|
96356
|
+
body.y = decision.y ?? 500;
|
|
96357
|
+
}
|
|
96358
|
+
return body;
|
|
96359
|
+
}
|
|
96360
|
+
case "swipe": {
|
|
96361
|
+
const swipe = resolveSwipeNorm(decision);
|
|
96362
|
+
if (!swipe)
|
|
96363
|
+
return null;
|
|
96364
|
+
return {
|
|
96365
|
+
kind: "swipe",
|
|
96366
|
+
x: swipe.x,
|
|
96367
|
+
y: swipe.y,
|
|
96368
|
+
x2: swipe.x2,
|
|
96369
|
+
y2: swipe.y2,
|
|
96370
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {}
|
|
96371
|
+
};
|
|
96372
|
+
}
|
|
96373
|
+
case "drag":
|
|
96374
|
+
if (decision.x == null || decision.y == null || decision.x2 == null || decision.y2 == null) {
|
|
96375
|
+
return null;
|
|
96376
|
+
}
|
|
96377
|
+
return {
|
|
96378
|
+
kind: "drag",
|
|
96379
|
+
x: decision.x,
|
|
96380
|
+
y: decision.y,
|
|
96381
|
+
x2: decision.x2,
|
|
96382
|
+
y2: decision.y2,
|
|
96383
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {}
|
|
96384
|
+
};
|
|
96385
|
+
case "type":
|
|
96386
|
+
case "input": {
|
|
96387
|
+
const body = { kind: "input", text: decision.text ?? "" };
|
|
96388
|
+
if (decision.id)
|
|
96389
|
+
body.id = decision.id;
|
|
96390
|
+
if (decision.label)
|
|
96391
|
+
body.label = decision.label;
|
|
96392
|
+
if (decision.x != null)
|
|
96393
|
+
body.x = decision.x;
|
|
96394
|
+
if (decision.y != null)
|
|
96395
|
+
body.y = decision.y;
|
|
96396
|
+
if (decision.description)
|
|
96397
|
+
body.description = decision.description;
|
|
96398
|
+
return body;
|
|
96399
|
+
}
|
|
96400
|
+
case "alert":
|
|
96401
|
+
return {
|
|
96402
|
+
kind: "alert",
|
|
96403
|
+
alertAction: decision.alertAction === "dismiss" ? "dismiss" : "accept"
|
|
96404
|
+
};
|
|
96405
|
+
case "activate-app":
|
|
96406
|
+
return appId ? { kind: "activate-app", appId } : null;
|
|
96407
|
+
case "terminate-app":
|
|
96408
|
+
return appId ? { kind: "terminate-app", appId } : null;
|
|
96409
|
+
case "restart-app":
|
|
96410
|
+
return appId ? { kind: "restart-app", appId } : null;
|
|
96411
|
+
case "background-app":
|
|
96412
|
+
return {
|
|
96413
|
+
kind: "background-app",
|
|
96414
|
+
...decision.seconds != null ? { seconds: decision.seconds } : {}
|
|
96415
|
+
};
|
|
96416
|
+
case "open-url":
|
|
96417
|
+
return decision.url ? { kind: "open-url", url: decision.url } : null;
|
|
96418
|
+
default:
|
|
96419
|
+
return null;
|
|
96420
|
+
}
|
|
96421
|
+
}
|
|
96422
|
+
var SCROLL_INTENT_RE = /\b(scroll|swipe|flick)\b/i;
|
|
96423
|
+
var SCROLL_UP_RE = /\b(scroll\s+up|back\s+up|to the top|content above)\b/i;
|
|
96424
|
+
var SCROLL_UNTIL_END_RE = /\b(scroll[\s\w]{0,48}(bottom|end)|cannot scroll|can not scroll|can't scroll|until you can'?t scroll|no longer scroll|not be able to scroll)\b/i;
|
|
96425
|
+
function looksLikeScrollIntent(text3) {
|
|
96426
|
+
return SCROLL_INTENT_RE.test(text3);
|
|
96427
|
+
}
|
|
96428
|
+
function isScrollUntilEndGoal(instructions, expectedResult) {
|
|
96429
|
+
return SCROLL_UNTIL_END_RE.test(`${instructions} ${expectedResult}`);
|
|
96430
|
+
}
|
|
96431
|
+
function inferFingerDirection(text3) {
|
|
96432
|
+
if (SCROLL_UP_RE.test(text3))
|
|
96433
|
+
return "down";
|
|
96434
|
+
if (/\bscroll\s+left\b/i.test(text3) || /\bswipe\s+left\b/i.test(text3))
|
|
96435
|
+
return "left";
|
|
96436
|
+
if (/\bscroll\s+right\b/i.test(text3) || /\bswipe\s+right\b/i.test(text3))
|
|
96437
|
+
return "right";
|
|
96438
|
+
return "up";
|
|
96439
|
+
}
|
|
96440
|
+
function asDirectionalSwipe(decision) {
|
|
96441
|
+
const text3 = `${decision.reason} ${decision.thoughts}`;
|
|
96442
|
+
return {
|
|
96443
|
+
type: "swipe",
|
|
96444
|
+
direction: inferFingerDirection(text3),
|
|
96445
|
+
reason: decision.reason,
|
|
96446
|
+
thoughts: decision.thoughts
|
|
96447
|
+
};
|
|
96448
|
+
}
|
|
96449
|
+
function coerceScrollIntentToSwipe(decision) {
|
|
96450
|
+
if (decision.type === "tap" && isSystemPermissionLabel(decision.label))
|
|
96451
|
+
return decision;
|
|
96452
|
+
if (decision.type !== "tap" && decision.type !== "wait")
|
|
96453
|
+
return decision;
|
|
96454
|
+
if (!looksLikeScrollIntent(`${decision.reason} ${decision.thoughts}`))
|
|
96455
|
+
return decision;
|
|
96456
|
+
return asDirectionalSwipe(decision);
|
|
96457
|
+
}
|
|
96458
|
+
function continueScrollingInsteadOfComplete(input) {
|
|
96459
|
+
if (input.decision.type !== "verify" && input.decision.type !== "done") {
|
|
96460
|
+
return input.decision;
|
|
96461
|
+
}
|
|
96462
|
+
const hadSwipe = input.recentActions.some((action) => action.type === "swipe");
|
|
96463
|
+
const untilEnd = isScrollUntilEndGoal(input.instructions, input.expectedResult);
|
|
96464
|
+
const claimedScroll = looksLikeScrollIntent(`${input.decision.reason} ${input.decision.thoughts}`);
|
|
96465
|
+
if (!hadSwipe && (untilEnd || claimedScroll)) {
|
|
96466
|
+
return asDirectionalSwipe(input.decision);
|
|
96467
|
+
}
|
|
96468
|
+
if (untilEnd && hadSwipe && input.lastSwipeMovedScreen) {
|
|
96469
|
+
return asDirectionalSwipe(input.decision);
|
|
96470
|
+
}
|
|
96471
|
+
return input.decision;
|
|
96472
|
+
}
|
|
96473
|
+
function screenshotFingerprint(base643) {
|
|
96474
|
+
return String(Bun.hash(base643));
|
|
96475
|
+
}
|
|
96076
96476
|
var SYSTEM_PERMISSION_LABELS = new Set([
|
|
96077
96477
|
"allow",
|
|
96078
96478
|
"don't allow",
|
|
@@ -96092,7 +96492,11 @@ function isSystemPermissionLabel(label) {
|
|
|
96092
96492
|
return SYSTEM_PERMISSION_LABELS.has(trimmed);
|
|
96093
96493
|
}
|
|
96094
96494
|
function prefersScreenshotTap(decision) {
|
|
96095
|
-
|
|
96495
|
+
if (isSystemPermissionLabel(decision.label))
|
|
96496
|
+
return false;
|
|
96497
|
+
if (decision.id?.trim())
|
|
96498
|
+
return false;
|
|
96499
|
+
return decision.x != null && decision.y != null;
|
|
96096
96500
|
}
|
|
96097
96501
|
var SYSTEM_PROMPT = `You are a mobile QA agent controlling an app via screenshots.
|
|
96098
96502
|
A screenshot of the current device screen is ALWAYS attached to the user message as an image. You can see it. Never claim that no screenshot was provided, missing, blank, or unavailable.
|
|
@@ -96106,20 +96510,51 @@ Valid shapes:
|
|
|
96106
96510
|
{"type":"tap","label":"Allow","reason":"...","thoughts":"..."}
|
|
96107
96511
|
{"type":"tap","id":"permission_allow_button","reason":"...","thoughts":"..."}
|
|
96108
96512
|
{"type":"tap","x":0-1000,"y":0-1000,"reason":"...","thoughts":"..."}
|
|
96513
|
+
{"type":"tap","description":"the blue Login button","reason":"...","thoughts":"..."}
|
|
96514
|
+
{"type":"tap","x":0-1000,"y":0-1000,"double":true,"reason":"...","thoughts":"..."}
|
|
96515
|
+
{"type":"tap","x":0-1000,"y":0-1000,"durationMs":2000,"reason":"...","thoughts":"..."}
|
|
96516
|
+
{"type":"swipe","direction":"up|down|left|right","reason":"...","thoughts":"..."}
|
|
96517
|
+
{"type":"swipe","x":0-1000,"y":0-1000,"x2":0-1000,"y2":0-1000,"reason":"...","thoughts":"..."}
|
|
96518
|
+
{"type":"drag","x":0-1000,"y":0-1000,"x2":0-1000,"y2":0-1000,"reason":"...","thoughts":"..."}
|
|
96519
|
+
{"type":"type","text":"...","x":0-1000,"y":0-1000,"reason":"...","thoughts":"..."}
|
|
96520
|
+
{"type":"input","text":"...","id":"email_field","reason":"...","thoughts":"..."}
|
|
96109
96521
|
{"type":"alert","alertAction":"accept","reason":"...","thoughts":"..."}
|
|
96110
|
-
{"type":"
|
|
96522
|
+
{"type":"activate-app","appId":"com.example.app","reason":"...","thoughts":"..."}
|
|
96523
|
+
{"type":"terminate-app","appId":"com.example.app","reason":"...","thoughts":"..."}
|
|
96524
|
+
{"type":"restart-app","appId":"com.example.app","reason":"...","thoughts":"..."}
|
|
96525
|
+
{"type":"background-app","seconds":3,"reason":"...","thoughts":"..."}
|
|
96526
|
+
{"type":"open-url","url":"https://example.com","reason":"...","thoughts":"..."}
|
|
96527
|
+
{"type":"assert","assertion":"visible|not-visible","text":"...","reason":"...","thoughts":"..."}
|
|
96111
96528
|
{"type":"wait","ms":500-3000,"reason":"...","thoughts":"..."}
|
|
96112
96529
|
{"type":"verify","reason":"...","thoughts":"..."}
|
|
96113
96530
|
{"type":"done","reason":"...","thoughts":"..."}
|
|
96114
96531
|
{"type":"fail","reason":"...","thoughts":"..."}
|
|
96115
96532
|
|
|
96116
|
-
Coordinates use a 0\u20131000 normalized grid (0,0 top-left of the attached screenshot).
|
|
96533
|
+
Coordinates use a 0\u20131000 normalized grid (0,0 top-left of the attached screenshot). The user message also includes a screen snapshot (cleaned accessibility tree) with the same grid \u2014 x,y is each element's TOP-LEFT, not centre. Centre = x + width/2, y + height/2.
|
|
96534
|
+
|
|
96535
|
+
Use BOTH the screenshot and the screen snapshot:
|
|
96536
|
+
- Prefer snapshot "id=\u2026" when present (same as yoqa action tap --id).
|
|
96537
|
+
- System permission / notification dialogs: {"type":"tap","label":"Allow"} or {"type":"alert","alertAction":"accept"}. Never guess coordinates for these.
|
|
96538
|
+
- For other in-app controls, tap screenshot x,y of the visible control if there is no id.
|
|
96539
|
+
- If the snapshot is empty or the control is only drawn (no label/id), use the screenshot.
|
|
96540
|
+
- If a previous action failed (see Last action error), pick a different target from the snapshot + screenshot \u2014 do not repeat the same id/label/point.
|
|
96541
|
+
|
|
96542
|
+
Swipe direction is the FINGER movement (same as Inspector):
|
|
96543
|
+
- "down" (finger down, y 200\u2192800): scroll UP \u2014 reveal content above
|
|
96544
|
+
- "up" (finger up, y 800\u2192200): scroll DOWN \u2014 reveal content below / to the bottom
|
|
96545
|
+
- "left" / "right": horizontal paging
|
|
96546
|
+
When the case says "scroll up", emit direction "down". When it says "scroll down" or "to the bottom", emit direction "up". Never tap to scroll. Never emit wait as a stand-in for scrolling.
|
|
96117
96547
|
|
|
96118
96548
|
Rules:
|
|
96119
|
-
- Look at the attached screenshot
|
|
96120
|
-
-
|
|
96121
|
-
-
|
|
96122
|
-
-
|
|
96549
|
+
- Look at the attached screenshot AND the screen snapshot before deciding.
|
|
96550
|
+
- Double-tap: {"type":"tap","double":true,...}. Long-press: {"type":"tap","durationMs":2000,...}. Drag a slider/chip with type drag, not swipe.
|
|
96551
|
+
- activate-app / terminate-app / restart-app / open-url / background-app match yoqa action. Omit appId to use the catalog app id provided in the user message.
|
|
96552
|
+
- Assert text on the tree with {"type":"assert","assertion":"visible","text":"\u2026"}.
|
|
96553
|
+
- To scroll a list or page, emit swipe \u2014 never a tap in the middle of the screen as a stand-in for scrolling.
|
|
96554
|
+
- "Scroll until you cannot scroll anymore" / "to the bottom": keep emitting swipe (direction "up") until a later screenshot shows the same content. Then verify. Unchanged screens after taps do not count \u2014 taps do not scroll.
|
|
96555
|
+
- You are given ONE current instruction. Complete only that instruction, then return verify or done. Later instructions are hidden on purpose \u2014 do not invent, skip ahead, or combine them.
|
|
96556
|
+
- Do not open unrelated tabs or screens (Wallet, Profile, \u2026) unless the current instruction says to. If the current target is not visible, scroll THIS screen to find it. If it cannot be found, fail with what you see.
|
|
96557
|
+
- When THIS instruction's expected result is visible (or the action is done and the UI has updated), return verify or done. That finishes this instruction only, not the whole case.
|
|
96123
96558
|
- Splash / loading / blank screens: return wait (do not tap imaginary tabs).
|
|
96124
96559
|
- Do not repeat the same tap target from Recent actions unless the screenshot still shows you are on the wrong screen.
|
|
96125
96560
|
- On fail, say what you see (e.g. "splash logo only, no bottom tabs") \u2014 never "no screenshot".`;
|
|
@@ -96137,12 +96572,30 @@ function formatRecentActions(actions) {
|
|
|
96137
96572
|
}
|
|
96138
96573
|
return `${n}. tap (${action.x ?? "?"},${action.y ?? "?"}) \u2014 ${action.reason ?? ""}`;
|
|
96139
96574
|
}
|
|
96575
|
+
if (action.type === "swipe") {
|
|
96576
|
+
if (action.direction) {
|
|
96577
|
+
return `${n}. swipe ${action.direction} \u2014 ${action.reason ?? ""}`;
|
|
96578
|
+
}
|
|
96579
|
+
return `${n}. swipe (${action.x ?? "?"},${action.y ?? "?"})\u2192(${action.x2 ?? "?"},${action.y2 ?? "?"}) \u2014 ${action.reason ?? ""}`;
|
|
96580
|
+
}
|
|
96140
96581
|
if (action.type === "alert") {
|
|
96141
96582
|
return `${n}. alert ${action.alertAction ?? "accept"} \u2014 ${action.reason ?? ""}`;
|
|
96142
96583
|
}
|
|
96143
|
-
if (action.type === "type") {
|
|
96584
|
+
if (action.type === "type" || action.type === "input") {
|
|
96144
96585
|
return `${n}. type "${action.text ?? ""}" \u2014 ${action.reason ?? ""}`;
|
|
96145
96586
|
}
|
|
96587
|
+
if (action.type === "drag") {
|
|
96588
|
+
return `${n}. drag (${action.x ?? "?"},${action.y ?? "?"})\u2192(${action.x2 ?? "?"},${action.y2 ?? "?"}) \u2014 ${action.reason ?? ""}`;
|
|
96589
|
+
}
|
|
96590
|
+
if (action.type === "assert") {
|
|
96591
|
+
return `${n}. assert ${action.assertion ?? "visible"} "${action.text ?? ""}" \u2014 ${action.reason ?? ""}`;
|
|
96592
|
+
}
|
|
96593
|
+
if (action.type === "activate-app" || action.type === "terminate-app" || action.type === "restart-app") {
|
|
96594
|
+
return `${n}. ${action.type} ${action.appId ?? ""} \u2014 ${action.reason ?? ""}`;
|
|
96595
|
+
}
|
|
96596
|
+
if (action.type === "open-url") {
|
|
96597
|
+
return `${n}. open-url ${action.url ?? ""} \u2014 ${action.reason ?? ""}`;
|
|
96598
|
+
}
|
|
96146
96599
|
if (action.type === "wait") {
|
|
96147
96600
|
return `${n}. wait ${action.ms ?? "?"}ms \u2014 ${action.reason ?? ""}`;
|
|
96148
96601
|
}
|
|
@@ -96150,29 +96603,101 @@ function formatRecentActions(actions) {
|
|
|
96150
96603
|
}).join(`
|
|
96151
96604
|
`);
|
|
96152
96605
|
}
|
|
96153
|
-
|
|
96154
|
-
|
|
96155
|
-
|
|
96156
|
-
|
|
96157
|
-
|
|
96606
|
+
var LIST_ITEM_START_RE = /^(?:\d+[.)]|\d+\s*[-\u2013]|[-*\u2022])\s+/;
|
|
96607
|
+
function splitInstructionSteps(instructions) {
|
|
96608
|
+
const trimmed = instructions.replace(/\r\n/g, `
|
|
96609
|
+
`).trim();
|
|
96610
|
+
if (!trimmed)
|
|
96611
|
+
return [""];
|
|
96612
|
+
const lines = trimmed.split(`
|
|
96613
|
+
`);
|
|
96614
|
+
const items = [];
|
|
96615
|
+
let current = [];
|
|
96616
|
+
let sawList = false;
|
|
96617
|
+
const pushCurrent = () => {
|
|
96618
|
+
const text3 = current.join(" ").replace(/\s+/g, " ").trim();
|
|
96619
|
+
if (text3)
|
|
96620
|
+
items.push(text3);
|
|
96621
|
+
current = [];
|
|
96622
|
+
};
|
|
96623
|
+
for (const rawLine of lines) {
|
|
96624
|
+
const line = rawLine.trim();
|
|
96625
|
+
if (!line)
|
|
96626
|
+
continue;
|
|
96627
|
+
if (LIST_ITEM_START_RE.test(line)) {
|
|
96628
|
+
sawList = true;
|
|
96629
|
+
pushCurrent();
|
|
96630
|
+
current = [line.replace(LIST_ITEM_START_RE, "").trim()];
|
|
96631
|
+
} else if (current.length > 0) {
|
|
96632
|
+
current.push(line);
|
|
96633
|
+
} else {
|
|
96634
|
+
current = [line];
|
|
96635
|
+
}
|
|
96636
|
+
}
|
|
96637
|
+
pushCurrent();
|
|
96638
|
+
if (!sawList)
|
|
96639
|
+
return [trimmed];
|
|
96640
|
+
return items.length > 0 ? items : [trimmed];
|
|
96641
|
+
}
|
|
96642
|
+
function flattenCaseInstructions(flows2) {
|
|
96643
|
+
const queue = [];
|
|
96644
|
+
for (const flow of flows2) {
|
|
96645
|
+
const steps = splitInstructionSteps(flow.instructions);
|
|
96646
|
+
const usable = steps.length > 0 ? steps : [""];
|
|
96647
|
+
for (let i = 0;i < usable.length; i++) {
|
|
96648
|
+
const last = i === usable.length - 1;
|
|
96649
|
+
queue.push({
|
|
96650
|
+
instructions: usable[i] ?? "",
|
|
96651
|
+
expectedResult: last ? flow.expectedResult : ""
|
|
96652
|
+
});
|
|
96653
|
+
}
|
|
96654
|
+
}
|
|
96655
|
+
return queue.length > 0 ? queue : [{ instructions: "", expectedResult: "" }];
|
|
96158
96656
|
}
|
|
96159
|
-
|
|
96160
|
-
|
|
96657
|
+
function formatCompletedInstructions(completed) {
|
|
96658
|
+
if (completed.length === 0)
|
|
96659
|
+
return "(none \u2014 this is the first instruction)";
|
|
96660
|
+
return completed.map((text3, index) => `${index + 1}. ${text3}`).join(`
|
|
96661
|
+
`);
|
|
96662
|
+
}
|
|
96663
|
+
function formatDecidePrompt(input) {
|
|
96664
|
+
const ordinal = input.instructionOrdinal ?? input.stepIndex + 1;
|
|
96665
|
+
const total = input.instructionCount ?? 0;
|
|
96666
|
+
const remaining = total > 0 ? Math.max(0, total - ordinal) : 0;
|
|
96667
|
+
const expected = input.expectedResult.trim() ? input.expectedResult : "(none for this instruction \u2014 return verify once you have performed it and the UI has updated)";
|
|
96668
|
+
const progress = total > 0 ? `Instruction ${ordinal} of ${total} \u2014 do ONLY this instruction${remaining > 0 ? `; ${remaining} remain after it` : " (last instruction)"}` : `Step ${input.stepIndex + 1}`;
|
|
96669
|
+
const later = remaining > 0 ? `Later instructions exist (${remaining}) but are hidden. Do not invent or perform them. Do not open unrelated screens to skip ahead.` : "This is the last instruction of the case.";
|
|
96670
|
+
return [
|
|
96161
96671
|
`App context: ${input.appContext || "(none)"}`,
|
|
96162
96672
|
`Test case: ${input.caseTitle}`,
|
|
96163
|
-
|
|
96164
|
-
|
|
96165
|
-
|
|
96673
|
+
progress,
|
|
96674
|
+
"Completed instructions (already done \u2014 do not repeat):",
|
|
96675
|
+
formatCompletedInstructions(input.completedInstructions ?? []),
|
|
96676
|
+
`Current instruction (do ONLY this): ${input.instructions || "(none)"}`,
|
|
96677
|
+
`Expected result for this instruction: ${expected}`,
|
|
96678
|
+
later,
|
|
96679
|
+
`Catalog app id (activate/terminate/restart): ${input.defaultAppId || "(unknown)"}`,
|
|
96166
96680
|
`Recent actions:
|
|
96167
96681
|
${formatRecentActions(input.recentActions ?? [])}`,
|
|
96168
|
-
|
|
96682
|
+
`Last action error: ${input.lastError || "(none)"}`,
|
|
96683
|
+
"Screen snapshot (cleaned accessibility tree, 0\u20131000 grid, x,y is top-left):",
|
|
96684
|
+
input.screenSnapshot || "(unavailable)",
|
|
96685
|
+
"Look at the attached screenshot image AND the screen snapshot and decide the next action. A screenshot is attached.",
|
|
96169
96686
|
'Reply with ONLY the JSON action object, including non-empty "reason" and "thoughts".'
|
|
96170
96687
|
].join(`
|
|
96171
96688
|
`);
|
|
96689
|
+
}
|
|
96690
|
+
function isAbsurdNoScreenshotFail(decision) {
|
|
96691
|
+
if (decision.type !== "fail")
|
|
96692
|
+
return false;
|
|
96693
|
+
const text3 = `${decision.reason} ${decision.thoughts}`.toLowerCase();
|
|
96694
|
+
return text3.includes("no screenshot") || text3.includes("screenshot provided") || text3.includes("screenshot was not") || text3.includes("without a screenshot") || text3.includes("missing screenshot") || text3.includes("screenshot unavailable") || text3.includes("cannot see the screen") || text3.includes("no image");
|
|
96695
|
+
}
|
|
96696
|
+
async function decideNextAction(input) {
|
|
96172
96697
|
return completeVision(input.auth, {
|
|
96173
96698
|
schema: agentDecisionSchema,
|
|
96174
96699
|
system: SYSTEM_PROMPT,
|
|
96175
|
-
prompt,
|
|
96700
|
+
prompt: formatDecidePrompt(input),
|
|
96176
96701
|
imageBase64: input.imageBase64
|
|
96177
96702
|
});
|
|
96178
96703
|
}
|
|
@@ -96187,18 +96712,24 @@ function buildScriptFromDecisions(decisions, sourceRunId) {
|
|
|
96187
96712
|
type: "tap",
|
|
96188
96713
|
x: decision.x,
|
|
96189
96714
|
y: decision.y,
|
|
96715
|
+
...decision.double ? { double: true } : {},
|
|
96716
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
|
|
96190
96717
|
reason: decision.reason
|
|
96191
96718
|
});
|
|
96192
96719
|
} else if (decision.label) {
|
|
96193
96720
|
actions.push({
|
|
96194
96721
|
type: "tap",
|
|
96195
96722
|
label: decision.label,
|
|
96723
|
+
...decision.double ? { double: true } : {},
|
|
96724
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
|
|
96196
96725
|
reason: decision.reason
|
|
96197
96726
|
});
|
|
96198
96727
|
} else if (decision.id) {
|
|
96199
96728
|
actions.push({
|
|
96200
96729
|
type: "tap",
|
|
96201
96730
|
id: decision.id,
|
|
96731
|
+
...decision.double ? { double: true } : {},
|
|
96732
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
|
|
96202
96733
|
reason: decision.reason
|
|
96203
96734
|
});
|
|
96204
96735
|
} else {
|
|
@@ -96209,12 +96740,70 @@ function buildScriptFromDecisions(decisions, sourceRunId) {
|
|
|
96209
96740
|
reason: decision.reason
|
|
96210
96741
|
});
|
|
96211
96742
|
}
|
|
96212
|
-
} else if (decision.type === "
|
|
96743
|
+
} else if (decision.type === "swipe") {
|
|
96744
|
+
const swipe = resolveSwipeNorm(decision);
|
|
96745
|
+
if (!swipe)
|
|
96746
|
+
continue;
|
|
96747
|
+
actions.push({
|
|
96748
|
+
type: "swipe",
|
|
96749
|
+
x: swipe.x,
|
|
96750
|
+
y: swipe.y,
|
|
96751
|
+
x2: swipe.x2,
|
|
96752
|
+
y2: swipe.y2,
|
|
96753
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
|
|
96754
|
+
reason: decision.reason
|
|
96755
|
+
});
|
|
96756
|
+
} else if (decision.type === "type" || decision.type === "input") {
|
|
96213
96757
|
actions.push({
|
|
96214
96758
|
type: "type",
|
|
96215
96759
|
text: decision.text ?? "",
|
|
96216
96760
|
reason: decision.reason
|
|
96217
96761
|
});
|
|
96762
|
+
} else if (decision.type === "drag") {
|
|
96763
|
+
if (decision.x == null || decision.y == null || decision.x2 == null || decision.y2 == null) {
|
|
96764
|
+
continue;
|
|
96765
|
+
}
|
|
96766
|
+
actions.push({
|
|
96767
|
+
type: "drag",
|
|
96768
|
+
x: decision.x,
|
|
96769
|
+
y: decision.y,
|
|
96770
|
+
x2: decision.x2,
|
|
96771
|
+
y2: decision.y2,
|
|
96772
|
+
...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
|
|
96773
|
+
reason: decision.reason
|
|
96774
|
+
});
|
|
96775
|
+
} else if (decision.type === "activate-app" || decision.type === "terminate-app" || decision.type === "restart-app") {
|
|
96776
|
+
if (!decision.appId)
|
|
96777
|
+
continue;
|
|
96778
|
+
actions.push({
|
|
96779
|
+
type: decision.type,
|
|
96780
|
+
appId: decision.appId,
|
|
96781
|
+
reason: decision.reason
|
|
96782
|
+
});
|
|
96783
|
+
} else if (decision.type === "background-app") {
|
|
96784
|
+
actions.push({
|
|
96785
|
+
type: "background-app",
|
|
96786
|
+
...decision.seconds != null ? { seconds: decision.seconds } : {},
|
|
96787
|
+
reason: decision.reason
|
|
96788
|
+
});
|
|
96789
|
+
} else if (decision.type === "open-url") {
|
|
96790
|
+
if (!decision.url)
|
|
96791
|
+
continue;
|
|
96792
|
+
actions.push({
|
|
96793
|
+
type: "open-url",
|
|
96794
|
+
url: decision.url,
|
|
96795
|
+
reason: decision.reason
|
|
96796
|
+
});
|
|
96797
|
+
} else if (decision.type === "assert") {
|
|
96798
|
+
if (!decision.text?.trim())
|
|
96799
|
+
continue;
|
|
96800
|
+
actions.push({
|
|
96801
|
+
type: "assert",
|
|
96802
|
+
assertion: decision.assertion === "not-visible" ? "not-visible" : "visible",
|
|
96803
|
+
text: decision.text,
|
|
96804
|
+
...decision.timeoutMs != null ? { timeoutMs: decision.timeoutMs } : {},
|
|
96805
|
+
reason: decision.reason
|
|
96806
|
+
});
|
|
96218
96807
|
} else if (decision.type === "wait") {
|
|
96219
96808
|
actions.push({
|
|
96220
96809
|
type: "wait",
|
|
@@ -98096,8 +98685,8 @@ ${treeSummary || "(none)"}`;
|
|
|
98096
98685
|
}
|
|
98097
98686
|
|
|
98098
98687
|
// src/domains/devices/interaction.ts
|
|
98099
|
-
|
|
98100
|
-
|
|
98688
|
+
var MJPEG_PAUSE_SETTLE_MS = 150;
|
|
98689
|
+
async function readScreen(session, options) {
|
|
98101
98690
|
const raw2 = await session.pageSource();
|
|
98102
98691
|
const window2 = await session.getWindowSize();
|
|
98103
98692
|
if (options.full) {
|
|
@@ -98110,6 +98699,14 @@ async function getScreen(session, options = {}) {
|
|
|
98110
98699
|
elements: cleaned.elements
|
|
98111
98700
|
};
|
|
98112
98701
|
}
|
|
98702
|
+
async function getScreen(session, options = {}) {
|
|
98703
|
+
if (options.pauseMjpeg !== false) {
|
|
98704
|
+
const paused = abortAllMjpegProxies();
|
|
98705
|
+
if (paused)
|
|
98706
|
+
await Bun.sleep(MJPEG_PAUSE_SETTLE_MS);
|
|
98707
|
+
}
|
|
98708
|
+
return await readScreen(session, options);
|
|
98709
|
+
}
|
|
98113
98710
|
|
|
98114
98711
|
class ActionValidationError extends Error {
|
|
98115
98712
|
constructor(message) {
|
|
@@ -98164,7 +98761,9 @@ async function performAction(session, body) {
|
|
|
98164
98761
|
throw new ActionValidationError(`${body.kind} requires x,y,x2,y2`);
|
|
98165
98762
|
}
|
|
98166
98763
|
if (body.kind === "swipe") {
|
|
98167
|
-
await session.swipe(x, y, body.x2, body.y2, body.durationMs
|
|
98764
|
+
await session.swipe(x, y, body.x2, body.y2, body.durationMs, {
|
|
98765
|
+
coordSpace: "screenshot"
|
|
98766
|
+
});
|
|
98168
98767
|
} else {
|
|
98169
98768
|
await session.drag(x, y, body.x2, body.y2, body.durationMs);
|
|
98170
98769
|
}
|
|
@@ -98232,14 +98831,77 @@ var defaultClock = {
|
|
|
98232
98831
|
sleep: (ms) => new Promise((resolve7) => setTimeout(resolve7, ms)),
|
|
98233
98832
|
now: () => Date.now()
|
|
98234
98833
|
};
|
|
98834
|
+
var noopSetCurrentCommand = async () => {};
|
|
98835
|
+
async function withCurrentCommand(setCurrentCommand, command, work) {
|
|
98836
|
+
await setCurrentCommand(command);
|
|
98837
|
+
try {
|
|
98838
|
+
return await work();
|
|
98839
|
+
} finally {
|
|
98840
|
+
await setCurrentCommand(null);
|
|
98841
|
+
}
|
|
98842
|
+
}
|
|
98843
|
+
function isRetriableActionError(error40) {
|
|
98844
|
+
return error40 instanceof ActionNotFoundError || error40 instanceof ActionValidationError;
|
|
98845
|
+
}
|
|
98846
|
+
async function readCleanedTree(readScreen2, session) {
|
|
98847
|
+
try {
|
|
98848
|
+
const screen = await readScreen2(session);
|
|
98849
|
+
const elements = screen.elements ?? [];
|
|
98850
|
+
return { snapshot: formatScreenSnapshot(elements), elements };
|
|
98851
|
+
} catch (error40) {
|
|
98852
|
+
if (isDeadSessionError(error40))
|
|
98853
|
+
throw error40;
|
|
98854
|
+
const message = error40 instanceof Error ? error40.message : String(error40);
|
|
98855
|
+
return { snapshot: `(screen tree unavailable: ${message})`, elements: [] };
|
|
98856
|
+
}
|
|
98857
|
+
}
|
|
98858
|
+
async function runTextAssert(input) {
|
|
98859
|
+
const deadline = input.clock.now() + input.timeoutMs;
|
|
98860
|
+
for (;; ) {
|
|
98861
|
+
if (input.isAborted()) {
|
|
98862
|
+
throw new Error("Aborted");
|
|
98863
|
+
}
|
|
98864
|
+
const screen = await input.readScreen(input.session);
|
|
98865
|
+
const found = screenHasText(screen.elements, input.text);
|
|
98866
|
+
if (input.assertion === "visible" && found)
|
|
98867
|
+
return;
|
|
98868
|
+
if (input.assertion === "not-visible" && !found)
|
|
98869
|
+
return;
|
|
98870
|
+
if (input.clock.now() >= deadline) {
|
|
98871
|
+
throw new Error(input.assertion === "visible" ? `Expected visible text not found within ${Math.round(input.timeoutMs / 1000)}s: ${input.text}` : `Unexpected text still visible after ${Math.round(input.timeoutMs / 1000)}s: ${input.text}`);
|
|
98872
|
+
}
|
|
98873
|
+
await input.clock.sleep(400);
|
|
98874
|
+
}
|
|
98875
|
+
}
|
|
98876
|
+
function commandForDecision(decision, defaultAppId) {
|
|
98877
|
+
if (decision.type === "verify" || decision.type === "done" || decision.type === "fail") {
|
|
98878
|
+
return null;
|
|
98879
|
+
}
|
|
98880
|
+
if (decision.type === "wait") {
|
|
98881
|
+
const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
|
|
98882
|
+
return formatSleepShellLine(waitMs / 1000);
|
|
98883
|
+
}
|
|
98884
|
+
if (decision.type === "assert") {
|
|
98885
|
+
const assertion = decision.assertion === "not-visible" ? "not-visible" : "visible";
|
|
98886
|
+
const timeoutMs = Math.min(60000, Math.max(1000, decision.timeoutMs ?? 5000));
|
|
98887
|
+
return formatAssertShellLine({
|
|
98888
|
+
assertion,
|
|
98889
|
+
text: decision.text ?? "",
|
|
98890
|
+
timeoutSeconds: timeoutMs / 1000
|
|
98891
|
+
});
|
|
98892
|
+
}
|
|
98893
|
+
const body = decisionToActionRequest(decision, { defaultAppId });
|
|
98894
|
+
return body ? formatActionShellLine(body) : null;
|
|
98895
|
+
}
|
|
98235
98896
|
async function executeScriptCase(deps) {
|
|
98236
98897
|
const perform = deps.performAction ?? performAction;
|
|
98237
|
-
const
|
|
98898
|
+
const readScreen2 = deps.readScreen ?? (async (session) => {
|
|
98238
98899
|
const screen = await getScreen(session, { full: false });
|
|
98239
98900
|
return { elements: screen.elements };
|
|
98240
98901
|
});
|
|
98241
98902
|
const clock = deps.clock ?? defaultClock;
|
|
98242
98903
|
const settleMs = deps.settleMs ?? POST_ACTION_SETTLE_MS;
|
|
98904
|
+
const setCurrentCommand = deps.setCurrentCommand ?? noopSetCurrentCommand;
|
|
98243
98905
|
let stepIdx = 0;
|
|
98244
98906
|
let lastScreenshotUri = null;
|
|
98245
98907
|
try {
|
|
@@ -98264,108 +98926,274 @@ async function executeScriptCase(deps) {
|
|
|
98264
98926
|
tapBody.x = action.x;
|
|
98265
98927
|
if (action.y != null)
|
|
98266
98928
|
tapBody.y = action.y;
|
|
98267
|
-
|
|
98268
|
-
|
|
98269
|
-
|
|
98270
|
-
|
|
98271
|
-
|
|
98272
|
-
|
|
98273
|
-
|
|
98274
|
-
|
|
98275
|
-
|
|
98276
|
-
|
|
98277
|
-
|
|
98278
|
-
|
|
98279
|
-
|
|
98280
|
-
|
|
98281
|
-
|
|
98282
|
-
|
|
98283
|
-
|
|
98929
|
+
if (action.double)
|
|
98930
|
+
tapBody.double = true;
|
|
98931
|
+
if (action.durationMs != null)
|
|
98932
|
+
tapBody.durationMs = action.durationMs;
|
|
98933
|
+
const command = formatActionShellLine(tapBody);
|
|
98934
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
98935
|
+
await perform(deps.session, tapBody);
|
|
98936
|
+
await clock.sleep(settleMs);
|
|
98937
|
+
await deps.appendStep({
|
|
98938
|
+
idx: stepIdx,
|
|
98939
|
+
action: {
|
|
98940
|
+
type: "tap",
|
|
98941
|
+
x: action.x,
|
|
98942
|
+
y: action.y,
|
|
98943
|
+
label: action.label,
|
|
98944
|
+
id: action.id,
|
|
98945
|
+
reason: action.reason ?? "Replayed saved script tap",
|
|
98946
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
98947
|
+
},
|
|
98948
|
+
screenshotUri: shot.path,
|
|
98949
|
+
ok: true,
|
|
98950
|
+
latencyMs,
|
|
98951
|
+
detail: action.reason ?? action.label ?? action.id ?? null,
|
|
98952
|
+
command
|
|
98953
|
+
});
|
|
98954
|
+
});
|
|
98955
|
+
} else if (action.type === "swipe") {
|
|
98956
|
+
const swipeBody = {
|
|
98957
|
+
kind: "swipe",
|
|
98958
|
+
x: action.x,
|
|
98959
|
+
y: action.y,
|
|
98960
|
+
x2: action.x2,
|
|
98961
|
+
y2: action.y2,
|
|
98962
|
+
...action.durationMs != null ? { durationMs: action.durationMs } : {}
|
|
98963
|
+
};
|
|
98964
|
+
const command = formatActionShellLine(swipeBody);
|
|
98965
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
98966
|
+
await perform(deps.session, swipeBody);
|
|
98967
|
+
await clock.sleep(settleMs);
|
|
98968
|
+
await deps.appendStep({
|
|
98969
|
+
idx: stepIdx,
|
|
98970
|
+
action: {
|
|
98971
|
+
type: "swipe",
|
|
98972
|
+
x: action.x,
|
|
98973
|
+
y: action.y,
|
|
98974
|
+
x2: action.x2,
|
|
98975
|
+
y2: action.y2,
|
|
98976
|
+
durationMs: action.durationMs,
|
|
98977
|
+
reason: action.reason ?? "Replayed saved script swipe",
|
|
98978
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
98979
|
+
},
|
|
98980
|
+
screenshotUri: shot.path,
|
|
98981
|
+
ok: true,
|
|
98982
|
+
latencyMs,
|
|
98983
|
+
detail: action.reason ?? null,
|
|
98984
|
+
command
|
|
98985
|
+
});
|
|
98986
|
+
});
|
|
98987
|
+
} else if (action.type === "drag") {
|
|
98988
|
+
const dragBody = {
|
|
98989
|
+
kind: "drag",
|
|
98990
|
+
x: action.x,
|
|
98991
|
+
y: action.y,
|
|
98992
|
+
x2: action.x2,
|
|
98993
|
+
y2: action.y2,
|
|
98994
|
+
...action.durationMs != null ? { durationMs: action.durationMs } : {}
|
|
98995
|
+
};
|
|
98996
|
+
const command = formatActionShellLine(dragBody);
|
|
98997
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
98998
|
+
await perform(deps.session, dragBody);
|
|
98999
|
+
await clock.sleep(settleMs);
|
|
99000
|
+
await deps.appendStep({
|
|
99001
|
+
idx: stepIdx,
|
|
99002
|
+
action: {
|
|
99003
|
+
type: "drag",
|
|
99004
|
+
x: action.x,
|
|
99005
|
+
y: action.y,
|
|
99006
|
+
x2: action.x2,
|
|
99007
|
+
y2: action.y2,
|
|
99008
|
+
durationMs: action.durationMs,
|
|
99009
|
+
reason: action.reason ?? "Replayed saved script drag",
|
|
99010
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99011
|
+
},
|
|
99012
|
+
screenshotUri: shot.path,
|
|
99013
|
+
ok: true,
|
|
99014
|
+
latencyMs,
|
|
99015
|
+
detail: action.reason ?? null,
|
|
99016
|
+
command
|
|
99017
|
+
});
|
|
99018
|
+
});
|
|
99019
|
+
} else if (action.type === "activate-app" || action.type === "terminate-app" || action.type === "restart-app") {
|
|
99020
|
+
const appBody = { kind: action.type, appId: action.appId };
|
|
99021
|
+
const command = formatActionShellLine(appBody);
|
|
99022
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99023
|
+
await perform(deps.session, appBody);
|
|
99024
|
+
await clock.sleep(settleMs);
|
|
99025
|
+
await deps.appendStep({
|
|
99026
|
+
idx: stepIdx,
|
|
99027
|
+
action: {
|
|
99028
|
+
type: action.type,
|
|
99029
|
+
appId: action.appId,
|
|
99030
|
+
reason: action.reason ?? `Replayed saved script ${action.type}`,
|
|
99031
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99032
|
+
},
|
|
99033
|
+
screenshotUri: shot.path,
|
|
99034
|
+
ok: true,
|
|
99035
|
+
latencyMs,
|
|
99036
|
+
detail: action.reason ?? action.appId,
|
|
99037
|
+
command
|
|
99038
|
+
});
|
|
99039
|
+
});
|
|
99040
|
+
} else if (action.type === "background-app") {
|
|
99041
|
+
const backgroundBody = {
|
|
99042
|
+
kind: "background-app",
|
|
99043
|
+
...action.seconds != null ? { seconds: action.seconds } : {}
|
|
99044
|
+
};
|
|
99045
|
+
const command = formatActionShellLine(backgroundBody);
|
|
99046
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99047
|
+
await perform(deps.session, backgroundBody);
|
|
99048
|
+
await clock.sleep(settleMs);
|
|
99049
|
+
await deps.appendStep({
|
|
99050
|
+
idx: stepIdx,
|
|
99051
|
+
action: {
|
|
99052
|
+
type: "background-app",
|
|
99053
|
+
seconds: action.seconds,
|
|
99054
|
+
reason: action.reason ?? "Replayed saved script background-app",
|
|
99055
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99056
|
+
},
|
|
99057
|
+
screenshotUri: shot.path,
|
|
99058
|
+
ok: true,
|
|
99059
|
+
latencyMs,
|
|
99060
|
+
detail: action.reason ?? null,
|
|
99061
|
+
command
|
|
99062
|
+
});
|
|
99063
|
+
});
|
|
99064
|
+
} else if (action.type === "open-url") {
|
|
99065
|
+
const urlBody = { kind: "open-url", url: action.url };
|
|
99066
|
+
const command = formatActionShellLine(urlBody);
|
|
99067
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99068
|
+
await perform(deps.session, urlBody);
|
|
99069
|
+
await clock.sleep(settleMs);
|
|
99070
|
+
await deps.appendStep({
|
|
99071
|
+
idx: stepIdx,
|
|
99072
|
+
action: {
|
|
99073
|
+
type: "open-url",
|
|
99074
|
+
url: action.url,
|
|
99075
|
+
reason: action.reason ?? "Replayed saved script open-url",
|
|
99076
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99077
|
+
},
|
|
99078
|
+
screenshotUri: shot.path,
|
|
99079
|
+
ok: true,
|
|
99080
|
+
latencyMs,
|
|
99081
|
+
detail: action.reason ?? action.url,
|
|
99082
|
+
command
|
|
99083
|
+
});
|
|
98284
99084
|
});
|
|
98285
99085
|
} else if (action.type === "type") {
|
|
98286
|
-
|
|
98287
|
-
|
|
98288
|
-
await
|
|
98289
|
-
|
|
98290
|
-
|
|
98291
|
-
|
|
98292
|
-
|
|
98293
|
-
|
|
98294
|
-
|
|
98295
|
-
|
|
98296
|
-
|
|
98297
|
-
|
|
98298
|
-
|
|
98299
|
-
|
|
99086
|
+
const typeBody = { kind: "input", text: action.text };
|
|
99087
|
+
const command = formatActionShellLine(typeBody);
|
|
99088
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99089
|
+
await perform(deps.session, typeBody);
|
|
99090
|
+
await clock.sleep(settleMs);
|
|
99091
|
+
await deps.appendStep({
|
|
99092
|
+
idx: stepIdx,
|
|
99093
|
+
action: {
|
|
99094
|
+
type: "type",
|
|
99095
|
+
text: action.text,
|
|
99096
|
+
reason: action.reason ?? "Replayed saved script type",
|
|
99097
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99098
|
+
},
|
|
99099
|
+
screenshotUri: shot.path,
|
|
99100
|
+
ok: true,
|
|
99101
|
+
latencyMs,
|
|
99102
|
+
detail: action.reason ?? null,
|
|
99103
|
+
command
|
|
99104
|
+
});
|
|
98300
99105
|
});
|
|
98301
99106
|
} else if (action.type === "assert") {
|
|
98302
99107
|
const timeoutMs = action.timeoutMs ?? 5000;
|
|
98303
99108
|
const assertion = action.assertion;
|
|
98304
|
-
const
|
|
98305
|
-
|
|
99109
|
+
const command = formatAssertShellLine({
|
|
99110
|
+
assertion,
|
|
99111
|
+
text: action.text,
|
|
99112
|
+
timeoutSeconds: timeoutMs / 1000
|
|
99113
|
+
});
|
|
99114
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99115
|
+
const deadline = clock.now() + timeoutMs;
|
|
99116
|
+
for (;; ) {
|
|
99117
|
+
if (deps.isAborted()) {
|
|
99118
|
+
return;
|
|
99119
|
+
}
|
|
99120
|
+
const screen = await readScreen2(deps.session);
|
|
99121
|
+
const found = screenHasText(screen.elements, action.text);
|
|
99122
|
+
if (assertion === "visible" && found)
|
|
99123
|
+
break;
|
|
99124
|
+
if (assertion === "not-visible" && !found)
|
|
99125
|
+
break;
|
|
99126
|
+
if (clock.now() >= deadline) {
|
|
99127
|
+
throw new Error(assertion === "visible" ? `Expected visible text not found within ${Math.round(timeoutMs / 1000)}s: ${action.text}` : `Unexpected text still visible after ${Math.round(timeoutMs / 1000)}s: ${action.text}`);
|
|
99128
|
+
}
|
|
99129
|
+
await clock.sleep(400);
|
|
99130
|
+
}
|
|
98306
99131
|
if (deps.isAborted()) {
|
|
98307
|
-
return
|
|
99132
|
+
return;
|
|
98308
99133
|
}
|
|
98309
|
-
|
|
98310
|
-
|
|
98311
|
-
|
|
98312
|
-
|
|
98313
|
-
|
|
98314
|
-
|
|
98315
|
-
|
|
98316
|
-
|
|
98317
|
-
|
|
98318
|
-
|
|
98319
|
-
|
|
98320
|
-
|
|
98321
|
-
|
|
98322
|
-
|
|
98323
|
-
|
|
98324
|
-
|
|
98325
|
-
text: action.text,
|
|
98326
|
-
timeoutMs,
|
|
98327
|
-
reason: action.reason ?? `Assert ${assertion}: ${action.text}`,
|
|
98328
|
-
thoughts: "Replaying the saved script without calling the AI agent."
|
|
98329
|
-
},
|
|
98330
|
-
screenshotUri: shot.path,
|
|
98331
|
-
ok: true,
|
|
98332
|
-
latencyMs,
|
|
98333
|
-
detail: action.reason ?? `${assertion}: ${action.text}`
|
|
99134
|
+
await deps.appendStep({
|
|
99135
|
+
idx: stepIdx,
|
|
99136
|
+
action: {
|
|
99137
|
+
type: "assert",
|
|
99138
|
+
assertion,
|
|
99139
|
+
text: action.text,
|
|
99140
|
+
timeoutMs,
|
|
99141
|
+
reason: action.reason ?? `Assert ${assertion}: ${action.text}`,
|
|
99142
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99143
|
+
},
|
|
99144
|
+
screenshotUri: shot.path,
|
|
99145
|
+
ok: true,
|
|
99146
|
+
latencyMs,
|
|
99147
|
+
detail: action.reason ?? `${assertion}: ${action.text}`,
|
|
99148
|
+
command
|
|
99149
|
+
});
|
|
98334
99150
|
});
|
|
99151
|
+
if (deps.isAborted()) {
|
|
99152
|
+
return "cancelled";
|
|
99153
|
+
}
|
|
98335
99154
|
} else if (action.type === "alert") {
|
|
98336
|
-
|
|
99155
|
+
const alertBody = {
|
|
98337
99156
|
kind: "alert",
|
|
98338
99157
|
alertAction: action.alertAction === "dismiss" ? "dismiss" : "accept"
|
|
98339
|
-
}
|
|
98340
|
-
|
|
98341
|
-
await
|
|
98342
|
-
|
|
98343
|
-
|
|
98344
|
-
|
|
98345
|
-
|
|
98346
|
-
|
|
98347
|
-
|
|
98348
|
-
|
|
98349
|
-
|
|
98350
|
-
|
|
98351
|
-
|
|
98352
|
-
|
|
99158
|
+
};
|
|
99159
|
+
const command = formatActionShellLine(alertBody);
|
|
99160
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99161
|
+
await perform(deps.session, alertBody);
|
|
99162
|
+
await clock.sleep(settleMs);
|
|
99163
|
+
await deps.appendStep({
|
|
99164
|
+
idx: stepIdx,
|
|
99165
|
+
action: {
|
|
99166
|
+
type: "alert",
|
|
99167
|
+
alertAction: action.alertAction === "dismiss" ? "dismiss" : "accept",
|
|
99168
|
+
reason: action.reason ?? "Replayed saved script alert",
|
|
99169
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99170
|
+
},
|
|
99171
|
+
screenshotUri: shot.path,
|
|
99172
|
+
ok: true,
|
|
99173
|
+
latencyMs,
|
|
99174
|
+
detail: action.reason ?? action.alertAction ?? "accept",
|
|
99175
|
+
command
|
|
99176
|
+
});
|
|
98353
99177
|
});
|
|
98354
99178
|
} else {
|
|
98355
99179
|
const waitMs = Math.min(3000, Math.max(500, action.ms));
|
|
98356
|
-
|
|
98357
|
-
await
|
|
98358
|
-
|
|
98359
|
-
|
|
98360
|
-
|
|
98361
|
-
|
|
98362
|
-
|
|
98363
|
-
|
|
98364
|
-
|
|
98365
|
-
|
|
98366
|
-
|
|
98367
|
-
|
|
98368
|
-
|
|
99180
|
+
const command = formatSleepShellLine(waitMs / 1000);
|
|
99181
|
+
await withCurrentCommand(setCurrentCommand, command, async () => {
|
|
99182
|
+
await clock.sleep(waitMs);
|
|
99183
|
+
await deps.appendStep({
|
|
99184
|
+
idx: stepIdx,
|
|
99185
|
+
action: {
|
|
99186
|
+
type: "wait",
|
|
99187
|
+
ms: waitMs,
|
|
99188
|
+
reason: action.reason ?? `wait ${waitMs}ms`,
|
|
99189
|
+
thoughts: "Replaying the saved script without calling the AI agent."
|
|
99190
|
+
},
|
|
99191
|
+
screenshotUri: shot.path,
|
|
99192
|
+
ok: true,
|
|
99193
|
+
latencyMs,
|
|
99194
|
+
detail: action.reason ?? `wait ${waitMs}ms`,
|
|
99195
|
+
command
|
|
99196
|
+
});
|
|
98369
99197
|
});
|
|
98370
99198
|
}
|
|
98371
99199
|
stepIdx += 1;
|
|
@@ -98380,7 +99208,8 @@ async function executeScriptCase(deps) {
|
|
|
98380
99208
|
screenshotUri: lastScreenshotUri,
|
|
98381
99209
|
ok: true,
|
|
98382
99210
|
latencyMs: 0,
|
|
98383
|
-
detail: "Saved script completed"
|
|
99211
|
+
detail: "Saved script completed",
|
|
99212
|
+
command: null
|
|
98384
99213
|
});
|
|
98385
99214
|
return "passed";
|
|
98386
99215
|
} catch (error40) {
|
|
@@ -98398,7 +99227,8 @@ async function executeScriptCase(deps) {
|
|
|
98398
99227
|
screenshotUri: lastScreenshotUri,
|
|
98399
99228
|
ok: false,
|
|
98400
99229
|
latencyMs: 0,
|
|
98401
|
-
detail: message
|
|
99230
|
+
detail: message,
|
|
99231
|
+
command: null
|
|
98402
99232
|
});
|
|
98403
99233
|
return "errored";
|
|
98404
99234
|
}
|
|
@@ -98406,165 +99236,214 @@ async function executeScriptCase(deps) {
|
|
|
98406
99236
|
async function executeAgentCase(deps) {
|
|
98407
99237
|
const decide = deps.decide ?? decideNextAction;
|
|
98408
99238
|
const perform = deps.performAction ?? performAction;
|
|
99239
|
+
const readScreen2 = deps.readScreen ?? (async (session) => {
|
|
99240
|
+
const screen = await getScreen(session, { full: false });
|
|
99241
|
+
return { elements: screen.elements };
|
|
99242
|
+
});
|
|
98409
99243
|
const clock = deps.clock ?? defaultClock;
|
|
98410
99244
|
const settleMs = deps.settleMs ?? POST_ACTION_SETTLE_MS;
|
|
98411
99245
|
const maxSteps = deps.maxStepsPerCase ?? MAX_STEPS_PER_CASE;
|
|
99246
|
+
const setCurrentCommand = deps.setCurrentCommand ?? noopSetCurrentCommand;
|
|
98412
99247
|
let stepIdx = 0;
|
|
98413
99248
|
let caseStatus = "passed";
|
|
98414
99249
|
let caseError = null;
|
|
98415
99250
|
let lastScreenshotUri = null;
|
|
98416
99251
|
const recordedDecisions = [];
|
|
99252
|
+
const decideOnce = async (input) => {
|
|
99253
|
+
const payload = {
|
|
99254
|
+
auth: deps.auth,
|
|
99255
|
+
appContext: deps.appContext,
|
|
99256
|
+
caseTitle: deps.catalogCase.name,
|
|
99257
|
+
instructions: input.flow.instructions,
|
|
99258
|
+
expectedResult: input.flow.expectedResult,
|
|
99259
|
+
stepIndex: stepIdx,
|
|
99260
|
+
imageBase64: input.imageBase64,
|
|
99261
|
+
recentActions: input.recentActions,
|
|
99262
|
+
screenSnapshot: input.screenSnapshot,
|
|
99263
|
+
lastError: input.lastError,
|
|
99264
|
+
defaultAppId: deps.defaultAppId,
|
|
99265
|
+
completedInstructions: [...input.completedInstructions],
|
|
99266
|
+
instructionOrdinal: input.instructionOrdinal,
|
|
99267
|
+
instructionCount: input.instructionCount
|
|
99268
|
+
};
|
|
99269
|
+
let decision = await decide(payload);
|
|
99270
|
+
if (isAbsurdNoScreenshotFail(decision)) {
|
|
99271
|
+
decision = await decide(payload);
|
|
99272
|
+
if (isAbsurdNoScreenshotFail(decision)) {
|
|
99273
|
+
decision = {
|
|
99274
|
+
type: "fail",
|
|
99275
|
+
reason: "Model ignored the attached screenshot (claimed none was provided). Try a stronger vision model in Settings \u2192 Provider.",
|
|
99276
|
+
thoughts: "The vision model returned fail claiming no screenshot was available even though an image was attached to the request. After one retry it still denied the screenshot, so the step was marked failed."
|
|
99277
|
+
};
|
|
99278
|
+
}
|
|
99279
|
+
}
|
|
99280
|
+
decision = coerceScrollIntentToSwipe(decision);
|
|
99281
|
+
decision = continueScrollingInsteadOfComplete({
|
|
99282
|
+
decision,
|
|
99283
|
+
instructions: input.flow.instructions,
|
|
99284
|
+
expectedResult: input.flow.expectedResult,
|
|
99285
|
+
recentActions: input.recentActions,
|
|
99286
|
+
lastSwipeMovedScreen: input.lastSwipeMovedScreen
|
|
99287
|
+
});
|
|
99288
|
+
if ((decision.type === "activate-app" || decision.type === "terminate-app" || decision.type === "restart-app") && !decision.appId && deps.defaultAppId) {
|
|
99289
|
+
decision = { ...decision, appId: deps.defaultAppId };
|
|
99290
|
+
}
|
|
99291
|
+
return decision;
|
|
99292
|
+
};
|
|
99293
|
+
const applyDecision = async (decision) => {
|
|
99294
|
+
if (decision.type === "wait") {
|
|
99295
|
+
const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
|
|
99296
|
+
await clock.sleep(waitMs);
|
|
99297
|
+
return "continue";
|
|
99298
|
+
}
|
|
99299
|
+
if (decision.type === "verify" || decision.type === "done") {
|
|
99300
|
+
return "done";
|
|
99301
|
+
}
|
|
99302
|
+
if (decision.type === "fail") {
|
|
99303
|
+
return "fail";
|
|
99304
|
+
}
|
|
99305
|
+
if (decision.type === "assert") {
|
|
99306
|
+
const assertion = decision.assertion === "not-visible" ? "not-visible" : "visible";
|
|
99307
|
+
const timeoutMs = Math.min(60000, Math.max(1000, decision.timeoutMs ?? 5000));
|
|
99308
|
+
await runTextAssert({
|
|
99309
|
+
session: deps.session,
|
|
99310
|
+
readScreen: readScreen2,
|
|
99311
|
+
clock,
|
|
99312
|
+
isAborted: deps.isAborted,
|
|
99313
|
+
assertion,
|
|
99314
|
+
text: decision.text ?? "",
|
|
99315
|
+
timeoutMs
|
|
99316
|
+
});
|
|
99317
|
+
return "continue";
|
|
99318
|
+
}
|
|
99319
|
+
const body = decisionToActionRequest(decision, { defaultAppId: deps.defaultAppId });
|
|
99320
|
+
if (!body) {
|
|
99321
|
+
throw new Error(`${decision.type} is missing required fields`);
|
|
99322
|
+
}
|
|
99323
|
+
try {
|
|
99324
|
+
await perform(deps.session, body);
|
|
99325
|
+
} catch (error40) {
|
|
99326
|
+
if (error40 instanceof ActionNotFoundError && (body.id || body.label) && decision.x != null && decision.y != null) {
|
|
99327
|
+
await perform(deps.session, {
|
|
99328
|
+
kind: body.kind === "input" ? "input" : "tap",
|
|
99329
|
+
x: decision.x,
|
|
99330
|
+
y: decision.y,
|
|
99331
|
+
text: body.text,
|
|
99332
|
+
double: body.double,
|
|
99333
|
+
durationMs: body.durationMs
|
|
99334
|
+
});
|
|
99335
|
+
} else {
|
|
99336
|
+
throw error40;
|
|
99337
|
+
}
|
|
99338
|
+
}
|
|
99339
|
+
await clock.sleep(settleMs);
|
|
99340
|
+
return "continue";
|
|
99341
|
+
};
|
|
98417
99342
|
try {
|
|
98418
|
-
const
|
|
98419
|
-
|
|
99343
|
+
const instructionQueue = flattenCaseInstructions(deps.catalogCase.flows.length > 0 ? deps.catalogCase.flows : [{ id: "empty", instructions: "", expectedResult: "", flowId: null }]);
|
|
99344
|
+
const recentActions = [];
|
|
99345
|
+
const completedInstructions = [];
|
|
99346
|
+
let prevFingerprint = null;
|
|
99347
|
+
for (let instructionIndex = 0;instructionIndex < instructionQueue.length; instructionIndex++) {
|
|
99348
|
+
const instruction = instructionQueue[instructionIndex];
|
|
99349
|
+
if (!instruction)
|
|
99350
|
+
break;
|
|
98420
99351
|
if (deps.isAborted()) {
|
|
98421
99352
|
caseStatus = "cancelled";
|
|
98422
99353
|
break;
|
|
98423
99354
|
}
|
|
98424
|
-
|
|
98425
|
-
let
|
|
98426
|
-
for (let attempt = 0;attempt < maxSteps && !flowDone; attempt++) {
|
|
99355
|
+
let instructionDone = false;
|
|
99356
|
+
for (let attempt = 0;attempt < maxSteps && !instructionDone; attempt++) {
|
|
98427
99357
|
if (deps.isAborted()) {
|
|
98428
99358
|
caseStatus = "cancelled";
|
|
98429
|
-
|
|
99359
|
+
instructionDone = true;
|
|
98430
99360
|
break;
|
|
98431
99361
|
}
|
|
98432
99362
|
const shotStarted = clock.now();
|
|
98433
99363
|
const shot = await deps.session.screenshot();
|
|
98434
99364
|
lastScreenshotUri = shot.path;
|
|
99365
|
+
const tree = await readCleanedTree(readScreen2, deps.session);
|
|
99366
|
+
const fingerprint = screenshotFingerprint(shot.base64);
|
|
99367
|
+
const lastAction = recentActions.at(-1);
|
|
99368
|
+
const lastSwipeMovedScreen = lastAction?.type === "swipe" && prevFingerprint != null && fingerprint !== prevFingerprint;
|
|
98435
99369
|
if (deps.isAborted()) {
|
|
98436
99370
|
caseStatus = "cancelled";
|
|
98437
|
-
|
|
99371
|
+
instructionDone = true;
|
|
98438
99372
|
break;
|
|
98439
99373
|
}
|
|
98440
|
-
|
|
98441
|
-
|
|
98442
|
-
appContext: deps.appContext,
|
|
98443
|
-
caseTitle: deps.catalogCase.name,
|
|
98444
|
-
instructions: flow.instructions,
|
|
98445
|
-
expectedResult: flow.expectedResult,
|
|
98446
|
-
stepIndex: stepIdx,
|
|
99374
|
+
const decideInput = {
|
|
99375
|
+
flow: instruction,
|
|
98447
99376
|
imageBase64: shot.base64,
|
|
98448
|
-
recentActions
|
|
98449
|
-
|
|
98450
|
-
|
|
98451
|
-
|
|
98452
|
-
|
|
98453
|
-
|
|
98454
|
-
|
|
98455
|
-
|
|
98456
|
-
|
|
98457
|
-
stepIndex: stepIdx,
|
|
98458
|
-
imageBase64: shot.base64,
|
|
98459
|
-
recentActions
|
|
98460
|
-
});
|
|
98461
|
-
if (isAbsurdNoScreenshotFail(decision)) {
|
|
98462
|
-
decision = {
|
|
98463
|
-
type: "fail",
|
|
98464
|
-
reason: "Model ignored the attached screenshot (claimed none was provided). Try a stronger vision model in Settings \u2192 Provider.",
|
|
98465
|
-
thoughts: "The vision model returned fail claiming no screenshot was available even though an image was attached to the request. After one retry it still denied the screenshot, so the step was marked failed."
|
|
98466
|
-
};
|
|
98467
|
-
}
|
|
98468
|
-
}
|
|
99377
|
+
recentActions,
|
|
99378
|
+
screenSnapshot: tree.snapshot,
|
|
99379
|
+
lastSwipeMovedScreen,
|
|
99380
|
+
completedInstructions,
|
|
99381
|
+
instructionOrdinal: instructionIndex + 1,
|
|
99382
|
+
instructionCount: instructionQueue.length
|
|
99383
|
+
};
|
|
99384
|
+
let decision = await decideOnce(decideInput);
|
|
99385
|
+
prevFingerprint = fingerprint;
|
|
98469
99386
|
const latencyMs = clock.now() - shotStarted;
|
|
98470
99387
|
if (deps.isAborted()) {
|
|
98471
99388
|
caseStatus = "cancelled";
|
|
98472
|
-
|
|
99389
|
+
instructionDone = true;
|
|
98473
99390
|
break;
|
|
98474
99391
|
}
|
|
98475
|
-
|
|
98476
|
-
|
|
98477
|
-
|
|
98478
|
-
|
|
98479
|
-
|
|
98480
|
-
|
|
98481
|
-
|
|
98482
|
-
|
|
98483
|
-
|
|
98484
|
-
|
|
98485
|
-
|
|
98486
|
-
|
|
98487
|
-
|
|
98488
|
-
|
|
98489
|
-
|
|
98490
|
-
tapBody.y = decision.y ?? 500;
|
|
99392
|
+
const applyWithRetry = async () => {
|
|
99393
|
+
try {
|
|
99394
|
+
return await applyDecision(decision);
|
|
99395
|
+
} catch (error40) {
|
|
99396
|
+
if (!isRetriableActionError(error40) || deps.isAborted())
|
|
99397
|
+
throw error40;
|
|
99398
|
+
const failed = decision;
|
|
99399
|
+
decision = await decideOnce({
|
|
99400
|
+
...decideInput,
|
|
99401
|
+
recentActions: [...recentActions, failed],
|
|
99402
|
+
lastError: error40 instanceof Error ? error40.message : String(error40)
|
|
99403
|
+
});
|
|
99404
|
+
const retryCommand = commandForDecision(decision, deps.defaultAppId);
|
|
99405
|
+
await setCurrentCommand(retryCommand);
|
|
99406
|
+
return await applyDecision(decision);
|
|
98491
99407
|
}
|
|
98492
|
-
|
|
98493
|
-
|
|
98494
|
-
|
|
98495
|
-
|
|
98496
|
-
action: decision,
|
|
98497
|
-
screenshotUri: shot.path,
|
|
98498
|
-
ok: true,
|
|
98499
|
-
latencyMs,
|
|
98500
|
-
detail: decision.reason ?? null
|
|
98501
|
-
});
|
|
98502
|
-
} else if (decision.type === "alert") {
|
|
98503
|
-
await perform(deps.session, {
|
|
98504
|
-
kind: "alert",
|
|
98505
|
-
alertAction: decision.alertAction === "dismiss" ? "dismiss" : "accept"
|
|
98506
|
-
});
|
|
98507
|
-
await clock.sleep(settleMs);
|
|
98508
|
-
await deps.appendStep({
|
|
98509
|
-
idx: stepIdx,
|
|
98510
|
-
action: decision,
|
|
98511
|
-
screenshotUri: shot.path,
|
|
98512
|
-
ok: true,
|
|
98513
|
-
latencyMs,
|
|
98514
|
-
detail: decision.reason ?? null
|
|
98515
|
-
});
|
|
98516
|
-
} else if (decision.type === "type") {
|
|
98517
|
-
await perform(deps.session, { kind: "input", text: decision.text ?? "" });
|
|
98518
|
-
await clock.sleep(settleMs);
|
|
98519
|
-
await deps.appendStep({
|
|
98520
|
-
idx: stepIdx,
|
|
98521
|
-
action: decision,
|
|
98522
|
-
screenshotUri: shot.path,
|
|
98523
|
-
ok: true,
|
|
98524
|
-
latencyMs,
|
|
98525
|
-
detail: decision.reason ?? null
|
|
98526
|
-
});
|
|
98527
|
-
} else if (decision.type === "wait") {
|
|
98528
|
-
const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
|
|
98529
|
-
await clock.sleep(waitMs);
|
|
98530
|
-
await deps.appendStep({
|
|
98531
|
-
idx: stepIdx,
|
|
98532
|
-
action: decision,
|
|
98533
|
-
screenshotUri: shot.path,
|
|
98534
|
-
ok: true,
|
|
98535
|
-
latencyMs,
|
|
98536
|
-
detail: decision.reason ?? `wait ${waitMs}ms`
|
|
98537
|
-
});
|
|
98538
|
-
} else if (decision.type === "verify" || decision.type === "done") {
|
|
98539
|
-
await deps.appendStep({
|
|
98540
|
-
idx: stepIdx,
|
|
98541
|
-
action: decision,
|
|
98542
|
-
screenshotUri: shot.path,
|
|
98543
|
-
ok: true,
|
|
98544
|
-
latencyMs,
|
|
98545
|
-
detail: decision.reason ?? null
|
|
98546
|
-
});
|
|
98547
|
-
flowDone = true;
|
|
98548
|
-
} else {
|
|
99408
|
+
};
|
|
99409
|
+
const recordStep = async (command, outcome2) => {
|
|
99410
|
+
recentActions.push(decision);
|
|
99411
|
+
recordedDecisions.push(decision);
|
|
98549
99412
|
await deps.appendStep({
|
|
98550
99413
|
idx: stepIdx,
|
|
98551
99414
|
action: decision,
|
|
98552
99415
|
screenshotUri: shot.path,
|
|
98553
|
-
ok:
|
|
99416
|
+
ok: outcome2 !== "fail",
|
|
98554
99417
|
latencyMs,
|
|
98555
|
-
detail: decision.reason ?? "Agent failed the step"
|
|
99418
|
+
detail: decision.type === "wait" ? decision.reason ?? `wait ${Math.min(3000, Math.max(500, decision.ms ?? 1500))}ms` : decision.reason ?? (outcome2 === "fail" ? "Agent failed the step" : null),
|
|
99419
|
+
command
|
|
98556
99420
|
});
|
|
99421
|
+
};
|
|
99422
|
+
const initialCommand = commandForDecision(decision, deps.defaultAppId);
|
|
99423
|
+
const outcome = initialCommand ? await withCurrentCommand(setCurrentCommand, initialCommand, async () => {
|
|
99424
|
+
const result = await applyWithRetry();
|
|
99425
|
+
await recordStep(commandForDecision(decision, deps.defaultAppId), result);
|
|
99426
|
+
return result;
|
|
99427
|
+
}) : await (async () => {
|
|
99428
|
+
const result = await applyWithRetry();
|
|
99429
|
+
await recordStep(null, result);
|
|
99430
|
+
return result;
|
|
99431
|
+
})();
|
|
99432
|
+
if (outcome === "done") {
|
|
99433
|
+
instructionDone = true;
|
|
99434
|
+
completedInstructions.push(instruction.instructions);
|
|
99435
|
+
} else if (outcome === "fail") {
|
|
98557
99436
|
caseStatus = "errored";
|
|
98558
|
-
caseError = decision.reason ?? "Agent marked the
|
|
98559
|
-
|
|
99437
|
+
caseError = decision.reason ?? "Agent marked the instruction as failed";
|
|
99438
|
+
instructionDone = true;
|
|
98560
99439
|
}
|
|
98561
99440
|
stepIdx += 1;
|
|
98562
99441
|
}
|
|
98563
99442
|
if (caseStatus === "errored" || caseStatus === "cancelled")
|
|
98564
99443
|
break;
|
|
98565
|
-
if (!
|
|
99444
|
+
if (!instructionDone) {
|
|
98566
99445
|
caseStatus = "errored";
|
|
98567
|
-
caseError = `Exceeded max steps (${maxSteps}) for
|
|
99446
|
+
caseError = `Exceeded max steps (${maxSteps}) for instruction ${instructionIndex + 1} of ${instructionQueue.length}`;
|
|
98568
99447
|
break;
|
|
98569
99448
|
}
|
|
98570
99449
|
}
|
|
@@ -98584,7 +99463,8 @@ async function executeAgentCase(deps) {
|
|
|
98584
99463
|
screenshotUri: lastScreenshotUri,
|
|
98585
99464
|
ok: false,
|
|
98586
99465
|
latencyMs: 0,
|
|
98587
|
-
detail: caseError
|
|
99466
|
+
detail: caseError,
|
|
99467
|
+
command: null
|
|
98588
99468
|
});
|
|
98589
99469
|
}
|
|
98590
99470
|
}
|
|
@@ -98668,6 +99548,7 @@ async function loadRun(runId) {
|
|
|
98668
99548
|
ok: step.ok === 1,
|
|
98669
99549
|
latencyMs: step.latencyMs,
|
|
98670
99550
|
detail: step.detail,
|
|
99551
|
+
command: step.command ?? null,
|
|
98671
99552
|
createdAt: step.createdAt
|
|
98672
99553
|
}));
|
|
98673
99554
|
const caseMode = test.executionMode === "script" || test.executionMode === "agent" ? test.executionMode : null;
|
|
@@ -98680,6 +99561,7 @@ async function loadRun(runId) {
|
|
|
98680
99561
|
error: test.error,
|
|
98681
99562
|
startedAt: test.startedAt,
|
|
98682
99563
|
finishedAt: test.finishedAt,
|
|
99564
|
+
currentCommand: test.currentCommand ?? null,
|
|
98683
99565
|
steps
|
|
98684
99566
|
});
|
|
98685
99567
|
}
|
|
@@ -98709,9 +99591,14 @@ async function appendStep(input) {
|
|
|
98709
99591
|
ok: input.ok ? 1 : 0,
|
|
98710
99592
|
latencyMs: Math.max(0, Math.round(input.latencyMs)),
|
|
98711
99593
|
detail: input.detail,
|
|
99594
|
+
command: input.command,
|
|
98712
99595
|
createdAt: Date.now()
|
|
98713
99596
|
});
|
|
98714
99597
|
}
|
|
99598
|
+
async function setCurrentCommand(runTestId, command) {
|
|
99599
|
+
const db = getCatalogDb();
|
|
99600
|
+
await db.update(runTests).set({ currentCommand: command }).where(eq(runTests.id, runTestId));
|
|
99601
|
+
}
|
|
98715
99602
|
async function updateCaseLastRun(caseId, status, at) {
|
|
98716
99603
|
const db = getCatalogDb();
|
|
98717
99604
|
await db.update(cases).set({
|
|
@@ -98734,7 +99621,8 @@ async function persistCancelled(runId) {
|
|
|
98734
99621
|
await db.update(runTests).set({
|
|
98735
99622
|
status: "cancelled",
|
|
98736
99623
|
error: "Cancelled by user",
|
|
98737
|
-
finishedAt
|
|
99624
|
+
finishedAt,
|
|
99625
|
+
currentCommand: null
|
|
98738
99626
|
}).where(eq(runTests.id, test.id));
|
|
98739
99627
|
}
|
|
98740
99628
|
}
|
|
@@ -98754,6 +99642,9 @@ async function executeScriptCase2(input) {
|
|
|
98754
99642
|
...step
|
|
98755
99643
|
});
|
|
98756
99644
|
},
|
|
99645
|
+
setCurrentCommand: async (command) => {
|
|
99646
|
+
await setCurrentCommand(input.runTestId, command);
|
|
99647
|
+
},
|
|
98757
99648
|
clock: { sleep, now: () => Date.now() }
|
|
98758
99649
|
});
|
|
98759
99650
|
}
|
|
@@ -98774,8 +99665,12 @@ async function executeAgentCase2(input) {
|
|
|
98774
99665
|
...step
|
|
98775
99666
|
});
|
|
98776
99667
|
},
|
|
99668
|
+
setCurrentCommand: async (command) => {
|
|
99669
|
+
await setCurrentCommand(input.runTestId, command);
|
|
99670
|
+
},
|
|
98777
99671
|
decide: decideNextAction,
|
|
98778
|
-
clock: { sleep, now: () => Date.now() }
|
|
99672
|
+
clock: { sleep, now: () => Date.now() },
|
|
99673
|
+
defaultAppId: input.defaultAppId
|
|
98779
99674
|
});
|
|
98780
99675
|
}
|
|
98781
99676
|
async function executeCase(input) {
|
|
@@ -98788,7 +99683,8 @@ async function executeCase(input) {
|
|
|
98788
99683
|
status: "running",
|
|
98789
99684
|
startedAt,
|
|
98790
99685
|
error: null,
|
|
98791
|
-
executionMode: input.caseMode
|
|
99686
|
+
executionMode: input.caseMode,
|
|
99687
|
+
currentCommand: null
|
|
98792
99688
|
}).where(eq(runTests.id, input.runTestId));
|
|
98793
99689
|
let caseStatus;
|
|
98794
99690
|
let caseError = null;
|
|
@@ -98815,7 +99711,8 @@ async function executeCase(input) {
|
|
|
98815
99711
|
caseId: input.caseId,
|
|
98816
99712
|
appContext: input.appContext,
|
|
98817
99713
|
session: input.session,
|
|
98818
|
-
auth: input.auth
|
|
99714
|
+
auth: input.auth,
|
|
99715
|
+
defaultAppId: input.defaultAppId
|
|
98819
99716
|
});
|
|
98820
99717
|
caseStatus = result.status;
|
|
98821
99718
|
decisions = result.decisions;
|
|
@@ -98828,7 +99725,8 @@ async function executeCase(input) {
|
|
|
98828
99725
|
await db.update(runTests).set({
|
|
98829
99726
|
status: caseStatus,
|
|
98830
99727
|
error: caseError,
|
|
98831
|
-
finishedAt
|
|
99728
|
+
finishedAt,
|
|
99729
|
+
currentCommand: null
|
|
98832
99730
|
}).where(eq(runTests.id, input.runTestId));
|
|
98833
99731
|
await updateCaseLastRun(input.caseId, caseStatus, finishedAt);
|
|
98834
99732
|
if (caseStatus === "passed" && input.caseMode === "agent") {
|
|
@@ -98926,7 +99824,8 @@ async function executeRun(runId) {
|
|
|
98926
99824
|
appContext: app.context,
|
|
98927
99825
|
session,
|
|
98928
99826
|
auth,
|
|
98929
|
-
caseMode
|
|
99827
|
+
caseMode,
|
|
99828
|
+
defaultAppId: run.platform === "ios" ? app.iosBundleId || undefined : app.androidApplicationId || undefined
|
|
98930
99829
|
});
|
|
98931
99830
|
if (status === "cancelled" || isAborted2(runId)) {
|
|
98932
99831
|
await persistCancelled(runId);
|
|
@@ -99406,7 +100305,9 @@ function createSessionRoutes() {
|
|
|
99406
100305
|
try {
|
|
99407
100306
|
const { session } = requireActiveSession();
|
|
99408
100307
|
const full = c.req.query("full") === "1" || c.req.query("full") === "true";
|
|
99409
|
-
const
|
|
100308
|
+
const pauseQuery = c.req.query("pauseMjpeg");
|
|
100309
|
+
const pauseMjpeg = pauseQuery !== "0" && pauseQuery !== "false";
|
|
100310
|
+
const result = await getScreen(session, { full, pauseMjpeg });
|
|
99410
100311
|
return c.json(screenResponseSchema.parse(result));
|
|
99411
100312
|
} catch (error40) {
|
|
99412
100313
|
const gone = sessionErrorResponse(error40);
|