@yoqa/runner 0.3.24 → 0.3.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +1147 -246
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -16513,7 +16513,8 @@ var runTests = sqliteTable("run_tests", {
16513
16513
  executionMode: text("execution_mode"),
16514
16514
  error: text("error"),
16515
16515
  startedAt: integer("started_at"),
16516
- finishedAt: integer("finished_at")
16516
+ finishedAt: integer("finished_at"),
16517
+ currentCommand: text("current_command")
16517
16518
  });
16518
16519
  var runSteps = sqliteTable("run_steps", {
16519
16520
  id: text("id").primaryKey(),
@@ -16524,6 +16525,7 @@ var runSteps = sqliteTable("run_steps", {
16524
16525
  ok: integer("ok").notNull().default(0),
16525
16526
  latencyMs: integer("latency_ms").notNull().default(0),
16526
16527
  detail: text("detail"),
16528
+ command: text("command"),
16527
16529
  createdAt: integer("created_at").notNull()
16528
16530
  });
16529
16531
 
@@ -16638,7 +16640,8 @@ function ensureSchema(sqlite) {
16638
16640
  execution_mode TEXT,
16639
16641
  error TEXT,
16640
16642
  started_at INTEGER,
16641
- finished_at INTEGER
16643
+ finished_at INTEGER,
16644
+ current_command TEXT
16642
16645
  );
16643
16646
 
16644
16647
  CREATE TABLE IF NOT EXISTS run_steps (
@@ -16650,6 +16653,7 @@ function ensureSchema(sqlite) {
16650
16653
  ok INTEGER NOT NULL DEFAULT 0,
16651
16654
  latency_ms INTEGER NOT NULL DEFAULT 0,
16652
16655
  detail TEXT,
16656
+ command TEXT,
16653
16657
  created_at INTEGER NOT NULL
16654
16658
  );
16655
16659
 
@@ -16668,6 +16672,7 @@ function ensureSchema(sqlite) {
16668
16672
  migrateAppsPrefix(sqlite);
16669
16673
  migrateCaseScripts(sqlite);
16670
16674
  migrateRunExecutionMode(sqlite);
16675
+ migrateRunCommands(sqlite);
16671
16676
  }
16672
16677
  function tableColumns(sqlite, table) {
16673
16678
  const rows = sqlite.query(`PRAGMA table_info(${table})`).all();
@@ -16696,6 +16701,16 @@ function migrateRunExecutionMode(sqlite) {
16696
16701
  addColumnIfMissing(sqlite, "run_tests", "execution_mode", "execution_mode TEXT", testCols);
16697
16702
  }
16698
16703
  }
16704
+ function migrateRunCommands(sqlite) {
16705
+ const testCols = tableColumns(sqlite, "run_tests");
16706
+ if (testCols.size > 0) {
16707
+ addColumnIfMissing(sqlite, "run_tests", "current_command", "current_command TEXT", testCols);
16708
+ }
16709
+ const stepCols = tableColumns(sqlite, "run_steps");
16710
+ if (stepCols.size > 0) {
16711
+ addColumnIfMissing(sqlite, "run_steps", "command", "command TEXT", stepCols);
16712
+ }
16713
+ }
16699
16714
  function migrateProvidersTable(sqlite) {
16700
16715
  const existing = tableColumns(sqlite, "providers");
16701
16716
  if (existing.size === 0)
@@ -16801,7 +16816,7 @@ function getCatalogDb() {
16801
16816
  // package.json
16802
16817
  var package_default = {
16803
16818
  name: "@yoqa/runner",
16804
- version: "0.3.24",
16819
+ version: "0.3.26",
16805
16820
  description: "Yoqa local runner \u2014 Appium + catalog HTTP API (Bun). Start with yoqa serve or the yoqa-runner bin.",
16806
16821
  license: "UNLICENSED",
16807
16822
  type: "module",
@@ -16884,12 +16899,14 @@ function trackMjpegProxy() {
16884
16899
  return controller;
16885
16900
  }
16886
16901
  function abortAllMjpegProxies() {
16902
+ const had = proxies.size > 0;
16887
16903
  for (const controller of [...proxies]) {
16888
16904
  try {
16889
16905
  controller.abort();
16890
16906
  } catch {}
16891
16907
  }
16892
16908
  proxies.clear();
16909
+ return had;
16893
16910
  }
16894
16911
 
16895
16912
  // src/domains/devices/session.ts
@@ -17574,6 +17591,38 @@ async function listDevices(platform, options = {}) {
17574
17591
  return listAndroidDevices(options);
17575
17592
  }
17576
17593
 
17594
+ // src/domains/devices/keyboard.ts
17595
+ function isUnsupportedCommandError(error) {
17596
+ const message = error instanceof Error ? error.message : String(error);
17597
+ return /unknown command|unknown method|not (yet )?(been )?implemented|not (yet )?supported|does not support/i.test(message);
17598
+ }
17599
+ function w3cKeyActions(text2) {
17600
+ const actions = [];
17601
+ for (const ch of text2) {
17602
+ actions.push({ type: "keyDown", value: ch }, { type: "keyUp", value: ch });
17603
+ }
17604
+ return [{ type: "key", id: "keyboard", actions }];
17605
+ }
17606
+ async function w3cType(browser, text2) {
17607
+ await browser.performActions(w3cKeyActions(text2));
17608
+ try {
17609
+ await browser.releaseActions();
17610
+ } catch {}
17611
+ }
17612
+ async function typeText(browser, text2) {
17613
+ if (text2.length === 0)
17614
+ return "noop";
17615
+ try {
17616
+ await browser.execute("mobile: type", { text: text2 });
17617
+ return "mobile";
17618
+ } catch (error) {
17619
+ if (!isUnsupportedCommandError(error))
17620
+ throw error;
17621
+ }
17622
+ await w3cType(browser, text2);
17623
+ return "w3c";
17624
+ }
17625
+
17577
17626
  // src/domains/devices/session.ts
17578
17627
  var YOQA_ROOT4 = join7(homedir6(), ".yoqa");
17579
17628
  var DEFAULT_MJPEG_PORT = Number(process.env.YOQA_MJPEG_PORT ?? "9100");
@@ -17597,6 +17646,7 @@ function isDeadSessionError(error) {
17597
17646
  const message = error instanceof Error ? error.message : String(error);
17598
17647
  return DEAD_SESSION_RE.test(message);
17599
17648
  }
17649
+ var DEAD_SESSION_RETRY_MS = 200;
17600
17650
  function mjpegSettingsForDevice(options) {
17601
17651
  const physicalIos = options.platform === "ios" && looksLikePhysicalIosUdid(options.deviceId);
17602
17652
  if (physicalIos) {
@@ -17981,10 +18031,17 @@ async function createDeviceSession(options) {
17981
18031
  try {
17982
18032
  return await fn();
17983
18033
  } catch (error) {
17984
- if (isDeadSessionError(error)) {
17985
- notifySessionDead();
18034
+ if (!isDeadSessionError(error))
18035
+ throw error;
18036
+ await Bun.sleep(DEAD_SESSION_RETRY_MS);
18037
+ try {
18038
+ return await fn();
18039
+ } catch (retryError) {
18040
+ if (isDeadSessionError(retryError)) {
18041
+ notifySessionDead();
18042
+ }
18043
+ throw retryError;
17986
18044
  }
17987
- throw error;
17988
18045
  }
17989
18046
  };
17990
18047
  const getWindowSize = async () => guard(() => browser.getWindowSize());
@@ -18048,10 +18105,10 @@ async function createDeviceSession(options) {
18048
18105
  });
18049
18106
  });
18050
18107
  };
18051
- const swipe = async (x1, y1, x2, y2, durationMs = 400) => {
18108
+ const swipe = async (x1, y1, x2, y2, durationMs = 400, swipeOptions) => {
18052
18109
  await gate.withLock(async () => {
18053
18110
  await guard(async () => {
18054
- const size = await getPointerSize();
18111
+ const size = swipeOptions?.coordSpace === "screenshot" ? await getScreenshotCoordSize() : await getPointerSize();
18055
18112
  await injectSwipe(browser, toPx(x1, size.width), toPx(y1, size.height), toPx(x2, size.width), toPx(y2, size.height), durationMs);
18056
18113
  });
18057
18114
  });
@@ -18062,7 +18119,7 @@ async function createDeviceSession(options) {
18062
18119
  const type = async (text2) => {
18063
18120
  await gate.withLock(async () => {
18064
18121
  await guard(async () => {
18065
- await browser.keys(text2.split(""));
18122
+ await typeText(browser, text2);
18066
18123
  });
18067
18124
  });
18068
18125
  };
@@ -24263,6 +24320,8 @@ var caseScriptTapSchema = exports_external.object({
24263
24320
  y: exports_external.number().min(0).max(1000).optional(),
24264
24321
  label: exports_external.string().min(1).optional(),
24265
24322
  id: exports_external.string().min(1).optional(),
24323
+ double: exports_external.boolean().optional(),
24324
+ durationMs: exports_external.number().min(50).max(5000).optional(),
24266
24325
  reason: exports_external.string().optional()
24267
24326
  }).superRefine((value, ctx) => {
24268
24327
  const hasCoords = value.x != null && value.y != null;
@@ -24273,8 +24332,45 @@ var caseScriptTapSchema = exports_external.object({
24273
24332
  });
24274
24333
  }
24275
24334
  });
24335
+ var caseScriptSwipeSchema = exports_external.object({
24336
+ type: exports_external.literal("swipe"),
24337
+ x: exports_external.number().min(0).max(1000),
24338
+ y: exports_external.number().min(0).max(1000),
24339
+ x2: exports_external.number().min(0).max(1000),
24340
+ y2: exports_external.number().min(0).max(1000),
24341
+ durationMs: exports_external.number().min(50).max(5000).optional(),
24342
+ reason: exports_external.string().optional()
24343
+ });
24344
+ var caseScriptDragSchema = exports_external.object({
24345
+ type: exports_external.literal("drag"),
24346
+ x: exports_external.number().min(0).max(1000),
24347
+ y: exports_external.number().min(0).max(1000),
24348
+ x2: exports_external.number().min(0).max(1000),
24349
+ y2: exports_external.number().min(0).max(1000),
24350
+ durationMs: exports_external.number().min(50).max(5000).optional(),
24351
+ reason: exports_external.string().optional()
24352
+ });
24353
+ var caseScriptAppIdSchema = exports_external.union([
24354
+ exports_external.object({
24355
+ type: exports_external.literal("activate-app"),
24356
+ appId: exports_external.string().min(1),
24357
+ reason: exports_external.string().optional()
24358
+ }),
24359
+ exports_external.object({
24360
+ type: exports_external.literal("terminate-app"),
24361
+ appId: exports_external.string().min(1),
24362
+ reason: exports_external.string().optional()
24363
+ }),
24364
+ exports_external.object({
24365
+ type: exports_external.literal("restart-app"),
24366
+ appId: exports_external.string().min(1),
24367
+ reason: exports_external.string().optional()
24368
+ })
24369
+ ]);
24276
24370
  var caseScriptActionSchema = exports_external.union([
24277
24371
  caseScriptTapSchema,
24372
+ caseScriptSwipeSchema,
24373
+ caseScriptDragSchema,
24278
24374
  exports_external.object({
24279
24375
  type: exports_external.literal("type"),
24280
24376
  text: exports_external.string(),
@@ -24296,6 +24392,17 @@ var caseScriptActionSchema = exports_external.union([
24296
24392
  type: exports_external.literal("alert"),
24297
24393
  alertAction: exports_external.union([exports_external.literal("accept"), exports_external.literal("dismiss")]).default("accept"),
24298
24394
  reason: exports_external.string().optional()
24395
+ }),
24396
+ caseScriptAppIdSchema,
24397
+ exports_external.object({
24398
+ type: exports_external.literal("background-app"),
24399
+ seconds: exports_external.number().min(0).max(120).optional(),
24400
+ reason: exports_external.string().optional()
24401
+ }),
24402
+ exports_external.object({
24403
+ type: exports_external.literal("open-url"),
24404
+ url: exports_external.string().min(1),
24405
+ reason: exports_external.string().optional()
24299
24406
  })
24300
24407
  ]);
24301
24408
  var caseScriptSchema = exports_external.object({
@@ -24552,6 +24659,7 @@ var runStepSchema = exports_external.object({
24552
24659
  ok: exports_external.boolean(),
24553
24660
  latencyMs: exports_external.number().nonnegative(),
24554
24661
  detail: exports_external.string().nullable(),
24662
+ command: exports_external.string().nullable().optional(),
24555
24663
  createdAt: exports_external.number().int().nonnegative()
24556
24664
  });
24557
24665
  var runTestSchema = exports_external.object({
@@ -24563,6 +24671,7 @@ var runTestSchema = exports_external.object({
24563
24671
  error: exports_external.string().nullable(),
24564
24672
  startedAt: exports_external.number().int().nonnegative().nullable(),
24565
24673
  finishedAt: exports_external.number().int().nonnegative().nullable(),
24674
+ currentCommand: exports_external.string().nullable().optional(),
24566
24675
  steps: exports_external.array(runStepSchema).optional()
24567
24676
  });
24568
24677
  var runSchema = exports_external.object({
@@ -24736,6 +24845,59 @@ function decodeXmlEntities(value) {
24736
24845
  }
24737
24846
 
24738
24847
  // ../../packages/runner-client/src/shell-script.ts
24848
+ function shellSingleQuote(value) {
24849
+ return `'${value.replace(/'/g, `'\\''`)}'`;
24850
+ }
24851
+ function formatActionShellLine(action) {
24852
+ const parts = ["yoqa", "action", action.kind];
24853
+ if (action.id) {
24854
+ parts.push("--id", shellSingleQuote(action.id));
24855
+ }
24856
+ if (action.label) {
24857
+ parts.push("--label", shellSingleQuote(action.label));
24858
+ }
24859
+ if (action.description) {
24860
+ parts.push("-d", shellSingleQuote(action.description));
24861
+ }
24862
+ if (action.x != null)
24863
+ parts.push("--x", String(Math.round(action.x)));
24864
+ if (action.y != null)
24865
+ parts.push("--y", String(Math.round(action.y)));
24866
+ if (action.x2 != null)
24867
+ parts.push("--x2", String(Math.round(action.x2)));
24868
+ if (action.y2 != null)
24869
+ parts.push("--y2", String(Math.round(action.y2)));
24870
+ if (action.durationMs != null)
24871
+ parts.push("--duration", String(Math.round(action.durationMs)));
24872
+ if (action.double)
24873
+ parts.push("--double");
24874
+ if (action.text != null)
24875
+ parts.push("--text", shellSingleQuote(action.text));
24876
+ if (action.appId != null)
24877
+ parts.push("--app-id", shellSingleQuote(action.appId));
24878
+ if (action.url != null)
24879
+ parts.push("--url", shellSingleQuote(action.url));
24880
+ if (action.seconds != null)
24881
+ parts.push("--seconds", String(action.seconds));
24882
+ if (action.kind === "alert") {
24883
+ if (action.alertAction === "dismiss")
24884
+ parts.push("--dismiss");
24885
+ }
24886
+ return parts.join(" ");
24887
+ }
24888
+ function formatSleepShellLine(seconds) {
24889
+ const safe = Math.max(0, seconds);
24890
+ const rounded = Number.isInteger(safe) ? String(safe) : String(Number(safe.toFixed(3)));
24891
+ return `sleep ${rounded}`;
24892
+ }
24893
+ function formatAssertShellLine(input) {
24894
+ const parts = ["yoqa", "assert", input.assertion, "--text", shellSingleQuote(input.text.trim())];
24895
+ const timeout = input.timeoutSeconds ?? 5;
24896
+ if (timeout !== 5) {
24897
+ parts.push("--timeout", String(timeout));
24898
+ }
24899
+ return parts.join(" ");
24900
+ }
24739
24901
  var DEFAULT_SHELL_SCRIPT_HEADER = [
24740
24902
  "#!/usr/bin/env bash",
24741
24903
  "# Yoqa inspector script",
@@ -25163,9 +25325,16 @@ function normalizeVisionJson(value) {
25163
25325
  out.x = clampNormCoord(out.x);
25164
25326
  if (typeof out.y === "number")
25165
25327
  out.y = clampNormCoord(out.y);
25328
+ if (typeof out.x2 === "number")
25329
+ out.x2 = clampNormCoord(out.x2);
25330
+ if (typeof out.y2 === "number")
25331
+ out.y2 = clampNormCoord(out.y2);
25166
25332
  if (typeof out.ms === "number" && Number.isFinite(out.ms)) {
25167
25333
  out.ms = Math.min(1e4, Math.max(0, out.ms));
25168
25334
  }
25335
+ if (typeof out.durationMs === "number" && Number.isFinite(out.durationMs)) {
25336
+ out.durationMs = Math.min(5000, Math.max(50, Math.round(out.durationMs)));
25337
+ }
25169
25338
  const reason = typeof out.reason === "string" ? out.reason.trim() : "";
25170
25339
  const thoughts = typeof out.thoughts === "string" ? out.thoughts.trim() : "";
25171
25340
  if (reason)
@@ -96053,26 +96222,257 @@ async function completeVision(auth, input) {
96053
96222
  }
96054
96223
 
96055
96224
  // src/domains/runs/agent.ts
96225
+ var swipeDirectionSchema = exports_external.union([
96226
+ exports_external.literal("up"),
96227
+ exports_external.literal("down"),
96228
+ exports_external.literal("left"),
96229
+ exports_external.literal("right")
96230
+ ]);
96056
96231
  var agentDecisionSchema = exports_external.object({
96057
96232
  type: exports_external.union([
96058
96233
  exports_external.literal("tap"),
96234
+ exports_external.literal("swipe"),
96235
+ exports_external.literal("drag"),
96059
96236
  exports_external.literal("type"),
96237
+ exports_external.literal("input"),
96060
96238
  exports_external.literal("wait"),
96061
96239
  exports_external.literal("alert"),
96240
+ exports_external.literal("activate-app"),
96241
+ exports_external.literal("terminate-app"),
96242
+ exports_external.literal("restart-app"),
96243
+ exports_external.literal("background-app"),
96244
+ exports_external.literal("open-url"),
96245
+ exports_external.literal("assert"),
96062
96246
  exports_external.literal("verify"),
96063
96247
  exports_external.literal("done"),
96064
96248
  exports_external.literal("fail")
96065
96249
  ]),
96066
96250
  x: exports_external.number().optional(),
96067
96251
  y: exports_external.number().optional(),
96252
+ x2: exports_external.number().optional(),
96253
+ y2: exports_external.number().optional(),
96254
+ direction: swipeDirectionSchema.optional(),
96255
+ durationMs: exports_external.number().optional(),
96256
+ double: exports_external.boolean().optional(),
96068
96257
  label: exports_external.string().min(1).optional(),
96069
96258
  id: exports_external.string().min(1).optional(),
96259
+ description: exports_external.string().min(1).optional(),
96070
96260
  text: exports_external.string().optional(),
96071
96261
  ms: exports_external.number().optional(),
96072
96262
  alertAction: exports_external.union([exports_external.literal("accept"), exports_external.literal("dismiss")]).optional(),
96263
+ appId: exports_external.string().min(1).optional(),
96264
+ url: exports_external.string().min(1).optional(),
96265
+ seconds: exports_external.number().optional(),
96266
+ assertion: exports_external.union([exports_external.literal("visible"), exports_external.literal("not-visible")]).optional(),
96267
+ timeoutMs: exports_external.number().optional(),
96073
96268
  reason: exports_external.string().min(1),
96074
96269
  thoughts: exports_external.string().min(1)
96270
+ }).superRefine((value, ctx) => {
96271
+ if (value.type === "swipe") {
96272
+ const hasCoords = value.x != null && value.y != null && value.x2 != null && value.y2 != null;
96273
+ if (!hasCoords && value.direction == null) {
96274
+ ctx.addIssue({
96275
+ code: exports_external.ZodIssueCode.custom,
96276
+ message: "swipe requires direction or x,y,x2,y2"
96277
+ });
96278
+ }
96279
+ }
96280
+ if (value.type === "drag") {
96281
+ if (value.x == null || value.y == null || value.x2 == null || value.y2 == null) {
96282
+ ctx.addIssue({
96283
+ code: exports_external.ZodIssueCode.custom,
96284
+ message: "drag requires x,y,x2,y2"
96285
+ });
96286
+ }
96287
+ }
96288
+ if ((value.type === "type" || value.type === "input") && !value.text?.trim()) {
96289
+ ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "type/input requires text" });
96290
+ }
96291
+ if (value.type === "open-url" && !value.url?.trim()) {
96292
+ ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "open-url requires url" });
96293
+ }
96294
+ if (value.type === "assert" && !value.text?.trim()) {
96295
+ ctx.addIssue({ code: exports_external.ZodIssueCode.custom, message: "assert requires text" });
96296
+ }
96075
96297
  });
96298
+ var FINGER_SWIPE_NORMS = {
96299
+ up: { x: 500, y: 800, x2: 500, y2: 200 },
96300
+ down: { x: 500, y: 200, x2: 500, y2: 800 },
96301
+ left: { x: 800, y: 500, x2: 200, y2: 500 },
96302
+ right: { x: 200, y: 500, x2: 800, y2: 500 }
96303
+ };
96304
+ function resolveSwipeNorm(decision) {
96305
+ if (decision.x != null && decision.y != null && decision.x2 != null && decision.y2 != null) {
96306
+ return { x: decision.x, y: decision.y, x2: decision.x2, y2: decision.y2 };
96307
+ }
96308
+ if (decision.direction)
96309
+ return FINGER_SWIPE_NORMS[decision.direction];
96310
+ return null;
96311
+ }
96312
+ var MAX_SCREEN_SNAPSHOT_ROWS = 80;
96313
+ function padNorm(n) {
96314
+ return String(Math.round(n)).padStart(4, " ");
96315
+ }
96316
+ function formatScreenSnapshot(elements) {
96317
+ if (elements == null)
96318
+ return "(screen tree unavailable)";
96319
+ if (elements.length === 0)
96320
+ return "(empty tree)";
96321
+ const ordered = [...elements].sort((a, b) => a.y - b.y || a.x - b.x);
96322
+ const shown = ordered.slice(0, MAX_SCREEN_SNAPSHOT_ROWS);
96323
+ const lines = shown.map((element) => {
96324
+ const id = element.id?.trim() ? ` id=${element.id.trim()}` : "";
96325
+ const label = element.label?.trim() || "(no label)";
96326
+ return `${padNorm(element.x)},${padNorm(element.y)} ${Math.round(element.width)}x${Math.round(element.height)}${id} ${label}`;
96327
+ });
96328
+ if (ordered.length > shown.length) {
96329
+ lines.push(`\u2026 ${ordered.length - shown.length} more elements omitted`);
96330
+ }
96331
+ return lines.join(`
96332
+ `);
96333
+ }
96334
+ function decisionToActionRequest(decision, options = {}) {
96335
+ const appId = decision.appId?.trim() || options.defaultAppId?.trim() || undefined;
96336
+ switch (decision.type) {
96337
+ case "tap": {
96338
+ const body = { kind: "tap" };
96339
+ if (decision.double)
96340
+ body.double = true;
96341
+ if (decision.durationMs != null)
96342
+ body.durationMs = decision.durationMs;
96343
+ if (decision.description)
96344
+ body.description = decision.description;
96345
+ if (prefersScreenshotTap(decision)) {
96346
+ body.x = decision.x;
96347
+ body.y = decision.y;
96348
+ } else if (decision.id) {
96349
+ body.id = decision.id;
96350
+ if (decision.label)
96351
+ body.label = decision.label;
96352
+ } else if (decision.label) {
96353
+ body.label = decision.label;
96354
+ } else if (decision.description) {} else {
96355
+ body.x = decision.x ?? 500;
96356
+ body.y = decision.y ?? 500;
96357
+ }
96358
+ return body;
96359
+ }
96360
+ case "swipe": {
96361
+ const swipe = resolveSwipeNorm(decision);
96362
+ if (!swipe)
96363
+ return null;
96364
+ return {
96365
+ kind: "swipe",
96366
+ x: swipe.x,
96367
+ y: swipe.y,
96368
+ x2: swipe.x2,
96369
+ y2: swipe.y2,
96370
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {}
96371
+ };
96372
+ }
96373
+ case "drag":
96374
+ if (decision.x == null || decision.y == null || decision.x2 == null || decision.y2 == null) {
96375
+ return null;
96376
+ }
96377
+ return {
96378
+ kind: "drag",
96379
+ x: decision.x,
96380
+ y: decision.y,
96381
+ x2: decision.x2,
96382
+ y2: decision.y2,
96383
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {}
96384
+ };
96385
+ case "type":
96386
+ case "input": {
96387
+ const body = { kind: "input", text: decision.text ?? "" };
96388
+ if (decision.id)
96389
+ body.id = decision.id;
96390
+ if (decision.label)
96391
+ body.label = decision.label;
96392
+ if (decision.x != null)
96393
+ body.x = decision.x;
96394
+ if (decision.y != null)
96395
+ body.y = decision.y;
96396
+ if (decision.description)
96397
+ body.description = decision.description;
96398
+ return body;
96399
+ }
96400
+ case "alert":
96401
+ return {
96402
+ kind: "alert",
96403
+ alertAction: decision.alertAction === "dismiss" ? "dismiss" : "accept"
96404
+ };
96405
+ case "activate-app":
96406
+ return appId ? { kind: "activate-app", appId } : null;
96407
+ case "terminate-app":
96408
+ return appId ? { kind: "terminate-app", appId } : null;
96409
+ case "restart-app":
96410
+ return appId ? { kind: "restart-app", appId } : null;
96411
+ case "background-app":
96412
+ return {
96413
+ kind: "background-app",
96414
+ ...decision.seconds != null ? { seconds: decision.seconds } : {}
96415
+ };
96416
+ case "open-url":
96417
+ return decision.url ? { kind: "open-url", url: decision.url } : null;
96418
+ default:
96419
+ return null;
96420
+ }
96421
+ }
96422
+ var SCROLL_INTENT_RE = /\b(scroll|swipe|flick)\b/i;
96423
+ var SCROLL_UP_RE = /\b(scroll\s+up|back\s+up|to the top|content above)\b/i;
96424
+ var SCROLL_UNTIL_END_RE = /\b(scroll[\s\w]{0,48}(bottom|end)|cannot scroll|can not scroll|can't scroll|until you can'?t scroll|no longer scroll|not be able to scroll)\b/i;
96425
+ function looksLikeScrollIntent(text3) {
96426
+ return SCROLL_INTENT_RE.test(text3);
96427
+ }
96428
+ function isScrollUntilEndGoal(instructions, expectedResult) {
96429
+ return SCROLL_UNTIL_END_RE.test(`${instructions} ${expectedResult}`);
96430
+ }
96431
+ function inferFingerDirection(text3) {
96432
+ if (SCROLL_UP_RE.test(text3))
96433
+ return "down";
96434
+ if (/\bscroll\s+left\b/i.test(text3) || /\bswipe\s+left\b/i.test(text3))
96435
+ return "left";
96436
+ if (/\bscroll\s+right\b/i.test(text3) || /\bswipe\s+right\b/i.test(text3))
96437
+ return "right";
96438
+ return "up";
96439
+ }
96440
+ function asDirectionalSwipe(decision) {
96441
+ const text3 = `${decision.reason} ${decision.thoughts}`;
96442
+ return {
96443
+ type: "swipe",
96444
+ direction: inferFingerDirection(text3),
96445
+ reason: decision.reason,
96446
+ thoughts: decision.thoughts
96447
+ };
96448
+ }
96449
+ function coerceScrollIntentToSwipe(decision) {
96450
+ if (decision.type === "tap" && isSystemPermissionLabel(decision.label))
96451
+ return decision;
96452
+ if (decision.type !== "tap" && decision.type !== "wait")
96453
+ return decision;
96454
+ if (!looksLikeScrollIntent(`${decision.reason} ${decision.thoughts}`))
96455
+ return decision;
96456
+ return asDirectionalSwipe(decision);
96457
+ }
96458
+ function continueScrollingInsteadOfComplete(input) {
96459
+ if (input.decision.type !== "verify" && input.decision.type !== "done") {
96460
+ return input.decision;
96461
+ }
96462
+ const hadSwipe = input.recentActions.some((action) => action.type === "swipe");
96463
+ const untilEnd = isScrollUntilEndGoal(input.instructions, input.expectedResult);
96464
+ const claimedScroll = looksLikeScrollIntent(`${input.decision.reason} ${input.decision.thoughts}`);
96465
+ if (!hadSwipe && (untilEnd || claimedScroll)) {
96466
+ return asDirectionalSwipe(input.decision);
96467
+ }
96468
+ if (untilEnd && hadSwipe && input.lastSwipeMovedScreen) {
96469
+ return asDirectionalSwipe(input.decision);
96470
+ }
96471
+ return input.decision;
96472
+ }
96473
+ function screenshotFingerprint(base643) {
96474
+ return String(Bun.hash(base643));
96475
+ }
96076
96476
  var SYSTEM_PERMISSION_LABELS = new Set([
96077
96477
  "allow",
96078
96478
  "don't allow",
@@ -96092,7 +96492,11 @@ function isSystemPermissionLabel(label) {
96092
96492
  return SYSTEM_PERMISSION_LABELS.has(trimmed);
96093
96493
  }
96094
96494
  function prefersScreenshotTap(decision) {
96095
- return decision.x != null && decision.y != null && !isSystemPermissionLabel(decision.label);
96495
+ if (isSystemPermissionLabel(decision.label))
96496
+ return false;
96497
+ if (decision.id?.trim())
96498
+ return false;
96499
+ return decision.x != null && decision.y != null;
96096
96500
  }
96097
96501
  var SYSTEM_PROMPT = `You are a mobile QA agent controlling an app via screenshots.
96098
96502
  A screenshot of the current device screen is ALWAYS attached to the user message as an image. You can see it. Never claim that no screenshot was provided, missing, blank, or unavailable.
@@ -96106,20 +96510,51 @@ Valid shapes:
96106
96510
  {"type":"tap","label":"Allow","reason":"...","thoughts":"..."}
96107
96511
  {"type":"tap","id":"permission_allow_button","reason":"...","thoughts":"..."}
96108
96512
  {"type":"tap","x":0-1000,"y":0-1000,"reason":"...","thoughts":"..."}
96513
+ {"type":"tap","description":"the blue Login button","reason":"...","thoughts":"..."}
96514
+ {"type":"tap","x":0-1000,"y":0-1000,"double":true,"reason":"...","thoughts":"..."}
96515
+ {"type":"tap","x":0-1000,"y":0-1000,"durationMs":2000,"reason":"...","thoughts":"..."}
96516
+ {"type":"swipe","direction":"up|down|left|right","reason":"...","thoughts":"..."}
96517
+ {"type":"swipe","x":0-1000,"y":0-1000,"x2":0-1000,"y2":0-1000,"reason":"...","thoughts":"..."}
96518
+ {"type":"drag","x":0-1000,"y":0-1000,"x2":0-1000,"y2":0-1000,"reason":"...","thoughts":"..."}
96519
+ {"type":"type","text":"...","x":0-1000,"y":0-1000,"reason":"...","thoughts":"..."}
96520
+ {"type":"input","text":"...","id":"email_field","reason":"...","thoughts":"..."}
96109
96521
  {"type":"alert","alertAction":"accept","reason":"...","thoughts":"..."}
96110
- {"type":"type","text":"...","reason":"...","thoughts":"..."}
96522
+ {"type":"activate-app","appId":"com.example.app","reason":"...","thoughts":"..."}
96523
+ {"type":"terminate-app","appId":"com.example.app","reason":"...","thoughts":"..."}
96524
+ {"type":"restart-app","appId":"com.example.app","reason":"...","thoughts":"..."}
96525
+ {"type":"background-app","seconds":3,"reason":"...","thoughts":"..."}
96526
+ {"type":"open-url","url":"https://example.com","reason":"...","thoughts":"..."}
96527
+ {"type":"assert","assertion":"visible|not-visible","text":"...","reason":"...","thoughts":"..."}
96111
96528
  {"type":"wait","ms":500-3000,"reason":"...","thoughts":"..."}
96112
96529
  {"type":"verify","reason":"...","thoughts":"..."}
96113
96530
  {"type":"done","reason":"...","thoughts":"..."}
96114
96531
  {"type":"fail","reason":"...","thoughts":"..."}
96115
96532
 
96116
- Coordinates use a 0\u20131000 normalized grid (0,0 top-left of the attached screenshot). For in-app controls, tap using screenshot x,y from what you see \u2014 do not prefer label or id over those coordinates.
96533
+ Coordinates use a 0\u20131000 normalized grid (0,0 top-left of the attached screenshot). The user message also includes a screen snapshot (cleaned accessibility tree) with the same grid \u2014 x,y is each element's TOP-LEFT, not centre. Centre = x + width/2, y + height/2.
96534
+
96535
+ Use BOTH the screenshot and the screen snapshot:
96536
+ - Prefer snapshot "id=\u2026" when present (same as yoqa action tap --id).
96537
+ - System permission / notification dialogs: {"type":"tap","label":"Allow"} or {"type":"alert","alertAction":"accept"}. Never guess coordinates for these.
96538
+ - For other in-app controls, tap screenshot x,y of the visible control if there is no id.
96539
+ - If the snapshot is empty or the control is only drawn (no label/id), use the screenshot.
96540
+ - If a previous action failed (see Last action error), pick a different target from the snapshot + screenshot \u2014 do not repeat the same id/label/point.
96541
+
96542
+ Swipe direction is the FINGER movement (same as Inspector):
96543
+ - "down" (finger down, y 200\u2192800): scroll UP \u2014 reveal content above
96544
+ - "up" (finger up, y 800\u2192200): scroll DOWN \u2014 reveal content below / to the bottom
96545
+ - "left" / "right": horizontal paging
96546
+ When the case says "scroll up", emit direction "down". When it says "scroll down" or "to the bottom", emit direction "up". Never tap to scroll. Never emit wait as a stand-in for scrolling.
96117
96547
 
96118
96548
  Rules:
96119
- - Look at the attached screenshot before deciding. Base taps on what is visible now.
96120
- - For ordinary app UI, emit {"type":"tap","x":\u2026,"y":\u2026} using the screenshot grid. Include label or id only for system permission / notification dialogs.
96121
- - System permission / notification dialogs (Allow, Don't allow, While using the app): use {"type":"tap","label":"Allow"} or {"type":"alert","alertAction":"accept"}. Never guess coordinates for these buttons \u2014 coordinate taps often miss.
96122
- - If the expected result (or the goal of the instructions) is already visible, return verify or done \u2014 do not keep tapping the same control.
96549
+ - Look at the attached screenshot AND the screen snapshot before deciding.
96550
+ - Double-tap: {"type":"tap","double":true,...}. Long-press: {"type":"tap","durationMs":2000,...}. Drag a slider/chip with type drag, not swipe.
96551
+ - activate-app / terminate-app / restart-app / open-url / background-app match yoqa action. Omit appId to use the catalog app id provided in the user message.
96552
+ - Assert text on the tree with {"type":"assert","assertion":"visible","text":"\u2026"}.
96553
+ - To scroll a list or page, emit swipe \u2014 never a tap in the middle of the screen as a stand-in for scrolling.
96554
+ - "Scroll until you cannot scroll anymore" / "to the bottom": keep emitting swipe (direction "up") until a later screenshot shows the same content. Then verify. Unchanged screens after taps do not count \u2014 taps do not scroll.
96555
+ - You are given ONE current instruction. Complete only that instruction, then return verify or done. Later instructions are hidden on purpose \u2014 do not invent, skip ahead, or combine them.
96556
+ - Do not open unrelated tabs or screens (Wallet, Profile, \u2026) unless the current instruction says to. If the current target is not visible, scroll THIS screen to find it. If it cannot be found, fail with what you see.
96557
+ - When THIS instruction's expected result is visible (or the action is done and the UI has updated), return verify or done. That finishes this instruction only, not the whole case.
96123
96558
  - Splash / loading / blank screens: return wait (do not tap imaginary tabs).
96124
96559
  - Do not repeat the same tap target from Recent actions unless the screenshot still shows you are on the wrong screen.
96125
96560
  - On fail, say what you see (e.g. "splash logo only, no bottom tabs") \u2014 never "no screenshot".`;
@@ -96137,12 +96572,30 @@ function formatRecentActions(actions) {
96137
96572
  }
96138
96573
  return `${n}. tap (${action.x ?? "?"},${action.y ?? "?"}) \u2014 ${action.reason ?? ""}`;
96139
96574
  }
96575
+ if (action.type === "swipe") {
96576
+ if (action.direction) {
96577
+ return `${n}. swipe ${action.direction} \u2014 ${action.reason ?? ""}`;
96578
+ }
96579
+ return `${n}. swipe (${action.x ?? "?"},${action.y ?? "?"})\u2192(${action.x2 ?? "?"},${action.y2 ?? "?"}) \u2014 ${action.reason ?? ""}`;
96580
+ }
96140
96581
  if (action.type === "alert") {
96141
96582
  return `${n}. alert ${action.alertAction ?? "accept"} \u2014 ${action.reason ?? ""}`;
96142
96583
  }
96143
- if (action.type === "type") {
96584
+ if (action.type === "type" || action.type === "input") {
96144
96585
  return `${n}. type "${action.text ?? ""}" \u2014 ${action.reason ?? ""}`;
96145
96586
  }
96587
+ if (action.type === "drag") {
96588
+ return `${n}. drag (${action.x ?? "?"},${action.y ?? "?"})\u2192(${action.x2 ?? "?"},${action.y2 ?? "?"}) \u2014 ${action.reason ?? ""}`;
96589
+ }
96590
+ if (action.type === "assert") {
96591
+ return `${n}. assert ${action.assertion ?? "visible"} "${action.text ?? ""}" \u2014 ${action.reason ?? ""}`;
96592
+ }
96593
+ if (action.type === "activate-app" || action.type === "terminate-app" || action.type === "restart-app") {
96594
+ return `${n}. ${action.type} ${action.appId ?? ""} \u2014 ${action.reason ?? ""}`;
96595
+ }
96596
+ if (action.type === "open-url") {
96597
+ return `${n}. open-url ${action.url ?? ""} \u2014 ${action.reason ?? ""}`;
96598
+ }
96146
96599
  if (action.type === "wait") {
96147
96600
  return `${n}. wait ${action.ms ?? "?"}ms \u2014 ${action.reason ?? ""}`;
96148
96601
  }
@@ -96150,29 +96603,101 @@ function formatRecentActions(actions) {
96150
96603
  }).join(`
96151
96604
  `);
96152
96605
  }
96153
- function isAbsurdNoScreenshotFail(decision) {
96154
- if (decision.type !== "fail")
96155
- return false;
96156
- const text3 = `${decision.reason} ${decision.thoughts}`.toLowerCase();
96157
- return text3.includes("no screenshot") || text3.includes("screenshot provided") || text3.includes("screenshot was not") || text3.includes("without a screenshot") || text3.includes("missing screenshot") || text3.includes("screenshot unavailable") || text3.includes("cannot see the screen") || text3.includes("no image");
96606
+ var LIST_ITEM_START_RE = /^(?:\d+[.)]|\d+\s*[-\u2013]|[-*\u2022])\s+/;
96607
+ function splitInstructionSteps(instructions) {
96608
+ const trimmed = instructions.replace(/\r\n/g, `
96609
+ `).trim();
96610
+ if (!trimmed)
96611
+ return [""];
96612
+ const lines = trimmed.split(`
96613
+ `);
96614
+ const items = [];
96615
+ let current = [];
96616
+ let sawList = false;
96617
+ const pushCurrent = () => {
96618
+ const text3 = current.join(" ").replace(/\s+/g, " ").trim();
96619
+ if (text3)
96620
+ items.push(text3);
96621
+ current = [];
96622
+ };
96623
+ for (const rawLine of lines) {
96624
+ const line = rawLine.trim();
96625
+ if (!line)
96626
+ continue;
96627
+ if (LIST_ITEM_START_RE.test(line)) {
96628
+ sawList = true;
96629
+ pushCurrent();
96630
+ current = [line.replace(LIST_ITEM_START_RE, "").trim()];
96631
+ } else if (current.length > 0) {
96632
+ current.push(line);
96633
+ } else {
96634
+ current = [line];
96635
+ }
96636
+ }
96637
+ pushCurrent();
96638
+ if (!sawList)
96639
+ return [trimmed];
96640
+ return items.length > 0 ? items : [trimmed];
96641
+ }
96642
+ function flattenCaseInstructions(flows2) {
96643
+ const queue = [];
96644
+ for (const flow of flows2) {
96645
+ const steps = splitInstructionSteps(flow.instructions);
96646
+ const usable = steps.length > 0 ? steps : [""];
96647
+ for (let i = 0;i < usable.length; i++) {
96648
+ const last = i === usable.length - 1;
96649
+ queue.push({
96650
+ instructions: usable[i] ?? "",
96651
+ expectedResult: last ? flow.expectedResult : ""
96652
+ });
96653
+ }
96654
+ }
96655
+ return queue.length > 0 ? queue : [{ instructions: "", expectedResult: "" }];
96158
96656
  }
96159
- async function decideNextAction(input) {
96160
- const prompt = [
96657
+ function formatCompletedInstructions(completed) {
96658
+ if (completed.length === 0)
96659
+ return "(none \u2014 this is the first instruction)";
96660
+ return completed.map((text3, index) => `${index + 1}. ${text3}`).join(`
96661
+ `);
96662
+ }
96663
+ function formatDecidePrompt(input) {
96664
+ const ordinal = input.instructionOrdinal ?? input.stepIndex + 1;
96665
+ const total = input.instructionCount ?? 0;
96666
+ const remaining = total > 0 ? Math.max(0, total - ordinal) : 0;
96667
+ const expected = input.expectedResult.trim() ? input.expectedResult : "(none for this instruction \u2014 return verify once you have performed it and the UI has updated)";
96668
+ const progress = total > 0 ? `Instruction ${ordinal} of ${total} \u2014 do ONLY this instruction${remaining > 0 ? `; ${remaining} remain after it` : " (last instruction)"}` : `Step ${input.stepIndex + 1}`;
96669
+ const later = remaining > 0 ? `Later instructions exist (${remaining}) but are hidden. Do not invent or perform them. Do not open unrelated screens to skip ahead.` : "This is the last instruction of the case.";
96670
+ return [
96161
96671
  `App context: ${input.appContext || "(none)"}`,
96162
96672
  `Test case: ${input.caseTitle}`,
96163
- `Step ${input.stepIndex + 1}`,
96164
- `Instructions: ${input.instructions || "(none)"}`,
96165
- `Expected result: ${input.expectedResult || "(none)"}`,
96673
+ progress,
96674
+ "Completed instructions (already done \u2014 do not repeat):",
96675
+ formatCompletedInstructions(input.completedInstructions ?? []),
96676
+ `Current instruction (do ONLY this): ${input.instructions || "(none)"}`,
96677
+ `Expected result for this instruction: ${expected}`,
96678
+ later,
96679
+ `Catalog app id (activate/terminate/restart): ${input.defaultAppId || "(unknown)"}`,
96166
96680
  `Recent actions:
96167
96681
  ${formatRecentActions(input.recentActions ?? [])}`,
96168
- "Look at the attached screenshot image and decide the next action. A screenshot is attached.",
96682
+ `Last action error: ${input.lastError || "(none)"}`,
96683
+ "Screen snapshot (cleaned accessibility tree, 0\u20131000 grid, x,y is top-left):",
96684
+ input.screenSnapshot || "(unavailable)",
96685
+ "Look at the attached screenshot image AND the screen snapshot and decide the next action. A screenshot is attached.",
96169
96686
  'Reply with ONLY the JSON action object, including non-empty "reason" and "thoughts".'
96170
96687
  ].join(`
96171
96688
  `);
96689
+ }
96690
+ function isAbsurdNoScreenshotFail(decision) {
96691
+ if (decision.type !== "fail")
96692
+ return false;
96693
+ const text3 = `${decision.reason} ${decision.thoughts}`.toLowerCase();
96694
+ return text3.includes("no screenshot") || text3.includes("screenshot provided") || text3.includes("screenshot was not") || text3.includes("without a screenshot") || text3.includes("missing screenshot") || text3.includes("screenshot unavailable") || text3.includes("cannot see the screen") || text3.includes("no image");
96695
+ }
96696
+ async function decideNextAction(input) {
96172
96697
  return completeVision(input.auth, {
96173
96698
  schema: agentDecisionSchema,
96174
96699
  system: SYSTEM_PROMPT,
96175
- prompt,
96700
+ prompt: formatDecidePrompt(input),
96176
96701
  imageBase64: input.imageBase64
96177
96702
  });
96178
96703
  }
@@ -96187,18 +96712,24 @@ function buildScriptFromDecisions(decisions, sourceRunId) {
96187
96712
  type: "tap",
96188
96713
  x: decision.x,
96189
96714
  y: decision.y,
96715
+ ...decision.double ? { double: true } : {},
96716
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
96190
96717
  reason: decision.reason
96191
96718
  });
96192
96719
  } else if (decision.label) {
96193
96720
  actions.push({
96194
96721
  type: "tap",
96195
96722
  label: decision.label,
96723
+ ...decision.double ? { double: true } : {},
96724
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
96196
96725
  reason: decision.reason
96197
96726
  });
96198
96727
  } else if (decision.id) {
96199
96728
  actions.push({
96200
96729
  type: "tap",
96201
96730
  id: decision.id,
96731
+ ...decision.double ? { double: true } : {},
96732
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
96202
96733
  reason: decision.reason
96203
96734
  });
96204
96735
  } else {
@@ -96209,12 +96740,70 @@ function buildScriptFromDecisions(decisions, sourceRunId) {
96209
96740
  reason: decision.reason
96210
96741
  });
96211
96742
  }
96212
- } else if (decision.type === "type") {
96743
+ } else if (decision.type === "swipe") {
96744
+ const swipe = resolveSwipeNorm(decision);
96745
+ if (!swipe)
96746
+ continue;
96747
+ actions.push({
96748
+ type: "swipe",
96749
+ x: swipe.x,
96750
+ y: swipe.y,
96751
+ x2: swipe.x2,
96752
+ y2: swipe.y2,
96753
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
96754
+ reason: decision.reason
96755
+ });
96756
+ } else if (decision.type === "type" || decision.type === "input") {
96213
96757
  actions.push({
96214
96758
  type: "type",
96215
96759
  text: decision.text ?? "",
96216
96760
  reason: decision.reason
96217
96761
  });
96762
+ } else if (decision.type === "drag") {
96763
+ if (decision.x == null || decision.y == null || decision.x2 == null || decision.y2 == null) {
96764
+ continue;
96765
+ }
96766
+ actions.push({
96767
+ type: "drag",
96768
+ x: decision.x,
96769
+ y: decision.y,
96770
+ x2: decision.x2,
96771
+ y2: decision.y2,
96772
+ ...decision.durationMs != null ? { durationMs: decision.durationMs } : {},
96773
+ reason: decision.reason
96774
+ });
96775
+ } else if (decision.type === "activate-app" || decision.type === "terminate-app" || decision.type === "restart-app") {
96776
+ if (!decision.appId)
96777
+ continue;
96778
+ actions.push({
96779
+ type: decision.type,
96780
+ appId: decision.appId,
96781
+ reason: decision.reason
96782
+ });
96783
+ } else if (decision.type === "background-app") {
96784
+ actions.push({
96785
+ type: "background-app",
96786
+ ...decision.seconds != null ? { seconds: decision.seconds } : {},
96787
+ reason: decision.reason
96788
+ });
96789
+ } else if (decision.type === "open-url") {
96790
+ if (!decision.url)
96791
+ continue;
96792
+ actions.push({
96793
+ type: "open-url",
96794
+ url: decision.url,
96795
+ reason: decision.reason
96796
+ });
96797
+ } else if (decision.type === "assert") {
96798
+ if (!decision.text?.trim())
96799
+ continue;
96800
+ actions.push({
96801
+ type: "assert",
96802
+ assertion: decision.assertion === "not-visible" ? "not-visible" : "visible",
96803
+ text: decision.text,
96804
+ ...decision.timeoutMs != null ? { timeoutMs: decision.timeoutMs } : {},
96805
+ reason: decision.reason
96806
+ });
96218
96807
  } else if (decision.type === "wait") {
96219
96808
  actions.push({
96220
96809
  type: "wait",
@@ -98096,8 +98685,8 @@ ${treeSummary || "(none)"}`;
98096
98685
  }
98097
98686
 
98098
98687
  // src/domains/devices/interaction.ts
98099
- async function getScreen(session, options = {}) {
98100
- abortAllMjpegProxies();
98688
+ var MJPEG_PAUSE_SETTLE_MS = 150;
98689
+ async function readScreen(session, options) {
98101
98690
  const raw2 = await session.pageSource();
98102
98691
  const window2 = await session.getWindowSize();
98103
98692
  if (options.full) {
@@ -98110,6 +98699,14 @@ async function getScreen(session, options = {}) {
98110
98699
  elements: cleaned.elements
98111
98700
  };
98112
98701
  }
98702
+ async function getScreen(session, options = {}) {
98703
+ if (options.pauseMjpeg !== false) {
98704
+ const paused = abortAllMjpegProxies();
98705
+ if (paused)
98706
+ await Bun.sleep(MJPEG_PAUSE_SETTLE_MS);
98707
+ }
98708
+ return await readScreen(session, options);
98709
+ }
98113
98710
 
98114
98711
  class ActionValidationError extends Error {
98115
98712
  constructor(message) {
@@ -98164,7 +98761,9 @@ async function performAction(session, body) {
98164
98761
  throw new ActionValidationError(`${body.kind} requires x,y,x2,y2`);
98165
98762
  }
98166
98763
  if (body.kind === "swipe") {
98167
- await session.swipe(x, y, body.x2, body.y2, body.durationMs);
98764
+ await session.swipe(x, y, body.x2, body.y2, body.durationMs, {
98765
+ coordSpace: "screenshot"
98766
+ });
98168
98767
  } else {
98169
98768
  await session.drag(x, y, body.x2, body.y2, body.durationMs);
98170
98769
  }
@@ -98232,14 +98831,77 @@ var defaultClock = {
98232
98831
  sleep: (ms) => new Promise((resolve7) => setTimeout(resolve7, ms)),
98233
98832
  now: () => Date.now()
98234
98833
  };
98834
+ var noopSetCurrentCommand = async () => {};
98835
+ async function withCurrentCommand(setCurrentCommand, command, work) {
98836
+ await setCurrentCommand(command);
98837
+ try {
98838
+ return await work();
98839
+ } finally {
98840
+ await setCurrentCommand(null);
98841
+ }
98842
+ }
98843
+ function isRetriableActionError(error40) {
98844
+ return error40 instanceof ActionNotFoundError || error40 instanceof ActionValidationError;
98845
+ }
98846
+ async function readCleanedTree(readScreen2, session) {
98847
+ try {
98848
+ const screen = await readScreen2(session);
98849
+ const elements = screen.elements ?? [];
98850
+ return { snapshot: formatScreenSnapshot(elements), elements };
98851
+ } catch (error40) {
98852
+ if (isDeadSessionError(error40))
98853
+ throw error40;
98854
+ const message = error40 instanceof Error ? error40.message : String(error40);
98855
+ return { snapshot: `(screen tree unavailable: ${message})`, elements: [] };
98856
+ }
98857
+ }
98858
+ async function runTextAssert(input) {
98859
+ const deadline = input.clock.now() + input.timeoutMs;
98860
+ for (;; ) {
98861
+ if (input.isAborted()) {
98862
+ throw new Error("Aborted");
98863
+ }
98864
+ const screen = await input.readScreen(input.session);
98865
+ const found = screenHasText(screen.elements, input.text);
98866
+ if (input.assertion === "visible" && found)
98867
+ return;
98868
+ if (input.assertion === "not-visible" && !found)
98869
+ return;
98870
+ if (input.clock.now() >= deadline) {
98871
+ throw new Error(input.assertion === "visible" ? `Expected visible text not found within ${Math.round(input.timeoutMs / 1000)}s: ${input.text}` : `Unexpected text still visible after ${Math.round(input.timeoutMs / 1000)}s: ${input.text}`);
98872
+ }
98873
+ await input.clock.sleep(400);
98874
+ }
98875
+ }
98876
+ function commandForDecision(decision, defaultAppId) {
98877
+ if (decision.type === "verify" || decision.type === "done" || decision.type === "fail") {
98878
+ return null;
98879
+ }
98880
+ if (decision.type === "wait") {
98881
+ const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
98882
+ return formatSleepShellLine(waitMs / 1000);
98883
+ }
98884
+ if (decision.type === "assert") {
98885
+ const assertion = decision.assertion === "not-visible" ? "not-visible" : "visible";
98886
+ const timeoutMs = Math.min(60000, Math.max(1000, decision.timeoutMs ?? 5000));
98887
+ return formatAssertShellLine({
98888
+ assertion,
98889
+ text: decision.text ?? "",
98890
+ timeoutSeconds: timeoutMs / 1000
98891
+ });
98892
+ }
98893
+ const body = decisionToActionRequest(decision, { defaultAppId });
98894
+ return body ? formatActionShellLine(body) : null;
98895
+ }
98235
98896
  async function executeScriptCase(deps) {
98236
98897
  const perform = deps.performAction ?? performAction;
98237
- const readScreen = deps.readScreen ?? (async (session) => {
98898
+ const readScreen2 = deps.readScreen ?? (async (session) => {
98238
98899
  const screen = await getScreen(session, { full: false });
98239
98900
  return { elements: screen.elements };
98240
98901
  });
98241
98902
  const clock = deps.clock ?? defaultClock;
98242
98903
  const settleMs = deps.settleMs ?? POST_ACTION_SETTLE_MS;
98904
+ const setCurrentCommand = deps.setCurrentCommand ?? noopSetCurrentCommand;
98243
98905
  let stepIdx = 0;
98244
98906
  let lastScreenshotUri = null;
98245
98907
  try {
@@ -98264,108 +98926,274 @@ async function executeScriptCase(deps) {
98264
98926
  tapBody.x = action.x;
98265
98927
  if (action.y != null)
98266
98928
  tapBody.y = action.y;
98267
- await perform(deps.session, tapBody);
98268
- await clock.sleep(settleMs);
98269
- await deps.appendStep({
98270
- idx: stepIdx,
98271
- action: {
98272
- type: "tap",
98273
- x: action.x,
98274
- y: action.y,
98275
- label: action.label,
98276
- id: action.id,
98277
- reason: action.reason ?? "Replayed saved script tap",
98278
- thoughts: "Replaying the saved script without calling the AI agent."
98279
- },
98280
- screenshotUri: shot.path,
98281
- ok: true,
98282
- latencyMs,
98283
- detail: action.reason ?? action.label ?? action.id ?? null
98929
+ if (action.double)
98930
+ tapBody.double = true;
98931
+ if (action.durationMs != null)
98932
+ tapBody.durationMs = action.durationMs;
98933
+ const command = formatActionShellLine(tapBody);
98934
+ await withCurrentCommand(setCurrentCommand, command, async () => {
98935
+ await perform(deps.session, tapBody);
98936
+ await clock.sleep(settleMs);
98937
+ await deps.appendStep({
98938
+ idx: stepIdx,
98939
+ action: {
98940
+ type: "tap",
98941
+ x: action.x,
98942
+ y: action.y,
98943
+ label: action.label,
98944
+ id: action.id,
98945
+ reason: action.reason ?? "Replayed saved script tap",
98946
+ thoughts: "Replaying the saved script without calling the AI agent."
98947
+ },
98948
+ screenshotUri: shot.path,
98949
+ ok: true,
98950
+ latencyMs,
98951
+ detail: action.reason ?? action.label ?? action.id ?? null,
98952
+ command
98953
+ });
98954
+ });
98955
+ } else if (action.type === "swipe") {
98956
+ const swipeBody = {
98957
+ kind: "swipe",
98958
+ x: action.x,
98959
+ y: action.y,
98960
+ x2: action.x2,
98961
+ y2: action.y2,
98962
+ ...action.durationMs != null ? { durationMs: action.durationMs } : {}
98963
+ };
98964
+ const command = formatActionShellLine(swipeBody);
98965
+ await withCurrentCommand(setCurrentCommand, command, async () => {
98966
+ await perform(deps.session, swipeBody);
98967
+ await clock.sleep(settleMs);
98968
+ await deps.appendStep({
98969
+ idx: stepIdx,
98970
+ action: {
98971
+ type: "swipe",
98972
+ x: action.x,
98973
+ y: action.y,
98974
+ x2: action.x2,
98975
+ y2: action.y2,
98976
+ durationMs: action.durationMs,
98977
+ reason: action.reason ?? "Replayed saved script swipe",
98978
+ thoughts: "Replaying the saved script without calling the AI agent."
98979
+ },
98980
+ screenshotUri: shot.path,
98981
+ ok: true,
98982
+ latencyMs,
98983
+ detail: action.reason ?? null,
98984
+ command
98985
+ });
98986
+ });
98987
+ } else if (action.type === "drag") {
98988
+ const dragBody = {
98989
+ kind: "drag",
98990
+ x: action.x,
98991
+ y: action.y,
98992
+ x2: action.x2,
98993
+ y2: action.y2,
98994
+ ...action.durationMs != null ? { durationMs: action.durationMs } : {}
98995
+ };
98996
+ const command = formatActionShellLine(dragBody);
98997
+ await withCurrentCommand(setCurrentCommand, command, async () => {
98998
+ await perform(deps.session, dragBody);
98999
+ await clock.sleep(settleMs);
99000
+ await deps.appendStep({
99001
+ idx: stepIdx,
99002
+ action: {
99003
+ type: "drag",
99004
+ x: action.x,
99005
+ y: action.y,
99006
+ x2: action.x2,
99007
+ y2: action.y2,
99008
+ durationMs: action.durationMs,
99009
+ reason: action.reason ?? "Replayed saved script drag",
99010
+ thoughts: "Replaying the saved script without calling the AI agent."
99011
+ },
99012
+ screenshotUri: shot.path,
99013
+ ok: true,
99014
+ latencyMs,
99015
+ detail: action.reason ?? null,
99016
+ command
99017
+ });
99018
+ });
99019
+ } else if (action.type === "activate-app" || action.type === "terminate-app" || action.type === "restart-app") {
99020
+ const appBody = { kind: action.type, appId: action.appId };
99021
+ const command = formatActionShellLine(appBody);
99022
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99023
+ await perform(deps.session, appBody);
99024
+ await clock.sleep(settleMs);
99025
+ await deps.appendStep({
99026
+ idx: stepIdx,
99027
+ action: {
99028
+ type: action.type,
99029
+ appId: action.appId,
99030
+ reason: action.reason ?? `Replayed saved script ${action.type}`,
99031
+ thoughts: "Replaying the saved script without calling the AI agent."
99032
+ },
99033
+ screenshotUri: shot.path,
99034
+ ok: true,
99035
+ latencyMs,
99036
+ detail: action.reason ?? action.appId,
99037
+ command
99038
+ });
99039
+ });
99040
+ } else if (action.type === "background-app") {
99041
+ const backgroundBody = {
99042
+ kind: "background-app",
99043
+ ...action.seconds != null ? { seconds: action.seconds } : {}
99044
+ };
99045
+ const command = formatActionShellLine(backgroundBody);
99046
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99047
+ await perform(deps.session, backgroundBody);
99048
+ await clock.sleep(settleMs);
99049
+ await deps.appendStep({
99050
+ idx: stepIdx,
99051
+ action: {
99052
+ type: "background-app",
99053
+ seconds: action.seconds,
99054
+ reason: action.reason ?? "Replayed saved script background-app",
99055
+ thoughts: "Replaying the saved script without calling the AI agent."
99056
+ },
99057
+ screenshotUri: shot.path,
99058
+ ok: true,
99059
+ latencyMs,
99060
+ detail: action.reason ?? null,
99061
+ command
99062
+ });
99063
+ });
99064
+ } else if (action.type === "open-url") {
99065
+ const urlBody = { kind: "open-url", url: action.url };
99066
+ const command = formatActionShellLine(urlBody);
99067
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99068
+ await perform(deps.session, urlBody);
99069
+ await clock.sleep(settleMs);
99070
+ await deps.appendStep({
99071
+ idx: stepIdx,
99072
+ action: {
99073
+ type: "open-url",
99074
+ url: action.url,
99075
+ reason: action.reason ?? "Replayed saved script open-url",
99076
+ thoughts: "Replaying the saved script without calling the AI agent."
99077
+ },
99078
+ screenshotUri: shot.path,
99079
+ ok: true,
99080
+ latencyMs,
99081
+ detail: action.reason ?? action.url,
99082
+ command
99083
+ });
98284
99084
  });
98285
99085
  } else if (action.type === "type") {
98286
- await perform(deps.session, { kind: "input", text: action.text });
98287
- await clock.sleep(settleMs);
98288
- await deps.appendStep({
98289
- idx: stepIdx,
98290
- action: {
98291
- type: "type",
98292
- text: action.text,
98293
- reason: action.reason ?? "Replayed saved script type",
98294
- thoughts: "Replaying the saved script without calling the AI agent."
98295
- },
98296
- screenshotUri: shot.path,
98297
- ok: true,
98298
- latencyMs,
98299
- detail: action.reason ?? null
99086
+ const typeBody = { kind: "input", text: action.text };
99087
+ const command = formatActionShellLine(typeBody);
99088
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99089
+ await perform(deps.session, typeBody);
99090
+ await clock.sleep(settleMs);
99091
+ await deps.appendStep({
99092
+ idx: stepIdx,
99093
+ action: {
99094
+ type: "type",
99095
+ text: action.text,
99096
+ reason: action.reason ?? "Replayed saved script type",
99097
+ thoughts: "Replaying the saved script without calling the AI agent."
99098
+ },
99099
+ screenshotUri: shot.path,
99100
+ ok: true,
99101
+ latencyMs,
99102
+ detail: action.reason ?? null,
99103
+ command
99104
+ });
98300
99105
  });
98301
99106
  } else if (action.type === "assert") {
98302
99107
  const timeoutMs = action.timeoutMs ?? 5000;
98303
99108
  const assertion = action.assertion;
98304
- const deadline = clock.now() + timeoutMs;
98305
- for (;; ) {
99109
+ const command = formatAssertShellLine({
99110
+ assertion,
99111
+ text: action.text,
99112
+ timeoutSeconds: timeoutMs / 1000
99113
+ });
99114
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99115
+ const deadline = clock.now() + timeoutMs;
99116
+ for (;; ) {
99117
+ if (deps.isAborted()) {
99118
+ return;
99119
+ }
99120
+ const screen = await readScreen2(deps.session);
99121
+ const found = screenHasText(screen.elements, action.text);
99122
+ if (assertion === "visible" && found)
99123
+ break;
99124
+ if (assertion === "not-visible" && !found)
99125
+ break;
99126
+ if (clock.now() >= deadline) {
99127
+ throw new Error(assertion === "visible" ? `Expected visible text not found within ${Math.round(timeoutMs / 1000)}s: ${action.text}` : `Unexpected text still visible after ${Math.round(timeoutMs / 1000)}s: ${action.text}`);
99128
+ }
99129
+ await clock.sleep(400);
99130
+ }
98306
99131
  if (deps.isAborted()) {
98307
- return "cancelled";
99132
+ return;
98308
99133
  }
98309
- const screen = await readScreen(deps.session);
98310
- const found = screenHasText(screen.elements, action.text);
98311
- if (assertion === "visible" && found)
98312
- break;
98313
- if (assertion === "not-visible" && !found)
98314
- break;
98315
- if (clock.now() >= deadline) {
98316
- throw new Error(assertion === "visible" ? `Expected visible text not found within ${Math.round(timeoutMs / 1000)}s: ${action.text}` : `Unexpected text still visible after ${Math.round(timeoutMs / 1000)}s: ${action.text}`);
98317
- }
98318
- await clock.sleep(400);
98319
- }
98320
- await deps.appendStep({
98321
- idx: stepIdx,
98322
- action: {
98323
- type: "assert",
98324
- assertion,
98325
- text: action.text,
98326
- timeoutMs,
98327
- reason: action.reason ?? `Assert ${assertion}: ${action.text}`,
98328
- thoughts: "Replaying the saved script without calling the AI agent."
98329
- },
98330
- screenshotUri: shot.path,
98331
- ok: true,
98332
- latencyMs,
98333
- detail: action.reason ?? `${assertion}: ${action.text}`
99134
+ await deps.appendStep({
99135
+ idx: stepIdx,
99136
+ action: {
99137
+ type: "assert",
99138
+ assertion,
99139
+ text: action.text,
99140
+ timeoutMs,
99141
+ reason: action.reason ?? `Assert ${assertion}: ${action.text}`,
99142
+ thoughts: "Replaying the saved script without calling the AI agent."
99143
+ },
99144
+ screenshotUri: shot.path,
99145
+ ok: true,
99146
+ latencyMs,
99147
+ detail: action.reason ?? `${assertion}: ${action.text}`,
99148
+ command
99149
+ });
98334
99150
  });
99151
+ if (deps.isAborted()) {
99152
+ return "cancelled";
99153
+ }
98335
99154
  } else if (action.type === "alert") {
98336
- await perform(deps.session, {
99155
+ const alertBody = {
98337
99156
  kind: "alert",
98338
99157
  alertAction: action.alertAction === "dismiss" ? "dismiss" : "accept"
98339
- });
98340
- await clock.sleep(settleMs);
98341
- await deps.appendStep({
98342
- idx: stepIdx,
98343
- action: {
98344
- type: "alert",
98345
- alertAction: action.alertAction === "dismiss" ? "dismiss" : "accept",
98346
- reason: action.reason ?? "Replayed saved script alert",
98347
- thoughts: "Replaying the saved script without calling the AI agent."
98348
- },
98349
- screenshotUri: shot.path,
98350
- ok: true,
98351
- latencyMs,
98352
- detail: action.reason ?? action.alertAction ?? "accept"
99158
+ };
99159
+ const command = formatActionShellLine(alertBody);
99160
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99161
+ await perform(deps.session, alertBody);
99162
+ await clock.sleep(settleMs);
99163
+ await deps.appendStep({
99164
+ idx: stepIdx,
99165
+ action: {
99166
+ type: "alert",
99167
+ alertAction: action.alertAction === "dismiss" ? "dismiss" : "accept",
99168
+ reason: action.reason ?? "Replayed saved script alert",
99169
+ thoughts: "Replaying the saved script without calling the AI agent."
99170
+ },
99171
+ screenshotUri: shot.path,
99172
+ ok: true,
99173
+ latencyMs,
99174
+ detail: action.reason ?? action.alertAction ?? "accept",
99175
+ command
99176
+ });
98353
99177
  });
98354
99178
  } else {
98355
99179
  const waitMs = Math.min(3000, Math.max(500, action.ms));
98356
- await clock.sleep(waitMs);
98357
- await deps.appendStep({
98358
- idx: stepIdx,
98359
- action: {
98360
- type: "wait",
98361
- ms: waitMs,
98362
- reason: action.reason ?? `wait ${waitMs}ms`,
98363
- thoughts: "Replaying the saved script without calling the AI agent."
98364
- },
98365
- screenshotUri: shot.path,
98366
- ok: true,
98367
- latencyMs,
98368
- detail: action.reason ?? `wait ${waitMs}ms`
99180
+ const command = formatSleepShellLine(waitMs / 1000);
99181
+ await withCurrentCommand(setCurrentCommand, command, async () => {
99182
+ await clock.sleep(waitMs);
99183
+ await deps.appendStep({
99184
+ idx: stepIdx,
99185
+ action: {
99186
+ type: "wait",
99187
+ ms: waitMs,
99188
+ reason: action.reason ?? `wait ${waitMs}ms`,
99189
+ thoughts: "Replaying the saved script without calling the AI agent."
99190
+ },
99191
+ screenshotUri: shot.path,
99192
+ ok: true,
99193
+ latencyMs,
99194
+ detail: action.reason ?? `wait ${waitMs}ms`,
99195
+ command
99196
+ });
98369
99197
  });
98370
99198
  }
98371
99199
  stepIdx += 1;
@@ -98380,7 +99208,8 @@ async function executeScriptCase(deps) {
98380
99208
  screenshotUri: lastScreenshotUri,
98381
99209
  ok: true,
98382
99210
  latencyMs: 0,
98383
- detail: "Saved script completed"
99211
+ detail: "Saved script completed",
99212
+ command: null
98384
99213
  });
98385
99214
  return "passed";
98386
99215
  } catch (error40) {
@@ -98398,7 +99227,8 @@ async function executeScriptCase(deps) {
98398
99227
  screenshotUri: lastScreenshotUri,
98399
99228
  ok: false,
98400
99229
  latencyMs: 0,
98401
- detail: message
99230
+ detail: message,
99231
+ command: null
98402
99232
  });
98403
99233
  return "errored";
98404
99234
  }
@@ -98406,165 +99236,214 @@ async function executeScriptCase(deps) {
98406
99236
  async function executeAgentCase(deps) {
98407
99237
  const decide = deps.decide ?? decideNextAction;
98408
99238
  const perform = deps.performAction ?? performAction;
99239
+ const readScreen2 = deps.readScreen ?? (async (session) => {
99240
+ const screen = await getScreen(session, { full: false });
99241
+ return { elements: screen.elements };
99242
+ });
98409
99243
  const clock = deps.clock ?? defaultClock;
98410
99244
  const settleMs = deps.settleMs ?? POST_ACTION_SETTLE_MS;
98411
99245
  const maxSteps = deps.maxStepsPerCase ?? MAX_STEPS_PER_CASE;
99246
+ const setCurrentCommand = deps.setCurrentCommand ?? noopSetCurrentCommand;
98412
99247
  let stepIdx = 0;
98413
99248
  let caseStatus = "passed";
98414
99249
  let caseError = null;
98415
99250
  let lastScreenshotUri = null;
98416
99251
  const recordedDecisions = [];
99252
+ const decideOnce = async (input) => {
99253
+ const payload = {
99254
+ auth: deps.auth,
99255
+ appContext: deps.appContext,
99256
+ caseTitle: deps.catalogCase.name,
99257
+ instructions: input.flow.instructions,
99258
+ expectedResult: input.flow.expectedResult,
99259
+ stepIndex: stepIdx,
99260
+ imageBase64: input.imageBase64,
99261
+ recentActions: input.recentActions,
99262
+ screenSnapshot: input.screenSnapshot,
99263
+ lastError: input.lastError,
99264
+ defaultAppId: deps.defaultAppId,
99265
+ completedInstructions: [...input.completedInstructions],
99266
+ instructionOrdinal: input.instructionOrdinal,
99267
+ instructionCount: input.instructionCount
99268
+ };
99269
+ let decision = await decide(payload);
99270
+ if (isAbsurdNoScreenshotFail(decision)) {
99271
+ decision = await decide(payload);
99272
+ if (isAbsurdNoScreenshotFail(decision)) {
99273
+ decision = {
99274
+ type: "fail",
99275
+ reason: "Model ignored the attached screenshot (claimed none was provided). Try a stronger vision model in Settings \u2192 Provider.",
99276
+ thoughts: "The vision model returned fail claiming no screenshot was available even though an image was attached to the request. After one retry it still denied the screenshot, so the step was marked failed."
99277
+ };
99278
+ }
99279
+ }
99280
+ decision = coerceScrollIntentToSwipe(decision);
99281
+ decision = continueScrollingInsteadOfComplete({
99282
+ decision,
99283
+ instructions: input.flow.instructions,
99284
+ expectedResult: input.flow.expectedResult,
99285
+ recentActions: input.recentActions,
99286
+ lastSwipeMovedScreen: input.lastSwipeMovedScreen
99287
+ });
99288
+ if ((decision.type === "activate-app" || decision.type === "terminate-app" || decision.type === "restart-app") && !decision.appId && deps.defaultAppId) {
99289
+ decision = { ...decision, appId: deps.defaultAppId };
99290
+ }
99291
+ return decision;
99292
+ };
99293
+ const applyDecision = async (decision) => {
99294
+ if (decision.type === "wait") {
99295
+ const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
99296
+ await clock.sleep(waitMs);
99297
+ return "continue";
99298
+ }
99299
+ if (decision.type === "verify" || decision.type === "done") {
99300
+ return "done";
99301
+ }
99302
+ if (decision.type === "fail") {
99303
+ return "fail";
99304
+ }
99305
+ if (decision.type === "assert") {
99306
+ const assertion = decision.assertion === "not-visible" ? "not-visible" : "visible";
99307
+ const timeoutMs = Math.min(60000, Math.max(1000, decision.timeoutMs ?? 5000));
99308
+ await runTextAssert({
99309
+ session: deps.session,
99310
+ readScreen: readScreen2,
99311
+ clock,
99312
+ isAborted: deps.isAborted,
99313
+ assertion,
99314
+ text: decision.text ?? "",
99315
+ timeoutMs
99316
+ });
99317
+ return "continue";
99318
+ }
99319
+ const body = decisionToActionRequest(decision, { defaultAppId: deps.defaultAppId });
99320
+ if (!body) {
99321
+ throw new Error(`${decision.type} is missing required fields`);
99322
+ }
99323
+ try {
99324
+ await perform(deps.session, body);
99325
+ } catch (error40) {
99326
+ if (error40 instanceof ActionNotFoundError && (body.id || body.label) && decision.x != null && decision.y != null) {
99327
+ await perform(deps.session, {
99328
+ kind: body.kind === "input" ? "input" : "tap",
99329
+ x: decision.x,
99330
+ y: decision.y,
99331
+ text: body.text,
99332
+ double: body.double,
99333
+ durationMs: body.durationMs
99334
+ });
99335
+ } else {
99336
+ throw error40;
99337
+ }
99338
+ }
99339
+ await clock.sleep(settleMs);
99340
+ return "continue";
99341
+ };
98417
99342
  try {
98418
- const flows2 = deps.catalogCase.flows.length > 0 ? deps.catalogCase.flows : [{ id: "empty", instructions: "", expectedResult: "", flowId: null }];
98419
- for (const flow of flows2) {
99343
+ const instructionQueue = flattenCaseInstructions(deps.catalogCase.flows.length > 0 ? deps.catalogCase.flows : [{ id: "empty", instructions: "", expectedResult: "", flowId: null }]);
99344
+ const recentActions = [];
99345
+ const completedInstructions = [];
99346
+ let prevFingerprint = null;
99347
+ for (let instructionIndex = 0;instructionIndex < instructionQueue.length; instructionIndex++) {
99348
+ const instruction = instructionQueue[instructionIndex];
99349
+ if (!instruction)
99350
+ break;
98420
99351
  if (deps.isAborted()) {
98421
99352
  caseStatus = "cancelled";
98422
99353
  break;
98423
99354
  }
98424
- const recentActions = [];
98425
- let flowDone = false;
98426
- for (let attempt = 0;attempt < maxSteps && !flowDone; attempt++) {
99355
+ let instructionDone = false;
99356
+ for (let attempt = 0;attempt < maxSteps && !instructionDone; attempt++) {
98427
99357
  if (deps.isAborted()) {
98428
99358
  caseStatus = "cancelled";
98429
- flowDone = true;
99359
+ instructionDone = true;
98430
99360
  break;
98431
99361
  }
98432
99362
  const shotStarted = clock.now();
98433
99363
  const shot = await deps.session.screenshot();
98434
99364
  lastScreenshotUri = shot.path;
99365
+ const tree = await readCleanedTree(readScreen2, deps.session);
99366
+ const fingerprint = screenshotFingerprint(shot.base64);
99367
+ const lastAction = recentActions.at(-1);
99368
+ const lastSwipeMovedScreen = lastAction?.type === "swipe" && prevFingerprint != null && fingerprint !== prevFingerprint;
98435
99369
  if (deps.isAborted()) {
98436
99370
  caseStatus = "cancelled";
98437
- flowDone = true;
99371
+ instructionDone = true;
98438
99372
  break;
98439
99373
  }
98440
- let decision = await decide({
98441
- auth: deps.auth,
98442
- appContext: deps.appContext,
98443
- caseTitle: deps.catalogCase.name,
98444
- instructions: flow.instructions,
98445
- expectedResult: flow.expectedResult,
98446
- stepIndex: stepIdx,
99374
+ const decideInput = {
99375
+ flow: instruction,
98447
99376
  imageBase64: shot.base64,
98448
- recentActions
98449
- });
98450
- if (isAbsurdNoScreenshotFail(decision)) {
98451
- decision = await decide({
98452
- auth: deps.auth,
98453
- appContext: deps.appContext,
98454
- caseTitle: deps.catalogCase.name,
98455
- instructions: flow.instructions,
98456
- expectedResult: flow.expectedResult,
98457
- stepIndex: stepIdx,
98458
- imageBase64: shot.base64,
98459
- recentActions
98460
- });
98461
- if (isAbsurdNoScreenshotFail(decision)) {
98462
- decision = {
98463
- type: "fail",
98464
- reason: "Model ignored the attached screenshot (claimed none was provided). Try a stronger vision model in Settings \u2192 Provider.",
98465
- thoughts: "The vision model returned fail claiming no screenshot was available even though an image was attached to the request. After one retry it still denied the screenshot, so the step was marked failed."
98466
- };
98467
- }
98468
- }
99377
+ recentActions,
99378
+ screenSnapshot: tree.snapshot,
99379
+ lastSwipeMovedScreen,
99380
+ completedInstructions,
99381
+ instructionOrdinal: instructionIndex + 1,
99382
+ instructionCount: instructionQueue.length
99383
+ };
99384
+ let decision = await decideOnce(decideInput);
99385
+ prevFingerprint = fingerprint;
98469
99386
  const latencyMs = clock.now() - shotStarted;
98470
99387
  if (deps.isAborted()) {
98471
99388
  caseStatus = "cancelled";
98472
- flowDone = true;
99389
+ instructionDone = true;
98473
99390
  break;
98474
99391
  }
98475
- recentActions.push(decision);
98476
- recordedDecisions.push(decision);
98477
- if (decision.type === "tap") {
98478
- const tapBody = { kind: "tap" };
98479
- if (prefersScreenshotTap(decision)) {
98480
- tapBody.x = decision.x;
98481
- tapBody.y = decision.y;
98482
- } else if (decision.label) {
98483
- tapBody.label = decision.label;
98484
- if (decision.id)
98485
- tapBody.id = decision.id;
98486
- } else if (decision.id) {
98487
- tapBody.id = decision.id;
98488
- } else {
98489
- tapBody.x = decision.x ?? 500;
98490
- tapBody.y = decision.y ?? 500;
99392
+ const applyWithRetry = async () => {
99393
+ try {
99394
+ return await applyDecision(decision);
99395
+ } catch (error40) {
99396
+ if (!isRetriableActionError(error40) || deps.isAborted())
99397
+ throw error40;
99398
+ const failed = decision;
99399
+ decision = await decideOnce({
99400
+ ...decideInput,
99401
+ recentActions: [...recentActions, failed],
99402
+ lastError: error40 instanceof Error ? error40.message : String(error40)
99403
+ });
99404
+ const retryCommand = commandForDecision(decision, deps.defaultAppId);
99405
+ await setCurrentCommand(retryCommand);
99406
+ return await applyDecision(decision);
98491
99407
  }
98492
- await perform(deps.session, tapBody);
98493
- await clock.sleep(settleMs);
98494
- await deps.appendStep({
98495
- idx: stepIdx,
98496
- action: decision,
98497
- screenshotUri: shot.path,
98498
- ok: true,
98499
- latencyMs,
98500
- detail: decision.reason ?? null
98501
- });
98502
- } else if (decision.type === "alert") {
98503
- await perform(deps.session, {
98504
- kind: "alert",
98505
- alertAction: decision.alertAction === "dismiss" ? "dismiss" : "accept"
98506
- });
98507
- await clock.sleep(settleMs);
98508
- await deps.appendStep({
98509
- idx: stepIdx,
98510
- action: decision,
98511
- screenshotUri: shot.path,
98512
- ok: true,
98513
- latencyMs,
98514
- detail: decision.reason ?? null
98515
- });
98516
- } else if (decision.type === "type") {
98517
- await perform(deps.session, { kind: "input", text: decision.text ?? "" });
98518
- await clock.sleep(settleMs);
98519
- await deps.appendStep({
98520
- idx: stepIdx,
98521
- action: decision,
98522
- screenshotUri: shot.path,
98523
- ok: true,
98524
- latencyMs,
98525
- detail: decision.reason ?? null
98526
- });
98527
- } else if (decision.type === "wait") {
98528
- const waitMs = Math.min(3000, Math.max(500, decision.ms ?? 1500));
98529
- await clock.sleep(waitMs);
98530
- await deps.appendStep({
98531
- idx: stepIdx,
98532
- action: decision,
98533
- screenshotUri: shot.path,
98534
- ok: true,
98535
- latencyMs,
98536
- detail: decision.reason ?? `wait ${waitMs}ms`
98537
- });
98538
- } else if (decision.type === "verify" || decision.type === "done") {
98539
- await deps.appendStep({
98540
- idx: stepIdx,
98541
- action: decision,
98542
- screenshotUri: shot.path,
98543
- ok: true,
98544
- latencyMs,
98545
- detail: decision.reason ?? null
98546
- });
98547
- flowDone = true;
98548
- } else {
99408
+ };
99409
+ const recordStep = async (command, outcome2) => {
99410
+ recentActions.push(decision);
99411
+ recordedDecisions.push(decision);
98549
99412
  await deps.appendStep({
98550
99413
  idx: stepIdx,
98551
99414
  action: decision,
98552
99415
  screenshotUri: shot.path,
98553
- ok: false,
99416
+ ok: outcome2 !== "fail",
98554
99417
  latencyMs,
98555
- detail: decision.reason ?? "Agent failed the step"
99418
+ detail: decision.type === "wait" ? decision.reason ?? `wait ${Math.min(3000, Math.max(500, decision.ms ?? 1500))}ms` : decision.reason ?? (outcome2 === "fail" ? "Agent failed the step" : null),
99419
+ command
98556
99420
  });
99421
+ };
99422
+ const initialCommand = commandForDecision(decision, deps.defaultAppId);
99423
+ const outcome = initialCommand ? await withCurrentCommand(setCurrentCommand, initialCommand, async () => {
99424
+ const result = await applyWithRetry();
99425
+ await recordStep(commandForDecision(decision, deps.defaultAppId), result);
99426
+ return result;
99427
+ }) : await (async () => {
99428
+ const result = await applyWithRetry();
99429
+ await recordStep(null, result);
99430
+ return result;
99431
+ })();
99432
+ if (outcome === "done") {
99433
+ instructionDone = true;
99434
+ completedInstructions.push(instruction.instructions);
99435
+ } else if (outcome === "fail") {
98557
99436
  caseStatus = "errored";
98558
- caseError = decision.reason ?? "Agent marked the flow as failed";
98559
- flowDone = true;
99437
+ caseError = decision.reason ?? "Agent marked the instruction as failed";
99438
+ instructionDone = true;
98560
99439
  }
98561
99440
  stepIdx += 1;
98562
99441
  }
98563
99442
  if (caseStatus === "errored" || caseStatus === "cancelled")
98564
99443
  break;
98565
- if (!flowDone) {
99444
+ if (!instructionDone) {
98566
99445
  caseStatus = "errored";
98567
- caseError = `Exceeded max steps (${maxSteps}) for a flow`;
99446
+ caseError = `Exceeded max steps (${maxSteps}) for instruction ${instructionIndex + 1} of ${instructionQueue.length}`;
98568
99447
  break;
98569
99448
  }
98570
99449
  }
@@ -98584,7 +99463,8 @@ async function executeAgentCase(deps) {
98584
99463
  screenshotUri: lastScreenshotUri,
98585
99464
  ok: false,
98586
99465
  latencyMs: 0,
98587
- detail: caseError
99466
+ detail: caseError,
99467
+ command: null
98588
99468
  });
98589
99469
  }
98590
99470
  }
@@ -98668,6 +99548,7 @@ async function loadRun(runId) {
98668
99548
  ok: step.ok === 1,
98669
99549
  latencyMs: step.latencyMs,
98670
99550
  detail: step.detail,
99551
+ command: step.command ?? null,
98671
99552
  createdAt: step.createdAt
98672
99553
  }));
98673
99554
  const caseMode = test.executionMode === "script" || test.executionMode === "agent" ? test.executionMode : null;
@@ -98680,6 +99561,7 @@ async function loadRun(runId) {
98680
99561
  error: test.error,
98681
99562
  startedAt: test.startedAt,
98682
99563
  finishedAt: test.finishedAt,
99564
+ currentCommand: test.currentCommand ?? null,
98683
99565
  steps
98684
99566
  });
98685
99567
  }
@@ -98709,9 +99591,14 @@ async function appendStep(input) {
98709
99591
  ok: input.ok ? 1 : 0,
98710
99592
  latencyMs: Math.max(0, Math.round(input.latencyMs)),
98711
99593
  detail: input.detail,
99594
+ command: input.command,
98712
99595
  createdAt: Date.now()
98713
99596
  });
98714
99597
  }
99598
+ async function setCurrentCommand(runTestId, command) {
99599
+ const db = getCatalogDb();
99600
+ await db.update(runTests).set({ currentCommand: command }).where(eq(runTests.id, runTestId));
99601
+ }
98715
99602
  async function updateCaseLastRun(caseId, status, at) {
98716
99603
  const db = getCatalogDb();
98717
99604
  await db.update(cases).set({
@@ -98734,7 +99621,8 @@ async function persistCancelled(runId) {
98734
99621
  await db.update(runTests).set({
98735
99622
  status: "cancelled",
98736
99623
  error: "Cancelled by user",
98737
- finishedAt
99624
+ finishedAt,
99625
+ currentCommand: null
98738
99626
  }).where(eq(runTests.id, test.id));
98739
99627
  }
98740
99628
  }
@@ -98754,6 +99642,9 @@ async function executeScriptCase2(input) {
98754
99642
  ...step
98755
99643
  });
98756
99644
  },
99645
+ setCurrentCommand: async (command) => {
99646
+ await setCurrentCommand(input.runTestId, command);
99647
+ },
98757
99648
  clock: { sleep, now: () => Date.now() }
98758
99649
  });
98759
99650
  }
@@ -98774,8 +99665,12 @@ async function executeAgentCase2(input) {
98774
99665
  ...step
98775
99666
  });
98776
99667
  },
99668
+ setCurrentCommand: async (command) => {
99669
+ await setCurrentCommand(input.runTestId, command);
99670
+ },
98777
99671
  decide: decideNextAction,
98778
- clock: { sleep, now: () => Date.now() }
99672
+ clock: { sleep, now: () => Date.now() },
99673
+ defaultAppId: input.defaultAppId
98779
99674
  });
98780
99675
  }
98781
99676
  async function executeCase(input) {
@@ -98788,7 +99683,8 @@ async function executeCase(input) {
98788
99683
  status: "running",
98789
99684
  startedAt,
98790
99685
  error: null,
98791
- executionMode: input.caseMode
99686
+ executionMode: input.caseMode,
99687
+ currentCommand: null
98792
99688
  }).where(eq(runTests.id, input.runTestId));
98793
99689
  let caseStatus;
98794
99690
  let caseError = null;
@@ -98815,7 +99711,8 @@ async function executeCase(input) {
98815
99711
  caseId: input.caseId,
98816
99712
  appContext: input.appContext,
98817
99713
  session: input.session,
98818
- auth: input.auth
99714
+ auth: input.auth,
99715
+ defaultAppId: input.defaultAppId
98819
99716
  });
98820
99717
  caseStatus = result.status;
98821
99718
  decisions = result.decisions;
@@ -98828,7 +99725,8 @@ async function executeCase(input) {
98828
99725
  await db.update(runTests).set({
98829
99726
  status: caseStatus,
98830
99727
  error: caseError,
98831
- finishedAt
99728
+ finishedAt,
99729
+ currentCommand: null
98832
99730
  }).where(eq(runTests.id, input.runTestId));
98833
99731
  await updateCaseLastRun(input.caseId, caseStatus, finishedAt);
98834
99732
  if (caseStatus === "passed" && input.caseMode === "agent") {
@@ -98926,7 +99824,8 @@ async function executeRun(runId) {
98926
99824
  appContext: app.context,
98927
99825
  session,
98928
99826
  auth,
98929
- caseMode
99827
+ caseMode,
99828
+ defaultAppId: run.platform === "ios" ? app.iosBundleId || undefined : app.androidApplicationId || undefined
98930
99829
  });
98931
99830
  if (status === "cancelled" || isAborted2(runId)) {
98932
99831
  await persistCancelled(runId);
@@ -99406,7 +100305,9 @@ function createSessionRoutes() {
99406
100305
  try {
99407
100306
  const { session } = requireActiveSession();
99408
100307
  const full = c.req.query("full") === "1" || c.req.query("full") === "true";
99409
- const result = await getScreen(session, { full });
100308
+ const pauseQuery = c.req.query("pauseMjpeg");
100309
+ const pauseMjpeg = pauseQuery !== "0" && pauseQuery !== "false";
100310
+ const result = await getScreen(session, { full, pauseMjpeg });
99410
100311
  return c.json(screenResponseSchema.parse(result));
99411
100312
  } catch (error40) {
99412
100313
  const gone = sessionErrorResponse(error40);