@yoqa/runner 0.3.25 → 0.3.27

package/dist/index.js

@@ -16816,7 +16816,7 @@ function getCatalogDb() {
 // package.json
 var package_default = {
   name: "@yoqa/runner",
-  version: "0.3.25",
+  version: "0.3.27",
   description: "Yoqa local runner \u2014 Appium + catalog HTTP API (Bun). Start with yoqa serve or the yoqa-runner bin.",
   license: "UNLICENSED",
   type: "module",
@@ -17591,6 +17591,41 @@ async function listDevices(platform, options = {}) {
   return listAndroidDevices(options);
 }
 
+// src/domains/devices/keyboard.ts
+function isUnsupportedCommandError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  return /unknown command|unknown method|not (yet )?(been )?implemented|not (yet )?supported|does not support/i.test(message);
+}
+var W3C_RETURN_KEY = "\uE007";
+function w3cKeyActions(text2) {
+  const actions = [];
+  for (const ch of text2) {
+    const value = ch === `
+` || ch === "\r" ? W3C_RETURN_KEY : ch;
+    actions.push({ type: "keyDown", value }, { type: "keyUp", value });
+  }
+  return [{ type: "key", id: "keyboard", actions }];
+}
+async function w3cType(browser, text2) {
+  await browser.performActions(w3cKeyActions(text2));
+  try {
+    await browser.releaseActions();
+  } catch {}
+}
+async function typeText(browser, text2) {
+  if (text2.length === 0)
+    return "noop";
+  try {
+    await browser.execute("mobile: type", { text: text2 });
+    return "mobile";
+  } catch (error) {
+    if (!isUnsupportedCommandError(error))
+      throw error;
+  }
+  await w3cType(browser, text2);
+  return "w3c";
+}
+
 // src/domains/devices/session.ts
 var YOQA_ROOT4 = join7(homedir6(), ".yoqa");
 var DEFAULT_MJPEG_PORT = Number(process.env.YOQA_MJPEG_PORT ?? "9100");
@@ -18087,7 +18122,7 @@ async function createDeviceSession(options) {
   const type = async (text2) => {
     await gate.withLock(async () => {
       await guard(async () => {
-        await browser.keys(text2.split(""));
+        await typeText(browser, text2);
       });
     });
   };
@@ -96466,7 +96501,7 @@ function prefersScreenshotTap(decision) {
     return false;
   return decision.x != null && decision.y != null;
 }
-var SYSTEM_PROMPT = `You are a mobile QA agent controlling an app via screenshots.
+var SYSTEM_PROMPT = `You are a mobile QA agent controlling an app using BOTH visual screenshots and screen snapshots (accessibility tree).
 A screenshot of the current device screen is ALWAYS attached to the user message as an image. You can see it. Never claim that no screenshot was provided, missing, blank, or unavailable.
 
 Your entire reply MUST be a single strict JSON object (double quotes only \u2014 never single quotes) and nothing else \u2014 no markdown fences, no commentary before or after.
@@ -96501,12 +96536,21 @@ Valid shapes:
 Coordinates use a 0\u20131000 normalized grid (0,0 top-left of the attached screenshot). The user message also includes a screen snapshot (cleaned accessibility tree) with the same grid \u2014 x,y is each element's TOP-LEFT, not centre. Centre = x + width/2, y + height/2.
 
 Use BOTH the screenshot and the screen snapshot:
-- Prefer snapshot "id=\u2026" when present (same as yoqa action tap --id).
+- Visual context (screenshot): Observe visual layout, colors, icons, canvas controls, modal overlays, and soft keyboard visibility.
+- Structural context (screen snapshot): Inspect accessibility ids ("id=..."), element boundaries, and text labels.
+- Decide whether to target by "id" or "x,y":
+  * Use "id" when the snapshot provides a clean, unique, and unambiguous identifier for the element.
+  * Use "x,y" coordinates (centre of the control on the screenshot grid) when controls lack IDs, are custom-drawn icons/canvas, or when coordinates are more direct and reliable than broad container IDs.
 - System permission / notification dialogs: {"type":"tap","label":"Allow"} or {"type":"alert","alertAction":"accept"}. Never guess coordinates for these.
-- For other in-app controls, tap screenshot x,y of the visible control if there is no id.
-- If the snapshot is empty or the control is only drawn (no label/id), use the screenshot.
 - If a previous action failed (see Last action error), pick a different target from the snapshot + screenshot \u2014 do not repeat the same id/label/point.
 
+Textfield input and keyboard handling:
+- Typing into a textfield brings up the virtual soft keyboard on the bottom of the screen (typically covering y ~ 550\u20131000), which may obscure buttons (like Submit, Login, Next) or lower inputs.
+- When typing inside a textfield or when the keyboard is open:
+  1. Dismiss keyboard by tapping away: Emit a tap on an empty, neutral background area outside textfields and above the keyboard (e.g. tap neutral header space or empty background) to dismiss the keyboard.
+  2. Nextline / Return on keyboard: Tap the Return / Done / Next / Search key on the virtual keyboard itself (using its x,y coordinates or label/id if present), or append "\\n" to your input text.
+  3. Dismiss then use app buttons: If an app button you need is obscured or covered by the soft keyboard, dismiss the keyboard first (by tapping away or pressing Return) so the button is revealed, then tap the app button on the subsequent step.
+
 Swipe direction is the FINGER movement (same as Inspector):
 - "down" (finger down, y 200\u2192800): scroll UP \u2014 reveal content above
 - "up" (finger up, y 800\u2192200): scroll DOWN \u2014 reveal content below / to the bottom
@@ -96650,7 +96694,7 @@ ${formatRecentActions(input.recentActions ?? [])}`,
     `Last action error: ${input.lastError || "(none)"}`,
     "Screen snapshot (cleaned accessibility tree, 0\u20131000 grid, x,y is top-left):",
     input.screenSnapshot || "(unavailable)",
-    "Look at the attached screenshot image AND the screen snapshot and decide the next action. A screenshot is attached.",
+    "Look at the attached screenshot image AND the screen snapshot and decide the next action. Decide whether to use x,y coordinates or an id based on both sources. A screenshot is attached.",
     'Reply with ONLY the JSON action object, including non-empty "reason" and "thoughts".'
   ].join(`
 `);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@yoqa/runner",
-	"version": "0.3.25",
+	"version": "0.3.27",
 	"description": "Yoqa local runner — Appium + catalog HTTP API (Bun). Start with yoqa serve or the yoqa-runner bin.",
 	"license": "UNLICENSED",
 	"type": "module",