0agent 1.0.69 → 1.0.71

package/dist/daemon.mjs

@@ -3027,7 +3027,7 @@ var init_GUICapability = __esm({
           properties: {
             action: {
               type: "string",
-              description: 'Browser (no Screen Recording): "click_text"|"type_in"|"get_elements"|"read_element"|"get_media_state"|"scroll_to"|"exec_js"|"browser_state"|"cdp_screenshot" | Native apps: "accessibility_click" | Navigation: "open_url"|"open_app" | Mouse/KB (Screen Recording for screenshots): "screenshot"|"click"|"double_click"|"right_click"|"move"|"type"|"hotkey"|"scroll"|"drag"|"find_and_click"|"get_screen_size"|"get_cursor_pos"|"wait"'
+              description: 'Browser (no Screen Recording): "click_text"|"type_in"|"get_elements"|"read_element"|"get_media_state"|"scroll_to"|"exec_js"|"browser_state"|"cdp_screenshot" | Native apps (no Screen Recording): "app_type"|"accessibility_click" | Navigation: "open_url"|"open_app" | Mouse/KB (Screen Recording for screenshots): "screenshot"|"click"|"double_click"|"right_click"|"move"|"type"|"hotkey"|"scroll"|"drag"|"find_and_click"|"get_screen_size"|"get_cursor_pos"|"wait"'
             },
             js: { type: "string", description: `JavaScript to execute in Chrome tab (use with exec_js). Example: "document.querySelector('video').paused"` },
             selector: { type: "string", description: 'CSS selector for read_element, type_in, scroll_to (e.g. "input[type=search]", ".title", "video")' },
@@ -3292,7 +3292,7 @@ print(f"Moved to ({${x}}, {${y}})")
           case "type": {
             if (!text) return null;
             return header + `
-pyautogui.write(${JSON.stringify(text)}, interval=${interval})
+pyautogui.typewrite(${JSON.stringify(text)}, interval=${interval})
 print("Typed successfully")
 `;
           }
@@ -3303,24 +3303,58 @@ print("Typed successfully")
             const pyKeys = JSON.stringify(parts);
             if (targetApp && platform2() === "darwin") {
               const safeApp = targetApp.replace(/'/g, "\\'");
-              const asKey = parts[parts.length - 1] ?? "";
-              const modifiers = parts.slice(0, -1).map((k) => {
+              const mainKey = parts[parts.length - 1] ?? "";
+              const modParts = parts.slice(0, -1);
+              const KEY_CODES = {
+                down: 125,
+                up: 126,
+                left: 123,
+                right: 124,
+                enter: 36,
+                return: 36,
+                escape: 53,
+                esc: 53,
+                tab: 48,
+                delete: 51,
+                backspace: 51,
+                "delete-forward": 117,
+                space: 49,
+                home: 115,
+                end: 119,
+                pageup: 116,
+                pagedown: 121,
+                f1: 122,
+                f2: 120,
+                f3: 99,
+                f4: 118,
+                f5: 96,
+                f6: 97,
+                f7: 98,
+                f8: 100,
+                f9: 101,
+                f10: 109,
+                f11: 103,
+                f12: 111
+              };
+              const keyCode = KEY_CODES[mainKey];
+              const modifiers = modParts.map((k) => {
                 if (k === "command") return "command down";
                 if (k === "ctrl") return "control down";
                 if (k === "shift") return "shift down";
                 if (k === "option") return "option down";
                 return "";
               }).filter(Boolean).join(", ");
-              const asModStr = modifiers ? ` using {${modifiers}}` : "";
+              const usingClause = modifiers ? ` using {${modifiers}}` : "";
+              const keyStatement = keyCode !== void 0 ? `key code ${keyCode}${usingClause}` : `keystroke "${mainKey}"${usingClause}`;
               return header + `
 import subprocess, time
-# Focus target app first
 subprocess.run(['osascript', '-e', 'tell application "${safeApp}" to activate'], capture_output=True)
 time.sleep(0.3)
-# Send keystroke via AppleScript (reliable \u2014 goes to the focused app, not Terminal)
-as_script = '''tell application "System Events"
-  keystroke "${asKey}"${asModStr}
-end tell'''
+as_script = """tell application "System Events"
+  tell process "${safeApp}"
+    ${keyStatement}
+  end tell
+end tell"""
 r = subprocess.run(['osascript', '-e', as_script], capture_output=True, text=True)
 if r.returncode == 0:
     print(f"Sent ${parts.join("+")} to ${safeApp}")
@@ -3587,6 +3621,48 @@ time.sleep(1.5)
 `;
           }
           // ── New high-level browser actions — no Screen Recording needed ───────────
+          case "app_type": {
+            const appName = String(input.app ?? "").trim();
+            const typeText = String(input.text ?? text ?? "").trim();
+            if (!appName || !typeText) return null;
+            const osName = platform2();
+            if (osName !== "darwin") return header + `print("app_type requires macOS")`;
+            const safeApp = appName.replace(/'/g, "\\'");
+            const textJson = JSON.stringify(typeText);
+            return header + `
+import subprocess, time, json
+
+text_to_type = json.loads(${textJson})
+
+# Step 1: copy to clipboard (handles unicode, special chars, long text)
+cp = subprocess.run(['pbcopy'], input=text_to_type.encode('utf-8'), capture_output=True)
+if cp.returncode != 0:
+    print(f"Clipboard copy failed: {cp.stderr.decode()[:100]}")
+    sys.exit(1)
+
+# Step 2: bring app to front
+subprocess.run(['osascript', '-e', 'tell application "${safeApp}" to activate'], capture_output=True)
+time.sleep(0.4)
+
+# Step 3: paste via AppleScript System Events (targets the specific process, not OS focus)
+paste_script = """tell application "System Events"
+  tell process "${safeApp}"
+    keystroke "v" using command down
+  end tell
+end tell"""
+r = subprocess.run(['osascript', '-e', paste_script], capture_output=True, text=True)
+if r.returncode == 0:
+    print(f"Typed in ${safeApp}: {text_to_type[:60]}")
+else:
+    # Accessibility permission might be needed
+    err = r.stderr.strip()
+    if 'not allowed' in err.lower() or 'accessibility' in err.lower():
+        subprocess.run(['open', 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility'], capture_output=True)
+        print(f"Accessibility permission needed for ${safeApp}. System Settings opened \u2014 Privacy & Security \u2192 Accessibility \u2192 enable Terminal.")
+    else:
+        print(f"app_type error: {err[:200]}")
+`;
+          }
           case "click_text": {
             if (!text) return null;
             if (platform2() !== "darwin") return header + `print("click_text requires macOS + Chrome")`;
@@ -4709,20 +4785,31 @@ content = element.text if element else page.get_all_text()` : `content = page.ge
           lines.push(
             ``,
             `Browser/GUI actions \u2014 ALL work without Screen Recording permission:`,
-            `\u2022 click_text {text:"Submit"} \u2014 click any button/link/tab by its visible text. Use get_elements first if unsure.`,
-            `\u2022 type_in {selector:"search", text:"Drake"} \u2014 fill form field by placeholder or aria-label. Handles React events.`,
-            `\u2022 get_elements \u2014 list every button/link/input/heading on the page. ALWAYS call this first when navigating.`,
-            `\u2022 read_element {selector:"h1"} \u2014 read text of any element. selector="" reads full page.`,
-            `\u2022 get_media_state \u2014 returns {state:PLAYING/PAUSED, time, duration, volume}. Call after play/pause to verify.`,
-            `\u2022 scroll_to {selector:".result"} or {direction:"down", amount:400} \u2014 scroll page.`,
-            `\u2022 exec_js {js:"..."} \u2014 run arbitrary JS. Use for anything not covered above.`,
-            `\u2022 browser_state \u2014 get current URL + title. Call after any navigation.`,
-            `\u2022 cdp_screenshot \u2014 screenshot via Chrome DevTools Protocol (if Chrome has --remote-debugging-port=9222).`,
-            `\u2022 accessibility_click {app:"WhatsApp", element:"Send"} \u2014 click native macOS app button (no Screen Recording).`,
-            `\u2022 open_url {url:"..."} \u2014 navigate. Returns actual URL + title + video state. Read it.`,
-            `\u2022 hotkey {keys:"k", app:"Google Chrome"} \u2014 send key to specific app. Without app param: goes to Terminal.`,
-            `WORKFLOW: navigate \u2192 get_elements \u2192 click_text/type_in \u2192 get_media_state/browser_state to verify.`,
-            `NEVER assume success. Always verify with get_media_state, browser_state, or read_element.`
+            `BROWSER (Chrome): click_text {text} | type_in {selector,text} | get_elements | read_element {selector}`,
+            `  get_media_state | scroll_to {selector|direction} | exec_js {js} | browser_state | cdp_screenshot`,
+            `NATIVE APPS (no Screen Recording \u2014 use these for WhatsApp, iMessage, Finder):`,
+            `  app_type {app:"WhatsApp", text:"hi"} \u2014 types via clipboard paste \u2192 cmd+v into the app.`,
+            `    Uses macOS clipboard so unicode/emoji/special chars all work. Target app gets the text`,
+            `    regardless of OS keyboard focus. ALWAYS use this for native app text input.`,
+            `  accessibility_click {app:"WhatsApp", element:"Send"} \u2014 click button via Accessibility API.`,
+            `  hotkey {keys:"cmd+f", app:"WhatsApp"} \u2014 send hotkey to specific app (not Terminal).`,
+            `NATIVE APP SEARCH NAVIGATION (generalised \u2014 works for WhatsApp, Spotlight, file dialogs, any search):`,
+            `  After typing in a search box, results appear in a dropdown/list.`,
+            `  Navigate them with: hotkey {keys:"down", app:"AppName"} to move to first result,`,
+            `  then hotkey {keys:"enter", app:"AppName"} to select/open it.`,
+            `  If first down+enter doesn't work, try: down, down, enter (some apps skip a header row).`,
+            `  Use wait {seconds:1} after typing to give results time to load before navigating.`,
+            `WHATSAPP EXACT WORKFLOW:`,
+            `  1. open_app {app:"WhatsApp"}`,
+            `  2. hotkey {keys:"cmd+f", app:"WhatsApp"} \u2014 open search bar`,
+            `  3. app_type {app:"WhatsApp", text:"ContactName"} \u2014 type contact name`,
+            `  4. wait {seconds:1} \u2014 wait for search results to load`,
+            `  5. hotkey {keys:"down", app:"WhatsApp"} \u2014 move focus to first result`,
+            `  6. hotkey {keys:"enter", app:"WhatsApp"} \u2014 open the conversation`,
+            `  7. app_type {app:"WhatsApp", text:"your message"} \u2014 type message`,
+            `  8. hotkey {keys:"enter", app:"WhatsApp"} \u2014 send`,
+            `NEVER use bare 'type' action for native apps \u2014 it goes to Terminal not the app.`,
+            `ALWAYS verify: browser_state after web nav, get_media_state after play/pause.`
           );
         }
         if (isSelfMod && this.agentRoot) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "0agent",
-  "version": "1.0.69",
+  "version": "1.0.71",
   "description": "A persistent, learning AI agent that runs on your machine. An agent that learns.",
   "private": false,
   "license": "Apache-2.0",