0agent 1.0.69 → 1.0.70

package/dist/daemon.mjs

@@ -3027,7 +3027,7 @@ var init_GUICapability = __esm({
           properties: {
             action: {
               type: "string",
-              description: 'Browser (no Screen Recording): "click_text"|"type_in"|"get_elements"|"read_element"|"get_media_state"|"scroll_to"|"exec_js"|"browser_state"|"cdp_screenshot" | Native apps: "accessibility_click" | Navigation: "open_url"|"open_app" | Mouse/KB (Screen Recording for screenshots): "screenshot"|"click"|"double_click"|"right_click"|"move"|"type"|"hotkey"|"scroll"|"drag"|"find_and_click"|"get_screen_size"|"get_cursor_pos"|"wait"'
+              description: 'Browser (no Screen Recording): "click_text"|"type_in"|"get_elements"|"read_element"|"get_media_state"|"scroll_to"|"exec_js"|"browser_state"|"cdp_screenshot" | Native apps (no Screen Recording): "app_type"|"accessibility_click" | Navigation: "open_url"|"open_app" | Mouse/KB (Screen Recording for screenshots): "screenshot"|"click"|"double_click"|"right_click"|"move"|"type"|"hotkey"|"scroll"|"drag"|"find_and_click"|"get_screen_size"|"get_cursor_pos"|"wait"'
             },
             js: { type: "string", description: `JavaScript to execute in Chrome tab (use with exec_js). Example: "document.querySelector('video').paused"` },
             selector: { type: "string", description: 'CSS selector for read_element, type_in, scroll_to (e.g. "input[type=search]", ".title", "video")' },
@@ -3292,7 +3292,7 @@ print(f"Moved to ({${x}}, {${y}})")
           case "type": {
             if (!text) return null;
             return header + `
-pyautogui.write(${JSON.stringify(text)}, interval=${interval})
+pyautogui.typewrite(${JSON.stringify(text)}, interval=${interval})
 print("Typed successfully")
 `;
           }
@@ -3587,6 +3587,48 @@ time.sleep(1.5)
 `;
           }
           // ── New high-level browser actions — no Screen Recording needed ───────────
+          case "app_type": {
+            const appName = String(input.app ?? "").trim();
+            const typeText = String(input.text ?? text ?? "").trim();
+            if (!appName || !typeText) return null;
+            const osName = platform2();
+            if (osName !== "darwin") return header + `print("app_type requires macOS")`;
+            const safeApp = appName.replace(/'/g, "\\'");
+            const textJson = JSON.stringify(typeText);
+            return header + `
+import subprocess, time, json
+
+text_to_type = json.loads(${textJson})
+
+# Step 1: copy to clipboard (handles unicode, special chars, long text)
+cp = subprocess.run(['pbcopy'], input=text_to_type.encode('utf-8'), capture_output=True)
+if cp.returncode != 0:
+    print(f"Clipboard copy failed: {cp.stderr.decode()[:100]}")
+    sys.exit(1)
+
+# Step 2: bring app to front
+subprocess.run(['osascript', '-e', 'tell application "${safeApp}" to activate'], capture_output=True)
+time.sleep(0.4)
+
+# Step 3: paste via AppleScript System Events (targets the specific process, not OS focus)
+paste_script = """tell application "System Events"
+  tell process "${safeApp}"
+    keystroke "v" using command down
+  end tell
+end tell"""
+r = subprocess.run(['osascript', '-e', paste_script], capture_output=True, text=True)
+if r.returncode == 0:
+    print(f"Typed in ${safeApp}: {text_to_type[:60]}")
+else:
+    # Accessibility permission might be needed
+    err = r.stderr.strip()
+    if 'not allowed' in err.lower() or 'accessibility' in err.lower():
+        subprocess.run(['open', 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility'], capture_output=True)
+        print(f"Accessibility permission needed for ${safeApp}. System Settings opened \u2014 Privacy & Security \u2192 Accessibility \u2192 enable Terminal.")
+    else:
+        print(f"app_type error: {err[:200]}")
+`;
+          }
           case "click_text": {
             if (!text) return null;
             if (platform2() !== "darwin") return header + `print("click_text requires macOS + Chrome")`;
@@ -4709,20 +4751,24 @@ content = element.text if element else page.get_all_text()` : `content = page.ge
           lines.push(
             ``,
             `Browser/GUI actions \u2014 ALL work without Screen Recording permission:`,
-            `\u2022 click_text {text:"Submit"} \u2014 click any button/link/tab by its visible text. Use get_elements first if unsure.`,
-            `\u2022 type_in {selector:"search", text:"Drake"} \u2014 fill form field by placeholder or aria-label. Handles React events.`,
-            `\u2022 get_elements \u2014 list every button/link/input/heading on the page. ALWAYS call this first when navigating.`,
-            `\u2022 read_element {selector:"h1"} \u2014 read text of any element. selector="" reads full page.`,
-            `\u2022 get_media_state \u2014 returns {state:PLAYING/PAUSED, time, duration, volume}. Call after play/pause to verify.`,
-            `\u2022 scroll_to {selector:".result"} or {direction:"down", amount:400} \u2014 scroll page.`,
-            `\u2022 exec_js {js:"..."} \u2014 run arbitrary JS. Use for anything not covered above.`,
-            `\u2022 browser_state \u2014 get current URL + title. Call after any navigation.`,
-            `\u2022 cdp_screenshot \u2014 screenshot via Chrome DevTools Protocol (if Chrome has --remote-debugging-port=9222).`,
-            `\u2022 accessibility_click {app:"WhatsApp", element:"Send"} \u2014 click native macOS app button (no Screen Recording).`,
-            `\u2022 open_url {url:"..."} \u2014 navigate. Returns actual URL + title + video state. Read it.`,
-            `\u2022 hotkey {keys:"k", app:"Google Chrome"} \u2014 send key to specific app. Without app param: goes to Terminal.`,
-            `WORKFLOW: navigate \u2192 get_elements \u2192 click_text/type_in \u2192 get_media_state/browser_state to verify.`,
-            `NEVER assume success. Always verify with get_media_state, browser_state, or read_element.`
+            `BROWSER (Chrome): click_text {text} | type_in {selector,text} | get_elements | read_element {selector}`,
+            `  get_media_state | scroll_to {selector|direction} | exec_js {js} | browser_state | cdp_screenshot`,
+            `NATIVE APPS (no Screen Recording \u2014 use these for WhatsApp, iMessage, Finder):`,
+            `  app_type {app:"WhatsApp", text:"hi"} \u2014 types via clipboard paste \u2192 cmd+v into the app.`,
+            `    Uses macOS clipboard so unicode/emoji/special chars all work. Target app gets the text`,
+            `    regardless of OS keyboard focus. ALWAYS use this for native app text input.`,
+            `  accessibility_click {app:"WhatsApp", element:"Send"} \u2014 click button via Accessibility API.`,
+            `  hotkey {keys:"cmd+f", app:"WhatsApp"} \u2014 send hotkey to specific app (not Terminal).`,
+            `WHATSAPP WORKFLOW (use this exact sequence):`,
+            `  1. open_app {app:"WhatsApp"}`,
+            `  2. hotkey {keys:"cmd+f", app:"WhatsApp"} \u2014 open search`,
+            `  3. app_type {app:"WhatsApp", text:"ContactName"} \u2014 search for contact`,
+            `  4. hotkey {keys:"enter", app:"WhatsApp"} \u2014 open the conversation`,
+            `  5. app_type {app:"WhatsApp", text:"your message"} \u2014 type message`,
+            `  6. hotkey {keys:"enter", app:"WhatsApp"} \u2014 send`,
+            `  7. accessibility_click {app:"WhatsApp", element:"Send"} \u2014 if enter doesn't send`,
+            `NEVER use bare 'type' action for native apps \u2014 it goes to Terminal not the app.`,
+            `ALWAYS verify: browser_state after web nav, get_media_state after play/pause, read_element for page content.`
           );
         }
         if (isSelfMod && this.agentRoot) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "0agent",
-  "version": "1.0.69",
+  "version": "1.0.70",
   "description": "A persistent, learning AI agent that runs on your machine. An agent that learns.",
   "private": false,
   "license": "Apache-2.0",