@phnx-labs/agents-cli 1.17.3 → 1.17.4

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 1.17.4
+
+**Browser**
+
+- `agents browser type` now detects rich-text editor frameworks (Lexical, ProseMirror, Slate, Draft.js, Quill, CKEditor5, Trix) by walking up to 5 ancestor levels from each textbox and tagging refs with `[editor=<framework>]`. Editor-tagged refs route through the WHATWG `beforeinput` dispatch (`InputEvent('beforeinput', { inputType: 'insertText', ... })`) for Lexical/ProseMirror/Slate/Quill/CKEditor5/Draft and `el.editor.insertString()` for Trix. `agents browser refs --json` surfaces the new `editor` field, and `type --clear` prepends a select-all + `deleteContentBackward` dispatch before inserting.
+- Plain-input reliability also improved: `typeText` now issues a single CDP `Input.insertText` instead of per-character `dispatchKeyEvent`, so framework-controlled inputs (React, Vue, Solid, MUI/Chakra/Mantine `TextField`, masked-number fields, Canva-style pickers) actually receive `beforeinput`/`input`/`textInput` events. `focusNode` falls back to the first focusable descendant when `DOM.focus` throws "Element is not focusable" — fixes wrapper-ref UIs like Slack composer, Linear comments, Notion blocks, and every MUI/Chakra/Mantine `TextField`. ([#12](https://github.com/phnx-labs/agents-cli/pull/12))
+
 ## 1.17.3
 
 **Browser**

package/dist/commands/browser.js

@@ -754,6 +754,7 @@ function registerTaskCommands(browser) {
         .option('-t, --tab <tabId>', 'Tab ID (defaults to current)')
         .option('--all', 'Include non-interactive elements')
         .option('-l, --limit <n>', 'Max elements (default 500)', '500')
+        .option('--json', 'Output machine-readable JSON')
         .action(async (task, opts) => {
         const response = await sendIPCRequest({
             action: 'refs',
@@ -763,9 +764,18 @@ function registerTaskCommands(browser) {
             limit: parseInt(opts.limit, 10),
         });
         if (!response.ok) {
-            console.error(response.error);
+            if (opts.json) {
+                console.log(JSON.stringify({ ok: false, error: response.error }));
+            }
+            else {
+                console.error(response.error);
+            }
             process.exit(1);
         }
+        if (opts.json) {
+            console.log(JSON.stringify(response.nodes ?? [], null, 2));
+            return;
+        }
         console.log(response.refs);
     });
     browser
@@ -789,6 +799,7 @@ function registerTaskCommands(browser) {
         .command('type <task> <ref> <text>')
         .description('Type text into an element by ref')
         .option('-t, --tab <tabId>', 'Tab ID (defaults to current)')
+        .option('--clear', 'Clear editor content before typing')
         .action(async (task, ref, text, opts) => {
         const response = await sendIPCRequest({
             action: 'type',
@@ -796,6 +807,7 @@ function registerTaskCommands(browser) {
             tabId: opts.tab,
             ref: parseInt(ref, 10),
             text,
+            clear: opts.clear,
         });
         if (!response.ok) {
             console.error(response.error);

package/dist/lib/browser/editor.d.ts

@@ -0,0 +1,3 @@
+import type { CDPClient } from './cdp.js';
+import type { RefNode } from './refs.js';
+export declare function typeEditorText(cdp: CDPClient, sessionId: string, node: RefNode, text: string, clear?: boolean): Promise<void>;

package/dist/lib/browser/editor.js

@@ -0,0 +1,50 @@
+const BEFOREINPUT_INSERT_FN = `(function(text) {
+  this.focus();
+  var sel = window.getSelection();
+  var range = document.createRange();
+  range.selectNodeContents(this);
+  range.collapse(false);
+  sel.removeAllRanges();
+  sel.addRange(range);
+  this.dispatchEvent(new InputEvent('beforeinput', {
+    inputType: 'insertText',
+    data: text,
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+  }));
+})`;
+const BEFOREINPUT_CLEAR_FN = `(function() {
+  this.focus();
+  var sel = window.getSelection();
+  var range = document.createRange();
+  range.selectNodeContents(this);
+  sel.removeAllRanges();
+  sel.addRange(range);
+  this.dispatchEvent(new InputEvent('beforeinput', {
+    inputType: 'deleteContentBackward',
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+  }));
+})`;
+const TRIX_INSERT_FN = `(function(text) { this.editor.insertString(text); })`;
+export async function typeEditorText(cdp, sessionId, node, text, clear = false) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId: node.backendNodeId }, sessionId);
+    if (!object.objectId)
+        throw new Error(`Could not resolve DOM node for ref ${node.ref}`);
+    const objectId = object.objectId;
+    try {
+        if (node.editor === 'trix') {
+            await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: TRIX_INSERT_FN, arguments: [{ value: text }], returnByValue: true }, sessionId);
+            return;
+        }
+        if (clear) {
+            await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: BEFOREINPUT_CLEAR_FN, arguments: [], returnByValue: true }, sessionId);
+        }
+        await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: BEFOREINPUT_INSERT_FN, arguments: [{ value: text }], returnByValue: true }, sessionId);
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}

package/dist/lib/browser/input.js

@@ -8,11 +8,13 @@ export async function hoverAtCoords(cdp, sessionId, x, y) {
 export async function scrollAtCoords(cdp, sessionId, x, y, deltaX, deltaY) {
     await cdp.send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX, deltaY }, sessionId);
 }
+// `Input.insertText` is the CDP native text-insertion method. It dispatches a
+// real `beforeinput`/`input`/`textInput` sequence on the focused element, which
+// is what framework-controlled inputs (React, Vue, Solid, contenteditable
+// editors) actually listen for. Per-character `dispatchKeyEvent` only fires
+// `keydown`/`keyup` with no input event, so controlled inputs ignore it.
 export async function typeText(cdp, sessionId, text) {
-    for (const char of text) {
-        await cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', text: char }, sessionId);
-        await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', text: char }, sessionId);
-    }
+    await cdp.send('Input.insertText', { text }, sessionId);
 }
 const KEY_CODES = {
     Enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
@@ -50,6 +52,41 @@ export async function pressKey(cdp, sessionId, keyName) {
         nativeVirtualKeyCode: keyInfo.keyCode,
     }, sessionId);
 }
+const FOCUS_DESCENDANT_FN = `(function() {
+  const selector = 'input:not([disabled]):not([type=hidden]),textarea:not([disabled]),select:not([disabled]),[contenteditable=""],[contenteditable=true],[tabindex]:not([tabindex="-1"])';
+  const candidates = this.querySelectorAll(selector);
+  for (const el of candidates) {
+    el.focus();
+    if (document.activeElement === el) return true;
+  }
+  return false;
+})`;
+// `DOM.focus` only works on natively focusable elements. UIs that wrap real
+// inputs in styled containers (Slack composer, Linear comments, Notion blocks,
+// Canva pickers, MUI/Chakra/Mantine TextField) often expose the wrapper as the
+// accessible "ref" — focusing it throws "Element is not focusable". When that
+// happens, walk the subtree for the first focusable descendant.
 export async function focusNode(cdp, sessionId, backendNodeId) {
-    await cdp.send('DOM.focus', { backendNodeId }, sessionId);
+    try {
+        await cdp.send('DOM.focus', { backendNodeId }, sessionId);
+        return;
+    }
+    catch (err) {
+        const focused = await focusFirstFocusableDescendant(cdp, sessionId, backendNodeId);
+        if (!focused)
+            throw err;
+    }
+}
+async function focusFirstFocusableDescendant(cdp, sessionId, backendNodeId) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return false;
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: FOCUS_DESCENDANT_FN, returnByValue: true }, sessionId);
+        return result.value === true;
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
 }

package/dist/lib/browser/ipc.js

@@ -164,11 +164,17 @@ export class BrowserIPCServer {
                 if (!request.task) {
                     return { ok: false, error: 'Task required' };
                 }
-                const { refs } = await this.service.refs(request.task, request.tabId, {
+                const { refs, nodeMap } = await this.service.refs(request.task, request.tabId, {
                     interactive: request.interactive ?? true,
                     limit: request.limit ?? 500,
                 });
-                return { ok: true, refs };
+                const nodes = Array.from(nodeMap.values()).map(n => {
+                    const entry = { ref: n.ref, role: n.role, name: n.name, attrs: n.attrs };
+                    if (n.editor !== undefined)
+                        entry.editor = n.editor;
+                    return entry;
+                });
+                return { ok: true, refs, nodes };
             }
             case 'click': {
                 if (!request.task || request.ref === undefined) {
@@ -181,7 +187,7 @@ export class BrowserIPCServer {
                 if (!request.task || request.ref === undefined || !request.text) {
                     return { ok: false, error: 'Task, ref, and text required' };
                 }
-                await this.service.type(request.task, request.ref, request.text, request.tabId);
+                await this.service.type(request.task, request.ref, request.text, request.tabId, request.clear);
                 return { ok: true };
             }
             case 'press': {

package/dist/lib/browser/refs.d.ts

@@ -10,6 +10,7 @@ export interface RefNode {
     name: string;
     attrs: string[];
     backendNodeId?: number;
+    editor?: string;
 }
 export declare function getRefs(cdp: CDPClient, sessionId: string, opts?: RefOpts): Promise<{
     refs: string;

package/dist/lib/browser/refs.js

@@ -1,3 +1,31 @@
+const EDITOR_DETECT_FN = `(function() {
+  let el = this;
+  for (let i = 0; i < 5; i++) {
+    if (!el || el === document.documentElement) break;
+    if (el.hasAttribute && el.hasAttribute('data-lexical-editor')) return 'lexical';
+    if (el.classList && el.classList.contains('ProseMirror')) return 'prosemirror';
+    if (el.hasAttribute && el.hasAttribute('data-slate-editor')) return 'slate';
+    if (el.classList && Array.from(el.classList).some(function(c) { return /^DraftEditor-/.test(c); })) return 'draft';
+    if (el.classList && el.classList.contains('ql-editor')) return 'quill';
+    if (el.classList && el.classList.contains('ck-editor__editable')) return 'ckeditor5';
+    if (el.tagName === 'TRIX-EDITOR') return 'trix';
+    el = el.parentElement;
+  }
+  return null;
+})`;
+async function detectEditorForNode(cdp, sessionId, backendNodeId) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return undefined;
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: EDITOR_DETECT_FN, returnByValue: true }, sessionId);
+        return result.value ?? undefined;
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}
 const INTERACTIVE_ROLES = new Set([
     'button',
     'link',
@@ -59,12 +87,18 @@ export async function getRefs(cdp, sessionId, opts = {}) {
             attrs,
             backendNodeId: node.backendDOMNodeId,
         };
+        if (role === 'textbox' && node.backendDOMNodeId) {
+            const editor = await detectEditorForNode(cdp, sessionId, node.backendDOMNodeId);
+            if (editor)
+                refNode.editor = editor;
+        }
         nodeMap.set(ref, refNode);
         const attrStr = attrs.length > 0 ? ` [${attrs.join('] [')}]` : '';
+        const editorStr = refNode.editor ? ` [editor=${refNode.editor}]` : '';
         const nameStr = name ? ` "${truncate(name, 50)}"` : '';
         const line = compact
-            ? `${role}${nameStr} [ref=${ref}]${attrStr}`
-            : `- ${role}${nameStr} [ref=${ref}]${attrStr}`;
+            ? `${role}${nameStr} [ref=${ref}]${attrStr}${editorStr}`
+            : `- ${role}${nameStr} [ref=${ref}]${attrStr}${editorStr}`;
         lines.push(line);
     }
     return { refs: lines.join('\n'), nodeMap };

package/dist/lib/browser/service.d.ts

@@ -99,7 +99,7 @@ export declare class BrowserService {
         nodeMap: Map<number, RefNode>;
     }>;
     click(taskId: string, ref: number, tabHint?: string): Promise<void>;
-    type(taskId: string, ref: number, text: string, tabHint?: string): Promise<void>;
+    type(taskId: string, ref: number, text: string, tabHint?: string, clear?: boolean): Promise<void>;
     press(taskId: string, key: string, tabHint?: string): Promise<void>;
     hover(taskId: string, ref: number, tabHint?: string): Promise<void>;
     scroll(taskId: string, deltaX: number, deltaY: number, atX?: number, atY?: number, tabHint?: string): Promise<void>;

package/dist/lib/browser/service.js

@@ -8,6 +8,7 @@ import { connectSSH } from './drivers/ssh.js';
 import { generateTaskId, generateShortId, generateFunName, } from './types.js';
 import { getRefs, resolveRefToCoords } from './refs.js';
 import { clickAtCoords, hoverAtCoords, scrollAtCoords, typeText, pressKey, focusNode } from './input.js';
+import { typeEditorText } from './editor.js';
 import { emit } from '../events.js';
 /**
  * Parse a `targetFilter` string into its kind + value, or return `null`
@@ -502,7 +503,7 @@ export class BrowserService {
         const { x, y } = await resolveRefToCoords(conn.cdp, sessionId, nodeMap, ref);
         await clickAtCoords(conn.cdp, sessionId, x, y);
     }
-    async type(taskId, ref, text, tabHint) {
+    async type(taskId, ref, text, tabHint, clear) {
         const { conn, task } = await this.findTask(taskId);
         const shortId = tabHint ? await this.resolveTabHint(conn, task, tabHint) : this.resolveCurrentTab(task);
         const cdpTargetId = this.getCdpTargetId(task, shortId);
@@ -514,10 +515,15 @@ export class BrowserService {
         const node = nodeMap.get(ref);
         if (!node)
             throw new Error(`Ref ${ref} not found`);
-        if (node.backendNodeId) {
-            await focusNode(conn.cdp, sessionId, node.backendNodeId);
+        if (node.editor) {
+            await typeEditorText(conn.cdp, sessionId, node, text, clear);
+        }
+        else {
+            if (node.backendNodeId) {
+                await focusNode(conn.cdp, sessionId, node.backendNodeId);
+            }
+            await typeText(conn.cdp, sessionId, text);
         }
-        await typeText(conn.cdp, sessionId, text);
     }
     async press(taskId, key, tabHint) {
         const { conn, task } = await this.findTask(taskId);

package/dist/lib/browser/types.d.ts

@@ -129,6 +129,7 @@ export interface IPCResponse {
     result?: unknown;
     path?: string;
     refs?: string;
+    nodes?: RefNodeJson[];
     port?: number;
     pid?: number;
     logs?: ConsoleEntry[];
@@ -160,6 +161,13 @@ export interface NetworkRequest {
     mimeType?: string;
     timestamp: number;
 }
+export interface RefNodeJson {
+    ref: number;
+    role: string;
+    name: string;
+    attrs: string[];
+    editor?: string;
+}
 export interface DeviceDescriptor {
     width: number;
     height: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@phnx-labs/agents-cli",
-  "version": "1.17.3",
+  "version": "1.17.4",
   "description": "One CLI for all your AI coding agents - versions, config, cloud dispatch, sessions, and teams",
   "type": "module",
   "main": "dist/index.js",
@@ -87,6 +87,7 @@
     "@types/diff": "^6.0.0",
     "@types/marked-terminal": "^6.1.1",
     "@types/node": "^22.0.0",
+    "playwright": "^1.44.0",
     "tsx": "^4.19.0",
     "typescript": "^5.5.0",
     "vitest": "^2.0.0"