@phnx-labs/agents-cli 1.17.3 → 1.17.5

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 1.17.4
+
+**Browser**
+
+- `agents browser type` now detects rich-text editor frameworks (Lexical, ProseMirror, Slate, Draft.js, Quill, CKEditor5, Trix) by walking up to 5 ancestor levels from each textbox and tagging refs with `[editor=<framework>]`. Editor-tagged refs route through the WHATWG `beforeinput` dispatch (`InputEvent('beforeinput', { inputType: 'insertText', ... })`) for Lexical/ProseMirror/Slate/Quill/CKEditor5/Draft and `el.editor.insertString()` for Trix. `agents browser refs --json` surfaces the new `editor` field, and `type --clear` prepends a select-all + `deleteContentBackward` dispatch before inserting.
+- Plain-input reliability also improved: `typeText` now issues a single CDP `Input.insertText` instead of per-character `dispatchKeyEvent`, so framework-controlled inputs (React, Vue, Solid, MUI/Chakra/Mantine `TextField`, masked-number fields, Canva-style pickers) actually receive `beforeinput`/`input`/`textInput` events. `focusNode` falls back to the first focusable descendant when `DOM.focus` throws "Element is not focusable" — fixes wrapper-ref UIs like Slack composer, Linear comments, Notion blocks, and every MUI/Chakra/Mantine `TextField`. ([#12](https://github.com/phnx-labs/agents-cli/pull/12))
+
 ## 1.17.3
 
 **Browser**

package/dist/commands/browser.js

@@ -754,6 +754,7 @@ function registerTaskCommands(browser) {
         .option('-t, --tab <tabId>', 'Tab ID (defaults to current)')
         .option('--all', 'Include non-interactive elements')
         .option('-l, --limit <n>', 'Max elements (default 500)', '500')
+        .option('--json', 'Output machine-readable JSON')
         .action(async (task, opts) => {
         const response = await sendIPCRequest({
             action: 'refs',
@@ -763,9 +764,18 @@ function registerTaskCommands(browser) {
             limit: parseInt(opts.limit, 10),
         });
         if (!response.ok) {
-            console.error(response.error);
+            if (opts.json) {
+                console.log(JSON.stringify({ ok: false, error: response.error }));
+            }
+            else {
+                console.error(response.error);
+            }
             process.exit(1);
         }
+        if (opts.json) {
+            console.log(JSON.stringify(response.nodes ?? [], null, 2));
+            return;
+        }
         console.log(response.refs);
     });
     browser
@@ -789,6 +799,7 @@ function registerTaskCommands(browser) {
         .command('type <task> <ref> <text>')
         .description('Type text into an element by ref')
         .option('-t, --tab <tabId>', 'Tab ID (defaults to current)')
+        .option('--clear', 'Clear editor content before typing')
         .action(async (task, ref, text, opts) => {
         const response = await sendIPCRequest({
             action: 'type',
@@ -796,6 +807,7 @@ function registerTaskCommands(browser) {
             tabId: opts.tab,
             ref: parseInt(ref, 10),
             text,
+            clear: opts.clear,
         });
         if (!response.ok) {
             console.error(response.error);
@@ -859,6 +871,53 @@ function registerTaskCommands(browser) {
         }
         console.log('Scrolled');
     });
+    browser
+        .command('upload <task>')
+        .description('Upload file(s) — supports hidden file inputs, drag-drop targets, and OS chooser interception')
+        .option('-t, --tab <tabId>', 'Tab ID (defaults to current)')
+        .option('-r, --ref <n>', 'Ref of the upload target element (file input or drop zone)', (v) => parseInt(v, 10))
+        .option('--trigger <n>', 'Ref of a button that opens the OS file chooser (Pattern C)', (v) => parseInt(v, 10))
+        .option('-f, --file <path...>', 'Absolute path(s) to file(s) to upload (repeatable)')
+        .option('--drop', 'Force drag-drop pattern even if ref is an <input type=file>')
+        .option('--input', 'Force file-input pattern (DOM.setFileInputFiles)')
+        .option('--timeout <ms>', 'Timeout for chooser interception (Pattern C)', (v) => parseInt(v, 10))
+        .action(async (task, opts) => {
+        const files = opts.file ?? [];
+        if (files.length === 0) {
+            console.error('--file <path> is required (repeat for multiple files)');
+            process.exit(1);
+        }
+        if (opts.ref === undefined && opts.trigger === undefined) {
+            console.error('--ref <n> or --trigger <n> is required');
+            process.exit(1);
+        }
+        if (opts.drop && opts.input) {
+            console.error('--drop and --input are mutually exclusive');
+            process.exit(1);
+        }
+        let mode = 'auto';
+        if (opts.trigger !== undefined)
+            mode = 'chooser';
+        else if (opts.drop)
+            mode = 'drop';
+        else if (opts.input)
+            mode = 'input';
+        const response = await sendIPCRequest({
+            action: 'upload',
+            task,
+            tabId: opts.tab,
+            ref: opts.ref,
+            trigger: opts.trigger,
+            files,
+            uploadMode: mode,
+            timeout: opts.timeout,
+        });
+        if (!response.ok) {
+            console.error(response.error);
+            process.exit(1);
+        }
+        console.log(`Uploaded ${files.length} file${files.length === 1 ? '' : 's'} (${response.uploadMode})`);
+    });
     // ─── Viewport & Device ───────────────────────────────────────────────────────
     const setCmd = browser.command('set').description('Set browser emulation options');
     setCmd

package/dist/lib/browser/editor.d.ts

@@ -0,0 +1,3 @@
+import type { CDPClient } from './cdp.js';
+import type { RefNode } from './refs.js';
+export declare function typeEditorText(cdp: CDPClient, sessionId: string, node: RefNode, text: string, clear?: boolean): Promise<void>;

package/dist/lib/browser/editor.js

@@ -0,0 +1,50 @@
+const BEFOREINPUT_INSERT_FN = `(function(text) {
+  this.focus();
+  var sel = window.getSelection();
+  var range = document.createRange();
+  range.selectNodeContents(this);
+  range.collapse(false);
+  sel.removeAllRanges();
+  sel.addRange(range);
+  this.dispatchEvent(new InputEvent('beforeinput', {
+    inputType: 'insertText',
+    data: text,
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+  }));
+})`;
+const BEFOREINPUT_CLEAR_FN = `(function() {
+  this.focus();
+  var sel = window.getSelection();
+  var range = document.createRange();
+  range.selectNodeContents(this);
+  sel.removeAllRanges();
+  sel.addRange(range);
+  this.dispatchEvent(new InputEvent('beforeinput', {
+    inputType: 'deleteContentBackward',
+    bubbles: true,
+    cancelable: true,
+    composed: true,
+  }));
+})`;
+const TRIX_INSERT_FN = `(function(text) { this.editor.insertString(text); })`;
+export async function typeEditorText(cdp, sessionId, node, text, clear = false) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId: node.backendNodeId }, sessionId);
+    if (!object.objectId)
+        throw new Error(`Could not resolve DOM node for ref ${node.ref}`);
+    const objectId = object.objectId;
+    try {
+        if (node.editor === 'trix') {
+            await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: TRIX_INSERT_FN, arguments: [{ value: text }], returnByValue: true }, sessionId);
+            return;
+        }
+        if (clear) {
+            await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: BEFOREINPUT_CLEAR_FN, arguments: [], returnByValue: true }, sessionId);
+        }
+        await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: BEFOREINPUT_INSERT_FN, arguments: [{ value: text }], returnByValue: true }, sessionId);
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}

package/dist/lib/browser/input.js

@@ -8,11 +8,13 @@ export async function hoverAtCoords(cdp, sessionId, x, y) {
 export async function scrollAtCoords(cdp, sessionId, x, y, deltaX, deltaY) {
     await cdp.send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX, deltaY }, sessionId);
 }
+// `Input.insertText` is the CDP native text-insertion method. It dispatches a
+// real `beforeinput`/`input`/`textInput` sequence on the focused element, which
+// is what framework-controlled inputs (React, Vue, Solid, contenteditable
+// editors) actually listen for. Per-character `dispatchKeyEvent` only fires
+// `keydown`/`keyup` with no input event, so controlled inputs ignore it.
 export async function typeText(cdp, sessionId, text) {
-    for (const char of text) {
-        await cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', text: char }, sessionId);
-        await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', text: char }, sessionId);
-    }
+    await cdp.send('Input.insertText', { text }, sessionId);
 }
 const KEY_CODES = {
     Enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
@@ -50,6 +52,41 @@ export async function pressKey(cdp, sessionId, keyName) {
         nativeVirtualKeyCode: keyInfo.keyCode,
     }, sessionId);
 }
+const FOCUS_DESCENDANT_FN = `(function() {
+  const selector = 'input:not([disabled]):not([type=hidden]),textarea:not([disabled]),select:not([disabled]),[contenteditable=""],[contenteditable=true],[tabindex]:not([tabindex="-1"])';
+  const candidates = this.querySelectorAll(selector);
+  for (const el of candidates) {
+    el.focus();
+    if (document.activeElement === el) return true;
+  }
+  return false;
+})`;
+// `DOM.focus` only works on natively focusable elements. UIs that wrap real
+// inputs in styled containers (Slack composer, Linear comments, Notion blocks,
+// Canva pickers, MUI/Chakra/Mantine TextField) often expose the wrapper as the
+// accessible "ref" — focusing it throws "Element is not focusable". When that
+// happens, walk the subtree for the first focusable descendant.
 export async function focusNode(cdp, sessionId, backendNodeId) {
-    await cdp.send('DOM.focus', { backendNodeId }, sessionId);
+    try {
+        await cdp.send('DOM.focus', { backendNodeId }, sessionId);
+        return;
+    }
+    catch (err) {
+        const focused = await focusFirstFocusableDescendant(cdp, sessionId, backendNodeId);
+        if (!focused)
+            throw err;
+    }
+}
+async function focusFirstFocusableDescendant(cdp, sessionId, backendNodeId) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return false;
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: FOCUS_DESCENDANT_FN, returnByValue: true }, sessionId);
+        return result.value === true;
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
 }

package/dist/lib/browser/ipc.js

@@ -164,11 +164,17 @@ export class BrowserIPCServer {
                 if (!request.task) {
                     return { ok: false, error: 'Task required' };
                 }
-                const { refs } = await this.service.refs(request.task, request.tabId, {
+                const { refs, nodeMap } = await this.service.refs(request.task, request.tabId, {
                     interactive: request.interactive ?? true,
                     limit: request.limit ?? 500,
                 });
-                return { ok: true, refs };
+                const nodes = Array.from(nodeMap.values()).map(n => {
+                    const entry = { ref: n.ref, role: n.role, name: n.name, attrs: n.attrs };
+                    if (n.editor !== undefined)
+                        entry.editor = n.editor;
+                    return entry;
+                });
+                return { ok: true, refs, nodes };
             }
             case 'click': {
                 if (!request.task || request.ref === undefined) {
@@ -181,7 +187,7 @@ export class BrowserIPCServer {
                 if (!request.task || request.ref === undefined || !request.text) {
                     return { ok: false, error: 'Task, ref, and text required' };
                 }
-                await this.service.type(request.task, request.ref, request.text, request.tabId);
+                await this.service.type(request.task, request.ref, request.text, request.tabId, request.clear);
                 return { ok: true };
             }
             case 'press': {
@@ -290,6 +296,19 @@ export class BrowserIPCServer {
                 const downloadPath = await this.service.waitForDownload(request.task, request.timeout);
                 return { ok: true, downloadPath };
             }
+            case 'upload': {
+                if (!request.task || !request.files || request.files.length === 0) {
+                    return { ok: false, error: 'Task and at least one file required' };
+                }
+                const result = await this.service.upload(request.task, request.files, {
+                    ref: request.ref,
+                    trigger: request.trigger,
+                    mode: request.uploadMode,
+                    tabHint: request.tabId,
+                    timeout: request.timeout,
+                });
+                return { ok: true, uploadMode: result.mode };
+            }
             default:
                 return { ok: false, error: `Unknown action: ${request.action}` };
         }

package/dist/lib/browser/refs.d.ts

@@ -10,6 +10,7 @@ export interface RefNode {
     name: string;
     attrs: string[];
     backendNodeId?: number;
+    editor?: string;
 }
 export declare function getRefs(cdp: CDPClient, sessionId: string, opts?: RefOpts): Promise<{
     refs: string;

package/dist/lib/browser/refs.js

@@ -1,3 +1,31 @@
+const EDITOR_DETECT_FN = `(function() {
+  let el = this;
+  for (let i = 0; i < 5; i++) {
+    if (!el || el === document.documentElement) break;
+    if (el.hasAttribute && el.hasAttribute('data-lexical-editor')) return 'lexical';
+    if (el.classList && el.classList.contains('ProseMirror')) return 'prosemirror';
+    if (el.hasAttribute && el.hasAttribute('data-slate-editor')) return 'slate';
+    if (el.classList && Array.from(el.classList).some(function(c) { return /^DraftEditor-/.test(c); })) return 'draft';
+    if (el.classList && el.classList.contains('ql-editor')) return 'quill';
+    if (el.classList && el.classList.contains('ck-editor__editable')) return 'ckeditor5';
+    if (el.tagName === 'TRIX-EDITOR') return 'trix';
+    el = el.parentElement;
+  }
+  return null;
+})`;
+async function detectEditorForNode(cdp, sessionId, backendNodeId) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return undefined;
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: EDITOR_DETECT_FN, returnByValue: true }, sessionId);
+        return result.value ?? undefined;
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}
 const INTERACTIVE_ROLES = new Set([
     'button',
     'link',
@@ -59,12 +87,18 @@ export async function getRefs(cdp, sessionId, opts = {}) {
             attrs,
             backendNodeId: node.backendDOMNodeId,
         };
+        if (role === 'textbox' && node.backendDOMNodeId) {
+            const editor = await detectEditorForNode(cdp, sessionId, node.backendDOMNodeId);
+            if (editor)
+                refNode.editor = editor;
+        }
         nodeMap.set(ref, refNode);
         const attrStr = attrs.length > 0 ? ` [${attrs.join('] [')}]` : '';
+        const editorStr = refNode.editor ? ` [editor=${refNode.editor}]` : '';
         const nameStr = name ? ` "${truncate(name, 50)}"` : '';
         const line = compact
-            ? `${role}${nameStr} [ref=${ref}]${attrStr}`
-            : `- ${role}${nameStr} [ref=${ref}]${attrStr}`;
+            ? `${role}${nameStr} [ref=${ref}]${attrStr}${editorStr}`
+            : `- ${role}${nameStr} [ref=${ref}]${attrStr}${editorStr}`;
         lines.push(line);
     }
     return { refs: lines.join('\n'), nodeMap };

package/dist/lib/browser/service.d.ts

@@ -1,6 +1,7 @@
 import { type TabInfo, type ProfileStatus, type HistoricalTask } from './types.js';
 import { type RefOpts, type RefNode } from './refs.js';
 import type { TargetFilter } from './types.js';
+export type UploadMode = 'auto' | 'input' | 'drop' | 'chooser';
 /**
  * Parse a `targetFilter` string into its kind + value, or return `null`
  * when the input is missing or malformed. Filter syntax:
@@ -99,10 +100,19 @@ export declare class BrowserService {
         nodeMap: Map<number, RefNode>;
     }>;
     click(taskId: string, ref: number, tabHint?: string): Promise<void>;
-    type(taskId: string, ref: number, text: string, tabHint?: string): Promise<void>;
+    type(taskId: string, ref: number, text: string, tabHint?: string, clear?: boolean): Promise<void>;
     press(taskId: string, key: string, tabHint?: string): Promise<void>;
     hover(taskId: string, ref: number, tabHint?: string): Promise<void>;
     scroll(taskId: string, deltaX: number, deltaY: number, atX?: number, atY?: number, tabHint?: string): Promise<void>;
+    upload(taskId: string, files: string[], options: {
+        ref?: number;
+        trigger?: number;
+        mode?: UploadMode;
+        tabHint?: string;
+        timeout?: number;
+    }): Promise<{
+        mode: 'input' | 'drop' | 'chooser';
+    }>;
     status(profileName?: string): Promise<ProfileStatus[]>;
     private reconcileFromDisk;
     setViewport(taskId: string, width: number, height: number, options?: {

package/dist/lib/browser/service.js

@@ -8,6 +8,8 @@ import { connectSSH } from './drivers/ssh.js';
 import { generateTaskId, generateShortId, generateFunName, } from './types.js';
 import { getRefs, resolveRefToCoords } from './refs.js';
 import { clickAtCoords, hoverAtCoords, scrollAtCoords, typeText, pressKey, focusNode } from './input.js';
+import { typeEditorText } from './editor.js';
+import { detectUploadPattern, uploadToDropTarget, uploadToFileInput, uploadViaFileChooser, } from './upload.js';
 import { emit } from '../events.js';
 /**
  * Parse a `targetFilter` string into its kind + value, or return `null`
@@ -502,7 +504,7 @@ export class BrowserService {
         const { x, y } = await resolveRefToCoords(conn.cdp, sessionId, nodeMap, ref);
         await clickAtCoords(conn.cdp, sessionId, x, y);
     }
-    async type(taskId, ref, text, tabHint) {
+    async type(taskId, ref, text, tabHint, clear) {
         const { conn, task } = await this.findTask(taskId);
         const shortId = tabHint ? await this.resolveTabHint(conn, task, tabHint) : this.resolveCurrentTab(task);
         const cdpTargetId = this.getCdpTargetId(task, shortId);
@@ -514,10 +516,15 @@ export class BrowserService {
         const node = nodeMap.get(ref);
         if (!node)
             throw new Error(`Ref ${ref} not found`);
-        if (node.backendNodeId) {
-            await focusNode(conn.cdp, sessionId, node.backendNodeId);
+        if (node.editor) {
+            await typeEditorText(conn.cdp, sessionId, node, text, clear);
+        }
+        else {
+            if (node.backendNodeId) {
+                await focusNode(conn.cdp, sessionId, node.backendNodeId);
+            }
+            await typeText(conn.cdp, sessionId, text);
         }
-        await typeText(conn.cdp, sessionId, text);
     }
     async press(taskId, key, tabHint) {
         const { conn, task } = await this.findTask(taskId);
@@ -551,6 +558,60 @@ export class BrowserService {
         const sessionId = await this.getSessionId(conn, target.targetId);
         await scrollAtCoords(conn.cdp, sessionId, atX ?? 0, atY ?? 0, deltaX, deltaY);
     }
+    async upload(taskId, files, options) {
+        const { conn, task } = await this.findTask(taskId);
+        const shortId = options.tabHint
+            ? await this.resolveTabHint(conn, task, options.tabHint)
+            : this.resolveCurrentTab(task);
+        const cdpTargetId = this.getCdpTargetId(task, shortId);
+        const target = await this.getTarget(conn, cdpTargetId);
+        if (!target)
+            throw new Error(`Tab ${shortId} not found`);
+        const sessionId = await this.getSessionId(conn, target.targetId);
+        // Match the user-facing ref numbering from `agents browser refs` (which
+        // defaults to interactive=true). The other action helpers in this file
+        // use interactive=false historically, but that produces ref numbers the
+        // user never sees — `--ref 1` then resolves to the RootWebArea instead of
+        // the first interactive element. Match the listing the user actually saw.
+        const { nodeMap } = await getRefs(conn.cdp, sessionId, { interactive: true, limit: 1000 });
+        const mode = options.mode ?? 'auto';
+        if (options.trigger !== undefined || mode === 'chooser') {
+            const ref = options.trigger ?? options.ref;
+            if (ref === undefined) {
+                throw new Error('chooser mode requires --trigger <ref> (or --ref) pointing at the button that opens the file dialog');
+            }
+            const node = nodeMap.get(ref);
+            if (!node)
+                throw new Error(`Ref ${ref} not found`);
+            await uploadViaFileChooser(conn.cdp, sessionId, { node, nodeMap }, files, options.timeout);
+            return { mode: 'chooser' };
+        }
+        if (options.ref === undefined) {
+            throw new Error('upload requires --ref <n> (target element) or --trigger <n> (button that opens chooser)');
+        }
+        const node = nodeMap.get(options.ref);
+        if (!node)
+            throw new Error(`Ref ${options.ref} not found`);
+        if (!node.backendNodeId)
+            throw new Error(`Ref ${options.ref} has no DOM node`);
+        let resolved;
+        if (mode === 'input') {
+            resolved = 'input';
+        }
+        else if (mode === 'drop') {
+            resolved = 'drop';
+        }
+        else {
+            resolved = await detectUploadPattern(conn.cdp, sessionId, node.backendNodeId);
+        }
+        if (resolved === 'input') {
+            await uploadToFileInput(conn.cdp, sessionId, node.backendNodeId, files);
+        }
+        else {
+            await uploadToDropTarget(conn.cdp, sessionId, node.backendNodeId, files);
+        }
+        return { mode: resolved };
+    }
     async status(profileName) {
         const seen = new Set();
         const statuses = [];

package/dist/lib/browser/types.d.ts

@@ -83,7 +83,7 @@ export interface HistoricalTask {
     domains: string[];
     tabCount: number;
 }
-export type IPCAction = 'start' | 'launch-profile' | 'done' | 'stop' | 'status' | 'history' | 'navigate' | 'tab-add' | 'tab-focus' | 'tab-close' | 'tab-list' | 'evaluate' | 'screenshot' | 'refs' | 'click' | 'type' | 'press' | 'hover' | 'scroll' | 'set-viewport' | 'set-device' | 'console' | 'errors' | 'requests' | 'response-body' | 'wait' | 'set-download-path' | 'wait-download';
+export type IPCAction = 'start' | 'launch-profile' | 'done' | 'stop' | 'status' | 'history' | 'navigate' | 'tab-add' | 'tab-focus' | 'tab-close' | 'tab-list' | 'evaluate' | 'screenshot' | 'refs' | 'click' | 'type' | 'press' | 'hover' | 'scroll' | 'set-viewport' | 'set-device' | 'console' | 'errors' | 'requests' | 'response-body' | 'wait' | 'set-download-path' | 'wait-download' | 'upload';
 export interface IPCRequest {
     action: IPCAction;
     task?: string;
@@ -116,6 +116,9 @@ export interface IPCRequest {
     waitValue?: string | number;
     timeout?: number;
     downloadPath?: string;
+    files?: string[];
+    trigger?: number;
+    uploadMode?: 'auto' | 'input' | 'drop' | 'chooser';
 }
 export interface IPCResponse {
     ok: boolean;
@@ -129,6 +132,7 @@ export interface IPCResponse {
     result?: unknown;
     path?: string;
     refs?: string;
+    nodes?: RefNodeJson[];
     port?: number;
     pid?: number;
     logs?: ConsoleEntry[];
@@ -137,6 +141,7 @@ export interface IPCResponse {
     body?: string;
     downloadPath?: string;
     devices?: string[];
+    uploadMode?: 'input' | 'drop' | 'chooser';
 }
 export interface ConsoleEntry {
     level: 'log' | 'info' | 'warn' | 'error';
@@ -160,6 +165,13 @@ export interface NetworkRequest {
     mimeType?: string;
     timestamp: number;
 }
+export interface RefNodeJson {
+    ref: number;
+    role: string;
+    name: string;
+    attrs: string[];
+    editor?: string;
+}
 export interface DeviceDescriptor {
     width: number;
     height: number;

package/dist/lib/browser/upload.d.ts

@@ -0,0 +1,54 @@
+import type { CDPClient } from './cdp.js';
+import { type RefNode } from './refs.js';
+/**
+ * File upload strategies for `agents browser upload`.
+ *
+ * Every uploader on the web is one of three patterns:
+ *
+ *   A. Direct file input — the page exposes (or hides) `<input type=file>`.
+ *      `DOM.setFileInputFiles` plants the paths directly. The cleanest
+ *      path; works whenever the input is in the DOM, even when CSS-hidden
+ *      or visually offscreen.
+ *
+ *   B. Drag-drop target — the page listens for `drop` events on a region
+ *      (Canva, Notion, Linear, GitHub PRs). We dispatch synthetic
+ *      `dragenter`/`dragover`/`drop` events with a `DataTransfer` whose
+ *      `files` list carries a `File` built from disk bytes. The dispatch
+ *      uses real elementFromPoint coordinates so React/DOM listeners fire.
+ *
+ *   C. Native chooser interception — the user clicks a button that calls
+ *      `input.click()` and the only file input is dynamically created in
+ *      response. `Page.setInterceptFileChooserDialog` flips the chooser
+ *      from a blocking OS dialog into a CDP event; we click the trigger,
+ *      wait for `Page.fileChooserOpened`, then satisfy it with
+ *      `Page.handleFileChooser({ action: 'accept', files })`. Lifecycle:
+ *      enable interception -> click -> wait -> accept -> disable.
+ */
+export interface UploadOptions {
+    files: string[];
+}
+/** Pattern A — direct file input via `DOM.setFileInputFiles`. */
+export declare function uploadToFileInput(cdp: CDPClient, sessionId: string, backendNodeId: number, files: string[]): Promise<void>;
+/** Pattern B — synthetic drag-drop onto a target node. */
+export declare function uploadToDropTarget(cdp: CDPClient, sessionId: string, backendNodeId: number, files: string[]): Promise<void>;
+/**
+ * Pattern C — click a trigger, intercept the OS file chooser, feed files.
+ *
+ * `Page.setInterceptFileChooserDialog` must be enabled before the click; the
+ * chooser only fires once, so we register a single-shot handler ahead of time.
+ * Auto-attached child sessions matter here: the chooser event arrives on the
+ * session whose page hosts the input, which is the same session we used for
+ * the click — so we filter event params by sessionId.
+ */
+export declare function uploadViaFileChooser(cdp: CDPClient, sessionId: string, triggerRef: {
+    node: RefNode;
+    nodeMap: Map<number, RefNode>;
+}, files: string[], timeoutMs?: number): Promise<void>;
+export declare function mimeFromExt(filePath: string): string;
+/**
+ * Inspect a ref's DOM node to decide which pattern fits when the caller
+ * didn't specify. Returns 'input' if the node is `<input type=file>`,
+ * otherwise 'drop'. Chooser interception (Pattern C) is never auto-selected
+ * because it requires clicking the ref, which mutates page state — opt-in only.
+ */
+export declare function detectUploadPattern(cdp: CDPClient, sessionId: string, backendNodeId: number): Promise<'input' | 'drop'>;

package/dist/lib/browser/upload.js

@@ -0,0 +1,264 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { clickAtCoords } from './input.js';
+import { resolveRefToCoords } from './refs.js';
+const RESOLVE_FILE_INPUT_FN = `(function() {
+  const start = this;
+  // Walk multiple paths to find an <input type=file>:
+  //   1. The node itself (when the AX backend node IS the input).
+  //   2. Its ancestors via parentElement (custom button wrappers).
+  //   3. closest('input[type=file]') (handles label/span/button inside or
+  //      near a file input).
+  //   4. Across user-agent shadow boundaries via getRootNode().host
+  //      (Chromium's internal shadow-button pseudo-element for file inputs).
+  //   5. If the AX backend node is associated with a <label for=...>, follow
+  //      the htmlFor relationship.
+  //   6. Last resort: if the start node was a click target that fires
+  //      input.click() inside a click handler, fall back to the unique
+  //      <input type=file> on the page (when there is exactly one).
+  let el = start;
+  for (let i = 0; i < 8 && el; i++) {
+    if (el.tagName === 'INPUT' && el.type === 'file') return el;
+    if (el.closest) {
+      const found = el.closest('input[type=file]');
+      if (found) return found;
+    }
+    if (el.tagName === 'LABEL' && el.htmlFor) {
+      const t = document.getElementById(el.htmlFor);
+      if (t && t.tagName === 'INPUT' && t.type === 'file') return t;
+    }
+    const root = el.getRootNode && el.getRootNode();
+    if (root && root.host && root !== document) {
+      el = root.host;
+      continue;
+    }
+    el = el.parentElement;
+  }
+  // Final fallback: if exactly one file input exists on the page, use it.
+  // This handles cases where the AX tree exposes the input as an internal
+  // pseudo-button whose parentElement is null. A page with a single uploader
+  // (Slack composer, Notion image block, Canva ingredient) hits this branch.
+  const all = document.querySelectorAll('input[type=file]');
+  if (all.length === 1) return all[0];
+  return null;
+})`;
+/** Pattern A — direct file input via `DOM.setFileInputFiles`. */
+export async function uploadToFileInput(cdp, sessionId, backendNodeId, files) {
+    validateFiles(files);
+    const resolvedId = await resolveActualFileInput(cdp, sessionId, backendNodeId);
+    await cdp.send('DOM.setFileInputFiles', { files, backendNodeId: resolvedId }, sessionId);
+}
+async function resolveActualFileInput(cdp, sessionId, backendNodeId) {
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return backendNodeId;
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: RESOLVE_FILE_INPUT_FN, returnByValue: false }, sessionId);
+        if (!result.objectId) {
+            throw new Error('Ref is not (and is not contained in) an <input type=file>');
+        }
+        const inputObjectId = result.objectId;
+        try {
+            const { node } = await cdp.send('DOM.describeNode', { objectId: inputObjectId }, sessionId);
+            return node.backendNodeId;
+        }
+        finally {
+            await cdp.send('Runtime.releaseObject', { objectId: inputObjectId }, sessionId);
+        }
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}
+const DRAG_DROP_FN = `(function(files) {
+  const el = this;
+  const rect = el.getBoundingClientRect();
+  const x = rect.left + rect.width / 2;
+  const y = rect.top + rect.height / 2;
+  const dt = new DataTransfer();
+  for (const f of files) {
+    const u8 = Uint8Array.from(atob(f.bytes), c => c.charCodeAt(0));
+    const blob = new Blob([u8], { type: f.type || 'application/octet-stream' });
+    const file = new File([blob], f.name, { type: f.type || 'application/octet-stream' });
+    dt.items.add(file);
+  }
+  function dispatch(type) {
+    // Chromium does not honor the dataTransfer field in DragEventInit for
+    // synthetic events. Build the DragEvent with no dataTransfer in the init,
+    // then override the event dataTransfer getter via defineProperty so
+    // page-level listeners see the File list we constructed.
+    const ev = new DragEvent(type, {
+      bubbles: true,
+      cancelable: true,
+      composed: true,
+      clientX: x,
+      clientY: y,
+    });
+    Object.defineProperty(ev, 'dataTransfer', { value: dt });
+    el.dispatchEvent(ev);
+  }
+  dispatch('dragenter');
+  dispatch('dragover');
+  dispatch('drop');
+  return { dispatched: 3, files: files.length };
+})`;
+/** Pattern B — synthetic drag-drop onto a target node. */
+export async function uploadToDropTarget(cdp, sessionId, backendNodeId, files) {
+    validateFiles(files);
+    const payload = files.map((p) => ({
+        name: path.basename(p),
+        type: mimeFromExt(p),
+        bytes: fs.readFileSync(p).toString('base64'),
+    }));
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId) {
+        throw new Error('Drop target node could not be resolved');
+    }
+    const objectId = object.objectId;
+    try {
+        const r = await cdp.send('Runtime.callFunctionOn', {
+            objectId,
+            functionDeclaration: DRAG_DROP_FN,
+            arguments: [{ value: payload }],
+            returnByValue: true,
+            awaitPromise: true,
+        }, sessionId);
+        if (r.exceptionDetails) {
+            throw new Error('Drop dispatch threw: ' + JSON.stringify(r.exceptionDetails));
+        }
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}
+/**
+ * Pattern C — click a trigger, intercept the OS file chooser, feed files.
+ *
+ * `Page.setInterceptFileChooserDialog` must be enabled before the click; the
+ * chooser only fires once, so we register a single-shot handler ahead of time.
+ * Auto-attached child sessions matter here: the chooser event arrives on the
+ * session whose page hosts the input, which is the same session we used for
+ * the click — so we filter event params by sessionId.
+ */
+export async function uploadViaFileChooser(cdp, sessionId, triggerRef, files, timeoutMs = 5000) {
+    validateFiles(files);
+    await cdp.send('Page.enable', {}, sessionId);
+    await cdp.send('Page.setInterceptFileChooserDialog', { enabled: true }, sessionId);
+    let opened = null;
+    let resolve;
+    let reject;
+    const wait = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    const handler = (params) => {
+        const ev = params;
+        if (typeof ev.backendNodeId === 'number') {
+            opened = { backendNodeId: ev.backendNodeId };
+            resolve();
+        }
+    };
+    cdp.on('Page.fileChooserOpened', handler);
+    const timer = setTimeout(() => {
+        reject(new Error(`File chooser did not open within ${timeoutMs}ms — is the trigger ref correct?`));
+    }, timeoutMs);
+    try {
+        const { x, y } = await resolveRefToCoords(cdp, sessionId, triggerRef.nodeMap, triggerRef.node.ref);
+        await clickAtCoords(cdp, sessionId, x, y);
+        await wait;
+        await cdp.send('Page.handleFileChooser', { action: 'accept', files }, sessionId);
+        // Some Chromium builds expect setFileInputFiles instead of handleFileChooser.
+        // We try handleFileChooser first because it's the documented path for
+        // intercepted dialogs; if Chromium rejects it (older protocol), fall back
+        // to setFileInputFiles using the backendNodeId from the event.
+    }
+    catch (err) {
+        if (opened && err instanceof Error && /not supported|not found|Method/i.test(err.message)) {
+            await cdp.send('DOM.setFileInputFiles', { files, backendNodeId: opened.backendNodeId }, sessionId);
+        }
+        else {
+            throw err;
+        }
+    }
+    finally {
+        clearTimeout(timer);
+        cdp.off('Page.fileChooserOpened', handler);
+        await cdp.send('Page.setInterceptFileChooserDialog', { enabled: false }, sessionId).catch(() => { });
+    }
+}
+function validateFiles(files) {
+    if (!files || files.length === 0) {
+        throw new Error('At least one file path is required');
+    }
+    for (const f of files) {
+        if (!path.isAbsolute(f)) {
+            throw new Error(`Upload path must be absolute: ${f}`);
+        }
+        if (!fs.existsSync(f)) {
+            throw new Error(`File not found: ${f}`);
+        }
+    }
+}
+const MIME_BY_EXT = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.svg': 'image/svg+xml',
+    '.pdf': 'application/pdf',
+    '.txt': 'text/plain',
+    '.csv': 'text/csv',
+    '.json': 'application/json',
+    '.zip': 'application/zip',
+    '.mp4': 'video/mp4',
+    '.mov': 'video/quicktime',
+    '.mp3': 'audio/mpeg',
+    '.wav': 'audio/wav',
+};
+export function mimeFromExt(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return MIME_BY_EXT[ext] ?? 'application/octet-stream';
+}
+/**
+ * Inspect a ref's DOM node to decide which pattern fits when the caller
+ * didn't specify. Returns 'input' if the node is `<input type=file>`,
+ * otherwise 'drop'. Chooser interception (Pattern C) is never auto-selected
+ * because it requires clicking the ref, which mutates page state — opt-in only.
+ */
+export async function detectUploadPattern(cdp, sessionId, backendNodeId) {
+    const { node } = await cdp.send('DOM.describeNode', { backendNodeId, depth: 0 }, sessionId);
+    if (isFileInputNode(node))
+        return 'input';
+    // The node itself isn't <input type=file>, but it might be a button or
+    // shadow-DOM descendant *inside* one — that's what the accessibility tree
+    // surfaces for file inputs. Walk up to confirm before falling back to drop.
+    const { object } = await cdp.send('DOM.resolveNode', { backendNodeId }, sessionId);
+    if (!object.objectId)
+        return 'drop';
+    const objectId = object.objectId;
+    try {
+        const { result } = await cdp.send('Runtime.callFunctionOn', { objectId, functionDeclaration: RESOLVE_FILE_INPUT_FN, returnByValue: false }, sessionId);
+        if (result.objectId) {
+            await cdp.send('Runtime.releaseObject', { objectId: result.objectId }, sessionId);
+            return 'input';
+        }
+        return 'drop';
+    }
+    finally {
+        await cdp.send('Runtime.releaseObject', { objectId }, sessionId);
+    }
+}
+function isFileInputNode(node) {
+    const tag = (node.nodeName ?? '').toLowerCase();
+    if (tag !== 'input')
+        return false;
+    const attrs = node.attributes ?? [];
+    for (let i = 0; i < attrs.length; i += 2) {
+        if (attrs[i] === 'type' && attrs[i + 1]?.toLowerCase() === 'file') {
+            return true;
+        }
+    }
+    return false;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@phnx-labs/agents-cli",
-  "version": "1.17.3",
+  "version": "1.17.5",
   "description": "One CLI for all your AI coding agents - versions, config, cloud dispatch, sessions, and teams",
   "type": "module",
   "main": "dist/index.js",
@@ -87,6 +87,7 @@
     "@types/diff": "^6.0.0",
     "@types/marked-terminal": "^6.1.1",
     "@types/node": "^22.0.0",
+    "playwright": "^1.44.0",
     "tsx": "^4.19.0",
     "typescript": "^5.5.0",
     "vitest": "^2.0.0"