@minionry/minion 0.7.36

package/dist/electron-host/dom-driver.js

@@ -0,0 +1,642 @@
+import { EventEmitter } from "node:events";
+export class DomDriverError extends Error {
+    code;
+    details;
+    constructor(code, message, details) {
+        super(message);
+        this.name = "DomDriverError";
+        this.code = code;
+        this.details = details;
+    }
+}
+export class DomDriver extends EventEmitter {
+    tabs;
+    leases;
+    arbitration;
+    nowMs;
+    setTimer;
+    consoleBufferSize;
+    networkBufferSize;
+    protocolVersion;
+    state = new Map();
+    constructor(opts) {
+        super();
+        this.tabs = opts.tabs;
+        this.leases = opts.leases ?? null;
+        this.arbitration = opts.arbitration ?? null;
+        this.nowMs = opts.now ?? (() => Date.now());
+        this.setTimer = opts.setTimeout ?? setTimeout;
+        this.consoleBufferSize = opts.consoleBufferSize ?? 500;
+        this.networkBufferSize = opts.networkBufferSize ?? 500;
+        this.protocolVersion = opts.protocolVersion ?? "1.3";
+    }
+    async snapshot(args) {
+        const { tab, dbg, state } = this.beginRead(args.tabId);
+        const resp = await dbg.sendCommand("Accessibility.getFullAXTree");
+        state.currentEpoch += 1;
+        state.refs.clear();
+        const tree = buildSnapshotTree(resp.nodes, state.currentEpoch, state.refs);
+        this.emit("snapshot", args.tabId, state.currentEpoch);
+        return {
+            url: tab.getURL(),
+            title: tab.getTitle(),
+            epoch: state.currentEpoch,
+            root: tree,
+        };
+    }
+    async click(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        const { x, y } = await this.resolveRefToCenter(args.tabId, args.ref);
+        const button = args.button ?? "left";
+        const clickCount = args.clickCount ?? 1;
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mousePressed",
+            x,
+            y,
+            button,
+            clickCount,
+        });
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mouseReleased",
+            x,
+            y,
+            button,
+            clickCount,
+        });
+    }
+    async type(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        await this.callOnRef(args.tabId, args.ref, "function(){ this.focus(); }");
+        for (const ch of args.text) {
+            await dbg.sendCommand("Input.dispatchKeyEvent", {
+                type: "keyDown",
+                text: ch,
+                unmodifiedText: ch,
+                key: ch,
+            });
+            await dbg.sendCommand("Input.dispatchKeyEvent", {
+                type: "keyUp",
+                key: ch,
+            });
+        }
+        if (args.submit) {
+            await dbg.sendCommand("Input.dispatchKeyEvent", {
+                type: "keyDown",
+                key: "Enter",
+                code: "Enter",
+            });
+            await dbg.sendCommand("Input.dispatchKeyEvent", {
+                type: "keyUp",
+                key: "Enter",
+                code: "Enter",
+            });
+        }
+    }
+    async hover(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        const { x, y } = await this.resolveRefToCenter(args.tabId, args.ref);
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mouseMoved",
+            x,
+            y,
+        });
+    }
+    async press_key(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        await dbg.sendCommand("Input.dispatchKeyEvent", {
+            type: "keyDown",
+            key: args.key,
+            code: args.key,
+        });
+        await dbg.sendCommand("Input.dispatchKeyEvent", {
+            type: "keyUp",
+            key: args.key,
+            code: args.key,
+        });
+    }
+    async select_option(args) {
+        this.beginMutate(args.tabId, args.agentId);
+        const fn = `function(){
+      const opts = Array.from(this.options || []);
+      const wanted = new Set(${JSON.stringify(args.values)});
+      for (const o of opts) o.selected = wanted.has(o.value);
+      this.dispatchEvent(new Event('input', { bubbles: true }));
+      this.dispatchEvent(new Event('change', { bubbles: true }));
+      return opts.filter(o => o.selected).map(o => o.value);
+    }`;
+        await this.callOnRef(args.tabId, args.ref, fn);
+    }
+    async fill_form(args) {
+        this.beginMutate(args.tabId, args.agentId);
+        for (const field of args.fields) {
+            const fn = `function(v){
+        if ('value' in this) this.value = v;
+        else this.textContent = v;
+        this.dispatchEvent(new Event('input', { bubbles: true }));
+        this.dispatchEvent(new Event('change', { bubbles: true }));
+      }`;
+            await this.callOnRef(args.tabId, field.ref, fn, [
+                { value: field.value },
+            ]);
+        }
+    }
+    async drag_drop(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        const from = await this.resolveRefToCenter(args.tabId, args.fromRef);
+        const to = await this.resolveRefToCenter(args.tabId, args.toRef);
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mouseMoved",
+            x: from.x,
+            y: from.y,
+        });
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mousePressed",
+            x: from.x,
+            y: from.y,
+            button: "left",
+            clickCount: 1,
+        });
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mouseMoved",
+            x: to.x,
+            y: to.y,
+        });
+        await dbg.sendCommand("Input.dispatchMouseEvent", {
+            type: "mouseReleased",
+            x: to.x,
+            y: to.y,
+            button: "left",
+            clickCount: 1,
+        });
+    }
+    async file_upload(args) {
+        const { dbg } = this.beginMutate(args.tabId, args.agentId);
+        const entry = this.requireRef(args.tabId, args.ref);
+        await dbg.sendCommand("DOM.setFileInputFiles", {
+            files: args.paths,
+            backendNodeId: entry.backendNodeId,
+        });
+    }
+    async handle_dialog(args) {
+        const { dbg, state } = this.beginMutate(args.tabId, args.agentId);
+        const cmd = { accept: args.accept };
+        if (args.promptText !== undefined)
+            cmd.promptText = args.promptText;
+        await dbg.sendCommand("Page.handleJavaScriptDialog", cmd);
+        state.pendingDialog = null;
+    }
+    async wait_for(args) {
+        const { dbg, tab } = this.beginRead(args.tabId);
+        if (this.leases && args.agentId) {
+            const deadline = this.nowMs() + args.timeoutMs + 5_000;
+            this.leases.extendLease(args.tabId, args.agentId, deadline);
+        }
+        const pollMs = Math.max(50, args.pollIntervalMs ?? 100);
+        const startedAt = this.nowMs();
+        const deadlineAt = startedAt + args.timeoutMs;
+        while (true) {
+            if (args.condition.type === "expression") {
+                const ok = await this.evalBoolean(dbg, args.condition.expr);
+                if (ok)
+                    return;
+            }
+            else {
+                if (!tab.isLoading())
+                    return;
+            }
+            if (this.nowMs() >= deadlineAt) {
+                throw new DomDriverError("timeout", `wait_for timed out after ${args.timeoutMs}ms`);
+            }
+            await this.sleep(pollMs);
+        }
+    }
+    async screenshot(args) {
+        const { dbg } = this.beginRead(args.tabId);
+        const format = args.format ?? "png";
+        const params = {
+            format,
+            captureBeyondViewport: !!args.fullPage,
+        };
+        if (format === "jpeg") {
+            const q = args.quality ?? 80;
+            if (q < 0 || q > 100) {
+                throw new DomDriverError("timeout", "screenshot quality must be 0-100", {
+                    quality: q,
+                });
+            }
+            params.quality = q;
+        }
+        if (args.clip) {
+            params.clip = { ...args.clip, scale: 1 };
+        }
+        const resp = await dbg.sendCommand("Page.captureScreenshot", params);
+        return { format, data: Buffer.from(resp.data, "base64") };
+    }
+    console(args) {
+        const { state } = this.beginRead(args.tabId);
+        return filterSince(state.console, args.since);
+    }
+    network(args) {
+        const { state } = this.beginRead(args.tabId);
+        return filterSince(state.network, args.since);
+    }
+    disposeTab(tabId) {
+        const state = this.state.get(tabId);
+        if (!state)
+            return;
+        const tab = this.tabs.getWebContents(tabId);
+        if (tab && !tab.isDestroyed()) {
+            try {
+                if (tab.debugger.isAttached())
+                    tab.debugger.detach();
+            }
+            catch {
+            }
+        }
+        this.state.delete(tabId);
+    }
+    beginRead(tabId) {
+        const tab = this.tabs.getWebContents(tabId);
+        if (!tab || tab.isDestroyed()) {
+            throw new DomDriverError("tab_not_found", `no such tab: ${tabId}`, {
+                tabId,
+            });
+        }
+        const state = this.attach(tabId, tab);
+        return { tab, dbg: tab.debugger, state };
+    }
+    beginMutate(tabId, agentId) {
+        const ctx = this.beginRead(tabId);
+        if (this.arbitration?.isUserActive(tabId)) {
+            throw new DomDriverError("arbitration_user_active", "human user is currently driving this tab", { tabId, agentId });
+        }
+        if (ctx.tab.isLoading()) {
+            throw new DomDriverError("navigation_in_progress", "tab is navigating; retry after onload", { tabId });
+        }
+        return ctx;
+    }
+    attach(tabId, tab) {
+        const existing = this.state.get(tabId);
+        if (existing)
+            return existing;
+        if (!tab.debugger.isAttached()) {
+            tab.debugger.attach(this.protocolVersion);
+        }
+        const messageListener = (_event, method, params) => {
+            this.handleCdpEvent(tabId, method, params);
+        };
+        const state = {
+            attached: true,
+            refs: new Map(),
+            currentEpoch: 0,
+            console: [],
+            consoleSeq: 0,
+            network: [],
+            networkSeq: 0,
+            pendingDialog: null,
+            messageListener,
+        };
+        this.state.set(tabId, state);
+        tab.debugger.on("message", messageListener);
+        void tab.debugger.sendCommand("Runtime.enable").catch(noop);
+        void tab.debugger.sendCommand("Page.enable").catch(noop);
+        void tab.debugger.sendCommand("Network.enable").catch(noop);
+        void tab.debugger.sendCommand("Log.enable").catch(noop);
+        void tab.debugger.sendCommand("DOM.enable").catch(noop);
+        void tab.debugger.sendCommand("Accessibility.enable").catch(noop);
+        return state;
+    }
+    handleCdpEvent(tabId, method, params) {
+        const state = this.state.get(tabId);
+        if (!state)
+            return;
+        switch (method) {
+            case "Runtime.consoleAPICalled": {
+                const entry = consoleEntryFromRuntime(params, ++state.consoleSeq, this.nowMs());
+                pushCapped(state.console, entry, this.consoleBufferSize);
+                return;
+            }
+            case "Log.entryAdded": {
+                const entry = consoleEntryFromLog(params, ++state.consoleSeq, this.nowMs());
+                if (entry)
+                    pushCapped(state.console, entry, this.consoleBufferSize);
+                return;
+            }
+            case "Network.requestWillBeSent": {
+                const entry = networkEntryFromRequest(params, ++state.networkSeq, this.nowMs());
+                if (entry)
+                    pushCapped(state.network, entry, this.networkBufferSize);
+                return;
+            }
+            case "Network.responseReceived": {
+                annotateResponse(state.network, params);
+                return;
+            }
+            case "Network.loadingFailed": {
+                annotateFailure(state.network, params, this.nowMs());
+                return;
+            }
+            case "Page.javascriptDialogOpening": {
+                state.pendingDialog = {
+                    url: String(params.url ?? ""),
+                    type: String(params.type ?? ""),
+                    message: String(params.message ?? ""),
+                };
+                return;
+            }
+            default:
+                return;
+        }
+    }
+    requireRef(tabId, ref) {
+        const state = this.state.get(tabId);
+        if (!state) {
+            throw new DomDriverError("ref_stale", `no snapshot for tab: ${tabId}`, {
+                tabId,
+                ref,
+            });
+        }
+        const entry = state.refs.get(ref);
+        if (!entry || entry.epoch !== state.currentEpoch) {
+            throw new DomDriverError("ref_stale", `ref no longer valid: ${ref}`, {
+                ref,
+                expectedEpoch: state.currentEpoch,
+                gotEpoch: entry?.epoch ?? null,
+            });
+        }
+        return entry;
+    }
+    async resolveRefToCenter(tabId, ref) {
+        const entry = this.requireRef(tabId, ref);
+        const { dbg } = this.beginRead(tabId);
+        let resolved;
+        try {
+            resolved = await dbg.sendCommand("DOM.resolveNode", { backendNodeId: entry.backendNodeId });
+        }
+        catch (err) {
+            throw new DomDriverError("ref_stale", `node no longer in DOM: ${ref}`, {
+                ref,
+                cause: errMessage(err),
+            });
+        }
+        const resp = await dbg.sendCommand("Runtime.callFunctionOn", {
+            objectId: resolved.object.objectId,
+            functionDeclaration: "function(){ const r = this.getBoundingClientRect(); return { x: r.x, y: r.y, w: r.width, h: r.height }; }",
+            returnByValue: true,
+        });
+        if (resp.exceptionDetails) {
+            throw new DomDriverError("ref_stale", "getBoundingClientRect threw", {
+                ref,
+                cause: resp.exceptionDetails.text,
+            });
+        }
+        const rect = resp.result.value;
+        if (!rect) {
+            throw new DomDriverError("ref_stale", "node has no rect", { ref });
+        }
+        return { x: rect.x + rect.w / 2, y: rect.y + rect.h / 2 };
+    }
+    async callOnRef(tabId, ref, functionDeclaration, args) {
+        const entry = this.requireRef(tabId, ref);
+        const { dbg } = this.beginRead(tabId);
+        const resolved = await dbg.sendCommand("DOM.resolveNode", { backendNodeId: entry.backendNodeId });
+        const resp = await dbg.sendCommand("Runtime.callFunctionOn", {
+            objectId: resolved.object.objectId,
+            functionDeclaration,
+            arguments: args,
+            returnByValue: true,
+        });
+        if (resp.exceptionDetails) {
+            throw new DomDriverError("ref_stale", "function call on ref threw", {
+                ref,
+                cause: resp.exceptionDetails.text,
+            });
+        }
+        return resp.result.value;
+    }
+    async evalBoolean(dbg, expr) {
+        const resp = await dbg.sendCommand("Runtime.evaluate", {
+            expression: expr,
+            returnByValue: true,
+        });
+        return !!resp.result.value;
+    }
+    sleep(ms) {
+        return new Promise((resolve) => {
+            this.setTimer(resolve, ms);
+        });
+    }
+}
+function noop() {
+}
+function errMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function pushCapped(buffer, entry, max) {
+    buffer.push(entry);
+    while (buffer.length > max)
+        buffer.shift();
+}
+function filterSince(buffer, since) {
+    if (since === undefined)
+        return buffer.slice();
+    return buffer.filter((e) => e.seq > since);
+}
+function axValue(v) {
+    return v?.value;
+}
+function stringValue(v) {
+    const x = axValue(v);
+    return typeof x === "string" ? x : undefined;
+}
+const AX_PROPERTY_MAPPER = {
+    level: (raw, out) => {
+        if (typeof raw === "number")
+            out.level = raw;
+    },
+    checked: (raw, out) => {
+        if (raw === "mixed")
+            out.checked = "mixed";
+        else if (typeof raw === "boolean")
+            out.checked = raw;
+    },
+    selected: (raw, out) => {
+        if (typeof raw === "boolean")
+            out.selected = raw;
+    },
+    expanded: (raw, out) => {
+        if (typeof raw === "boolean")
+            out.expanded = raw;
+    },
+    disabled: (raw, out) => {
+        if (typeof raw === "boolean")
+            out.disabled = raw;
+    },
+    focused: (raw, out) => {
+        if (typeof raw === "boolean")
+            out.focused = raw;
+    },
+};
+function mintRef(ctx, n) {
+    const ref = ctx.mint();
+    if (n.backendDOMNodeId !== undefined) {
+        ctx.refs.set(ref, { epoch: ctx.epoch, backendNodeId: n.backendDOMNodeId });
+    }
+    return ref;
+}
+function visitChildren(ctx, n) {
+    const built = [];
+    for (const childId of n.childIds ?? []) {
+        const c = ctx.byId.get(childId);
+        if (!c)
+            continue;
+        const child = visitNode(ctx, c);
+        if (child)
+            built.push(child);
+    }
+    return built;
+}
+function visitIgnoredNode(ctx, n) {
+    const inlined = visitChildren(ctx, n);
+    if (inlined.length === 1)
+        return inlined[0];
+    if (inlined.length === 0)
+        return null;
+    const ref = mintRef(ctx, n);
+    return { ref, role: "generic", children: inlined };
+}
+function projectAxFields(n, out) {
+    const name = stringValue(n.name);
+    if (name)
+        out.name = name;
+    const description = stringValue(n.description);
+    if (description)
+        out.description = description;
+    const value = stringValue(n.value);
+    if (value)
+        out.value = value;
+    for (const p of n.properties ?? []) {
+        const mapper = AX_PROPERTY_MAPPER[p.name];
+        if (mapper)
+            mapper(p.value.value, out);
+    }
+}
+function visitNode(ctx, n) {
+    if (n.ignored)
+        return visitIgnoredNode(ctx, n);
+    const out = {
+        ref: mintRef(ctx, n),
+        role: stringValue(n.role) ?? "unknown",
+        children: [],
+    };
+    projectAxFields(n, out);
+    out.children = visitChildren(ctx, n);
+    return out;
+}
+function buildSnapshotTree(nodes, epoch, refs) {
+    if (!Array.isArray(nodes) || nodes.length === 0) {
+        return { ref: `e${epoch}n0`, role: "RootWebArea", children: [] };
+    }
+    const byId = new Map();
+    for (const n of nodes)
+        byId.set(n.nodeId, n);
+    let idxCounter = 0;
+    const ctx = { byId, refs, epoch, mint: () => `e${epoch}n${++idxCounter}` };
+    const root = visitNode(ctx, nodes[0]);
+    if (!root) {
+        return { ref: `e${epoch}n0`, role: "RootWebArea", children: [] };
+    }
+    return root;
+}
+function consoleEntryFromRuntime(params, seq, ts) {
+    const typeRaw = String(params.type ?? "log");
+    const level = normalizeLevel(typeRaw);
+    const argsArr = Array.isArray(params.args)
+        ? params.args
+        : [];
+    const text = argsArr
+        .map((a) => {
+        if (a && a.value !== undefined)
+            return String(a.value);
+        if (a && a.description !== undefined)
+            return String(a.description);
+        return "";
+    })
+        .join(" ");
+    const sf = params
+        .stackTrace?.callFrames?.[0];
+    const entry = { seq, ts, level, text };
+    if (sf?.url)
+        entry.url = sf.url;
+    if (typeof sf?.lineNumber === "number")
+        entry.lineNumber = sf.lineNumber;
+    return entry;
+}
+function consoleEntryFromLog(params, seq, ts) {
+    const e = params
+        .entry;
+    if (!e || typeof e.text !== "string")
+        return null;
+    return {
+        seq,
+        ts,
+        level: normalizeLevel(e.level ?? "log"),
+        text: e.text,
+        url: e.url,
+        lineNumber: e.lineNumber,
+    };
+}
+function normalizeLevel(raw) {
+    switch (raw) {
+        case "info":
+        case "warning":
+        case "warn":
+        case "error":
+        case "debug":
+            return raw === "warning" ? "warn" : raw;
+        default:
+            return "log";
+    }
+}
+function networkEntryFromRequest(params, seq, ts) {
+    const requestId = params.requestId;
+    const request = params.request;
+    if (typeof requestId !== "string" || !request || typeof request.url !== "string") {
+        return null;
+    }
+    const resourceTypeRaw = params.type;
+    return {
+        seq,
+        ts,
+        requestId,
+        url: request.url,
+        method: request.method ?? "GET",
+        resourceType: typeof resourceTypeRaw === "string" ? resourceTypeRaw : undefined,
+    };
+}
+function annotateResponse(buffer, params) {
+    const requestId = params.requestId;
+    const response = params.response;
+    if (typeof requestId !== "string" || !response)
+        return;
+    for (let i = buffer.length - 1; i >= 0; i--) {
+        if (buffer[i].requestId === requestId) {
+            buffer[i].status = response.status;
+            buffer[i].statusText = response.statusText;
+            return;
+        }
+    }
+}
+function annotateFailure(buffer, params, _ts) {
+    const requestId = params.requestId;
+    const errorText = params.errorText;
+    if (typeof requestId !== "string")
+        return;
+    for (let i = buffer.length - 1; i >= 0; i--) {
+        if (buffer[i].requestId === requestId) {
+            buffer[i].failed = true;
+            buffer[i].errorText = typeof errorText === "string" ? errorText : undefined;
+            return;
+        }
+    }
+}