alumnium 0.14.0

package/dist/accessibility/ChromiumAccessibilityTree.js

@@ -0,0 +1,216 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export class ChromiumAccessibilityTree extends BaseAccessibilityTree {
+    constructor(cdpResponse) {
+        super();
+        this.nextRawId = 0;
+        this.raw = null;
+        this.cdpResponse = cdpResponse;
+    }
+    static fromXml(xmlString) {
+        const instance = new ChromiumAccessibilityTree({ nodes: [] });
+        instance.raw = xmlString;
+        return instance;
+    }
+    toStr() {
+        if (this.raw !== null) {
+            return this.raw;
+        }
+        const nodes = this.cdpResponse.nodes;
+        const nodeLookup = {};
+        for (const node of nodes) {
+            nodeLookup[node.nodeId] = node;
+        }
+        // Build tree structure and convert to XML
+        const rootNodes = [];
+        for (const node of nodes) {
+            if (!node.parentId) {
+                const xmlNode = this.nodeToXml(node, nodeLookup);
+                rootNodes.push(xmlNode);
+            }
+        }
+        // Combine all root nodes into a single XML string
+        let xmlString = "";
+        for (const root of rootNodes) {
+            xmlString += this.elementToString(root, 0);
+        }
+        this.raw = xmlString;
+        return this.raw;
+    }
+    nodeToXml(node, nodeLookup) {
+        const role = node.role?.value || "unknown";
+        const elem = {
+            tag: role,
+            attributes: {},
+            children: [],
+        };
+        // Add our own sequential raw_id attribute
+        this.nextRawId++;
+        elem.attributes.raw_id = String(this.nextRawId);
+        // Add all node attributes as XML attributes
+        if (node.backendDOMNodeId !== undefined) {
+            elem.attributes.backendDOMNodeId = String(node.backendDOMNodeId);
+        }
+        if (node.nodeId !== undefined) {
+            elem.attributes.nodeId = String(node.nodeId);
+        }
+        if (node.ignored !== undefined) {
+            elem.attributes.ignored = String(node.ignored);
+        }
+        // Add name as attribute if present
+        if (node.name?.value) {
+            elem.attributes.name = node.name.value;
+        }
+        // Add properties as attributes
+        if (node.properties) {
+            for (const prop of node.properties) {
+                const propName = prop.name || "";
+                const propValue = prop.value;
+                if (propValue &&
+                    typeof propValue === "object" &&
+                    "value" in propValue) {
+                    elem.attributes[propName] = String(propValue.value);
+                }
+                else if (typeof propValue === "object") {
+                    // Complex property values (like nodeList) are converted to empty string
+                    elem.attributes[propName] = "";
+                }
+                else {
+                    elem.attributes[propName] = String(propValue);
+                }
+            }
+        }
+        // Process children recursively
+        if (node.childIds) {
+            for (const childId of node.childIds) {
+                if (childId in nodeLookup) {
+                    const childElem = this.nodeToXml(nodeLookup[childId], nodeLookup);
+                    elem.children.push(childElem);
+                }
+            }
+        }
+        return elem;
+    }
+    elementToString(elem, indent) {
+        const indentStr = "  ".repeat(indent);
+        let result = `${indentStr}<${elem.tag}`;
+        // Add attributes
+        for (const [key, value] of Object.entries(elem.attributes)) {
+            // Escape XML special characters
+            const escapedValue = value
+                .replace(/&/g, "&amp;")
+                .replace(/</g, "&lt;")
+                .replace(/>/g, "&gt;")
+                .replace(/"/g, "&quot;")
+                .replace(/'/g, "&apos;");
+            result += ` ${key}="${escapedValue}"`;
+        }
+        if (elem.children.length === 0 && !elem.text) {
+            result += " />\n";
+        }
+        else {
+            result += ">";
+            if (elem.text) {
+                result += elem.text;
+            }
+            if (elem.children.length > 0) {
+                result += "\n";
+                for (const child of elem.children) {
+                    result += this.elementToString(child, indent + 1);
+                }
+                result += indentStr;
+            }
+            result += `</${elem.tag}>\n`;
+        }
+        return result;
+    }
+    elementById(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            throw new Error(`No element with raw_id=${rawId} found`);
+        }
+        const backendNodeIdStr = element.attributes.backendDOMNodeId;
+        if (!backendNodeIdStr) {
+            throw new Error(`Element with raw_id=${rawId} has no backendDOMNodeId attribute`);
+        }
+        return new AccessibilityElement(undefined, undefined, undefined, element.tag, undefined, parseInt(backendNodeIdStr));
+    }
+    scopeToArea(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            return this;
+        }
+        const scopedXml = this.elementToString(element, 0);
+        return ChromiumAccessibilityTree.fromXml(scopedXml);
+    }
+    findElementByRawId(xmlString, targetRawId) {
+        // Parse the XML string to find the element
+        // This is a simple XML parser for our specific use case
+        const elements = this.parseSimpleXml(xmlString);
+        const search = (elem) => {
+            if (elem.attributes.raw_id === String(targetRawId)) {
+                return elem;
+            }
+            for (const child of elem.children) {
+                const result = search(child);
+                if (result)
+                    return result;
+            }
+            return null;
+        };
+        for (const elem of elements) {
+            const result = search(elem);
+            if (result)
+                return result;
+        }
+        return null;
+    }
+    parseSimpleXml(xmlString) {
+        const elements = [];
+        const stack = [];
+        // Combined regex for all XML tokens (opening tags, closing tags, self-closing tags)
+        const tokenRegex = /<\/?([a-zA-Z_][\w:.-]*)((?:\s+[\w:.-]+="[^"]*")*)\s*(\/?)>/g;
+        const attrRegex = /([\w:.-]+)="([^"]*)"/g;
+        let match;
+        while ((match = tokenRegex.exec(xmlString)) !== null) {
+            const fullMatch = match[0];
+            const tagName = match[1];
+            const attrsString = match[2];
+            const selfClosing = match[3] === "/";
+            const isClosingTag = fullMatch.startsWith("</");
+            if (isClosingTag) {
+                // Handle closing tag - pop from stack
+                if (stack.length > 0 && stack[stack.length - 1].tag === tagName) {
+                    stack.pop();
+                }
+            }
+            else {
+                // Handle opening or self-closing tag
+                // Parse attributes
+                const attributes = {};
+                let attrMatch;
+                attrRegex.lastIndex = 0;
+                while ((attrMatch = attrRegex.exec(attrsString)) !== null) {
+                    attributes[attrMatch[1]] = attrMatch[2];
+                }
+                const elem = {
+                    tag: tagName,
+                    attributes,
+                    children: [],
+                };
+                if (stack.length > 0) {
+                    stack[stack.length - 1].children.push(elem);
+                }
+                else {
+                    elements.push(elem);
+                }
+                if (!selfClosing) {
+                    stack.push(elem);
+                }
+            }
+        }
+        return elements;
+    }
+}

package/dist/accessibility/UIAutomator2AccessibilityTree.d.ts

@@ -0,0 +1,15 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export declare class UIAutomator2AccessibilityTree extends BaseAccessibilityTree {
+    private xmlString;
+    private nextRawId;
+    private raw;
+    constructor(xmlString: string);
+    toStr(): string;
+    private addRawIds;
+    elementById(rawId: number): AccessibilityElement;
+    scopeToArea(rawId: number): UIAutomator2AccessibilityTree;
+    private findElementByRawId;
+    private elementToString;
+    private parseSimpleXml;
+}

package/dist/accessibility/UIAutomator2AccessibilityTree.js

@@ -0,0 +1,149 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export class UIAutomator2AccessibilityTree extends BaseAccessibilityTree {
+    constructor(xmlString) {
+        super();
+        this.nextRawId = 0;
+        this.raw = null;
+        // Clean multiple XML declaration lines from page source
+        const xmlDeclarationPattern = /^\s*<\?xml.*\?>\s*$/gm;
+        const lines = xmlString.split("\n");
+        const cleanedLines = lines.filter((line) => !xmlDeclarationPattern.test(line));
+        const cleanedXmlContent = cleanedLines.join("\n");
+        this.xmlString = `<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n<root>\n${cleanedXmlContent}\n</root>`;
+    }
+    toStr() {
+        if (this.raw !== null) {
+            return this.raw;
+        }
+        // Parse the XML
+        const elements = this.parseSimpleXml(this.xmlString);
+        // Add raw_id attributes recursively
+        for (const elem of elements) {
+            this.addRawIds(elem);
+        }
+        // Serialize back to string
+        let result = "";
+        for (const elem of elements) {
+            result += this.elementToString(elem, 0);
+        }
+        this.raw = result;
+        return this.raw;
+    }
+    addRawIds(elem) {
+        this.nextRawId++;
+        elem.attributes.raw_id = String(this.nextRawId);
+        for (const child of elem.children) {
+            this.addRawIds(child);
+        }
+    }
+    elementById(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            throw new Error(`No element with raw_id=${rawId} found`);
+        }
+        // Extract properties for UIAutomator2
+        return new AccessibilityElement(rawId, undefined, undefined, element.attributes.class || element.tag, undefined, undefined, element.attributes["resource-id"], undefined, element.attributes.text, element.attributes["content-desc"], element.attributes.bounds);
+    }
+    scopeToArea(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            return this;
+        }
+        const scopedXml = this.elementToString(element, 0);
+        return new UIAutomator2AccessibilityTree(scopedXml);
+    }
+    findElementByRawId(xmlString, targetRawId) {
+        const elements = this.parseSimpleXml(xmlString);
+        const search = (elem) => {
+            if (elem.attributes.raw_id === String(targetRawId)) {
+                return elem;
+            }
+            for (const child of elem.children) {
+                const result = search(child);
+                if (result)
+                    return result;
+            }
+            return null;
+        };
+        for (const elem of elements) {
+            const result = search(elem);
+            if (result)
+                return result;
+        }
+        return null;
+    }
+    elementToString(elem, indent) {
+        const indentStr = "  ".repeat(indent);
+        let result = `${indentStr}<${elem.tag}`;
+        // Add attributes
+        for (const [key, value] of Object.entries(elem.attributes)) {
+            const escapedValue = value
+                .replace(/&/g, "&amp;")
+                .replace(/</g, "&lt;")
+                .replace(/>/g, "&gt;")
+                .replace(/"/g, "&quot;")
+                .replace(/'/g, "&apos;");
+            result += ` ${key}="${escapedValue}"`;
+        }
+        if (elem.children.length === 0) {
+            result += " />\n";
+        }
+        else {
+            result += ">\n";
+            for (const child of elem.children) {
+                result += this.elementToString(child, indent + 1);
+            }
+            result += `${indentStr}</${elem.tag}>\n`;
+        }
+        return result;
+    }
+    parseSimpleXml(xmlString) {
+        const elements = [];
+        const stack = [];
+        // Combined regex for all XML tokens (opening tags, closing tags, self-closing tags)
+        const tokenRegex = /<\/?([a-zA-Z_][\w:.-]*)((?:\s+[\w:.-]+="[^"]*")*)\s*(\/?)>/g;
+        const attrRegex = /([\w:.-]+)="([^"]*)"/g;
+        let match;
+        while ((match = tokenRegex.exec(xmlString)) !== null) {
+            const fullMatch = match[0];
+            const tagName = match[1];
+            const attrsString = match[2];
+            const selfClosing = match[3] === "/";
+            const isClosingTag = fullMatch.startsWith("</");
+            if (isClosingTag) {
+                // Handle closing tag - pop from stack
+                if (stack.length > 0 && stack[stack.length - 1].tag === tagName) {
+                    stack.pop();
+                }
+            }
+            else {
+                // Handle opening or self-closing tag
+                // Parse attributes
+                const attributes = {};
+                let attrMatch;
+                attrRegex.lastIndex = 0;
+                while ((attrMatch = attrRegex.exec(attrsString)) !== null) {
+                    attributes[attrMatch[1]] = attrMatch[2];
+                }
+                const elem = {
+                    tag: tagName,
+                    attributes,
+                    children: [],
+                };
+                if (stack.length > 0) {
+                    stack[stack.length - 1].children.push(elem);
+                }
+                else {
+                    elements.push(elem);
+                }
+                if (!selfClosing) {
+                    stack.push(elem);
+                }
+            }
+        }
+        return elements;
+    }
+}

package/dist/accessibility/XCUITestAccessibilityTree.d.ts

@@ -0,0 +1,15 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export declare class XCUITestAccessibilityTree extends BaseAccessibilityTree {
+    private xmlString;
+    private nextRawId;
+    private raw;
+    constructor(xmlString: string);
+    toStr(): string;
+    private addRawIds;
+    elementById(rawId: number): AccessibilityElement;
+    scopeToArea(rawId: number): XCUITestAccessibilityTree;
+    private findElementByRawId;
+    private elementToString;
+    private parseSimpleXml;
+}

package/dist/accessibility/XCUITestAccessibilityTree.js

@@ -0,0 +1,144 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export class XCUITestAccessibilityTree extends BaseAccessibilityTree {
+    constructor(xmlString) {
+        super();
+        this.nextRawId = 0;
+        this.raw = null;
+        this.xmlString = xmlString;
+    }
+    toStr() {
+        if (this.raw !== null) {
+            return this.raw;
+        }
+        // Parse the XML
+        const elements = this.parseSimpleXml(this.xmlString);
+        // Add raw_id attributes recursively
+        for (const elem of elements) {
+            this.addRawIds(elem);
+        }
+        // Serialize back to string
+        let result = "";
+        for (const elem of elements) {
+            result += this.elementToString(elem, 0);
+        }
+        this.raw = result;
+        return this.raw;
+    }
+    addRawIds(elem) {
+        this.nextRawId++;
+        elem.attributes.raw_id = String(this.nextRawId);
+        for (const child of elem.children) {
+            this.addRawIds(child);
+        }
+    }
+    elementById(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            throw new Error(`No element with raw_id=${rawId} found`);
+        }
+        // Extract properties for XCUITest
+        return new AccessibilityElement(rawId, element.attributes.name, element.attributes.label, element.tag, element.attributes.value);
+    }
+    scopeToArea(rawId) {
+        const rawXml = this.toStr();
+        const element = this.findElementByRawId(rawXml, rawId);
+        if (!element) {
+            return this;
+        }
+        const scopedXml = this.elementToString(element, 0);
+        return new XCUITestAccessibilityTree(scopedXml);
+    }
+    findElementByRawId(xmlString, targetRawId) {
+        const elements = this.parseSimpleXml(xmlString);
+        const search = (elem) => {
+            if (elem.attributes.raw_id === String(targetRawId)) {
+                return elem;
+            }
+            for (const child of elem.children) {
+                const result = search(child);
+                if (result)
+                    return result;
+            }
+            return null;
+        };
+        for (const elem of elements) {
+            const result = search(elem);
+            if (result)
+                return result;
+        }
+        return null;
+    }
+    elementToString(elem, indent) {
+        const indentStr = "  ".repeat(indent);
+        let result = `${indentStr}<${elem.tag}`;
+        // Add attributes
+        for (const [key, value] of Object.entries(elem.attributes)) {
+            const escapedValue = value
+                .replace(/&/g, "&amp;")
+                .replace(/</g, "&lt;")
+                .replace(/>/g, "&gt;")
+                .replace(/"/g, "&quot;")
+                .replace(/'/g, "&apos;");
+            result += ` ${key}="${escapedValue}"`;
+        }
+        if (elem.children.length === 0) {
+            result += " />\n";
+        }
+        else {
+            result += ">\n";
+            for (const child of elem.children) {
+                result += this.elementToString(child, indent + 1);
+            }
+            result += `${indentStr}</${elem.tag}>\n`;
+        }
+        return result;
+    }
+    parseSimpleXml(xmlString) {
+        const elements = [];
+        const stack = [];
+        // Combined regex for all XML tokens (opening tags, closing tags, self-closing tags)
+        const tokenRegex = /<\/?([a-zA-Z_][\w:.-]*)((?:\s+[\w:.-]+="[^"]*")*)\s*(\/?)>/g;
+        const attrRegex = /([\w:.-]+)="([^"]*)"/g;
+        let match;
+        while ((match = tokenRegex.exec(xmlString)) !== null) {
+            const fullMatch = match[0];
+            const tagName = match[1];
+            const attrsString = match[2];
+            const selfClosing = match[3] === "/";
+            const isClosingTag = fullMatch.startsWith("</");
+            if (isClosingTag) {
+                // Handle closing tag - pop from stack
+                if (stack.length > 0 && stack[stack.length - 1].tag === tagName) {
+                    stack.pop();
+                }
+            }
+            else {
+                // Handle opening or self-closing tag
+                // Parse attributes
+                const attributes = {};
+                let attrMatch;
+                attrRegex.lastIndex = 0;
+                while ((attrMatch = attrRegex.exec(attrsString)) !== null) {
+                    attributes[attrMatch[1]] = attrMatch[2];
+                }
+                const elem = {
+                    tag: tagName,
+                    attributes,
+                    children: [],
+                };
+                if (stack.length > 0) {
+                    stack[stack.length - 1].children.push(elem);
+                }
+                else {
+                    elements.push(elem);
+                }
+                if (!selfClosing) {
+                    stack.push(elem);
+                }
+            }
+        }
+        return elements;
+    }
+}

package/dist/accessibility/index.d.ts

@@ -0,0 +1,5 @@
+export { AccessibilityElement } from "./AccessibilityElement.js";
+export { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export { ChromiumAccessibilityTree } from "./ChromiumAccessibilityTree.js";
+export { UIAutomator2AccessibilityTree } from "./UIAutomator2AccessibilityTree.js";
+export { XCUITestAccessibilityTree } from "./XCUITestAccessibilityTree.js";

package/dist/accessibility/index.js

@@ -0,0 +1,5 @@
+export { AccessibilityElement } from "./AccessibilityElement.js";
+export { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+export { ChromiumAccessibilityTree } from "./ChromiumAccessibilityTree.js";
+export { UIAutomator2AccessibilityTree } from "./UIAutomator2AccessibilityTree.js";
+export { XCUITestAccessibilityTree } from "./XCUITestAccessibilityTree.js";

package/dist/clients/HttpClient.d.ts

@@ -0,0 +1,27 @@
+import { BaseTool } from "../tools/BaseTool.js";
+import { Data } from "./typecasting.js";
+export declare class HttpClient {
+    private platform;
+    private tools;
+    private baseUrl;
+    private sessionId;
+    private sessionPromise;
+    private timeout;
+    constructor(baseUrl: string, platform: string, tools: Record<string, new (...args: any[]) => BaseTool>);
+    private fetchWithTimeout;
+    private ensureSession;
+    quit(): Promise<void>;
+    planActions(goal: string, accessibilityTree: string): Promise<string[]>;
+    addExample(goal: string, actions: string[]): Promise<void>;
+    clearExamples(): Promise<void>;
+    executeAction(goal: string, step: string, accessibilityTree: string): Promise<any[]>;
+    retrieve(statement: string, accessibilityTree: string, title: string, url: string, screenshot?: string): Promise<[string, Data]>;
+    findArea(description: string, accessibilityTree: string): Promise<{
+        id: number;
+        explanation: string;
+    }>;
+    findElement(description: string, accessibilityTree: string): Promise<any>;
+    saveCache(): Promise<void>;
+    discardCache(): Promise<void>;
+    getStats(): Promise<Record<string, Record<string, number>>>;
+}