alumnium 0.14.0

package/.mocharc.json

@@ -0,0 +1,10 @@
+{
+  "spec": "examples/mocha/**/*.test.ts",
+  "node-option": [
+    "loader=ts-node/esm",
+    "no-warnings"
+  ],
+  "loader": "ts-node/esm",
+  "require": "examples/mocha/helpers.ts",
+  "timeout": 300000
+}

package/.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "dbaeumer.vscode-eslint",
+        "ms-vscode.vscode-typescript-next",
+    ]
+}

package/.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+    "[javascript]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit",
+            "source.fixAll": "explicit",
+        },
+    },
+    "[typescript]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit",
+            "source.fixAll": "explicit",
+        },
+    },
+}

package/README.md

@@ -0,0 +1,75 @@
+<h1>
+    <p align="center">
+        <img src="https://raw.githubusercontent.com/alumnium-hq/alumnium.github.io/efb2afaf0ced7ec07c241445e7b381914281edaf/src/assets/logo.svg" height="128" alt="Logo" />
+        <br />
+        Alumnium
+    </p>
+</h1>
+<p align="center">
+    Pave the way towards AI-powered test automation.
+    <br />
+    <a href="#installation">Installation</a>
+    ·
+    <a href="#quick-start">Quick Start</a>
+    ·
+    <a href="https://alumnium.ai/docs/">Documentation</a>
+</p>
+
+Alumnium is an experimental project that builds upon the existing test automation ecosystem, offering a higher-level abstraction for testing. It simplifies interactions with applications and provide more robust mechanisms for verifying assertions. It works with Appium, Playwright, or Selenium.
+
+https://github.com/user-attachments/assets/b1a548c0-f1e1-4ffe-bec9-d814770ba2ae
+
+Currently in the very early stages of development and not recommended for production use.
+
+## Installation
+
+```bash
+npm install alumnium
+```
+
+## Quick Start
+
+1. Run Alumnium server:
+
+```sh
+docker run --rm -p 8013:8013 -e OPENAI_API_KEY=... alumnium/alumnium
+```
+
+2. Run your tests:
+
+```javascript
+import { Alumni } from "alumnium";
+import { Builder } from "selenium-webdriver";
+
+const driver = await new Builder().forBrowser("chrome").build();
+const al = new Alumni(driver);
+
+await driver.get("https://search.brave.com");
+await al.do("type 'selenium' into the search field, then press 'Enter'");
+await al.check("page title contains selenium");
+await al.check("search results contain selenium.dev");
+console.log("Atomic number:", await al.get("atomic number")); // 34
+
+await al.quit();
+```
+
+Check out [documentation][1] and more [examples][2]!
+
+## Contributing
+
+See the [contributing guidelines][4] for information on how to get involved in the project and develop locally.
+
+## Acknowledgments
+
+[![LambdaTest](https://www.lambdatest.com/resources/images/logos/logo.svg)][5]
+
+Alumnium is a member of [LambdaTest][5] Open Source Program, which supports the project community and development with
+the necessary tools. Thank you! 💚
+
+
+
+[1]: https://alumnium.ai/docs/
+[2]: https://github.com/alumnium-hq/alumnium/tree/main/packages/typescript/examples/
+[3]: https://alumnium.ai/docs/getting-started/configuration/
+[4]: https://github.com/alumnium-hq/alumnium/tree/main/CONTRIBUTING.md
+[5]: https://www.lambdatest.com/

package/dist/Alumni.d.ts

@@ -0,0 +1,33 @@
+import { Page } from "playwright";
+import { WebDriver } from "selenium-webdriver";
+import type { Browser } from "webdriverio";
+import { Area } from "./Area.js";
+import { Cache } from "./Cache.js";
+import { Data } from "./clients/typecasting.js";
+import { BaseDriver } from "./drivers/BaseDriver.js";
+import { Element } from "./drivers/index.js";
+import { Model } from "./Model.js";
+import { BaseTool } from "./tools/BaseTool.js";
+export interface AlumniOptions {
+    url?: string;
+    model?: Model;
+    extraTools?: (new (...args: any[]) => BaseTool)[];
+}
+export declare class Alumni {
+    driver: BaseDriver;
+    private client;
+    private tools;
+    cache: Cache;
+    private url;
+    private model;
+    constructor(driver: WebDriver | Page | Browser, options?: AlumniOptions);
+    quit(): Promise<void>;
+    do(goal: string): Promise<void>;
+    check(statement: string, vision?: boolean): Promise<string>;
+    get(data: string, vision?: boolean): Promise<Data>;
+    find(description: string): Promise<Element>;
+    area(description: string): Promise<Area>;
+    learn(goal: string, actions: string[]): Promise<void>;
+    clearLearnExamples(): Promise<void>;
+    getStats(): Promise<Record<string, Record<string, number>>>;
+}

package/dist/Alumni.js

@@ -0,0 +1,116 @@
+import { WebDriver } from "selenium-webdriver";
+import { Area } from "./Area.js";
+import { Cache } from "./Cache.js";
+import { HttpClient } from "./clients/HttpClient.js";
+import { AppiumDriver } from "./drivers/AppiumDriver.js";
+import { PlaywrightDriver } from "./drivers/PlaywrightDriver.js";
+import { SeleniumDriver } from "./drivers/SeleniumDriver.js";
+import { Model } from "./Model.js";
+import { BaseTool } from "./tools/BaseTool.js";
+import { ClickTool } from "./tools/ClickTool.js";
+import { DragAndDropTool } from "./tools/DragAndDropTool.js";
+import { HoverTool } from "./tools/HoverTool.js";
+import { PressKeyTool } from "./tools/PressKeyTool.js";
+import { SelectTool } from "./tools/SelectTool.js";
+import { TypeTool } from "./tools/TypeTool.js";
+import { getLogger } from "./utils/logger.js";
+const logger = getLogger(["Alumni"]);
+export class Alumni {
+    constructor(driver, options = {}) {
+        this.url = options.url || "http://localhost:8013";
+        this.model = options.model || Model.current;
+        // Wrap driver or use directly if already wrapped
+        if (driver instanceof WebDriver) {
+            this.driver = new SeleniumDriver(driver);
+        }
+        else if (driver.context) {
+            this.driver = new PlaywrightDriver(driver);
+        }
+        else if (driver.capabilities &&
+            driver.getPageSource) {
+            // WebdriverIO Browser (Appium)
+            this.driver = new AppiumDriver(driver);
+        }
+        else {
+            throw new Error(`Unsupported driver type '${typeof driver}'`);
+        }
+        // Initialize tools
+        this.tools = {
+            ClickTool,
+            TypeTool,
+            HoverTool,
+            SelectTool,
+            PressKeyTool,
+            DragAndDropTool,
+        };
+        // Add extra tools if provided
+        if (options.extraTools) {
+            for (const tool of options.extraTools) {
+                this.tools[tool.name] = tool;
+            }
+        }
+        // Initialize HTTP client
+        this.client = new HttpClient(this.url, this.driver.platform, this.tools);
+        this.cache = new Cache(this.client);
+        logger.info(`Using model: ${this.model.provider}/${this.model.name}`);
+        logger.info(`Using HTTP client with server: ${this.url}`);
+    }
+    async quit() {
+        await this.client.quit();
+        await this.driver.quit();
+    }
+    async do(goal) {
+        const initialAccessibilityTree = await this.driver.getAccessibilityTree();
+        const steps = await this.client.planActions(goal, initialAccessibilityTree.toStr());
+        for (let idx = 0; idx < steps.length; idx++) {
+            const step = steps[idx];
+            // Use initial tree for first step, fresh tree for subsequent steps
+            const accessibilityTree = idx === 0
+                ? initialAccessibilityTree
+                : await this.driver.getAccessibilityTree();
+            const actorResponse = await this.client.executeAction(goal, step, accessibilityTree.toStr());
+            // Execute tool calls
+            for (const toolCall of actorResponse) {
+                await BaseTool.executeToolCall(toolCall, this.tools, this.client, this.driver);
+            }
+        }
+    }
+    async check(statement, vision = false) {
+        const screenshot = vision ? await this.driver.screenshot() : undefined;
+        const accessibilityTree = await this.driver.getAccessibilityTree();
+        const [explanation, value] = await this.client.retrieve(`Is the following true or false - ${statement}`, accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
+        if (!value) {
+            throw new Error(explanation);
+        }
+        return explanation;
+    }
+    async get(data, vision = false) {
+        const screenshot = vision ? await this.driver.screenshot() : undefined;
+        const accessibilityTree = await this.driver.getAccessibilityTree();
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [_explanation, value] = await this.client.retrieve(data, accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
+        return value;
+    }
+    async find(description) {
+        const accessibilityTree = await this.driver.getAccessibilityTree();
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        const response = await this.client.findElement(description, accessibilityTree.toStr());
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        return this.driver.findElement(response.id);
+    }
+    async area(description) {
+        const accessibilityTree = await this.driver.getAccessibilityTree();
+        const response = await this.client.findArea(description, accessibilityTree.toStr());
+        const scopedTree = accessibilityTree.scopeToArea(response.id);
+        return new Area(response.id, response.explanation, scopedTree, this.driver, this.tools, this.client);
+    }
+    async learn(goal, actions) {
+        await this.client.addExample(goal, actions);
+    }
+    async clearLearnExamples() {
+        await this.client.clearExamples();
+    }
+    async getStats() {
+        return await this.client.getStats();
+    }
+}

package/dist/Area.d.ts

@@ -0,0 +1,19 @@
+import { BaseAccessibilityTree } from "./accessibility/BaseAccessibilityTree.js";
+import { HttpClient } from "./clients/HttpClient.js";
+import { Data } from "./clients/typecasting.js";
+import { BaseDriver } from "./drivers/BaseDriver.js";
+import { Element } from "./drivers/index.js";
+import { BaseTool } from "./tools/BaseTool.js";
+export declare class Area {
+    id: number;
+    description: string;
+    private accessibilityTree;
+    private driver;
+    private tools;
+    private client;
+    constructor(id: number, description: string, accessibilityTree: BaseAccessibilityTree, driver: BaseDriver, tools: Record<string, new (...args: any[]) => BaseTool>, client: HttpClient);
+    do(goal: string): Promise<void>;
+    check(statement: string, vision?: boolean): Promise<string>;
+    get(data: string, vision?: boolean): Promise<Data>;
+    find(description: string): Promise<Element>;
+}

package/dist/Area.js

@@ -0,0 +1,43 @@
+import { BaseTool } from "./tools/BaseTool.js";
+export class Area {
+    constructor(id, description, accessibilityTree, driver, 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tools, client) {
+        this.id = id;
+        this.description = description;
+        this.accessibilityTree = accessibilityTree;
+        this.driver = driver;
+        this.tools = tools;
+        this.client = client;
+    }
+    async do(goal) {
+        const steps = await this.client.planActions(goal, this.accessibilityTree.toStr());
+        for (const step of steps) {
+            const actorResponse = await this.client.executeAction(goal, step, this.accessibilityTree.toStr());
+            // Execute tool calls
+            for (const toolCall of actorResponse) {
+                await BaseTool.executeToolCall(toolCall, this.tools, this.client, this.driver);
+            }
+        }
+    }
+    async check(statement, vision = false) {
+        const screenshot = vision ? await this.driver.screenshot() : undefined;
+        const [explanation, value] = await this.client.retrieve(`Is the following true or false - ${statement}`, this.accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
+        if (!value) {
+            throw new Error(explanation);
+        }
+        return explanation;
+    }
+    async get(data, vision = false) {
+        const screenshot = vision ? await this.driver.screenshot() : undefined;
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [_explanation, value] = await this.client.retrieve(data, this.accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
+        return value;
+    }
+    async find(description) {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        const response = await this.client.findElement(description, this.accessibilityTree.toStr());
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        return this.driver.findElement(response.id);
+    }
+}

package/dist/Cache.d.ts

@@ -0,0 +1,7 @@
+import { HttpClient } from "./clients/HttpClient.js";
+export declare class Cache {
+    private client;
+    constructor(client: HttpClient);
+    save(): Promise<void>;
+    discard(): Promise<void>;
+}

package/dist/Cache.js

@@ -0,0 +1,11 @@
+export class Cache {
+    constructor(client) {
+        this.client = client;
+    }
+    async save() {
+        await this.client.saveCache();
+    }
+    async discard() {
+        await this.client.discardCache();
+    }
+}

package/dist/Model.d.ts

@@ -0,0 +1,21 @@
+export declare enum Provider {
+    AZURE_OPENAI = "azure_openai",
+    ANTHROPIC = "anthropic",
+    AWS_ANTHROPIC = "aws_anthropic",
+    AWS_META = "aws_meta",
+    DEEPSEEK = "deepseek",
+    GOOGLE = "google",
+    MISTRALAI = "mistralai",
+    OLLAMA = "ollama",
+    OPENAI = "openai"
+}
+export declare class ModelName {
+    static readonly DEFAULT: Record<Provider, string>;
+}
+export declare class Model {
+    provider: Provider;
+    name: string;
+    static current: Model;
+    constructor(provider?: Provider | string, name?: string);
+    static initialize(): void;
+}

package/dist/Model.js

@@ -0,0 +1,47 @@
+export var Provider;
+(function (Provider) {
+    Provider["AZURE_OPENAI"] = "azure_openai";
+    Provider["ANTHROPIC"] = "anthropic";
+    Provider["AWS_ANTHROPIC"] = "aws_anthropic";
+    Provider["AWS_META"] = "aws_meta";
+    Provider["DEEPSEEK"] = "deepseek";
+    Provider["GOOGLE"] = "google";
+    Provider["MISTRALAI"] = "mistralai";
+    Provider["OLLAMA"] = "ollama";
+    Provider["OPENAI"] = "openai";
+})(Provider || (Provider = {}));
+export class ModelName {
+}
+ModelName.DEFAULT = {
+    [Provider.AZURE_OPENAI]: "gpt-4o-mini",
+    [Provider.ANTHROPIC]: "claude-3-haiku-20240307",
+    [Provider.AWS_ANTHROPIC]: "anthropic.claude-3-haiku-20240307-v1:0",
+    [Provider.AWS_META]: "us.meta.llama4-maverick-17b-instruct-v1:0",
+    [Provider.DEEPSEEK]: "deepseek-chat",
+    [Provider.GOOGLE]: "gemini-2.0-flash-001",
+    [Provider.MISTRALAI]: "mistral-medium-2505",
+    [Provider.OLLAMA]: "mistral-small3.1",
+    [Provider.OPENAI]: "gpt-4o-mini-2024-07-18",
+};
+export class Model {
+    constructor(provider, name) {
+        // Convert string to Provider enum if needed
+        if (typeof provider === "string") {
+            this.provider =
+                Provider[provider.toUpperCase()] ||
+                    Provider.OPENAI;
+        }
+        else {
+            this.provider = provider || Provider.OPENAI;
+        }
+        this.name = name || ModelName.DEFAULT[this.provider];
+    }
+    static initialize() {
+        const alumniumModel = process.env.ALUMNIUM_MODEL || "openai";
+        const [provider, ...nameParts] = alumniumModel.toLowerCase().split("/");
+        const name = nameParts.length > 0 ? nameParts[0] : undefined;
+        Model.current = new Model(provider, name);
+    }
+}
+// Initialize on module load
+Model.initialize();

package/dist/accessibility/AccessibilityElement.d.ts

@@ -0,0 +1,14 @@
+export declare class AccessibilityElement {
+    id?: number;
+    backendNodeId?: number;
+    name?: string;
+    label?: string;
+    type?: string;
+    value?: string;
+    androidResourceId?: string;
+    androidClass?: string;
+    androidText?: string;
+    androidContentDesc?: string;
+    androidBounds?: string;
+    constructor(id?: number, name?: string, label?: string, type?: string, value?: string, backendNodeId?: number, androidResourceId?: string, androidClass?: string, androidText?: string, androidContentDesc?: string, androidBounds?: string);
+}

package/dist/accessibility/AccessibilityElement.js

@@ -0,0 +1,15 @@
+export class AccessibilityElement {
+    constructor(id, name, label, type, value, backendNodeId, androidResourceId, androidClass, androidText, androidContentDesc, androidBounds) {
+        this.id = id;
+        this.backendNodeId = backendNodeId;
+        this.name = name;
+        this.label = label;
+        this.type = type;
+        this.value = value;
+        this.androidResourceId = androidResourceId;
+        this.androidClass = androidClass;
+        this.androidText = androidText;
+        this.androidContentDesc = androidContentDesc;
+        this.androidBounds = androidBounds;
+    }
+}

package/dist/accessibility/BaseAccessibilityTree.d.ts

@@ -0,0 +1,6 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+export declare abstract class BaseAccessibilityTree {
+    abstract toStr(): string;
+    abstract elementById(id: number): AccessibilityElement;
+    abstract scopeToArea(rawId: number): BaseAccessibilityTree;
+}

package/dist/accessibility/BaseAccessibilityTree.js

@@ -0,0 +1,2 @@
+export class BaseAccessibilityTree {
+}

package/dist/accessibility/ChromiumAccessibilityTree.d.ts

@@ -0,0 +1,20 @@
+import { AccessibilityElement } from "./AccessibilityElement.js";
+import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
+interface CDPResponse {
+    nodes: any[];
+}
+export declare class ChromiumAccessibilityTree extends BaseAccessibilityTree {
+    private cdpResponse;
+    private nextRawId;
+    private raw;
+    constructor(cdpResponse: CDPResponse);
+    private static fromXml;
+    toStr(): string;
+    private nodeToXml;
+    private elementToString;
+    elementById(rawId: number): AccessibilityElement;
+    scopeToArea(rawId: number): ChromiumAccessibilityTree;
+    private findElementByRawId;
+    private parseSimpleXml;
+}
+export {};