alumnium 0.14.0 → 0.16.0

package/README.md

@@ -59,14 +59,6 @@ Check out [documentation][1] and more [examples][2]!
 
 See the [contributing guidelines][4] for information on how to get involved in the project and develop locally.
 
-## Acknowledgments
-
-[![LambdaTest](https://www.lambdatest.com/resources/images/logos/logo.svg)][5]
-
-Alumnium is a member of [LambdaTest][5] Open Source Program, which supports the project community and development with
-the necessary tools. Thank you! 💚
-
-
 
 [1]: https://alumnium.ai/docs/
 [2]: https://github.com/alumnium-hq/alumnium/tree/main/packages/typescript/examples/

package/dist/Alumni.d.ts

@@ -7,11 +7,14 @@ import { Data } from "./clients/typecasting.js";
 import { BaseDriver } from "./drivers/BaseDriver.js";
 import { Element } from "./drivers/index.js";
 import { Model } from "./Model.js";
-import { BaseTool } from "./tools/BaseTool.js";
+import { ToolClass } from "./tools/BaseTool.js";
 export interface AlumniOptions {
     url?: string;
     model?: Model;
-    extraTools?: (new (...args: any[]) => BaseTool)[];
+    extraTools?: ToolClass[];
+}
+export interface VisionOptions {
+    vision?: boolean;
 }
 export declare class Alumni {
     driver: BaseDriver;
@@ -23,8 +26,8 @@ export declare class Alumni {
     constructor(driver: WebDriver | Page | Browser, options?: AlumniOptions);
     quit(): Promise<void>;
     do(goal: string): Promise<void>;
-    check(statement: string, vision?: boolean): Promise<string>;
-    get(data: string, vision?: boolean): Promise<Data>;
+    check(statement: string, options?: VisionOptions): Promise<string>;
+    get(data: string, options?: VisionOptions): Promise<Data>;
     find(description: string): Promise<Element>;
     area(description: string): Promise<Area>;
     learn(goal: string, actions: string[]): Promise<void>;

package/dist/Alumni.js

@@ -5,6 +5,7 @@ import { HttpClient } from "./clients/HttpClient.js";
 import { AppiumDriver } from "./drivers/AppiumDriver.js";
 import { PlaywrightDriver } from "./drivers/PlaywrightDriver.js";
 import { SeleniumDriver } from "./drivers/SeleniumDriver.js";
+import { AssertionError } from "./errors/AssertionError.js";
 import { Model } from "./Model.js";
 import { BaseTool } from "./tools/BaseTool.js";
 import { ClickTool } from "./tools/ClickTool.js";
@@ -50,7 +51,7 @@ export class Alumni {
             }
         }
         // Initialize HTTP client
-        this.client = new HttpClient(this.url, this.driver.platform, this.tools);
+        this.client = new HttpClient(this.url, this.model, this.driver.platform, this.tools);
         this.cache = new Cache(this.client);
         logger.info(`Using model: ${this.model.provider}/${this.model.name}`);
         logger.info(`Using HTTP client with server: ${this.url}`);
@@ -71,21 +72,25 @@ export class Alumni {
             const actorResponse = await this.client.executeAction(goal, step, accessibilityTree.toStr());
             // Execute tool calls
             for (const toolCall of actorResponse) {
-                await BaseTool.executeToolCall(toolCall, this.tools, this.client, this.driver);
+                await BaseTool.executeToolCall(toolCall, this.tools, this.driver);
             }
         }
     }
-    async check(statement, vision = false) {
-        const screenshot = vision ? await this.driver.screenshot() : undefined;
+    async check(statement, options = {}) {
+        const screenshot = options.vision
+            ? await this.driver.screenshot()
+            : undefined;
         const accessibilityTree = await this.driver.getAccessibilityTree();
         const [explanation, value] = await this.client.retrieve(`Is the following true or false - ${statement}`, accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
         if (!value) {
-            throw new Error(explanation);
+            throw new AssertionError(explanation);
         }
         return explanation;
     }
-    async get(data, vision = false) {
-        const screenshot = vision ? await this.driver.screenshot() : undefined;
+    async get(data, options = {}) {
+        const screenshot = options.vision
+            ? await this.driver.screenshot()
+            : undefined;
         const accessibilityTree = await this.driver.getAccessibilityTree();
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const [_explanation, value] = await this.client.retrieve(data, accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
@@ -93,9 +98,7 @@ export class Alumni {
     }
     async find(description) {
         const accessibilityTree = await this.driver.getAccessibilityTree();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
         const response = await this.client.findElement(description, accessibilityTree.toStr());
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
         return this.driver.findElement(response.id);
     }
     async area(description) {

package/dist/Area.d.ts

@@ -1,9 +1,10 @@
 import { BaseAccessibilityTree } from "./accessibility/BaseAccessibilityTree.js";
+import { VisionOptions } from "./Alumni.js";
 import { HttpClient } from "./clients/HttpClient.js";
 import { Data } from "./clients/typecasting.js";
 import { BaseDriver } from "./drivers/BaseDriver.js";
 import { Element } from "./drivers/index.js";
-import { BaseTool } from "./tools/BaseTool.js";
+import { ToolClass } from "./tools/BaseTool.js";
 export declare class Area {
     id: number;
     description: string;
@@ -11,9 +12,9 @@ export declare class Area {
     private driver;
     private tools;
     private client;
-    constructor(id: number, description: string, accessibilityTree: BaseAccessibilityTree, driver: BaseDriver, tools: Record<string, new (...args: any[]) => BaseTool>, client: HttpClient);
+    constructor(id: number, description: string, accessibilityTree: BaseAccessibilityTree, driver: BaseDriver, tools: Record<string, ToolClass>, client: HttpClient);
     do(goal: string): Promise<void>;
-    check(statement: string, vision?: boolean): Promise<string>;
-    get(data: string, vision?: boolean): Promise<Data>;
+    check(statement: string, options?: VisionOptions): Promise<string>;
+    get(data: string, options?: VisionOptions): Promise<Data>;
     find(description: string): Promise<Element>;
 }

package/dist/Area.js

@@ -1,8 +1,7 @@
+import { AssertionError } from "./errors/AssertionError.js";
 import { BaseTool } from "./tools/BaseTool.js";
 export class Area {
-    constructor(id, description, accessibilityTree, driver, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    tools, client) {
+    constructor(id, description, accessibilityTree, driver, tools, client) {
         this.id = id;
         this.description = description;
         this.accessibilityTree = accessibilityTree;
@@ -16,28 +15,30 @@ export class Area {
             const actorResponse = await this.client.executeAction(goal, step, this.accessibilityTree.toStr());
             // Execute tool calls
             for (const toolCall of actorResponse) {
-                await BaseTool.executeToolCall(toolCall, this.tools, this.client, this.driver);
+                await BaseTool.executeToolCall(toolCall, this.tools, this.driver);
             }
         }
     }
-    async check(statement, vision = false) {
-        const screenshot = vision ? await this.driver.screenshot() : undefined;
+    async check(statement, options = {}) {
+        const screenshot = options.vision
+            ? await this.driver.screenshot()
+            : undefined;
         const [explanation, value] = await this.client.retrieve(`Is the following true or false - ${statement}`, this.accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
         if (!value) {
-            throw new Error(explanation);
+            throw new AssertionError(explanation);
         }
         return explanation;
     }
-    async get(data, vision = false) {
-        const screenshot = vision ? await this.driver.screenshot() : undefined;
+    async get(data, options = {}) {
+        const screenshot = options.vision
+            ? await this.driver.screenshot()
+            : undefined;
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const [_explanation, value] = await this.client.retrieve(data, this.accessibilityTree.toStr(), await this.driver.title(), await this.driver.url(), screenshot);
         return value;
     }
     async find(description) {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
         const response = await this.client.findElement(description, this.accessibilityTree.toStr());
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
         return this.driver.findElement(response.id);
     }
 }

package/dist/Model.d.ts

@@ -4,10 +4,12 @@ export declare enum Provider {
     AWS_ANTHROPIC = "aws_anthropic",
     AWS_META = "aws_meta",
     DEEPSEEK = "deepseek",
+    GITHUB = "github",
     GOOGLE = "google",
     MISTRALAI = "mistralai",
     OLLAMA = "ollama",
-    OPENAI = "openai"
+    OPENAI = "openai",
+    XAI = "xai"
 }
 export declare class ModelName {
     static readonly DEFAULT: Record<Provider, string>;

package/dist/Model.js

@@ -5,23 +5,27 @@ export var Provider;
     Provider["AWS_ANTHROPIC"] = "aws_anthropic";
     Provider["AWS_META"] = "aws_meta";
     Provider["DEEPSEEK"] = "deepseek";
+    Provider["GITHUB"] = "github";
     Provider["GOOGLE"] = "google";
     Provider["MISTRALAI"] = "mistralai";
     Provider["OLLAMA"] = "ollama";
     Provider["OPENAI"] = "openai";
+    Provider["XAI"] = "xai";
 })(Provider || (Provider = {}));
 export class ModelName {
 }
 ModelName.DEFAULT = {
     [Provider.AZURE_OPENAI]: "gpt-4o-mini",
-    [Provider.ANTHROPIC]: "claude-3-haiku-20240307",
-    [Provider.AWS_ANTHROPIC]: "anthropic.claude-3-haiku-20240307-v1:0",
+    [Provider.ANTHROPIC]: "claude-haiku-4-5-20251001",
+    [Provider.AWS_ANTHROPIC]: "us.anthropic.claude-haiku-4-5-20251001-v1:0",
     [Provider.AWS_META]: "us.meta.llama4-maverick-17b-instruct-v1:0",
     [Provider.DEEPSEEK]: "deepseek-chat",
+    [Provider.GITHUB]: "gpt-4o-mini",
     [Provider.GOOGLE]: "gemini-2.0-flash-001",
     [Provider.MISTRALAI]: "mistral-medium-2505",
     [Provider.OLLAMA]: "mistral-small3.1",
     [Provider.OPENAI]: "gpt-4o-mini-2024-07-18",
+    [Provider.XAI]: "grok-4-fast-non-reasoning",
 };
 export class Model {
     constructor(provider, name) {
@@ -37,9 +41,12 @@ export class Model {
         this.name = name || ModelName.DEFAULT[this.provider];
     }
     static initialize() {
-        const alumniumModel = process.env.ALUMNIUM_MODEL || "openai";
-        const [provider, ...nameParts] = alumniumModel.toLowerCase().split("/");
-        const name = nameParts.length > 0 ? nameParts[0] : undefined;
+        const alumniumModel = process.env.ALUMNIUM_MODEL || "";
+        let [provider, name] = alumniumModel.toLowerCase().split("/");
+        if (!provider && process.env.GITHUB_ACTIONS) {
+            provider = "github";
+            name = ModelName.DEFAULT[Provider.GITHUB];
+        }
         Model.current = new Model(provider, name);
     }
 }

package/dist/accessibility/ChromiumAccessibilityTree.d.ts

@@ -1,13 +1,10 @@
 import { AccessibilityElement } from "./AccessibilityElement.js";
 import { BaseAccessibilityTree } from "./BaseAccessibilityTree.js";
-interface CDPResponse {
-    nodes: any[];
-}
 export declare class ChromiumAccessibilityTree extends BaseAccessibilityTree {
     private cdpResponse;
     private nextRawId;
     private raw;
-    constructor(cdpResponse: CDPResponse);
+    constructor(cdpResponse: unknown);
     private static fromXml;
     toStr(): string;
     private nodeToXml;
@@ -17,4 +14,3 @@ export declare class ChromiumAccessibilityTree extends BaseAccessibilityTree {
     private findElementByRawId;
     private parseSimpleXml;
 }
-export {};

package/dist/clients/ApiModels.d.ts

@@ -0,0 +1,73 @@
+/**
+ * TypeScript interfaces for API request and response models.
+ *
+ * IMPORTANT: These interfaces must be kept in sync with the Python API models
+ * defined in packages/python/src/alumnium/server/api_models.py
+ */
+export interface SessionRequest {
+    platform: "chromium" | "uiautomator2" | "xcuitest";
+    provider: string;
+    name?: string;
+    tools: {
+        [key: string]: any;
+    }[];
+}
+export interface SessionResponse {
+    session_id: string;
+}
+export interface PlanRequest {
+    goal: string;
+    accessibility_tree: string;
+    url?: string;
+    title?: string;
+}
+export interface PlanResponse {
+    steps: string[];
+}
+export interface StepRequest {
+    goal: string;
+    step: string;
+    accessibility_tree: string;
+}
+export interface StepResponse {
+    actions: {
+        [key: string]: any;
+    }[];
+}
+export interface StatementRequest {
+    statement: string;
+    accessibility_tree: string;
+    url?: string;
+    title?: string;
+    screenshot?: string | null;
+}
+export interface StatementResponse {
+    result: string | string[];
+    explanation: string;
+}
+export interface AreaRequest {
+    description: string;
+    accessibility_tree: string;
+}
+export interface AreaResponse {
+    id: number;
+    explanation: string;
+}
+export interface FindRequest {
+    description: string;
+    accessibility_tree: string;
+}
+export interface FindResponse {
+    elements: {
+        [key: string]: number | string;
+    }[];
+}
+export interface AddExampleRequest {
+    goal: string;
+    actions: string[];
+}
+export interface StatsResponse {
+    [key: string]: {
+        [key: string]: number;
+    };
+}

package/dist/clients/ApiModels.js

@@ -0,0 +1,7 @@
+/**
+ * TypeScript interfaces for API request and response models.
+ *
+ * IMPORTANT: These interfaces must be kept in sync with the Python API models
+ * defined in packages/python/src/alumnium/server/api_models.py
+ */
+export {};

package/dist/clients/HttpClient.d.ts

@@ -1,27 +1,30 @@
-import { BaseTool } from "../tools/BaseTool.js";
+import { Model } from "../Model.js";
+import { ToolClass } from "../tools/BaseTool.js";
+import { FindResponse, StatsResponse, StepResponse } from "./ApiModels.js";
 import { Data } from "./typecasting.js";
 export declare class HttpClient {
+    private model;
     private platform;
     private tools;
     private baseUrl;
     private sessionId;
     private sessionPromise;
     private timeout;
-    constructor(baseUrl: string, platform: string, tools: Record<string, new (...args: any[]) => BaseTool>);
+    constructor(baseUrl: string, model: Model, platform: string, tools: Record<string, ToolClass>);
     private fetchWithTimeout;
     private ensureSession;
     quit(): Promise<void>;
     planActions(goal: string, accessibilityTree: string): Promise<string[]>;
     addExample(goal: string, actions: string[]): Promise<void>;
     clearExamples(): Promise<void>;
-    executeAction(goal: string, step: string, accessibilityTree: string): Promise<any[]>;
+    executeAction(goal: string, step: string, accessibilityTree: string): Promise<StepResponse["actions"]>;
     retrieve(statement: string, accessibilityTree: string, title: string, url: string, screenshot?: string): Promise<[string, Data]>;
     findArea(description: string, accessibilityTree: string): Promise<{
         id: number;
         explanation: string;
     }>;
-    findElement(description: string, accessibilityTree: string): Promise<any>;
+    findElement(description: string, accessibilityTree: string): Promise<FindResponse["elements"][0]>;
     saveCache(): Promise<void>;
     discardCache(): Promise<void>;
-    getStats(): Promise<Record<string, Record<string, number>>>;
+    getStats(): Promise<StatsResponse>;
 }

package/dist/clients/HttpClient.js

@@ -1,14 +1,10 @@
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-/* eslint-disable @typescript-eslint/no-unsafe-return */
-import { Model } from "../Model.js";
 import { convertToolsToSchemas } from "../tools/toolToSchemaConverter.js";
 import { getLogger } from "../utils/logger.js";
 import { looselyTypecast } from "./typecasting.js";
 const logger = getLogger(["HttpClient"]);
 export class HttpClient {
-    constructor(baseUrl, platform, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    tools) {
+    constructor(baseUrl, model, platform, tools) {
+        this.model = model;
         this.platform = platform;
         this.tools = tools;
         this.sessionId = null;
@@ -17,21 +13,14 @@ export class HttpClient {
         this.baseUrl = baseUrl.replace(/\/$/, "");
     }
     async fetchWithTimeout(url, options = {}) {
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), this.timeout);
-        try {
-            const response = await fetch(url, {
-                ...options,
-                signal: controller.signal,
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP error! status: ${response.status}`);
-            }
-            return response;
-        }
-        finally {
-            clearTimeout(timeoutId);
+        const response = await fetch(url, {
+            ...options,
+            signal: AbortSignal.timeout(this.timeout),
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
         }
+        return response;
     }
     async ensureSession() {
         if (this.sessionId) {
@@ -40,21 +29,20 @@ export class HttpClient {
         if (!this.sessionPromise) {
             this.sessionPromise = (async () => {
                 const toolSchemas = convertToolsToSchemas(this.tools);
+                const requestBody = {
+                    provider: this.model.provider,
+                    name: this.model.name,
+                    platform: this.platform,
+                    tools: toolSchemas,
+                };
                 const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions`, {
                     method: "POST",
                     headers: {
                         "Content-Type": "application/json",
                     },
-                    body: JSON.stringify({
-                        provider: Model.current.provider,
-                        name: Model.current.name,
-                        platform: this.platform,
-                        tools: toolSchemas,
-                    }),
+                    body: JSON.stringify(requestBody),
                 });
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const data = await response.json();
-                // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+                const data = (await response.json());
                 this.sessionId = data.session_id;
                 logger.debug(`Session initialized with ID: ${this.sessionId}`);
             })();
@@ -71,32 +59,32 @@ export class HttpClient {
     }
     async planActions(goal, accessibilityTree) {
         await this.ensureSession();
+        const requestBody = {
+            goal,
+            accessibility_tree: accessibilityTree,
+        };
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/plans`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                goal,
-                accessibility_tree: accessibilityTree,
-            }),
+            body: JSON.stringify(requestBody),
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const responseData = await response.json();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        const responseData = (await response.json());
         return responseData.steps;
     }
     async addExample(goal, actions) {
         await this.ensureSession();
+        const requestBody = {
+            goal,
+            actions,
+        };
         await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/examples`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                goal,
-                actions,
-            }),
+            body: JSON.stringify(requestBody),
         });
     }
     async clearExamples() {
@@ -105,80 +93,72 @@ export class HttpClient {
             method: "DELETE",
         });
     }
-    async executeAction(goal, step, accessibilityTree
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    ) {
+    async executeAction(goal, step, accessibilityTree) {
         await this.ensureSession();
+        const requestBody = {
+            goal,
+            step,
+            accessibility_tree: accessibilityTree,
+        };
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/steps`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                goal,
-                step,
-                accessibility_tree: accessibilityTree,
-            }),
+            body: JSON.stringify(requestBody),
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const responseData = await response.json();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        const responseData = (await response.json());
         return responseData.actions;
     }
     async retrieve(statement, accessibilityTree, title, url, screenshot) {
         await this.ensureSession();
+        const requestBody = {
+            statement,
+            accessibility_tree: accessibilityTree,
+            title,
+            url,
+            screenshot: screenshot || null,
+        };
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/statements`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                statement,
-                accessibility_tree: accessibilityTree,
-                title,
-                url,
-                screenshot: screenshot || null,
-            }),
+            body: JSON.stringify(requestBody),
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const responseData = await response.json();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument
+        const responseData = (await response.json());
         return [responseData.explanation, looselyTypecast(responseData.result)];
     }
     async findArea(description, accessibilityTree) {
         await this.ensureSession();
+        const requestBody = {
+            description,
+            accessibility_tree: accessibilityTree,
+        };
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/areas`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                description,
-                accessibility_tree: accessibilityTree,
-            }),
+            body: JSON.stringify(requestBody),
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const responseData = await response.json();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        const responseData = (await response.json());
         return { id: responseData.id, explanation: responseData.explanation };
     }
-    async findElement(description, accessibilityTree
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    ) {
+    async findElement(description, accessibilityTree) {
         await this.ensureSession();
+        const requestBody = {
+            description,
+            accessibility_tree: accessibilityTree,
+        };
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/elements`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify({
-                description,
-                accessibility_tree: accessibilityTree,
-            }),
+            body: JSON.stringify(requestBody),
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const responseData = await response.json();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        const responseData = (await response.json());
         return responseData.elements[0];
     }
     async saveCache() {
@@ -198,7 +178,6 @@ export class HttpClient {
         const response = await this.fetchWithTimeout(`${this.baseUrl}/v1/sessions/${this.sessionId}/stats`, {
             method: "GET",
         });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return response.json();
+        return (await response.json());
     }
 }

package/dist/drivers/AppiumDriver.d.ts

@@ -4,10 +4,11 @@ import { BaseDriver } from "./BaseDriver.js";
 import { Key } from "./keys.js";
 export declare class AppiumDriver extends BaseDriver {
     private driver;
-    platform: string;
+    platform: "xcuitest" | "uiautomator2";
     supportedTools: Set<string>;
     autoswitchContexts: boolean;
     delay: number;
+    doubleFetchPageSource: boolean;
     hideKeyboardAfterTyping: boolean;
     constructor(driver: Browser);
     getAccessibilityTree(): Promise<BaseAccessibilityTree>;