@samsara-dev/appwright 0.2.0

package/dist/utils.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.boxedStep = boxedStep;
+exports.validateBuildPath = validateBuildPath;
+exports.getLatestBuildToolsVersions = getLatestBuildToolsVersions;
+exports.longestDeterministicGroup = longestDeterministicGroup;
+exports.basePath = basePath;
+const test_1 = __importDefault(require("@playwright/test"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function boxedStep(target, context) {
+    return function replacementMethod(...args) {
+        const path = this.selector ? `("${this.selector}")` : "";
+        const argsString = args.length
+            ? "(" +
+                Array.from(args)
+                    .map((a) => JSON.stringify(a))
+                    .join(" , ") +
+                ")"
+            : "";
+        const name = `${context.name}${path}${argsString}`;
+        return test_1.default.step(name, async () => {
+            return await target.call(this, ...args);
+        }, { box: true });
+    };
+}
+function validateBuildPath(buildPath, expectedExtension) {
+    if (!buildPath) {
+        throw new Error(`Build path not found. Please set the build path in appwright.config.ts`);
+    }
+    if (!buildPath.endsWith(expectedExtension)) {
+        throw new Error(`File path is not supported for the given combination of platform and provider. Please provide build with ${expectedExtension} file extension in the appwright.config.ts`);
+    }
+    if (!fs_1.default.existsSync(buildPath)) {
+        throw new Error(`File not found at given path: ${buildPath}
+Please provide the correct path of the build.`);
+    }
+}
+function getLatestBuildToolsVersions(versions) {
+    return versions.sort((a, b) => (a > b ? -1 : 1))[0];
+}
+function longestDeterministicGroup(pattern) {
+    const patternToString = pattern.toString();
+    const matches = [...patternToString.matchAll(/\(([^)]+)\)/g)].map((match) => match[1]);
+    if (!matches || !matches.length) {
+        return undefined;
+    }
+    const noSpecialChars = matches.filter((match) => {
+        if (!match) {
+            return false;
+        }
+        const regexSpecialCharsPattern = /[.*+?^${}()|[\]\\]/;
+        return !regexSpecialCharsPattern.test(match);
+    });
+    const longestString = noSpecialChars.reduce((max, str) => (str.length > max.length ? str : max), "");
+    if (longestString == "") {
+        return undefined;
+    }
+    return longestString;
+}
+function basePath() {
+    return path_1.default.join(process.cwd(), "playwright-report", "data", "videos-store");
+}

package/dist/vision/index.d.ts

@@ -0,0 +1,64 @@
+import { Client as WebDriverClient } from "webdriver";
+import { Device } from "../device";
+import { z } from "zod";
+import { LLMModel } from "@empiricalrun/llm";
+import { ExtractType } from "../types";
+export interface AppwrightVision {
+    /**
+     * Extracts text from the screenshot based on the specified prompt.
+     * Ensure the `OPENAI_API_KEY` environment variable is set to authenticate the API request.
+     *
+     * **Usage:**
+     * ```js
+     * await device.beta.query("Extract contact details present in the footer from the screenshot");
+     * ```
+     *
+     * @param prompt that defines the specific area or context from which text should be extracted.
+     * @returns
+     */
+    query<T extends z.ZodType>(prompt: string, options?: {
+        responseFormat?: T;
+        model?: LLMModel;
+        screenshot?: string;
+        telemetry?: {
+            tags?: string[];
+        };
+    }): Promise<ExtractType<T>>;
+    /**
+     * Performs a tap action on the screen based on the provided prompt.
+     * Ensure the `EMPIRICAL_API_KEY` environment variable is set to authenticate the API request.
+     *
+     * **Usage:**
+     * ```js
+     * await device.beta.tap("Tap on the search button");
+     * ```
+     *
+     * @param prompt that defines where on the screen the tap action should occur
+     */
+    tap(prompt: string, options?: {
+        useCache?: boolean;
+        telemetry?: {
+            tags?: string[];
+        };
+    }): Promise<{
+        x: number;
+        y: number;
+    }>;
+}
+export declare class VisionProvider {
+    private device;
+    private webDriverClient;
+    constructor(device: Device, webDriverClient: WebDriverClient);
+    query<T extends z.ZodType>(prompt: string, options?: {
+        responseFormat?: T;
+        model?: LLMModel;
+        screenshot?: string;
+    }): Promise<ExtractType<T>>;
+    tap(prompt: string, options?: {
+        useCache?: boolean;
+    }): Promise<{
+        x: number;
+        y: number;
+    }>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/vision/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vision/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,eAAe,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAGnC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAGvC,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,cAAc,CAAC,EAAE,CAAC,CAAC;QACnB,KAAK,CAAC,EAAE,QAAQ,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE;YACV,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACH,GACA,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3B;;;;;;;;;;OAUG;IACH,GAAG,CACD,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE;YACV,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACH,GACA,OAAO,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtC;AAED,qBAAa,cAAc;IAEvB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;gBADf,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe;IAIpC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAC7B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,cAAc,CAAC,EAAE,CAAC,CAAC;QACnB,KAAK,CAAC,EAAE,QAAQ,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GACA,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAapB,GAAG,CACP,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/B,OAAO,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAsCrC"}

package/dist/vision/index.js

@@ -0,0 +1,106 @@
+"use strict";
+var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
+    var useValue = arguments.length > 2;
+    for (var i = 0; i < initializers.length; i++) {
+        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
+    }
+    return useValue ? value : void 0;
+};
+var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
+    function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
+    var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
+    var target = !descriptorIn && ctor ? contextIn["static"] ? ctor : ctor.prototype : null;
+    var descriptor = descriptorIn || (target ? Object.getOwnPropertyDescriptor(target, contextIn.name) : {});
+    var _, done = false;
+    for (var i = decorators.length - 1; i >= 0; i--) {
+        var context = {};
+        for (var p in contextIn) context[p] = p === "access" ? {} : contextIn[p];
+        for (var p in contextIn.access) context.access[p] = contextIn.access[p];
+        context.addInitializer = function (f) { if (done) throw new TypeError("Cannot add initializers after decoration has completed"); extraInitializers.push(accept(f || null)); };
+        var result = (0, decorators[i])(kind === "accessor" ? { get: descriptor.get, set: descriptor.set } : descriptor[key], context);
+        if (kind === "accessor") {
+            if (result === void 0) continue;
+            if (result === null || typeof result !== "object") throw new TypeError("Object expected");
+            if (_ = accept(result.get)) descriptor.get = _;
+            if (_ = accept(result.set)) descriptor.set = _;
+            if (_ = accept(result.init)) initializers.unshift(_);
+        }
+        else if (_ = accept(result)) {
+            if (kind === "field") initializers.unshift(_);
+            else descriptor[key] = _;
+        }
+    }
+    if (target) Object.defineProperty(target, contextIn.name, descriptor);
+    done = true;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VisionProvider = void 0;
+const vision_1 = require("@empiricalrun/llm/vision");
+const point_1 = require("@empiricalrun/llm/vision/point");
+const fs_1 = __importDefault(require("fs"));
+const test_1 = __importDefault(require("@playwright/test"));
+const utils_1 = require("../utils");
+const logger_1 = require("../logger");
+let VisionProvider = (() => {
+    let _instanceExtraInitializers = [];
+    let _query_decorators;
+    let _tap_decorators;
+    return class VisionProvider {
+        static {
+            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(null) : void 0;
+            _query_decorators = [utils_1.boxedStep];
+            _tap_decorators = [utils_1.boxedStep];
+            __esDecorate(this, null, _query_decorators, { kind: "method", name: "query", static: false, private: false, access: { has: obj => "query" in obj, get: obj => obj.query }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _tap_decorators, { kind: "method", name: "tap", static: false, private: false, access: { has: obj => "tap" in obj, get: obj => obj.tap }, metadata: _metadata }, null, _instanceExtraInitializers);
+            if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+        }
+        device = __runInitializers(this, _instanceExtraInitializers);
+        webDriverClient;
+        constructor(device, webDriverClient) {
+            this.device = device;
+            this.webDriverClient = webDriverClient;
+        }
+        async query(prompt, options) {
+            test_1.default.skip(!process.env.OPENAI_API_KEY, "LLM vision based extract text is not enabled. Set the OPENAI_API_KEY environment variable to enable it");
+            let base64Screenshot = options?.screenshot;
+            if (!base64Screenshot) {
+                base64Screenshot = await this.webDriverClient.takeScreenshot();
+            }
+            return await (0, vision_1.query)(base64Screenshot, prompt, options);
+        }
+        async tap(prompt, options) {
+            test_1.default.skip(!process.env.EMPIRICAL_API_KEY, "LLM vision based tap is not enabled. Set the EMPIRICAL_API_KEY environment variable to enable it");
+            const base64Image = await this.webDriverClient.takeScreenshot();
+            const coordinates = await (0, point_1.getCoordinatesFor)(prompt, base64Image, options);
+            if (coordinates.annotatedImage) {
+                const random = Math.floor(1000 + Math.random() * 9000);
+                const file = test_1.default.info().outputPath(`${random}.png`);
+                await fs_1.default.promises.writeFile(file, Buffer.from(coordinates.annotatedImage, "base64"));
+                await test_1.default.info().attach(`${random}`, { path: file });
+            }
+            const driverSize = await this.webDriverClient.getWindowRect();
+            const { container: imageSize, x, y } = coordinates;
+            const scaleFactorWidth = imageSize.width / driverSize.width;
+            const scaleFactorHeight = imageSize.height / driverSize.height;
+            if (scaleFactorWidth !== scaleFactorHeight) {
+                logger_1.logger.warn(`Scale factors are different: ${scaleFactorWidth} vs ${scaleFactorHeight}`);
+            }
+            const tapTargetX = x / scaleFactorWidth;
+            // This uses the width scale factor because getWindowRect on LambdaTest returns a smaller
+            // height value than the screenshot height, which causes disproportionate scaling
+            // for width and height.
+            // For example, Pixel 8 screenshot is 1080 (w) x 2400 (h), but LambdaTest returns
+            // 1080 (w) x 2142 (h) for getWindowRect.
+            const tapTargetY = y / scaleFactorWidth;
+            await this.device.tap({
+                x: tapTargetX,
+                y: tapTargetY,
+            });
+            return { x: tapTargetX, y: tapTargetY };
+        }
+    };
+})();
+exports.VisionProvider = VisionProvider;

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@samsara-dev/appwright",
+  "version": "0.2.0",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "main": "dist/index.js",
+  "engines": {
+    "node": ">=20.19.0"
+  },
+  "bin": {
+    "appwright": "dist/bin/index.js"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "test": "vitest --config ./src/tests/vitest.config.mts",
+    "build": "tsc --build",
+    "changeset": "changeset",
+    "clean": "tsc --build --clean",
+    "release": "changeset publish",
+    "build:doc": "typedoc --out api-references src"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bheemreddy-samsara/appwright.git"
+  },
+  "keywords": [
+    "e2e",
+    "automation",
+    "ios",
+    "android",
+    "testing"
+  ],
+  "author": "bheem.reddy@samsara.com",
+  "license": "Apache-2.0",
+  "description": "E2E mobile app testing done right, with the Playwright test runner",
+  "dependencies": {
+    "@empiricalrun/llm": "^0.9.25",
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
+    "@playwright/test": "^1.56.1",
+    "appium": "^3.1.0",
+    "appium-uiautomator2-driver": "^5.0.7",
+    "appium-xcuitest-driver": "^10.2.2",
+    "async-retry": "^1.3.3",
+    "fluent-ffmpeg": "^2.1.3",
+    "form-data": "4.0.0",
+    "node-fetch": "^3.3.2",
+    "picocolors": "^1.1.0",
+    "webdriver": "^8.36.1"
+  },
+  "devDependencies": {
+    "@changesets/cli": "^2.27.8",
+    "@empiricalrun/eslint-config": "^0.4.1",
+    "@empiricalrun/typescript-config": "^0.3.0",
+    "@types/async-retry": "^1.4.8",
+    "@types/fluent-ffmpeg": "^2.1.26",
+    "@types/node": "^22.5.2",
+    "eslint": "8.57.0",
+    "typedoc": "0.26.7",
+    "vitest": "^2.1.3"
+  }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "@empiricalrun/typescript-config/base",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "src/**/*.test.ts"
+  ]
+}