@appium/images-plugin 4.0.4 → 4.1.0

package/lib/{image-element.js → image-element.ts}

@@ -1,46 +1,33 @@
 import _ from 'lodash';
 import {errors} from 'appium/driver';
 import {util} from '@appium/support';
-import log from './logger';
+import {log} from './logger';
 import {
-  IMAGE_STRATEGY, DEFAULT_SETTINGS, IMAGE_TAP_STRATEGIES,
-  IMAGE_ELEMENT_PREFIX, IMAGE_EL_TAP_STRATEGY_W3C,
+  IMAGE_STRATEGY,
+  DEFAULT_SETTINGS,
+  IMAGE_TAP_STRATEGIES,
+  IMAGE_ELEMENT_PREFIX,
+  IMAGE_EL_TAP_STRATEGY_W3C,
 } from './constants';
+import type {Rect, Element, ExternalDriver, ActionSequence} from '@appium/types';
+import {ImageElementFinder} from './finder';
+import type {ImageSettings, Dimension, Position, ImageElementOpts} from './types';
 
 const TAP_DURATION_MS = 125;
 
-/**
- * @typedef Dimension
- * @property {number} width - width of rect
- * @property {number} height - height of rect
- */
-
-/**
- * @typedef Position
- * @property {number} x - x coordinate
- * @property {number} y - y coordinate
- */
-
-/**
- * @typedef ImageElementOpts
- * @property {Buffer} template - the image which was used to find this ImageElement
- * @property {Rect} rect - bounds of matched image element
- * @property {number} score The similarity score as a float number in range [0.0, 1.0].
- * 1.0 is the highest score (means both images are totally equal).
- * @property {Buffer?} match - the image which has matched marks. Defaults to null.
- * @property {import('./finder').default?} finder - the finder we can use to re-check stale elements
- * @property {import('@appium/types').Rect?} containerRect - The bounding
- * rectangle to limit the search in
- */
-
 /**
  * Representation of an "image element", which is simply a set of coordinates
  * and methods that can be used on that set of coordinates via the driver
  */
-export default class ImageElement {
-  /**
-   * @param {ImageElementOpts} options
-   */
+export class ImageElement {
+  readonly template: Buffer;
+  rect: Rect;
+  readonly id: string;
+  readonly match: Buffer | null;
+  readonly score: number;
+  readonly finder: ImageElementFinder | null;
+  readonly containerRect: Rect | null;
+
   constructor({
     template,
     rect,
@@ -48,7 +35,7 @@ export default class ImageElement {
     match = null,
     finder = null,
     containerRect = null,
-  }) {
+  }: ImageElementOpts) {
     this.template = template;
     this.rect = rect;
     this.id = `${IMAGE_ELEMENT_PREFIX}${util.uuidV4()}`;
@@ -59,23 +46,23 @@ export default class ImageElement {
   }
 
   /**
-   * @returns {Dimension} - dimension of element
+   * @returns dimension of element
    */
-  get size() {
+  get size(): Dimension {
     return {width: this.rect.width, height: this.rect.height};
   }
 
   /**
-   * @returns {Position} - coordinates of top-left corner of element
+   * @returns coordinates of top-left corner of element
    */
-  get location() {
+  get location(): Position {
     return {x: this.rect.x, y: this.rect.y};
   }
 
   /**
-   * @returns {Position} - coordinates of center of element
+   * @returns coordinates of center of element
    */
-  get center() {
+  get center(): Position {
     return {
       x: this.rect.x + this.rect.width / 2,
       y: this.rect.y + this.rect.height / 2,
@@ -83,34 +70,33 @@ export default class ImageElement {
   }
 
   /**
-   * @returns {string} - the base64-encoded original image used for matching
+   * @returns the base64-encoded original image used for matching
    */
-  get originalImage() {
+  get originalImage(): string {
     return this.template.toString('base64');
   }
 
   /**
-   * @returns {string|null} - the base64-encoded image which has matched marks
+   * @returns the base64-encoded image which has matched marks
    */
-  get matchedImage() {
+  get matchedImage(): string | null {
     return this.match?.toString('base64') ?? null;
   }
 
   /**
    *
-   * @returns {Element} - this image element as a WebElement
+   * @returns this image element as a WebElement
    */
-  asElement() {
+  asElement(): Element {
     return util.wrapElement(this.id);
   }
 
   /**
-   * @param {ImageElement} other - an ImageElement to compare with this one
+   * @param other - an ImageElement to compare with this one
    *
-   * @returns {boolean} - whether the other element and this one have the same
-   * properties
+   * @returns whether the other element and this one have the same properties
    */
-  equals(other) {
+  equals(other: ImageElement): boolean {
     return (
       this.rect.x === other.rect.x &&
       this.rect.y === other.rect.y &&
@@ -123,12 +109,12 @@ export default class ImageElement {
    * Use a driver to tap the screen at the center of this ImageElement's
    * position
    *
-   * @param {import('appium/driver').BaseDriver} driver - driver for calling actions with
+   * @param driver - driver for calling actions with
    */
-  async click(driver) {
+  async click(driver: ExternalDriver): Promise<void> {
     // before we click we need to make sure the element is actually still there
     // where we expect it to be
-    const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
+    const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
     const {
       autoUpdateImageElementPosition: updatePos,
       checkForImageElementStaleness,
@@ -144,19 +130,20 @@ export default class ImageElement {
       );
     }
 
-    let newImgEl;
+    let newImgEl: ImageElement | undefined;
     if (checkForImageElementStaleness || updatePos) {
       log.info('Checking image element for staleness before clicking');
       try {
-        newImgEl = /** @type {ImageElement} */ (await (/** @type {import('./finder').default} */ (this.finder))
-          .findByImage(
-            this.template, driver, {
-            shouldCheckStaleness: true,
-            // Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
-            // managed inside Appium after finidng image by template which managed by a user
-            ignoreDefaultImageTemplateScale: true,
-            containerRect: this.containerRect,
-          }));
+        if (!this.finder) {
+          throw new errors.StaleElementReferenceError();
+        }
+        newImgEl = (await this.finder.findByImage(this.template, driver, {
+          shouldCheckStaleness: true,
+          // Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
+          // managed inside Appium after finidng image by template which managed by a user
+          ignoreDefaultImageTemplateScale: true,
+          containerRect: this.containerRect,
+        })) as ImageElement;
       } catch {
         throw new errors.StaleElementReferenceError();
       }
@@ -188,7 +175,7 @@ export default class ImageElement {
     if (imageElementTapStrategy === IMAGE_EL_TAP_STRATEGY_W3C) {
       // set up a W3C action to click on the image by position
       log.info('Will tap using W3C actions');
-      const action = {
+      const action: ActionSequence = {
         type: 'pointer',
         id: 'mouse',
         parameters: {pointerType: 'touch'},
@@ -231,34 +218,36 @@ export default class ImageElement {
   /**
    * Perform lookup of image element(s) inside of the current element
    *
-   * @param {boolean} multiple - Whether to lookup multiple elements
-   * @param {import('appium/driver').BaseDriver} driver - The driver to use for commands
-   * @param  {string[]} args = Rest of arguments for executeScripts
-   * @returns {Promise<Element|Element[]|ImageElement>} - WebDriver element with a special id prefix
+   * @param multiple - Whether to lookup multiple elements
+   * @param driver - The driver to use for commands
+   * @param args - Rest of arguments for executeScripts
+   * @returns WebDriver element with a special id prefix
    */
-  async find(multiple, driver, ...args) {
+  async find(multiple: boolean, driver: ExternalDriver, ...args: any[]): Promise<Element | Element[] | ImageElement> {
     const [strategy, selector] = args;
     if (strategy !== IMAGE_STRATEGY) {
       throw new errors.InvalidSelectorError(`Lookup strategies other than '${IMAGE_STRATEGY}' are not supported`);
     }
-    return await (/** @type {import('./finder').default} */ (this.finder)).findByImage(
-      Buffer.from(selector, 'base64'),
-      driver,
-      {multiple, containerRect: this.rect}
-    );
+    if (!this.finder) {
+      throw new Error('Finder is not available');
+    }
+    return await this.finder.findByImage(Buffer.from(selector, 'base64'), driver, {
+      multiple,
+      containerRect: this.rect,
+    });
   }
 
   /**
    * Handle various Appium commands that involve an image element
    *
-   * @param {import('appium/driver').BaseDriver} driver - the driver to use for commands
-   * @param {string} cmd - the name of the driver command
-   * @param {any} imgEl - image element object
-   * @param {string[]} args - Rest of arguments for executeScripts
+   * @param driver - the driver to use for commands
+   * @param cmd - the name of the driver command
+   * @param imgEl - image element object
+   * @param args - Rest of arguments for executeScripts
    *
-   * @returns {Promise<any>} - the result of running a command
+   * @returns the result of running a command
    */
-  static async execute(driver, imgEl, cmd, ...args) {
+  static async execute(driver: ExternalDriver, imgEl: ImageElement, cmd: string, ...args: any[]): Promise<any> {
     switch (cmd) {
       case 'click':
         return await imgEl.click(driver);
@@ -294,10 +283,3 @@ export default class ImageElement {
     }
   }
 }
-
-export {ImageElement};
-
-/**
- * @typedef {import('@appium/types').Rect} Rect
- * @typedef {import('@appium/types').Element} Element
- */

package/lib/index.ts

@@ -0,0 +1,7 @@
+export {ImageElementPlugin, getImgElFromArgs} from './plugin';
+export {IMAGE_STRATEGY} from './constants';
+export {ImageElementFinder} from './finder';
+export {ImageElement} from './image-element';
+export * from './constants';
+export {compareImages} from './compare';
+export type * from './types';

package/lib/logger.ts

@@ -0,0 +1,3 @@
+import {logger} from 'appium/support';
+
+export const log = logger.getLogger('ImageElementPlugin');

package/lib/{plugin.js → plugin.ts}

@@ -3,22 +3,26 @@ import {errors} from 'appium/driver';
 import {util} from '@appium/support';
 import {BasePlugin} from 'appium/plugin';
 import {compareImages} from './compare';
-import ImageElementFinder from './finder';
+import {ImageElementFinder} from './finder';
 import {ImageElement} from './image-element';
 import {IMAGE_STRATEGY, IMAGE_ELEMENT_PREFIX} from './constants';
+import type {ExternalDriver, Element, ActionSequence, MethodMap} from '@appium/types';
+import type {MatchingOptions, SimilarityOptions, OccurrenceOptions} from '@appium/opencv';
 
-function getImgElFromArgs(args) {
+export function getImgElFromArgs(args: any[]): string | undefined {
   return args.find((arg) => _.isString(arg) && arg.startsWith(IMAGE_ELEMENT_PREFIX));
 }
 
-export default class ImageElementPlugin extends BasePlugin {
-  constructor(pluginName) {
+export class ImageElementPlugin extends BasePlugin {
+  readonly finder: ImageElementFinder;
+
+  constructor(pluginName: string) {
     super(pluginName);
     this.finder = new ImageElementFinder();
   }
 
   // this plugin supports a non-standard 'compare images' command
-  static newMethodMap = /** @type {const} */ ({
+  static newMethodMap: MethodMap<ImageElementPlugin> = {
     '/session/:sessionId/appium/compare_images': {
       POST: {
         command: 'compareImages',
@@ -29,30 +33,33 @@ export default class ImageElementPlugin extends BasePlugin {
         neverProxy: true,
       },
     },
-  });
-
-  async compareImages(next, driver, ...args) {
-    // @ts-ignore Arguments should be ok there
-    return await compareImages(...args);
+  } as const;
+
+  async compareImages(
+    next: () => Promise<any>,
+    driver: ExternalDriver,
+    mode: string,
+    firstImage: string | Buffer,
+    secondImage: string | Buffer,
+    options?: MatchingOptions | SimilarityOptions | OccurrenceOptions
+  ): Promise<any> {
+    return await compareImages(mode, firstImage, secondImage, options);
   }
 
-  async findElement(next, driver, ...args) {
+  async findElement(next: () => Promise<any>, driver: ExternalDriver, ...args: any[]): Promise<any> {
     return await this._find(false, next, driver, ...args);
   }
 
-  async findElements(next, driver, ...args) {
+  async findElements(next: () => Promise<any>, driver: ExternalDriver, ...args: any[]): Promise<any> {
     return await this._find(true, next, driver, ...args);
   }
 
-  /**
-   *
-   * @param {boolean} multiple
-   * @param {*} next
-   * @param {*} driver
-   * @param  {...any} args
-   * @returns {Promise<any>}
-   */
-  async _find(multiple, next, driver, ...args) {
+  private async _find(
+    multiple: boolean,
+    next: () => Promise<any>,
+    driver: ExternalDriver,
+    ...args: any[]
+  ): Promise<any> {
     const [strategy, selector] = args;
 
     // if we're not actually finding by image, just do the normal thing
@@ -63,7 +70,12 @@ export default class ImageElementPlugin extends BasePlugin {
     return await this.finder.findByImage(Buffer.from(selector, 'base64'), driver, {multiple});
   }
 
-  async handle(next, driver, cmdName, ...args) {
+  async handle(
+    next: () => Promise<any>,
+    driver: ExternalDriver,
+    cmdName: string,
+    ...args: any[]
+  ): Promise<any> {
     // if we have a command that involves an image element id, attempt to find the image element
     // and execute the command on it
     const imgElId = getImgElFromArgs(args);
@@ -83,28 +95,22 @@ export default class ImageElementPlugin extends BasePlugin {
     return await next();
   }
 
-  async performActions(next, driver, ...args) {
+  async performActions(
+    next: () => Promise<any>,
+    driver: ExternalDriver,
+    actionSequences: ActionSequence[]
+  ): Promise<any> {
     // Replace with coordinates when ActionSequence includes image elements.
-    const [actionSequences] = /** @type {[import('@appium/types').ActionSequence[]]} */ (args);
     for (const actionSequence of actionSequences) {
       for (const action of actionSequence.actions) {
         // The actions that can have an Element as the origin are "pointerMove" and "scroll".
-        if (
-          !_.isPlainObject(
-            /** @type {{origin?: "viewport" | "pointer" | import('@appium/types').Element}} */ (
-              action
-            ).origin,
-          )
-        ) {
+        if (!_.isPlainObject((action as any).origin)) {
           continue;
         }
 
-        const actionWithEl =
-          /** @type {import('@appium/types').PointerMoveAction | import('@appium/types').ScrollAction} */ (
-            action
-          );
+        const actionWithEl = action as any;
 
-        const elId = util.unwrapElement(/** @type {import('@appium/types').Element} */ (actionWithEl.origin));
+        const elId = util.unwrapElement(actionWithEl.origin as Element);
         if (!_.startsWith(elId, IMAGE_ELEMENT_PREFIX)) {
           continue;
         }
@@ -125,5 +131,3 @@ export default class ImageElementPlugin extends BasePlugin {
     return await next();
   }
 }
-
-export {ImageElementPlugin, getImgElFromArgs, IMAGE_STRATEGY};

package/lib/types.ts

@@ -0,0 +1,187 @@
+import type {Rect} from '@appium/types';
+import type {ImageElementFinder} from './finder';
+import type {MatchingResult, OccurrenceResult, SimilarityResult} from '@appium/opencv';
+import {
+  IMAGE_EL_TAP_STRATEGY_W3C,
+  IMAGE_EL_TAP_STRATEGY_MJSONWP,
+} from './constants';
+
+/**
+ * Image settings interface for device settings
+ */
+export interface ImageSettings {
+  // value between 0 and 1 representing match strength, below which an image
+  // element will not be found
+  imageMatchThreshold: number;
+
+  // One of possible image matching methods.
+  // Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
+  // for more details.
+  // TM_CCOEFF_NORMED by default
+  imageMatchMethod: string;
+
+  // if the image returned by getScreenshot differs in size or aspect ratio
+  // from the screen, attempt to fix it automatically
+  fixImageFindScreenshotDims: boolean;
+
+  // whether Appium should ensure that an image template sent in during image
+  // element find should have its size adjusted so the match algorithm will not
+  // complain
+  fixImageTemplateSize: boolean;
+
+  // whether Appium should ensure that an image template sent in during image
+  // element find should have its scale adjusted to display size so the match
+  // algorithm will not complain.
+  // e.g. iOS has `width=375, height=667` window rect, but its screenshot is
+  //      `width=750 × height=1334` pixels. This setting help to adjust the scale
+  //      if a user use `width=750 × height=1334` pixels's base template image.
+  fixImageTemplateScale: boolean;
+
+  // Users might have scaled template image to reduce their storage size.
+  // This setting allows users to scale a template image they send to Appium server
+  // so that the Appium server compares the actual scale users originally had.
+  // e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
+  //      the user can set {defaultImageTemplateScale: 4.0} to scale the small image
+  //      to the original one so that Appium can compare it as the original one.
+  defaultImageTemplateScale: number;
+
+  // whether Appium should re-check that an image element can be matched
+  // against the current screenshot before clicking it
+  checkForImageElementStaleness: boolean;
+
+  // whether before clicking on an image element Appium should re-determine the
+  // position of the element on screen
+  autoUpdateImageElementPosition: boolean;
+
+  // which method to use for tapping by coordinate for image elements. the
+  // options are 'w3c' or 'mjsonwp'
+  imageElementTapStrategy: typeof IMAGE_EL_TAP_STRATEGY_W3C | typeof IMAGE_EL_TAP_STRATEGY_MJSONWP;
+
+  // which method to use to save the matched image area in ImageElement class.
+  // It is used for debugging purpose.
+  getMatchedImageResult: boolean;
+}
+
+/**
+ * Options for finding elements by image
+ */
+export interface FindByImageOptions {
+  /** whether this call to find an image is merely to check staleness.
+   * If so we can bypass a lot of logic */
+  shouldCheckStaleness?: boolean;
+  /** Whether we are finding one element or multiple */
+  multiple?: boolean;
+  /** Whether we ignore defaultImageTemplateScale. It can be used when you would like to
+   * scale template with defaultImageTemplateScale setting. */
+  ignoreDefaultImageTemplateScale?: boolean;
+  /** The bounding rectangle to limit the search in */
+  containerRect?: Rect | null;
+}
+
+/**
+ * Options for creating an ImageElement
+ */
+export interface ImageElementOpts {
+  /** the image which was used to find this ImageElement */
+  template: Buffer;
+  /** bounds of matched image element */
+  rect: Rect;
+  /** The similarity score as a float number in range [0.0, 1.0]. 1.0 is the highest
+   * score (means both images are totally equal). */
+  score: number;
+  /** the image which has matched marks. Defaults to null. */
+  match?: Buffer | null;
+  /** the finder we can use to re-check stale elements */
+  finder?: ImageElementFinder | null;
+  /** The bounding rectangle to limit the search in */
+  containerRect?: Rect | null;
+}
+
+/**
+ * Dimension interface for width and height
+ */
+export interface Dimension {
+  width: number;
+  height: number;
+}
+
+/**
+ * Position interface for x and y coordinates
+ */
+export interface Position {
+  x: number;
+  y: number;
+}
+
+/**
+ * Screenshot interface
+ */
+export interface Screenshot {
+  screenshot: Buffer;
+}
+
+/**
+ * Screenshot scale interface
+ */
+export interface ScreenshotScale {
+  xScale: number;
+  yScale: number;
+}
+
+/**
+ * Image template settings for scaling
+ */
+export interface ImageTemplateSettings {
+  /** fixImageTemplateScale in device-settings */
+  fixImageTemplateScale?: boolean;
+  /** defaultImageTemplateScale in device-settings */
+  defaultImageTemplateScale?: number;
+  /** Ignore defaultImageTemplateScale if it has true. If the template
+   * has been scaled to defaultImageTemplateScale or should ignore the scale,
+   * this parameter should be true. e.g. click in image-element module */
+  ignoreDefaultImageTemplateScale?: boolean;
+  /** Scale ratio for width */
+  xScale?: number;
+  /** Scale ratio for height */
+  yScale?: number;
+}
+
+/**
+ * Occurrence result with visualization (string instead of Buffer)
+ */
+export interface OccurrenceResultWithVisualization {
+  rect: Rect;
+  score: number;
+  visualization?: string | null;
+}
+
+/**
+ * Interface for results with string visualization instead of Buffer
+ */
+export interface Visualized {
+  visualization?: string | null;
+}
+
+/**
+ * Matching result with string visualization
+ */
+export type VisualizedMatchingResult = MatchingResult & Visualized;
+
+/**
+ * Occurrence result with string visualization
+ */
+export type VisualizedOccurrenceResult = OccurrenceResult & Visualized;
+
+/**
+ * Similarity result with string visualization
+ */
+export type VisualizedSimilarityResult = SimilarityResult & Visualized;
+
+/**
+ * Result type for image comparison operations
+ */
+export type ComparisonResult =
+  | VisualizedMatchingResult
+  | VisualizedOccurrenceResult
+  | VisualizedSimilarityResult
+  | VisualizedSimilarityResult[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appium/images-plugin",
-  "version": "4.0.4",
+  "version": "4.1.0",
   "description": "Plugin for working with images and image elements in Appium",
   "keywords": [
     "automation",
@@ -23,28 +23,28 @@
   },
   "license": "Apache-2.0",
   "author": "https://github.com/appium",
-  "types": "./build/lib/plugin.d.ts",
+  "main": "./build/lib/index.js",
+  "types": "./build/lib/index.d.ts",
   "files": [
     "build",
     "docs",
     "lib",
-    "index.js",
     "tsconfig.json",
-    "!build/tsconfig.tsbuildinfo"
+    "!build/tsconfig.tsbuildinfo",
+    "!build/test"
   ],
   "scripts": {
     "test": "npm run test:unit",
-    "test:e2e": "mocha --exit --timeout 40s --slow 20s \"./test/e2e/**/*.spec.*js\"",
-    "test:smoke": "node ./index.js",
-    "test:unit": "mocha \"./test/unit/**/*.spec.*js\""
+    "test:smoke": "node ./build/lib/index.js",
+    "test:unit": "mocha \"./test/unit/**/*.spec.*ts\"",
+    "test:e2e": "mocha --exit --timeout 40s --slow 20s \"./test/e2e/**/*.spec.*ts\""
   },
   "dependencies": {
-    "@appium/opencv": "^4.0.3",
-    "@appium/support": "^7.0.4",
-    "lodash": "4.17.21",
-    "lru-cache": "11.2.4",
-    "sharp": "0.34.5",
-    "source-map-support": "0.5.21"
+    "@appium/opencv": "^4.1.0",
+    "@appium/support": "^7.0.5",
+    "lodash": "4.17.23",
+    "lru-cache": "11.2.5",
+    "sharp": "0.34.5"
   },
   "peerDependencies": {
     "appium": "^3.0.0-beta.0"
@@ -60,5 +60,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "9004554879687ddad51d3afdf8c711b027efbd99"
+  "gitHead": "f7b20335eab4022e5cbbb627ec86866a994444f8"
 }

package/tsconfig.json

@@ -6,10 +6,11 @@
     "strict": false,
     "paths": {
       "@appium/opencv": ["../opencv"]
-    }
+    },
+    "types": ["mocha", "chai", "sinon", "chai-as-promised"]
   },
   "extends": "@appium/tsconfig/tsconfig.plugin.json",
-  "include": ["lib"],
+  "include": ["lib", "test"],
   "references": [
     {
       "path": "../opencv"