@appium/images-plugin 4.0.4 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/lib/compare.d.ts +8 -22
- package/build/lib/compare.d.ts.map +1 -1
- package/build/lib/compare.js +17 -23
- package/build/lib/compare.js.map +1 -1
- package/build/lib/constants.d.ts +13 -12
- package/build/lib/constants.d.ts.map +1 -1
- package/build/lib/constants.js +0 -31
- package/build/lib/constants.js.map +1 -1
- package/build/lib/finder.d.ts +22 -121
- package/build/lib/finder.d.ts.map +1 -1
- package/build/lib/finder.js +60 -100
- package/build/lib/finder.js.map +1 -1
- package/build/lib/image-element.d.ts +36 -108
- package/build/lib/image-element.d.ts.map +1 -1
- package/build/lib/image-element.js +45 -60
- package/build/lib/image-element.js.map +1 -1
- package/build/lib/index.d.ts +8 -0
- package/build/lib/index.d.ts.map +1 -0
- package/build/lib/index.js +30 -0
- package/build/lib/index.js.map +1 -0
- package/build/lib/logger.d.ts +1 -2
- package/build/lib/logger.d.ts.map +1 -1
- package/build/lib/logger.js +2 -2
- package/build/lib/logger.js.map +1 -1
- package/build/lib/plugin.d.ts +15 -34
- package/build/lib/plugin.d.ts.map +1 -1
- package/build/lib/plugin.js +12 -25
- package/build/lib/plugin.js.map +1 -1
- package/build/lib/types.d.ts +127 -0
- package/build/lib/types.d.ts.map +1 -0
- package/build/lib/types.js +3 -0
- package/build/lib/types.js.map +1 -0
- package/lib/compare.ts +100 -0
- package/lib/constants.ts +31 -0
- package/lib/{finder.js → finder.ts} +109 -136
- package/lib/{image-element.js → image-element.ts} +67 -85
- package/lib/index.ts +7 -0
- package/lib/logger.ts +3 -0
- package/lib/{plugin.js → plugin.ts} +42 -38
- package/lib/types.ts +187 -0
- package/package.json +14 -14
- package/tsconfig.json +3 -2
- package/index.js +0 -1
- package/lib/compare.js +0 -96
- package/lib/constants.js +0 -70
- package/lib/logger.js +0 -5
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../lib/types.ts"],"names":[],"mappings":""}
|
package/lib/compare.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import _ from 'lodash';
|
|
2
|
+
import {errors} from 'appium/driver';
|
|
3
|
+
import {
|
|
4
|
+
getImagesMatches,
|
|
5
|
+
getImagesSimilarity,
|
|
6
|
+
getImageOccurrence,
|
|
7
|
+
type MatchingResult,
|
|
8
|
+
type OccurrenceResult,
|
|
9
|
+
type SimilarityResult,
|
|
10
|
+
type MatchingOptions,
|
|
11
|
+
type SimilarityOptions,
|
|
12
|
+
type OccurrenceOptions,
|
|
13
|
+
} from '@appium/opencv';
|
|
14
|
+
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE} from './constants';
|
|
15
|
+
import type {ComparisonResult} from './types';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Performs images comparison using OpenCV framework features.
|
|
19
|
+
* It is expected that both OpenCV framework and opencv4nodejs
|
|
20
|
+
* module are installed on the machine where Appium server is running.
|
|
21
|
+
*
|
|
22
|
+
* @param mode - One of possible comparison modes:
|
|
23
|
+
* matchFeatures, getSimilarity, matchTemplate
|
|
24
|
+
* @param firstImage - Base64-encoded image file.
|
|
25
|
+
* All image formats, that OpenCV library itself accepts, are supported.
|
|
26
|
+
* @param secondImage - Base64-encoded image file.
|
|
27
|
+
* All image formats, that OpenCV library itself accepts, are supported.
|
|
28
|
+
* @param options - The content of this dictionary depends
|
|
29
|
+
* on the actual `mode` value. See the documentation on `@appium/support`
|
|
30
|
+
* module for more details.
|
|
31
|
+
* @returns The content of the resulting dictionary depends
|
|
32
|
+
* on the actual `mode` and `options` values. See the documentation on
|
|
33
|
+
* `@appium/support` module for more details.
|
|
34
|
+
* @throws {Error} If required OpenCV modules are not installed or
|
|
35
|
+
* if `mode` value is incorrect or if there was an unexpected issue while
|
|
36
|
+
* matching the images.
|
|
37
|
+
*/
|
|
38
|
+
export async function compareImages(
|
|
39
|
+
mode: string,
|
|
40
|
+
firstImage: string | Buffer,
|
|
41
|
+
secondImage: string | Buffer,
|
|
42
|
+
options: MatchingOptions | SimilarityOptions | OccurrenceOptions = {}
|
|
43
|
+
): Promise<ComparisonResult> {
|
|
44
|
+
const img1 = Buffer.isBuffer(firstImage) ? firstImage : Buffer.from(firstImage, 'base64');
|
|
45
|
+
const img2 = Buffer.isBuffer(secondImage) ? secondImage : Buffer.from(secondImage, 'base64');
|
|
46
|
+
let result: MatchingResult | SimilarityResult | OccurrenceResult;
|
|
47
|
+
switch (_.toLower(mode)) {
|
|
48
|
+
case MATCH_FEATURES_MODE.toLowerCase():
|
|
49
|
+
try {
|
|
50
|
+
result = await getImagesMatches(img1, img2, options as MatchingOptions);
|
|
51
|
+
} catch {
|
|
52
|
+
// might throw if no matches
|
|
53
|
+
result = {count: 0} as MatchingResult;
|
|
54
|
+
}
|
|
55
|
+
break;
|
|
56
|
+
case GET_SIMILARITY_MODE.toLowerCase():
|
|
57
|
+
result = await getImagesSimilarity(img1, img2, options as SimilarityOptions);
|
|
58
|
+
break;
|
|
59
|
+
case MATCH_TEMPLATE_MODE.toLowerCase(): {
|
|
60
|
+
const opts = options as OccurrenceOptions;
|
|
61
|
+
// firstImage/img1 is the full image and secondImage/img2 is the partial one
|
|
62
|
+
result = await getImageOccurrence(img1, img2, opts);
|
|
63
|
+
if (opts.multiple && (result as OccurrenceResult).multiple) {
|
|
64
|
+
const multipleResults = (result as OccurrenceResult).multiple;
|
|
65
|
+
if (multipleResults) {
|
|
66
|
+
return multipleResults.map(convertVisualizationToBase64);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
default:
|
|
72
|
+
throw new errors.InvalidArgumentError(
|
|
73
|
+
`'${mode}' images comparison mode is unknown. ` +
|
|
74
|
+
`Only ${JSON.stringify([
|
|
75
|
+
MATCH_FEATURES_MODE,
|
|
76
|
+
GET_SIMILARITY_MODE,
|
|
77
|
+
MATCH_TEMPLATE_MODE,
|
|
78
|
+
])} modes are supported.`
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
return convertVisualizationToBase64(result);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* base64 encodes the visualization part of the result
|
|
86
|
+
* (if necessary)
|
|
87
|
+
*
|
|
88
|
+
* @param element - occurrence result
|
|
89
|
+
* @returns result with base64-encoded visualization
|
|
90
|
+
**/
|
|
91
|
+
function convertVisualizationToBase64(
|
|
92
|
+
element: Partial<{visualization: Buffer | null}>
|
|
93
|
+
): any {
|
|
94
|
+
return Buffer.isBuffer(element.visualization)
|
|
95
|
+
? {
|
|
96
|
+
...element,
|
|
97
|
+
visualization: element.visualization.toString('base64'),
|
|
98
|
+
}
|
|
99
|
+
: element;
|
|
100
|
+
}
|
package/lib/constants.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import {node} from 'appium/support';
|
|
2
|
+
import type {ImageSettings} from './types';
|
|
3
|
+
|
|
4
|
+
export const IMAGE_STRATEGY = '-image';
|
|
5
|
+
|
|
6
|
+
export const IMAGE_ELEMENT_PREFIX = 'appium-image-element-';
|
|
7
|
+
export const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions';
|
|
8
|
+
export const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions';
|
|
9
|
+
export const IMAGE_TAP_STRATEGIES = [IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C] as const;
|
|
10
|
+
export const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
|
|
11
|
+
|
|
12
|
+
export const MATCH_FEATURES_MODE = 'matchFeatures';
|
|
13
|
+
export const GET_SIMILARITY_MODE = 'getSimilarity';
|
|
14
|
+
export const MATCH_TEMPLATE_MODE = 'matchTemplate';
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_MATCH_THRESHOLD = 0.4;
|
|
17
|
+
|
|
18
|
+
export const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
|
|
19
|
+
|
|
20
|
+
export const DEFAULT_SETTINGS = node.deepFreeze({
|
|
21
|
+
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
|
|
22
|
+
imageMatchMethod: '',
|
|
23
|
+
fixImageFindScreenshotDims: true,
|
|
24
|
+
fixImageTemplateSize: false,
|
|
25
|
+
fixImageTemplateScale: false,
|
|
26
|
+
defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
|
|
27
|
+
checkForImageElementStaleness: true,
|
|
28
|
+
autoUpdateImageElementPosition: false,
|
|
29
|
+
imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
|
|
30
|
+
getMatchedImageResult: false,
|
|
31
|
+
}) as ImageSettings;
|
|
@@ -3,12 +3,23 @@ import {LRUCache} from 'lru-cache';
|
|
|
3
3
|
import {errors} from 'appium/driver';
|
|
4
4
|
import {ImageElement} from './image-element';
|
|
5
5
|
import {compareImages} from './compare';
|
|
6
|
-
import log from './logger';
|
|
6
|
+
import {log} from './logger';
|
|
7
7
|
import {
|
|
8
|
-
DEFAULT_SETTINGS,
|
|
8
|
+
DEFAULT_SETTINGS,
|
|
9
|
+
MATCH_TEMPLATE_MODE,
|
|
10
|
+
DEFAULT_TEMPLATE_IMAGE_SCALE,
|
|
9
11
|
DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
|
10
12
|
} from './constants';
|
|
11
13
|
import sharp from 'sharp';
|
|
14
|
+
import type {ExternalDriver, Element, Rect, Size} from '@appium/types';
|
|
15
|
+
import type {
|
|
16
|
+
ImageSettings,
|
|
17
|
+
FindByImageOptions,
|
|
18
|
+
Screenshot,
|
|
19
|
+
ScreenshotScale,
|
|
20
|
+
ImageTemplateSettings,
|
|
21
|
+
OccurrenceResultWithVisualization,
|
|
22
|
+
} from './types';
|
|
12
23
|
|
|
13
24
|
// Used to compare ratio and screen width
|
|
14
25
|
// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
|
|
@@ -19,28 +30,26 @@ const MAX_CACHE_AGE_MS = 24 * 60 * 60 * 1000;
|
|
|
19
30
|
/**
|
|
20
31
|
* Checks if one rect fully contains another
|
|
21
32
|
*
|
|
22
|
-
* @param
|
|
23
|
-
* @param
|
|
24
|
-
* @returns
|
|
33
|
+
* @param templateRect The bounding rect
|
|
34
|
+
* @param rect The rect to be checked for containment
|
|
35
|
+
* @returns True if templateRect contains rect
|
|
25
36
|
*/
|
|
26
|
-
function containsRect(templateRect, rect) {
|
|
27
|
-
return
|
|
28
|
-
|
|
29
|
-
|
|
37
|
+
function containsRect(templateRect: Rect, rect: Rect): boolean {
|
|
38
|
+
return (
|
|
39
|
+
templateRect.x <= rect.x &&
|
|
40
|
+
templateRect.y <= rect.y &&
|
|
41
|
+
rect.width <= templateRect.x + templateRect.width - rect.x &&
|
|
42
|
+
rect.height <= templateRect.y + templateRect.height - rect.y
|
|
43
|
+
);
|
|
30
44
|
}
|
|
31
45
|
|
|
32
46
|
const NO_OCCURRENCES_PATTERN = /Cannot find any occurrences/;
|
|
33
47
|
const CONDITION_UNMET_PATTERN = /Condition unmet/;
|
|
34
48
|
|
|
49
|
+
export class ImageElementFinder {
|
|
50
|
+
private _imgElCache: LRUCache<string, ImageElement>;
|
|
35
51
|
|
|
36
|
-
|
|
37
|
-
/** @type {LRUCache<string,ImageElement>} */
|
|
38
|
-
_imgElCache;
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* @param {number} max
|
|
42
|
-
*/
|
|
43
|
-
constructor(max = MAX_CACHE_ITEMS) {
|
|
52
|
+
constructor(max: number = MAX_CACHE_ITEMS) {
|
|
44
53
|
this._imgElCache = new LRUCache({
|
|
45
54
|
ttl: MAX_CACHE_AGE_MS,
|
|
46
55
|
updateAgeOnGet: true,
|
|
@@ -48,57 +57,40 @@ export default class ImageElementFinder {
|
|
|
48
57
|
});
|
|
49
58
|
}
|
|
50
59
|
|
|
51
|
-
|
|
52
|
-
* @param {ImageElement} imgEl
|
|
53
|
-
* @returns {Element}
|
|
54
|
-
*/
|
|
55
|
-
registerImageElement(imgEl) {
|
|
60
|
+
registerImageElement(imgEl: ImageElement): Element {
|
|
56
61
|
this._imgElCache.set(imgEl.id, imgEl);
|
|
57
62
|
return imgEl.asElement();
|
|
58
63
|
}
|
|
59
64
|
|
|
60
|
-
|
|
61
|
-
* @param {string} imgElId
|
|
62
|
-
* @returns {ImageElement|undefined}
|
|
63
|
-
*/
|
|
64
|
-
getImageElement(imgElId) {
|
|
65
|
+
getImageElement(imgElId: string): ImageElement | undefined {
|
|
65
66
|
return this._imgElCache.get(imgElId);
|
|
66
67
|
}
|
|
67
68
|
|
|
68
|
-
clearImageElements() {
|
|
69
|
+
clearImageElements(): void {
|
|
69
70
|
this._imgElCache.clear();
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
/**
|
|
73
|
-
* @typedef FindByImageOptions
|
|
74
|
-
* @property {boolean} [shouldCheckStaleness=false] - whether this call to find an
|
|
75
|
-
* image is merely to check staleness. If so we can bypass a lot of logic
|
|
76
|
-
* @property {boolean} [multiple=false] - Whether we are finding one element or
|
|
77
|
-
* multiple
|
|
78
|
-
* @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
|
|
79
|
-
* ignore defaultImageTemplateScale. It can be used when you would like to
|
|
80
|
-
* scale template with defaultImageTemplateScale setting.
|
|
81
|
-
* @property {import('@appium/types').Rect?} [containerRect=null] - The bounding
|
|
82
|
-
* rectangle to limit the search in
|
|
83
|
-
*/
|
|
84
|
-
|
|
85
73
|
/**
|
|
86
74
|
* Find a screen rect represented by an ImageElement corresponding to an image
|
|
87
75
|
* template sent in by the client
|
|
88
76
|
*
|
|
89
|
-
* @param
|
|
90
|
-
*
|
|
91
|
-
* @param
|
|
92
|
-
* @param {FindByImageOptions} opts - additional options
|
|
77
|
+
* @param template - image used as a template to be matched in the screenshot
|
|
78
|
+
* @param driver
|
|
79
|
+
* @param opts - additional options
|
|
93
80
|
*
|
|
94
|
-
* @returns
|
|
81
|
+
* @returns WebDriver element with a special id prefix
|
|
95
82
|
*/
|
|
96
83
|
async findByImage(
|
|
97
|
-
template,
|
|
98
|
-
driver,
|
|
99
|
-
{
|
|
100
|
-
|
|
101
|
-
|
|
84
|
+
template: Buffer,
|
|
85
|
+
driver: ExternalDriver,
|
|
86
|
+
{
|
|
87
|
+
shouldCheckStaleness = false,
|
|
88
|
+
multiple = false,
|
|
89
|
+
ignoreDefaultImageTemplateScale = false,
|
|
90
|
+
containerRect = null,
|
|
91
|
+
}: FindByImageOptions = {}
|
|
92
|
+
): Promise<Element | Element[] | ImageElement> {
|
|
93
|
+
const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
|
102
94
|
const {
|
|
103
95
|
imageMatchThreshold: threshold,
|
|
104
96
|
imageMatchMethod,
|
|
@@ -112,7 +104,7 @@ export default class ImageElementFinder {
|
|
|
112
104
|
if (!driver.getWindowRect && !_.has(driver, 'getWindowSize')) {
|
|
113
105
|
throw new Error("This driver does not support the required 'getWindowRect' command");
|
|
114
106
|
}
|
|
115
|
-
let screenSize;
|
|
107
|
+
let screenSize: Size;
|
|
116
108
|
if (driver.getWindowRect) {
|
|
117
109
|
const screenRect = await driver.getWindowRect();
|
|
118
110
|
screenSize = {
|
|
@@ -120,7 +112,8 @@ export default class ImageElementFinder {
|
|
|
120
112
|
height: screenRect.height,
|
|
121
113
|
};
|
|
122
114
|
} else {
|
|
123
|
-
//
|
|
115
|
+
// TODO: Drop the deprecated endpoint
|
|
116
|
+
// @ts-expect-error - deprecated getWindowSize method
|
|
124
117
|
screenSize = await driver.getWindowSize();
|
|
125
118
|
}
|
|
126
119
|
|
|
@@ -135,11 +128,10 @@ export default class ImageElementFinder {
|
|
|
135
128
|
});
|
|
136
129
|
}
|
|
137
130
|
|
|
138
|
-
const results = [];
|
|
131
|
+
const results: OccurrenceResultWithVisualization[] = [];
|
|
139
132
|
let didFixTemplateImageScale = false;
|
|
140
|
-
const performLookup = async () => {
|
|
133
|
+
const performLookup = async (): Promise<boolean> => {
|
|
141
134
|
try {
|
|
142
|
-
|
|
143
135
|
const {screenshot, scale} = await this.getScreenshotForImageFind(driver, screenSize);
|
|
144
136
|
|
|
145
137
|
if (!didFixTemplateImageScale) {
|
|
@@ -154,7 +146,7 @@ export default class ImageElementFinder {
|
|
|
154
146
|
didFixTemplateImageScale = true;
|
|
155
147
|
}
|
|
156
148
|
|
|
157
|
-
const comparisonOpts = {
|
|
149
|
+
const comparisonOpts: any = {
|
|
158
150
|
threshold,
|
|
159
151
|
visualize,
|
|
160
152
|
multiple,
|
|
@@ -163,23 +155,26 @@ export default class ImageElementFinder {
|
|
|
163
155
|
comparisonOpts.method = imageMatchMethod;
|
|
164
156
|
}
|
|
165
157
|
|
|
166
|
-
const pushIfOk = (el) => {
|
|
167
|
-
|
|
158
|
+
const pushIfOk = (el: any): boolean => {
|
|
159
|
+
const result: OccurrenceResultWithVisualization = {
|
|
160
|
+
rect: el.rect,
|
|
161
|
+
score: el.score,
|
|
162
|
+
visualization: el.visualization,
|
|
163
|
+
};
|
|
164
|
+
if (containerRect && !containsRect(containerRect, result.rect)) {
|
|
168
165
|
log.debug(
|
|
169
|
-
`The matched element rectangle ${JSON.stringify(
|
|
170
|
-
|
|
166
|
+
`The matched element rectangle ${JSON.stringify(result.rect)} is not located ` +
|
|
167
|
+
`inside of the bounding rectangle ${JSON.stringify(containerRect)}, thus rejected`
|
|
171
168
|
);
|
|
172
169
|
return false;
|
|
173
170
|
}
|
|
174
|
-
results.push(
|
|
171
|
+
results.push(result);
|
|
175
172
|
return true;
|
|
176
173
|
};
|
|
177
174
|
|
|
178
|
-
const elOrEls = await compareImages(
|
|
179
|
-
MATCH_TEMPLATE_MODE, screenshot, template, comparisonOpts
|
|
180
|
-
);
|
|
175
|
+
const elOrEls = await compareImages(MATCH_TEMPLATE_MODE, screenshot, template, comparisonOpts);
|
|
181
176
|
return _.some((_.isArray(elOrEls) ? elOrEls : [elOrEls]).map(pushIfOk));
|
|
182
|
-
} catch (err) {
|
|
177
|
+
} catch (err: any) {
|
|
183
178
|
// if compareImages fails, we'll get a specific error, but we should
|
|
184
179
|
// retry, so trap that and just return false to trigger the next round of
|
|
185
180
|
// implicitly waiting. For other errors, throw them to get out of the
|
|
@@ -193,7 +188,7 @@ export default class ImageElementFinder {
|
|
|
193
188
|
|
|
194
189
|
try {
|
|
195
190
|
await driver.implicitWaitForCondition(performLookup);
|
|
196
|
-
} catch (err) {
|
|
191
|
+
} catch (err: any) {
|
|
197
192
|
// this `implicitWaitForCondition` method will throw a 'Condition unmet'
|
|
198
193
|
// error if an element is not found eventually. In that case, we will
|
|
199
194
|
// handle the element not found response below. In the case where get some
|
|
@@ -239,12 +234,12 @@ export default class ImageElementFinder {
|
|
|
239
234
|
/**
|
|
240
235
|
* Ensure that the image template sent in for a find is of a suitable size
|
|
241
236
|
*
|
|
242
|
-
* @param
|
|
243
|
-
* @param
|
|
237
|
+
* @param template - template image
|
|
238
|
+
* @param maxSize - size of the bounding rectangle
|
|
244
239
|
*
|
|
245
|
-
* @returns
|
|
240
|
+
* @returns image, potentially resized
|
|
246
241
|
*/
|
|
247
|
-
async ensureTemplateSize(template, maxSize) {
|
|
242
|
+
async ensureTemplateSize(template: Buffer, maxSize: Size): Promise<Buffer> {
|
|
248
243
|
const imgObj = sharp(template);
|
|
249
244
|
const {width: tplWidth, height: tplHeight} = await imgObj.metadata();
|
|
250
245
|
if (_.isNil(tplWidth) || _.isNil(tplHeight)) {
|
|
@@ -261,32 +256,36 @@ export default class ImageElementFinder {
|
|
|
261
256
|
|
|
262
257
|
log.info(
|
|
263
258
|
`Scaling template image from ${tplWidth}x${tplHeight} to match ` +
|
|
264
|
-
|
|
259
|
+
`the bounding rectangle at ${maxSize.width}x${maxSize.height}`
|
|
265
260
|
);
|
|
266
261
|
// otherwise, scale it to fit inside the bounding rectangle dimensions:
|
|
267
262
|
// https://sharp.pixelplumbing.com/api-resize
|
|
268
|
-
return await imgObj
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
263
|
+
return await imgObj
|
|
264
|
+
.resize({
|
|
265
|
+
width: Math.trunc(maxSize.width),
|
|
266
|
+
height: Math.trunc(maxSize.height),
|
|
267
|
+
fit: 'inside',
|
|
268
|
+
})
|
|
269
|
+
.toBuffer();
|
|
274
270
|
}
|
|
275
271
|
|
|
276
272
|
/**
|
|
277
273
|
* Get the screenshot image that will be used for find by element, potentially
|
|
278
274
|
* altering it in various ways based on user-requested settings
|
|
279
275
|
*
|
|
280
|
-
* @param
|
|
281
|
-
* @param
|
|
276
|
+
* @param driver
|
|
277
|
+
* @param screenSize - The original size of the screen
|
|
282
278
|
*
|
|
283
|
-
* @returns
|
|
279
|
+
* @returns PNG screenshot and ScreenshotScale
|
|
284
280
|
*/
|
|
285
|
-
async getScreenshotForImageFind(
|
|
281
|
+
async getScreenshotForImageFind(
|
|
282
|
+
driver: ExternalDriver,
|
|
283
|
+
screenSize: Size
|
|
284
|
+
): Promise<Screenshot & {scale?: ScreenshotScale}> {
|
|
286
285
|
if (!driver.getScreenshot) {
|
|
287
286
|
throw new Error("This driver does not support the required 'getScreenshot' command");
|
|
288
287
|
}
|
|
289
|
-
const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
|
288
|
+
const settings: ImageSettings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
|
|
290
289
|
const {fixImageFindScreenshotDims} = settings;
|
|
291
290
|
|
|
292
291
|
const screenshot = Buffer.from(await driver.getScreenshot(), 'base64');
|
|
@@ -301,7 +300,7 @@ export default class ImageElementFinder {
|
|
|
301
300
|
if (screenSize.width < 1 || screenSize.height < 1) {
|
|
302
301
|
log.warn(
|
|
303
302
|
`The retrieved screen size ${screenSize.width}x${screenSize.height} does ` +
|
|
304
|
-
|
|
303
|
+
`not seem to be valid. No changes will be applied to the screenshot`
|
|
305
304
|
);
|
|
306
305
|
return {screenshot};
|
|
307
306
|
}
|
|
@@ -316,7 +315,7 @@ export default class ImageElementFinder {
|
|
|
316
315
|
if (!shotWidth || shotWidth < 1 || !shotHeight || shotHeight < 1) {
|
|
317
316
|
log.warn(
|
|
318
317
|
`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +
|
|
319
|
-
|
|
318
|
+
`not seem to be valid. No changes will be applied to the screenshot`
|
|
320
319
|
);
|
|
321
320
|
return {screenshot};
|
|
322
321
|
}
|
|
@@ -334,21 +333,21 @@ export default class ImageElementFinder {
|
|
|
334
333
|
// are two potential types of mismatch: aspect ratio mismatch and scale
|
|
335
334
|
// mismatch. We need to detect and fix both
|
|
336
335
|
|
|
337
|
-
const scale = {xScale: 1.0, yScale: 1.0};
|
|
336
|
+
const scale: ScreenshotScale = {xScale: 1.0, yScale: 1.0};
|
|
338
337
|
|
|
339
338
|
const screenAR = screenSize.width / screenSize.height;
|
|
340
339
|
const shotAR = shotWidth / shotHeight;
|
|
341
340
|
if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {
|
|
342
341
|
log.info(
|
|
343
342
|
`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +
|
|
344
|
-
|
|
343
|
+
`screen aspect ratio '${screenAR}' (${screenSize.width}x${screenSize.height})`
|
|
345
344
|
);
|
|
346
345
|
} else {
|
|
347
346
|
log.warn(
|
|
348
347
|
`When trying to find an element, determined that the screen ` +
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
348
|
+
`aspect ratio and screenshot aspect ratio are different. Screen ` +
|
|
349
|
+
`is ${screenSize.width}x${screenSize.height} whereas screenshot is ` +
|
|
350
|
+
`${shotWidth}x${shotHeight}.`
|
|
352
351
|
);
|
|
353
352
|
|
|
354
353
|
// In the case where the x-scale and y-scale are different, we need to decide
|
|
@@ -365,13 +364,12 @@ export default class ImageElementFinder {
|
|
|
365
364
|
const xScale = (1.0 * shotWidth) / screenSize.width;
|
|
366
365
|
const yScale = (1.0 * shotHeight) / screenSize.height;
|
|
367
366
|
const scaleFactor = Math.min(xScale, yScale);
|
|
368
|
-
const [newWidth, newHeight] = [shotWidth * scaleFactor, shotHeight * scaleFactor]
|
|
369
|
-
.map(Math.trunc);
|
|
367
|
+
const [newWidth, newHeight] = [shotWidth * scaleFactor, shotHeight * scaleFactor].map(Math.trunc);
|
|
370
368
|
|
|
371
369
|
log.warn(
|
|
372
370
|
`Resizing screenshot to ${newWidth}x${newHeight} to match ` +
|
|
373
|
-
|
|
374
|
-
|
|
371
|
+
`screen aspect ratio so that image element coordinates have a ` +
|
|
372
|
+
`greater chance of being correct.`
|
|
375
373
|
);
|
|
376
374
|
imgObj = imgObj.resize({
|
|
377
375
|
width: newWidth,
|
|
@@ -381,7 +379,8 @@ export default class ImageElementFinder {
|
|
|
381
379
|
|
|
382
380
|
scale.xScale *= scaleFactor;
|
|
383
381
|
scale.yScale *= scaleFactor;
|
|
384
|
-
|
|
382
|
+
shotWidth = newWidth;
|
|
383
|
+
shotHeight = newHeight;
|
|
385
384
|
}
|
|
386
385
|
|
|
387
386
|
// Resize based on the screen dimensions only if both width and height are mismatched
|
|
@@ -391,7 +390,7 @@ export default class ImageElementFinder {
|
|
|
391
390
|
if (screenSize.width !== shotWidth && screenSize.height !== shotHeight) {
|
|
392
391
|
log.info(
|
|
393
392
|
`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
|
|
394
|
-
|
|
393
|
+
`screen at ${screenSize.width}x${screenSize.height}`
|
|
395
394
|
);
|
|
396
395
|
imgObj = imgObj.resize({
|
|
397
396
|
width: Math.trunc(screenSize.width),
|
|
@@ -409,40 +408,32 @@ export default class ImageElementFinder {
|
|
|
409
408
|
};
|
|
410
409
|
}
|
|
411
410
|
|
|
412
|
-
/**
|
|
413
|
-
* @typedef ImageTemplateSettings
|
|
414
|
-
* @property {boolean} [fixImageTemplateScale=false] - fixImageTemplateScale in device-settings
|
|
415
|
-
* @property {number} [defaultImageTemplateScale=DEFAULT_TEMPLATE_IMAGE_SCALE] - defaultImageTemplateScale in device-settings
|
|
416
|
-
* @property {boolean} [ignoreDefaultImageTemplateScale=false] - Ignore defaultImageTemplateScale if it has true.
|
|
417
|
-
* If the template has been scaled to defaultImageTemplateScale or should ignore the scale,
|
|
418
|
-
* this parameter should be true. e.g. click in image-element module
|
|
419
|
-
* @property {number} [xScale=DEFAULT_FIX_IMAGE_TEMPLATE_SCALE] - Scale ratio for width
|
|
420
|
-
* @property {number} [yScale=DEFAULT_FIX_IMAGE_TEMPLATE_SCALE] - Scale ratio for height
|
|
421
|
-
|
|
422
|
-
*/
|
|
423
411
|
/**
|
|
424
412
|
* Get a image that will be used for template matching.
|
|
425
413
|
* Returns scaled image if scale ratio is provided.
|
|
426
414
|
*
|
|
427
|
-
* @param
|
|
428
|
-
*
|
|
429
|
-
* @param {ImageTemplateSettings} opts - Image template scale related options
|
|
415
|
+
* @param template - image used as a template to be matched in the screenshot
|
|
416
|
+
* @param opts - Image template scale related options
|
|
430
417
|
*
|
|
431
|
-
* @returns
|
|
418
|
+
* @returns scaled template screenshot
|
|
432
419
|
*/
|
|
433
|
-
async fixImageTemplateScale(template, opts) {
|
|
420
|
+
async fixImageTemplateScale(template: Buffer, opts?: ImageTemplateSettings): Promise<Buffer> {
|
|
434
421
|
if (!opts) {
|
|
435
422
|
return template;
|
|
436
423
|
}
|
|
437
424
|
|
|
438
|
-
|
|
425
|
+
const {
|
|
439
426
|
fixImageTemplateScale: fixTplScale = false,
|
|
440
|
-
defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
|
|
427
|
+
defaultImageTemplateScale: initialDefaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
|
|
441
428
|
ignoreDefaultImageTemplateScale = false,
|
|
442
|
-
xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
|
443
|
-
yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
|
429
|
+
xScale: initialXScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
|
430
|
+
yScale: initialYScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
|
|
444
431
|
} = opts;
|
|
445
432
|
|
|
433
|
+
let defaultImageTemplateScale = initialDefaultImageTemplateScale;
|
|
434
|
+
let xScale = initialXScale;
|
|
435
|
+
let yScale = initialYScale;
|
|
436
|
+
|
|
446
437
|
if (ignoreDefaultImageTemplateScale) {
|
|
447
438
|
defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;
|
|
448
439
|
}
|
|
@@ -467,12 +458,10 @@ export default class ImageElementFinder {
|
|
|
467
458
|
|
|
468
459
|
// Return if the scale is default, 1, value
|
|
469
460
|
if (
|
|
470
|
-
Math.round(xScale * FLOAT_PRECISION) ===
|
|
471
|
-
Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION) &&
|
|
461
|
+
Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION) &&
|
|
472
462
|
Math.round(
|
|
473
463
|
Number(
|
|
474
|
-
yScale * FLOAT_PRECISION ===
|
|
475
|
-
Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
|
|
464
|
+
yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
|
|
476
465
|
)
|
|
477
466
|
)
|
|
478
467
|
) {
|
|
@@ -499,19 +488,3 @@ export default class ImageElementFinder {
|
|
|
499
488
|
return await imgObj.toBuffer();
|
|
500
489
|
}
|
|
501
490
|
}
|
|
502
|
-
|
|
503
|
-
/**
|
|
504
|
-
* @typedef {import('@appium/types').ExternalDriver} ExternalDriver
|
|
505
|
-
* @typedef {import('@appium/types').Element} Element
|
|
506
|
-
*/
|
|
507
|
-
|
|
508
|
-
/**
|
|
509
|
-
* @typedef Screenshot
|
|
510
|
-
* @property {Buffer} screenshot - screenshot image as PNG
|
|
511
|
-
*/
|
|
512
|
-
|
|
513
|
-
/**
|
|
514
|
-
* @typedef ScreenshotScale
|
|
515
|
-
* @property {number} xScale - Scale ratio for width
|
|
516
|
-
* @property {number} yScale - Scale ratio for height
|
|
517
|
-
*/
|