askui 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/ai-element/ai-element-collection.d.ts +10 -0
- package/dist/cjs/core/ai-element/ai-element-collection.js +78 -0
- package/dist/cjs/core/ai-element/ai-element-error.d.ts +2 -0
- package/dist/cjs/core/ai-element/ai-element-error.js +6 -0
- package/dist/cjs/core/ai-element/ai-element.d.ts +27 -0
- package/dist/cjs/core/ai-element/ai-element.js +31 -0
- package/dist/cjs/core/model/custom-element-json.d.ts +32 -15
- package/dist/cjs/core/model/custom-element.d.ts +3 -2
- package/dist/cjs/core/model/custom-element.js +4 -2
- package/dist/cjs/core/reporting/step-reporter.js +15 -6
- package/dist/cjs/execution/dsl.d.ts +488 -113
- package/dist/cjs/execution/dsl.js +519 -113
- package/dist/cjs/execution/index.d.ts +1 -1
- package/dist/cjs/execution/index.js +11 -3
- package/dist/cjs/execution/inference-client.js +9 -4
- package/dist/cjs/execution/ui-control-client-dependency-builder.d.ts +1 -0
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -3
- package/dist/cjs/execution/ui-control-client.d.ts +101 -16
- package/dist/cjs/execution/ui-control-client.js +142 -52
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/cjs/execution/ui-controller-client.d.ts +1 -0
- package/dist/cjs/execution/ui-controller-client.js +11 -1
- package/dist/cjs/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/cjs/execution/ui-controller-not-connected-error.js +12 -0
- package/dist/cjs/main.d.ts +1 -1
- package/dist/cjs/main.js +12 -3
- package/dist/cjs/utils/analytics/analytics.d.ts +3 -1
- package/dist/cjs/utils/analytics/analytics.js +5 -0
- package/dist/cjs/utils/http/custom-errors/index.js +1 -1
- package/dist/cjs/utils/http/http-client-got.js +46 -20
- package/dist/esm/core/ai-element/ai-element-collection.d.ts +10 -0
- package/dist/esm/core/ai-element/ai-element-collection.js +71 -0
- package/dist/esm/core/ai-element/ai-element-error.d.ts +2 -0
- package/dist/esm/core/ai-element/ai-element-error.js +2 -0
- package/dist/esm/core/ai-element/ai-element.d.ts +27 -0
- package/dist/esm/core/ai-element/ai-element.js +28 -0
- package/dist/esm/core/model/custom-element-json.d.ts +32 -15
- package/dist/esm/core/model/custom-element.d.ts +3 -2
- package/dist/esm/core/model/custom-element.js +4 -2
- package/dist/esm/core/reporting/step-reporter.js +15 -6
- package/dist/esm/execution/dsl.d.ts +488 -113
- package/dist/esm/execution/dsl.js +519 -113
- package/dist/esm/execution/index.d.ts +1 -1
- package/dist/esm/execution/index.js +1 -1
- package/dist/esm/execution/inference-client.js +9 -4
- package/dist/esm/execution/ui-control-client-dependency-builder.d.ts +1 -0
- package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -3
- package/dist/esm/execution/ui-control-client.d.ts +101 -16
- package/dist/esm/execution/ui-control-client.js +142 -49
- package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/esm/execution/ui-controller-client.d.ts +1 -0
- package/dist/esm/execution/ui-controller-client.js +11 -1
- package/dist/esm/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/esm/execution/ui-controller-not-connected-error.js +8 -0
- package/dist/esm/main.d.ts +1 -1
- package/dist/esm/main.js +1 -1
- package/dist/esm/utils/analytics/analytics.d.ts +3 -1
- package/dist/esm/utils/analytics/analytics.js +5 -0
- package/dist/esm/utils/http/custom-errors/index.js +1 -1
- package/dist/esm/utils/http/http-client-got.js +46 -20
- package/package.json +1 -1
|
@@ -31,7 +31,11 @@ class FluentBase {
|
|
|
31
31
|
if (this instanceof FluentCommand) {
|
|
32
32
|
const fluentCommand = this;
|
|
33
33
|
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
|
|
34
|
-
|
|
34
|
+
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
|
|
35
|
+
return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), {
|
|
36
|
+
customElementsJson: customElements,
|
|
37
|
+
aiElementNames,
|
|
38
|
+
});
|
|
35
39
|
}
|
|
36
40
|
if (!this.prev) {
|
|
37
41
|
throw new Error('Prev element not defined');
|
|
@@ -44,7 +48,11 @@ class FluentBase {
|
|
|
44
48
|
if (this instanceof Getter) {
|
|
45
49
|
const getter = this;
|
|
46
50
|
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
|
|
47
|
-
|
|
51
|
+
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
|
|
52
|
+
return getter.getterExecutor(newCurrentInstruction.trim(), {
|
|
53
|
+
customElementsJson: customElements,
|
|
54
|
+
aiElementNames,
|
|
55
|
+
});
|
|
48
56
|
}
|
|
49
57
|
if (!this.prev) {
|
|
50
58
|
throw new Error('Prev element not defined');
|
|
@@ -147,9 +155,11 @@ class FluentFilters extends FluentBase {
|
|
|
147
155
|
*
|
|
148
156
|
* **Examples:**
|
|
149
157
|
* ```typescript
|
|
150
|
-
* await aui.
|
|
158
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
151
159
|
* ```
|
|
152
160
|
*
|
|
161
|
+
* 
|
|
162
|
+
*
|
|
153
163
|
* @return {FluentFiltersOrRelations}
|
|
154
164
|
*/
|
|
155
165
|
button() {
|
|
@@ -209,13 +219,20 @@ class FluentFilters extends FluentBase {
|
|
|
209
219
|
/**
|
|
210
220
|
* Filters for an UI element 'text'.
|
|
211
221
|
*
|
|
212
|
-
*
|
|
222
|
+
* Takes an optional parameter to filter for a specific text.
|
|
223
|
+
* See the examples below.
|
|
224
|
+
*
|
|
213
225
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
214
226
|
*
|
|
215
227
|
* **Examples:**
|
|
216
228
|
* ```typescript
|
|
217
|
-
* await aui.click().text().
|
|
229
|
+
* await aui.click().text().exec();
|
|
230
|
+
* await aui.click().text('Username').exec();
|
|
231
|
+
*
|
|
232
|
+
* // Matching with an exact text
|
|
218
233
|
* await aui.click().text().withExactText('Username').exec();
|
|
234
|
+
*
|
|
235
|
+
* // Matching with a regex
|
|
219
236
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
220
237
|
* ```
|
|
221
238
|
*
|
|
@@ -239,6 +256,8 @@ class FluentFilters extends FluentBase {
|
|
|
239
256
|
* icon().withText('plus')
|
|
240
257
|
* ```
|
|
241
258
|
*
|
|
259
|
+
* 
|
|
260
|
+
*
|
|
242
261
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
243
262
|
*
|
|
244
263
|
* @return {FluentFiltersOrRelations}
|
|
@@ -249,7 +268,9 @@ class FluentFilters extends FluentBase {
|
|
|
249
268
|
return new FluentFiltersOrRelations(this);
|
|
250
269
|
}
|
|
251
270
|
/**
|
|
252
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
271
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
272
|
+
*
|
|
273
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
253
274
|
*
|
|
254
275
|
* **Example**
|
|
255
276
|
* ```typescript
|
|
@@ -259,6 +280,7 @@ class FluentFilters extends FluentBase {
|
|
|
259
280
|
* customImage: './logo.png', // required
|
|
260
281
|
* name: 'myLogo', // optional
|
|
261
282
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
283
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
262
284
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
263
285
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
264
286
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -273,12 +295,13 @@ class FluentFilters extends FluentBase {
|
|
|
273
295
|
* - **name** (*`string`, optional*):
|
|
274
296
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
275
297
|
* - **threshold** (*`number`, optional*):
|
|
276
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
298
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
299
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
300
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
277
301
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
278
302
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
279
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
280
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
281
|
-
* of the given custom image.
|
|
303
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
304
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
282
305
|
*
|
|
283
306
|
*
|
|
284
307
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -292,6 +315,23 @@ class FluentFilters extends FluentBase {
|
|
|
292
315
|
this._params.set('customElement', customElement);
|
|
293
316
|
return new FluentFiltersOrRelations(this);
|
|
294
317
|
}
|
|
318
|
+
/**
|
|
319
|
+
* Detects an AI Element created with the workflow creator.
|
|
320
|
+
*
|
|
321
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
322
|
+
*
|
|
323
|
+
* @return {FluentFiltersOrRelations}
|
|
324
|
+
*/
|
|
325
|
+
aiElement(aiElementName) {
|
|
326
|
+
this._textStr = '';
|
|
327
|
+
this._textStr += 'ai';
|
|
328
|
+
this._textStr += ' element';
|
|
329
|
+
this._textStr += ' with';
|
|
330
|
+
this._textStr += ' name';
|
|
331
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
332
|
+
this._params.set('aiElementName', aiElementName);
|
|
333
|
+
return new FluentFiltersOrRelations(this);
|
|
334
|
+
}
|
|
295
335
|
/**
|
|
296
336
|
* Filters for a UI element 'image'.
|
|
297
337
|
*
|
|
@@ -301,10 +341,14 @@ class FluentFilters extends FluentBase {
|
|
|
301
341
|
* await aui.click().image().exec();
|
|
302
342
|
*
|
|
303
343
|
* // Works if you have an image with
|
|
304
|
-
* // a
|
|
305
|
-
* await aui.click().image().above().text().withText('
|
|
344
|
+
* // a text below
|
|
345
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
306
346
|
* ```
|
|
307
347
|
*
|
|
348
|
+
* 
|
|
349
|
+
*
|
|
350
|
+
*
|
|
351
|
+
*
|
|
308
352
|
* @return {FluentFiltersOrRelations}
|
|
309
353
|
*/
|
|
310
354
|
image() {
|
|
@@ -325,6 +369,10 @@ class FluentFilters extends FluentBase {
|
|
|
325
369
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
326
370
|
* ```
|
|
327
371
|
*
|
|
372
|
+
* 
|
|
373
|
+
*
|
|
374
|
+
*
|
|
375
|
+
*
|
|
328
376
|
* @return {FluentFiltersOrRelations}
|
|
329
377
|
*/
|
|
330
378
|
textfield() {
|
|
@@ -333,7 +381,13 @@ class FluentFilters extends FluentBase {
|
|
|
333
381
|
return new FluentFiltersOrRelations(this);
|
|
334
382
|
}
|
|
335
383
|
/**
|
|
336
|
-
* Filters for similar
|
|
384
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
385
|
+
*
|
|
386
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
387
|
+
*
|
|
388
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
389
|
+
*
|
|
390
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
337
391
|
*
|
|
338
392
|
* **Examples:**
|
|
339
393
|
* ```typescript
|
|
@@ -348,6 +402,11 @@ class FluentFilters extends FluentBase {
|
|
|
348
402
|
* // usually false
|
|
349
403
|
* 'atebxtc' === withText('text') => false
|
|
350
404
|
* 'other' === withText('text') => false
|
|
405
|
+
*
|
|
406
|
+
* // optional parameter: similarity_score
|
|
407
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
408
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
409
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
351
410
|
* ```
|
|
352
411
|
* 
|
|
353
412
|
*
|
|
@@ -378,6 +437,10 @@ class FluentFilters extends FluentBase {
|
|
|
378
437
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
379
438
|
* ```
|
|
380
439
|
*
|
|
440
|
+
* 
|
|
441
|
+
*
|
|
442
|
+
*
|
|
443
|
+
*
|
|
381
444
|
* @param {string} regex_pattern - A regex pattern
|
|
382
445
|
*
|
|
383
446
|
* @return {FluentFiltersOrRelations}
|
|
@@ -406,6 +469,10 @@ class FluentFilters extends FluentBase {
|
|
|
406
469
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
407
470
|
* ```
|
|
408
471
|
*
|
|
472
|
+
* 
|
|
473
|
+
*
|
|
474
|
+
*
|
|
475
|
+
*
|
|
409
476
|
* @param {string} text - A text to be matched.
|
|
410
477
|
*
|
|
411
478
|
* @return {FluentFiltersOrRelations}
|
|
@@ -442,17 +509,28 @@ class FluentFilters extends FluentBase {
|
|
|
442
509
|
/**
|
|
443
510
|
* Filters elements based on a textual description.
|
|
444
511
|
*
|
|
445
|
-
*
|
|
512
|
+
* **What Should I Write as Matching Text**
|
|
513
|
+
*
|
|
446
514
|
* The text description inside the `matching()` should describe the element visually.
|
|
447
515
|
* It understands color, some famous company/product names, general descriptions.
|
|
448
516
|
*
|
|
449
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
450
|
-
* E.g. `puzzle piece` can fail
|
|
517
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
518
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
451
519
|
* Generally the more detail the better.
|
|
452
520
|
*
|
|
453
521
|
* **Examples:**
|
|
454
522
|
* ```typescript
|
|
455
|
-
*
|
|
523
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
524
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
525
|
+
*
|
|
526
|
+
* // Select an image that has text in it
|
|
527
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
528
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
529
|
+
*
|
|
530
|
+
* // Target a logo/image by describing it
|
|
531
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
532
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
533
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
456
534
|
* ```
|
|
457
535
|
*
|
|
458
536
|
* @param {string} text - A description of the target element.
|
|
@@ -611,14 +689,19 @@ class FluentFiltersOrRelations extends FluentFilters {
|
|
|
611
689
|
/**
|
|
612
690
|
* Filters for an element right of another element.
|
|
613
691
|
*
|
|
692
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
693
|
+
*
|
|
614
694
|
* **Examples:**
|
|
615
695
|
* ```typescript
|
|
616
|
-
* -------------- --------------
|
|
617
|
-
* | leftEl | |
|
|
618
|
-
* -------------- --------------
|
|
696
|
+
* -------------- -------------- --------------
|
|
697
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
698
|
+
* -------------- -------------- --------------
|
|
619
699
|
*
|
|
620
|
-
* // Returns
|
|
700
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
621
701
|
* ...rightEl().rightOf().leftEl()
|
|
702
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
703
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
704
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
622
705
|
* // Returns no element because leftEl is left of rightEl
|
|
623
706
|
* ...leftEl().rightOf().rightEl()
|
|
624
707
|
* ```
|
|
@@ -638,14 +721,19 @@ class FluentFiltersOrRelations extends FluentFilters {
|
|
|
638
721
|
/**
|
|
639
722
|
* Filters for an element left of another element.
|
|
640
723
|
*
|
|
724
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
725
|
+
*
|
|
641
726
|
* **Examples:**
|
|
642
727
|
* ```typescript
|
|
643
|
-
* -------------- --------------
|
|
644
|
-
* |
|
|
645
|
-
* -------------- --------------
|
|
728
|
+
* -------------- -------------- --------------
|
|
729
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
730
|
+
* -------------- -------------- --------------
|
|
646
731
|
*
|
|
647
|
-
* // Returns
|
|
732
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
648
733
|
* ...leftEl().leftOf().rightEl()
|
|
734
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
735
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
736
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
649
737
|
* // Returns no element because rightEl is left of leftEl
|
|
650
738
|
* ...rightEl().leftOf().leftEl()
|
|
651
739
|
* ```
|
|
@@ -665,17 +753,25 @@ class FluentFiltersOrRelations extends FluentFilters {
|
|
|
665
753
|
/**
|
|
666
754
|
* Filters for an element below another element.
|
|
667
755
|
*
|
|
756
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
757
|
+
*
|
|
668
758
|
* **Examples:**
|
|
669
759
|
* ```typescript
|
|
670
760
|
* --------------
|
|
671
761
|
* | text |
|
|
672
762
|
* --------------
|
|
673
763
|
* --------------
|
|
674
|
-
* |
|
|
764
|
+
* | button0 |
|
|
765
|
+
* --------------
|
|
766
|
+
* --------------
|
|
767
|
+
* | button1 |
|
|
675
768
|
* --------------
|
|
676
769
|
*
|
|
677
|
-
* // Returns
|
|
770
|
+
* // Returns button0 because button0 is the first button below text
|
|
678
771
|
* ...button().below().text()
|
|
772
|
+
* ...button().below(0).text()
|
|
773
|
+
* // Returns button1 because button1 is the second button below text
|
|
774
|
+
* ...button().below(1).text()
|
|
679
775
|
* // Returns no element because text is above button
|
|
680
776
|
* ...text().below().button()
|
|
681
777
|
* ```
|
|
@@ -694,17 +790,25 @@ class FluentFiltersOrRelations extends FluentFilters {
|
|
|
694
790
|
/**
|
|
695
791
|
* Filters for an element above another element.
|
|
696
792
|
*
|
|
793
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
794
|
+
*
|
|
697
795
|
* **Examples:**
|
|
698
796
|
* ```typescript
|
|
699
797
|
* --------------
|
|
700
|
-
* |
|
|
798
|
+
* | text1 |
|
|
799
|
+
* --------------
|
|
800
|
+
* --------------
|
|
801
|
+
* | text0 |
|
|
701
802
|
* --------------
|
|
702
803
|
* --------------
|
|
703
804
|
* | button |
|
|
704
805
|
* --------------
|
|
705
806
|
*
|
|
706
|
-
* // Returns
|
|
807
|
+
* // Returns text0 because text0 is the first element above button
|
|
707
808
|
* ...text().above().button()
|
|
809
|
+
* ...text().above(0).button()
|
|
810
|
+
* // Returns text1 because text1 is the second element above button
|
|
811
|
+
* ...text().above(1).button()
|
|
708
812
|
* // Returns no element because button is below text
|
|
709
813
|
* ...button().above().text()
|
|
710
814
|
* ```
|
|
@@ -870,9 +974,11 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
870
974
|
*
|
|
871
975
|
* **Examples:**
|
|
872
976
|
* ```typescript
|
|
873
|
-
* await aui.
|
|
977
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
874
978
|
* ```
|
|
875
979
|
*
|
|
980
|
+
* 
|
|
981
|
+
*
|
|
876
982
|
* @return {FluentFiltersOrRelationsCondition}
|
|
877
983
|
*/
|
|
878
984
|
button() {
|
|
@@ -932,13 +1038,20 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
932
1038
|
/**
|
|
933
1039
|
* Filters for an UI element 'text'.
|
|
934
1040
|
*
|
|
935
|
-
*
|
|
1041
|
+
* Takes an optional parameter to filter for a specific text.
|
|
1042
|
+
* See the examples below.
|
|
1043
|
+
*
|
|
936
1044
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
937
1045
|
*
|
|
938
1046
|
* **Examples:**
|
|
939
1047
|
* ```typescript
|
|
940
|
-
* await aui.click().text().
|
|
1048
|
+
* await aui.click().text().exec();
|
|
1049
|
+
* await aui.click().text('Username').exec();
|
|
1050
|
+
*
|
|
1051
|
+
* // Matching with an exact text
|
|
941
1052
|
* await aui.click().text().withExactText('Username').exec();
|
|
1053
|
+
*
|
|
1054
|
+
* // Matching with a regex
|
|
942
1055
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
943
1056
|
* ```
|
|
944
1057
|
*
|
|
@@ -962,6 +1075,8 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
962
1075
|
* icon().withText('plus')
|
|
963
1076
|
* ```
|
|
964
1077
|
*
|
|
1078
|
+
* 
|
|
1079
|
+
*
|
|
965
1080
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
966
1081
|
*
|
|
967
1082
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -972,7 +1087,9 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
972
1087
|
return new FluentFiltersOrRelationsCondition(this);
|
|
973
1088
|
}
|
|
974
1089
|
/**
|
|
975
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
1090
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
1091
|
+
*
|
|
1092
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
976
1093
|
*
|
|
977
1094
|
* **Example**
|
|
978
1095
|
* ```typescript
|
|
@@ -982,6 +1099,7 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
982
1099
|
* customImage: './logo.png', // required
|
|
983
1100
|
* name: 'myLogo', // optional
|
|
984
1101
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
1102
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
985
1103
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
986
1104
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
987
1105
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -996,12 +1114,13 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
996
1114
|
* - **name** (*`string`, optional*):
|
|
997
1115
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
998
1116
|
* - **threshold** (*`number`, optional*):
|
|
999
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1117
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1118
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
1119
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
1000
1120
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
1001
1121
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
1002
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
1003
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
1004
|
-
* of the given custom image.
|
|
1122
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
1123
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
1005
1124
|
*
|
|
1006
1125
|
*
|
|
1007
1126
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -1015,6 +1134,23 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1015
1134
|
this._params.set('customElement', customElement);
|
|
1016
1135
|
return new FluentFiltersOrRelationsCondition(this);
|
|
1017
1136
|
}
|
|
1137
|
+
/**
|
|
1138
|
+
* Detects an AI Element created with the workflow creator.
|
|
1139
|
+
*
|
|
1140
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
1141
|
+
*
|
|
1142
|
+
* @return {FluentFiltersOrRelationsCondition}
|
|
1143
|
+
*/
|
|
1144
|
+
aiElement(aiElementName) {
|
|
1145
|
+
this._textStr = '';
|
|
1146
|
+
this._textStr += 'ai';
|
|
1147
|
+
this._textStr += ' element';
|
|
1148
|
+
this._textStr += ' with';
|
|
1149
|
+
this._textStr += ' name';
|
|
1150
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
1151
|
+
this._params.set('aiElementName', aiElementName);
|
|
1152
|
+
return new FluentFiltersOrRelationsCondition(this);
|
|
1153
|
+
}
|
|
1018
1154
|
/**
|
|
1019
1155
|
* Filters for a UI element 'image'.
|
|
1020
1156
|
*
|
|
@@ -1024,10 +1160,14 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1024
1160
|
* await aui.click().image().exec();
|
|
1025
1161
|
*
|
|
1026
1162
|
* // Works if you have an image with
|
|
1027
|
-
* // a
|
|
1028
|
-
* await aui.click().image().above().text().withText('
|
|
1163
|
+
* // a text below
|
|
1164
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
1029
1165
|
* ```
|
|
1030
1166
|
*
|
|
1167
|
+
* 
|
|
1168
|
+
*
|
|
1169
|
+
*
|
|
1170
|
+
*
|
|
1031
1171
|
* @return {FluentFiltersOrRelationsCondition}
|
|
1032
1172
|
*/
|
|
1033
1173
|
image() {
|
|
@@ -1048,6 +1188,10 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1048
1188
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
1049
1189
|
* ```
|
|
1050
1190
|
*
|
|
1191
|
+
* 
|
|
1192
|
+
*
|
|
1193
|
+
*
|
|
1194
|
+
*
|
|
1051
1195
|
* @return {FluentFiltersOrRelationsCondition}
|
|
1052
1196
|
*/
|
|
1053
1197
|
textfield() {
|
|
@@ -1056,7 +1200,13 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1056
1200
|
return new FluentFiltersOrRelationsCondition(this);
|
|
1057
1201
|
}
|
|
1058
1202
|
/**
|
|
1059
|
-
* Filters for similar
|
|
1203
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
1204
|
+
*
|
|
1205
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
1206
|
+
*
|
|
1207
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
1208
|
+
*
|
|
1209
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
1060
1210
|
*
|
|
1061
1211
|
* **Examples:**
|
|
1062
1212
|
* ```typescript
|
|
@@ -1071,6 +1221,11 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1071
1221
|
* // usually false
|
|
1072
1222
|
* 'atebxtc' === withText('text') => false
|
|
1073
1223
|
* 'other' === withText('text') => false
|
|
1224
|
+
*
|
|
1225
|
+
* // optional parameter: similarity_score
|
|
1226
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
1227
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
1228
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
1074
1229
|
* ```
|
|
1075
1230
|
* 
|
|
1076
1231
|
*
|
|
@@ -1101,6 +1256,10 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1101
1256
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
1102
1257
|
* ```
|
|
1103
1258
|
*
|
|
1259
|
+
* 
|
|
1260
|
+
*
|
|
1261
|
+
*
|
|
1262
|
+
*
|
|
1104
1263
|
* @param {string} regex_pattern - A regex pattern
|
|
1105
1264
|
*
|
|
1106
1265
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -1129,6 +1288,10 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1129
1288
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
1130
1289
|
* ```
|
|
1131
1290
|
*
|
|
1291
|
+
* 
|
|
1292
|
+
*
|
|
1293
|
+
*
|
|
1294
|
+
*
|
|
1132
1295
|
* @param {string} text - A text to be matched.
|
|
1133
1296
|
*
|
|
1134
1297
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -1165,17 +1328,28 @@ class FluentFiltersCondition extends FluentBase {
|
|
|
1165
1328
|
/**
|
|
1166
1329
|
* Filters elements based on a textual description.
|
|
1167
1330
|
*
|
|
1168
|
-
*
|
|
1331
|
+
* **What Should I Write as Matching Text**
|
|
1332
|
+
*
|
|
1169
1333
|
* The text description inside the `matching()` should describe the element visually.
|
|
1170
1334
|
* It understands color, some famous company/product names, general descriptions.
|
|
1171
1335
|
*
|
|
1172
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
1173
|
-
* E.g. `puzzle piece` can fail
|
|
1336
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
1337
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
1174
1338
|
* Generally the more detail the better.
|
|
1175
1339
|
*
|
|
1176
1340
|
* **Examples:**
|
|
1177
1341
|
* ```typescript
|
|
1178
|
-
*
|
|
1342
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
1343
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
1344
|
+
*
|
|
1345
|
+
* // Select an image that has text in it
|
|
1346
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
1347
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
1348
|
+
*
|
|
1349
|
+
* // Target a logo/image by describing it
|
|
1350
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
1351
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
1352
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
1179
1353
|
* ```
|
|
1180
1354
|
*
|
|
1181
1355
|
* @param {string} text - A description of the target element.
|
|
@@ -1334,14 +1508,19 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1334
1508
|
/**
|
|
1335
1509
|
* Filters for an element right of another element.
|
|
1336
1510
|
*
|
|
1511
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1512
|
+
*
|
|
1337
1513
|
* **Examples:**
|
|
1338
1514
|
* ```typescript
|
|
1339
|
-
* -------------- --------------
|
|
1340
|
-
* | leftEl | |
|
|
1341
|
-
* -------------- --------------
|
|
1515
|
+
* -------------- -------------- --------------
|
|
1516
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
1517
|
+
* -------------- -------------- --------------
|
|
1342
1518
|
*
|
|
1343
|
-
* // Returns
|
|
1519
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
1344
1520
|
* ...rightEl().rightOf().leftEl()
|
|
1521
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
1522
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
1523
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
1345
1524
|
* // Returns no element because leftEl is left of rightEl
|
|
1346
1525
|
* ...leftEl().rightOf().rightEl()
|
|
1347
1526
|
* ```
|
|
@@ -1361,14 +1540,19 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1361
1540
|
/**
|
|
1362
1541
|
* Filters for an element left of another element.
|
|
1363
1542
|
*
|
|
1543
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1544
|
+
*
|
|
1364
1545
|
* **Examples:**
|
|
1365
1546
|
* ```typescript
|
|
1366
|
-
* -------------- --------------
|
|
1367
|
-
* |
|
|
1368
|
-
* -------------- --------------
|
|
1547
|
+
* -------------- -------------- --------------
|
|
1548
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
1549
|
+
* -------------- -------------- --------------
|
|
1369
1550
|
*
|
|
1370
|
-
* // Returns
|
|
1551
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
1371
1552
|
* ...leftEl().leftOf().rightEl()
|
|
1553
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
1554
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
1555
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
1372
1556
|
* // Returns no element because rightEl is left of leftEl
|
|
1373
1557
|
* ...rightEl().leftOf().leftEl()
|
|
1374
1558
|
* ```
|
|
@@ -1388,17 +1572,25 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1388
1572
|
/**
|
|
1389
1573
|
* Filters for an element below another element.
|
|
1390
1574
|
*
|
|
1575
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1576
|
+
*
|
|
1391
1577
|
* **Examples:**
|
|
1392
1578
|
* ```typescript
|
|
1393
1579
|
* --------------
|
|
1394
1580
|
* | text |
|
|
1395
1581
|
* --------------
|
|
1396
1582
|
* --------------
|
|
1397
|
-
* |
|
|
1583
|
+
* | button0 |
|
|
1584
|
+
* --------------
|
|
1585
|
+
* --------------
|
|
1586
|
+
* | button1 |
|
|
1398
1587
|
* --------------
|
|
1399
1588
|
*
|
|
1400
|
-
* // Returns
|
|
1589
|
+
* // Returns button0 because button0 is the first button below text
|
|
1401
1590
|
* ...button().below().text()
|
|
1591
|
+
* ...button().below(0).text()
|
|
1592
|
+
* // Returns button1 because button1 is the second button below text
|
|
1593
|
+
* ...button().below(1).text()
|
|
1402
1594
|
* // Returns no element because text is above button
|
|
1403
1595
|
* ...text().below().button()
|
|
1404
1596
|
* ```
|
|
@@ -1417,17 +1609,25 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1417
1609
|
/**
|
|
1418
1610
|
* Filters for an element above another element.
|
|
1419
1611
|
*
|
|
1612
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1613
|
+
*
|
|
1420
1614
|
* **Examples:**
|
|
1421
1615
|
* ```typescript
|
|
1422
1616
|
* --------------
|
|
1423
|
-
* |
|
|
1617
|
+
* | text1 |
|
|
1618
|
+
* --------------
|
|
1619
|
+
* --------------
|
|
1620
|
+
* | text0 |
|
|
1424
1621
|
* --------------
|
|
1425
1622
|
* --------------
|
|
1426
1623
|
* | button |
|
|
1427
1624
|
* --------------
|
|
1428
1625
|
*
|
|
1429
|
-
* // Returns
|
|
1626
|
+
* // Returns text0 because text0 is the first element above button
|
|
1430
1627
|
* ...text().above().button()
|
|
1628
|
+
* ...text().above(0).button()
|
|
1629
|
+
* // Returns text1 because text1 is the second element above button
|
|
1630
|
+
* ...text().above(1).button()
|
|
1431
1631
|
* // Returns no element because button is below text
|
|
1432
1632
|
* ...button().above().text()
|
|
1433
1633
|
* ```
|
|
@@ -1511,12 +1711,12 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1511
1711
|
* **Examples:**
|
|
1512
1712
|
* ```typescript
|
|
1513
1713
|
* // Stops execution at this point when the element does not exist.
|
|
1514
|
-
* await aui.expect().text(
|
|
1714
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1515
1715
|
*
|
|
1516
1716
|
* // This will catch the error and log a message
|
|
1517
1717
|
* // But the execution will continue afterwards
|
|
1518
1718
|
* try {
|
|
1519
|
-
* await aui.expect().text(
|
|
1719
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1520
1720
|
* } catch (error) {
|
|
1521
1721
|
* console.log('Too bad we could not find the element!');
|
|
1522
1722
|
* }
|
|
@@ -1539,12 +1739,12 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1539
1739
|
* **Examples:**
|
|
1540
1740
|
* ```typescript
|
|
1541
1741
|
* // Stops execution at this point when the element does exist.
|
|
1542
|
-
* await aui.expect().text(
|
|
1742
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1543
1743
|
*
|
|
1544
1744
|
* // This will catch the error and log a message
|
|
1545
1745
|
* // But the execution will continue afterwards
|
|
1546
1746
|
* try {
|
|
1547
|
-
* await aui.expect().text(
|
|
1747
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1548
1748
|
* } catch (error) {
|
|
1549
1749
|
* console.log('Too bad we could find the element!');
|
|
1550
1750
|
* }
|
|
@@ -1574,8 +1774,8 @@ class FluentCommand extends FluentBase {
|
|
|
1574
1774
|
*
|
|
1575
1775
|
* **Examples:**
|
|
1576
1776
|
* ```typescript
|
|
1577
|
-
* await aui.expect().text(
|
|
1578
|
-
* await aui.expect().text(
|
|
1777
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1778
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1579
1779
|
* ```
|
|
1580
1780
|
*
|
|
1581
1781
|
* @return {FluentFiltersCondition}
|
|
@@ -1592,9 +1792,13 @@ class FluentCommand extends FluentBase {
|
|
|
1592
1792
|
*
|
|
1593
1793
|
* **Example:**
|
|
1594
1794
|
* ```typescript
|
|
1595
|
-
* await aui.click().button().withText('
|
|
1795
|
+
* await aui.click().button().withText('Google Search').exec();
|
|
1596
1796
|
* ```
|
|
1597
1797
|
*
|
|
1798
|
+
* 
|
|
1799
|
+
*
|
|
1800
|
+
*
|
|
1801
|
+
*
|
|
1598
1802
|
* @return {FluentFilters}
|
|
1599
1803
|
*/
|
|
1600
1804
|
click() {
|
|
@@ -1608,9 +1812,13 @@ class FluentCommand extends FluentBase {
|
|
|
1608
1812
|
*
|
|
1609
1813
|
* **Example:**
|
|
1610
1814
|
* ```typescript
|
|
1611
|
-
* await aui.moveMouseTo().
|
|
1815
|
+
* await aui.moveMouseTo().text().withText('Grinning_Face').exec()
|
|
1612
1816
|
* ```
|
|
1613
1817
|
*
|
|
1818
|
+
* 
|
|
1819
|
+
*
|
|
1820
|
+
*
|
|
1821
|
+
*
|
|
1614
1822
|
* @return {FluentFilters}
|
|
1615
1823
|
*/
|
|
1616
1824
|
moveMouseTo() {
|
|
@@ -1652,9 +1860,11 @@ class FluentCommand extends FluentBase {
|
|
|
1652
1860
|
*
|
|
1653
1861
|
* **Example:**
|
|
1654
1862
|
* ```typescript
|
|
1655
|
-
* await aui.
|
|
1863
|
+
* await aui.scrollInside(0,-500).text().withText('Bottom sheet').exec();
|
|
1656
1864
|
* ```
|
|
1657
1865
|
*
|
|
1866
|
+
* 
|
|
1867
|
+
*
|
|
1658
1868
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1659
1869
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1660
1870
|
*
|
|
@@ -1731,12 +1941,14 @@ class FluentCommand extends FluentBase {
|
|
|
1731
1941
|
*
|
|
1732
1942
|
* **Examples:**
|
|
1733
1943
|
* ```typescript
|
|
1734
|
-
* await aui.type('
|
|
1944
|
+
* await aui.type('askui@askui.com').exec()
|
|
1735
1945
|
*
|
|
1736
1946
|
* // mask the text so it is not send to the askui-inference server
|
|
1737
1947
|
* await aui.type('Type some text', { isSecret: true, secretMask: '**' }).exec()
|
|
1738
1948
|
* ```
|
|
1739
1949
|
*
|
|
1950
|
+
* 
|
|
1951
|
+
*
|
|
1740
1952
|
* @param {string} text - A text to type
|
|
1741
1953
|
*
|
|
1742
1954
|
* @return {Exec}
|
|
@@ -1752,9 +1964,11 @@ class FluentCommand extends FluentBase {
|
|
|
1752
1964
|
*
|
|
1753
1965
|
* **Example:**
|
|
1754
1966
|
* ```typescript
|
|
1755
|
-
* await aui.moveMouseRelatively(
|
|
1967
|
+
* await aui.moveMouseRelatively(0, 50).exec();
|
|
1756
1968
|
* ```
|
|
1757
1969
|
*
|
|
1970
|
+
* 
|
|
1971
|
+
*
|
|
1758
1972
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1759
1973
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1760
1974
|
*
|
|
@@ -1784,6 +1998,8 @@ class FluentCommand extends FluentBase {
|
|
|
1784
1998
|
* await aui.moveMouse(500, 500).exec();
|
|
1785
1999
|
* ```
|
|
1786
2000
|
*
|
|
2001
|
+
* 
|
|
2002
|
+
*
|
|
1787
2003
|
* @param {number} x_coordinate - A (positive/negative) x coordinate.
|
|
1788
2004
|
* @param {number} y_coordinate - A (positive/negative) y coordinate.
|
|
1789
2005
|
*
|
|
@@ -1809,10 +2025,12 @@ class FluentCommand extends FluentBase {
|
|
|
1809
2025
|
*
|
|
1810
2026
|
* **Example:**
|
|
1811
2027
|
* ```typescript
|
|
1812
|
-
* // Scroll
|
|
1813
|
-
* await aui.scroll(0,
|
|
2028
|
+
* // Scroll 500 pixels down in y direction
|
|
2029
|
+
* await aui.scroll(0, -500).exec()
|
|
1814
2030
|
* ```
|
|
1815
2031
|
*
|
|
2032
|
+
* 
|
|
2033
|
+
*
|
|
1816
2034
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1817
2035
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1818
2036
|
*
|
|
@@ -1930,6 +2148,8 @@ class FluentCommand extends FluentBase {
|
|
|
1930
2148
|
* await aui.mouseDoubleLeftClick().exec();
|
|
1931
2149
|
* ```
|
|
1932
2150
|
*
|
|
2151
|
+
* 
|
|
2152
|
+
*
|
|
1933
2153
|
* @return {Exec}
|
|
1934
2154
|
*/
|
|
1935
2155
|
mouseDoubleLeftClick() {
|
|
@@ -1980,11 +2200,17 @@ class FluentCommand extends FluentBase {
|
|
|
1980
2200
|
/**
|
|
1981
2201
|
* Toggles mouse down (Left mouse key/tap).
|
|
1982
2202
|
*
|
|
2203
|
+
* This is the equivalent to **mouse-left-press-and-hold**. It holds the mouse button until the `mouseToogleUp()` is called. Often combined with `mouseToggleUP` to automate **drag-and-drop**.
|
|
2204
|
+
*
|
|
1983
2205
|
* **Example:**
|
|
1984
2206
|
* ```typescript
|
|
1985
2207
|
* await aui.mouseToggleDown().exec();
|
|
2208
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
2209
|
+
* await aui.mouseToggleUp().exec();
|
|
1986
2210
|
* ```
|
|
1987
2211
|
*
|
|
2212
|
+
* 
|
|
2213
|
+
*
|
|
1988
2214
|
* @return {Exec}
|
|
1989
2215
|
*/
|
|
1990
2216
|
mouseToggleDown() {
|
|
@@ -1995,11 +2221,17 @@ class FluentCommand extends FluentBase {
|
|
|
1995
2221
|
/**
|
|
1996
2222
|
* Toggles mouse up (Left mouse key/tap).
|
|
1997
2223
|
*
|
|
2224
|
+
* This is the equivalent to releasing the pressing mouse left button. Often combined with `mouseToggleDown()` to automate **drag-and-drop**.
|
|
2225
|
+
*
|
|
1998
2226
|
* **Example:**
|
|
1999
2227
|
* ```typescript
|
|
2228
|
+
* await aui.mouseToggleDown().exec();
|
|
2229
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
2000
2230
|
* await aui.mouseToggleUp().exec();
|
|
2001
2231
|
* ```
|
|
2002
2232
|
*
|
|
2233
|
+
* 
|
|
2234
|
+
*
|
|
2003
2235
|
* @return {Exec}
|
|
2004
2236
|
*/
|
|
2005
2237
|
mouseToggleUp() {
|
|
@@ -2012,8 +2244,13 @@ class FluentCommand extends FluentBase {
|
|
|
2012
2244
|
*
|
|
2013
2245
|
* **Operating system specific mappings:**
|
|
2014
2246
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
2015
|
-
* ---
|
|
2016
2247
|
*
|
|
2248
|
+
* **Examples:**
|
|
2249
|
+
* ```typescript
|
|
2250
|
+
* await aui.pressThreeKeys('control', 'command' 'space').exec();
|
|
2251
|
+
* ```
|
|
2252
|
+
*
|
|
2253
|
+
* 
|
|
2017
2254
|
*
|
|
2018
2255
|
* @param {MODIFIER_KEY} first_key - A modifier key
|
|
2019
2256
|
* @param {MODIFIER_KEY} second_key - A modifier key
|
|
@@ -2056,8 +2293,13 @@ class FluentCommand extends FluentBase {
|
|
|
2056
2293
|
*
|
|
2057
2294
|
* **Operating system specific mappings:**
|
|
2058
2295
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
2059
|
-
* ---
|
|
2060
2296
|
*
|
|
2297
|
+
* **Examples:**
|
|
2298
|
+
* ```typescript
|
|
2299
|
+
* await aui.pressKey('tab').exec();
|
|
2300
|
+
* ```
|
|
2301
|
+
*
|
|
2302
|
+
* 
|
|
2061
2303
|
*
|
|
2062
2304
|
* @param {PC_AND_MODIFIER_KEY} key - A key
|
|
2063
2305
|
*
|
|
@@ -2090,7 +2332,15 @@ class FluentCommand extends FluentBase {
|
|
|
2090
2332
|
return new Exec(this);
|
|
2091
2333
|
}
|
|
2092
2334
|
/**
|
|
2093
|
-
* Press two Android keys like `
|
|
2335
|
+
* Press two Android keys like `volume_down+power`
|
|
2336
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
2337
|
+
*
|
|
2338
|
+
* **Examples:**
|
|
2339
|
+
* ```typescript
|
|
2340
|
+
* await aui.pressAndroidTwoKey('volume_down', 'power').exec();
|
|
2341
|
+
* ```
|
|
2342
|
+
*
|
|
2343
|
+
* 
|
|
2094
2344
|
*
|
|
2095
2345
|
* @param {ANDROID_KEY} first_key - A Android key
|
|
2096
2346
|
* @param {ANDROID_KEY} second_key - A Android key
|
|
@@ -2107,7 +2357,15 @@ class FluentCommand extends FluentBase {
|
|
|
2107
2357
|
return new Exec(this);
|
|
2108
2358
|
}
|
|
2109
2359
|
/**
|
|
2110
|
-
* Press one Android key like `
|
|
2360
|
+
* Press one Android key like `del`
|
|
2361
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
2362
|
+
*
|
|
2363
|
+
* **Examples:**
|
|
2364
|
+
* ```typescript
|
|
2365
|
+
* await aui.pressAndroidKey('notification').exec();
|
|
2366
|
+
* ```
|
|
2367
|
+
*
|
|
2368
|
+
* 
|
|
2111
2369
|
*
|
|
2112
2370
|
* @param {ANDROID_KEY} key - A Android key
|
|
2113
2371
|
*
|
|
@@ -2216,9 +2474,11 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2216
2474
|
*
|
|
2217
2475
|
* **Examples:**
|
|
2218
2476
|
* ```typescript
|
|
2219
|
-
* await aui.
|
|
2477
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
2220
2478
|
* ```
|
|
2221
2479
|
*
|
|
2480
|
+
* 
|
|
2481
|
+
*
|
|
2222
2482
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2223
2483
|
*/
|
|
2224
2484
|
button() {
|
|
@@ -2278,13 +2538,20 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2278
2538
|
/**
|
|
2279
2539
|
* Filters for an UI element 'text'.
|
|
2280
2540
|
*
|
|
2281
|
-
*
|
|
2541
|
+
* Takes an optional parameter to filter for a specific text.
|
|
2542
|
+
* See the examples below.
|
|
2543
|
+
*
|
|
2282
2544
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
2283
2545
|
*
|
|
2284
2546
|
* **Examples:**
|
|
2285
2547
|
* ```typescript
|
|
2286
|
-
* await aui.click().text().
|
|
2548
|
+
* await aui.click().text().exec();
|
|
2549
|
+
* await aui.click().text('Username').exec();
|
|
2550
|
+
*
|
|
2551
|
+
* // Matching with an exact text
|
|
2287
2552
|
* await aui.click().text().withExactText('Username').exec();
|
|
2553
|
+
*
|
|
2554
|
+
* // Matching with a regex
|
|
2288
2555
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
2289
2556
|
* ```
|
|
2290
2557
|
*
|
|
@@ -2308,6 +2575,8 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2308
2575
|
* icon().withText('plus')
|
|
2309
2576
|
* ```
|
|
2310
2577
|
*
|
|
2578
|
+
* 
|
|
2579
|
+
*
|
|
2311
2580
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
2312
2581
|
*
|
|
2313
2582
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2318,7 +2587,9 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2318
2587
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2319
2588
|
}
|
|
2320
2589
|
/**
|
|
2321
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
2590
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
2591
|
+
*
|
|
2592
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
2322
2593
|
*
|
|
2323
2594
|
* **Example**
|
|
2324
2595
|
* ```typescript
|
|
@@ -2328,6 +2599,7 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2328
2599
|
* customImage: './logo.png', // required
|
|
2329
2600
|
* name: 'myLogo', // optional
|
|
2330
2601
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
2602
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
2331
2603
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
2332
2604
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
2333
2605
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -2342,12 +2614,13 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2342
2614
|
* - **name** (*`string`, optional*):
|
|
2343
2615
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
2344
2616
|
* - **threshold** (*`number`, optional*):
|
|
2345
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
2617
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
2618
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
2619
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
2346
2620
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
2347
2621
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
2348
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
2349
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
2350
|
-
* of the given custom image.
|
|
2622
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
2623
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
2351
2624
|
*
|
|
2352
2625
|
*
|
|
2353
2626
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -2361,6 +2634,23 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2361
2634
|
this._params.set('customElement', customElement);
|
|
2362
2635
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2363
2636
|
}
|
|
2637
|
+
/**
|
|
2638
|
+
* Detects an AI Element created with the workflow creator.
|
|
2639
|
+
*
|
|
2640
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
2641
|
+
*
|
|
2642
|
+
* @return {FluentFiltersOrRelationsGetter}
|
|
2643
|
+
*/
|
|
2644
|
+
aiElement(aiElementName) {
|
|
2645
|
+
this._textStr = '';
|
|
2646
|
+
this._textStr += 'ai';
|
|
2647
|
+
this._textStr += ' element';
|
|
2648
|
+
this._textStr += ' with';
|
|
2649
|
+
this._textStr += ' name';
|
|
2650
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
2651
|
+
this._params.set('aiElementName', aiElementName);
|
|
2652
|
+
return new FluentFiltersOrRelationsGetter(this);
|
|
2653
|
+
}
|
|
2364
2654
|
/**
|
|
2365
2655
|
* Filters for a UI element 'image'.
|
|
2366
2656
|
*
|
|
@@ -2370,10 +2660,14 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2370
2660
|
* await aui.click().image().exec();
|
|
2371
2661
|
*
|
|
2372
2662
|
* // Works if you have an image with
|
|
2373
|
-
* // a
|
|
2374
|
-
* await aui.click().image().above().text().withText('
|
|
2663
|
+
* // a text below
|
|
2664
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
2375
2665
|
* ```
|
|
2376
2666
|
*
|
|
2667
|
+
* 
|
|
2668
|
+
*
|
|
2669
|
+
*
|
|
2670
|
+
*
|
|
2377
2671
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2378
2672
|
*/
|
|
2379
2673
|
image() {
|
|
@@ -2394,6 +2688,10 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2394
2688
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
2395
2689
|
* ```
|
|
2396
2690
|
*
|
|
2691
|
+
* 
|
|
2692
|
+
*
|
|
2693
|
+
*
|
|
2694
|
+
*
|
|
2397
2695
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2398
2696
|
*/
|
|
2399
2697
|
textfield() {
|
|
@@ -2402,7 +2700,13 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2402
2700
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2403
2701
|
}
|
|
2404
2702
|
/**
|
|
2405
|
-
* Filters for similar
|
|
2703
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
2704
|
+
*
|
|
2705
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
2706
|
+
*
|
|
2707
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
2708
|
+
*
|
|
2709
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
2406
2710
|
*
|
|
2407
2711
|
* **Examples:**
|
|
2408
2712
|
* ```typescript
|
|
@@ -2417,6 +2721,11 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2417
2721
|
* // usually false
|
|
2418
2722
|
* 'atebxtc' === withText('text') => false
|
|
2419
2723
|
* 'other' === withText('text') => false
|
|
2724
|
+
*
|
|
2725
|
+
* // optional parameter: similarity_score
|
|
2726
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
2727
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
2728
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
2420
2729
|
* ```
|
|
2421
2730
|
* 
|
|
2422
2731
|
*
|
|
@@ -2447,6 +2756,10 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2447
2756
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
2448
2757
|
* ```
|
|
2449
2758
|
*
|
|
2759
|
+
* 
|
|
2760
|
+
*
|
|
2761
|
+
*
|
|
2762
|
+
*
|
|
2450
2763
|
* @param {string} regex_pattern - A regex pattern
|
|
2451
2764
|
*
|
|
2452
2765
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2475,6 +2788,10 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2475
2788
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
2476
2789
|
* ```
|
|
2477
2790
|
*
|
|
2791
|
+
* 
|
|
2792
|
+
*
|
|
2793
|
+
*
|
|
2794
|
+
*
|
|
2478
2795
|
* @param {string} text - A text to be matched.
|
|
2479
2796
|
*
|
|
2480
2797
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2511,17 +2828,28 @@ class FluentFiltersGetter extends FluentBase {
|
|
|
2511
2828
|
/**
|
|
2512
2829
|
* Filters elements based on a textual description.
|
|
2513
2830
|
*
|
|
2514
|
-
*
|
|
2831
|
+
* **What Should I Write as Matching Text**
|
|
2832
|
+
*
|
|
2515
2833
|
* The text description inside the `matching()` should describe the element visually.
|
|
2516
2834
|
* It understands color, some famous company/product names, general descriptions.
|
|
2517
2835
|
*
|
|
2518
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
2519
|
-
* E.g. `puzzle piece` can fail
|
|
2836
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
2837
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
2520
2838
|
* Generally the more detail the better.
|
|
2521
2839
|
*
|
|
2522
2840
|
* **Examples:**
|
|
2523
2841
|
* ```typescript
|
|
2524
|
-
*
|
|
2842
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
2843
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
2844
|
+
*
|
|
2845
|
+
* // Select an image that has text in it
|
|
2846
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
2847
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
2848
|
+
*
|
|
2849
|
+
* // Target a logo/image by describing it
|
|
2850
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
2851
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
2852
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
2525
2853
|
* ```
|
|
2526
2854
|
*
|
|
2527
2855
|
* @param {string} text - A description of the target element.
|
|
@@ -2680,14 +3008,19 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2680
3008
|
/**
|
|
2681
3009
|
* Filters for an element right of another element.
|
|
2682
3010
|
*
|
|
3011
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3012
|
+
*
|
|
2683
3013
|
* **Examples:**
|
|
2684
3014
|
* ```typescript
|
|
2685
|
-
* -------------- --------------
|
|
2686
|
-
* | leftEl | |
|
|
2687
|
-
* -------------- --------------
|
|
3015
|
+
* -------------- -------------- --------------
|
|
3016
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
3017
|
+
* -------------- -------------- --------------
|
|
2688
3018
|
*
|
|
2689
|
-
* // Returns
|
|
3019
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
2690
3020
|
* ...rightEl().rightOf().leftEl()
|
|
3021
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
3022
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
3023
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
2691
3024
|
* // Returns no element because leftEl is left of rightEl
|
|
2692
3025
|
* ...leftEl().rightOf().rightEl()
|
|
2693
3026
|
* ```
|
|
@@ -2707,14 +3040,19 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2707
3040
|
/**
|
|
2708
3041
|
* Filters for an element left of another element.
|
|
2709
3042
|
*
|
|
3043
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3044
|
+
*
|
|
2710
3045
|
* **Examples:**
|
|
2711
3046
|
* ```typescript
|
|
2712
|
-
* -------------- --------------
|
|
2713
|
-
* |
|
|
2714
|
-
* -------------- --------------
|
|
3047
|
+
* -------------- -------------- --------------
|
|
3048
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
3049
|
+
* -------------- -------------- --------------
|
|
2715
3050
|
*
|
|
2716
|
-
* // Returns
|
|
3051
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
2717
3052
|
* ...leftEl().leftOf().rightEl()
|
|
3053
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
3054
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
3055
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
2718
3056
|
* // Returns no element because rightEl is left of leftEl
|
|
2719
3057
|
* ...rightEl().leftOf().leftEl()
|
|
2720
3058
|
* ```
|
|
@@ -2734,17 +3072,25 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2734
3072
|
/**
|
|
2735
3073
|
* Filters for an element below another element.
|
|
2736
3074
|
*
|
|
3075
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3076
|
+
*
|
|
2737
3077
|
* **Examples:**
|
|
2738
3078
|
* ```typescript
|
|
2739
3079
|
* --------------
|
|
2740
3080
|
* | text |
|
|
2741
3081
|
* --------------
|
|
2742
3082
|
* --------------
|
|
2743
|
-
* |
|
|
3083
|
+
* | button0 |
|
|
3084
|
+
* --------------
|
|
3085
|
+
* --------------
|
|
3086
|
+
* | button1 |
|
|
2744
3087
|
* --------------
|
|
2745
3088
|
*
|
|
2746
|
-
* // Returns
|
|
3089
|
+
* // Returns button0 because button0 is the first button below text
|
|
2747
3090
|
* ...button().below().text()
|
|
3091
|
+
* ...button().below(0).text()
|
|
3092
|
+
* // Returns button1 because button1 is the second button below text
|
|
3093
|
+
* ...button().below(1).text()
|
|
2748
3094
|
* // Returns no element because text is above button
|
|
2749
3095
|
* ...text().below().button()
|
|
2750
3096
|
* ```
|
|
@@ -2763,17 +3109,25 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2763
3109
|
/**
|
|
2764
3110
|
* Filters for an element above another element.
|
|
2765
3111
|
*
|
|
3112
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3113
|
+
*
|
|
2766
3114
|
* **Examples:**
|
|
2767
3115
|
* ```typescript
|
|
2768
3116
|
* --------------
|
|
2769
|
-
* |
|
|
3117
|
+
* | text1 |
|
|
3118
|
+
* --------------
|
|
3119
|
+
* --------------
|
|
3120
|
+
* | text0 |
|
|
2770
3121
|
* --------------
|
|
2771
3122
|
* --------------
|
|
2772
3123
|
* | button |
|
|
2773
3124
|
* --------------
|
|
2774
3125
|
*
|
|
2775
|
-
* // Returns
|
|
3126
|
+
* // Returns text0 because text0 is the first element above button
|
|
2776
3127
|
* ...text().above().button()
|
|
3128
|
+
* ...text().above(0).button()
|
|
3129
|
+
* // Returns text1 because text1 is the second element above button
|
|
3130
|
+
* ...text().above(1).button()
|
|
2777
3131
|
* // Returns no element because button is below text
|
|
2778
3132
|
* ...button().above().text()
|
|
2779
3133
|
* ```
|
|
@@ -2868,22 +3222,74 @@ class Getter extends FluentCommand {
|
|
|
2868
3222
|
*
|
|
2869
3223
|
* **Examples:**
|
|
2870
3224
|
* ```typescript
|
|
2871
|
-
* const text = await aui.get().text(
|
|
3225
|
+
* const text = await aui.get().text('Sign').exec();
|
|
2872
3226
|
* console.log(text);
|
|
3227
|
+
*
|
|
3228
|
+
* // Console output
|
|
3229
|
+
* [
|
|
3230
|
+
* DetectedElement {
|
|
3231
|
+
* name: 'TEXT',
|
|
3232
|
+
* text: 'Sign In',
|
|
3233
|
+
* bndbox: BoundingBox {
|
|
3234
|
+
* xmin: 1128.2720982142857,
|
|
3235
|
+
* ymin: 160.21332310267857,
|
|
3236
|
+
* xmax: 1178.8204241071428,
|
|
3237
|
+
* ymax: 180.83512834821428
|
|
3238
|
+
* }
|
|
3239
|
+
* }
|
|
3240
|
+
* ]
|
|
2873
3241
|
* ```
|
|
2874
|
-
*
|
|
2875
|
-
*
|
|
3242
|
+
*
|
|
3243
|
+
* // *************************************************** //
|
|
3244
|
+
* // Examples on how to work with the returned elements //
|
|
3245
|
+
* // *************************************************** //
|
|
3246
|
+
* const texts = await aui.get().text().below().textfield().exec();
|
|
3247
|
+
*
|
|
3248
|
+
* // We can get a lot of elements this way
|
|
3249
|
+
* console.log(texts);
|
|
3250
|
+
*
|
|
3251
|
+
* // Console output
|
|
3252
|
+
* [
|
|
2876
3253
|
* DetectedElement {
|
|
2877
|
-
*
|
|
2878
|
-
*
|
|
2879
|
-
*
|
|
2880
|
-
*
|
|
2881
|
-
*
|
|
2882
|
-
*
|
|
2883
|
-
*
|
|
2884
|
-
*
|
|
2885
|
-
*
|
|
2886
|
-
*
|
|
3254
|
+
* name: 'TEXT',
|
|
3255
|
+
* text: 'Sign In',
|
|
3256
|
+
* bndbox: BoundingBox {
|
|
3257
|
+
* xmin: 1128.2720982142857,
|
|
3258
|
+
* ymin: 160.21332310267857,
|
|
3259
|
+
* xmax: 1178.8204241071428,
|
|
3260
|
+
* ymax: 180.83512834821428
|
|
3261
|
+
* },
|
|
3262
|
+
* },
|
|
3263
|
+
* DetectedElement {
|
|
3264
|
+
* name: 'TEXT',
|
|
3265
|
+
* text: 'Login',
|
|
3266
|
+
* bndbox: BoundingBox {
|
|
3267
|
+
* xmin: 250.8204241071428,
|
|
3268
|
+
* ymin: 300.21332310267857,
|
|
3269
|
+
* xmax: 450.6304241071428,
|
|
3270
|
+
* ymax: 950.47812834821428
|
|
3271
|
+
* },
|
|
3272
|
+
* },
|
|
3273
|
+
* ... 10 more items
|
|
3274
|
+
* ]
|
|
3275
|
+
*
|
|
3276
|
+
* // Extract the FIRST element
|
|
3277
|
+
* // Arrays start with index 0!
|
|
3278
|
+
* const firstTextElement = texts[0];
|
|
3279
|
+
* const textOfFirstElement = firstElement.text;
|
|
3280
|
+
*
|
|
3281
|
+
* console.log(textOfFirstElement);
|
|
3282
|
+
*
|
|
3283
|
+
* // Console output
|
|
3284
|
+
* Sign In
|
|
3285
|
+
*
|
|
3286
|
+
* // Log the text of the SECOND element
|
|
3287
|
+
* // with shorter code
|
|
3288
|
+
* const texts = await aui.get().text().below().textfield().exec();
|
|
3289
|
+
* console.log(texts[1].text)
|
|
3290
|
+
*
|
|
3291
|
+
* // Console output
|
|
3292
|
+
* Login
|
|
2887
3293
|
* ```
|
|
2888
3294
|
*
|
|
2889
3295
|
* @return {FluentFiltersGetter}
|