askui 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/model/custom-element-json.d.ts +32 -15
- package/dist/cjs/core/model/custom-element.d.ts +3 -2
- package/dist/cjs/core/model/custom-element.js +4 -2
- package/dist/cjs/core/reporting/step-reporter.js +15 -6
- package/dist/cjs/execution/dsl.d.ts +361 -96
- package/dist/cjs/execution/dsl.js +361 -96
- package/dist/cjs/execution/index.d.ts +1 -1
- package/dist/cjs/execution/inference-client.js +9 -4
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +3 -3
- package/dist/cjs/execution/ui-control-client.d.ts +154 -13
- package/dist/cjs/execution/ui-control-client.js +219 -21
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/cjs/execution/ui-controller-client.d.ts +1 -0
- package/dist/cjs/execution/ui-controller-client.js +11 -1
- package/dist/cjs/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/cjs/execution/ui-controller-not-connected-error.js +12 -0
- package/dist/cjs/lib/interactive_cli/create-example-project.js +1 -1
- package/dist/cjs/main.d.ts +1 -1
- package/dist/cjs/utils/analytics/analytics.d.ts +2 -0
- package/dist/cjs/utils/analytics/analytics.js +5 -0
- package/dist/cjs/utils/http/custom-errors/index.js +1 -1
- package/dist/cjs/utils/http/http-client-got.js +46 -20
- package/dist/esm/core/model/custom-element-json.d.ts +32 -15
- package/dist/esm/core/model/custom-element.d.ts +3 -2
- package/dist/esm/core/model/custom-element.js +4 -2
- package/dist/esm/core/reporting/step-reporter.js +15 -6
- package/dist/esm/execution/dsl.d.ts +361 -96
- package/dist/esm/execution/dsl.js +361 -96
- package/dist/esm/execution/index.d.ts +1 -1
- package/dist/esm/execution/inference-client.js +9 -4
- package/dist/esm/execution/ui-control-client-dependency-builder.js +3 -3
- package/dist/esm/execution/ui-control-client.d.ts +154 -13
- package/dist/esm/execution/ui-control-client.js +216 -21
- package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/esm/execution/ui-controller-client.d.ts +1 -0
- package/dist/esm/execution/ui-controller-client.js +11 -1
- package/dist/esm/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/esm/execution/ui-controller-not-connected-error.js +8 -0
- package/dist/esm/lib/interactive_cli/create-example-project.js +1 -1
- package/dist/esm/main.d.ts +1 -1
- package/dist/esm/utils/analytics/analytics.d.ts +2 -0
- package/dist/esm/utils/analytics/analytics.js +5 -0
- package/dist/esm/utils/http/custom-errors/index.js +1 -1
- package/dist/esm/utils/http/http-client-got.js +46 -20
- package/dist/example_projects_templates/typescript/.eslintrc.json-template +3 -2
- package/package.json +1 -1
|
@@ -85,9 +85,11 @@ export declare class FluentFilters extends FluentBase {
|
|
|
85
85
|
*
|
|
86
86
|
* **Examples:**
|
|
87
87
|
* ```typescript
|
|
88
|
-
* await aui.
|
|
88
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
89
89
|
* ```
|
|
90
90
|
*
|
|
91
|
+
* 
|
|
92
|
+
*
|
|
91
93
|
* @return {FluentFiltersOrRelations}
|
|
92
94
|
*/
|
|
93
95
|
button(): FluentFiltersOrRelations;
|
|
@@ -121,13 +123,20 @@ export declare class FluentFilters extends FluentBase {
|
|
|
121
123
|
/**
|
|
122
124
|
* Filters for an UI element 'text'.
|
|
123
125
|
*
|
|
124
|
-
*
|
|
126
|
+
* Takes an optional parameter to filter for a specific text.
|
|
127
|
+
* See the examples below.
|
|
128
|
+
*
|
|
125
129
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
126
130
|
*
|
|
127
131
|
* **Examples:**
|
|
128
132
|
* ```typescript
|
|
129
|
-
* await aui.click().text().
|
|
133
|
+
* await aui.click().text().exec();
|
|
134
|
+
* await aui.click().text('Username').exec();
|
|
135
|
+
*
|
|
136
|
+
* // Matching with an exact text
|
|
130
137
|
* await aui.click().text().withExactText('Username').exec();
|
|
138
|
+
*
|
|
139
|
+
* // Matching with a regex
|
|
131
140
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
132
141
|
* ```
|
|
133
142
|
*
|
|
@@ -144,13 +153,17 @@ export declare class FluentFilters extends FluentBase {
|
|
|
144
153
|
* icon().withText('plus')
|
|
145
154
|
* ```
|
|
146
155
|
*
|
|
156
|
+
* 
|
|
157
|
+
*
|
|
147
158
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
148
159
|
*
|
|
149
160
|
* @return {FluentFiltersOrRelations}
|
|
150
161
|
*/
|
|
151
162
|
icon(): FluentFiltersOrRelations;
|
|
152
163
|
/**
|
|
153
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
164
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
165
|
+
*
|
|
166
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
154
167
|
*
|
|
155
168
|
* **Example**
|
|
156
169
|
* ```typescript
|
|
@@ -160,6 +173,7 @@ export declare class FluentFilters extends FluentBase {
|
|
|
160
173
|
* customImage: './logo.png', // required
|
|
161
174
|
* name: 'myLogo', // optional
|
|
162
175
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
176
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
163
177
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
164
178
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
165
179
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -174,12 +188,13 @@ export declare class FluentFilters extends FluentBase {
|
|
|
174
188
|
* - **name** (*`string`, optional*):
|
|
175
189
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
176
190
|
* - **threshold** (*`number`, optional*):
|
|
177
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
191
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
192
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
193
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
178
194
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
179
195
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
180
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
181
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
182
|
-
* of the given custom image.
|
|
196
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
197
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
183
198
|
*
|
|
184
199
|
*
|
|
185
200
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -196,10 +211,14 @@ export declare class FluentFilters extends FluentBase {
|
|
|
196
211
|
* await aui.click().image().exec();
|
|
197
212
|
*
|
|
198
213
|
* // Works if you have an image with
|
|
199
|
-
* // a
|
|
200
|
-
* await aui.click().image().above().text().withText('
|
|
214
|
+
* // a text below
|
|
215
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
201
216
|
* ```
|
|
202
217
|
*
|
|
218
|
+
* 
|
|
219
|
+
*
|
|
220
|
+
*
|
|
221
|
+
*
|
|
203
222
|
* @return {FluentFiltersOrRelations}
|
|
204
223
|
*/
|
|
205
224
|
image(): FluentFiltersOrRelations;
|
|
@@ -216,11 +235,21 @@ export declare class FluentFilters extends FluentBase {
|
|
|
216
235
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
217
236
|
* ```
|
|
218
237
|
*
|
|
238
|
+
* 
|
|
239
|
+
*
|
|
240
|
+
*
|
|
241
|
+
*
|
|
219
242
|
* @return {FluentFiltersOrRelations}
|
|
220
243
|
*/
|
|
221
244
|
textfield(): FluentFiltersOrRelations;
|
|
222
245
|
/**
|
|
223
|
-
* Filters for similar
|
|
246
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
247
|
+
*
|
|
248
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
249
|
+
*
|
|
250
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
251
|
+
*
|
|
252
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
224
253
|
*
|
|
225
254
|
* **Examples:**
|
|
226
255
|
* ```typescript
|
|
@@ -235,6 +264,11 @@ export declare class FluentFilters extends FluentBase {
|
|
|
235
264
|
* // usually false
|
|
236
265
|
* 'atebxtc' === withText('text') => false
|
|
237
266
|
* 'other' === withText('text') => false
|
|
267
|
+
*
|
|
268
|
+
* // optional parameter: similarity_score
|
|
269
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
270
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
271
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
238
272
|
* ```
|
|
239
273
|
* 
|
|
240
274
|
*
|
|
@@ -256,6 +290,10 @@ export declare class FluentFilters extends FluentBase {
|
|
|
256
290
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
257
291
|
* ```
|
|
258
292
|
*
|
|
293
|
+
* 
|
|
294
|
+
*
|
|
295
|
+
*
|
|
296
|
+
*
|
|
259
297
|
* @param {string} regex_pattern - A regex pattern
|
|
260
298
|
*
|
|
261
299
|
* @return {FluentFiltersOrRelations}
|
|
@@ -277,6 +315,10 @@ export declare class FluentFilters extends FluentBase {
|
|
|
277
315
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
278
316
|
* ```
|
|
279
317
|
*
|
|
318
|
+
* 
|
|
319
|
+
*
|
|
320
|
+
*
|
|
321
|
+
*
|
|
280
322
|
* @param {string} text - A text to be matched.
|
|
281
323
|
*
|
|
282
324
|
* @return {FluentFiltersOrRelations}
|
|
@@ -301,12 +343,13 @@ export declare class FluentFilters extends FluentBase {
|
|
|
301
343
|
/**
|
|
302
344
|
* Filters elements based on a textual description.
|
|
303
345
|
*
|
|
304
|
-
*
|
|
346
|
+
* **What Should I Write as Matching Text**
|
|
347
|
+
*
|
|
305
348
|
* The text description inside the `matching()` should describe the element visually.
|
|
306
349
|
* It understands color, some famous company/product names, general descriptions.
|
|
307
350
|
*
|
|
308
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
309
|
-
* E.g. `puzzle piece` can fail
|
|
351
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
352
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
310
353
|
* Generally the more detail the better.
|
|
311
354
|
*
|
|
312
355
|
* **Examples:**
|
|
@@ -451,14 +494,19 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
|
|
|
451
494
|
/**
|
|
452
495
|
* Filters for an element right of another element.
|
|
453
496
|
*
|
|
497
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
498
|
+
*
|
|
454
499
|
* **Examples:**
|
|
455
500
|
* ```typescript
|
|
456
|
-
* -------------- --------------
|
|
457
|
-
* | leftEl | |
|
|
458
|
-
* -------------- --------------
|
|
501
|
+
* -------------- -------------- --------------
|
|
502
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
503
|
+
* -------------- -------------- --------------
|
|
459
504
|
*
|
|
460
|
-
* // Returns
|
|
505
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
461
506
|
* ...rightEl().rightOf().leftEl()
|
|
507
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
508
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
509
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
462
510
|
* // Returns no element because leftEl is left of rightEl
|
|
463
511
|
* ...leftEl().rightOf().rightEl()
|
|
464
512
|
* ```
|
|
@@ -470,14 +518,19 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
|
|
|
470
518
|
/**
|
|
471
519
|
* Filters for an element left of another element.
|
|
472
520
|
*
|
|
521
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
522
|
+
*
|
|
473
523
|
* **Examples:**
|
|
474
524
|
* ```typescript
|
|
475
|
-
* -------------- --------------
|
|
476
|
-
* |
|
|
477
|
-
* -------------- --------------
|
|
525
|
+
* -------------- -------------- --------------
|
|
526
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
527
|
+
* -------------- -------------- --------------
|
|
478
528
|
*
|
|
479
|
-
* // Returns
|
|
529
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
480
530
|
* ...leftEl().leftOf().rightEl()
|
|
531
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
532
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
533
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
481
534
|
* // Returns no element because rightEl is left of leftEl
|
|
482
535
|
* ...rightEl().leftOf().leftEl()
|
|
483
536
|
* ```
|
|
@@ -489,17 +542,25 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
|
|
|
489
542
|
/**
|
|
490
543
|
* Filters for an element below another element.
|
|
491
544
|
*
|
|
545
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
546
|
+
*
|
|
492
547
|
* **Examples:**
|
|
493
548
|
* ```typescript
|
|
494
549
|
* --------------
|
|
495
550
|
* | text |
|
|
496
551
|
* --------------
|
|
497
552
|
* --------------
|
|
498
|
-
* |
|
|
553
|
+
* | button0 |
|
|
554
|
+
* --------------
|
|
555
|
+
* --------------
|
|
556
|
+
* | button1 |
|
|
499
557
|
* --------------
|
|
500
558
|
*
|
|
501
|
-
* // Returns
|
|
559
|
+
* // Returns button0 because button0 is the first button below text
|
|
502
560
|
* ...button().below().text()
|
|
561
|
+
* ...button().below(0).text()
|
|
562
|
+
* // Returns button1 because button1 is the second button below text
|
|
563
|
+
* ...button().below(1).text()
|
|
503
564
|
* // Returns no element because text is above button
|
|
504
565
|
* ...text().below().button()
|
|
505
566
|
* ```
|
|
@@ -511,17 +572,25 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
|
|
|
511
572
|
/**
|
|
512
573
|
* Filters for an element above another element.
|
|
513
574
|
*
|
|
575
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
576
|
+
*
|
|
514
577
|
* **Examples:**
|
|
515
578
|
* ```typescript
|
|
516
579
|
* --------------
|
|
517
|
-
* |
|
|
580
|
+
* | text1 |
|
|
581
|
+
* --------------
|
|
582
|
+
* --------------
|
|
583
|
+
* | text0 |
|
|
518
584
|
* --------------
|
|
519
585
|
* --------------
|
|
520
586
|
* | button |
|
|
521
587
|
* --------------
|
|
522
588
|
*
|
|
523
|
-
* // Returns
|
|
589
|
+
* // Returns text0 because text0 is the first element above button
|
|
524
590
|
* ...text().above().button()
|
|
591
|
+
* ...text().above(0).button()
|
|
592
|
+
* // Returns text1 because text1 is the second element above button
|
|
593
|
+
* ...text().above(1).button()
|
|
525
594
|
* // Returns no element because button is below text
|
|
526
595
|
* ...button().above().text()
|
|
527
596
|
* ```
|
|
@@ -641,9 +710,11 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
641
710
|
*
|
|
642
711
|
* **Examples:**
|
|
643
712
|
* ```typescript
|
|
644
|
-
* await aui.
|
|
713
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
645
714
|
* ```
|
|
646
715
|
*
|
|
716
|
+
* 
|
|
717
|
+
*
|
|
647
718
|
* @return {FluentFiltersOrRelationsCondition}
|
|
648
719
|
*/
|
|
649
720
|
button(): FluentFiltersOrRelationsCondition;
|
|
@@ -677,13 +748,20 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
677
748
|
/**
|
|
678
749
|
* Filters for an UI element 'text'.
|
|
679
750
|
*
|
|
680
|
-
*
|
|
751
|
+
* Takes an optional parameter to filter for a specific text.
|
|
752
|
+
* See the examples below.
|
|
753
|
+
*
|
|
681
754
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
682
755
|
*
|
|
683
756
|
* **Examples:**
|
|
684
757
|
* ```typescript
|
|
685
|
-
* await aui.click().text().
|
|
758
|
+
* await aui.click().text().exec();
|
|
759
|
+
* await aui.click().text('Username').exec();
|
|
760
|
+
*
|
|
761
|
+
* // Matching with an exact text
|
|
686
762
|
* await aui.click().text().withExactText('Username').exec();
|
|
763
|
+
*
|
|
764
|
+
* // Matching with a regex
|
|
687
765
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
688
766
|
* ```
|
|
689
767
|
*
|
|
@@ -700,13 +778,17 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
700
778
|
* icon().withText('plus')
|
|
701
779
|
* ```
|
|
702
780
|
*
|
|
781
|
+
* 
|
|
782
|
+
*
|
|
703
783
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
704
784
|
*
|
|
705
785
|
* @return {FluentFiltersOrRelationsCondition}
|
|
706
786
|
*/
|
|
707
787
|
icon(): FluentFiltersOrRelationsCondition;
|
|
708
788
|
/**
|
|
709
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
789
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
790
|
+
*
|
|
791
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
710
792
|
*
|
|
711
793
|
* **Example**
|
|
712
794
|
* ```typescript
|
|
@@ -716,6 +798,7 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
716
798
|
* customImage: './logo.png', // required
|
|
717
799
|
* name: 'myLogo', // optional
|
|
718
800
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
801
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
719
802
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
720
803
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
721
804
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -730,12 +813,13 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
730
813
|
* - **name** (*`string`, optional*):
|
|
731
814
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
732
815
|
* - **threshold** (*`number`, optional*):
|
|
733
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
816
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
817
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
818
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
734
819
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
735
820
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
736
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
737
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
738
|
-
* of the given custom image.
|
|
821
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
822
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
739
823
|
*
|
|
740
824
|
*
|
|
741
825
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -752,10 +836,14 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
752
836
|
* await aui.click().image().exec();
|
|
753
837
|
*
|
|
754
838
|
* // Works if you have an image with
|
|
755
|
-
* // a
|
|
756
|
-
* await aui.click().image().above().text().withText('
|
|
839
|
+
* // a text below
|
|
840
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
757
841
|
* ```
|
|
758
842
|
*
|
|
843
|
+
* 
|
|
844
|
+
*
|
|
845
|
+
*
|
|
846
|
+
*
|
|
759
847
|
* @return {FluentFiltersOrRelationsCondition}
|
|
760
848
|
*/
|
|
761
849
|
image(): FluentFiltersOrRelationsCondition;
|
|
@@ -772,11 +860,21 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
772
860
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
773
861
|
* ```
|
|
774
862
|
*
|
|
863
|
+
* 
|
|
864
|
+
*
|
|
865
|
+
*
|
|
866
|
+
*
|
|
775
867
|
* @return {FluentFiltersOrRelationsCondition}
|
|
776
868
|
*/
|
|
777
869
|
textfield(): FluentFiltersOrRelationsCondition;
|
|
778
870
|
/**
|
|
779
|
-
* Filters for similar
|
|
871
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
872
|
+
*
|
|
873
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
874
|
+
*
|
|
875
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
876
|
+
*
|
|
877
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
780
878
|
*
|
|
781
879
|
* **Examples:**
|
|
782
880
|
* ```typescript
|
|
@@ -791,6 +889,11 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
791
889
|
* // usually false
|
|
792
890
|
* 'atebxtc' === withText('text') => false
|
|
793
891
|
* 'other' === withText('text') => false
|
|
892
|
+
*
|
|
893
|
+
* // optional parameter: similarity_score
|
|
894
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
895
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
896
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
794
897
|
* ```
|
|
795
898
|
* 
|
|
796
899
|
*
|
|
@@ -812,6 +915,10 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
812
915
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
813
916
|
* ```
|
|
814
917
|
*
|
|
918
|
+
* 
|
|
919
|
+
*
|
|
920
|
+
*
|
|
921
|
+
*
|
|
815
922
|
* @param {string} regex_pattern - A regex pattern
|
|
816
923
|
*
|
|
817
924
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -833,6 +940,10 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
833
940
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
834
941
|
* ```
|
|
835
942
|
*
|
|
943
|
+
* 
|
|
944
|
+
*
|
|
945
|
+
*
|
|
946
|
+
*
|
|
836
947
|
* @param {string} text - A text to be matched.
|
|
837
948
|
*
|
|
838
949
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -857,12 +968,13 @@ export declare class FluentFiltersCondition extends FluentBase {
|
|
|
857
968
|
/**
|
|
858
969
|
* Filters elements based on a textual description.
|
|
859
970
|
*
|
|
860
|
-
*
|
|
971
|
+
* **What Should I Write as Matching Text**
|
|
972
|
+
*
|
|
861
973
|
* The text description inside the `matching()` should describe the element visually.
|
|
862
974
|
* It understands color, some famous company/product names, general descriptions.
|
|
863
975
|
*
|
|
864
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
865
|
-
* E.g. `puzzle piece` can fail
|
|
976
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
977
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
866
978
|
* Generally the more detail the better.
|
|
867
979
|
*
|
|
868
980
|
* **Examples:**
|
|
@@ -1007,14 +1119,19 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1007
1119
|
/**
|
|
1008
1120
|
* Filters for an element right of another element.
|
|
1009
1121
|
*
|
|
1122
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1123
|
+
*
|
|
1010
1124
|
* **Examples:**
|
|
1011
1125
|
* ```typescript
|
|
1012
|
-
* -------------- --------------
|
|
1013
|
-
* | leftEl | |
|
|
1014
|
-
* -------------- --------------
|
|
1126
|
+
* -------------- -------------- --------------
|
|
1127
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
1128
|
+
* -------------- -------------- --------------
|
|
1015
1129
|
*
|
|
1016
|
-
* // Returns
|
|
1130
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
1017
1131
|
* ...rightEl().rightOf().leftEl()
|
|
1132
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
1133
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
1134
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
1018
1135
|
* // Returns no element because leftEl is left of rightEl
|
|
1019
1136
|
* ...leftEl().rightOf().rightEl()
|
|
1020
1137
|
* ```
|
|
@@ -1026,14 +1143,19 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1026
1143
|
/**
|
|
1027
1144
|
* Filters for an element left of another element.
|
|
1028
1145
|
*
|
|
1146
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1147
|
+
*
|
|
1029
1148
|
* **Examples:**
|
|
1030
1149
|
* ```typescript
|
|
1031
|
-
* -------------- --------------
|
|
1032
|
-
* |
|
|
1033
|
-
* -------------- --------------
|
|
1150
|
+
* -------------- -------------- --------------
|
|
1151
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
1152
|
+
* -------------- -------------- --------------
|
|
1034
1153
|
*
|
|
1035
|
-
* // Returns
|
|
1154
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
1036
1155
|
* ...leftEl().leftOf().rightEl()
|
|
1156
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
1157
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
1158
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
1037
1159
|
* // Returns no element because rightEl is left of leftEl
|
|
1038
1160
|
* ...rightEl().leftOf().leftEl()
|
|
1039
1161
|
* ```
|
|
@@ -1045,17 +1167,25 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1045
1167
|
/**
|
|
1046
1168
|
* Filters for an element below another element.
|
|
1047
1169
|
*
|
|
1170
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1171
|
+
*
|
|
1048
1172
|
* **Examples:**
|
|
1049
1173
|
* ```typescript
|
|
1050
1174
|
* --------------
|
|
1051
1175
|
* | text |
|
|
1052
1176
|
* --------------
|
|
1053
1177
|
* --------------
|
|
1054
|
-
* |
|
|
1178
|
+
* | button0 |
|
|
1179
|
+
* --------------
|
|
1180
|
+
* --------------
|
|
1181
|
+
* | button1 |
|
|
1055
1182
|
* --------------
|
|
1056
1183
|
*
|
|
1057
|
-
* // Returns
|
|
1184
|
+
* // Returns button0 because button0 is the first button below text
|
|
1058
1185
|
* ...button().below().text()
|
|
1186
|
+
* ...button().below(0).text()
|
|
1187
|
+
* // Returns button1 because button1 is the second button below text
|
|
1188
|
+
* ...button().below(1).text()
|
|
1059
1189
|
* // Returns no element because text is above button
|
|
1060
1190
|
* ...text().below().button()
|
|
1061
1191
|
* ```
|
|
@@ -1067,17 +1197,25 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1067
1197
|
/**
|
|
1068
1198
|
* Filters for an element above another element.
|
|
1069
1199
|
*
|
|
1200
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1201
|
+
*
|
|
1070
1202
|
* **Examples:**
|
|
1071
1203
|
* ```typescript
|
|
1072
1204
|
* --------------
|
|
1073
|
-
* |
|
|
1205
|
+
* | text1 |
|
|
1206
|
+
* --------------
|
|
1207
|
+
* --------------
|
|
1208
|
+
* | text0 |
|
|
1074
1209
|
* --------------
|
|
1075
1210
|
* --------------
|
|
1076
1211
|
* | button |
|
|
1077
1212
|
* --------------
|
|
1078
1213
|
*
|
|
1079
|
-
* // Returns
|
|
1214
|
+
* // Returns text0 because text0 is the first element above button
|
|
1080
1215
|
* ...text().above().button()
|
|
1216
|
+
* ...text().above(0).button()
|
|
1217
|
+
* // Returns text1 because text1 is the second element above button
|
|
1218
|
+
* ...text().above(1).button()
|
|
1081
1219
|
* // Returns no element because button is below text
|
|
1082
1220
|
* ...button().above().text()
|
|
1083
1221
|
* ```
|
|
@@ -1145,12 +1283,12 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1145
1283
|
* **Examples:**
|
|
1146
1284
|
* ```typescript
|
|
1147
1285
|
* // Stops execution at this point when the element does not exist.
|
|
1148
|
-
* await aui.expect().text(
|
|
1286
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1149
1287
|
*
|
|
1150
1288
|
* // This will catch the error and log a message
|
|
1151
1289
|
* // But the execution will continue afterwards
|
|
1152
1290
|
* try {
|
|
1153
|
-
* await aui.expect().text(
|
|
1291
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1154
1292
|
* } catch (error) {
|
|
1155
1293
|
* console.log('Too bad we could not find the element!');
|
|
1156
1294
|
* }
|
|
@@ -1169,12 +1307,12 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1169
1307
|
* **Examples:**
|
|
1170
1308
|
* ```typescript
|
|
1171
1309
|
* // Stops execution at this point when the element does exist.
|
|
1172
|
-
* await aui.expect().text(
|
|
1310
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1173
1311
|
*
|
|
1174
1312
|
* // This will catch the error and log a message
|
|
1175
1313
|
* // But the execution will continue afterwards
|
|
1176
1314
|
* try {
|
|
1177
|
-
* await aui.expect().text(
|
|
1315
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1178
1316
|
* } catch (error) {
|
|
1179
1317
|
* console.log('Too bad we could find the element!');
|
|
1180
1318
|
* }
|
|
@@ -1195,8 +1333,8 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1195
1333
|
*
|
|
1196
1334
|
* **Examples:**
|
|
1197
1335
|
* ```typescript
|
|
1198
|
-
* await aui.expect().text(
|
|
1199
|
-
* await aui.expect().text(
|
|
1336
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1337
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1200
1338
|
* ```
|
|
1201
1339
|
*
|
|
1202
1340
|
* @return {FluentFiltersCondition}
|
|
@@ -1209,9 +1347,13 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1209
1347
|
*
|
|
1210
1348
|
* **Example:**
|
|
1211
1349
|
* ```typescript
|
|
1212
|
-
* await aui.click().button().withText('
|
|
1350
|
+
* await aui.click().button().withText('Google Search').exec();
|
|
1213
1351
|
* ```
|
|
1214
1352
|
*
|
|
1353
|
+
* 
|
|
1354
|
+
*
|
|
1355
|
+
*
|
|
1356
|
+
*
|
|
1215
1357
|
* @return {FluentFilters}
|
|
1216
1358
|
*/
|
|
1217
1359
|
click(): FluentFilters;
|
|
@@ -1220,9 +1362,13 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1220
1362
|
*
|
|
1221
1363
|
* **Example:**
|
|
1222
1364
|
* ```typescript
|
|
1223
|
-
* await aui.moveMouseTo().
|
|
1365
|
+
* await aui.moveMouseTo().text().withText('Grinning_Face').exec()
|
|
1224
1366
|
* ```
|
|
1225
1367
|
*
|
|
1368
|
+
* 
|
|
1369
|
+
*
|
|
1370
|
+
*
|
|
1371
|
+
*
|
|
1226
1372
|
* @return {FluentFilters}
|
|
1227
1373
|
*/
|
|
1228
1374
|
moveMouseTo(): FluentFilters;
|
|
@@ -1252,9 +1398,11 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1252
1398
|
*
|
|
1253
1399
|
* **Example:**
|
|
1254
1400
|
* ```typescript
|
|
1255
|
-
* await aui.
|
|
1401
|
+
* await aui.scrollInside(0,-500).text().withText('Bottom sheet').exec();
|
|
1256
1402
|
* ```
|
|
1257
1403
|
*
|
|
1404
|
+
* 
|
|
1405
|
+
*
|
|
1258
1406
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1259
1407
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1260
1408
|
*
|
|
@@ -1304,12 +1452,14 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1304
1452
|
*
|
|
1305
1453
|
* **Examples:**
|
|
1306
1454
|
* ```typescript
|
|
1307
|
-
* await aui.type('
|
|
1455
|
+
* await aui.type('askui@askui.com').exec()
|
|
1308
1456
|
*
|
|
1309
1457
|
* // mask the text so it is not send to the askui-inference server
|
|
1310
1458
|
* await aui.type('Type some text', { isSecret: true, secretMask: '**' }).exec()
|
|
1311
1459
|
* ```
|
|
1312
1460
|
*
|
|
1461
|
+
* 
|
|
1462
|
+
*
|
|
1313
1463
|
* @param {string} text - A text to type
|
|
1314
1464
|
*
|
|
1315
1465
|
* @return {Exec}
|
|
@@ -1320,9 +1470,11 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1320
1470
|
*
|
|
1321
1471
|
* **Example:**
|
|
1322
1472
|
* ```typescript
|
|
1323
|
-
* await aui.moveMouseRelatively(
|
|
1473
|
+
* await aui.moveMouseRelatively(0, 50).exec();
|
|
1324
1474
|
* ```
|
|
1325
1475
|
*
|
|
1476
|
+
* 
|
|
1477
|
+
*
|
|
1326
1478
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1327
1479
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1328
1480
|
*
|
|
@@ -1339,6 +1491,8 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1339
1491
|
* await aui.moveMouse(500, 500).exec();
|
|
1340
1492
|
* ```
|
|
1341
1493
|
*
|
|
1494
|
+
* 
|
|
1495
|
+
*
|
|
1342
1496
|
* @param {number} x_coordinate - A (positive/negative) x coordinate.
|
|
1343
1497
|
* @param {number} y_coordinate - A (positive/negative) y coordinate.
|
|
1344
1498
|
*
|
|
@@ -1354,10 +1508,12 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1354
1508
|
*
|
|
1355
1509
|
* **Example:**
|
|
1356
1510
|
* ```typescript
|
|
1357
|
-
* // Scroll
|
|
1358
|
-
* await aui.scroll(0,
|
|
1511
|
+
* // Scroll 500 pixels down in y direction
|
|
1512
|
+
* await aui.scroll(0, -500).exec()
|
|
1359
1513
|
* ```
|
|
1360
1514
|
*
|
|
1515
|
+
* 
|
|
1516
|
+
*
|
|
1361
1517
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1362
1518
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1363
1519
|
*
|
|
@@ -1448,6 +1604,8 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1448
1604
|
* await aui.mouseDoubleLeftClick().exec();
|
|
1449
1605
|
* ```
|
|
1450
1606
|
*
|
|
1607
|
+
* 
|
|
1608
|
+
*
|
|
1451
1609
|
* @return {Exec}
|
|
1452
1610
|
*/
|
|
1453
1611
|
mouseDoubleLeftClick(): Exec;
|
|
@@ -1486,22 +1644,34 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1486
1644
|
/**
|
|
1487
1645
|
* Toggles mouse down (Left mouse key/tap).
|
|
1488
1646
|
*
|
|
1647
|
+
* This is the equivalent to **mouse-left-press-and-hold**. It holds the mouse button until the `mouseToogleUp()` is called. Often combined with `mouseToggleUP` to automate **drag-and-drop**.
|
|
1648
|
+
*
|
|
1489
1649
|
* **Example:**
|
|
1490
1650
|
* ```typescript
|
|
1491
1651
|
* await aui.mouseToggleDown().exec();
|
|
1652
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
1653
|
+
* await aui.mouseToggleUp().exec();
|
|
1492
1654
|
* ```
|
|
1493
1655
|
*
|
|
1656
|
+
* 
|
|
1657
|
+
*
|
|
1494
1658
|
* @return {Exec}
|
|
1495
1659
|
*/
|
|
1496
1660
|
mouseToggleDown(): Exec;
|
|
1497
1661
|
/**
|
|
1498
1662
|
* Toggles mouse up (Left mouse key/tap).
|
|
1499
1663
|
*
|
|
1664
|
+
* This is the equivalent to releasing the pressing mouse left button. Often combined with `mouseToggleDown()` to automate **drag-and-drop**.
|
|
1665
|
+
*
|
|
1500
1666
|
* **Example:**
|
|
1501
1667
|
* ```typescript
|
|
1668
|
+
* await aui.mouseToggleDown().exec();
|
|
1669
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
1502
1670
|
* await aui.mouseToggleUp().exec();
|
|
1503
1671
|
* ```
|
|
1504
1672
|
*
|
|
1673
|
+
* 
|
|
1674
|
+
*
|
|
1505
1675
|
* @return {Exec}
|
|
1506
1676
|
*/
|
|
1507
1677
|
mouseToggleUp(): Exec;
|
|
@@ -1510,8 +1680,13 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1510
1680
|
*
|
|
1511
1681
|
* **Operating system specific mappings:**
|
|
1512
1682
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
1513
|
-
* ---
|
|
1514
1683
|
*
|
|
1684
|
+
* **Examples:**
|
|
1685
|
+
* ```typescript
|
|
1686
|
+
* await aui.pressThreeKeys('control', 'command' 'space').exec();
|
|
1687
|
+
* ```
|
|
1688
|
+
*
|
|
1689
|
+
* 
|
|
1515
1690
|
*
|
|
1516
1691
|
* @param {MODIFIER_KEY} first_key - A modifier key
|
|
1517
1692
|
* @param {MODIFIER_KEY} second_key - A modifier key
|
|
@@ -1539,8 +1714,13 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1539
1714
|
*
|
|
1540
1715
|
* **Operating system specific mappings:**
|
|
1541
1716
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
1542
|
-
* ---
|
|
1543
1717
|
*
|
|
1718
|
+
* **Examples:**
|
|
1719
|
+
* ```typescript
|
|
1720
|
+
* await aui.pressKey('tab').exec();
|
|
1721
|
+
* ```
|
|
1722
|
+
*
|
|
1723
|
+
* 
|
|
1544
1724
|
*
|
|
1545
1725
|
* @param {PC_AND_MODIFIER_KEY} key - A key
|
|
1546
1726
|
*
|
|
@@ -1558,7 +1738,15 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1558
1738
|
*/
|
|
1559
1739
|
pressAndroidThirdKey(first_key: ANDROID_KEY, second_key: ANDROID_KEY, third_key: ANDROID_KEY): Exec;
|
|
1560
1740
|
/**
|
|
1561
|
-
* Press two Android keys like `
|
|
1741
|
+
* Press two Android keys like `volume_down+power`
|
|
1742
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
1743
|
+
*
|
|
1744
|
+
* **Examples:**
|
|
1745
|
+
* ```typescript
|
|
1746
|
+
* await aui.pressAndroidTwoKey('volume_down', 'power').exec();
|
|
1747
|
+
* ```
|
|
1748
|
+
*
|
|
1749
|
+
* 
|
|
1562
1750
|
*
|
|
1563
1751
|
* @param {ANDROID_KEY} first_key - A Android key
|
|
1564
1752
|
* @param {ANDROID_KEY} second_key - A Android key
|
|
@@ -1567,7 +1755,15 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
1567
1755
|
*/
|
|
1568
1756
|
pressAndroidTwoKey(first_key: ANDROID_KEY, second_key: ANDROID_KEY): Exec;
|
|
1569
1757
|
/**
|
|
1570
|
-
* Press one Android key like `
|
|
1758
|
+
* Press one Android key like `del`
|
|
1759
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
1760
|
+
*
|
|
1761
|
+
* **Examples:**
|
|
1762
|
+
* ```typescript
|
|
1763
|
+
* await aui.pressAndroidKey('notification').exec();
|
|
1764
|
+
* ```
|
|
1765
|
+
*
|
|
1766
|
+
* 
|
|
1571
1767
|
*
|
|
1572
1768
|
* @param {ANDROID_KEY} key - A Android key
|
|
1573
1769
|
*
|
|
@@ -1642,9 +1838,11 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1642
1838
|
*
|
|
1643
1839
|
* **Examples:**
|
|
1644
1840
|
* ```typescript
|
|
1645
|
-
* await aui.
|
|
1841
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
1646
1842
|
* ```
|
|
1647
1843
|
*
|
|
1844
|
+
* 
|
|
1845
|
+
*
|
|
1648
1846
|
* @return {FluentFiltersOrRelationsGetter}
|
|
1649
1847
|
*/
|
|
1650
1848
|
button(): FluentFiltersOrRelationsGetter;
|
|
@@ -1678,13 +1876,20 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1678
1876
|
/**
|
|
1679
1877
|
* Filters for an UI element 'text'.
|
|
1680
1878
|
*
|
|
1681
|
-
*
|
|
1879
|
+
* Takes an optional parameter to filter for a specific text.
|
|
1880
|
+
* See the examples below.
|
|
1881
|
+
*
|
|
1682
1882
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
1683
1883
|
*
|
|
1684
1884
|
* **Examples:**
|
|
1685
1885
|
* ```typescript
|
|
1686
|
-
* await aui.click().text().
|
|
1886
|
+
* await aui.click().text().exec();
|
|
1887
|
+
* await aui.click().text('Username').exec();
|
|
1888
|
+
*
|
|
1889
|
+
* // Matching with an exact text
|
|
1687
1890
|
* await aui.click().text().withExactText('Username').exec();
|
|
1891
|
+
*
|
|
1892
|
+
* // Matching with a regex
|
|
1688
1893
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
1689
1894
|
* ```
|
|
1690
1895
|
*
|
|
@@ -1701,13 +1906,17 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1701
1906
|
* icon().withText('plus')
|
|
1702
1907
|
* ```
|
|
1703
1908
|
*
|
|
1909
|
+
* 
|
|
1910
|
+
*
|
|
1704
1911
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
1705
1912
|
*
|
|
1706
1913
|
* @return {FluentFiltersOrRelationsGetter}
|
|
1707
1914
|
*/
|
|
1708
1915
|
icon(): FluentFiltersOrRelationsGetter;
|
|
1709
1916
|
/**
|
|
1710
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
1917
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
1918
|
+
*
|
|
1919
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
1711
1920
|
*
|
|
1712
1921
|
* **Example**
|
|
1713
1922
|
* ```typescript
|
|
@@ -1717,6 +1926,7 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1717
1926
|
* customImage: './logo.png', // required
|
|
1718
1927
|
* name: 'myLogo', // optional
|
|
1719
1928
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
1929
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
1720
1930
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
1721
1931
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
1722
1932
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -1731,12 +1941,13 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1731
1941
|
* - **name** (*`string`, optional*):
|
|
1732
1942
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
1733
1943
|
* - **threshold** (*`number`, optional*):
|
|
1734
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1944
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1945
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
1946
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
1735
1947
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
1736
1948
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
1737
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
1738
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
1739
|
-
* of the given custom image.
|
|
1949
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
1950
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
1740
1951
|
*
|
|
1741
1952
|
*
|
|
1742
1953
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -1753,10 +1964,14 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1753
1964
|
* await aui.click().image().exec();
|
|
1754
1965
|
*
|
|
1755
1966
|
* // Works if you have an image with
|
|
1756
|
-
* // a
|
|
1757
|
-
* await aui.click().image().above().text().withText('
|
|
1967
|
+
* // a text below
|
|
1968
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
1758
1969
|
* ```
|
|
1759
1970
|
*
|
|
1971
|
+
* 
|
|
1972
|
+
*
|
|
1973
|
+
*
|
|
1974
|
+
*
|
|
1760
1975
|
* @return {FluentFiltersOrRelationsGetter}
|
|
1761
1976
|
*/
|
|
1762
1977
|
image(): FluentFiltersOrRelationsGetter;
|
|
@@ -1773,11 +1988,21 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1773
1988
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
1774
1989
|
* ```
|
|
1775
1990
|
*
|
|
1991
|
+
* 
|
|
1992
|
+
*
|
|
1993
|
+
*
|
|
1994
|
+
*
|
|
1776
1995
|
* @return {FluentFiltersOrRelationsGetter}
|
|
1777
1996
|
*/
|
|
1778
1997
|
textfield(): FluentFiltersOrRelationsGetter;
|
|
1779
1998
|
/**
|
|
1780
|
-
* Filters for similar
|
|
1999
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
2000
|
+
*
|
|
2001
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
2002
|
+
*
|
|
2003
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
2004
|
+
*
|
|
2005
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
1781
2006
|
*
|
|
1782
2007
|
* **Examples:**
|
|
1783
2008
|
* ```typescript
|
|
@@ -1792,6 +2017,11 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1792
2017
|
* // usually false
|
|
1793
2018
|
* 'atebxtc' === withText('text') => false
|
|
1794
2019
|
* 'other' === withText('text') => false
|
|
2020
|
+
*
|
|
2021
|
+
* // optional parameter: similarity_score
|
|
2022
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
2023
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
2024
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
1795
2025
|
* ```
|
|
1796
2026
|
* 
|
|
1797
2027
|
*
|
|
@@ -1813,6 +2043,10 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1813
2043
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
1814
2044
|
* ```
|
|
1815
2045
|
*
|
|
2046
|
+
* 
|
|
2047
|
+
*
|
|
2048
|
+
*
|
|
2049
|
+
*
|
|
1816
2050
|
* @param {string} regex_pattern - A regex pattern
|
|
1817
2051
|
*
|
|
1818
2052
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -1834,6 +2068,10 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1834
2068
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
1835
2069
|
* ```
|
|
1836
2070
|
*
|
|
2071
|
+
* 
|
|
2072
|
+
*
|
|
2073
|
+
*
|
|
2074
|
+
*
|
|
1837
2075
|
* @param {string} text - A text to be matched.
|
|
1838
2076
|
*
|
|
1839
2077
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -1858,12 +2096,13 @@ export declare class FluentFiltersGetter extends FluentBase {
|
|
|
1858
2096
|
/**
|
|
1859
2097
|
* Filters elements based on a textual description.
|
|
1860
2098
|
*
|
|
1861
|
-
*
|
|
2099
|
+
* **What Should I Write as Matching Text**
|
|
2100
|
+
*
|
|
1862
2101
|
* The text description inside the `matching()` should describe the element visually.
|
|
1863
2102
|
* It understands color, some famous company/product names, general descriptions.
|
|
1864
2103
|
*
|
|
1865
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
1866
|
-
* E.g. `puzzle piece` can fail
|
|
2104
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
2105
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
1867
2106
|
* Generally the more detail the better.
|
|
1868
2107
|
*
|
|
1869
2108
|
* **Examples:**
|
|
@@ -2008,14 +2247,19 @@ export declare class FluentFiltersOrRelationsGetter extends FluentFiltersGetter
|
|
|
2008
2247
|
/**
|
|
2009
2248
|
* Filters for an element right of another element.
|
|
2010
2249
|
*
|
|
2250
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
2251
|
+
*
|
|
2011
2252
|
* **Examples:**
|
|
2012
2253
|
* ```typescript
|
|
2013
|
-
* -------------- --------------
|
|
2014
|
-
* | leftEl | |
|
|
2015
|
-
* -------------- --------------
|
|
2254
|
+
* -------------- -------------- --------------
|
|
2255
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
2256
|
+
* -------------- -------------- --------------
|
|
2016
2257
|
*
|
|
2017
|
-
* // Returns
|
|
2258
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
2018
2259
|
* ...rightEl().rightOf().leftEl()
|
|
2260
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
2261
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
2262
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
2019
2263
|
* // Returns no element because leftEl is left of rightEl
|
|
2020
2264
|
* ...leftEl().rightOf().rightEl()
|
|
2021
2265
|
* ```
|
|
@@ -2027,14 +2271,19 @@ export declare class FluentFiltersOrRelationsGetter extends FluentFiltersGetter
|
|
|
2027
2271
|
/**
|
|
2028
2272
|
* Filters for an element left of another element.
|
|
2029
2273
|
*
|
|
2274
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
2275
|
+
*
|
|
2030
2276
|
* **Examples:**
|
|
2031
2277
|
* ```typescript
|
|
2032
|
-
* -------------- --------------
|
|
2033
|
-
* |
|
|
2034
|
-
* -------------- --------------
|
|
2278
|
+
* -------------- -------------- --------------
|
|
2279
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
2280
|
+
* -------------- -------------- --------------
|
|
2035
2281
|
*
|
|
2036
|
-
* // Returns
|
|
2282
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
2037
2283
|
* ...leftEl().leftOf().rightEl()
|
|
2284
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
2285
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
2286
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
2038
2287
|
* // Returns no element because rightEl is left of leftEl
|
|
2039
2288
|
* ...rightEl().leftOf().leftEl()
|
|
2040
2289
|
* ```
|
|
@@ -2046,17 +2295,25 @@ export declare class FluentFiltersOrRelationsGetter extends FluentFiltersGetter
|
|
|
2046
2295
|
/**
|
|
2047
2296
|
* Filters for an element below another element.
|
|
2048
2297
|
*
|
|
2298
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
2299
|
+
*
|
|
2049
2300
|
* **Examples:**
|
|
2050
2301
|
* ```typescript
|
|
2051
2302
|
* --------------
|
|
2052
2303
|
* | text |
|
|
2053
2304
|
* --------------
|
|
2054
2305
|
* --------------
|
|
2055
|
-
* |
|
|
2306
|
+
* | button0 |
|
|
2307
|
+
* --------------
|
|
2308
|
+
* --------------
|
|
2309
|
+
* | button1 |
|
|
2056
2310
|
* --------------
|
|
2057
2311
|
*
|
|
2058
|
-
* // Returns
|
|
2312
|
+
* // Returns button0 because button0 is the first button below text
|
|
2059
2313
|
* ...button().below().text()
|
|
2314
|
+
* ...button().below(0).text()
|
|
2315
|
+
* // Returns button1 because button1 is the second button below text
|
|
2316
|
+
* ...button().below(1).text()
|
|
2060
2317
|
* // Returns no element because text is above button
|
|
2061
2318
|
* ...text().below().button()
|
|
2062
2319
|
* ```
|
|
@@ -2068,17 +2325,25 @@ export declare class FluentFiltersOrRelationsGetter extends FluentFiltersGetter
|
|
|
2068
2325
|
/**
|
|
2069
2326
|
* Filters for an element above another element.
|
|
2070
2327
|
*
|
|
2328
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
2329
|
+
*
|
|
2071
2330
|
* **Examples:**
|
|
2072
2331
|
* ```typescript
|
|
2073
2332
|
* --------------
|
|
2074
|
-
* |
|
|
2333
|
+
* | text1 |
|
|
2334
|
+
* --------------
|
|
2335
|
+
* --------------
|
|
2336
|
+
* | text0 |
|
|
2075
2337
|
* --------------
|
|
2076
2338
|
* --------------
|
|
2077
2339
|
* | button |
|
|
2078
2340
|
* --------------
|
|
2079
2341
|
*
|
|
2080
|
-
* // Returns
|
|
2342
|
+
* // Returns text0 because text0 is the first element above button
|
|
2081
2343
|
* ...text().above().button()
|
|
2344
|
+
* ...text().above(0).button()
|
|
2345
|
+
* // Returns text1 because text1 is the second element above button
|
|
2346
|
+
* ...text().above(1).button()
|
|
2082
2347
|
* // Returns no element because button is below text
|
|
2083
2348
|
* ...button().above().text()
|
|
2084
2349
|
* ```
|
|
@@ -2153,7 +2418,7 @@ export declare abstract class Getter extends FluentCommand {
|
|
|
2153
2418
|
*
|
|
2154
2419
|
* **Examples:**
|
|
2155
2420
|
* ```typescript
|
|
2156
|
-
* const text = await aui.get().text(
|
|
2421
|
+
* const text = await aui.get().text('Sign').exec();
|
|
2157
2422
|
* console.log(text);
|
|
2158
2423
|
* ```
|
|
2159
2424
|
* ```text
|