askui 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/ai-element/ai-element-collection.d.ts +10 -0
- package/dist/cjs/core/ai-element/ai-element-collection.js +78 -0
- package/dist/cjs/core/ai-element/ai-element-error.d.ts +2 -0
- package/dist/cjs/core/ai-element/ai-element-error.js +6 -0
- package/dist/cjs/core/ai-element/ai-element.d.ts +27 -0
- package/dist/cjs/core/ai-element/ai-element.js +31 -0
- package/dist/cjs/core/model/custom-element-json.d.ts +32 -15
- package/dist/cjs/core/model/custom-element.d.ts +3 -2
- package/dist/cjs/core/model/custom-element.js +4 -2
- package/dist/cjs/core/reporting/step-reporter.js +15 -6
- package/dist/cjs/execution/dsl.d.ts +488 -113
- package/dist/cjs/execution/dsl.js +519 -113
- package/dist/cjs/execution/index.d.ts +1 -1
- package/dist/cjs/execution/index.js +11 -3
- package/dist/cjs/execution/inference-client.js +9 -4
- package/dist/cjs/execution/ui-control-client-dependency-builder.d.ts +1 -0
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -3
- package/dist/cjs/execution/ui-control-client.d.ts +101 -16
- package/dist/cjs/execution/ui-control-client.js +142 -52
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/cjs/execution/ui-controller-client.d.ts +1 -0
- package/dist/cjs/execution/ui-controller-client.js +11 -1
- package/dist/cjs/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/cjs/execution/ui-controller-not-connected-error.js +12 -0
- package/dist/cjs/main.d.ts +1 -1
- package/dist/cjs/main.js +12 -3
- package/dist/cjs/utils/analytics/analytics.d.ts +3 -1
- package/dist/cjs/utils/analytics/analytics.js +5 -0
- package/dist/cjs/utils/http/custom-errors/index.js +1 -1
- package/dist/cjs/utils/http/http-client-got.js +46 -20
- package/dist/esm/core/ai-element/ai-element-collection.d.ts +10 -0
- package/dist/esm/core/ai-element/ai-element-collection.js +71 -0
- package/dist/esm/core/ai-element/ai-element-error.d.ts +2 -0
- package/dist/esm/core/ai-element/ai-element-error.js +2 -0
- package/dist/esm/core/ai-element/ai-element.d.ts +27 -0
- package/dist/esm/core/ai-element/ai-element.js +28 -0
- package/dist/esm/core/model/custom-element-json.d.ts +32 -15
- package/dist/esm/core/model/custom-element.d.ts +3 -2
- package/dist/esm/core/model/custom-element.js +4 -2
- package/dist/esm/core/reporting/step-reporter.js +15 -6
- package/dist/esm/execution/dsl.d.ts +488 -113
- package/dist/esm/execution/dsl.js +519 -113
- package/dist/esm/execution/index.d.ts +1 -1
- package/dist/esm/execution/index.js +1 -1
- package/dist/esm/execution/inference-client.js +9 -4
- package/dist/esm/execution/ui-control-client-dependency-builder.d.ts +1 -0
- package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -3
- package/dist/esm/execution/ui-control-client.d.ts +101 -16
- package/dist/esm/execution/ui-control-client.js +142 -49
- package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/esm/execution/ui-controller-client.d.ts +1 -0
- package/dist/esm/execution/ui-controller-client.js +11 -1
- package/dist/esm/execution/ui-controller-not-connected-error.d.ts +4 -0
- package/dist/esm/execution/ui-controller-not-connected-error.js +8 -0
- package/dist/esm/main.d.ts +1 -1
- package/dist/esm/main.js +1 -1
- package/dist/esm/utils/analytics/analytics.d.ts +3 -1
- package/dist/esm/utils/analytics/analytics.js +5 -0
- package/dist/esm/utils/http/custom-errors/index.js +1 -1
- package/dist/esm/utils/http/http-client-got.js +46 -20
- package/package.json +1 -1
|
@@ -28,7 +28,11 @@ class FluentBase {
|
|
|
28
28
|
if (this instanceof FluentCommand) {
|
|
29
29
|
const fluentCommand = this;
|
|
30
30
|
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
|
|
31
|
-
|
|
31
|
+
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
|
|
32
|
+
return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), {
|
|
33
|
+
customElementsJson: customElements,
|
|
34
|
+
aiElementNames,
|
|
35
|
+
});
|
|
32
36
|
}
|
|
33
37
|
if (!this.prev) {
|
|
34
38
|
throw new Error('Prev element not defined');
|
|
@@ -41,7 +45,11 @@ class FluentBase {
|
|
|
41
45
|
if (this instanceof Getter) {
|
|
42
46
|
const getter = this;
|
|
43
47
|
const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
|
|
44
|
-
|
|
48
|
+
const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
|
|
49
|
+
return getter.getterExecutor(newCurrentInstruction.trim(), {
|
|
50
|
+
customElementsJson: customElements,
|
|
51
|
+
aiElementNames,
|
|
52
|
+
});
|
|
45
53
|
}
|
|
46
54
|
if (!this.prev) {
|
|
47
55
|
throw new Error('Prev element not defined');
|
|
@@ -143,9 +151,11 @@ export class FluentFilters extends FluentBase {
|
|
|
143
151
|
*
|
|
144
152
|
* **Examples:**
|
|
145
153
|
* ```typescript
|
|
146
|
-
* await aui.
|
|
154
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
147
155
|
* ```
|
|
148
156
|
*
|
|
157
|
+
* 
|
|
158
|
+
*
|
|
149
159
|
* @return {FluentFiltersOrRelations}
|
|
150
160
|
*/
|
|
151
161
|
button() {
|
|
@@ -205,13 +215,20 @@ export class FluentFilters extends FluentBase {
|
|
|
205
215
|
/**
|
|
206
216
|
* Filters for an UI element 'text'.
|
|
207
217
|
*
|
|
208
|
-
*
|
|
218
|
+
* Takes an optional parameter to filter for a specific text.
|
|
219
|
+
* See the examples below.
|
|
220
|
+
*
|
|
209
221
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
210
222
|
*
|
|
211
223
|
* **Examples:**
|
|
212
224
|
* ```typescript
|
|
213
|
-
* await aui.click().text().
|
|
225
|
+
* await aui.click().text().exec();
|
|
226
|
+
* await aui.click().text('Username').exec();
|
|
227
|
+
*
|
|
228
|
+
* // Matching with an exact text
|
|
214
229
|
* await aui.click().text().withExactText('Username').exec();
|
|
230
|
+
*
|
|
231
|
+
* // Matching with a regex
|
|
215
232
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
216
233
|
* ```
|
|
217
234
|
*
|
|
@@ -235,6 +252,8 @@ export class FluentFilters extends FluentBase {
|
|
|
235
252
|
* icon().withText('plus')
|
|
236
253
|
* ```
|
|
237
254
|
*
|
|
255
|
+
* 
|
|
256
|
+
*
|
|
238
257
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
239
258
|
*
|
|
240
259
|
* @return {FluentFiltersOrRelations}
|
|
@@ -245,7 +264,9 @@ export class FluentFilters extends FluentBase {
|
|
|
245
264
|
return new FluentFiltersOrRelations(this);
|
|
246
265
|
}
|
|
247
266
|
/**
|
|
248
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
267
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
268
|
+
*
|
|
269
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
249
270
|
*
|
|
250
271
|
* **Example**
|
|
251
272
|
* ```typescript
|
|
@@ -255,6 +276,7 @@ export class FluentFilters extends FluentBase {
|
|
|
255
276
|
* customImage: './logo.png', // required
|
|
256
277
|
* name: 'myLogo', // optional
|
|
257
278
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
279
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
258
280
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
259
281
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
260
282
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -269,12 +291,13 @@ export class FluentFilters extends FluentBase {
|
|
|
269
291
|
* - **name** (*`string`, optional*):
|
|
270
292
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
271
293
|
* - **threshold** (*`number`, optional*):
|
|
272
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
294
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
295
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
296
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
273
297
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
274
298
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
275
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
276
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
277
|
-
* of the given custom image.
|
|
299
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
300
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
278
301
|
*
|
|
279
302
|
*
|
|
280
303
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -288,6 +311,23 @@ export class FluentFilters extends FluentBase {
|
|
|
288
311
|
this._params.set('customElement', customElement);
|
|
289
312
|
return new FluentFiltersOrRelations(this);
|
|
290
313
|
}
|
|
314
|
+
/**
|
|
315
|
+
* Detects an AI Element created with the workflow creator.
|
|
316
|
+
*
|
|
317
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
318
|
+
*
|
|
319
|
+
* @return {FluentFiltersOrRelations}
|
|
320
|
+
*/
|
|
321
|
+
aiElement(aiElementName) {
|
|
322
|
+
this._textStr = '';
|
|
323
|
+
this._textStr += 'ai';
|
|
324
|
+
this._textStr += ' element';
|
|
325
|
+
this._textStr += ' with';
|
|
326
|
+
this._textStr += ' name';
|
|
327
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
328
|
+
this._params.set('aiElementName', aiElementName);
|
|
329
|
+
return new FluentFiltersOrRelations(this);
|
|
330
|
+
}
|
|
291
331
|
/**
|
|
292
332
|
* Filters for a UI element 'image'.
|
|
293
333
|
*
|
|
@@ -297,10 +337,14 @@ export class FluentFilters extends FluentBase {
|
|
|
297
337
|
* await aui.click().image().exec();
|
|
298
338
|
*
|
|
299
339
|
* // Works if you have an image with
|
|
300
|
-
* // a
|
|
301
|
-
* await aui.click().image().above().text().withText('
|
|
340
|
+
* // a text below
|
|
341
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
302
342
|
* ```
|
|
303
343
|
*
|
|
344
|
+
* 
|
|
345
|
+
*
|
|
346
|
+
*
|
|
347
|
+
*
|
|
304
348
|
* @return {FluentFiltersOrRelations}
|
|
305
349
|
*/
|
|
306
350
|
image() {
|
|
@@ -321,6 +365,10 @@ export class FluentFilters extends FluentBase {
|
|
|
321
365
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
322
366
|
* ```
|
|
323
367
|
*
|
|
368
|
+
* 
|
|
369
|
+
*
|
|
370
|
+
*
|
|
371
|
+
*
|
|
324
372
|
* @return {FluentFiltersOrRelations}
|
|
325
373
|
*/
|
|
326
374
|
textfield() {
|
|
@@ -329,7 +377,13 @@ export class FluentFilters extends FluentBase {
|
|
|
329
377
|
return new FluentFiltersOrRelations(this);
|
|
330
378
|
}
|
|
331
379
|
/**
|
|
332
|
-
* Filters for similar
|
|
380
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
381
|
+
*
|
|
382
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
383
|
+
*
|
|
384
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
385
|
+
*
|
|
386
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
333
387
|
*
|
|
334
388
|
* **Examples:**
|
|
335
389
|
* ```typescript
|
|
@@ -344,6 +398,11 @@ export class FluentFilters extends FluentBase {
|
|
|
344
398
|
* // usually false
|
|
345
399
|
* 'atebxtc' === withText('text') => false
|
|
346
400
|
* 'other' === withText('text') => false
|
|
401
|
+
*
|
|
402
|
+
* // optional parameter: similarity_score
|
|
403
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
404
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
405
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
347
406
|
* ```
|
|
348
407
|
* 
|
|
349
408
|
*
|
|
@@ -374,6 +433,10 @@ export class FluentFilters extends FluentBase {
|
|
|
374
433
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
375
434
|
* ```
|
|
376
435
|
*
|
|
436
|
+
* 
|
|
437
|
+
*
|
|
438
|
+
*
|
|
439
|
+
*
|
|
377
440
|
* @param {string} regex_pattern - A regex pattern
|
|
378
441
|
*
|
|
379
442
|
* @return {FluentFiltersOrRelations}
|
|
@@ -402,6 +465,10 @@ export class FluentFilters extends FluentBase {
|
|
|
402
465
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
403
466
|
* ```
|
|
404
467
|
*
|
|
468
|
+
* 
|
|
469
|
+
*
|
|
470
|
+
*
|
|
471
|
+
*
|
|
405
472
|
* @param {string} text - A text to be matched.
|
|
406
473
|
*
|
|
407
474
|
* @return {FluentFiltersOrRelations}
|
|
@@ -438,17 +505,28 @@ export class FluentFilters extends FluentBase {
|
|
|
438
505
|
/**
|
|
439
506
|
* Filters elements based on a textual description.
|
|
440
507
|
*
|
|
441
|
-
*
|
|
508
|
+
* **What Should I Write as Matching Text**
|
|
509
|
+
*
|
|
442
510
|
* The text description inside the `matching()` should describe the element visually.
|
|
443
511
|
* It understands color, some famous company/product names, general descriptions.
|
|
444
512
|
*
|
|
445
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
446
|
-
* E.g. `puzzle piece` can fail
|
|
513
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
514
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
447
515
|
* Generally the more detail the better.
|
|
448
516
|
*
|
|
449
517
|
* **Examples:**
|
|
450
518
|
* ```typescript
|
|
451
|
-
*
|
|
519
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
520
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
521
|
+
*
|
|
522
|
+
* // Select an image that has text in it
|
|
523
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
524
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
525
|
+
*
|
|
526
|
+
* // Target a logo/image by describing it
|
|
527
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
528
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
529
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
452
530
|
* ```
|
|
453
531
|
*
|
|
454
532
|
* @param {string} text - A description of the target element.
|
|
@@ -606,14 +684,19 @@ export class FluentFiltersOrRelations extends FluentFilters {
|
|
|
606
684
|
/**
|
|
607
685
|
* Filters for an element right of another element.
|
|
608
686
|
*
|
|
687
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
688
|
+
*
|
|
609
689
|
* **Examples:**
|
|
610
690
|
* ```typescript
|
|
611
|
-
* -------------- --------------
|
|
612
|
-
* | leftEl | |
|
|
613
|
-
* -------------- --------------
|
|
691
|
+
* -------------- -------------- --------------
|
|
692
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
693
|
+
* -------------- -------------- --------------
|
|
614
694
|
*
|
|
615
|
-
* // Returns
|
|
695
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
616
696
|
* ...rightEl().rightOf().leftEl()
|
|
697
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
698
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
699
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
617
700
|
* // Returns no element because leftEl is left of rightEl
|
|
618
701
|
* ...leftEl().rightOf().rightEl()
|
|
619
702
|
* ```
|
|
@@ -633,14 +716,19 @@ export class FluentFiltersOrRelations extends FluentFilters {
|
|
|
633
716
|
/**
|
|
634
717
|
* Filters for an element left of another element.
|
|
635
718
|
*
|
|
719
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
720
|
+
*
|
|
636
721
|
* **Examples:**
|
|
637
722
|
* ```typescript
|
|
638
|
-
* -------------- --------------
|
|
639
|
-
* |
|
|
640
|
-
* -------------- --------------
|
|
723
|
+
* -------------- -------------- --------------
|
|
724
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
725
|
+
* -------------- -------------- --------------
|
|
641
726
|
*
|
|
642
|
-
* // Returns
|
|
727
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
643
728
|
* ...leftEl().leftOf().rightEl()
|
|
729
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
730
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
731
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
644
732
|
* // Returns no element because rightEl is left of leftEl
|
|
645
733
|
* ...rightEl().leftOf().leftEl()
|
|
646
734
|
* ```
|
|
@@ -660,17 +748,25 @@ export class FluentFiltersOrRelations extends FluentFilters {
|
|
|
660
748
|
/**
|
|
661
749
|
* Filters for an element below another element.
|
|
662
750
|
*
|
|
751
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
752
|
+
*
|
|
663
753
|
* **Examples:**
|
|
664
754
|
* ```typescript
|
|
665
755
|
* --------------
|
|
666
756
|
* | text |
|
|
667
757
|
* --------------
|
|
668
758
|
* --------------
|
|
669
|
-
* |
|
|
759
|
+
* | button0 |
|
|
760
|
+
* --------------
|
|
761
|
+
* --------------
|
|
762
|
+
* | button1 |
|
|
670
763
|
* --------------
|
|
671
764
|
*
|
|
672
|
-
* // Returns
|
|
765
|
+
* // Returns button0 because button0 is the first button below text
|
|
673
766
|
* ...button().below().text()
|
|
767
|
+
* ...button().below(0).text()
|
|
768
|
+
* // Returns button1 because button1 is the second button below text
|
|
769
|
+
* ...button().below(1).text()
|
|
674
770
|
* // Returns no element because text is above button
|
|
675
771
|
* ...text().below().button()
|
|
676
772
|
* ```
|
|
@@ -689,17 +785,25 @@ export class FluentFiltersOrRelations extends FluentFilters {
|
|
|
689
785
|
/**
|
|
690
786
|
* Filters for an element above another element.
|
|
691
787
|
*
|
|
788
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
789
|
+
*
|
|
692
790
|
* **Examples:**
|
|
693
791
|
* ```typescript
|
|
694
792
|
* --------------
|
|
695
|
-
* |
|
|
793
|
+
* | text1 |
|
|
794
|
+
* --------------
|
|
795
|
+
* --------------
|
|
796
|
+
* | text0 |
|
|
696
797
|
* --------------
|
|
697
798
|
* --------------
|
|
698
799
|
* | button |
|
|
699
800
|
* --------------
|
|
700
801
|
*
|
|
701
|
-
* // Returns
|
|
802
|
+
* // Returns text0 because text0 is the first element above button
|
|
702
803
|
* ...text().above().button()
|
|
804
|
+
* ...text().above(0).button()
|
|
805
|
+
* // Returns text1 because text1 is the second element above button
|
|
806
|
+
* ...text().above(1).button()
|
|
703
807
|
* // Returns no element because button is below text
|
|
704
808
|
* ...button().above().text()
|
|
705
809
|
* ```
|
|
@@ -864,9 +968,11 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
864
968
|
*
|
|
865
969
|
* **Examples:**
|
|
866
970
|
* ```typescript
|
|
867
|
-
* await aui.
|
|
971
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
868
972
|
* ```
|
|
869
973
|
*
|
|
974
|
+
* 
|
|
975
|
+
*
|
|
870
976
|
* @return {FluentFiltersOrRelationsCondition}
|
|
871
977
|
*/
|
|
872
978
|
button() {
|
|
@@ -926,13 +1032,20 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
926
1032
|
/**
|
|
927
1033
|
* Filters for an UI element 'text'.
|
|
928
1034
|
*
|
|
929
|
-
*
|
|
1035
|
+
* Takes an optional parameter to filter for a specific text.
|
|
1036
|
+
* See the examples below.
|
|
1037
|
+
*
|
|
930
1038
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
931
1039
|
*
|
|
932
1040
|
* **Examples:**
|
|
933
1041
|
* ```typescript
|
|
934
|
-
* await aui.click().text().
|
|
1042
|
+
* await aui.click().text().exec();
|
|
1043
|
+
* await aui.click().text('Username').exec();
|
|
1044
|
+
*
|
|
1045
|
+
* // Matching with an exact text
|
|
935
1046
|
* await aui.click().text().withExactText('Username').exec();
|
|
1047
|
+
*
|
|
1048
|
+
* // Matching with a regex
|
|
936
1049
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
937
1050
|
* ```
|
|
938
1051
|
*
|
|
@@ -956,6 +1069,8 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
956
1069
|
* icon().withText('plus')
|
|
957
1070
|
* ```
|
|
958
1071
|
*
|
|
1072
|
+
* 
|
|
1073
|
+
*
|
|
959
1074
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
960
1075
|
*
|
|
961
1076
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -966,7 +1081,9 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
966
1081
|
return new FluentFiltersOrRelationsCondition(this);
|
|
967
1082
|
}
|
|
968
1083
|
/**
|
|
969
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
1084
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
1085
|
+
*
|
|
1086
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
970
1087
|
*
|
|
971
1088
|
* **Example**
|
|
972
1089
|
* ```typescript
|
|
@@ -976,6 +1093,7 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
976
1093
|
* customImage: './logo.png', // required
|
|
977
1094
|
* name: 'myLogo', // optional
|
|
978
1095
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
1096
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
979
1097
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
980
1098
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
981
1099
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -990,12 +1108,13 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
990
1108
|
* - **name** (*`string`, optional*):
|
|
991
1109
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
992
1110
|
* - **threshold** (*`number`, optional*):
|
|
993
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1111
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
1112
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
1113
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
994
1114
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
995
1115
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
996
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
997
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
998
|
-
* of the given custom image.
|
|
1116
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
1117
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
999
1118
|
*
|
|
1000
1119
|
*
|
|
1001
1120
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -1009,6 +1128,23 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1009
1128
|
this._params.set('customElement', customElement);
|
|
1010
1129
|
return new FluentFiltersOrRelationsCondition(this);
|
|
1011
1130
|
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Detects an AI Element created with the workflow creator.
|
|
1133
|
+
*
|
|
1134
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
1135
|
+
*
|
|
1136
|
+
* @return {FluentFiltersOrRelationsCondition}
|
|
1137
|
+
*/
|
|
1138
|
+
aiElement(aiElementName) {
|
|
1139
|
+
this._textStr = '';
|
|
1140
|
+
this._textStr += 'ai';
|
|
1141
|
+
this._textStr += ' element';
|
|
1142
|
+
this._textStr += ' with';
|
|
1143
|
+
this._textStr += ' name';
|
|
1144
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
1145
|
+
this._params.set('aiElementName', aiElementName);
|
|
1146
|
+
return new FluentFiltersOrRelationsCondition(this);
|
|
1147
|
+
}
|
|
1012
1148
|
/**
|
|
1013
1149
|
* Filters for a UI element 'image'.
|
|
1014
1150
|
*
|
|
@@ -1018,10 +1154,14 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1018
1154
|
* await aui.click().image().exec();
|
|
1019
1155
|
*
|
|
1020
1156
|
* // Works if you have an image with
|
|
1021
|
-
* // a
|
|
1022
|
-
* await aui.click().image().above().text().withText('
|
|
1157
|
+
* // a text below
|
|
1158
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
1023
1159
|
* ```
|
|
1024
1160
|
*
|
|
1161
|
+
* 
|
|
1162
|
+
*
|
|
1163
|
+
*
|
|
1164
|
+
*
|
|
1025
1165
|
* @return {FluentFiltersOrRelationsCondition}
|
|
1026
1166
|
*/
|
|
1027
1167
|
image() {
|
|
@@ -1042,6 +1182,10 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1042
1182
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
1043
1183
|
* ```
|
|
1044
1184
|
*
|
|
1185
|
+
* 
|
|
1186
|
+
*
|
|
1187
|
+
*
|
|
1188
|
+
*
|
|
1045
1189
|
* @return {FluentFiltersOrRelationsCondition}
|
|
1046
1190
|
*/
|
|
1047
1191
|
textfield() {
|
|
@@ -1050,7 +1194,13 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1050
1194
|
return new FluentFiltersOrRelationsCondition(this);
|
|
1051
1195
|
}
|
|
1052
1196
|
/**
|
|
1053
|
-
* Filters for similar
|
|
1197
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
1198
|
+
*
|
|
1199
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
1200
|
+
*
|
|
1201
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
1202
|
+
*
|
|
1203
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
1054
1204
|
*
|
|
1055
1205
|
* **Examples:**
|
|
1056
1206
|
* ```typescript
|
|
@@ -1065,6 +1215,11 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1065
1215
|
* // usually false
|
|
1066
1216
|
* 'atebxtc' === withText('text') => false
|
|
1067
1217
|
* 'other' === withText('text') => false
|
|
1218
|
+
*
|
|
1219
|
+
* // optional parameter: similarity_score
|
|
1220
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
1221
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
1222
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
1068
1223
|
* ```
|
|
1069
1224
|
* 
|
|
1070
1225
|
*
|
|
@@ -1095,6 +1250,10 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1095
1250
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
1096
1251
|
* ```
|
|
1097
1252
|
*
|
|
1253
|
+
* 
|
|
1254
|
+
*
|
|
1255
|
+
*
|
|
1256
|
+
*
|
|
1098
1257
|
* @param {string} regex_pattern - A regex pattern
|
|
1099
1258
|
*
|
|
1100
1259
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -1123,6 +1282,10 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1123
1282
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
1124
1283
|
* ```
|
|
1125
1284
|
*
|
|
1285
|
+
* 
|
|
1286
|
+
*
|
|
1287
|
+
*
|
|
1288
|
+
*
|
|
1126
1289
|
* @param {string} text - A text to be matched.
|
|
1127
1290
|
*
|
|
1128
1291
|
* @return {FluentFiltersOrRelationsCondition}
|
|
@@ -1159,17 +1322,28 @@ export class FluentFiltersCondition extends FluentBase {
|
|
|
1159
1322
|
/**
|
|
1160
1323
|
* Filters elements based on a textual description.
|
|
1161
1324
|
*
|
|
1162
|
-
*
|
|
1325
|
+
* **What Should I Write as Matching Text**
|
|
1326
|
+
*
|
|
1163
1327
|
* The text description inside the `matching()` should describe the element visually.
|
|
1164
1328
|
* It understands color, some famous company/product names, general descriptions.
|
|
1165
1329
|
*
|
|
1166
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
1167
|
-
* E.g. `puzzle piece` can fail
|
|
1330
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
1331
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
1168
1332
|
* Generally the more detail the better.
|
|
1169
1333
|
*
|
|
1170
1334
|
* **Examples:**
|
|
1171
1335
|
* ```typescript
|
|
1172
|
-
*
|
|
1336
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
1337
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
1338
|
+
*
|
|
1339
|
+
* // Select an image that has text in it
|
|
1340
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
1341
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
1342
|
+
*
|
|
1343
|
+
* // Target a logo/image by describing it
|
|
1344
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
1345
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
1346
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
1173
1347
|
* ```
|
|
1174
1348
|
*
|
|
1175
1349
|
* @param {string} text - A description of the target element.
|
|
@@ -1327,14 +1501,19 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1327
1501
|
/**
|
|
1328
1502
|
* Filters for an element right of another element.
|
|
1329
1503
|
*
|
|
1504
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1505
|
+
*
|
|
1330
1506
|
* **Examples:**
|
|
1331
1507
|
* ```typescript
|
|
1332
|
-
* -------------- --------------
|
|
1333
|
-
* | leftEl | |
|
|
1334
|
-
* -------------- --------------
|
|
1508
|
+
* -------------- -------------- --------------
|
|
1509
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
1510
|
+
* -------------- -------------- --------------
|
|
1335
1511
|
*
|
|
1336
|
-
* // Returns
|
|
1512
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
1337
1513
|
* ...rightEl().rightOf().leftEl()
|
|
1514
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
1515
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
1516
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
1338
1517
|
* // Returns no element because leftEl is left of rightEl
|
|
1339
1518
|
* ...leftEl().rightOf().rightEl()
|
|
1340
1519
|
* ```
|
|
@@ -1354,14 +1533,19 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1354
1533
|
/**
|
|
1355
1534
|
* Filters for an element left of another element.
|
|
1356
1535
|
*
|
|
1536
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1537
|
+
*
|
|
1357
1538
|
* **Examples:**
|
|
1358
1539
|
* ```typescript
|
|
1359
|
-
* -------------- --------------
|
|
1360
|
-
* |
|
|
1361
|
-
* -------------- --------------
|
|
1540
|
+
* -------------- -------------- --------------
|
|
1541
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
1542
|
+
* -------------- -------------- --------------
|
|
1362
1543
|
*
|
|
1363
|
-
* // Returns
|
|
1544
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
1364
1545
|
* ...leftEl().leftOf().rightEl()
|
|
1546
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
1547
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
1548
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
1365
1549
|
* // Returns no element because rightEl is left of leftEl
|
|
1366
1550
|
* ...rightEl().leftOf().leftEl()
|
|
1367
1551
|
* ```
|
|
@@ -1381,17 +1565,25 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1381
1565
|
/**
|
|
1382
1566
|
* Filters for an element below another element.
|
|
1383
1567
|
*
|
|
1568
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1569
|
+
*
|
|
1384
1570
|
* **Examples:**
|
|
1385
1571
|
* ```typescript
|
|
1386
1572
|
* --------------
|
|
1387
1573
|
* | text |
|
|
1388
1574
|
* --------------
|
|
1389
1575
|
* --------------
|
|
1390
|
-
* |
|
|
1576
|
+
* | button0 |
|
|
1577
|
+
* --------------
|
|
1578
|
+
* --------------
|
|
1579
|
+
* | button1 |
|
|
1391
1580
|
* --------------
|
|
1392
1581
|
*
|
|
1393
|
-
* // Returns
|
|
1582
|
+
* // Returns button0 because button0 is the first button below text
|
|
1394
1583
|
* ...button().below().text()
|
|
1584
|
+
* ...button().below(0).text()
|
|
1585
|
+
* // Returns button1 because button1 is the second button below text
|
|
1586
|
+
* ...button().below(1).text()
|
|
1395
1587
|
* // Returns no element because text is above button
|
|
1396
1588
|
* ...text().below().button()
|
|
1397
1589
|
* ```
|
|
@@ -1410,17 +1602,25 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1410
1602
|
/**
|
|
1411
1603
|
* Filters for an element above another element.
|
|
1412
1604
|
*
|
|
1605
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
1606
|
+
*
|
|
1413
1607
|
* **Examples:**
|
|
1414
1608
|
* ```typescript
|
|
1415
1609
|
* --------------
|
|
1416
|
-
* |
|
|
1610
|
+
* | text1 |
|
|
1611
|
+
* --------------
|
|
1612
|
+
* --------------
|
|
1613
|
+
* | text0 |
|
|
1417
1614
|
* --------------
|
|
1418
1615
|
* --------------
|
|
1419
1616
|
* | button |
|
|
1420
1617
|
* --------------
|
|
1421
1618
|
*
|
|
1422
|
-
* // Returns
|
|
1619
|
+
* // Returns text0 because text0 is the first element above button
|
|
1423
1620
|
* ...text().above().button()
|
|
1621
|
+
* ...text().above(0).button()
|
|
1622
|
+
* // Returns text1 because text1 is the second element above button
|
|
1623
|
+
* ...text().above(1).button()
|
|
1424
1624
|
* // Returns no element because button is below text
|
|
1425
1625
|
* ...button().above().text()
|
|
1426
1626
|
* ```
|
|
@@ -1504,12 +1704,12 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1504
1704
|
* **Examples:**
|
|
1505
1705
|
* ```typescript
|
|
1506
1706
|
* // Stops execution at this point when the element does not exist.
|
|
1507
|
-
* await aui.expect().text(
|
|
1707
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1508
1708
|
*
|
|
1509
1709
|
* // This will catch the error and log a message
|
|
1510
1710
|
* // But the execution will continue afterwards
|
|
1511
1711
|
* try {
|
|
1512
|
-
* await aui.expect().text(
|
|
1712
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1513
1713
|
* } catch (error) {
|
|
1514
1714
|
* console.log('Too bad we could not find the element!');
|
|
1515
1715
|
* }
|
|
@@ -1532,12 +1732,12 @@ export class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
|
|
|
1532
1732
|
* **Examples:**
|
|
1533
1733
|
* ```typescript
|
|
1534
1734
|
* // Stops execution at this point when the element does exist.
|
|
1535
|
-
* await aui.expect().text(
|
|
1735
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1536
1736
|
*
|
|
1537
1737
|
* // This will catch the error and log a message
|
|
1538
1738
|
* // But the execution will continue afterwards
|
|
1539
1739
|
* try {
|
|
1540
|
-
* await aui.expect().text(
|
|
1740
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1541
1741
|
* } catch (error) {
|
|
1542
1742
|
* console.log('Too bad we could find the element!');
|
|
1543
1743
|
* }
|
|
@@ -1566,8 +1766,8 @@ export class FluentCommand extends FluentBase {
|
|
|
1566
1766
|
*
|
|
1567
1767
|
* **Examples:**
|
|
1568
1768
|
* ```typescript
|
|
1569
|
-
* await aui.expect().text(
|
|
1570
|
-
* await aui.expect().text(
|
|
1769
|
+
* await aui.expect().text('Login').exists().exec()
|
|
1770
|
+
* await aui.expect().text('Login').notExists().exec()
|
|
1571
1771
|
* ```
|
|
1572
1772
|
*
|
|
1573
1773
|
* @return {FluentFiltersCondition}
|
|
@@ -1584,9 +1784,13 @@ export class FluentCommand extends FluentBase {
|
|
|
1584
1784
|
*
|
|
1585
1785
|
* **Example:**
|
|
1586
1786
|
* ```typescript
|
|
1587
|
-
* await aui.click().button().withText('
|
|
1787
|
+
* await aui.click().button().withText('Google Search').exec();
|
|
1588
1788
|
* ```
|
|
1589
1789
|
*
|
|
1790
|
+
* 
|
|
1791
|
+
*
|
|
1792
|
+
*
|
|
1793
|
+
*
|
|
1590
1794
|
* @return {FluentFilters}
|
|
1591
1795
|
*/
|
|
1592
1796
|
click() {
|
|
@@ -1600,9 +1804,13 @@ export class FluentCommand extends FluentBase {
|
|
|
1600
1804
|
*
|
|
1601
1805
|
* **Example:**
|
|
1602
1806
|
* ```typescript
|
|
1603
|
-
* await aui.moveMouseTo().
|
|
1807
|
+
* await aui.moveMouseTo().text().withText('Grinning_Face').exec()
|
|
1604
1808
|
* ```
|
|
1605
1809
|
*
|
|
1810
|
+
* 
|
|
1811
|
+
*
|
|
1812
|
+
*
|
|
1813
|
+
*
|
|
1606
1814
|
* @return {FluentFilters}
|
|
1607
1815
|
*/
|
|
1608
1816
|
moveMouseTo() {
|
|
@@ -1644,9 +1852,11 @@ export class FluentCommand extends FluentBase {
|
|
|
1644
1852
|
*
|
|
1645
1853
|
* **Example:**
|
|
1646
1854
|
* ```typescript
|
|
1647
|
-
* await aui.
|
|
1855
|
+
* await aui.scrollInside(0,-500).text().withText('Bottom sheet').exec();
|
|
1648
1856
|
* ```
|
|
1649
1857
|
*
|
|
1858
|
+
* 
|
|
1859
|
+
*
|
|
1650
1860
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1651
1861
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1652
1862
|
*
|
|
@@ -1723,12 +1933,14 @@ export class FluentCommand extends FluentBase {
|
|
|
1723
1933
|
*
|
|
1724
1934
|
* **Examples:**
|
|
1725
1935
|
* ```typescript
|
|
1726
|
-
* await aui.type('
|
|
1936
|
+
* await aui.type('askui@askui.com').exec()
|
|
1727
1937
|
*
|
|
1728
1938
|
* // mask the text so it is not send to the askui-inference server
|
|
1729
1939
|
* await aui.type('Type some text', { isSecret: true, secretMask: '**' }).exec()
|
|
1730
1940
|
* ```
|
|
1731
1941
|
*
|
|
1942
|
+
* 
|
|
1943
|
+
*
|
|
1732
1944
|
* @param {string} text - A text to type
|
|
1733
1945
|
*
|
|
1734
1946
|
* @return {Exec}
|
|
@@ -1744,9 +1956,11 @@ export class FluentCommand extends FluentBase {
|
|
|
1744
1956
|
*
|
|
1745
1957
|
* **Example:**
|
|
1746
1958
|
* ```typescript
|
|
1747
|
-
* await aui.moveMouseRelatively(
|
|
1959
|
+
* await aui.moveMouseRelatively(0, 50).exec();
|
|
1748
1960
|
* ```
|
|
1749
1961
|
*
|
|
1962
|
+
* 
|
|
1963
|
+
*
|
|
1750
1964
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1751
1965
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1752
1966
|
*
|
|
@@ -1776,6 +1990,8 @@ export class FluentCommand extends FluentBase {
|
|
|
1776
1990
|
* await aui.moveMouse(500, 500).exec();
|
|
1777
1991
|
* ```
|
|
1778
1992
|
*
|
|
1993
|
+
* 
|
|
1994
|
+
*
|
|
1779
1995
|
* @param {number} x_coordinate - A (positive/negative) x coordinate.
|
|
1780
1996
|
* @param {number} y_coordinate - A (positive/negative) y coordinate.
|
|
1781
1997
|
*
|
|
@@ -1801,10 +2017,12 @@ export class FluentCommand extends FluentBase {
|
|
|
1801
2017
|
*
|
|
1802
2018
|
* **Example:**
|
|
1803
2019
|
* ```typescript
|
|
1804
|
-
* // Scroll
|
|
1805
|
-
* await aui.scroll(0,
|
|
2020
|
+
* // Scroll 500 pixels down in y direction
|
|
2021
|
+
* await aui.scroll(0, -500).exec()
|
|
1806
2022
|
* ```
|
|
1807
2023
|
*
|
|
2024
|
+
* 
|
|
2025
|
+
*
|
|
1808
2026
|
* @param {number} x_offset - A (positive/negative) x direction.
|
|
1809
2027
|
* @param {number} y_offset - A (positive/negative) y direction.
|
|
1810
2028
|
*
|
|
@@ -1922,6 +2140,8 @@ export class FluentCommand extends FluentBase {
|
|
|
1922
2140
|
* await aui.mouseDoubleLeftClick().exec();
|
|
1923
2141
|
* ```
|
|
1924
2142
|
*
|
|
2143
|
+
* 
|
|
2144
|
+
*
|
|
1925
2145
|
* @return {Exec}
|
|
1926
2146
|
*/
|
|
1927
2147
|
mouseDoubleLeftClick() {
|
|
@@ -1972,11 +2192,17 @@ export class FluentCommand extends FluentBase {
|
|
|
1972
2192
|
/**
|
|
1973
2193
|
* Toggles mouse down (Left mouse key/tap).
|
|
1974
2194
|
*
|
|
2195
|
+
* This is the equivalent to **mouse-left-press-and-hold**. It holds the mouse button until the `mouseToogleUp()` is called. Often combined with `mouseToggleUP` to automate **drag-and-drop**.
|
|
2196
|
+
*
|
|
1975
2197
|
* **Example:**
|
|
1976
2198
|
* ```typescript
|
|
1977
2199
|
* await aui.mouseToggleDown().exec();
|
|
2200
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
2201
|
+
* await aui.mouseToggleUp().exec();
|
|
1978
2202
|
* ```
|
|
1979
2203
|
*
|
|
2204
|
+
* 
|
|
2205
|
+
*
|
|
1980
2206
|
* @return {Exec}
|
|
1981
2207
|
*/
|
|
1982
2208
|
mouseToggleDown() {
|
|
@@ -1987,11 +2213,17 @@ export class FluentCommand extends FluentBase {
|
|
|
1987
2213
|
/**
|
|
1988
2214
|
* Toggles mouse up (Left mouse key/tap).
|
|
1989
2215
|
*
|
|
2216
|
+
* This is the equivalent to releasing the pressing mouse left button. Often combined with `mouseToggleDown()` to automate **drag-and-drop**.
|
|
2217
|
+
*
|
|
1990
2218
|
* **Example:**
|
|
1991
2219
|
* ```typescript
|
|
2220
|
+
* await aui.mouseToggleDown().exec();
|
|
2221
|
+
* await aui.moveMouseRelatively(-400,0).exec();
|
|
1992
2222
|
* await aui.mouseToggleUp().exec();
|
|
1993
2223
|
* ```
|
|
1994
2224
|
*
|
|
2225
|
+
* 
|
|
2226
|
+
*
|
|
1995
2227
|
* @return {Exec}
|
|
1996
2228
|
*/
|
|
1997
2229
|
mouseToggleUp() {
|
|
@@ -2004,8 +2236,13 @@ export class FluentCommand extends FluentBase {
|
|
|
2004
2236
|
*
|
|
2005
2237
|
* **Operating system specific mappings:**
|
|
2006
2238
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
2007
|
-
* ---
|
|
2008
2239
|
*
|
|
2240
|
+
* **Examples:**
|
|
2241
|
+
* ```typescript
|
|
2242
|
+
* await aui.pressThreeKeys('control', 'command' 'space').exec();
|
|
2243
|
+
* ```
|
|
2244
|
+
*
|
|
2245
|
+
* 
|
|
2009
2246
|
*
|
|
2010
2247
|
* @param {MODIFIER_KEY} first_key - A modifier key
|
|
2011
2248
|
* @param {MODIFIER_KEY} second_key - A modifier key
|
|
@@ -2048,8 +2285,13 @@ export class FluentCommand extends FluentBase {
|
|
|
2048
2285
|
*
|
|
2049
2286
|
* **Operating system specific mappings:**
|
|
2050
2287
|
* 1. Windows: `command`-key maps to `windows`-key
|
|
2051
|
-
* ---
|
|
2052
2288
|
*
|
|
2289
|
+
* **Examples:**
|
|
2290
|
+
* ```typescript
|
|
2291
|
+
* await aui.pressKey('tab').exec();
|
|
2292
|
+
* ```
|
|
2293
|
+
*
|
|
2294
|
+
* 
|
|
2053
2295
|
*
|
|
2054
2296
|
* @param {PC_AND_MODIFIER_KEY} key - A key
|
|
2055
2297
|
*
|
|
@@ -2082,7 +2324,15 @@ export class FluentCommand extends FluentBase {
|
|
|
2082
2324
|
return new Exec(this);
|
|
2083
2325
|
}
|
|
2084
2326
|
/**
|
|
2085
|
-
* Press two Android keys like `
|
|
2327
|
+
* Press two Android keys like `volume_down+power`
|
|
2328
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
2329
|
+
*
|
|
2330
|
+
* **Examples:**
|
|
2331
|
+
* ```typescript
|
|
2332
|
+
* await aui.pressAndroidTwoKey('volume_down', 'power').exec();
|
|
2333
|
+
* ```
|
|
2334
|
+
*
|
|
2335
|
+
* 
|
|
2086
2336
|
*
|
|
2087
2337
|
* @param {ANDROID_KEY} first_key - A Android key
|
|
2088
2338
|
* @param {ANDROID_KEY} second_key - A Android key
|
|
@@ -2099,7 +2349,15 @@ export class FluentCommand extends FluentBase {
|
|
|
2099
2349
|
return new Exec(this);
|
|
2100
2350
|
}
|
|
2101
2351
|
/**
|
|
2102
|
-
* Press one Android key like `
|
|
2352
|
+
* Press one Android key like `del`
|
|
2353
|
+
* See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
|
|
2354
|
+
*
|
|
2355
|
+
* **Examples:**
|
|
2356
|
+
* ```typescript
|
|
2357
|
+
* await aui.pressAndroidKey('notification').exec();
|
|
2358
|
+
* ```
|
|
2359
|
+
*
|
|
2360
|
+
* 
|
|
2103
2361
|
*
|
|
2104
2362
|
* @param {ANDROID_KEY} key - A Android key
|
|
2105
2363
|
*
|
|
@@ -2206,9 +2464,11 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2206
2464
|
*
|
|
2207
2465
|
* **Examples:**
|
|
2208
2466
|
* ```typescript
|
|
2209
|
-
* await aui.
|
|
2467
|
+
* await aui.click().button().contains().text().withText('Google Search').exec()
|
|
2210
2468
|
* ```
|
|
2211
2469
|
*
|
|
2470
|
+
* 
|
|
2471
|
+
*
|
|
2212
2472
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2213
2473
|
*/
|
|
2214
2474
|
button() {
|
|
@@ -2268,13 +2528,20 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2268
2528
|
/**
|
|
2269
2529
|
* Filters for an UI element 'text'.
|
|
2270
2530
|
*
|
|
2271
|
-
*
|
|
2531
|
+
* Takes an optional parameter to filter for a specific text.
|
|
2532
|
+
* See the examples below.
|
|
2533
|
+
*
|
|
2272
2534
|
* See also the filters `withTextRegex()` and `withExactText()`
|
|
2273
2535
|
*
|
|
2274
2536
|
* **Examples:**
|
|
2275
2537
|
* ```typescript
|
|
2276
|
-
* await aui.click().text().
|
|
2538
|
+
* await aui.click().text().exec();
|
|
2539
|
+
* await aui.click().text('Username').exec();
|
|
2540
|
+
*
|
|
2541
|
+
* // Matching with an exact text
|
|
2277
2542
|
* await aui.click().text().withExactText('Username').exec();
|
|
2543
|
+
*
|
|
2544
|
+
* // Matching with a regex
|
|
2278
2545
|
* await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
|
|
2279
2546
|
* ```
|
|
2280
2547
|
*
|
|
@@ -2298,6 +2565,8 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2298
2565
|
* icon().withText('plus')
|
|
2299
2566
|
* ```
|
|
2300
2567
|
*
|
|
2568
|
+
* 
|
|
2569
|
+
*
|
|
2301
2570
|
* **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
|
|
2302
2571
|
*
|
|
2303
2572
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2308,7 +2577,9 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2308
2577
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2309
2578
|
}
|
|
2310
2579
|
/**
|
|
2311
|
-
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element
|
|
2580
|
+
* Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
|
|
2581
|
+
*
|
|
2582
|
+
* See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
|
|
2312
2583
|
*
|
|
2313
2584
|
* **Example**
|
|
2314
2585
|
* ```typescript
|
|
@@ -2318,6 +2589,7 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2318
2589
|
* customImage: './logo.png', // required
|
|
2319
2590
|
* name: 'myLogo', // optional
|
|
2320
2591
|
* threshold: 0.9, // optional, defaults to 0.9
|
|
2592
|
+
* stopThreshold: 0.9, // optional, defaults to 0.9
|
|
2321
2593
|
* rotationDegreePerStep: 0, // optional, defaults to 0
|
|
2322
2594
|
* imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
|
|
2323
2595
|
* // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
|
|
@@ -2332,12 +2604,13 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2332
2604
|
* - **name** (*`string`, optional*):
|
|
2333
2605
|
* - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
|
|
2334
2606
|
* - **threshold** (*`number`, optional*):
|
|
2335
|
-
* - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
2607
|
+
* - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
|
|
2608
|
+
* - **stopThreshold** (*`number`, optional*):
|
|
2609
|
+
* - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
|
|
2336
2610
|
* - **rotationDegreePerStep** (*`number`, optional*):
|
|
2337
2611
|
* - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
|
|
2338
|
-
* - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
|
|
2339
|
-
* - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three
|
|
2340
|
-
* of the given custom image.
|
|
2612
|
+
* - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
|
|
2613
|
+
* - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
|
|
2341
2614
|
*
|
|
2342
2615
|
*
|
|
2343
2616
|
* @param {CustomElementJson} customElement - The custom element to filter for.
|
|
@@ -2351,6 +2624,23 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2351
2624
|
this._params.set('customElement', customElement);
|
|
2352
2625
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2353
2626
|
}
|
|
2627
|
+
/**
|
|
2628
|
+
* Detects an AI Element created with the workflow creator.
|
|
2629
|
+
*
|
|
2630
|
+
* @param {string} aiElementName - Name of the AI Element.
|
|
2631
|
+
*
|
|
2632
|
+
* @return {FluentFiltersOrRelationsGetter}
|
|
2633
|
+
*/
|
|
2634
|
+
aiElement(aiElementName) {
|
|
2635
|
+
this._textStr = '';
|
|
2636
|
+
this._textStr += 'ai';
|
|
2637
|
+
this._textStr += ' element';
|
|
2638
|
+
this._textStr += ' with';
|
|
2639
|
+
this._textStr += ' name';
|
|
2640
|
+
this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
|
|
2641
|
+
this._params.set('aiElementName', aiElementName);
|
|
2642
|
+
return new FluentFiltersOrRelationsGetter(this);
|
|
2643
|
+
}
|
|
2354
2644
|
/**
|
|
2355
2645
|
* Filters for a UI element 'image'.
|
|
2356
2646
|
*
|
|
@@ -2360,10 +2650,14 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2360
2650
|
* await aui.click().image().exec();
|
|
2361
2651
|
*
|
|
2362
2652
|
* // Works if you have an image with
|
|
2363
|
-
* // a
|
|
2364
|
-
* await aui.click().image().above().text().withText('
|
|
2653
|
+
* // a text below
|
|
2654
|
+
* await aui.click().image().above().text().withText('Automating WebGL').exec();
|
|
2365
2655
|
* ```
|
|
2366
2656
|
*
|
|
2657
|
+
* 
|
|
2658
|
+
*
|
|
2659
|
+
*
|
|
2660
|
+
*
|
|
2367
2661
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2368
2662
|
*/
|
|
2369
2663
|
image() {
|
|
@@ -2384,6 +2678,10 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2384
2678
|
* await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
|
|
2385
2679
|
* ```
|
|
2386
2680
|
*
|
|
2681
|
+
* 
|
|
2682
|
+
*
|
|
2683
|
+
*
|
|
2684
|
+
*
|
|
2387
2685
|
* @return {FluentFiltersOrRelationsGetter}
|
|
2388
2686
|
*/
|
|
2389
2687
|
textfield() {
|
|
@@ -2392,7 +2690,13 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2392
2690
|
return new FluentFiltersOrRelationsGetter(this);
|
|
2393
2691
|
}
|
|
2394
2692
|
/**
|
|
2395
|
-
* Filters for similar
|
|
2693
|
+
* Filters for similar -- meaning >70% similar -- text.
|
|
2694
|
+
*
|
|
2695
|
+
* Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
|
|
2696
|
+
*
|
|
2697
|
+
* _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
|
|
2698
|
+
*
|
|
2699
|
+
* `1 - (distance / (lengthString1 + lengthString2))`
|
|
2396
2700
|
*
|
|
2397
2701
|
* **Examples:**
|
|
2398
2702
|
* ```typescript
|
|
@@ -2407,6 +2711,11 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2407
2711
|
* // usually false
|
|
2408
2712
|
* 'atebxtc' === withText('text') => false
|
|
2409
2713
|
* 'other' === withText('text') => false
|
|
2714
|
+
*
|
|
2715
|
+
* // optional parameter: similarity_score
|
|
2716
|
+
* '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
|
|
2717
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
|
|
2718
|
+
* '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
|
|
2410
2719
|
* ```
|
|
2411
2720
|
* 
|
|
2412
2721
|
*
|
|
@@ -2437,6 +2746,10 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2437
2746
|
* await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
|
|
2438
2747
|
* ```
|
|
2439
2748
|
*
|
|
2749
|
+
* 
|
|
2750
|
+
*
|
|
2751
|
+
*
|
|
2752
|
+
*
|
|
2440
2753
|
* @param {string} regex_pattern - A regex pattern
|
|
2441
2754
|
*
|
|
2442
2755
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2465,6 +2778,10 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2465
2778
|
* await aui.moveMouseTo().text().withExactText('Password').exec()
|
|
2466
2779
|
* ```
|
|
2467
2780
|
*
|
|
2781
|
+
* 
|
|
2782
|
+
*
|
|
2783
|
+
*
|
|
2784
|
+
*
|
|
2468
2785
|
* @param {string} text - A text to be matched.
|
|
2469
2786
|
*
|
|
2470
2787
|
* @return {FluentFiltersOrRelationsGetter}
|
|
@@ -2501,17 +2818,28 @@ export class FluentFiltersGetter extends FluentBase {
|
|
|
2501
2818
|
/**
|
|
2502
2819
|
* Filters elements based on a textual description.
|
|
2503
2820
|
*
|
|
2504
|
-
*
|
|
2821
|
+
* **What Should I Write as Matching Text**
|
|
2822
|
+
*
|
|
2505
2823
|
* The text description inside the `matching()` should describe the element visually.
|
|
2506
2824
|
* It understands color, some famous company/product names, general descriptions.
|
|
2507
2825
|
*
|
|
2508
|
-
* It sometimes requires a bit of playing to find a matching description:
|
|
2509
|
-
* E.g. `puzzle piece` can fail
|
|
2826
|
+
* It sometimes requires a bit of playing around to find a matching description:
|
|
2827
|
+
* E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
|
|
2510
2828
|
* Generally the more detail the better.
|
|
2511
2829
|
*
|
|
2512
2830
|
* **Examples:**
|
|
2513
2831
|
* ```typescript
|
|
2514
|
-
*
|
|
2832
|
+
* // Select the black sneaker from a bunch of sneakers
|
|
2833
|
+
* await aui.click().element().matching('a black sneaker shoe').exec();
|
|
2834
|
+
*
|
|
2835
|
+
* // Select an image that has text in it
|
|
2836
|
+
* await aui.click().element().matching('has Burger King in it').exec();
|
|
2837
|
+
* await aui.click().element().matching('has adidas in it').exec();
|
|
2838
|
+
*
|
|
2839
|
+
* // Target a logo/image by describing it
|
|
2840
|
+
* await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
|
|
2841
|
+
* await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
|
|
2842
|
+
* await aui.click().element().matching('logo looking like a seashell').exec();
|
|
2515
2843
|
* ```
|
|
2516
2844
|
*
|
|
2517
2845
|
* @param {string} text - A description of the target element.
|
|
@@ -2669,14 +2997,19 @@ export class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2669
2997
|
/**
|
|
2670
2998
|
* Filters for an element right of another element.
|
|
2671
2999
|
*
|
|
3000
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3001
|
+
*
|
|
2672
3002
|
* **Examples:**
|
|
2673
3003
|
* ```typescript
|
|
2674
|
-
* -------------- --------------
|
|
2675
|
-
* | leftEl | |
|
|
2676
|
-
* -------------- --------------
|
|
3004
|
+
* -------------- -------------- --------------
|
|
3005
|
+
* | leftEl | | rightEl0 | | rightEl1 |
|
|
3006
|
+
* -------------- -------------- --------------
|
|
2677
3007
|
*
|
|
2678
|
-
* // Returns
|
|
3008
|
+
* // Returns rightEl0 because rightEl0 is the first element right of leftEl
|
|
2679
3009
|
* ...rightEl().rightOf().leftEl()
|
|
3010
|
+
* ...rightEl().rightOf(0).leftEl()
|
|
3011
|
+
* // Returns rightEl1 because rightEl1 is the second element right of leftEl
|
|
3012
|
+
* ...rightEl().rightOf(1).leftEl()
|
|
2680
3013
|
* // Returns no element because leftEl is left of rightEl
|
|
2681
3014
|
* ...leftEl().rightOf().rightEl()
|
|
2682
3015
|
* ```
|
|
@@ -2696,14 +3029,19 @@ export class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2696
3029
|
/**
|
|
2697
3030
|
* Filters for an element left of another element.
|
|
2698
3031
|
*
|
|
3032
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3033
|
+
*
|
|
2699
3034
|
* **Examples:**
|
|
2700
3035
|
* ```typescript
|
|
2701
|
-
* -------------- --------------
|
|
2702
|
-
* |
|
|
2703
|
-
* -------------- --------------
|
|
3036
|
+
* -------------- -------------- --------------
|
|
3037
|
+
* | leftEl1 | | leftEl0 | | rightEl |
|
|
3038
|
+
* -------------- -------------- --------------
|
|
2704
3039
|
*
|
|
2705
|
-
* // Returns
|
|
3040
|
+
* // Returns leftEl0 because leftEl0 is the first element left of rightEl
|
|
2706
3041
|
* ...leftEl().leftOf().rightEl()
|
|
3042
|
+
* ...leftEl().leftOf(0).rightEl()
|
|
3043
|
+
* // Returns leftEl1 because leftEl1 is the second element left of rightEl
|
|
3044
|
+
* ...leftEl().leftOf(1).rightEl()
|
|
2707
3045
|
* // Returns no element because rightEl is left of leftEl
|
|
2708
3046
|
* ...rightEl().leftOf().leftEl()
|
|
2709
3047
|
* ```
|
|
@@ -2723,17 +3061,25 @@ export class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2723
3061
|
/**
|
|
2724
3062
|
* Filters for an element below another element.
|
|
2725
3063
|
*
|
|
3064
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3065
|
+
*
|
|
2726
3066
|
* **Examples:**
|
|
2727
3067
|
* ```typescript
|
|
2728
3068
|
* --------------
|
|
2729
3069
|
* | text |
|
|
2730
3070
|
* --------------
|
|
2731
3071
|
* --------------
|
|
2732
|
-
* |
|
|
3072
|
+
* | button0 |
|
|
3073
|
+
* --------------
|
|
3074
|
+
* --------------
|
|
3075
|
+
* | button1 |
|
|
2733
3076
|
* --------------
|
|
2734
3077
|
*
|
|
2735
|
-
* // Returns
|
|
3078
|
+
* // Returns button0 because button0 is the first button below text
|
|
2736
3079
|
* ...button().below().text()
|
|
3080
|
+
* ...button().below(0).text()
|
|
3081
|
+
* // Returns button1 because button1 is the second button below text
|
|
3082
|
+
* ...button().below(1).text()
|
|
2737
3083
|
* // Returns no element because text is above button
|
|
2738
3084
|
* ...text().below().button()
|
|
2739
3085
|
* ```
|
|
@@ -2752,17 +3098,25 @@ export class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
|
|
|
2752
3098
|
/**
|
|
2753
3099
|
* Filters for an element above another element.
|
|
2754
3100
|
*
|
|
3101
|
+
* Takes an optional parameter `index` to select the `nth` element (starting with 0)
|
|
3102
|
+
*
|
|
2755
3103
|
* **Examples:**
|
|
2756
3104
|
* ```typescript
|
|
2757
3105
|
* --------------
|
|
2758
|
-
* |
|
|
3106
|
+
* | text1 |
|
|
3107
|
+
* --------------
|
|
3108
|
+
* --------------
|
|
3109
|
+
* | text0 |
|
|
2759
3110
|
* --------------
|
|
2760
3111
|
* --------------
|
|
2761
3112
|
* | button |
|
|
2762
3113
|
* --------------
|
|
2763
3114
|
*
|
|
2764
|
-
* // Returns
|
|
3115
|
+
* // Returns text0 because text0 is the first element above button
|
|
2765
3116
|
* ...text().above().button()
|
|
3117
|
+
* ...text().above(0).button()
|
|
3118
|
+
* // Returns text1 because text1 is the second element above button
|
|
3119
|
+
* ...text().above(1).button()
|
|
2766
3120
|
* // Returns no element because button is below text
|
|
2767
3121
|
* ...button().above().text()
|
|
2768
3122
|
* ```
|
|
@@ -2856,22 +3210,74 @@ export class Getter extends FluentCommand {
|
|
|
2856
3210
|
*
|
|
2857
3211
|
* **Examples:**
|
|
2858
3212
|
* ```typescript
|
|
2859
|
-
* const text = await aui.get().text(
|
|
3213
|
+
* const text = await aui.get().text('Sign').exec();
|
|
2860
3214
|
* console.log(text);
|
|
3215
|
+
*
|
|
3216
|
+
* // Console output
|
|
3217
|
+
* [
|
|
3218
|
+
* DetectedElement {
|
|
3219
|
+
* name: 'TEXT',
|
|
3220
|
+
* text: 'Sign In',
|
|
3221
|
+
* bndbox: BoundingBox {
|
|
3222
|
+
* xmin: 1128.2720982142857,
|
|
3223
|
+
* ymin: 160.21332310267857,
|
|
3224
|
+
* xmax: 1178.8204241071428,
|
|
3225
|
+
* ymax: 180.83512834821428
|
|
3226
|
+
* }
|
|
3227
|
+
* }
|
|
3228
|
+
* ]
|
|
2861
3229
|
* ```
|
|
2862
|
-
*
|
|
2863
|
-
*
|
|
3230
|
+
*
|
|
3231
|
+
* // *************************************************** //
|
|
3232
|
+
* // Examples on how to work with the returned elements //
|
|
3233
|
+
* // *************************************************** //
|
|
3234
|
+
* const texts = await aui.get().text().below().textfield().exec();
|
|
3235
|
+
*
|
|
3236
|
+
* // We can get a lot of elements this way
|
|
3237
|
+
* console.log(texts);
|
|
3238
|
+
*
|
|
3239
|
+
* // Console output
|
|
3240
|
+
* [
|
|
2864
3241
|
* DetectedElement {
|
|
2865
|
-
*
|
|
2866
|
-
*
|
|
2867
|
-
*
|
|
2868
|
-
*
|
|
2869
|
-
*
|
|
2870
|
-
*
|
|
2871
|
-
*
|
|
2872
|
-
*
|
|
2873
|
-
*
|
|
2874
|
-
*
|
|
3242
|
+
* name: 'TEXT',
|
|
3243
|
+
* text: 'Sign In',
|
|
3244
|
+
* bndbox: BoundingBox {
|
|
3245
|
+
* xmin: 1128.2720982142857,
|
|
3246
|
+
* ymin: 160.21332310267857,
|
|
3247
|
+
* xmax: 1178.8204241071428,
|
|
3248
|
+
* ymax: 180.83512834821428
|
|
3249
|
+
* },
|
|
3250
|
+
* },
|
|
3251
|
+
* DetectedElement {
|
|
3252
|
+
* name: 'TEXT',
|
|
3253
|
+
* text: 'Login',
|
|
3254
|
+
* bndbox: BoundingBox {
|
|
3255
|
+
* xmin: 250.8204241071428,
|
|
3256
|
+
* ymin: 300.21332310267857,
|
|
3257
|
+
* xmax: 450.6304241071428,
|
|
3258
|
+
* ymax: 950.47812834821428
|
|
3259
|
+
* },
|
|
3260
|
+
* },
|
|
3261
|
+
* ... 10 more items
|
|
3262
|
+
* ]
|
|
3263
|
+
*
|
|
3264
|
+
* // Extract the FIRST element
|
|
3265
|
+
* // Arrays start with index 0!
|
|
3266
|
+
* const firstTextElement = texts[0];
|
|
3267
|
+
* const textOfFirstElement = firstElement.text;
|
|
3268
|
+
*
|
|
3269
|
+
* console.log(textOfFirstElement);
|
|
3270
|
+
*
|
|
3271
|
+
* // Console output
|
|
3272
|
+
* Sign In
|
|
3273
|
+
*
|
|
3274
|
+
* // Log the text of the SECOND element
|
|
3275
|
+
* // with shorter code
|
|
3276
|
+
* const texts = await aui.get().text().below().textfield().exec();
|
|
3277
|
+
* console.log(texts[1].text)
|
|
3278
|
+
*
|
|
3279
|
+
* // Console output
|
|
3280
|
+
* Login
|
|
2875
3281
|
* ```
|
|
2876
3282
|
*
|
|
2877
3283
|
* @return {FluentFiltersGetter}
|