askui 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/cjs/core/ai-element/ai-element-collection.d.ts +10 -0
  2. package/dist/cjs/core/ai-element/ai-element-collection.js +78 -0
  3. package/dist/cjs/core/ai-element/ai-element-error.d.ts +2 -0
  4. package/dist/cjs/core/ai-element/ai-element-error.js +6 -0
  5. package/dist/cjs/core/ai-element/ai-element.d.ts +27 -0
  6. package/dist/cjs/core/ai-element/ai-element.js +31 -0
  7. package/dist/cjs/core/model/custom-element-json.d.ts +32 -15
  8. package/dist/cjs/core/model/custom-element.d.ts +3 -2
  9. package/dist/cjs/core/model/custom-element.js +4 -2
  10. package/dist/cjs/core/reporting/step-reporter.js +15 -6
  11. package/dist/cjs/execution/dsl.d.ts +488 -113
  12. package/dist/cjs/execution/dsl.js +519 -113
  13. package/dist/cjs/execution/index.d.ts +1 -1
  14. package/dist/cjs/execution/index.js +11 -3
  15. package/dist/cjs/execution/inference-client.js +9 -4
  16. package/dist/cjs/execution/ui-control-client-dependency-builder.d.ts +1 -0
  17. package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -3
  18. package/dist/cjs/execution/ui-control-client.d.ts +101 -16
  19. package/dist/cjs/execution/ui-control-client.js +142 -52
  20. package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
  21. package/dist/cjs/execution/ui-controller-client.d.ts +1 -0
  22. package/dist/cjs/execution/ui-controller-client.js +11 -1
  23. package/dist/cjs/execution/ui-controller-not-connected-error.d.ts +4 -0
  24. package/dist/cjs/execution/ui-controller-not-connected-error.js +12 -0
  25. package/dist/cjs/main.d.ts +1 -1
  26. package/dist/cjs/main.js +12 -3
  27. package/dist/cjs/utils/analytics/analytics.d.ts +3 -1
  28. package/dist/cjs/utils/analytics/analytics.js +5 -0
  29. package/dist/cjs/utils/http/custom-errors/index.js +1 -1
  30. package/dist/cjs/utils/http/http-client-got.js +46 -20
  31. package/dist/esm/core/ai-element/ai-element-collection.d.ts +10 -0
  32. package/dist/esm/core/ai-element/ai-element-collection.js +71 -0
  33. package/dist/esm/core/ai-element/ai-element-error.d.ts +2 -0
  34. package/dist/esm/core/ai-element/ai-element-error.js +2 -0
  35. package/dist/esm/core/ai-element/ai-element.d.ts +27 -0
  36. package/dist/esm/core/ai-element/ai-element.js +28 -0
  37. package/dist/esm/core/model/custom-element-json.d.ts +32 -15
  38. package/dist/esm/core/model/custom-element.d.ts +3 -2
  39. package/dist/esm/core/model/custom-element.js +4 -2
  40. package/dist/esm/core/reporting/step-reporter.js +15 -6
  41. package/dist/esm/execution/dsl.d.ts +488 -113
  42. package/dist/esm/execution/dsl.js +519 -113
  43. package/dist/esm/execution/index.d.ts +1 -1
  44. package/dist/esm/execution/index.js +1 -1
  45. package/dist/esm/execution/inference-client.js +9 -4
  46. package/dist/esm/execution/ui-control-client-dependency-builder.d.ts +1 -0
  47. package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -3
  48. package/dist/esm/execution/ui-control-client.d.ts +101 -16
  49. package/dist/esm/execution/ui-control-client.js +142 -49
  50. package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
  51. package/dist/esm/execution/ui-controller-client.d.ts +1 -0
  52. package/dist/esm/execution/ui-controller-client.js +11 -1
  53. package/dist/esm/execution/ui-controller-not-connected-error.d.ts +4 -0
  54. package/dist/esm/execution/ui-controller-not-connected-error.js +8 -0
  55. package/dist/esm/main.d.ts +1 -1
  56. package/dist/esm/main.js +1 -1
  57. package/dist/esm/utils/analytics/analytics.d.ts +3 -1
  58. package/dist/esm/utils/analytics/analytics.js +5 -0
  59. package/dist/esm/utils/http/custom-errors/index.js +1 -1
  60. package/dist/esm/utils/http/http-client-got.js +46 -20
  61. package/package.json +1 -1
@@ -31,7 +31,11 @@ class FluentBase {
31
31
  if (this instanceof FluentCommand) {
32
32
  const fluentCommand = this;
33
33
  const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
34
- return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), customElements);
34
+ const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
35
+ return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), {
36
+ customElementsJson: customElements,
37
+ aiElementNames,
38
+ });
35
39
  }
36
40
  if (!this.prev) {
37
41
  throw new Error('Prev element not defined');
@@ -44,7 +48,11 @@ class FluentBase {
44
48
  if (this instanceof Getter) {
45
49
  const getter = this;
46
50
  const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
47
- return getter.getterExecutor(newCurrentInstruction.trim(), customElements);
51
+ const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
52
+ return getter.getterExecutor(newCurrentInstruction.trim(), {
53
+ customElementsJson: customElements,
54
+ aiElementNames,
55
+ });
48
56
  }
49
57
  if (!this.prev) {
50
58
  throw new Error('Prev element not defined');
@@ -147,9 +155,11 @@ class FluentFilters extends FluentBase {
147
155
  *
148
156
  * **Examples:**
149
157
  * ```typescript
150
- * await aui.moveMouseTo().button().exec()
158
+ * await aui.click().button().contains().text().withText('Google Search').exec()
151
159
  * ```
152
160
  *
161
+ * ![](https://docs.askui.com/img/gif/button.gif)
162
+ *
153
163
  * @return {FluentFiltersOrRelations}
154
164
  */
155
165
  button() {
@@ -209,13 +219,20 @@ class FluentFilters extends FluentBase {
209
219
  /**
210
220
  * Filters for an UI element 'text'.
211
221
  *
212
- * Often combined with the filter `withText()` as shown in the below examples.
222
+ * Takes an optional parameter to filter for a specific text.
223
+ * See the examples below.
224
+ *
213
225
  * See also the filters `withTextRegex()` and `withExactText()`
214
226
  *
215
227
  * **Examples:**
216
228
  * ```typescript
217
- * await aui.click().text().withText('Password').exec();
229
+ * await aui.click().text().exec();
230
+ * await aui.click().text('Username').exec();
231
+ *
232
+ * // Matching with an exact text
218
233
  * await aui.click().text().withExactText('Username').exec();
234
+ *
235
+ * // Matching with a regex
219
236
  * await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
220
237
  * ```
221
238
  *
@@ -239,6 +256,8 @@ class FluentFilters extends FluentBase {
239
256
  * icon().withText('plus')
240
257
  * ```
241
258
  *
259
+ * ![](https://docs.askui.com/img/gif/icon.gif)
260
+ *
242
261
  * **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
243
262
  *
244
263
  * @return {FluentFiltersOrRelations}
@@ -249,7 +268,9 @@ class FluentFilters extends FluentBase {
249
268
  return new FluentFiltersOrRelations(this);
250
269
  }
251
270
  /**
252
- * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element that is not recognized by our machine learning models by default. It can also be used for pixel assertions of elements using classical [template matching](https://en.wikipedia.org/wiki/Template_matching).
271
+ * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
272
+ *
273
+ * See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
253
274
  *
254
275
  * **Example**
255
276
  * ```typescript
@@ -259,6 +280,7 @@ class FluentFilters extends FluentBase {
259
280
  * customImage: './logo.png', // required
260
281
  * name: 'myLogo', // optional
261
282
  * threshold: 0.9, // optional, defaults to 0.9
283
+ * stopThreshold: 0.9, // optional, defaults to 0.9
262
284
  * rotationDegreePerStep: 0, // optional, defaults to 0
263
285
  * imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
264
286
  * // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
@@ -273,12 +295,13 @@ class FluentFilters extends FluentBase {
273
295
  * - **name** (*`string`, optional*):
274
296
  * - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
275
297
  * - **threshold** (*`number`, optional*):
276
- * - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
298
+ * - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
299
+ * - **stopThreshold** (*`number`, optional*):
300
+ * - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
277
301
  * - **rotationDegreePerStep** (*`number`, optional*):
278
302
  * - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
279
- * - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
280
- * - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three color. Defaults to 'grayscale'.
281
- * of the given custom image.
303
+ * - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
304
+ * - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
282
305
  *
283
306
  *
284
307
  * @param {CustomElementJson} customElement - The custom element to filter for.
@@ -292,6 +315,23 @@ class FluentFilters extends FluentBase {
292
315
  this._params.set('customElement', customElement);
293
316
  return new FluentFiltersOrRelations(this);
294
317
  }
318
+ /**
319
+ * Detects an AI Element created with the workflow creator.
320
+ *
321
+ * @param {string} aiElementName - Name of the AI Element.
322
+ *
323
+ * @return {FluentFiltersOrRelations}
324
+ */
325
+ aiElement(aiElementName) {
326
+ this._textStr = '';
327
+ this._textStr += 'ai';
328
+ this._textStr += ' element';
329
+ this._textStr += ' with';
330
+ this._textStr += ' name';
331
+ this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
332
+ this._params.set('aiElementName', aiElementName);
333
+ return new FluentFiltersOrRelations(this);
334
+ }
295
335
  /**
296
336
  * Filters for a UI element 'image'.
297
337
  *
@@ -301,10 +341,14 @@ class FluentFilters extends FluentBase {
301
341
  * await aui.click().image().exec();
302
342
  *
303
343
  * // Works if you have an image with
304
- * // a caption text below
305
- * await aui.click().image().above().text().withText('The caption').exec();
344
+ * // a text below
345
+ * await aui.click().image().above().text().withText('Automating WebGL').exec();
306
346
  * ```
307
347
  *
348
+ * ![](https://docs.askui.com/img/gif/image.gif)
349
+ *
350
+ *
351
+ *
308
352
  * @return {FluentFiltersOrRelations}
309
353
  */
310
354
  image() {
@@ -325,6 +369,10 @@ class FluentFilters extends FluentBase {
325
369
  * await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
326
370
  * ```
327
371
  *
372
+ * ![](https://docs.askui.com/img/gif/textfield.gif)
373
+ *
374
+ *
375
+ *
328
376
  * @return {FluentFiltersOrRelations}
329
377
  */
330
378
  textfield() {
@@ -333,7 +381,13 @@ class FluentFilters extends FluentBase {
333
381
  return new FluentFiltersOrRelations(this);
334
382
  }
335
383
  /**
336
- * Filters for similar (doesn't need to be a 100% equal) text.
384
+ * Filters for similar -- meaning >70% similar -- text.
385
+ *
386
+ * Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
387
+ *
388
+ * _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
389
+ *
390
+ * `1 - (distance / (lengthString1 + lengthString2))`
337
391
  *
338
392
  * **Examples:**
339
393
  * ```typescript
@@ -348,6 +402,11 @@ class FluentFilters extends FluentBase {
348
402
  * // usually false
349
403
  * 'atebxtc' === withText('text') => false
350
404
  * 'other' === withText('text') => false
405
+ *
406
+ * // optional parameter: similarity_score
407
+ * '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
408
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
409
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
351
410
  * ```
352
411
  * ![](https://docs.askui.com/img/gif/withText.gif)
353
412
  *
@@ -378,6 +437,10 @@ class FluentFilters extends FluentBase {
378
437
  * await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
379
438
  * ```
380
439
  *
440
+ * ![](https://docs.askui.com/img/gif/withtextregex.gif)
441
+ *
442
+ *
443
+ *
381
444
  * @param {string} regex_pattern - A regex pattern
382
445
  *
383
446
  * @return {FluentFiltersOrRelations}
@@ -406,6 +469,10 @@ class FluentFilters extends FluentBase {
406
469
  * await aui.moveMouseTo().text().withExactText('Password').exec()
407
470
  * ```
408
471
  *
472
+ * ![](https://docs.askui.com/img/gif/withexacttext.gif)
473
+ *
474
+ *
475
+ *
409
476
  * @param {string} text - A text to be matched.
410
477
  *
411
478
  * @return {FluentFiltersOrRelations}
@@ -442,17 +509,28 @@ class FluentFilters extends FluentBase {
442
509
  /**
443
510
  * Filters elements based on a textual description.
444
511
  *
445
- * ## What Should I Write as Matching Text
512
+ * **What Should I Write as Matching Text**
513
+ *
446
514
  * The text description inside the `matching()` should describe the element visually.
447
515
  * It understands color, some famous company/product names, general descriptions.
448
516
  *
449
- * It sometimes requires a bit of playing to find a matching description:
450
- * E.g. `puzzle piece` can fail here while `an icon showing a puzzle piece` might work.
517
+ * It sometimes requires a bit of playing around to find a matching description:
518
+ * E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
451
519
  * Generally the more detail the better.
452
520
  *
453
521
  * **Examples:**
454
522
  * ```typescript
455
- * await aui.click().matching('a mask on purple background and a firefox logo').exec()
523
+ * // Select the black sneaker from a bunch of sneakers
524
+ * await aui.click().element().matching('a black sneaker shoe').exec();
525
+ *
526
+ * // Select an image that has text in it
527
+ * await aui.click().element().matching('has Burger King in it').exec();
528
+ * await aui.click().element().matching('has adidas in it').exec();
529
+ *
530
+ * // Target a logo/image by describing it
531
+ * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
532
+ * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
533
+ * await aui.click().element().matching('logo looking like a seashell').exec();
456
534
  * ```
457
535
  *
458
536
  * @param {string} text - A description of the target element.
@@ -611,14 +689,19 @@ class FluentFiltersOrRelations extends FluentFilters {
611
689
  /**
612
690
  * Filters for an element right of another element.
613
691
  *
692
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
693
+ *
614
694
  * **Examples:**
615
695
  * ```typescript
616
- * -------------- --------------
617
- * | leftEl | | rightEl |
618
- * -------------- --------------
696
+ * -------------- -------------- --------------
697
+ * | leftEl | | rightEl0 | | rightEl1 |
698
+ * -------------- -------------- --------------
619
699
  *
620
- * // Returns rightEl because rightEl is right of leftEl
700
+ * // Returns rightEl0 because rightEl0 is the first element right of leftEl
621
701
  * ...rightEl().rightOf().leftEl()
702
+ * ...rightEl().rightOf(0).leftEl()
703
+ * // Returns rightEl1 because rightEl1 is the second element right of leftEl
704
+ * ...rightEl().rightOf(1).leftEl()
622
705
  * // Returns no element because leftEl is left of rightEl
623
706
  * ...leftEl().rightOf().rightEl()
624
707
  * ```
@@ -638,14 +721,19 @@ class FluentFiltersOrRelations extends FluentFilters {
638
721
  /**
639
722
  * Filters for an element left of another element.
640
723
  *
724
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
725
+ *
641
726
  * **Examples:**
642
727
  * ```typescript
643
- * -------------- --------------
644
- * | leftEl | | rightEl |
645
- * -------------- --------------
728
+ * -------------- -------------- --------------
729
+ * | leftEl1 | | leftEl0 | | rightEl |
730
+ * -------------- -------------- --------------
646
731
  *
647
- * // Returns leftEl because leftEl is left of rightEl
732
+ * // Returns leftEl0 because leftEl0 is the first element left of rightEl
648
733
  * ...leftEl().leftOf().rightEl()
734
+ * ...leftEl().leftOf(0).rightEl()
735
+ * // Returns leftEl1 because leftEl1 is the second element left of rightEl
736
+ * ...leftEl().leftOf(1).rightEl()
649
737
  * // Returns no element because rightEl is left of leftEl
650
738
  * ...rightEl().leftOf().leftEl()
651
739
  * ```
@@ -665,17 +753,25 @@ class FluentFiltersOrRelations extends FluentFilters {
665
753
  /**
666
754
  * Filters for an element below another element.
667
755
  *
756
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
757
+ *
668
758
  * **Examples:**
669
759
  * ```typescript
670
760
  * --------------
671
761
  * | text |
672
762
  * --------------
673
763
  * --------------
674
- * | button |
764
+ * | button0 |
765
+ * --------------
766
+ * --------------
767
+ * | button1 |
675
768
  * --------------
676
769
  *
677
- * // Returns button because button is below text
770
+ * // Returns button0 because button0 is the first button below text
678
771
  * ...button().below().text()
772
+ * ...button().below(0).text()
773
+ * // Returns button1 because button1 is the second button below text
774
+ * ...button().below(1).text()
679
775
  * // Returns no element because text is above button
680
776
  * ...text().below().button()
681
777
  * ```
@@ -694,17 +790,25 @@ class FluentFiltersOrRelations extends FluentFilters {
694
790
  /**
695
791
  * Filters for an element above another element.
696
792
  *
793
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
794
+ *
697
795
  * **Examples:**
698
796
  * ```typescript
699
797
  * --------------
700
- * | text |
798
+ * | text1 |
799
+ * --------------
800
+ * --------------
801
+ * | text0 |
701
802
  * --------------
702
803
  * --------------
703
804
  * | button |
704
805
  * --------------
705
806
  *
706
- * // Returns text because text is above button
807
+ * // Returns text0 because text0 is the first element above button
707
808
  * ...text().above().button()
809
+ * ...text().above(0).button()
810
+ * // Returns text1 because text1 is the second element above button
811
+ * ...text().above(1).button()
708
812
  * // Returns no element because button is below text
709
813
  * ...button().above().text()
710
814
  * ```
@@ -870,9 +974,11 @@ class FluentFiltersCondition extends FluentBase {
870
974
  *
871
975
  * **Examples:**
872
976
  * ```typescript
873
- * await aui.moveMouseTo().button().exec()
977
+ * await aui.click().button().contains().text().withText('Google Search').exec()
874
978
  * ```
875
979
  *
980
+ * ![](https://docs.askui.com/img/gif/button.gif)
981
+ *
876
982
  * @return {FluentFiltersOrRelationsCondition}
877
983
  */
878
984
  button() {
@@ -932,13 +1038,20 @@ class FluentFiltersCondition extends FluentBase {
932
1038
  /**
933
1039
  * Filters for an UI element 'text'.
934
1040
  *
935
- * Often combined with the filter `withText()` as shown in the below examples.
1041
+ * Takes an optional parameter to filter for a specific text.
1042
+ * See the examples below.
1043
+ *
936
1044
  * See also the filters `withTextRegex()` and `withExactText()`
937
1045
  *
938
1046
  * **Examples:**
939
1047
  * ```typescript
940
- * await aui.click().text().withText('Password').exec();
1048
+ * await aui.click().text().exec();
1049
+ * await aui.click().text('Username').exec();
1050
+ *
1051
+ * // Matching with an exact text
941
1052
  * await aui.click().text().withExactText('Username').exec();
1053
+ *
1054
+ * // Matching with a regex
942
1055
  * await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
943
1056
  * ```
944
1057
  *
@@ -962,6 +1075,8 @@ class FluentFiltersCondition extends FluentBase {
962
1075
  * icon().withText('plus')
963
1076
  * ```
964
1077
  *
1078
+ * ![](https://docs.askui.com/img/gif/icon.gif)
1079
+ *
965
1080
  * **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
966
1081
  *
967
1082
  * @return {FluentFiltersOrRelationsCondition}
@@ -972,7 +1087,9 @@ class FluentFiltersCondition extends FluentBase {
972
1087
  return new FluentFiltersOrRelationsCondition(this);
973
1088
  }
974
1089
  /**
975
- * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element that is not recognized by our machine learning models by default. It can also be used for pixel assertions of elements using classical [template matching](https://en.wikipedia.org/wiki/Template_matching).
1090
+ * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
1091
+ *
1092
+ * See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
976
1093
  *
977
1094
  * **Example**
978
1095
  * ```typescript
@@ -982,6 +1099,7 @@ class FluentFiltersCondition extends FluentBase {
982
1099
  * customImage: './logo.png', // required
983
1100
  * name: 'myLogo', // optional
984
1101
  * threshold: 0.9, // optional, defaults to 0.9
1102
+ * stopThreshold: 0.9, // optional, defaults to 0.9
985
1103
  * rotationDegreePerStep: 0, // optional, defaults to 0
986
1104
  * imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
987
1105
  * // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
@@ -996,12 +1114,13 @@ class FluentFiltersCondition extends FluentBase {
996
1114
  * - **name** (*`string`, optional*):
997
1115
  * - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
998
1116
  * - **threshold** (*`number`, optional*):
999
- * - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
1117
+ * - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
1118
+ * - **stopThreshold** (*`number`, optional*):
1119
+ * - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
1000
1120
  * - **rotationDegreePerStep** (*`number`, optional*):
1001
1121
  * - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
1002
- * - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
1003
- * - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three color. Defaults to 'grayscale'.
1004
- * of the given custom image.
1122
+ * - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
1123
+ * - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
1005
1124
  *
1006
1125
  *
1007
1126
  * @param {CustomElementJson} customElement - The custom element to filter for.
@@ -1015,6 +1134,23 @@ class FluentFiltersCondition extends FluentBase {
1015
1134
  this._params.set('customElement', customElement);
1016
1135
  return new FluentFiltersOrRelationsCondition(this);
1017
1136
  }
1137
+ /**
1138
+ * Detects an AI Element created with the workflow creator.
1139
+ *
1140
+ * @param {string} aiElementName - Name of the AI Element.
1141
+ *
1142
+ * @return {FluentFiltersOrRelationsCondition}
1143
+ */
1144
+ aiElement(aiElementName) {
1145
+ this._textStr = '';
1146
+ this._textStr += 'ai';
1147
+ this._textStr += ' element';
1148
+ this._textStr += ' with';
1149
+ this._textStr += ' name';
1150
+ this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
1151
+ this._params.set('aiElementName', aiElementName);
1152
+ return new FluentFiltersOrRelationsCondition(this);
1153
+ }
1018
1154
  /**
1019
1155
  * Filters for a UI element 'image'.
1020
1156
  *
@@ -1024,10 +1160,14 @@ class FluentFiltersCondition extends FluentBase {
1024
1160
  * await aui.click().image().exec();
1025
1161
  *
1026
1162
  * // Works if you have an image with
1027
- * // a caption text below
1028
- * await aui.click().image().above().text().withText('The caption').exec();
1163
+ * // a text below
1164
+ * await aui.click().image().above().text().withText('Automating WebGL').exec();
1029
1165
  * ```
1030
1166
  *
1167
+ * ![](https://docs.askui.com/img/gif/image.gif)
1168
+ *
1169
+ *
1170
+ *
1031
1171
  * @return {FluentFiltersOrRelationsCondition}
1032
1172
  */
1033
1173
  image() {
@@ -1048,6 +1188,10 @@ class FluentFiltersCondition extends FluentBase {
1048
1188
  * await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
1049
1189
  * ```
1050
1190
  *
1191
+ * ![](https://docs.askui.com/img/gif/textfield.gif)
1192
+ *
1193
+ *
1194
+ *
1051
1195
  * @return {FluentFiltersOrRelationsCondition}
1052
1196
  */
1053
1197
  textfield() {
@@ -1056,7 +1200,13 @@ class FluentFiltersCondition extends FluentBase {
1056
1200
  return new FluentFiltersOrRelationsCondition(this);
1057
1201
  }
1058
1202
  /**
1059
- * Filters for similar (doesn't need to be a 100% equal) text.
1203
+ * Filters for similar -- meaning >70% similar -- text.
1204
+ *
1205
+ * Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
1206
+ *
1207
+ * _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
1208
+ *
1209
+ * `1 - (distance / (lengthString1 + lengthString2))`
1060
1210
  *
1061
1211
  * **Examples:**
1062
1212
  * ```typescript
@@ -1071,6 +1221,11 @@ class FluentFiltersCondition extends FluentBase {
1071
1221
  * // usually false
1072
1222
  * 'atebxtc' === withText('text') => false
1073
1223
  * 'other' === withText('text') => false
1224
+ *
1225
+ * // optional parameter: similarity_score
1226
+ * '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
1227
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
1228
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
1074
1229
  * ```
1075
1230
  * ![](https://docs.askui.com/img/gif/withText.gif)
1076
1231
  *
@@ -1101,6 +1256,10 @@ class FluentFiltersCondition extends FluentBase {
1101
1256
  * await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
1102
1257
  * ```
1103
1258
  *
1259
+ * ![](https://docs.askui.com/img/gif/withtextregex.gif)
1260
+ *
1261
+ *
1262
+ *
1104
1263
  * @param {string} regex_pattern - A regex pattern
1105
1264
  *
1106
1265
  * @return {FluentFiltersOrRelationsCondition}
@@ -1129,6 +1288,10 @@ class FluentFiltersCondition extends FluentBase {
1129
1288
  * await aui.moveMouseTo().text().withExactText('Password').exec()
1130
1289
  * ```
1131
1290
  *
1291
+ * ![](https://docs.askui.com/img/gif/withexacttext.gif)
1292
+ *
1293
+ *
1294
+ *
1132
1295
  * @param {string} text - A text to be matched.
1133
1296
  *
1134
1297
  * @return {FluentFiltersOrRelationsCondition}
@@ -1165,17 +1328,28 @@ class FluentFiltersCondition extends FluentBase {
1165
1328
  /**
1166
1329
  * Filters elements based on a textual description.
1167
1330
  *
1168
- * ## What Should I Write as Matching Text
1331
+ * **What Should I Write as Matching Text**
1332
+ *
1169
1333
  * The text description inside the `matching()` should describe the element visually.
1170
1334
  * It understands color, some famous company/product names, general descriptions.
1171
1335
  *
1172
- * It sometimes requires a bit of playing to find a matching description:
1173
- * E.g. `puzzle piece` can fail here while `an icon showing a puzzle piece` might work.
1336
+ * It sometimes requires a bit of playing around to find a matching description:
1337
+ * E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
1174
1338
  * Generally the more detail the better.
1175
1339
  *
1176
1340
  * **Examples:**
1177
1341
  * ```typescript
1178
- * await aui.click().matching('a mask on purple background and a firefox logo').exec()
1342
+ * // Select the black sneaker from a bunch of sneakers
1343
+ * await aui.click().element().matching('a black sneaker shoe').exec();
1344
+ *
1345
+ * // Select an image that has text in it
1346
+ * await aui.click().element().matching('has Burger King in it').exec();
1347
+ * await aui.click().element().matching('has adidas in it').exec();
1348
+ *
1349
+ * // Target a logo/image by describing it
1350
+ * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
1351
+ * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
1352
+ * await aui.click().element().matching('logo looking like a seashell').exec();
1179
1353
  * ```
1180
1354
  *
1181
1355
  * @param {string} text - A description of the target element.
@@ -1334,14 +1508,19 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1334
1508
  /**
1335
1509
  * Filters for an element right of another element.
1336
1510
  *
1511
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
1512
+ *
1337
1513
  * **Examples:**
1338
1514
  * ```typescript
1339
- * -------------- --------------
1340
- * | leftEl | | rightEl |
1341
- * -------------- --------------
1515
+ * -------------- -------------- --------------
1516
+ * | leftEl | | rightEl0 | | rightEl1 |
1517
+ * -------------- -------------- --------------
1342
1518
  *
1343
- * // Returns rightEl because rightEl is right of leftEl
1519
+ * // Returns rightEl0 because rightEl0 is the first element right of leftEl
1344
1520
  * ...rightEl().rightOf().leftEl()
1521
+ * ...rightEl().rightOf(0).leftEl()
1522
+ * // Returns rightEl1 because rightEl1 is the second element right of leftEl
1523
+ * ...rightEl().rightOf(1).leftEl()
1345
1524
  * // Returns no element because leftEl is left of rightEl
1346
1525
  * ...leftEl().rightOf().rightEl()
1347
1526
  * ```
@@ -1361,14 +1540,19 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1361
1540
  /**
1362
1541
  * Filters for an element left of another element.
1363
1542
  *
1543
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
1544
+ *
1364
1545
  * **Examples:**
1365
1546
  * ```typescript
1366
- * -------------- --------------
1367
- * | leftEl | | rightEl |
1368
- * -------------- --------------
1547
+ * -------------- -------------- --------------
1548
+ * | leftEl1 | | leftEl0 | | rightEl |
1549
+ * -------------- -------------- --------------
1369
1550
  *
1370
- * // Returns leftEl because leftEl is left of rightEl
1551
+ * // Returns leftEl0 because leftEl0 is the first element left of rightEl
1371
1552
  * ...leftEl().leftOf().rightEl()
1553
+ * ...leftEl().leftOf(0).rightEl()
1554
+ * // Returns leftEl1 because leftEl1 is the second element left of rightEl
1555
+ * ...leftEl().leftOf(1).rightEl()
1372
1556
  * // Returns no element because rightEl is left of leftEl
1373
1557
  * ...rightEl().leftOf().leftEl()
1374
1558
  * ```
@@ -1388,17 +1572,25 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1388
1572
  /**
1389
1573
  * Filters for an element below another element.
1390
1574
  *
1575
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
1576
+ *
1391
1577
  * **Examples:**
1392
1578
  * ```typescript
1393
1579
  * --------------
1394
1580
  * | text |
1395
1581
  * --------------
1396
1582
  * --------------
1397
- * | button |
1583
+ * | button0 |
1584
+ * --------------
1585
+ * --------------
1586
+ * | button1 |
1398
1587
  * --------------
1399
1588
  *
1400
- * // Returns button because button is below text
1589
+ * // Returns button0 because button0 is the first button below text
1401
1590
  * ...button().below().text()
1591
+ * ...button().below(0).text()
1592
+ * // Returns button1 because button1 is the second button below text
1593
+ * ...button().below(1).text()
1402
1594
  * // Returns no element because text is above button
1403
1595
  * ...text().below().button()
1404
1596
  * ```
@@ -1417,17 +1609,25 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1417
1609
  /**
1418
1610
  * Filters for an element above another element.
1419
1611
  *
1612
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
1613
+ *
1420
1614
  * **Examples:**
1421
1615
  * ```typescript
1422
1616
  * --------------
1423
- * | text |
1617
+ * | text1 |
1618
+ * --------------
1619
+ * --------------
1620
+ * | text0 |
1424
1621
  * --------------
1425
1622
  * --------------
1426
1623
  * | button |
1427
1624
  * --------------
1428
1625
  *
1429
- * // Returns text because text is above button
1626
+ * // Returns text0 because text0 is the first element above button
1430
1627
  * ...text().above().button()
1628
+ * ...text().above(0).button()
1629
+ * // Returns text1 because text1 is the second element above button
1630
+ * ...text().above(1).button()
1431
1631
  * // Returns no element because button is below text
1432
1632
  * ...button().above().text()
1433
1633
  * ```
@@ -1511,12 +1711,12 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1511
1711
  * **Examples:**
1512
1712
  * ```typescript
1513
1713
  * // Stops execution at this point when the element does not exist.
1514
- * await aui.expect().text().withText('Login').exists().exec()
1714
+ * await aui.expect().text('Login').exists().exec()
1515
1715
  *
1516
1716
  * // This will catch the error and log a message
1517
1717
  * // But the execution will continue afterwards
1518
1718
  * try {
1519
- * await aui.expect().text().withText('Login').exists().exec()
1719
+ * await aui.expect().text('Login').exists().exec()
1520
1720
  * } catch (error) {
1521
1721
  * console.log('Too bad we could not find the element!');
1522
1722
  * }
@@ -1539,12 +1739,12 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
1539
1739
  * **Examples:**
1540
1740
  * ```typescript
1541
1741
  * // Stops execution at this point when the element does exist.
1542
- * await aui.expect().text().withText('Login').notExists().exec()
1742
+ * await aui.expect().text('Login').notExists().exec()
1543
1743
  *
1544
1744
  * // This will catch the error and log a message
1545
1745
  * // But the execution will continue afterwards
1546
1746
  * try {
1547
- * await aui.expect().text().withText('Login').notExists().exec()
1747
+ * await aui.expect().text('Login').notExists().exec()
1548
1748
  * } catch (error) {
1549
1749
  * console.log('Too bad we could find the element!');
1550
1750
  * }
@@ -1574,8 +1774,8 @@ class FluentCommand extends FluentBase {
1574
1774
  *
1575
1775
  * **Examples:**
1576
1776
  * ```typescript
1577
- * await aui.expect().text().withText('Login').exists().exec()
1578
- * await aui.expect().text().withText('Login').notExists().exec()
1777
+ * await aui.expect().text('Login').exists().exec()
1778
+ * await aui.expect().text('Login').notExists().exec()
1579
1779
  * ```
1580
1780
  *
1581
1781
  * @return {FluentFiltersCondition}
@@ -1592,9 +1792,13 @@ class FluentCommand extends FluentBase {
1592
1792
  *
1593
1793
  * **Example:**
1594
1794
  * ```typescript
1595
- * await aui.click().button().withText('Submit').exec()
1795
+ * await aui.click().button().withText('Google Search').exec();
1596
1796
  * ```
1597
1797
  *
1798
+ * ![](https://docs.askui.com/img/gif/click.gif)
1799
+ *
1800
+ *
1801
+ *
1598
1802
  * @return {FluentFilters}
1599
1803
  */
1600
1804
  click() {
@@ -1608,9 +1812,13 @@ class FluentCommand extends FluentBase {
1608
1812
  *
1609
1813
  * **Example:**
1610
1814
  * ```typescript
1611
- * await aui.moveMouseTo().button().withText('Submit').exec()
1815
+ * await aui.moveMouseTo().text().withText('Grinning_Face').exec()
1612
1816
  * ```
1613
1817
  *
1818
+ * ![](https://docs.askui.com/img/gif/movemouseto.gif)
1819
+ *
1820
+ *
1821
+ *
1614
1822
  * @return {FluentFilters}
1615
1823
  */
1616
1824
  moveMouseTo() {
@@ -1652,9 +1860,11 @@ class FluentCommand extends FluentBase {
1652
1860
  *
1653
1861
  * **Example:**
1654
1862
  * ```typescript
1655
- * await aui.scroll(0, 10).textarea().exec()
1863
+ * await aui.scrollInside(0,-500).text().withText('Bottom sheet').exec();
1656
1864
  * ```
1657
1865
  *
1866
+ * ![](https://docs.askui.com/img/gif/scrollinside.gif)
1867
+ *
1658
1868
  * @param {number} x_offset - A (positive/negative) x direction.
1659
1869
  * @param {number} y_offset - A (positive/negative) y direction.
1660
1870
  *
@@ -1731,12 +1941,14 @@ class FluentCommand extends FluentBase {
1731
1941
  *
1732
1942
  * **Examples:**
1733
1943
  * ```typescript
1734
- * await aui.type('Type some text').exec()
1944
+ * await aui.type('askui@askui.com').exec()
1735
1945
  *
1736
1946
  * // mask the text so it is not send to the askui-inference server
1737
1947
  * await aui.type('Type some text', { isSecret: true, secretMask: '**' }).exec()
1738
1948
  * ```
1739
1949
  *
1950
+ * ![](https://docs.askui.com/img/gif/type.gif)
1951
+ *
1740
1952
  * @param {string} text - A text to type
1741
1953
  *
1742
1954
  * @return {Exec}
@@ -1752,9 +1964,11 @@ class FluentCommand extends FluentBase {
1752
1964
  *
1753
1965
  * **Example:**
1754
1966
  * ```typescript
1755
- * await aui.moveMouseRelatively(20, 20).exec();
1967
+ * await aui.moveMouseRelatively(0, 50).exec();
1756
1968
  * ```
1757
1969
  *
1970
+ * ![](https://docs.askui.com/img/gif/movemouserelatively.gif)
1971
+ *
1758
1972
  * @param {number} x_offset - A (positive/negative) x direction.
1759
1973
  * @param {number} y_offset - A (positive/negative) y direction.
1760
1974
  *
@@ -1784,6 +1998,8 @@ class FluentCommand extends FluentBase {
1784
1998
  * await aui.moveMouse(500, 500).exec();
1785
1999
  * ```
1786
2000
  *
2001
+ * ![](https://docs.askui.com/img/gif/moveMouse.gif)
2002
+ *
1787
2003
  * @param {number} x_coordinate - A (positive/negative) x coordinate.
1788
2004
  * @param {number} y_coordinate - A (positive/negative) y coordinate.
1789
2005
  *
@@ -1809,10 +2025,12 @@ class FluentCommand extends FluentBase {
1809
2025
  *
1810
2026
  * **Example:**
1811
2027
  * ```typescript
1812
- * // Scroll 10 up in y direction
1813
- * await aui.scroll(0, 10).exec()
2028
+ * // Scroll 500 pixels down in y direction
2029
+ * await aui.scroll(0, -500).exec()
1814
2030
  * ```
1815
2031
  *
2032
+ * ![](https://docs.askui.com/img/gif/scroll.gif)
2033
+ *
1816
2034
  * @param {number} x_offset - A (positive/negative) x direction.
1817
2035
  * @param {number} y_offset - A (positive/negative) y direction.
1818
2036
  *
@@ -1930,6 +2148,8 @@ class FluentCommand extends FluentBase {
1930
2148
  * await aui.mouseDoubleLeftClick().exec();
1931
2149
  * ```
1932
2150
  *
2151
+ * ![](https://docs.askui.com/img/gif/mousedoubleleftclick.gif)
2152
+ *
1933
2153
  * @return {Exec}
1934
2154
  */
1935
2155
  mouseDoubleLeftClick() {
@@ -1980,11 +2200,17 @@ class FluentCommand extends FluentBase {
1980
2200
  /**
1981
2201
  * Toggles mouse down (Left mouse key/tap).
1982
2202
  *
2203
+ * This is the equivalent to **mouse-left-press-and-hold**. It holds the mouse button until the `mouseToogleUp()` is called. Often combined with `mouseToggleUP` to automate **drag-and-drop**.
2204
+ *
1983
2205
  * **Example:**
1984
2206
  * ```typescript
1985
2207
  * await aui.mouseToggleDown().exec();
2208
+ * await aui.moveMouseRelatively(-400,0).exec();
2209
+ * await aui.mouseToggleUp().exec();
1986
2210
  * ```
1987
2211
  *
2212
+ * ![](https://docs.askui.com/img/gif/mouseToggleDownUp.gif)
2213
+ *
1988
2214
  * @return {Exec}
1989
2215
  */
1990
2216
  mouseToggleDown() {
@@ -1995,11 +2221,17 @@ class FluentCommand extends FluentBase {
1995
2221
  /**
1996
2222
  * Toggles mouse up (Left mouse key/tap).
1997
2223
  *
2224
+ * This is the equivalent to releasing the pressing mouse left button. Often combined with `mouseToggleDown()` to automate **drag-and-drop**.
2225
+ *
1998
2226
  * **Example:**
1999
2227
  * ```typescript
2228
+ * await aui.mouseToggleDown().exec();
2229
+ * await aui.moveMouseRelatively(-400,0).exec();
2000
2230
  * await aui.mouseToggleUp().exec();
2001
2231
  * ```
2002
2232
  *
2233
+ * ![](https://docs.askui.com/img/gif/mouseToggleDownUp.gif)
2234
+ *
2003
2235
  * @return {Exec}
2004
2236
  */
2005
2237
  mouseToggleUp() {
@@ -2012,8 +2244,13 @@ class FluentCommand extends FluentBase {
2012
2244
  *
2013
2245
  * **Operating system specific mappings:**
2014
2246
  * 1. Windows: `command`-key maps to `windows`-key
2015
- * ---
2016
2247
  *
2248
+ * **Examples:**
2249
+ * ```typescript
2250
+ * await aui.pressThreeKeys('control', 'command' 'space').exec();
2251
+ * ```
2252
+ *
2253
+ * ![](https://docs.askui.com/img/gif/pressThreeKeys.gif)
2017
2254
  *
2018
2255
  * @param {MODIFIER_KEY} first_key - A modifier key
2019
2256
  * @param {MODIFIER_KEY} second_key - A modifier key
@@ -2056,8 +2293,13 @@ class FluentCommand extends FluentBase {
2056
2293
  *
2057
2294
  * **Operating system specific mappings:**
2058
2295
  * 1. Windows: `command`-key maps to `windows`-key
2059
- * ---
2060
2296
  *
2297
+ * **Examples:**
2298
+ * ```typescript
2299
+ * await aui.pressKey('tab').exec();
2300
+ * ```
2301
+ *
2302
+ * ![](https://docs.askui.com/img/gif/pressKey.gif)
2061
2303
  *
2062
2304
  * @param {PC_AND_MODIFIER_KEY} key - A key
2063
2305
  *
@@ -2090,7 +2332,15 @@ class FluentCommand extends FluentBase {
2090
2332
  return new Exec(this);
2091
2333
  }
2092
2334
  /**
2093
- * Press two Android keys like `ALT+F4`
2335
+ * Press two Android keys like `volume_down+power`
2336
+ * See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
2337
+ *
2338
+ * **Examples:**
2339
+ * ```typescript
2340
+ * await aui.pressAndroidTwoKey('volume_down', 'power').exec();
2341
+ * ```
2342
+ *
2343
+ * ![](https://docs.askui.com/img/gif/pressAndroidTwoKey.gif)
2094
2344
  *
2095
2345
  * @param {ANDROID_KEY} first_key - A Android key
2096
2346
  * @param {ANDROID_KEY} second_key - A Android key
@@ -2107,7 +2357,15 @@ class FluentCommand extends FluentBase {
2107
2357
  return new Exec(this);
2108
2358
  }
2109
2359
  /**
2110
- * Press one Android key like `DEL`
2360
+ * Press one Android key like `del`
2361
+ * See [API docs](https://docs.askui.com/docs/api/Actions/pressandroidtwokey) for available keys.
2362
+ *
2363
+ * **Examples:**
2364
+ * ```typescript
2365
+ * await aui.pressAndroidKey('notification').exec();
2366
+ * ```
2367
+ *
2368
+ * ![](https://docs.askui.com/img/gif/pressAndroidKey.gif)
2111
2369
  *
2112
2370
  * @param {ANDROID_KEY} key - A Android key
2113
2371
  *
@@ -2216,9 +2474,11 @@ class FluentFiltersGetter extends FluentBase {
2216
2474
  *
2217
2475
  * **Examples:**
2218
2476
  * ```typescript
2219
- * await aui.moveMouseTo().button().exec()
2477
+ * await aui.click().button().contains().text().withText('Google Search').exec()
2220
2478
  * ```
2221
2479
  *
2480
+ * ![](https://docs.askui.com/img/gif/button.gif)
2481
+ *
2222
2482
  * @return {FluentFiltersOrRelationsGetter}
2223
2483
  */
2224
2484
  button() {
@@ -2278,13 +2538,20 @@ class FluentFiltersGetter extends FluentBase {
2278
2538
  /**
2279
2539
  * Filters for an UI element 'text'.
2280
2540
  *
2281
- * Often combined with the filter `withText()` as shown in the below examples.
2541
+ * Takes an optional parameter to filter for a specific text.
2542
+ * See the examples below.
2543
+ *
2282
2544
  * See also the filters `withTextRegex()` and `withExactText()`
2283
2545
  *
2284
2546
  * **Examples:**
2285
2547
  * ```typescript
2286
- * await aui.click().text().withText('Password').exec();
2548
+ * await aui.click().text().exec();
2549
+ * await aui.click().text('Username').exec();
2550
+ *
2551
+ * // Matching with an exact text
2287
2552
  * await aui.click().text().withExactText('Username').exec();
2553
+ *
2554
+ * // Matching with a regex
2288
2555
  * await aui.click().text().withTextRegex('\b[Ss]\w+').exec();
2289
2556
  * ```
2290
2557
  *
@@ -2308,6 +2575,8 @@ class FluentFiltersGetter extends FluentBase {
2308
2575
  * icon().withText('plus')
2309
2576
  * ```
2310
2577
  *
2578
+ * ![](https://docs.askui.com/img/gif/icon.gif)
2579
+ *
2311
2580
  * **Note:** This is an alpha feature. The prediction of the icon name is sometimes unstable. Use custom elements as an alternative.
2312
2581
  *
2313
2582
  * @return {FluentFiltersOrRelationsGetter}
@@ -2318,7 +2587,9 @@ class FluentFiltersGetter extends FluentBase {
2318
2587
  return new FluentFiltersOrRelationsGetter(this);
2319
2588
  }
2320
2589
  /**
2321
- * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element that is not recognized by our machine learning models by default. It can also be used for pixel assertions of elements using classical [template matching](https://en.wikipedia.org/wiki/Template_matching).
2590
+ * Filters for a 'custom element', that is a UI element which is defined by providing an image and other parameters such as degree of rotation. It allows filtering for a UI element based on an image instead of using text or element descriptions like `button().withText('Submit')` in `await aui.click().button().withText('Submit').exec()`.
2591
+ *
2592
+ * See the tutorial - [Custom Element](https://docs.askui.com/docs/general/Tutorials/custom-element) for more detail.
2322
2593
  *
2323
2594
  * **Example**
2324
2595
  * ```typescript
@@ -2328,6 +2599,7 @@ class FluentFiltersGetter extends FluentBase {
2328
2599
  * customImage: './logo.png', // required
2329
2600
  * name: 'myLogo', // optional
2330
2601
  * threshold: 0.9, // optional, defaults to 0.9
2602
+ * stopThreshold: 0.9, // optional, defaults to 0.9
2331
2603
  * rotationDegreePerStep: 0, // optional, defaults to 0
2332
2604
  * imageCompareFormat: 'grayscale', // optional, defaults to 'grayscale'
2333
2605
  * // mask:{x:0, y:0}[] // optional, a polygon to match only a certain area of the custom element
@@ -2342,12 +2614,13 @@ class FluentFiltersGetter extends FluentBase {
2342
2614
  * - **name** (*`string`, optional*):
2343
2615
  * - A unique name that can be used for filtering for the custom element. If not given, any text inside the custom image will be detected via OCR.
2344
2616
  * - **threshold** (*`number`, optional*):
2345
- * - A threshold for how much a UI element needs to be similar to the custom element as defined. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
2617
+ * - A threshold for how much a UI element needs to be similar to the custom element as defined by the image. Takes values between `0.0` (== all elements are recognized as the custom element which is probably not what you want) and `1.0` (== elements need to look exactly like the `customImage` which is unlikely to be achieved as even minor differences count). Defaults to `0.9`.
2618
+ * - **stopThreshold** (*`number`, optional*):
2619
+ * - A threshold for when to stop searching for UI elements similar to the custom element. As soon as UI elements have been found that are at least as similar as the `stopThreshold`, the search is going to stop. After that elements are filtered using the `threshold`. Because of that the `stopThreshold` should be greater than or equal to `threshold`. It is primarily to be used as a speed improvement (by lowering the value). Takes values between `0.0` and `1.0`. Defaults to `0.9`.
2346
2620
  * - **rotationDegreePerStep** (*`number`, optional*):
2347
2621
  * - Step size in rotation degree. Rotates the custom image by this step size until 360° is exceeded. The range is from `0` to `360`. Defaults to `0`.
2348
- * - **imageCompareFormat** (*`'RGB' | 'grayscale'`, optional*):
2349
- * - The color compare style. 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three color. Defaults to 'grayscale'.
2350
- * of the given custom image.
2622
+ * - **imageCompareFormat** (*`'RGB' | 'grayscale' | 'edges'`, optional*):
2623
+ * - The color compare style. 'edges' compares only edges, 'greyscale' compares the brightness of each pixel whereas 'RGB' compares all three colors (red, green, blue). Defaults to 'grayscale'.
2351
2624
  *
2352
2625
  *
2353
2626
  * @param {CustomElementJson} customElement - The custom element to filter for.
@@ -2361,6 +2634,23 @@ class FluentFiltersGetter extends FluentBase {
2361
2634
  this._params.set('customElement', customElement);
2362
2635
  return new FluentFiltersOrRelationsGetter(this);
2363
2636
  }
2637
+ /**
2638
+ * Detects an AI Element created with the workflow creator.
2639
+ *
2640
+ * @param {string} aiElementName - Name of the AI Element.
2641
+ *
2642
+ * @return {FluentFiltersOrRelationsGetter}
2643
+ */
2644
+ aiElement(aiElementName) {
2645
+ this._textStr = '';
2646
+ this._textStr += 'ai';
2647
+ this._textStr += ' element';
2648
+ this._textStr += ' with';
2649
+ this._textStr += ' name';
2650
+ this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
2651
+ this._params.set('aiElementName', aiElementName);
2652
+ return new FluentFiltersOrRelationsGetter(this);
2653
+ }
2364
2654
  /**
2365
2655
  * Filters for a UI element 'image'.
2366
2656
  *
@@ -2370,10 +2660,14 @@ class FluentFiltersGetter extends FluentBase {
2370
2660
  * await aui.click().image().exec();
2371
2661
  *
2372
2662
  * // Works if you have an image with
2373
- * // a caption text below
2374
- * await aui.click().image().above().text().withText('The caption').exec();
2663
+ * // a text below
2664
+ * await aui.click().image().above().text().withText('Automating WebGL').exec();
2375
2665
  * ```
2376
2666
  *
2667
+ * ![](https://docs.askui.com/img/gif/image.gif)
2668
+ *
2669
+ *
2670
+ *
2377
2671
  * @return {FluentFiltersOrRelationsGetter}
2378
2672
  */
2379
2673
  image() {
@@ -2394,6 +2688,10 @@ class FluentFiltersGetter extends FluentBase {
2394
2688
  * await aui.typeIn('Oh yeah').textfield().below().text().withText('E-Mail Address').exec();
2395
2689
  * ```
2396
2690
  *
2691
+ * ![](https://docs.askui.com/img/gif/textfield.gif)
2692
+ *
2693
+ *
2694
+ *
2397
2695
  * @return {FluentFiltersOrRelationsGetter}
2398
2696
  */
2399
2697
  textfield() {
@@ -2402,7 +2700,13 @@ class FluentFiltersGetter extends FluentBase {
2402
2700
  return new FluentFiltersOrRelationsGetter(this);
2403
2701
  }
2404
2702
  /**
2405
- * Filters for similar (doesn't need to be a 100% equal) text.
2703
+ * Filters for similar -- meaning >70% similar -- text.
2704
+ *
2705
+ * Takes an optional parameter to specify the similarity. Usually you need the optional parameter for long texts you want to match precisely.
2706
+ *
2707
+ * _We use [RapidFuzz](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#ratio) which calculates the similarity like this:_
2708
+ *
2709
+ * `1 - (distance / (lengthString1 + lengthString2))`
2406
2710
  *
2407
2711
  * **Examples:**
2408
2712
  * ```typescript
@@ -2417,6 +2721,11 @@ class FluentFiltersGetter extends FluentBase {
2417
2721
  * // usually false
2418
2722
  * 'atebxtc' === withText('text') => false
2419
2723
  * 'other' === withText('text') => false
2724
+ *
2725
+ * // optional parameter: similarity_score
2726
+ * '978-0-201-00650-6' == withText('978-0-201-00') => true with 82.76 similarity
2727
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => false with 82.76 < 90 similarity
2728
+ * '978-0-201-00650-6' == withText('978-0-201-00', 90) => true with 93.75 < 90 similarity
2420
2729
  * ```
2421
2730
  * ![](https://docs.askui.com/img/gif/withText.gif)
2422
2731
  *
@@ -2447,6 +2756,10 @@ class FluentFiltersGetter extends FluentBase {
2447
2756
  * await aui.get().text().withTextRegex('\b[Ss]\w+').exec()
2448
2757
  * ```
2449
2758
  *
2759
+ * ![](https://docs.askui.com/img/gif/withtextregex.gif)
2760
+ *
2761
+ *
2762
+ *
2450
2763
  * @param {string} regex_pattern - A regex pattern
2451
2764
  *
2452
2765
  * @return {FluentFiltersOrRelationsGetter}
@@ -2475,6 +2788,10 @@ class FluentFiltersGetter extends FluentBase {
2475
2788
  * await aui.moveMouseTo().text().withExactText('Password').exec()
2476
2789
  * ```
2477
2790
  *
2791
+ * ![](https://docs.askui.com/img/gif/withexacttext.gif)
2792
+ *
2793
+ *
2794
+ *
2478
2795
  * @param {string} text - A text to be matched.
2479
2796
  *
2480
2797
  * @return {FluentFiltersOrRelationsGetter}
@@ -2511,17 +2828,28 @@ class FluentFiltersGetter extends FluentBase {
2511
2828
  /**
2512
2829
  * Filters elements based on a textual description.
2513
2830
  *
2514
- * ## What Should I Write as Matching Text
2831
+ * **What Should I Write as Matching Text**
2832
+ *
2515
2833
  * The text description inside the `matching()` should describe the element visually.
2516
2834
  * It understands color, some famous company/product names, general descriptions.
2517
2835
  *
2518
- * It sometimes requires a bit of playing to find a matching description:
2519
- * E.g. `puzzle piece` can fail here while `an icon showing a puzzle piece` might work.
2836
+ * It sometimes requires a bit of playing around to find a matching description:
2837
+ * E.g. `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
2520
2838
  * Generally the more detail the better.
2521
2839
  *
2522
2840
  * **Examples:**
2523
2841
  * ```typescript
2524
- * await aui.click().matching('a mask on purple background and a firefox logo').exec()
2842
+ * // Select the black sneaker from a bunch of sneakers
2843
+ * await aui.click().element().matching('a black sneaker shoe').exec();
2844
+ *
2845
+ * // Select an image that has text in it
2846
+ * await aui.click().element().matching('has Burger King in it').exec();
2847
+ * await aui.click().element().matching('has adidas in it').exec();
2848
+ *
2849
+ * // Target a logo/image by describing it
2850
+ * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
2851
+ * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
2852
+ * await aui.click().element().matching('logo looking like a seashell').exec();
2525
2853
  * ```
2526
2854
  *
2527
2855
  * @param {string} text - A description of the target element.
@@ -2680,14 +3008,19 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
2680
3008
  /**
2681
3009
  * Filters for an element right of another element.
2682
3010
  *
3011
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
3012
+ *
2683
3013
  * **Examples:**
2684
3014
  * ```typescript
2685
- * -------------- --------------
2686
- * | leftEl | | rightEl |
2687
- * -------------- --------------
3015
+ * -------------- -------------- --------------
3016
+ * | leftEl | | rightEl0 | | rightEl1 |
3017
+ * -------------- -------------- --------------
2688
3018
  *
2689
- * // Returns rightEl because rightEl is right of leftEl
3019
+ * // Returns rightEl0 because rightEl0 is the first element right of leftEl
2690
3020
  * ...rightEl().rightOf().leftEl()
3021
+ * ...rightEl().rightOf(0).leftEl()
3022
+ * // Returns rightEl1 because rightEl1 is the second element right of leftEl
3023
+ * ...rightEl().rightOf(1).leftEl()
2691
3024
  * // Returns no element because leftEl is left of rightEl
2692
3025
  * ...leftEl().rightOf().rightEl()
2693
3026
  * ```
@@ -2707,14 +3040,19 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
2707
3040
  /**
2708
3041
  * Filters for an element left of another element.
2709
3042
  *
3043
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
3044
+ *
2710
3045
  * **Examples:**
2711
3046
  * ```typescript
2712
- * -------------- --------------
2713
- * | leftEl | | rightEl |
2714
- * -------------- --------------
3047
+ * -------------- -------------- --------------
3048
+ * | leftEl1 | | leftEl0 | | rightEl |
3049
+ * -------------- -------------- --------------
2715
3050
  *
2716
- * // Returns leftEl because leftEl is left of rightEl
3051
+ * // Returns leftEl0 because leftEl0 is the first element left of rightEl
2717
3052
  * ...leftEl().leftOf().rightEl()
3053
+ * ...leftEl().leftOf(0).rightEl()
3054
+ * // Returns leftEl1 because leftEl1 is the second element left of rightEl
3055
+ * ...leftEl().leftOf(1).rightEl()
2718
3056
  * // Returns no element because rightEl is left of leftEl
2719
3057
  * ...rightEl().leftOf().leftEl()
2720
3058
  * ```
@@ -2734,17 +3072,25 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
2734
3072
  /**
2735
3073
  * Filters for an element below another element.
2736
3074
  *
3075
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
3076
+ *
2737
3077
  * **Examples:**
2738
3078
  * ```typescript
2739
3079
  * --------------
2740
3080
  * | text |
2741
3081
  * --------------
2742
3082
  * --------------
2743
- * | button |
3083
+ * | button0 |
3084
+ * --------------
3085
+ * --------------
3086
+ * | button1 |
2744
3087
  * --------------
2745
3088
  *
2746
- * // Returns button because button is below text
3089
+ * // Returns button0 because button0 is the first button below text
2747
3090
  * ...button().below().text()
3091
+ * ...button().below(0).text()
3092
+ * // Returns button1 because button1 is the second button below text
3093
+ * ...button().below(1).text()
2748
3094
  * // Returns no element because text is above button
2749
3095
  * ...text().below().button()
2750
3096
  * ```
@@ -2763,17 +3109,25 @@ class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
2763
3109
  /**
2764
3110
  * Filters for an element above another element.
2765
3111
  *
3112
+ * Takes an optional parameter `index` to select the `nth` element (starting with 0)
3113
+ *
2766
3114
  * **Examples:**
2767
3115
  * ```typescript
2768
3116
  * --------------
2769
- * | text |
3117
+ * | text1 |
3118
+ * --------------
3119
+ * --------------
3120
+ * | text0 |
2770
3121
  * --------------
2771
3122
  * --------------
2772
3123
  * | button |
2773
3124
  * --------------
2774
3125
  *
2775
- * // Returns text because text is above button
3126
+ * // Returns text0 because text0 is the first element above button
2776
3127
  * ...text().above().button()
3128
+ * ...text().above(0).button()
3129
+ * // Returns text1 because text1 is the second element above button
3130
+ * ...text().above(1).button()
2777
3131
  * // Returns no element because button is below text
2778
3132
  * ...button().above().text()
2779
3133
  * ```
@@ -2868,22 +3222,74 @@ class Getter extends FluentCommand {
2868
3222
  *
2869
3223
  * **Examples:**
2870
3224
  * ```typescript
2871
- * const text = await aui.get().text().withText('Sign').exec();
3225
+ * const text = await aui.get().text('Sign').exec();
2872
3226
  * console.log(text);
3227
+ *
3228
+ * // Console output
3229
+ * [
3230
+ * DetectedElement {
3231
+ * name: 'TEXT',
3232
+ * text: 'Sign In',
3233
+ * bndbox: BoundingBox {
3234
+ * xmin: 1128.2720982142857,
3235
+ * ymin: 160.21332310267857,
3236
+ * xmax: 1178.8204241071428,
3237
+ * ymax: 180.83512834821428
3238
+ * }
3239
+ * }
3240
+ * ]
2873
3241
  * ```
2874
- * ```text
2875
- * console output: [
3242
+ *
3243
+ * // *************************************************** //
3244
+ * // Examples on how to work with the returned elements //
3245
+ * // *************************************************** //
3246
+ * const texts = await aui.get().text().below().textfield().exec();
3247
+ *
3248
+ * // We can get a lot of elements this way
3249
+ * console.log(texts);
3250
+ *
3251
+ * // Console output
3252
+ * [
2876
3253
  * DetectedElement {
2877
- * name: 'TEXT',
2878
- * text: 'Sign In',
2879
- * bndbox: BoundingBox {
2880
- * xmin: 1128.2720982142857,
2881
- * ymin: 160.21332310267857,
2882
- * xmax: 1178.8204241071428,
2883
- * ymax: 180.83512834821428
2884
- * }
2885
- * }
2886
- * ]
3254
+ * name: 'TEXT',
3255
+ * text: 'Sign In',
3256
+ * bndbox: BoundingBox {
3257
+ * xmin: 1128.2720982142857,
3258
+ * ymin: 160.21332310267857,
3259
+ * xmax: 1178.8204241071428,
3260
+ * ymax: 180.83512834821428
3261
+ * },
3262
+ * },
3263
+ * DetectedElement {
3264
+ * name: 'TEXT',
3265
+ * text: 'Login',
3266
+ * bndbox: BoundingBox {
3267
+ * xmin: 250.8204241071428,
3268
+ * ymin: 300.21332310267857,
3269
+ * xmax: 450.6304241071428,
3270
+ * ymax: 950.47812834821428
3271
+ * },
3272
+ * },
3273
+ * ... 10 more items
3274
+ * ]
3275
+ *
3276
+ * // Extract the FIRST element
3277
+ * // Arrays start with index 0!
3278
+ * const firstTextElement = texts[0];
3279
+ * const textOfFirstElement = firstElement.text;
3280
+ *
3281
+ * console.log(textOfFirstElement);
3282
+ *
3283
+ * // Console output
3284
+ * Sign In
3285
+ *
3286
+ * // Log the text of the SECOND element
3287
+ * // with shorter code
3288
+ * const texts = await aui.get().text().below().textfield().exec();
3289
+ * console.log(texts[1].text)
3290
+ *
3291
+ * // Console output
3292
+ * Login
2887
3293
  * ```
2888
3294
  *
2889
3295
  * @return {FluentFiltersGetter}