askui 0.1.5 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/README.md +6 -0
  2. package/bin/askui-postinstall +35 -0
  3. package/dist/cjs/core/model/test-case-dto/custom-element.d.ts +2 -2
  4. package/dist/cjs/core/model/test-case-dto/custom-element.js +2 -2
  5. package/dist/cjs/core/model/test-case-dto/custom-element.spec.js +7 -6
  6. package/dist/cjs/core/model/test-case-dto/index.js +1 -3
  7. package/dist/cjs/core/model/test-case-dto/test-step.d.ts +3 -7
  8. package/dist/cjs/core/model/test-case-dto/test-step.js +0 -12
  9. package/dist/cjs/core/ui-control-commands/control-command.d.ts +1 -0
  10. package/dist/cjs/core/ui-control-commands/control-command.js +5 -0
  11. package/dist/cjs/core/ui-control-commands/input-event.d.ts +1 -2
  12. package/dist/cjs/core/ui-control-commands/input-event.js +0 -1
  13. package/dist/cjs/execution/dsl.d.ts +6 -17
  14. package/dist/cjs/execution/dsl.js +4 -21
  15. package/dist/cjs/execution/execution-runtime.d.ts +7 -5
  16. package/dist/cjs/execution/execution-runtime.js +35 -23
  17. package/dist/cjs/execution/index.d.ts +2 -1
  18. package/dist/cjs/execution/index.js +5 -3
  19. package/dist/cjs/execution/{control-your-ui-api.d.ts → inference-client.d.ts} +4 -4
  20. package/dist/cjs/execution/{control-your-ui-api.js → inference-client.js} +9 -9
  21. package/dist/cjs/execution/ui-control-client-error.d.ts +2 -0
  22. package/dist/cjs/execution/ui-control-client-error.js +6 -0
  23. package/dist/cjs/execution/ui-control-client.d.ts +79 -0
  24. package/dist/cjs/execution/ui-control-client.js +185 -0
  25. package/dist/cjs/execution/{client-connection-state.d.ts → ui-controller-client-connection-state.d.ts} +1 -1
  26. package/dist/cjs/execution/ui-controller-client-connection-state.js +10 -0
  27. package/dist/cjs/execution/ui-controller-client-interface.d.ts +27 -0
  28. package/dist/cjs/execution/{client-interface.js → ui-controller-client-interface.js} +0 -0
  29. package/dist/cjs/execution/{control-your-ui-client.d.ts → ui-controller-client.d.ts} +6 -6
  30. package/dist/cjs/execution/{control-your-ui-client.js → ui-controller-client.js} +28 -28
  31. package/dist/cjs/lib/copy-example-project.js +2 -1
  32. package/dist/cjs/lib/download-binaries.js +10 -9
  33. package/dist/cjs/lib/index.d.ts +1 -1
  34. package/dist/cjs/lib/index.js +3 -3
  35. package/dist/cjs/lib/libfuse-error.d.ts +2 -0
  36. package/dist/cjs/lib/libfuse-error.js +6 -0
  37. package/dist/cjs/lib/logger.js +1 -1
  38. package/dist/cjs/lib/timeout-error.d.ts +0 -2
  39. package/dist/cjs/lib/timeout-error.js +0 -5
  40. package/dist/cjs/lib/ui-controller-args.d.ts +47 -0
  41. package/dist/cjs/lib/{control-ui-server-args.js → ui-controller-args.js} +0 -0
  42. package/dist/cjs/lib/{control-ui-server-args.spec.d.ts → ui-controller-args.spec.d.ts} +0 -0
  43. package/dist/cjs/lib/{control-ui-server-args.spec.js → ui-controller-args.spec.js} +7 -7
  44. package/dist/cjs/lib/ui-controller-darwin.d.ts +6 -0
  45. package/dist/cjs/lib/{control-ui-server-darwin.js → ui-controller-darwin.js} +8 -8
  46. package/dist/cjs/lib/ui-controller-facade.d.ts +17 -0
  47. package/dist/cjs/lib/{control-ui-server-facade.js → ui-controller-facade.js} +22 -16
  48. package/dist/cjs/lib/ui-controller-linux.d.ts +5 -0
  49. package/dist/cjs/lib/ui-controller-linux.js +52 -0
  50. package/dist/cjs/lib/ui-controller-win32.d.ts +3 -0
  51. package/dist/cjs/lib/ui-controller-win32.js +7 -0
  52. package/dist/cjs/lib/ui-controller.d.ts +8 -0
  53. package/dist/cjs/lib/{askui-control-server.js → ui-controller.js} +9 -9
  54. package/dist/cjs/lib/unkown-error.d.ts +0 -2
  55. package/dist/cjs/lib/unkown-error.js +0 -5
  56. package/dist/cjs/lib/wayland-error.d.ts +2 -0
  57. package/dist/cjs/lib/wayland-error.js +6 -0
  58. package/dist/cjs/main.d.ts +3 -2
  59. package/dist/cjs/main.js +6 -3
  60. package/dist/cjs/shared/index.d.ts +1 -0
  61. package/dist/cjs/shared/index.js +5 -0
  62. package/dist/cjs/utils/analytics/analytics-interface.d.ts +3 -0
  63. package/dist/cjs/utils/analytics/analytics-interface.js +2 -0
  64. package/dist/cjs/utils/analytics/analytics.d.ts +5 -0
  65. package/dist/cjs/utils/analytics/analytics.js +38 -0
  66. package/dist/cjs/utils/analytics/index.d.ts +1 -0
  67. package/dist/cjs/utils/analytics/index.js +5 -0
  68. package/dist/cjs/utils/analytics/installation-timestamp-create-error.d.ts +4 -0
  69. package/dist/cjs/utils/analytics/installation-timestamp-create-error.js +9 -0
  70. package/dist/cjs/utils/analytics/installation-timestamp-get-error.d.ts +4 -0
  71. package/dist/cjs/utils/analytics/installation-timestamp-get-error.js +9 -0
  72. package/dist/cjs/utils/analytics/installation-timestamp.d.ts +7 -0
  73. package/dist/cjs/utils/analytics/installation-timestamp.js +68 -0
  74. package/dist/cjs/utils/analytics/user-identifier-interface.d.ts +3 -0
  75. package/dist/cjs/utils/analytics/user-identifier-interface.js +2 -0
  76. package/dist/cjs/utils/analytics/user-identifier.d.ts +4 -0
  77. package/dist/cjs/utils/analytics/user-identifier.js +22 -0
  78. package/dist/cjs/utils/base_64_image/base-64-image-error.d.ts +2 -0
  79. package/dist/cjs/utils/base_64_image/base-64-image-error.js +6 -0
  80. package/dist/cjs/utils/base_64_image/base-64-image-string-error.d.ts +4 -0
  81. package/dist/cjs/utils/base_64_image/base-64-image-string-error.js +11 -0
  82. package/dist/cjs/utils/base_64_image/base-64-image.d.ts +18 -0
  83. package/dist/cjs/utils/base_64_image/base-64-image.js +86 -0
  84. package/dist/cjs/utils/http/custom-errors/http-client-error.d.ts +0 -2
  85. package/dist/cjs/utils/http/custom-errors/http-client-error.js +0 -5
  86. package/dist/cjs/utils/http/http-client-got.d.ts +6 -5
  87. package/dist/cjs/utils/http/http-client-got.js +11 -11
  88. package/dist/cjs/utils/image-resize-errors/image-resizing-base-error.d.ts +0 -2
  89. package/dist/cjs/utils/image-resize-errors/image-resizing-base-error.js +0 -5
  90. package/dist/cjs/utils/image-resize-errors/index.d.ts +0 -1
  91. package/dist/cjs/utils/image-resize-errors/index.js +1 -3
  92. package/dist/cjs/utils/path.d.ts +1 -0
  93. package/dist/cjs/utils/path.js +11 -0
  94. package/dist/cjs/utils/transformations.d.ts +0 -2
  95. package/dist/cjs/utils/transformations.js +11 -44
  96. package/dist/esm/core/model/test-case-dto/custom-element.d.ts +2 -2
  97. package/dist/esm/core/model/test-case-dto/custom-element.js +2 -2
  98. package/dist/esm/core/model/test-case-dto/custom-element.spec.js +7 -6
  99. package/dist/esm/core/model/test-case-dto/index.js +0 -1
  100. package/dist/esm/core/model/test-case-dto/test-step.d.ts +3 -7
  101. package/dist/esm/core/model/test-case-dto/test-step.js +1 -10
  102. package/dist/esm/core/ui-control-commands/control-command.d.ts +1 -0
  103. package/dist/esm/core/ui-control-commands/control-command.js +5 -0
  104. package/dist/esm/core/ui-control-commands/input-event.d.ts +1 -2
  105. package/dist/esm/core/ui-control-commands/input-event.js +0 -1
  106. package/dist/esm/execution/dsl.d.ts +6 -17
  107. package/dist/esm/execution/dsl.js +3 -21
  108. package/dist/esm/execution/execution-runtime.d.ts +7 -5
  109. package/dist/esm/execution/execution-runtime.js +35 -23
  110. package/dist/esm/execution/index.d.ts +2 -1
  111. package/dist/esm/execution/index.js +2 -1
  112. package/dist/esm/execution/{control-your-ui-api.d.ts → inference-client.d.ts} +4 -4
  113. package/dist/esm/execution/{control-your-ui-api.js → inference-client.js} +7 -7
  114. package/dist/esm/execution/ui-control-client-error.d.ts +2 -0
  115. package/dist/esm/execution/ui-control-client-error.js +2 -0
  116. package/dist/esm/execution/ui-control-client.d.ts +79 -0
  117. package/dist/esm/execution/ui-control-client.js +181 -0
  118. package/dist/esm/execution/{client-connection-state.d.ts → ui-controller-client-connection-state.d.ts} +1 -1
  119. package/dist/esm/execution/ui-controller-client-connection-state.js +7 -0
  120. package/dist/esm/execution/ui-controller-client-interface.d.ts +27 -0
  121. package/dist/esm/execution/{client-interface.js → ui-controller-client-interface.js} +0 -0
  122. package/dist/esm/execution/{control-your-ui-client.d.ts → ui-controller-client.d.ts} +6 -6
  123. package/dist/esm/execution/{control-your-ui-client.js → ui-controller-client.js} +26 -26
  124. package/dist/esm/lib/copy-example-project.js +2 -1
  125. package/dist/esm/lib/download-binaries.js +10 -9
  126. package/dist/esm/lib/index.d.ts +1 -1
  127. package/dist/esm/lib/index.js +1 -1
  128. package/dist/esm/lib/libfuse-error.d.ts +2 -0
  129. package/dist/esm/lib/libfuse-error.js +2 -0
  130. package/dist/esm/lib/logger.js +1 -1
  131. package/dist/esm/lib/timeout-error.d.ts +0 -2
  132. package/dist/esm/lib/timeout-error.js +0 -5
  133. package/dist/esm/lib/ui-controller-args.d.ts +47 -0
  134. package/dist/esm/lib/{control-ui-server-args.js → ui-controller-args.js} +0 -0
  135. package/dist/esm/lib/{control-ui-server-args.spec.d.ts → ui-controller-args.spec.d.ts} +0 -0
  136. package/dist/esm/lib/{control-ui-server-args.spec.js → ui-controller-args.spec.js} +1 -1
  137. package/dist/esm/lib/ui-controller-darwin.d.ts +6 -0
  138. package/dist/esm/lib/{control-ui-server-darwin.js → ui-controller-darwin.js} +6 -6
  139. package/dist/esm/lib/ui-controller-facade.d.ts +17 -0
  140. package/dist/esm/lib/{control-ui-server-facade.js → ui-controller-facade.js} +17 -11
  141. package/dist/esm/lib/ui-controller-linux.d.ts +5 -0
  142. package/dist/esm/lib/ui-controller-linux.js +48 -0
  143. package/dist/esm/lib/ui-controller-win32.d.ts +3 -0
  144. package/dist/esm/lib/ui-controller-win32.js +3 -0
  145. package/dist/esm/lib/ui-controller.d.ts +8 -0
  146. package/dist/esm/lib/{askui-control-server.js → ui-controller.js} +7 -7
  147. package/dist/esm/lib/unkown-error.d.ts +0 -2
  148. package/dist/esm/lib/unkown-error.js +0 -5
  149. package/dist/esm/lib/wayland-error.d.ts +2 -0
  150. package/dist/esm/lib/wayland-error.js +2 -0
  151. package/dist/esm/main.d.ts +3 -2
  152. package/dist/esm/main.js +3 -2
  153. package/dist/esm/shared/index.d.ts +1 -0
  154. package/dist/esm/shared/index.js +1 -0
  155. package/dist/esm/utils/analytics/analytics-interface.d.ts +3 -0
  156. package/dist/esm/utils/analytics/analytics-interface.js +1 -0
  157. package/dist/esm/utils/analytics/analytics.d.ts +5 -0
  158. package/dist/esm/utils/analytics/analytics.js +31 -0
  159. package/dist/esm/utils/analytics/index.d.ts +1 -0
  160. package/dist/esm/utils/analytics/index.js +1 -0
  161. package/dist/esm/utils/analytics/installation-timestamp-create-error.d.ts +4 -0
  162. package/dist/esm/utils/analytics/installation-timestamp-create-error.js +5 -0
  163. package/dist/esm/utils/analytics/installation-timestamp-get-error.d.ts +4 -0
  164. package/dist/esm/utils/analytics/installation-timestamp-get-error.js +5 -0
  165. package/dist/esm/utils/analytics/installation-timestamp.d.ts +7 -0
  166. package/dist/esm/utils/analytics/installation-timestamp.js +61 -0
  167. package/dist/esm/utils/analytics/user-identifier-interface.d.ts +3 -0
  168. package/dist/esm/utils/analytics/user-identifier-interface.js +1 -0
  169. package/dist/esm/utils/analytics/user-identifier.d.ts +4 -0
  170. package/dist/esm/utils/analytics/user-identifier.js +18 -0
  171. package/dist/esm/utils/base_64_image/base-64-image-error.d.ts +2 -0
  172. package/dist/esm/utils/base_64_image/base-64-image-error.js +2 -0
  173. package/dist/esm/utils/base_64_image/base-64-image-string-error.d.ts +4 -0
  174. package/dist/esm/utils/base_64_image/base-64-image-string-error.js +7 -0
  175. package/dist/esm/utils/base_64_image/base-64-image.d.ts +18 -0
  176. package/dist/esm/utils/base_64_image/base-64-image.js +79 -0
  177. package/dist/esm/utils/http/custom-errors/http-client-error.d.ts +0 -2
  178. package/dist/esm/utils/http/custom-errors/http-client-error.js +0 -5
  179. package/dist/esm/utils/http/http-client-got.d.ts +6 -5
  180. package/dist/esm/utils/http/http-client-got.js +11 -11
  181. package/dist/esm/utils/image-resize-errors/image-resizing-base-error.d.ts +0 -2
  182. package/dist/esm/utils/image-resize-errors/image-resizing-base-error.js +0 -5
  183. package/dist/esm/utils/image-resize-errors/index.d.ts +0 -1
  184. package/dist/esm/utils/image-resize-errors/index.js +0 -1
  185. package/dist/esm/utils/path.d.ts +1 -0
  186. package/dist/esm/utils/path.js +4 -0
  187. package/dist/esm/utils/transformations.d.ts +0 -2
  188. package/dist/esm/utils/transformations.js +11 -39
  189. package/dist/example_projects_templates/typescript_jest/test/helper/jest.setup.ts +7 -7
  190. package/dist/example_projects_templates/typescript_jest/test/jest.config.ts +3 -0
  191. package/package.json +15 -8
  192. package/dist/cjs/execution/client-connection-state.js +0 -10
  193. package/dist/cjs/execution/client-error.d.ts +0 -2
  194. package/dist/cjs/execution/client-error.js +0 -6
  195. package/dist/cjs/execution/client-interface.d.ts +0 -13
  196. package/dist/cjs/execution/client.d.ts +0 -25
  197. package/dist/cjs/execution/client.js +0 -116
  198. package/dist/cjs/lib/askui-control-server.d.ts +0 -8
  199. package/dist/cjs/lib/control-ui-server-args.d.ts +0 -21
  200. package/dist/cjs/lib/control-ui-server-darwin.d.ts +0 -6
  201. package/dist/cjs/lib/control-ui-server-facade.d.ts +0 -16
  202. package/dist/cjs/lib/control-ui-server-linux.d.ts +0 -4
  203. package/dist/cjs/lib/control-ui-server-linux.js +0 -13
  204. package/dist/cjs/lib/control-ui-server-win32.d.ts +0 -3
  205. package/dist/cjs/lib/control-ui-server-win32.js +0 -7
  206. package/dist/cjs/utils/image-resize-errors/invalid-base64-image-error.d.ts +0 -3
  207. package/dist/cjs/utils/image-resize-errors/invalid-base64-image-error.js +0 -7
  208. package/dist/esm/execution/client-connection-state.js +0 -7
  209. package/dist/esm/execution/client-error.d.ts +0 -2
  210. package/dist/esm/execution/client-error.js +0 -2
  211. package/dist/esm/execution/client-interface.d.ts +0 -13
  212. package/dist/esm/execution/client.d.ts +0 -25
  213. package/dist/esm/execution/client.js +0 -112
  214. package/dist/esm/lib/askui-control-server.d.ts +0 -8
  215. package/dist/esm/lib/control-ui-server-args.d.ts +0 -21
  216. package/dist/esm/lib/control-ui-server-darwin.d.ts +0 -6
  217. package/dist/esm/lib/control-ui-server-facade.d.ts +0 -16
  218. package/dist/esm/lib/control-ui-server-linux.d.ts +0 -4
  219. package/dist/esm/lib/control-ui-server-linux.js +0 -9
  220. package/dist/esm/lib/control-ui-server-win32.d.ts +0 -3
  221. package/dist/esm/lib/control-ui-server-win32.js +0 -3
  222. package/dist/esm/utils/image-resize-errors/invalid-base64-image-error.d.ts +0 -3
  223. package/dist/esm/utils/image-resize-errors/invalid-base64-image-error.js +0 -3
@@ -1,13 +1,14 @@
1
1
  import { TestStep } from '../core/model/test-case-dto';
2
- import { ControlYourUiClient } from './control-your-ui-client';
3
- import { ControlYourUiApi } from './control-your-ui-api';
2
+ import { UiControllerClient } from './ui-controller-client';
3
+ import { InferenceClient } from './inference-client';
4
4
  import { Annotation } from '../core/annotation/annotation';
5
5
  import { CustomElementJson } from '../core/model/test-case-dto/custom-element-json';
6
6
  export declare class ExecutionRuntime {
7
- private client;
8
- private api;
9
- constructor(client: ControlYourUiClient, api: ControlYourUiApi);
7
+ private uiControllerClient;
8
+ private inferenceClient;
9
+ constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient);
10
10
  executeTestStep(step: TestStep): Promise<void>;
11
+ private requestControl;
11
12
  /**
12
13
  * @param {TestStep} step - Test step used for predicting command.
13
14
  */
@@ -20,6 +21,7 @@ export declare class ExecutionRuntime {
20
21
  * --> retry with linear back-off
21
22
  */
22
23
  private predictCommandWithRetry;
24
+ private getImageIfRequired;
23
25
  private predictCommand;
24
26
  annotateInteractively(): Promise<void>;
25
27
  takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
@@ -12,12 +12,12 @@ import { CustomElement } from '../core/model/test-case-dto';
12
12
  import { RepeatError } from './repeat-error';
13
13
  import { delay } from './misc';
14
14
  import { ControlCommandError } from './control-command-error';
15
- import { toBase64Image } from '../utils/transformations';
16
15
  import { logger } from '../lib/logger';
16
+ import { Base64Image } from '../utils/base_64_image/base-64-image';
17
17
  export class ExecutionRuntime {
18
- constructor(client, api) {
19
- this.client = client;
20
- this.api = api;
18
+ constructor(uiControllerClient, inferenceClient) {
19
+ this.uiControllerClient = uiControllerClient;
20
+ this.inferenceClient = inferenceClient;
21
21
  this.EXEC_REPETITION_COUNT = 25;
22
22
  this.PREDICT_COMMAND_RETRY_COUNT = 2;
23
23
  }
@@ -27,6 +27,11 @@ export class ExecutionRuntime {
27
27
  yield this.executeCommand(step);
28
28
  });
29
29
  }
30
+ requestControl(controlCommand) {
31
+ return __awaiter(this, void 0, void 0, function* () {
32
+ yield this.uiControllerClient.requestControl(controlCommand);
33
+ });
34
+ }
30
35
  /**
31
36
  * @param {TestStep} step - Test step used for predicting command.
32
37
  */
@@ -35,15 +40,13 @@ export class ExecutionRuntime {
35
40
  return __awaiter(this, void 0, void 0, function* () {
36
41
  const controlCommand = yield this.predictCommandWithRetry(step);
37
42
  if (controlCommand.code === ControlCommandCode.OK) {
38
- yield this.client.requestControl(controlCommand);
39
- }
40
- else if (controlCommand.tryToRepeat) {
41
- yield this.client.requestControl(controlCommand);
42
- this.executeCommandRepeatedly(step);
43
+ return this.requestControl(controlCommand);
43
44
  }
44
- else {
45
- throw new ControlCommandError(((_a = controlCommand.actions[0]) === null || _a === void 0 ? void 0 : _a.text) || '');
45
+ if (controlCommand.tryToRepeat) {
46
+ yield this.requestControl(controlCommand);
47
+ return this.executeCommandRepeatedly(step);
46
48
  }
49
+ throw new ControlCommandError(((_a = controlCommand.actions[0]) === null || _a === void 0 ? void 0 : _a.text) || '');
47
50
  });
48
51
  }
49
52
  executeCommandRepeatedly(step) {
@@ -60,8 +63,8 @@ export class ExecutionRuntime {
60
63
  if (controlCommand.code === ControlCommandCode.OK) {
61
64
  break;
62
65
  }
63
- else if (controlCommand.tryToRepeat) {
64
- yield this.client.requestControl(controlCommand);
66
+ if (controlCommand.tryToRepeat) {
67
+ yield this.requestControl(controlCommand);
65
68
  }
66
69
  else {
67
70
  throw new ControlCommandError(((_a = controlCommand.actions[0]) === null || _a === void 0 ? void 0 : _a.text) || '');
@@ -91,31 +94,40 @@ export class ExecutionRuntime {
91
94
  return command;
92
95
  });
93
96
  }
97
+ getImageIfRequired(instruction) {
98
+ return __awaiter(this, void 0, void 0, function* () {
99
+ const isImageRequired = yield this.inferenceClient.isImageRequired(instruction);
100
+ if (!isImageRequired) {
101
+ return undefined;
102
+ }
103
+ const screenshotResponse = yield this.uiControllerClient.requestScreenshot();
104
+ return screenshotResponse.data.image;
105
+ });
106
+ }
94
107
  predictCommand(step) {
95
108
  return __awaiter(this, void 0, void 0, function* () {
96
- const isImageRequired = yield this.api.isImageRequired(step.instruction);
97
- let image;
98
- if (isImageRequired) {
99
- const screenshotResponse = yield this.client.requestScreenshot();
100
- image = screenshotResponse.data.image;
109
+ const image = yield this.getImageIfRequired(step.instruction);
110
+ const controlCommand = yield this.inferenceClient.predictControlCommand(step.instruction, step.customElements, image);
111
+ if (step.secretText !== undefined) {
112
+ controlCommand.setTextToBeTyped(step.secretText);
101
113
  }
102
- return this.api.predictControlCommand(step.instruction, step.customElements, image);
114
+ return controlCommand;
103
115
  });
104
116
  }
105
117
  annotateInteractively() {
106
118
  return __awaiter(this, void 0, void 0, function* () {
107
119
  const annotationResponse = yield this.annotateImage();
108
- yield this.client.annotateInteractively(annotationResponse.objects, annotationResponse.image);
120
+ yield this.uiControllerClient.annotateInteractively(annotationResponse.objects, annotationResponse.image);
109
121
  });
110
122
  }
111
123
  takeScreenshotIfImageisNotProvided(imagePath) {
112
124
  return __awaiter(this, void 0, void 0, function* () {
113
125
  let base64Image = '';
114
126
  if (imagePath !== undefined) {
115
- base64Image = yield toBase64Image(imagePath);
127
+ base64Image = (yield Base64Image.fromPath(imagePath)).toString();
116
128
  }
117
129
  if (imagePath === undefined) {
118
- const screenshotResponse = yield this.client.requestScreenshot();
130
+ const screenshotResponse = yield this.uiControllerClient.requestScreenshot();
119
131
  base64Image = screenshotResponse.data.image;
120
132
  }
121
133
  return base64Image;
@@ -128,7 +140,7 @@ export class ExecutionRuntime {
128
140
  if (customElementJson !== undefined) {
129
141
  customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
130
142
  }
131
- return this.api.predictImageAnnotation(base64Image, customElements);
143
+ return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
132
144
  });
133
145
  }
134
146
  }
@@ -1 +1,2 @@
1
- export { AskuiClient } from './client';
1
+ export { AnnotationLevel } from './annotation-level';
2
+ export { UiControlClient } from './ui-control-client';
@@ -1 +1,2 @@
1
- export { AskuiClient } from './client';
1
+ export { AnnotationLevel } from './annotation-level';
2
+ export { UiControlClient } from './ui-control-client';
@@ -2,12 +2,12 @@ import { HttpClientGot } from '../utils/http/http-client-got';
2
2
  import { ControlCommand } from '../core/ui-control-commands';
3
3
  import { CustomElement } from '../core/model/test-case-dto';
4
4
  import { Annotation } from '../core/annotation/annotation';
5
- export declare class ControlYourUiApi {
6
- apiEndpointUrl: string;
5
+ export declare class InferenceClient {
6
+ url: string;
7
7
  httpClient: HttpClientGot;
8
- constructor(apiEndpointUrl: string, httpClient: HttpClientGot);
8
+ constructor(url: string, httpClient: HttpClientGot);
9
9
  isImageRequired(instruction: string): Promise<boolean>;
10
10
  private resizeIfNeeded;
11
- predictControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand>;
11
+ predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
12
12
  predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
13
13
  }
@@ -11,14 +11,14 @@ import urljoin from 'url-join';
11
11
  import { ControlCommand } from '../core/ui-control-commands';
12
12
  import { Annotation } from '../core/annotation/annotation';
13
13
  import { resizeBase64ImageWithSameRatio } from '../utils/transformations';
14
- export class ControlYourUiApi {
15
- constructor(apiEndpointUrl, httpClient) {
16
- this.apiEndpointUrl = apiEndpointUrl;
14
+ export class InferenceClient {
15
+ constructor(url, httpClient) {
16
+ this.url = url;
17
17
  this.httpClient = httpClient;
18
18
  }
19
19
  isImageRequired(instruction) {
20
20
  return __awaiter(this, void 0, void 0, function* () {
21
- const url = urljoin(this.apiEndpointUrl, 'instruction', 'is-image-required');
21
+ const url = urljoin(this.url, 'instruction', 'is-image-required');
22
22
  const httpBody = {
23
23
  instruction,
24
24
  };
@@ -35,7 +35,7 @@ export class ControlYourUiApi {
35
35
  return resizeBase64ImageWithSameRatio(image);
36
36
  });
37
37
  }
38
- predictControlCommand(instruction, customElements, image) {
38
+ predictControlCommand(instruction, customElements = [], image) {
39
39
  return __awaiter(this, void 0, void 0, function* () {
40
40
  const resizedImage = yield this.resizeIfNeeded(customElements, image);
41
41
  const httpBody = {
@@ -43,7 +43,7 @@ export class ControlYourUiApi {
43
43
  instruction,
44
44
  customElements,
45
45
  };
46
- const url = urljoin(this.apiEndpointUrl, 'api', 'v1', 'predict-command');
46
+ const url = urljoin(this.url, 'api', 'v1', 'predict-command');
47
47
  const httpResponse = yield this.httpClient.post(url, httpBody);
48
48
  return ControlCommand.fromJson(httpResponse, resizedImage.resizeRatio);
49
49
  });
@@ -55,7 +55,7 @@ export class ControlYourUiApi {
55
55
  image: resizedImage.base64Image,
56
56
  customElements,
57
57
  };
58
- const url = urljoin(this.apiEndpointUrl, 'annotate', '?format=json');
58
+ const url = urljoin(this.url, 'annotate', '?format=json');
59
59
  const httpResponse = yield this.httpClient.post(url, httpBody);
60
60
  return Annotation.fromJson(Object.assign(Object.assign({}, httpResponse), { image }), resizedImage.resizeRatio);
61
61
  });
@@ -0,0 +1,2 @@
1
+ export declare class UiControlClientError extends Error {
2
+ }
@@ -0,0 +1,2 @@
1
+ export class UiControlClientError extends Error {
2
+ }
@@ -0,0 +1,79 @@
1
+ import { CustomElementJson } from '../core/model/test-case-dto';
2
+ import { Exec, Executable, FluentCommand, FluentFilters } from './dsl';
3
+ import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
4
+ import { Annotation } from '../core/annotation/annotation';
5
+ import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
6
+ import { ClientArgs } from './ui-controller-client-interface';
7
+ export declare class UiControlClient extends FluentCommand {
8
+ private httpClient;
9
+ private clientArgs;
10
+ private _uiControllerClient?;
11
+ private constructor();
12
+ static build(clientArgs?: ClientArgs): Promise<UiControlClient>;
13
+ private get uiControllerClient();
14
+ private get inferenceClient();
15
+ private get executionRuntime();
16
+ private annotateByDefault;
17
+ connect(): Promise<UiControllerClientConnectionState>;
18
+ annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
19
+ annotateInteractively(): Promise<void>;
20
+ exec(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
21
+ private secretText;
22
+ /**
23
+ * Types a text inside the filtered element.
24
+ *
25
+ * By default, the `text` is included in the logs and sent over to the askui Inference server to
26
+ * predict in which context the typing has to occur. You can exclude the `text` from the logs
27
+ * and the request to the askui Inference server setting `options.isSecret` to `true`.
28
+ * This should not change the quality of the prediction of the askui Inference server. In this
29
+ * case, `options.secretMask` is included in logs and sent over instead of the `text`.
30
+ *
31
+ * @param {string} text - A text to type.
32
+ * @param {Object} [options]
33
+ * @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
34
+ * logs of askui nor sent over to askui Inference for prediction.
35
+ * @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
36
+ * is included in logs and sent over to askui Inference for prediction instead of the `text`.
37
+ *
38
+ * @return {FluentFilters}
39
+ */
40
+ typeIn(text: string, { isSecret, secretMask }?: {
41
+ isSecret?: boolean | undefined;
42
+ secretMask?: string | undefined;
43
+ }): FluentFilters;
44
+ /**
45
+ * Types a text at the current position.
46
+ *
47
+ * By default, the `text` is included in the logs and sent over to the askui Inference server to
48
+ * predict in which context the typing has to occur. You can exclude the `text` from the logs
49
+ * and the request to the askui Inference server setting `options.isSecret` to `true`.
50
+ * This should not change the quality of the prediction of the askui Inference server. In this
51
+ * case, `options.secretMask` is included in logs and sent over instead of the `text`.
52
+ *
53
+ * @param {string} text - A text to type.
54
+ * @param {Object} options
55
+ * @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
56
+ * logs of askui nor sent over to askui Inference for prediction.
57
+ * @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
58
+ * is included in logs and sent over to askui Inference for prediction instead of the `text`.
59
+ *
60
+ * @return {Exec}
61
+ */
62
+ type(text: string, { isSecret, secretMask }?: {
63
+ isSecret?: boolean | undefined;
64
+ secretMask?: string | undefined;
65
+ }): Exec;
66
+ /**
67
+ * Waits for `<delayInMs>` ms, e.g., 1000 ms. The exact delay may be a little longer
68
+ * than `<delayInMs>` but never shorter than that.
69
+ *
70
+ * @param {number} delayInMs - The delay in ms to wait for.
71
+ *
72
+ * @return {Executable}
73
+ */
74
+ waitFor(delayInMs: number): Executable;
75
+ /**
76
+ * closes the connection to the controlui-server`.
77
+ */
78
+ close(): void;
79
+ }
@@ -0,0 +1,181 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { CustomElement } from '../core/model/test-case-dto';
11
+ import { FluentCommand, } from './dsl';
12
+ import { HttpClientGot } from '../utils/http/http-client-got';
13
+ import { UiControllerClient } from './ui-controller-client';
14
+ import { ExecutionRuntime } from './execution-runtime';
15
+ import { InferenceClient } from './inference-client';
16
+ import { AnnotationWriter } from '../core/annotation/annotation-writer';
17
+ import { logger } from '../lib/logger';
18
+ import { TestStepState } from '../core/model/test-case-result-dto';
19
+ import { AnnotationLevel } from './annotation-level';
20
+ import { UiControlClientError } from './ui-control-client-error';
21
+ import { envCredentials } from './read-environment-credentials';
22
+ import { Analytics } from '../utils/analytics';
23
+ const getClientArgsWithDefaults = (clientArgs = {}) => (Object.assign({ uiControllerUrl: 'http://localhost:6769', inferenceServerUrl: 'https://inference.askui.com', annotationLevel: AnnotationLevel.DISABLED }, clientArgs));
24
+ export class UiControlClient extends FluentCommand {
25
+ constructor(httpClient, clientArgs) {
26
+ super();
27
+ this.httpClient = httpClient;
28
+ this.clientArgs = clientArgs;
29
+ this.secretText = undefined;
30
+ }
31
+ static build(clientArgs) {
32
+ return __awaiter(this, void 0, void 0, function* () {
33
+ const analytics = new Analytics();
34
+ const analyticsHeaders = yield analytics.getAnalyticsHeaders();
35
+ const cas = getClientArgsWithDefaults(clientArgs);
36
+ const httpClient = new HttpClientGot(cas.credentials || envCredentials(), analyticsHeaders);
37
+ return new UiControlClient(httpClient, cas);
38
+ });
39
+ }
40
+ get uiControllerClient() {
41
+ if (!this._uiControllerClient) {
42
+ this._uiControllerClient = new UiControllerClient(this.clientArgs.uiControllerUrl);
43
+ }
44
+ return this._uiControllerClient;
45
+ }
46
+ get inferenceClient() {
47
+ return new InferenceClient(this.clientArgs.inferenceServerUrl, this.httpClient);
48
+ }
49
+ get executionRuntime() {
50
+ return new ExecutionRuntime(this.uiControllerClient, this.inferenceClient);
51
+ }
52
+ annotateByDefault(testStepState, customElements = []) {
53
+ return __awaiter(this, void 0, void 0, function* () {
54
+ if ((testStepState === TestStepState.FAILED
55
+ && this.clientArgs.annotationLevel === AnnotationLevel.DISABLED)
56
+ || (testStepState === TestStepState.PASSED
57
+ && this.clientArgs.annotationLevel !== AnnotationLevel.ALL)) {
58
+ return;
59
+ }
60
+ yield this.annotate({
61
+ customElements,
62
+ fileNamePrefix: `${testStepState.toLowerCase()}_testStep_annotation`,
63
+ });
64
+ });
65
+ }
66
+ connect() {
67
+ return __awaiter(this, void 0, void 0, function* () {
68
+ const connectionState = yield this.uiControllerClient.connect();
69
+ return connectionState;
70
+ });
71
+ }
72
+ annotate(annotationRequest = {}) {
73
+ return __awaiter(this, void 0, void 0, function* () {
74
+ const annotation = yield this.executionRuntime.annotateImage(annotationRequest.imagePath, annotationRequest.customElements);
75
+ AnnotationWriter.write(annotation.toHtml(), annotationRequest.outputPath, annotationRequest.fileNamePrefix);
76
+ return annotation;
77
+ });
78
+ }
79
+ annotateInteractively() {
80
+ return __awaiter(this, void 0, void 0, function* () {
81
+ try {
82
+ yield this.executionRuntime.annotateInteractively();
83
+ }
84
+ catch (err) {
85
+ logger.error(err);
86
+ }
87
+ });
88
+ }
89
+ exec(instruction, customElementJson = []) {
90
+ return __awaiter(this, void 0, void 0, function* () {
91
+ const customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
92
+ const { secretText } = this;
93
+ try {
94
+ yield this.executionRuntime.executeTestStep({
95
+ instruction,
96
+ customElements,
97
+ secretText,
98
+ });
99
+ yield this.annotateByDefault(TestStepState.PASSED, customElements);
100
+ return yield Promise.resolve();
101
+ }
102
+ catch (error) {
103
+ yield this.annotateByDefault(TestStepState.FAILED, customElements);
104
+ return Promise.reject(new UiControlClientError(`A problem occures while executing the instruction: ${instruction}. Reason ${error}`));
105
+ }
106
+ });
107
+ }
108
+ /**
109
+ * Types a text inside the filtered element.
110
+ *
111
+ * By default, the `text` is included in the logs and sent over to the askui Inference server to
112
+ * predict in which context the typing has to occur. You can exclude the `text` from the logs
113
+ * and the request to the askui Inference server setting `options.isSecret` to `true`.
114
+ * This should not change the quality of the prediction of the askui Inference server. In this
115
+ * case, `options.secretMask` is included in logs and sent over instead of the `text`.
116
+ *
117
+ * @param {string} text - A text to type.
118
+ * @param {Object} [options]
119
+ * @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
120
+ * logs of askui nor sent over to askui Inference for prediction.
121
+ * @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
122
+ * is included in logs and sent over to askui Inference for prediction instead of the `text`.
123
+ *
124
+ * @return {FluentFilters}
125
+ */
126
+ typeIn(text, { isSecret = false, secretMask = '****' } = {}) {
127
+ if (isSecret) {
128
+ this.secretText = text;
129
+ return super.typeIn(secretMask);
130
+ }
131
+ return super.typeIn(text);
132
+ }
133
+ /**
134
+ * Types a text at the current position.
135
+ *
136
+ * By default, the `text` is included in the logs and sent over to the askui Inference server to
137
+ * predict in which context the typing has to occur. You can exclude the `text` from the logs
138
+ * and the request to the askui Inference server setting `options.isSecret` to `true`.
139
+ * This should not change the quality of the prediction of the askui Inference server. In this
140
+ * case, `options.secretMask` is included in logs and sent over instead of the `text`.
141
+ *
142
+ * @param {string} text - A text to type.
143
+ * @param {Object} options
144
+ * @param {boolean} [options.isSecret = false] - If set to `true`, `text` is neither included in
145
+ * logs of askui nor sent over to askui Inference for prediction.
146
+ * @param {string} [options.secretMask = '****'] - If `options.isSecret` is set to `true`, this
147
+ * is included in logs and sent over to askui Inference for prediction instead of the `text`.
148
+ *
149
+ * @return {Exec}
150
+ */
151
+ type(text, { isSecret = false, secretMask = '****' } = {}) {
152
+ if (isSecret) {
153
+ this.secretText = text;
154
+ return super.type(secretMask);
155
+ }
156
+ return super.type(text);
157
+ }
158
+ /**
159
+ * Waits for `<delayInMs>` ms, e.g., 1000 ms. The exact delay may be a little longer
160
+ * than `<delayInMs>` but never shorter than that.
161
+ *
162
+ * @param {number} delayInMs - The delay in ms to wait for.
163
+ *
164
+ * @return {Executable}
165
+ */
166
+ // eslint-disable-next-line class-methods-use-this
167
+ waitFor(delayInMs) {
168
+ return {
169
+ exec() {
170
+ logger.debug(`Wait for ${delayInMs} ms`);
171
+ return new Promise((resolve) => { setTimeout(() => resolve(), delayInMs); });
172
+ },
173
+ };
174
+ }
175
+ /**
176
+ * closes the connection to the controlui-server`.
177
+ */
178
+ close() {
179
+ this.uiControllerClient.close();
180
+ }
181
+ }
@@ -1,4 +1,4 @@
1
- export declare enum ClientConnectionState {
1
+ export declare enum UiControllerClientConnectionState {
2
2
  NOT_CONNECTED = 0,
3
3
  CONNECTING = 1,
4
4
  CONNECTED = 2,
@@ -0,0 +1,7 @@
1
+ export var UiControllerClientConnectionState;
2
+ (function (UiControllerClientConnectionState) {
3
+ UiControllerClientConnectionState[UiControllerClientConnectionState["NOT_CONNECTED"] = 0] = "NOT_CONNECTED";
4
+ UiControllerClientConnectionState[UiControllerClientConnectionState["CONNECTING"] = 1] = "CONNECTING";
5
+ UiControllerClientConnectionState[UiControllerClientConnectionState["CONNECTED"] = 2] = "CONNECTED";
6
+ UiControllerClientConnectionState[UiControllerClientConnectionState["ERROR"] = 3] = "ERROR";
7
+ })(UiControllerClientConnectionState || (UiControllerClientConnectionState = {}));
@@ -0,0 +1,27 @@
1
+ import { CredentialArgs } from '../utils/http/credentials';
2
+ import { AnnotationLevel } from './annotation-level';
3
+ /**
4
+ * Configuration options for the askui UI Control Client
5
+ *
6
+ * @param {string} uiControllerUrl - Default: http://localhost:6769
7
+ * The adress of the askui UI Controller server.
8
+ * @param {string} inferenceClientUrl - Default: https://inference.askui.com`
9
+ * Address of the askui Inference server.
10
+ * @param {AnnotationLevel} annotationLevel - Default: AnnotationLevel.DISABLED
11
+ * Usage of annotate command
12
+ * after execution of test steps.
13
+ * You have three options: `DISABLED`, `ON_FAILURE`, `ALL`.
14
+ * @param {CredentialArgs} credentials - We need to provide credentials for
15
+ * the authentication of the askui Inference Server.
16
+ */
17
+ export interface ClientArgs {
18
+ readonly uiControllerUrl?: string;
19
+ readonly inferenceServerUrl?: string;
20
+ readonly annotationLevel?: AnnotationLevel;
21
+ readonly credentials?: CredentialArgs;
22
+ }
23
+ export interface ClientArgsWithDefaults extends ClientArgs {
24
+ readonly uiControllerUrl: string;
25
+ readonly inferenceServerUrl: string;
26
+ readonly annotationLevel: AnnotationLevel;
27
+ }
@@ -2,21 +2,21 @@ import WebSocket from 'ws';
2
2
  import { DetectedElement } from '../core/model/annotation-result/detected-element';
3
3
  import { CaptureScreenshotResponse, ControlResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, GetProcessPidResponse } from '../core/runner-protocol';
4
4
  import { ControlCommand } from '../core/ui-control-commands';
5
- import { ClientConnectionState } from './client-connection-state';
6
- export declare class ControlYourUiClient {
7
- controlServerUrl: string;
5
+ import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
6
+ export declare class UiControllerClient {
7
+ url: string;
8
8
  private static readonly EMPTY_REJECT;
9
9
  private static readonly EMPTY_RESOLVE;
10
10
  private static readonly REQUEST_TIMEOUT_IN_MS;
11
11
  ws: WebSocket;
12
- connectionState: ClientConnectionState;
12
+ connectionState: UiControllerClientConnectionState;
13
13
  private timeout?;
14
14
  private currentReject;
15
15
  private currentResolve;
16
- constructor(controlServerUrl: string);
16
+ constructor(url: string);
17
17
  private clearResponse;
18
18
  private onMessage;
19
- connect(): Promise<ClientConnectionState>;
19
+ connect(): Promise<UiControllerClientConnectionState>;
20
20
  close(): void;
21
21
  private sendAndReceive;
22
22
  private send;