askui 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +6 -12
  2. package/dist/cjs/core/annotation/annotation-writer.js +2 -4
  3. package/dist/cjs/core/annotation/annotation.d.ts +3 -4
  4. package/dist/cjs/core/annotation/annotation.js +5 -4
  5. package/dist/cjs/core/inference-response/inference-response.d.ts +15 -0
  6. package/dist/cjs/core/inference-response/inference-response.js +25 -0
  7. package/dist/cjs/core/model/annotation-result/boundary-box.d.ts +23 -0
  8. package/dist/cjs/core/model/annotation-result/boundary-box.js +27 -0
  9. package/dist/cjs/core/model/annotation-result/detected-element.d.ts +9 -3
  10. package/dist/cjs/core/model/annotation-result/detected-element.js +10 -4
  11. package/dist/cjs/core/ui-control-commands/control-command.d.ts +1 -1
  12. package/dist/cjs/core/ui-control-commands/control-command.js +2 -1
  13. package/dist/cjs/core/ui-control-commands/index.d.ts +1 -0
  14. package/dist/cjs/core/ui-control-commands/index.js +3 -1
  15. package/dist/cjs/execution/dsl.d.ts +691 -2
  16. package/dist/cjs/execution/dsl.js +968 -7
  17. package/dist/cjs/execution/dsl.spec.js +4 -4
  18. package/dist/cjs/execution/execution-runtime.d.ts +2 -0
  19. package/dist/cjs/execution/execution-runtime.js +11 -1
  20. package/dist/cjs/execution/inference-client.d.ts +3 -0
  21. package/dist/cjs/execution/inference-client.js +28 -12
  22. package/dist/cjs/execution/inference-response-error.d.ts +2 -0
  23. package/dist/cjs/execution/inference-response-error.js +6 -0
  24. package/dist/cjs/execution/ui-control-client.d.ts +5 -3
  25. package/dist/cjs/execution/ui-control-client.js +10 -2
  26. package/dist/cjs/utils/http/http-client-got.js +5 -1
  27. package/dist/esm/core/annotation/annotation-writer.js +2 -4
  28. package/dist/esm/core/annotation/annotation.d.ts +3 -4
  29. package/dist/esm/core/annotation/annotation.js +5 -4
  30. package/dist/esm/core/inference-response/inference-response.d.ts +15 -0
  31. package/dist/esm/core/inference-response/inference-response.js +21 -0
  32. package/dist/esm/core/model/annotation-result/boundary-box.d.ts +23 -0
  33. package/dist/esm/core/model/annotation-result/boundary-box.js +27 -0
  34. package/dist/esm/core/model/annotation-result/detected-element.d.ts +9 -3
  35. package/dist/esm/core/model/annotation-result/detected-element.js +10 -4
  36. package/dist/esm/core/ui-control-commands/control-command.d.ts +1 -1
  37. package/dist/esm/core/ui-control-commands/control-command.js +2 -1
  38. package/dist/esm/core/ui-control-commands/index.d.ts +1 -0
  39. package/dist/esm/core/ui-control-commands/index.js +1 -0
  40. package/dist/esm/execution/dsl.d.ts +691 -2
  41. package/dist/esm/execution/dsl.js +962 -6
  42. package/dist/esm/execution/dsl.spec.js +4 -4
  43. package/dist/esm/execution/execution-runtime.d.ts +2 -0
  44. package/dist/esm/execution/execution-runtime.js +11 -1
  45. package/dist/esm/execution/inference-client.d.ts +3 -0
  46. package/dist/esm/execution/inference-client.js +29 -13
  47. package/dist/esm/execution/inference-response-error.d.ts +2 -0
  48. package/dist/esm/execution/inference-response-error.js +2 -0
  49. package/dist/esm/execution/ui-control-client.d.ts +5 -3
  50. package/dist/esm/execution/ui-control-client.js +11 -3
  51. package/dist/esm/utils/http/http-client-got.js +5 -1
  52. package/package.json +1 -1
@@ -12,7 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
12
12
  const dsl_1 = require("./dsl");
13
13
  class TestCommand extends dsl_1.FluentCommand {
14
14
  // eslint-disable-next-line class-methods-use-this
15
- exec(instruction, customElements) {
15
+ fluentCommandExecutor(instruction, customElements) {
16
16
  return __awaiter(this, void 0, void 0, function* () {
17
17
  // eslint-disable-next-line no-console
18
18
  console.log(`${instruction} ${customElements}`);
@@ -24,14 +24,14 @@ describe('DSL', () => {
24
24
  describe('custom element', () => {
25
25
  test('should call exec function with zero custom element', () => __awaiter(void 0, void 0, void 0, function* () {
26
26
  const underTest = new TestCommand();
27
- const testCommandSpy = jest.spyOn(underTest, 'exec');
27
+ const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
28
28
  yield underTest.click().button()
29
29
  .exec();
30
30
  expect(testCommandSpy).toHaveBeenCalledWith('Click on button', []);
31
31
  }));
32
32
  test('should call exec function with one custom element', () => __awaiter(void 0, void 0, void 0, function* () {
33
33
  const underTest = new TestCommand();
34
- const testCommandSpy = jest.spyOn(underTest, 'exec');
34
+ const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
35
35
  yield underTest.click().customElement({
36
36
  customImage: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==',
37
37
  imageCompareFormat: 'grayscale',
@@ -46,7 +46,7 @@ describe('DSL', () => {
46
46
  }));
47
47
  test('should call exec function with two custom element', () => __awaiter(void 0, void 0, void 0, function* () {
48
48
  const underTest = new TestCommand();
49
- const testCommandSpy = jest.spyOn(underTest, 'exec');
49
+ const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
50
50
  yield underTest.click().customElement({
51
51
  customImage: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==',
52
52
  imageCompareFormat: 'grayscale',
@@ -3,6 +3,7 @@ import { UiControllerClient } from './ui-controller-client';
3
3
  import { InferenceClient } from './inference-client';
4
4
  import { Annotation } from '../core/annotation/annotation';
5
5
  import { CustomElementJson } from '../core/model/test-case-dto/custom-element-json';
6
+ import { DetectedElement } from '../core/model/annotation-result/detected-element';
6
7
  export declare class ExecutionRuntime {
7
8
  private uiControllerClient;
8
9
  private inferenceClient;
@@ -25,5 +26,6 @@ export declare class ExecutionRuntime {
25
26
  private predictCommand;
26
27
  annotateInteractively(): Promise<void>;
27
28
  takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
29
+ getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
28
30
  annotateImage(imagePath?: string, customElementJson?: CustomElementJson[]): Promise<Annotation>;
29
31
  }
@@ -119,7 +119,7 @@ class ExecutionRuntime {
119
119
  annotateInteractively() {
120
120
  return __awaiter(this, void 0, void 0, function* () {
121
121
  const annotationResponse = yield this.annotateImage();
122
- yield this.uiControllerClient.annotateInteractively(annotationResponse.objects, annotationResponse.image);
122
+ yield this.uiControllerClient.annotateInteractively(annotationResponse.detected_elements, annotationResponse.image);
123
123
  });
124
124
  }
125
125
  takeScreenshotIfImageisNotProvided(imagePath) {
@@ -135,6 +135,16 @@ class ExecutionRuntime {
135
135
  return base64Image;
136
136
  });
137
137
  }
138
+ getDetectedElements(instruction, customElementJson) {
139
+ return __awaiter(this, void 0, void 0, function* () {
140
+ let customElements = [];
141
+ const base64Image = yield this.takeScreenshotIfImageisNotProvided();
142
+ if (customElementJson !== undefined) {
143
+ customElements = yield test_case_dto_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
144
+ }
145
+ return this.inferenceClient.getDetectedElements(instruction, base64Image, customElements);
146
+ });
147
+ }
138
148
  annotateImage(imagePath, customElementJson) {
139
149
  return __awaiter(this, void 0, void 0, function* () {
140
150
  let customElements = [];
@@ -2,6 +2,7 @@ import { HttpClientGot } from '../utils/http/http-client-got';
2
2
  import { ControlCommand } from '../core/ui-control-commands';
3
3
  import { CustomElement } from '../core/model/test-case-dto';
4
4
  import { Annotation } from '../core/annotation/annotation';
5
+ import { DetectedElement } from '../core/model/annotation-result/detected-element';
5
6
  export declare class InferenceClient {
6
7
  baseUrl: string;
7
8
  httpClient: HttpClientGot;
@@ -11,6 +12,8 @@ export declare class InferenceClient {
11
12
  constructor(baseUrl: string, httpClient: HttpClientGot, workspaceId?: string | undefined, apiVersion?: string);
12
13
  isImageRequired(instruction: string): Promise<boolean>;
13
14
  private resizeIfNeeded;
15
+ inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
14
16
  predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
17
+ getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
15
18
  predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
16
19
  }
@@ -17,8 +17,9 @@ const url_join_1 = __importDefault(require("url-join"));
17
17
  const ui_control_commands_1 = require("../core/ui-control-commands");
18
18
  const annotation_1 = require("../core/annotation/annotation");
19
19
  const transformations_1 = require("../utils/transformations");
20
+ const inference_response_error_1 = require("./inference-response-error");
20
21
  class InferenceClient {
21
- constructor(baseUrl, httpClient, workspaceId, apiVersion = 'v2') {
22
+ constructor(baseUrl, httpClient, workspaceId, apiVersion = 'v3') {
22
23
  this.baseUrl = baseUrl;
23
24
  this.httpClient = httpClient;
24
25
  this.workspaceId = workspaceId;
@@ -45,7 +46,7 @@ class InferenceClient {
45
46
  return (0, transformations_1.resizeBase64ImageWithSameRatio)(image);
46
47
  });
47
48
  }
48
- predictControlCommand(instruction, customElements = [], image) {
49
+ inference(customElements = [], image, instruction) {
49
50
  return __awaiter(this, void 0, void 0, function* () {
50
51
  const resizedImage = yield this.resizeIfNeeded(customElements, image);
51
52
  const httpBody = {
@@ -53,21 +54,36 @@ class InferenceClient {
53
54
  instruction,
54
55
  customElements,
55
56
  };
56
- const url = (0, url_join_1.default)(this.url, 'predict-command');
57
+ const url = (0, url_join_1.default)(this.url, 'inference');
57
58
  const httpResponse = yield this.httpClient.post(url, httpBody);
58
- return ui_control_commands_1.ControlCommand.fromJson(httpResponse, resizedImage.resizeRatio);
59
+ return ui_control_commands_1.InferenceResponse.fromJson(httpResponse, resizedImage.resizeRatio, image);
60
+ });
61
+ }
62
+ predictControlCommand(instruction, customElements = [], image) {
63
+ return __awaiter(this, void 0, void 0, function* () {
64
+ const inferenceResponse = yield this.inference(customElements, image, instruction);
65
+ if (!(inferenceResponse instanceof ui_control_commands_1.ControlCommand)) {
66
+ throw new inference_response_error_1.InferenceResponseError('Internal Error. Can not execute command');
67
+ }
68
+ return inferenceResponse;
69
+ });
70
+ }
71
+ getDetectedElements(instruction, image, customElements = []) {
72
+ return __awaiter(this, void 0, void 0, function* () {
73
+ const inferenceResponse = yield this.inference(customElements, image, instruction);
74
+ if (!(inferenceResponse instanceof annotation_1.Annotation)) {
75
+ throw new inference_response_error_1.InferenceResponseError('Internal Error. Unable to get the detected elements');
76
+ }
77
+ return inferenceResponse.detected_elements;
59
78
  });
60
79
  }
61
80
  predictImageAnnotation(image, customElements = []) {
62
81
  return __awaiter(this, void 0, void 0, function* () {
63
- const resizedImage = yield this.resizeIfNeeded(customElements, image);
64
- const httpBody = {
65
- image: resizedImage.base64Image,
66
- customElements,
67
- };
68
- const url = (0, url_join_1.default)(this.url, 'annotate', '?format=json');
69
- const httpResponse = yield this.httpClient.post(url, httpBody);
70
- return annotation_1.Annotation.fromJson(Object.assign(Object.assign({}, httpResponse), { image }), resizedImage.resizeRatio);
82
+ const inferenceResponse = yield this.inference(customElements, image);
83
+ if (!(inferenceResponse instanceof annotation_1.Annotation)) {
84
+ throw new inference_response_error_1.InferenceResponseError('Internal Error. Can not execute annotation');
85
+ }
86
+ return inferenceResponse;
71
87
  });
72
88
  }
73
89
  }
@@ -0,0 +1,2 @@
1
+ export declare class InferenceResponseError extends Error {
2
+ }
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.InferenceResponseError = void 0;
4
+ class InferenceResponseError extends Error {
5
+ }
6
+ exports.InferenceResponseError = InferenceResponseError;
@@ -1,10 +1,11 @@
1
1
  import { CustomElementJson } from '../core/model/test-case-dto';
2
- import { Exec, Executable, FluentCommand, FluentFilters } from './dsl';
2
+ import { Exec, Executable, FluentFilters, ApiCommands } from './dsl';
3
3
  import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
4
4
  import { Annotation } from '../core/annotation/annotation';
5
5
  import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
6
6
  import { ClientArgs } from './ui-controller-client-interface';
7
- export declare class UiControlClient extends FluentCommand {
7
+ import { DetectedElement } from '../core/model/annotation-result/detected-element';
8
+ export declare class UiControlClient extends ApiCommands {
8
9
  private httpClient;
9
10
  private clientArgs;
10
11
  private workspaceId?;
@@ -19,7 +20,8 @@ export declare class UiControlClient extends FluentCommand {
19
20
  annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
20
21
  annotateInteractively(): Promise<void>;
21
22
  private escapeSeparatorString;
22
- exec(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
23
+ fluentCommandExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
24
+ getterExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
23
25
  private secretText;
24
26
  /**
25
27
  * Types a text inside the filtered element.
@@ -24,7 +24,7 @@ const ui_control_client_error_1 = require("./ui-control-client-error");
24
24
  const read_environment_credentials_1 = require("./read-environment-credentials");
25
25
  const analytics_1 = require("../utils/analytics");
26
26
  const getClientArgsWithDefaults = (clientArgs = {}) => (Object.assign({ uiControllerUrl: 'http://127.0.0.1:6769', inferenceServerUrl: 'https://inference.askui.com', annotationLevel: annotation_level_1.AnnotationLevel.DISABLED }, clientArgs));
27
- class UiControlClient extends dsl_1.FluentCommand {
27
+ class UiControlClient extends dsl_1.ApiCommands {
28
28
  constructor(httpClient, clientArgs, workspaceId) {
29
29
  super();
30
30
  this.httpClient = httpClient;
@@ -96,7 +96,7 @@ class UiControlClient extends dsl_1.FluentCommand {
96
96
  escapeSeparatorString(instruction) {
97
97
  return instruction.split(dsl_1.Separators.STRING).join('"');
98
98
  }
99
- exec(instruction, customElementJson = []) {
99
+ fluentCommandExecutor(instruction, customElementJson = []) {
100
100
  return __awaiter(this, void 0, void 0, function* () {
101
101
  const { secretText } = this;
102
102
  const customElements = yield test_case_dto_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
@@ -117,6 +117,14 @@ class UiControlClient extends dsl_1.FluentCommand {
117
117
  }
118
118
  });
119
119
  }
120
+ getterExecutor(instruction, customElementJson = []) {
121
+ return __awaiter(this, void 0, void 0, function* () {
122
+ const customElements = yield test_case_dto_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
123
+ const stringWithoutSeparators = this.escapeSeparatorString(instruction);
124
+ logger_1.logger.debug(stringWithoutSeparators);
125
+ return this.executionRuntime.getDetectedElements(instruction, customElements);
126
+ });
127
+ }
120
128
  /**
121
129
  * Types a text inside the filtered element.
122
130
  *
@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
15
15
  exports.HttpClientGot = void 0;
16
16
  const got_1 = __importDefault(require("got"));
17
17
  const tough_cookie_1 = require("tough-cookie");
18
+ const lib_1 = require("../../lib");
18
19
  const credentials_1 = require("./credentials");
19
20
  const custom_errors_1 = require("./custom-errors");
20
21
  class HttpClientGot {
@@ -39,7 +40,10 @@ class HttpClientGot {
39
40
  post(url, data) {
40
41
  return __awaiter(this, void 0, void 0, function* () {
41
42
  const options = this.injectHeadersAndCookies(url, { json: data, responseType: 'json', throwHttpErrors: false });
42
- const { body, statusCode } = yield got_1.default.post(url, options);
43
+ const { body, statusCode, headers } = yield got_1.default.post(url, options);
44
+ if (headers['deprecation'] !== undefined) {
45
+ lib_1.logger.warn(headers['deprecation']);
46
+ }
43
47
  if (statusCode !== 200) {
44
48
  throw (0, custom_errors_1.httpClientErrorHandler)(statusCode, JSON.stringify(body));
45
49
  }
@@ -4,10 +4,8 @@ import { logger } from '../../lib/logger';
4
4
  export class AnnotationWriter {
5
5
  static write(html, outputFolder = 'report', fileNamePrefix = 'annotation') {
6
6
  const currentDateTime = new Date();
7
- const formattedTime = `${currentDateTime.getDay()}-${currentDateTime.getMonth()}`
8
- + `-${currentDateTime.getFullYear()}_${currentDateTime.getHours()}`
9
- + `-${currentDateTime.getMinutes()}-${currentDateTime.getSeconds()}`;
10
- const fileName = `${fileNamePrefix}_${formattedTime}.html`;
7
+ const currentTimeStringOnlyNumbers = currentDateTime.toISOString().replace(/\D/g, '');
8
+ const fileName = `${currentTimeStringOnlyNumbers}_${fileNamePrefix}.html`;
11
9
  const outputFilePath = path.join(outputFolder, fileName);
12
10
  if (!(fs.existsSync(outputFolder))) {
13
11
  fs.mkdirSync(outputFolder, { recursive: true });
@@ -1,11 +1,10 @@
1
1
  import { JSDOM } from 'jsdom';
2
2
  import { DetectedElement } from '../model/annotation-result/detected-element';
3
- import { AnnotationJson } from './annotation-json';
4
3
  export declare class Annotation {
5
4
  image: string;
6
- objects: DetectedElement[];
7
- constructor(image: string, objects?: DetectedElement[]);
5
+ detected_elements: DetectedElement[];
6
+ constructor(image: string, detected_elements?: DetectedElement[]);
8
7
  toHtml(): JSDOM;
9
- static fromJson(json: AnnotationJson, resizeRatio?: number): Annotation;
8
+ static fromJson(json: unknown, resizeRatio?: number): Annotation;
10
9
  private static getHtmlTemplate;
11
10
  }
@@ -3,9 +3,9 @@ import path from 'path';
3
3
  import { JSDOM } from 'jsdom';
4
4
  import { DetectedElement } from '../model/annotation-result/detected-element';
5
5
  export class Annotation {
6
- constructor(image, objects = []) {
6
+ constructor(image, detected_elements = []) {
7
7
  this.image = image;
8
- this.objects = objects;
8
+ this.detected_elements = detected_elements;
9
9
  }
10
10
  toHtml() {
11
11
  const template = Annotation.getHtmlTemplate();
@@ -14,13 +14,14 @@ export class Annotation {
14
14
  var el = document.getElementsByTagName("bounding-box-renderer");
15
15
  el[0].setAttribute("shouldrenderimage", true);
16
16
  el[0].setAttribute("imagestr", "${this.image.trim()}");
17
- el[0].setAttribute("detectedobjects", JSON.stringify(${JSON.stringify(this.objects)}));
17
+ el[0].setAttribute("detectedobjects", JSON.stringify(${JSON.stringify(this.detected_elements)}));
18
18
  `;
19
19
  template.window.document.body.appendChild(script);
20
20
  return template;
21
21
  }
22
22
  static fromJson(json, resizeRatio = 1) {
23
- return new Annotation(json.image, json.objects.map((object) => DetectedElement.fromJson(object, resizeRatio)));
23
+ const annotation = json;
24
+ return new Annotation(annotation.image, annotation.detected_elements.map((data) => DetectedElement.fromJson(data, resizeRatio)));
24
25
  }
25
26
  static getHtmlTemplate() {
26
27
  const templatePath = path.join(__dirname, 'template.html');
@@ -0,0 +1,15 @@
1
+ import { ControlCommand } from '../ui-control-commands/control-command';
2
+ import { Annotation } from '../annotation/annotation';
3
+ export declare class InferenceResponse {
4
+ type: string;
5
+ data: ControlCommand | Annotation;
6
+ constructor(type: string, data: ControlCommand | Annotation);
7
+ static fromJson(json: unknown, resizeRatio?: number, image?: string): ControlCommand | Annotation;
8
+ static createModels(type: string, data: ControlCommand | Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;
9
+ static models: Models;
10
+ }
11
+ interface Models {
12
+ DETECTED_ELEMENTS: CallableFunction;
13
+ COMMANDS: CallableFunction;
14
+ }
15
+ export {};
@@ -0,0 +1,21 @@
1
+ import { ControlCommand } from '../ui-control-commands/control-command';
2
+ import { Annotation } from '../annotation/annotation';
3
+ export class InferenceResponse {
4
+ constructor(type, data) {
5
+ this.type = type;
6
+ this.data = data;
7
+ }
8
+ static fromJson(json, resizeRatio = 1, image) {
9
+ const inferenceResponse = json;
10
+ return this.createModels(inferenceResponse.type, inferenceResponse.data, resizeRatio, image);
11
+ }
12
+ static createModels(type, data, resizeRatio, image) {
13
+ return this.models[type](data, resizeRatio, image);
14
+ }
15
+ }
16
+ InferenceResponse.models = {
17
+ DETECTED_ELEMENTS: (data, resizeRatio, image) => Annotation
18
+ .fromJson({ image, detected_elements: data.detected_elements }, resizeRatio),
19
+ COMMANDS: (data, resizeRatio) => ControlCommand
20
+ .fromJson(data, resizeRatio),
21
+ };
@@ -1,3 +1,14 @@
1
+ /**
2
+ *
3
+ * @remarks
4
+ * The screen top left corner is the origin
5
+ *
6
+ * @param {number} xmin - The bounding box xmin coordinate in pixels
7
+ * @param {number} ymin - The bounding box ymin coordinate in pixels
8
+ * @param {number} xmax - The bounding box xmax coordinate in pixels
9
+ * @param {number} ymax - The bounding box ymax coordinate in pixels
10
+ *
11
+ */
1
12
  export declare class BoundingBox {
2
13
  xmin: number;
3
14
  ymin: number;
@@ -5,4 +16,16 @@ export declare class BoundingBox {
5
16
  ymax: number;
6
17
  constructor(xmin: number, ymin: number, xmax: number, ymax: number);
7
18
  static fromJson(boundinBox: BoundingBox, resizeRatio?: number): BoundingBox;
19
+ /**
20
+ *
21
+ * @returns {number} The bounding box height in pixels
22
+ *
23
+ */
24
+ get_height(): number;
25
+ /**
26
+ *
27
+ * @returns {number} The bounding box width in pixels
28
+ *
29
+ */
30
+ get_width(): number;
8
31
  }
@@ -1,3 +1,14 @@
1
+ /**
2
+ *
3
+ * @remarks
4
+ * The screen top left corner is the origin
5
+ *
6
+ * @param {number} xmin - The bounding box xmin coordinate in pixels
7
+ * @param {number} ymin - The bounding box ymin coordinate in pixels
8
+ * @param {number} xmax - The bounding box xmax coordinate in pixels
9
+ * @param {number} ymax - The bounding box ymax coordinate in pixels
10
+ *
11
+ */
1
12
  export class BoundingBox {
2
13
  constructor(xmin, ymin, xmax, ymax) {
3
14
  this.xmin = xmin;
@@ -8,4 +19,20 @@ export class BoundingBox {
8
19
  static fromJson(boundinBox, resizeRatio = 1) {
9
20
  return new BoundingBox(boundinBox.xmin * resizeRatio, boundinBox.ymin * resizeRatio, boundinBox.xmax * resizeRatio, boundinBox.ymax * resizeRatio);
10
21
  }
22
+ /**
23
+ *
24
+ * @returns {number} The bounding box height in pixels
25
+ *
26
+ */
27
+ get_height() {
28
+ return this.ymax - this.ymin;
29
+ }
30
+ /**
31
+ *
32
+ * @returns {number} The bounding box width in pixels
33
+ *
34
+ */
35
+ get_width() {
36
+ return this.xmax - this.xmin;
37
+ }
11
38
  }
@@ -1,11 +1,17 @@
1
1
  import { BoundingBox } from './boundary-box';
2
+ /**
3
+ *
4
+ * @param {string} name - The element type e.g text, button
5
+ * @param {string} text - The detected text inside the element
6
+ * @param {string[]} colors - The element top 3 dominate colors
7
+ * @param {BoundingBox} bndbox - The element bounding box
8
+ *
9
+ */
2
10
  export declare class DetectedElement {
3
11
  name: string;
4
- truncated: number;
5
- difficult: number;
6
12
  text: string;
7
13
  colors: string[];
8
14
  bndbox: BoundingBox;
9
- constructor(name: string, truncated: number, difficult: number, text: string, colors: string[], bndbox: BoundingBox);
15
+ constructor(name: string, text: string, colors: string[], bndbox: BoundingBox);
10
16
  static fromJson(detectedElement: DetectedElement, resizeRatio?: number): DetectedElement;
11
17
  }
@@ -1,14 +1,20 @@
1
1
  import { BoundingBox } from './boundary-box';
2
+ /**
3
+ *
4
+ * @param {string} name - The element type e.g text, button
5
+ * @param {string} text - The detected text inside the element
6
+ * @param {string[]} colors - The element top 3 dominate colors
7
+ * @param {BoundingBox} bndbox - The element bounding box
8
+ *
9
+ */
2
10
  export class DetectedElement {
3
- constructor(name, truncated, difficult, text, colors, bndbox) {
11
+ constructor(name, text, colors, bndbox) {
4
12
  this.name = name;
5
- this.truncated = truncated;
6
- this.difficult = difficult;
7
13
  this.text = text;
8
14
  this.colors = colors;
9
15
  this.bndbox = bndbox;
10
16
  }
11
17
  static fromJson(detectedElement, resizeRatio = 1) {
12
- return new DetectedElement(detectedElement.name, detectedElement.truncated, detectedElement.difficult, detectedElement.text, detectedElement.colors, BoundingBox.fromJson(detectedElement.bndbox, resizeRatio));
18
+ return new DetectedElement(detectedElement.name, detectedElement.text, detectedElement.colors, BoundingBox.fromJson(detectedElement.bndbox, resizeRatio));
13
19
  }
14
20
  }
@@ -5,6 +5,6 @@ export declare class ControlCommand {
5
5
  actions: Action[];
6
6
  tryToRepeat: boolean;
7
7
  constructor(code: ControlCommandCode, actions: Action[], tryToRepeat?: boolean);
8
- static fromJson(json: ControlCommand, resizeRatio?: number): ControlCommand;
8
+ static fromJson(json: unknown, resizeRatio?: number): ControlCommand;
9
9
  setTextToBeTyped(text: string): void;
10
10
  }
@@ -8,7 +8,8 @@ export class ControlCommand {
8
8
  this.tryToRepeat = tryToRepeat;
9
9
  }
10
10
  static fromJson(json, resizeRatio = 1) {
11
- return new ControlCommand(ControlCommandCode[json.code], json.actions.map((action) => Action.fromJson(action, resizeRatio)), json.tryToRepeat);
11
+ const controlCommand = json;
12
+ return new ControlCommand(ControlCommandCode[controlCommand.code], controlCommand.actions.map((action) => Action.fromJson(action, resizeRatio)), controlCommand.tryToRepeat);
12
13
  }
13
14
  setTextToBeTyped(text) {
14
15
  this.actions = this.actions.map((action) => ([InputEvent.TYPE, InputEvent.TYPE_TEXT].includes(action.inputEvent)
@@ -1,2 +1,3 @@
1
1
  export { ControlCommand } from './control-command';
2
2
  export { ControlCommandCode } from './control-command-code';
3
+ export { InferenceResponse } from '../inference-response/inference-response';
@@ -1,2 +1,3 @@
1
1
  export { ControlCommand } from './control-command';
2
2
  export { ControlCommandCode } from './control-command-code';
3
+ export { InferenceResponse } from '../inference-response/inference-response';