askui 0.20.7 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/inference-response/inference-response.d.ts +5 -0
- package/dist/cjs/core/inference-response/model-type.d.ts +1 -1
- package/dist/cjs/core/model/custom-element.d.ts +5 -5
- package/dist/cjs/core/model/custom-element.js +2 -2
- package/dist/cjs/core/reporting/index.js +5 -1
- package/dist/cjs/core/reporting/snapshot-detail-level.d.ts +1 -1
- package/dist/cjs/core/reporting/snapshot-detail-level.js +1 -2
- package/dist/cjs/core/reporting/step-status-end.d.ts +1 -1
- package/dist/cjs/core/reporting/step-status.d.ts +1 -1
- package/dist/cjs/core/runner-protocol/index.js +5 -1
- package/dist/cjs/core/ui-control-commands/action.d.ts +1 -1
- package/dist/cjs/core/ui-control-commands/control-command-code.js +1 -1
- package/dist/cjs/core/ui-control-commands/input-event.js +1 -1
- package/dist/cjs/execution/dsl.d.ts +6 -6
- package/dist/cjs/execution/dsl.js +1 -1
- package/dist/cjs/execution/execution-runtime.d.ts +4 -2
- package/dist/cjs/execution/execution-runtime.js +14 -7
- package/dist/cjs/execution/index.d.ts +1 -0
- package/dist/cjs/execution/index.js +6 -1
- package/dist/cjs/execution/inference-client.d.ts +3 -0
- package/dist/cjs/execution/inference-client.js +33 -8
- package/dist/cjs/execution/misc.js +1 -2
- package/dist/cjs/execution/read-credentials.js +1 -2
- package/dist/cjs/execution/read-environment-credentials.js +1 -2
- package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/index.d.ts +4 -0
- package/dist/cjs/execution/retry-strategies/index.js +20 -0
- package/dist/cjs/execution/retry-strategies/linear-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/linear-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/retry-strategy.js +2 -0
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -5
- package/dist/cjs/execution/ui-control-client.d.ts +92 -3
- package/dist/cjs/execution/ui-control-client.js +115 -15
- package/dist/cjs/execution/ui-controller-client-connection-state.js +1 -1
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +6 -0
- package/dist/cjs/execution/ui-controller-client.js +2 -1
- package/dist/cjs/lib/download-binaries.d.ts +0 -1
- package/dist/cjs/lib/download-binaries.js +3 -4
- package/dist/cjs/lib/interactive_cli/add-remove-script-package-json.js +2 -3
- package/dist/cjs/lib/interactive_cli/cli.js +1 -2
- package/dist/cjs/lib/logger.d.ts +1 -1
- package/dist/cjs/lib/ui-controller-args.js +2 -3
- package/dist/cjs/lib/ui-controller-facade.js +4 -4
- package/dist/cjs/main.js +5 -1
- package/dist/cjs/shared/log-levels.js +1 -1
- package/dist/cjs/shared/proxy-agent-args.d.ts +0 -1
- package/dist/cjs/utils/analytics/installation-timestamp-create-error.d.ts +0 -1
- package/dist/cjs/utils/analytics/installation-timestamp-get-error.d.ts +0 -1
- package/dist/cjs/utils/base_64_image/sharp.js +6 -3
- package/dist/cjs/utils/http/custom-errors/index.js +2 -2
- package/dist/cjs/utils/http/http-client-got.d.ts +0 -1
- package/dist/cjs/utils/http/http-client-got.js +2 -2
- package/dist/cjs/utils/path.js +1 -2
- package/dist/cjs/utils/proxy/proxy-builder.js +7 -3
- package/dist/cjs/utils/transformations.js +3 -4
- package/dist/esm/core/inference-response/inference-response.d.ts +5 -0
- package/dist/esm/core/inference-response/model-type.d.ts +1 -1
- package/dist/esm/core/model/custom-element.d.ts +5 -5
- package/dist/esm/core/model/custom-element.js +2 -2
- package/dist/esm/core/reporting/snapshot-detail-level.d.ts +1 -1
- package/dist/esm/core/reporting/step-status-end.d.ts +1 -1
- package/dist/esm/core/reporting/step-status.d.ts +1 -1
- package/dist/esm/core/ui-control-commands/action.d.ts +1 -1
- package/dist/esm/execution/dsl.d.ts +6 -6
- package/dist/esm/execution/execution-runtime.d.ts +4 -2
- package/dist/esm/execution/execution-runtime.js +14 -7
- package/dist/esm/execution/index.d.ts +1 -0
- package/dist/esm/execution/index.js +1 -0
- package/dist/esm/execution/inference-client.d.ts +3 -0
- package/dist/esm/execution/inference-client.js +33 -8
- package/dist/esm/execution/retry-strategies/exponential-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/exponential-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/fixed-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/fixed-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/index.d.ts +4 -0
- package/dist/esm/execution/retry-strategies/index.js +4 -0
- package/dist/esm/execution/retry-strategies/linear-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/linear-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/retry-strategy.js +1 -0
- package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -5
- package/dist/esm/execution/ui-control-client.d.ts +92 -3
- package/dist/esm/execution/ui-control-client.js +115 -15
- package/dist/esm/execution/ui-controller-client-interface.d.ts +6 -0
- package/dist/esm/execution/ui-controller-client.js +2 -1
- package/dist/esm/lib/download-binaries.d.ts +0 -1
- package/dist/esm/lib/logger.d.ts +1 -1
- package/dist/esm/lib/ui-controller-facade.js +4 -4
- package/dist/esm/shared/proxy-agent-args.d.ts +0 -1
- package/dist/esm/utils/analytics/installation-timestamp-create-error.d.ts +0 -1
- package/dist/esm/utils/analytics/installation-timestamp-get-error.d.ts +0 -1
- package/dist/esm/utils/http/http-client-got.d.ts +0 -1
- package/dist/esm/utils/http/http-client-got.js +2 -2
- package/dist/esm/utils/transformations.js +2 -2
- package/package.json +28 -28
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LinearRetryStrategy = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* LinearRetryStrategy implements a retry strategy that uses a linear backoff algorithm.
|
|
6
|
+
*/
|
|
7
|
+
class LinearRetryStrategy {
|
|
8
|
+
/**
|
|
9
|
+
* @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
|
|
10
|
+
* @param retryCount - The maximum number of retries (default is 3)
|
|
11
|
+
*/
|
|
12
|
+
constructor(baseDelayMs = 1000, retryCount = 3) {
|
|
13
|
+
this.baseDelayMs = baseDelayMs;
|
|
14
|
+
this.retryCount = retryCount;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Calculates the delay for a given retry attempt using linear backoff.
|
|
18
|
+
* @param attempt - The current retry attempt number (1-based)
|
|
19
|
+
* @returns The delay in milliseconds for the current attempt
|
|
20
|
+
*/
|
|
21
|
+
getDelay(attempt) {
|
|
22
|
+
return this.baseDelayMs * attempt;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
exports.LinearRetryStrategy = LinearRetryStrategy;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interface representing a retry strategy for operations.
|
|
3
|
+
*/
|
|
4
|
+
export interface RetryStrategy {
|
|
5
|
+
/**
|
|
6
|
+
* The base delay in milliseconds before the first retry.
|
|
7
|
+
*/
|
|
8
|
+
baseDelayMs: number;
|
|
9
|
+
/**
|
|
10
|
+
* The maximum number of retry attempts.
|
|
11
|
+
*/
|
|
12
|
+
retryCount: number;
|
|
13
|
+
/**
|
|
14
|
+
* Function to calculate the delay before the next retry attempt.
|
|
15
|
+
* @param attempt - The current retry attempt number (0-based).
|
|
16
|
+
* @returns The delay in milliseconds for the next retry.
|
|
17
|
+
*/
|
|
18
|
+
getDelay(attempt: number): number;
|
|
19
|
+
}
|
|
@@ -22,10 +22,11 @@ const proxy_builder_1 = require("../utils/proxy/proxy-builder");
|
|
|
22
22
|
const execution_runtime_1 = require("./execution-runtime");
|
|
23
23
|
const reporting_1 = require("../core/reporting");
|
|
24
24
|
const read_credentials_1 = require("./read-credentials");
|
|
25
|
+
const linear_retry_strategy_1 = require("./retry-strategies/linear-retry-strategy");
|
|
25
26
|
class UiControlClientDependencyBuilder {
|
|
26
27
|
static buildHttpClient(clientArgs) {
|
|
27
|
-
var _a;
|
|
28
28
|
return __awaiter(this, void 0, void 0, function* () {
|
|
29
|
+
var _a;
|
|
29
30
|
const analytics = new analytics_1.Analytics();
|
|
30
31
|
const analyticsHeaders = yield analytics.getAnalyticsHeaders(clientArgs.context);
|
|
31
32
|
const analyticsCookies = yield analytics.getAnalyticsCookies();
|
|
@@ -33,8 +34,8 @@ class UiControlClientDependencyBuilder {
|
|
|
33
34
|
});
|
|
34
35
|
}
|
|
35
36
|
static buildInferenceClient(clientArgs) {
|
|
36
|
-
var _a;
|
|
37
37
|
return __awaiter(this, void 0, void 0, function* () {
|
|
38
|
+
var _a;
|
|
38
39
|
const httpClient = yield UiControlClientDependencyBuilder.buildHttpClient(clientArgs);
|
|
39
40
|
return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition, clientArgs.inferenceServerApiVersion);
|
|
40
41
|
});
|
|
@@ -43,22 +44,22 @@ class UiControlClientDependencyBuilder {
|
|
|
43
44
|
return new ui_controller_client_1.UiControllerClient(clientArgs.uiControllerUrl);
|
|
44
45
|
}
|
|
45
46
|
static build(clientArgs) {
|
|
46
|
-
var _a;
|
|
47
47
|
return __awaiter(this, void 0, void 0, function* () {
|
|
48
|
+
var _a, _b;
|
|
48
49
|
const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs);
|
|
49
50
|
const inferenceClient = yield UiControlClientDependencyBuilder.buildInferenceClient(clientArgs);
|
|
50
51
|
const stepReporter = new reporting_1.StepReporter(clientArgs.reporter);
|
|
51
52
|
const workspaceId = (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId;
|
|
52
53
|
return {
|
|
53
|
-
executionRuntime: new execution_runtime_1.ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter),
|
|
54
|
+
executionRuntime: new execution_runtime_1.ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter, (_b = clientArgs.retryStrategy) !== null && _b !== void 0 ? _b : new linear_retry_strategy_1.LinearRetryStrategy()),
|
|
54
55
|
stepReporter,
|
|
55
56
|
workspaceId,
|
|
56
57
|
};
|
|
57
58
|
});
|
|
58
59
|
}
|
|
59
60
|
static getClientArgsWithDefaults(clientArgs) {
|
|
60
|
-
var _a, _b, _c, _d, _e, _f;
|
|
61
61
|
return __awaiter(this, void 0, void 0, function* () {
|
|
62
|
+
var _a, _b, _c, _d, _e, _f;
|
|
62
63
|
return Object.assign(Object.assign({}, clientArgs), { context: {
|
|
63
64
|
isCi: (_b = (_a = clientArgs.context) === null || _a === void 0 ? void 0 : _a.isCi) !== null && _b !== void 0 ? _b : is_ci_1.default,
|
|
64
65
|
}, credentials: (0, read_credentials_1.readCredentials)(clientArgs), inferenceServerApiVersion: (_c = clientArgs.inferenceServerApiVersion) !== null && _c !== void 0 ? _c : 'v3', inferenceServerUrl: (_d = clientArgs.inferenceServerUrl) !== null && _d !== void 0 ? _d : 'https://inference.askui.com', proxyAgents: (_e = clientArgs.proxyAgents) !== null && _e !== void 0 ? _e : (yield (0, proxy_builder_1.envProxyAgents)()), uiControllerUrl: (_f = clientArgs.uiControllerUrl) !== null && _f !== void 0 ? _f : 'http://127.0.0.1:6769' });
|
|
@@ -4,9 +4,9 @@ import { Annotation } from '../core/annotation/annotation';
|
|
|
4
4
|
import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
|
|
5
5
|
import { DetectedElement } from '../core/model/annotation-result/detected-element';
|
|
6
6
|
import { ClientArgs } from './ui-controller-client-interface';
|
|
7
|
-
export
|
|
8
|
-
export
|
|
9
|
-
export
|
|
7
|
+
export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
|
|
8
|
+
export type TextMatchingOption = 'similar' | 'exact' | 'regex';
|
|
9
|
+
export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
|
|
10
10
|
export interface ElementExistsQueryText {
|
|
11
11
|
value: string;
|
|
12
12
|
matching?: TextMatchingOption;
|
|
@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
|
|
|
59
59
|
private getAIElementsByNames;
|
|
60
60
|
fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
|
|
61
61
|
getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
|
|
62
|
+
/**
|
|
63
|
+
* Takes a prompt that contains a question you want to be answered
|
|
64
|
+
* or the data you want to have extracted from your screen.
|
|
65
|
+
*
|
|
66
|
+
* The optional 'config' can be used to specifiy the JSON schema the
|
|
67
|
+
* returned object shall have (https://json-schema.org).
|
|
68
|
+
*
|
|
69
|
+
* See the following examples on how to use it:
|
|
70
|
+
*
|
|
71
|
+
* let isWidgetsNew =
|
|
72
|
+
* await aui.ask(
|
|
73
|
+
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
|
|
74
|
+
* {
|
|
75
|
+
* json_schema: {
|
|
76
|
+
* "type": "boolean"
|
|
77
|
+
* }
|
|
78
|
+
* });
|
|
79
|
+
*
|
|
80
|
+
* Output of console.log(isWidgetsNew): true
|
|
81
|
+
*
|
|
82
|
+
* let newClients =
|
|
83
|
+
* await aui.ask(
|
|
84
|
+
* "How many new clients?",
|
|
85
|
+
* {
|
|
86
|
+
* json_schema: {
|
|
87
|
+
* "type": "number"
|
|
88
|
+
* }
|
|
89
|
+
* });
|
|
90
|
+
*
|
|
91
|
+
* Output of console.log(newClients): 9123
|
|
92
|
+
*
|
|
93
|
+
* let userNames =
|
|
94
|
+
* await aui.ask(
|
|
95
|
+
* "Return a list with the users names.",
|
|
96
|
+
* {
|
|
97
|
+
* json_schema: {
|
|
98
|
+
* "type": "array",
|
|
99
|
+
* "items": {
|
|
100
|
+
* "type": "string"
|
|
101
|
+
* }
|
|
102
|
+
* }
|
|
103
|
+
* });
|
|
104
|
+
*
|
|
105
|
+
* Output of console.log(userNames):
|
|
106
|
+
* [
|
|
107
|
+
* 'Yiorgos Avraamu',
|
|
108
|
+
* 'Avram Tsarios',
|
|
109
|
+
* 'Quintin Ed',
|
|
110
|
+
* 'Enéas Kwadwo',
|
|
111
|
+
* 'Agapetus Tadeáš'
|
|
112
|
+
* ]
|
|
113
|
+
*
|
|
114
|
+
* let users =
|
|
115
|
+
* await aui.ask(
|
|
116
|
+
* "Extract the users from the table.",
|
|
117
|
+
* {
|
|
118
|
+
* json_schema: {
|
|
119
|
+
* "type": "array",
|
|
120
|
+
* "items": {
|
|
121
|
+
* "type": "object",
|
|
122
|
+
* "properties": {
|
|
123
|
+
* "name": {
|
|
124
|
+
* "type": "string"
|
|
125
|
+
* },
|
|
126
|
+
* "usage": {
|
|
127
|
+
* "type": "number"
|
|
128
|
+
* }
|
|
129
|
+
* },
|
|
130
|
+
* "additionalProperties": false,
|
|
131
|
+
* "required": ["name", "usage"]
|
|
132
|
+
* },
|
|
133
|
+
* },
|
|
134
|
+
* });
|
|
135
|
+
*
|
|
136
|
+
* Output of console.log(users):
|
|
137
|
+
* [
|
|
138
|
+
* { name: 'Yiorgos Avraamu', usage: 50 },
|
|
139
|
+
* { name: 'Avram Tarasios', usage: 10 },
|
|
140
|
+
* { name: 'Quintin Ed', usage: 74 },
|
|
141
|
+
* { name: 'Eneás Kwadwo', usage: 98 },
|
|
142
|
+
* { name: 'Agapetus Tadeáš', usage: 22 }
|
|
143
|
+
* ]
|
|
144
|
+
*
|
|
145
|
+
* @param {string} prompt - The question you want to be answered or
|
|
146
|
+
* the data you want to have extracted.
|
|
147
|
+
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
|
|
148
|
+
* @returns {any} - The answer as JSON specified in the config object.
|
|
149
|
+
*/
|
|
150
|
+
ask(prompt: string, config?: object): Promise<any>;
|
|
62
151
|
private secretText;
|
|
63
152
|
private getAndResetSecretText;
|
|
64
153
|
/**
|
|
@@ -25,8 +25,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
25
25
|
this.stepReporter = stepReporter;
|
|
26
26
|
this.secretText = undefined;
|
|
27
27
|
}
|
|
28
|
-
static build(
|
|
29
|
-
return __awaiter(this,
|
|
28
|
+
static build() {
|
|
29
|
+
return __awaiter(this, arguments, void 0, function* (clientArgs = {}) {
|
|
30
30
|
const builder = ui_control_client_dependency_builder_1.UiControlClientDependencyBuilder;
|
|
31
31
|
const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
|
|
32
32
|
const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
|
|
@@ -75,8 +75,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
75
75
|
|| (this.stepReporter.config.withDetectedElements === 'always');
|
|
76
76
|
}
|
|
77
77
|
afterCommandExecution(instruction, error) {
|
|
78
|
-
var _a;
|
|
79
78
|
return __awaiter(this, void 0, void 0, function* () {
|
|
79
|
+
var _a;
|
|
80
80
|
const createdAt = new Date();
|
|
81
81
|
let annotation;
|
|
82
82
|
let screenshot;
|
|
@@ -93,8 +93,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
93
93
|
}, error);
|
|
94
94
|
});
|
|
95
95
|
}
|
|
96
|
-
annotate(
|
|
97
|
-
return __awaiter(this,
|
|
96
|
+
annotate() {
|
|
97
|
+
return __awaiter(this, arguments, void 0, function* (annotationRequest = {}) {
|
|
98
98
|
const annotation = yield this.executionRuntime.annotateImage(annotationRequest.imagePath, annotationRequest.customElements, annotationRequest.elements);
|
|
99
99
|
annotation_writer_1.AnnotationWriter.write(annotation.toHtml(), annotationRequest.outputPath, annotationRequest.fileNamePrefix);
|
|
100
100
|
return annotation;
|
|
@@ -114,8 +114,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
114
114
|
escapeSeparatorString(instruction) {
|
|
115
115
|
return instruction.split(dsl_1.Separators.STRING).join('"');
|
|
116
116
|
}
|
|
117
|
-
buildInstruction(
|
|
118
|
-
return __awaiter(this,
|
|
117
|
+
buildInstruction(instructionString_1) {
|
|
118
|
+
return __awaiter(this, arguments, void 0, function* (instructionString, customElementJson = []) {
|
|
119
119
|
return {
|
|
120
120
|
customElements: yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson),
|
|
121
121
|
secretText: this.getAndResetSecretText(),
|
|
@@ -134,8 +134,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
134
134
|
return workspaceAIElementCollection.getByNames(names);
|
|
135
135
|
});
|
|
136
136
|
}
|
|
137
|
-
fluentCommandExecutor(
|
|
138
|
-
return __awaiter(this,
|
|
137
|
+
fluentCommandExecutor(instructionString_1) {
|
|
138
|
+
return __awaiter(this, arguments, void 0, function* (instructionString, context = { customElementsJson: [], aiElementNames: [] }) {
|
|
139
139
|
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
|
|
140
140
|
const instruction = yield this.buildInstruction(instructionString, [
|
|
141
141
|
...context.customElementsJson,
|
|
@@ -154,8 +154,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
154
154
|
}
|
|
155
155
|
});
|
|
156
156
|
}
|
|
157
|
-
getterExecutor(
|
|
158
|
-
return __awaiter(this,
|
|
157
|
+
getterExecutor(instruction_1) {
|
|
158
|
+
return __awaiter(this, arguments, void 0, function* (instruction, context = { customElementsJson: [], aiElementNames: [] }) {
|
|
159
159
|
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
|
|
160
160
|
const customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(context.customElementsJson);
|
|
161
161
|
const stringWithoutSeparators = this.escapeSeparatorString(instruction);
|
|
@@ -166,6 +166,100 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
166
166
|
]);
|
|
167
167
|
});
|
|
168
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* Takes a prompt that contains a question you want to be answered
|
|
171
|
+
* or the data you want to have extracted from your screen.
|
|
172
|
+
*
|
|
173
|
+
* The optional 'config' can be used to specifiy the JSON schema the
|
|
174
|
+
* returned object shall have (https://json-schema.org).
|
|
175
|
+
*
|
|
176
|
+
* See the following examples on how to use it:
|
|
177
|
+
*
|
|
178
|
+
* let isWidgetsNew =
|
|
179
|
+
* await aui.ask(
|
|
180
|
+
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
|
|
181
|
+
* {
|
|
182
|
+
* json_schema: {
|
|
183
|
+
* "type": "boolean"
|
|
184
|
+
* }
|
|
185
|
+
* });
|
|
186
|
+
*
|
|
187
|
+
* Output of console.log(isWidgetsNew): true
|
|
188
|
+
*
|
|
189
|
+
* let newClients =
|
|
190
|
+
* await aui.ask(
|
|
191
|
+
* "How many new clients?",
|
|
192
|
+
* {
|
|
193
|
+
* json_schema: {
|
|
194
|
+
* "type": "number"
|
|
195
|
+
* }
|
|
196
|
+
* });
|
|
197
|
+
*
|
|
198
|
+
* Output of console.log(newClients): 9123
|
|
199
|
+
*
|
|
200
|
+
* let userNames =
|
|
201
|
+
* await aui.ask(
|
|
202
|
+
* "Return a list with the users names.",
|
|
203
|
+
* {
|
|
204
|
+
* json_schema: {
|
|
205
|
+
* "type": "array",
|
|
206
|
+
* "items": {
|
|
207
|
+
* "type": "string"
|
|
208
|
+
* }
|
|
209
|
+
* }
|
|
210
|
+
* });
|
|
211
|
+
*
|
|
212
|
+
* Output of console.log(userNames):
|
|
213
|
+
* [
|
|
214
|
+
* 'Yiorgos Avraamu',
|
|
215
|
+
* 'Avram Tsarios',
|
|
216
|
+
* 'Quintin Ed',
|
|
217
|
+
* 'Enéas Kwadwo',
|
|
218
|
+
* 'Agapetus Tadeáš'
|
|
219
|
+
* ]
|
|
220
|
+
*
|
|
221
|
+
* let users =
|
|
222
|
+
* await aui.ask(
|
|
223
|
+
* "Extract the users from the table.",
|
|
224
|
+
* {
|
|
225
|
+
* json_schema: {
|
|
226
|
+
* "type": "array",
|
|
227
|
+
* "items": {
|
|
228
|
+
* "type": "object",
|
|
229
|
+
* "properties": {
|
|
230
|
+
* "name": {
|
|
231
|
+
* "type": "string"
|
|
232
|
+
* },
|
|
233
|
+
* "usage": {
|
|
234
|
+
* "type": "number"
|
|
235
|
+
* }
|
|
236
|
+
* },
|
|
237
|
+
* "additionalProperties": false,
|
|
238
|
+
* "required": ["name", "usage"]
|
|
239
|
+
* },
|
|
240
|
+
* },
|
|
241
|
+
* });
|
|
242
|
+
*
|
|
243
|
+
* Output of console.log(users):
|
|
244
|
+
* [
|
|
245
|
+
* { name: 'Yiorgos Avraamu', usage: 50 },
|
|
246
|
+
* { name: 'Avram Tarasios', usage: 10 },
|
|
247
|
+
* { name: 'Quintin Ed', usage: 74 },
|
|
248
|
+
* { name: 'Eneás Kwadwo', usage: 98 },
|
|
249
|
+
* { name: 'Agapetus Tadeáš', usage: 22 }
|
|
250
|
+
* ]
|
|
251
|
+
*
|
|
252
|
+
* @param {string} prompt - The question you want to be answered or
|
|
253
|
+
* the data you want to have extracted.
|
|
254
|
+
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
|
|
255
|
+
* @returns {any} - The answer as JSON specified in the config object.
|
|
256
|
+
*/
|
|
257
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
258
|
+
ask(prompt, config) {
|
|
259
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
260
|
+
return this.executionRuntime.predictVQA(prompt, config);
|
|
261
|
+
});
|
|
262
|
+
}
|
|
169
263
|
getAndResetSecretText() {
|
|
170
264
|
const { secretText } = this;
|
|
171
265
|
this.secretText = undefined;
|
|
@@ -190,6 +284,9 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
190
284
|
* @return {FluentFilters}
|
|
191
285
|
*/
|
|
192
286
|
typeIn(text, { isSecret = false, secretMask = '****' } = {}) {
|
|
287
|
+
if (text.length === 0) {
|
|
288
|
+
throw new yup_1.ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
|
|
289
|
+
}
|
|
193
290
|
if (isSecret) {
|
|
194
291
|
this.secretText = text;
|
|
195
292
|
return super.typeIn(secretMask);
|
|
@@ -215,6 +312,9 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
215
312
|
* @return {Exec}
|
|
216
313
|
*/
|
|
217
314
|
type(text, { isSecret = false, secretMask = '****' } = {}) {
|
|
315
|
+
if (text.length === 0) {
|
|
316
|
+
throw new yup_1.ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
|
|
317
|
+
}
|
|
218
318
|
if (isSecret) {
|
|
219
319
|
this.secretText = text;
|
|
220
320
|
return super.type(secretMask);
|
|
@@ -245,8 +345,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
245
345
|
*
|
|
246
346
|
* @param {number} times
|
|
247
347
|
*/
|
|
248
|
-
pressKeyNTimes(
|
|
249
|
-
return __awaiter(this,
|
|
348
|
+
pressKeyNTimes(key_1) {
|
|
349
|
+
return __awaiter(this, arguments, void 0, function* (key, times = 2) {
|
|
250
350
|
/* eslint-disable no-await-in-loop */
|
|
251
351
|
for (let i = 0; i < times; i += 1) {
|
|
252
352
|
yield this.pressKey(key).exec();
|
|
@@ -332,8 +432,8 @@ class UiControlClient extends dsl_1.ApiCommands {
|
|
|
332
432
|
* @param {number} maxTry - Number of maximum retries
|
|
333
433
|
* @param {number} waitTime - Time in milliseconds
|
|
334
434
|
*/
|
|
335
|
-
waitUntil(
|
|
336
|
-
return __awaiter(this,
|
|
435
|
+
waitUntil(AskUICommand_1) {
|
|
436
|
+
return __awaiter(this, arguments, void 0, function* (AskUICommand, maxTry = 5, waitTime = 2000) {
|
|
337
437
|
try {
|
|
338
438
|
yield AskUICommand.exec();
|
|
339
439
|
}
|
|
@@ -7,4 +7,4 @@ var UiControllerClientConnectionState;
|
|
|
7
7
|
UiControllerClientConnectionState[UiControllerClientConnectionState["CONNECTING"] = 1] = "CONNECTING";
|
|
8
8
|
UiControllerClientConnectionState[UiControllerClientConnectionState["CONNECTED"] = 2] = "CONNECTED";
|
|
9
9
|
UiControllerClientConnectionState[UiControllerClientConnectionState["ERROR"] = 3] = "ERROR";
|
|
10
|
-
})(UiControllerClientConnectionState
|
|
10
|
+
})(UiControllerClientConnectionState || (exports.UiControllerClientConnectionState = UiControllerClientConnectionState = {}));
|
|
@@ -3,6 +3,7 @@ import { ProxyAgentArgs } from '../shared/proxy-agent-args';
|
|
|
3
3
|
import { ModelCompositionBranch } from './model-composition-branch';
|
|
4
4
|
import { Reporter } from '../core/reporting';
|
|
5
5
|
import { Context } from './context';
|
|
6
|
+
import { RetryStrategy } from './retry-strategies/retry-strategy';
|
|
6
7
|
/**
|
|
7
8
|
* Context object to provide additional information about the context of (test) automation.
|
|
8
9
|
*
|
|
@@ -43,6 +44,9 @@ export interface ContextArgs {
|
|
|
43
44
|
* @property {(Context | undefined)} [context] - Optional. Context object to provide additional
|
|
44
45
|
* information about the context of (test) automation, e.g., to allow for optimizations based on
|
|
45
46
|
* the environment, e.g., CI/CD.
|
|
47
|
+
* @property {(RetryStrategy | undefined)} [retryStrategy] - Default: `new LinearRetryStrategy()`.
|
|
48
|
+
* Strategy for retrying failed requests to the inference server. This can help manage transient
|
|
49
|
+
* errors or network issues, improving the reliability of interactions with the server.
|
|
46
50
|
*/
|
|
47
51
|
export interface ClientArgs {
|
|
48
52
|
readonly uiControllerUrl?: string;
|
|
@@ -54,10 +58,12 @@ export interface ClientArgs {
|
|
|
54
58
|
readonly reporter?: Reporter | Reporter[] | undefined;
|
|
55
59
|
readonly context?: ContextArgs | undefined;
|
|
56
60
|
readonly inferenceServerApiVersion?: string;
|
|
61
|
+
readonly retryStrategy?: RetryStrategy;
|
|
57
62
|
}
|
|
58
63
|
export interface ClientArgsWithDefaults extends ClientArgs {
|
|
59
64
|
readonly uiControllerUrl: string;
|
|
60
65
|
readonly inferenceServerUrl: string;
|
|
61
66
|
readonly context: Context;
|
|
62
67
|
readonly inferenceServerApiVersion: string;
|
|
68
|
+
readonly retryStrategy?: RetryStrategy;
|
|
63
69
|
}
|
|
@@ -26,7 +26,8 @@ class UiControllerClient {
|
|
|
26
26
|
clearTimeout(this.timeout);
|
|
27
27
|
const response = JSON.parse(data.toString());
|
|
28
28
|
if (response.data.error) {
|
|
29
|
-
|
|
29
|
+
logger_1.logger.error(response.data.error);
|
|
30
|
+
this.currentReject(new ui_controller_client_error_1.UiControllerClientError(response.data.error));
|
|
30
31
|
this.clearResponse();
|
|
31
32
|
return;
|
|
32
33
|
}
|
|
@@ -12,7 +12,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
12
12
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.
|
|
15
|
+
exports.platform = platform;
|
|
16
|
+
exports.getBinaryFilePath = getBinaryFilePath;
|
|
17
|
+
exports.downloadServerBinaries = downloadServerBinaries;
|
|
16
18
|
const fs_1 = __importDefault(require("fs"));
|
|
17
19
|
const got_1 = __importDefault(require("got"));
|
|
18
20
|
const os_1 = __importDefault(require("os"));
|
|
@@ -42,7 +44,6 @@ function platform() {
|
|
|
42
44
|
}
|
|
43
45
|
throw new Error(`Platform "${pf}" is not supported.`);
|
|
44
46
|
}
|
|
45
|
-
exports.platform = platform;
|
|
46
47
|
function buildBinaryNotAvailableError(binaryVersion) {
|
|
47
48
|
return new Error(`There was an error during downloading and creating the askui UI Controller. You can try a fresh install.
|
|
48
49
|
If you specified a binary version it is possible that the askui UI Controller version "${binaryVersion}"
|
|
@@ -52,7 +53,6 @@ function buildBinaryNotAvailableError(binaryVersion) {
|
|
|
52
53
|
function getBinaryFilePath(version) {
|
|
53
54
|
return path_1.default.join((0, path_2.getPathToNodeModulesRoot)(), 'release', version, ...binarySubPathsByPlatform[platform()]);
|
|
54
55
|
}
|
|
55
|
-
exports.getBinaryFilePath = getBinaryFilePath;
|
|
56
56
|
function getBinaryDownloadUrl(binaryVersion) {
|
|
57
57
|
const baseUrl = `https://files.askui.com/releases/askui-ui-controller/${binaryVersion}`;
|
|
58
58
|
const arch = os_1.default.arch();
|
|
@@ -86,4 +86,3 @@ function downloadServerBinaries(binaryVersion, proxyAgent) {
|
|
|
86
86
|
return Promise.resolve();
|
|
87
87
|
});
|
|
88
88
|
}
|
|
89
|
-
exports.downloadServerBinaries = downloadServerBinaries;
|
|
@@ -12,7 +12,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
12
12
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.
|
|
15
|
+
exports.addScript = addScript;
|
|
16
|
+
exports.removeScript = removeScript;
|
|
16
17
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
17
18
|
const logger_1 = require("../logger");
|
|
18
19
|
function addScript(filePath, name, command) {
|
|
@@ -35,7 +36,6 @@ function addScript(filePath, name, command) {
|
|
|
35
36
|
}
|
|
36
37
|
});
|
|
37
38
|
}
|
|
38
|
-
exports.addScript = addScript;
|
|
39
39
|
function removeScript(filePath, name) {
|
|
40
40
|
return __awaiter(this, void 0, void 0, function* () {
|
|
41
41
|
try {
|
|
@@ -54,4 +54,3 @@ function removeScript(filePath, name) {
|
|
|
54
54
|
}
|
|
55
55
|
});
|
|
56
56
|
}
|
|
57
|
-
exports.removeScript = removeScript;
|
|
@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
12
12
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.init =
|
|
15
|
+
exports.init = init;
|
|
16
16
|
const inquirer_1 = __importDefault(require("inquirer"));
|
|
17
17
|
const commander_1 = require("commander");
|
|
18
18
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
@@ -85,4 +85,3 @@ function init(argv) {
|
|
|
85
85
|
}));
|
|
86
86
|
return program.parse(args);
|
|
87
87
|
}
|
|
88
|
-
exports.init = init;
|
package/dist/cjs/lib/logger.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
declare const logger: import("pino").Logger<"verbose">;
|
|
1
|
+
declare const logger: import("pino").Logger<"verbose", boolean>;
|
|
2
2
|
export { logger };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.createArgsWithDefaults = createArgsWithDefaults;
|
|
4
|
+
exports.createCliFlagsFromArgs = createCliFlagsFromArgs;
|
|
4
5
|
function createArgsWithDefaults(args) {
|
|
5
6
|
const defaults = {
|
|
6
7
|
actionDelayInMs: 1000,
|
|
@@ -14,7 +15,6 @@ function createArgsWithDefaults(args) {
|
|
|
14
15
|
};
|
|
15
16
|
return Object.assign(defaults, args);
|
|
16
17
|
}
|
|
17
|
-
exports.createArgsWithDefaults = createArgsWithDefaults;
|
|
18
18
|
function createCliFlagsFromArgs(args) {
|
|
19
19
|
return [
|
|
20
20
|
`-d ${args.display.toString()}`,
|
|
@@ -26,4 +26,3 @@ function createCliFlagsFromArgs(args) {
|
|
|
26
26
|
(args === null || args === void 0 ? void 0 : args.logFilePath) ? `--log-file ${args.logFilePath}` : '',
|
|
27
27
|
].filter((arg) => !!arg);
|
|
28
28
|
}
|
|
29
|
-
exports.createCliFlagsFromArgs = createCliFlagsFromArgs;
|
|
@@ -92,8 +92,8 @@ class UiControllerFacade {
|
|
|
92
92
|
output: (process === null || process === void 0 ? void 0 : process.env['LOG_LEVEL']) === 'verbose' ? 'dots' : 'silent',
|
|
93
93
|
port: args.port,
|
|
94
94
|
timeout: timeoutInMs,
|
|
95
|
-
}).then((
|
|
96
|
-
if (open) {
|
|
95
|
+
}).then((returnObject) => {
|
|
96
|
+
if (returnObject.open) {
|
|
97
97
|
logger_1.logger.info('The Control UI Server has been started.');
|
|
98
98
|
return resolve();
|
|
99
99
|
}
|
|
@@ -105,8 +105,8 @@ class UiControllerFacade {
|
|
|
105
105
|
}
|
|
106
106
|
});
|
|
107
107
|
}
|
|
108
|
-
getBinary(
|
|
109
|
-
return __awaiter(this,
|
|
108
|
+
getBinary(binaryVersion_1) {
|
|
109
|
+
return __awaiter(this, arguments, void 0, function* (binaryVersion, overWriteBinary = false, proxyAgent) {
|
|
110
110
|
if (!fs_extra_1.default.existsSync(this.binaryFilePath) || overWriteBinary) {
|
|
111
111
|
logger_1.logger.debug(`Currently, no binary of the UI Controller is available at "${this.binaryFilePath}"`);
|
|
112
112
|
yield (0, download_binaries_1.downloadServerBinaries)(binaryVersion, proxyAgent);
|
package/dist/cjs/main.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -28,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
28
32
|
});
|
|
29
33
|
};
|
|
30
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
31
|
-
exports.getSharpFactory =
|
|
35
|
+
exports.getSharpFactory = getSharpFactory;
|
|
32
36
|
let lazilyInitializedSharpFactory;
|
|
33
37
|
class SharpImportError extends Error {
|
|
34
38
|
}
|
|
@@ -53,4 +57,3 @@ function getSharpFactory() {
|
|
|
53
57
|
return lazilyInitializedSharpFactory;
|
|
54
58
|
});
|
|
55
59
|
}
|
|
56
|
-
exports.getSharpFactory = getSharpFactory;
|