askui 0.20.7 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/inference-response/inference-response.d.ts +5 -0
- package/dist/cjs/core/inference-response/model-type.d.ts +1 -1
- package/dist/cjs/core/model/custom-element.d.ts +5 -5
- package/dist/cjs/core/model/custom-element.js +2 -2
- package/dist/cjs/core/reporting/index.js +5 -1
- package/dist/cjs/core/reporting/snapshot-detail-level.d.ts +1 -1
- package/dist/cjs/core/reporting/snapshot-detail-level.js +1 -2
- package/dist/cjs/core/reporting/step-status-end.d.ts +1 -1
- package/dist/cjs/core/reporting/step-status.d.ts +1 -1
- package/dist/cjs/core/runner-protocol/index.js +5 -1
- package/dist/cjs/core/ui-control-commands/action.d.ts +1 -1
- package/dist/cjs/core/ui-control-commands/control-command-code.js +1 -1
- package/dist/cjs/core/ui-control-commands/input-event.js +1 -1
- package/dist/cjs/execution/dsl.d.ts +6 -6
- package/dist/cjs/execution/dsl.js +1 -1
- package/dist/cjs/execution/execution-runtime.d.ts +4 -2
- package/dist/cjs/execution/execution-runtime.js +14 -7
- package/dist/cjs/execution/index.d.ts +1 -0
- package/dist/cjs/execution/index.js +6 -1
- package/dist/cjs/execution/inference-client.d.ts +3 -0
- package/dist/cjs/execution/inference-client.js +33 -8
- package/dist/cjs/execution/misc.js +1 -2
- package/dist/cjs/execution/read-credentials.js +1 -2
- package/dist/cjs/execution/read-environment-credentials.js +1 -2
- package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/index.d.ts +4 -0
- package/dist/cjs/execution/retry-strategies/index.js +20 -0
- package/dist/cjs/execution/retry-strategies/linear-retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/linear-retry-strategy.js +25 -0
- package/dist/cjs/execution/retry-strategies/retry-strategy.d.ts +19 -0
- package/dist/cjs/execution/retry-strategies/retry-strategy.js +2 -0
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -5
- package/dist/cjs/execution/ui-control-client.d.ts +92 -3
- package/dist/cjs/execution/ui-control-client.js +115 -15
- package/dist/cjs/execution/ui-controller-client-connection-state.js +1 -1
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +6 -0
- package/dist/cjs/execution/ui-controller-client.js +2 -1
- package/dist/cjs/lib/download-binaries.d.ts +0 -1
- package/dist/cjs/lib/download-binaries.js +3 -4
- package/dist/cjs/lib/interactive_cli/add-remove-script-package-json.js +2 -3
- package/dist/cjs/lib/interactive_cli/cli.js +1 -2
- package/dist/cjs/lib/logger.d.ts +1 -1
- package/dist/cjs/lib/ui-controller-args.js +2 -3
- package/dist/cjs/lib/ui-controller-facade.js +4 -4
- package/dist/cjs/main.js +5 -1
- package/dist/cjs/shared/log-levels.js +1 -1
- package/dist/cjs/shared/proxy-agent-args.d.ts +0 -1
- package/dist/cjs/utils/analytics/installation-timestamp-create-error.d.ts +0 -1
- package/dist/cjs/utils/analytics/installation-timestamp-get-error.d.ts +0 -1
- package/dist/cjs/utils/base_64_image/sharp.js +6 -3
- package/dist/cjs/utils/http/custom-errors/index.js +2 -2
- package/dist/cjs/utils/http/http-client-got.d.ts +0 -1
- package/dist/cjs/utils/http/http-client-got.js +2 -2
- package/dist/cjs/utils/path.js +1 -2
- package/dist/cjs/utils/proxy/proxy-builder.js +7 -3
- package/dist/cjs/utils/transformations.js +3 -4
- package/dist/esm/core/inference-response/inference-response.d.ts +5 -0
- package/dist/esm/core/inference-response/model-type.d.ts +1 -1
- package/dist/esm/core/model/custom-element.d.ts +5 -5
- package/dist/esm/core/model/custom-element.js +2 -2
- package/dist/esm/core/reporting/snapshot-detail-level.d.ts +1 -1
- package/dist/esm/core/reporting/step-status-end.d.ts +1 -1
- package/dist/esm/core/reporting/step-status.d.ts +1 -1
- package/dist/esm/core/ui-control-commands/action.d.ts +1 -1
- package/dist/esm/execution/dsl.d.ts +6 -6
- package/dist/esm/execution/execution-runtime.d.ts +4 -2
- package/dist/esm/execution/execution-runtime.js +14 -7
- package/dist/esm/execution/index.d.ts +1 -0
- package/dist/esm/execution/index.js +1 -0
- package/dist/esm/execution/inference-client.d.ts +3 -0
- package/dist/esm/execution/inference-client.js +33 -8
- package/dist/esm/execution/retry-strategies/exponential-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/exponential-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/fixed-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/fixed-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/index.d.ts +4 -0
- package/dist/esm/execution/retry-strategies/index.js +4 -0
- package/dist/esm/execution/retry-strategies/linear-retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/linear-retry-strategy.js +21 -0
- package/dist/esm/execution/retry-strategies/retry-strategy.d.ts +19 -0
- package/dist/esm/execution/retry-strategies/retry-strategy.js +1 -0
- package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -5
- package/dist/esm/execution/ui-control-client.d.ts +92 -3
- package/dist/esm/execution/ui-control-client.js +115 -15
- package/dist/esm/execution/ui-controller-client-interface.d.ts +6 -0
- package/dist/esm/execution/ui-controller-client.js +2 -1
- package/dist/esm/lib/download-binaries.d.ts +0 -1
- package/dist/esm/lib/logger.d.ts +1 -1
- package/dist/esm/lib/ui-controller-facade.js +4 -4
- package/dist/esm/shared/proxy-agent-args.d.ts +0 -1
- package/dist/esm/utils/analytics/installation-timestamp-create-error.d.ts +0 -1
- package/dist/esm/utils/analytics/installation-timestamp-get-error.d.ts +0 -1
- package/dist/esm/utils/http/http-client-got.d.ts +0 -1
- package/dist/esm/utils/http/http-client-got.js +2 -2
- package/dist/esm/utils/transformations.js +2 -2
- package/package.json +28 -28
|
@@ -16,10 +16,11 @@ import { envProxyAgents } from '../utils/proxy/proxy-builder';
|
|
|
16
16
|
import { ExecutionRuntime } from './execution-runtime';
|
|
17
17
|
import { StepReporter } from '../core/reporting';
|
|
18
18
|
import { readCredentials } from './read-credentials';
|
|
19
|
+
import { LinearRetryStrategy } from './retry-strategies/linear-retry-strategy';
|
|
19
20
|
export class UiControlClientDependencyBuilder {
|
|
20
21
|
static buildHttpClient(clientArgs) {
|
|
21
|
-
var _a;
|
|
22
22
|
return __awaiter(this, void 0, void 0, function* () {
|
|
23
|
+
var _a;
|
|
23
24
|
const analytics = new Analytics();
|
|
24
25
|
const analyticsHeaders = yield analytics.getAnalyticsHeaders(clientArgs.context);
|
|
25
26
|
const analyticsCookies = yield analytics.getAnalyticsCookies();
|
|
@@ -27,8 +28,8 @@ export class UiControlClientDependencyBuilder {
|
|
|
27
28
|
});
|
|
28
29
|
}
|
|
29
30
|
static buildInferenceClient(clientArgs) {
|
|
30
|
-
var _a;
|
|
31
31
|
return __awaiter(this, void 0, void 0, function* () {
|
|
32
|
+
var _a;
|
|
32
33
|
const httpClient = yield UiControlClientDependencyBuilder.buildHttpClient(clientArgs);
|
|
33
34
|
return new InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition, clientArgs.inferenceServerApiVersion);
|
|
34
35
|
});
|
|
@@ -37,22 +38,22 @@ export class UiControlClientDependencyBuilder {
|
|
|
37
38
|
return new UiControllerClient(clientArgs.uiControllerUrl);
|
|
38
39
|
}
|
|
39
40
|
static build(clientArgs) {
|
|
40
|
-
var _a;
|
|
41
41
|
return __awaiter(this, void 0, void 0, function* () {
|
|
42
|
+
var _a, _b;
|
|
42
43
|
const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs);
|
|
43
44
|
const inferenceClient = yield UiControlClientDependencyBuilder.buildInferenceClient(clientArgs);
|
|
44
45
|
const stepReporter = new StepReporter(clientArgs.reporter);
|
|
45
46
|
const workspaceId = (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId;
|
|
46
47
|
return {
|
|
47
|
-
executionRuntime: new ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter),
|
|
48
|
+
executionRuntime: new ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter, (_b = clientArgs.retryStrategy) !== null && _b !== void 0 ? _b : new LinearRetryStrategy()),
|
|
48
49
|
stepReporter,
|
|
49
50
|
workspaceId,
|
|
50
51
|
};
|
|
51
52
|
});
|
|
52
53
|
}
|
|
53
54
|
static getClientArgsWithDefaults(clientArgs) {
|
|
54
|
-
var _a, _b, _c, _d, _e, _f;
|
|
55
55
|
return __awaiter(this, void 0, void 0, function* () {
|
|
56
|
+
var _a, _b, _c, _d, _e, _f;
|
|
56
57
|
return Object.assign(Object.assign({}, clientArgs), { context: {
|
|
57
58
|
isCi: (_b = (_a = clientArgs.context) === null || _a === void 0 ? void 0 : _a.isCi) !== null && _b !== void 0 ? _b : isCI,
|
|
58
59
|
}, credentials: readCredentials(clientArgs), inferenceServerApiVersion: (_c = clientArgs.inferenceServerApiVersion) !== null && _c !== void 0 ? _c : 'v3', inferenceServerUrl: (_d = clientArgs.inferenceServerUrl) !== null && _d !== void 0 ? _d : 'https://inference.askui.com', proxyAgents: (_e = clientArgs.proxyAgents) !== null && _e !== void 0 ? _e : (yield envProxyAgents()), uiControllerUrl: (_f = clientArgs.uiControllerUrl) !== null && _f !== void 0 ? _f : 'http://127.0.0.1:6769' });
|
|
@@ -4,9 +4,9 @@ import { Annotation } from '../core/annotation/annotation';
|
|
|
4
4
|
import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
|
|
5
5
|
import { DetectedElement } from '../core/model/annotation-result/detected-element';
|
|
6
6
|
import { ClientArgs } from './ui-controller-client-interface';
|
|
7
|
-
export
|
|
8
|
-
export
|
|
9
|
-
export
|
|
7
|
+
export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
|
|
8
|
+
export type TextMatchingOption = 'similar' | 'exact' | 'regex';
|
|
9
|
+
export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
|
|
10
10
|
export interface ElementExistsQueryText {
|
|
11
11
|
value: string;
|
|
12
12
|
matching?: TextMatchingOption;
|
|
@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
|
|
|
59
59
|
private getAIElementsByNames;
|
|
60
60
|
fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
|
|
61
61
|
getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
|
|
62
|
+
/**
|
|
63
|
+
* Takes a prompt that contains a question you want to be answered
|
|
64
|
+
* or the data you want to have extracted from your screen.
|
|
65
|
+
*
|
|
66
|
+
* The optional 'config' can be used to specifiy the JSON schema the
|
|
67
|
+
* returned object shall have (https://json-schema.org).
|
|
68
|
+
*
|
|
69
|
+
* See the following examples on how to use it:
|
|
70
|
+
*
|
|
71
|
+
* let isWidgetsNew =
|
|
72
|
+
* await aui.ask(
|
|
73
|
+
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
|
|
74
|
+
* {
|
|
75
|
+
* json_schema: {
|
|
76
|
+
* "type": "boolean"
|
|
77
|
+
* }
|
|
78
|
+
* });
|
|
79
|
+
*
|
|
80
|
+
* Output of console.log(isWidgetsNew): true
|
|
81
|
+
*
|
|
82
|
+
* let newClients =
|
|
83
|
+
* await aui.ask(
|
|
84
|
+
* "How many new clients?",
|
|
85
|
+
* {
|
|
86
|
+
* json_schema: {
|
|
87
|
+
* "type": "number"
|
|
88
|
+
* }
|
|
89
|
+
* });
|
|
90
|
+
*
|
|
91
|
+
* Output of console.log(newClients): 9123
|
|
92
|
+
*
|
|
93
|
+
* let userNames =
|
|
94
|
+
* await aui.ask(
|
|
95
|
+
* "Return a list with the users names.",
|
|
96
|
+
* {
|
|
97
|
+
* json_schema: {
|
|
98
|
+
* "type": "array",
|
|
99
|
+
* "items": {
|
|
100
|
+
* "type": "string"
|
|
101
|
+
* }
|
|
102
|
+
* }
|
|
103
|
+
* });
|
|
104
|
+
*
|
|
105
|
+
* Output of console.log(userNames):
|
|
106
|
+
* [
|
|
107
|
+
* 'Yiorgos Avraamu',
|
|
108
|
+
* 'Avram Tsarios',
|
|
109
|
+
* 'Quintin Ed',
|
|
110
|
+
* 'Enéas Kwadwo',
|
|
111
|
+
* 'Agapetus Tadeáš'
|
|
112
|
+
* ]
|
|
113
|
+
*
|
|
114
|
+
* let users =
|
|
115
|
+
* await aui.ask(
|
|
116
|
+
* "Extract the users from the table.",
|
|
117
|
+
* {
|
|
118
|
+
* json_schema: {
|
|
119
|
+
* "type": "array",
|
|
120
|
+
* "items": {
|
|
121
|
+
* "type": "object",
|
|
122
|
+
* "properties": {
|
|
123
|
+
* "name": {
|
|
124
|
+
* "type": "string"
|
|
125
|
+
* },
|
|
126
|
+
* "usage": {
|
|
127
|
+
* "type": "number"
|
|
128
|
+
* }
|
|
129
|
+
* },
|
|
130
|
+
* "additionalProperties": false,
|
|
131
|
+
* "required": ["name", "usage"]
|
|
132
|
+
* },
|
|
133
|
+
* },
|
|
134
|
+
* });
|
|
135
|
+
*
|
|
136
|
+
* Output of console.log(users):
|
|
137
|
+
* [
|
|
138
|
+
* { name: 'Yiorgos Avraamu', usage: 50 },
|
|
139
|
+
* { name: 'Avram Tarasios', usage: 10 },
|
|
140
|
+
* { name: 'Quintin Ed', usage: 74 },
|
|
141
|
+
* { name: 'Eneás Kwadwo', usage: 98 },
|
|
142
|
+
* { name: 'Agapetus Tadeáš', usage: 22 }
|
|
143
|
+
* ]
|
|
144
|
+
*
|
|
145
|
+
* @param {string} prompt - The question you want to be answered or
|
|
146
|
+
* the data you want to have extracted.
|
|
147
|
+
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
|
|
148
|
+
* @returns {any} - The answer as JSON specified in the config object.
|
|
149
|
+
*/
|
|
150
|
+
ask(prompt: string, config?: object): Promise<any>;
|
|
62
151
|
private secretText;
|
|
63
152
|
private getAndResetSecretText;
|
|
64
153
|
/**
|
|
@@ -22,8 +22,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
22
22
|
this.stepReporter = stepReporter;
|
|
23
23
|
this.secretText = undefined;
|
|
24
24
|
}
|
|
25
|
-
static build(
|
|
26
|
-
return __awaiter(this,
|
|
25
|
+
static build() {
|
|
26
|
+
return __awaiter(this, arguments, void 0, function* (clientArgs = {}) {
|
|
27
27
|
const builder = UiControlClientDependencyBuilder;
|
|
28
28
|
const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
|
|
29
29
|
const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
|
|
@@ -72,8 +72,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
72
72
|
|| (this.stepReporter.config.withDetectedElements === 'always');
|
|
73
73
|
}
|
|
74
74
|
afterCommandExecution(instruction, error) {
|
|
75
|
-
var _a;
|
|
76
75
|
return __awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
var _a;
|
|
77
77
|
const createdAt = new Date();
|
|
78
78
|
let annotation;
|
|
79
79
|
let screenshot;
|
|
@@ -90,8 +90,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
90
90
|
}, error);
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
|
-
annotate(
|
|
94
|
-
return __awaiter(this,
|
|
93
|
+
annotate() {
|
|
94
|
+
return __awaiter(this, arguments, void 0, function* (annotationRequest = {}) {
|
|
95
95
|
const annotation = yield this.executionRuntime.annotateImage(annotationRequest.imagePath, annotationRequest.customElements, annotationRequest.elements);
|
|
96
96
|
AnnotationWriter.write(annotation.toHtml(), annotationRequest.outputPath, annotationRequest.fileNamePrefix);
|
|
97
97
|
return annotation;
|
|
@@ -111,8 +111,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
111
111
|
escapeSeparatorString(instruction) {
|
|
112
112
|
return instruction.split(Separators.STRING).join('"');
|
|
113
113
|
}
|
|
114
|
-
buildInstruction(
|
|
115
|
-
return __awaiter(this,
|
|
114
|
+
buildInstruction(instructionString_1) {
|
|
115
|
+
return __awaiter(this, arguments, void 0, function* (instructionString, customElementJson = []) {
|
|
116
116
|
return {
|
|
117
117
|
customElements: yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson),
|
|
118
118
|
secretText: this.getAndResetSecretText(),
|
|
@@ -131,8 +131,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
131
131
|
return workspaceAIElementCollection.getByNames(names);
|
|
132
132
|
});
|
|
133
133
|
}
|
|
134
|
-
fluentCommandExecutor(
|
|
135
|
-
return __awaiter(this,
|
|
134
|
+
fluentCommandExecutor(instructionString_1) {
|
|
135
|
+
return __awaiter(this, arguments, void 0, function* (instructionString, context = { customElementsJson: [], aiElementNames: [] }) {
|
|
136
136
|
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
|
|
137
137
|
const instruction = yield this.buildInstruction(instructionString, [
|
|
138
138
|
...context.customElementsJson,
|
|
@@ -151,8 +151,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
151
151
|
}
|
|
152
152
|
});
|
|
153
153
|
}
|
|
154
|
-
getterExecutor(
|
|
155
|
-
return __awaiter(this,
|
|
154
|
+
getterExecutor(instruction_1) {
|
|
155
|
+
return __awaiter(this, arguments, void 0, function* (instruction, context = { customElementsJson: [], aiElementNames: [] }) {
|
|
156
156
|
const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
|
|
157
157
|
const customElements = yield CustomElement.fromJsonListWithImagePathOrImage(context.customElementsJson);
|
|
158
158
|
const stringWithoutSeparators = this.escapeSeparatorString(instruction);
|
|
@@ -163,6 +163,100 @@ export class UiControlClient extends ApiCommands {
|
|
|
163
163
|
]);
|
|
164
164
|
});
|
|
165
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* Takes a prompt that contains a question you want to be answered
|
|
168
|
+
* or the data you want to have extracted from your screen.
|
|
169
|
+
*
|
|
170
|
+
* The optional 'config' can be used to specifiy the JSON schema the
|
|
171
|
+
* returned object shall have (https://json-schema.org).
|
|
172
|
+
*
|
|
173
|
+
* See the following examples on how to use it:
|
|
174
|
+
*
|
|
175
|
+
* let isWidgetsNew =
|
|
176
|
+
* await aui.ask(
|
|
177
|
+
* "Does the sidebar element 'Widgets' have a 'NEW' tag?",
|
|
178
|
+
* {
|
|
179
|
+
* json_schema: {
|
|
180
|
+
* "type": "boolean"
|
|
181
|
+
* }
|
|
182
|
+
* });
|
|
183
|
+
*
|
|
184
|
+
* Output of console.log(isWidgetsNew): true
|
|
185
|
+
*
|
|
186
|
+
* let newClients =
|
|
187
|
+
* await aui.ask(
|
|
188
|
+
* "How many new clients?",
|
|
189
|
+
* {
|
|
190
|
+
* json_schema: {
|
|
191
|
+
* "type": "number"
|
|
192
|
+
* }
|
|
193
|
+
* });
|
|
194
|
+
*
|
|
195
|
+
* Output of console.log(newClients): 9123
|
|
196
|
+
*
|
|
197
|
+
* let userNames =
|
|
198
|
+
* await aui.ask(
|
|
199
|
+
* "Return a list with the users names.",
|
|
200
|
+
* {
|
|
201
|
+
* json_schema: {
|
|
202
|
+
* "type": "array",
|
|
203
|
+
* "items": {
|
|
204
|
+
* "type": "string"
|
|
205
|
+
* }
|
|
206
|
+
* }
|
|
207
|
+
* });
|
|
208
|
+
*
|
|
209
|
+
* Output of console.log(userNames):
|
|
210
|
+
* [
|
|
211
|
+
* 'Yiorgos Avraamu',
|
|
212
|
+
* 'Avram Tsarios',
|
|
213
|
+
* 'Quintin Ed',
|
|
214
|
+
* 'Enéas Kwadwo',
|
|
215
|
+
* 'Agapetus Tadeáš'
|
|
216
|
+
* ]
|
|
217
|
+
*
|
|
218
|
+
* let users =
|
|
219
|
+
* await aui.ask(
|
|
220
|
+
* "Extract the users from the table.",
|
|
221
|
+
* {
|
|
222
|
+
* json_schema: {
|
|
223
|
+
* "type": "array",
|
|
224
|
+
* "items": {
|
|
225
|
+
* "type": "object",
|
|
226
|
+
* "properties": {
|
|
227
|
+
* "name": {
|
|
228
|
+
* "type": "string"
|
|
229
|
+
* },
|
|
230
|
+
* "usage": {
|
|
231
|
+
* "type": "number"
|
|
232
|
+
* }
|
|
233
|
+
* },
|
|
234
|
+
* "additionalProperties": false,
|
|
235
|
+
* "required": ["name", "usage"]
|
|
236
|
+
* },
|
|
237
|
+
* },
|
|
238
|
+
* });
|
|
239
|
+
*
|
|
240
|
+
* Output of console.log(users):
|
|
241
|
+
* [
|
|
242
|
+
* { name: 'Yiorgos Avraamu', usage: 50 },
|
|
243
|
+
* { name: 'Avram Tarasios', usage: 10 },
|
|
244
|
+
* { name: 'Quintin Ed', usage: 74 },
|
|
245
|
+
* { name: 'Eneás Kwadwo', usage: 98 },
|
|
246
|
+
* { name: 'Agapetus Tadeáš', usage: 22 }
|
|
247
|
+
* ]
|
|
248
|
+
*
|
|
249
|
+
* @param {string} prompt - The question you want to be answered or
|
|
250
|
+
* the data you want to have extracted.
|
|
251
|
+
* @param {Object} config - object that specifies the return json: {json_schema: {...}}.
|
|
252
|
+
* @returns {any} - The answer as JSON specified in the config object.
|
|
253
|
+
*/
|
|
254
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
255
|
+
ask(prompt, config) {
|
|
256
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
257
|
+
return this.executionRuntime.predictVQA(prompt, config);
|
|
258
|
+
});
|
|
259
|
+
}
|
|
166
260
|
getAndResetSecretText() {
|
|
167
261
|
const { secretText } = this;
|
|
168
262
|
this.secretText = undefined;
|
|
@@ -187,6 +281,9 @@ export class UiControlClient extends ApiCommands {
|
|
|
187
281
|
* @return {FluentFilters}
|
|
188
282
|
*/
|
|
189
283
|
typeIn(text, { isSecret = false, secretMask = '****' } = {}) {
|
|
284
|
+
if (text.length === 0) {
|
|
285
|
+
throw new ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
|
|
286
|
+
}
|
|
190
287
|
if (isSecret) {
|
|
191
288
|
this.secretText = text;
|
|
192
289
|
return super.typeIn(secretMask);
|
|
@@ -212,6 +309,9 @@ export class UiControlClient extends ApiCommands {
|
|
|
212
309
|
* @return {Exec}
|
|
213
310
|
*/
|
|
214
311
|
type(text, { isSecret = false, secretMask = '****' } = {}) {
|
|
312
|
+
if (text.length === 0) {
|
|
313
|
+
throw new ValidationError('Empty string is not allowed. Typing of an empty string was rejected.');
|
|
314
|
+
}
|
|
215
315
|
if (isSecret) {
|
|
216
316
|
this.secretText = text;
|
|
217
317
|
return super.type(secretMask);
|
|
@@ -242,8 +342,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
242
342
|
*
|
|
243
343
|
* @param {number} times
|
|
244
344
|
*/
|
|
245
|
-
pressKeyNTimes(
|
|
246
|
-
return __awaiter(this,
|
|
345
|
+
pressKeyNTimes(key_1) {
|
|
346
|
+
return __awaiter(this, arguments, void 0, function* (key, times = 2) {
|
|
247
347
|
/* eslint-disable no-await-in-loop */
|
|
248
348
|
for (let i = 0; i < times; i += 1) {
|
|
249
349
|
yield this.pressKey(key).exec();
|
|
@@ -329,8 +429,8 @@ export class UiControlClient extends ApiCommands {
|
|
|
329
429
|
* @param {number} maxTry - Number of maximum retries
|
|
330
430
|
* @param {number} waitTime - Time in milliseconds
|
|
331
431
|
*/
|
|
332
|
-
waitUntil(
|
|
333
|
-
return __awaiter(this,
|
|
432
|
+
waitUntil(AskUICommand_1) {
|
|
433
|
+
return __awaiter(this, arguments, void 0, function* (AskUICommand, maxTry = 5, waitTime = 2000) {
|
|
334
434
|
try {
|
|
335
435
|
yield AskUICommand.exec();
|
|
336
436
|
}
|
|
@@ -3,6 +3,7 @@ import { ProxyAgentArgs } from '../shared/proxy-agent-args';
|
|
|
3
3
|
import { ModelCompositionBranch } from './model-composition-branch';
|
|
4
4
|
import { Reporter } from '../core/reporting';
|
|
5
5
|
import { Context } from './context';
|
|
6
|
+
import { RetryStrategy } from './retry-strategies/retry-strategy';
|
|
6
7
|
/**
|
|
7
8
|
* Context object to provide additional information about the context of (test) automation.
|
|
8
9
|
*
|
|
@@ -43,6 +44,9 @@ export interface ContextArgs {
|
|
|
43
44
|
* @property {(Context | undefined)} [context] - Optional. Context object to provide additional
|
|
44
45
|
* information about the context of (test) automation, e.g., to allow for optimizations based on
|
|
45
46
|
* the environment, e.g., CI/CD.
|
|
47
|
+
* @property {(RetryStrategy | undefined)} [retryStrategy] - Default: `new LinearRetryStrategy()`.
|
|
48
|
+
* Strategy for retrying failed requests to the inference server. This can help manage transient
|
|
49
|
+
* errors or network issues, improving the reliability of interactions with the server.
|
|
46
50
|
*/
|
|
47
51
|
export interface ClientArgs {
|
|
48
52
|
readonly uiControllerUrl?: string;
|
|
@@ -54,10 +58,12 @@ export interface ClientArgs {
|
|
|
54
58
|
readonly reporter?: Reporter | Reporter[] | undefined;
|
|
55
59
|
readonly context?: ContextArgs | undefined;
|
|
56
60
|
readonly inferenceServerApiVersion?: string;
|
|
61
|
+
readonly retryStrategy?: RetryStrategy;
|
|
57
62
|
}
|
|
58
63
|
export interface ClientArgsWithDefaults extends ClientArgs {
|
|
59
64
|
readonly uiControllerUrl: string;
|
|
60
65
|
readonly inferenceServerUrl: string;
|
|
61
66
|
readonly context: Context;
|
|
62
67
|
readonly inferenceServerApiVersion: string;
|
|
68
|
+
readonly retryStrategy?: RetryStrategy;
|
|
63
69
|
}
|
|
@@ -20,7 +20,8 @@ export class UiControllerClient {
|
|
|
20
20
|
clearTimeout(this.timeout);
|
|
21
21
|
const response = JSON.parse(data.toString());
|
|
22
22
|
if (response.data.error) {
|
|
23
|
-
|
|
23
|
+
logger.error(response.data.error);
|
|
24
|
+
this.currentReject(new UiControllerClientError(response.data.error));
|
|
24
25
|
this.clearResponse();
|
|
25
26
|
return;
|
|
26
27
|
}
|
package/dist/esm/lib/logger.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
declare const logger: import("pino").Logger<"verbose">;
|
|
1
|
+
declare const logger: import("pino").Logger<"verbose", boolean>;
|
|
2
2
|
export { logger };
|
|
@@ -86,8 +86,8 @@ export class UiControllerFacade {
|
|
|
86
86
|
output: (process === null || process === void 0 ? void 0 : process.env['LOG_LEVEL']) === 'verbose' ? 'dots' : 'silent',
|
|
87
87
|
port: args.port,
|
|
88
88
|
timeout: timeoutInMs,
|
|
89
|
-
}).then((
|
|
90
|
-
if (open) {
|
|
89
|
+
}).then((returnObject) => {
|
|
90
|
+
if (returnObject.open) {
|
|
91
91
|
logger.info('The Control UI Server has been started.');
|
|
92
92
|
return resolve();
|
|
93
93
|
}
|
|
@@ -99,8 +99,8 @@ export class UiControllerFacade {
|
|
|
99
99
|
}
|
|
100
100
|
});
|
|
101
101
|
}
|
|
102
|
-
getBinary(
|
|
103
|
-
return __awaiter(this,
|
|
102
|
+
getBinary(binaryVersion_1) {
|
|
103
|
+
return __awaiter(this, arguments, void 0, function* (binaryVersion, overWriteBinary = false, proxyAgent) {
|
|
104
104
|
if (!fs.existsSync(this.binaryFilePath) || overWriteBinary) {
|
|
105
105
|
logger.debug(`Currently, no binary of the UI Controller is available at "${this.binaryFilePath}"`);
|
|
106
106
|
yield downloadServerBinaries(binaryVersion, proxyAgent);
|
|
@@ -127,8 +127,8 @@ export class HttpClientGot {
|
|
|
127
127
|
return { body, headers };
|
|
128
128
|
});
|
|
129
129
|
}
|
|
130
|
-
get(
|
|
131
|
-
return __awaiter(this,
|
|
130
|
+
get(url_1) {
|
|
131
|
+
return __awaiter(this, arguments, void 0, function* (url, options = { responseType: 'json' }) {
|
|
132
132
|
const response = yield this.askuiGot.get(url, this.injectHeadersAndCookies(url, options));
|
|
133
133
|
return response.body;
|
|
134
134
|
});
|
|
@@ -20,8 +20,8 @@ import { ImageResizingError } from './image-resize-errors';
|
|
|
20
20
|
* @returns {Promise<ResizedImage>} ResiziedImage Interface,
|
|
21
21
|
* the true pixel values = controlui-api response * resizeRatio.
|
|
22
22
|
*/
|
|
23
|
-
export function resizeBase64ImageWithSameRatio(
|
|
24
|
-
return __awaiter(this,
|
|
23
|
+
export function resizeBase64ImageWithSameRatio(base64ImageString_1) {
|
|
24
|
+
return __awaiter(this, arguments, void 0, function* (base64ImageString, maxEdge = 1400) {
|
|
25
25
|
logger.debug('Image resizing');
|
|
26
26
|
try {
|
|
27
27
|
const image = yield Base64Image.fromString(base64ImageString);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "askui",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.9",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
|
|
6
6
|
"description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",
|
|
@@ -52,40 +52,40 @@
|
|
|
52
52
|
],
|
|
53
53
|
"dependencies": {
|
|
54
54
|
"chalk": "4.1.1",
|
|
55
|
-
"commander": "
|
|
55
|
+
"commander": "12.1.0",
|
|
56
56
|
"fkill": "7.2.1",
|
|
57
|
-
"fs-extra": "
|
|
57
|
+
"fs-extra": "11.2.0",
|
|
58
58
|
"got": "11.8.6",
|
|
59
59
|
"inquirer": "8.2.5",
|
|
60
60
|
"is-ci": "3.0.1",
|
|
61
|
-
"jsdom": "
|
|
61
|
+
"jsdom": "25.0.1",
|
|
62
62
|
"listr": "0.14.3",
|
|
63
63
|
"lodash": "^4.17.21",
|
|
64
64
|
"node-machine-id": "1.1.12",
|
|
65
65
|
"nunjucks": "3.2.4",
|
|
66
|
-
"pino": "
|
|
67
|
-
"pino-pretty": "
|
|
68
|
-
"tough-cookie": "
|
|
66
|
+
"pino": "9.4.0",
|
|
67
|
+
"pino-pretty": "11.2.2",
|
|
68
|
+
"tough-cookie": "5.0.0",
|
|
69
69
|
"url-join": "4.0.1",
|
|
70
|
-
"wait-port": "
|
|
71
|
-
"ws": "
|
|
72
|
-
"yup": "
|
|
70
|
+
"wait-port": "1.1.0",
|
|
71
|
+
"ws": "8.18.0",
|
|
72
|
+
"yup": "1.4.0"
|
|
73
73
|
},
|
|
74
74
|
"devDependencies": {
|
|
75
75
|
"@release-it/bumper": "6.0.1",
|
|
76
|
-
"@release-it/conventional-changelog": "8.0.
|
|
77
|
-
"@types/fs-extra": "
|
|
78
|
-
"@types/inquirer": "9.0.
|
|
76
|
+
"@release-it/conventional-changelog": "8.0.2",
|
|
77
|
+
"@types/fs-extra": "11.0.4",
|
|
78
|
+
"@types/inquirer": "9.0.7",
|
|
79
79
|
"@types/is-ci": "^3.0.4",
|
|
80
|
-
"@types/jest": "29.5.
|
|
81
|
-
"@types/jsdom": "
|
|
80
|
+
"@types/jest": "29.5.13",
|
|
81
|
+
"@types/jsdom": "21.1.7",
|
|
82
82
|
"@types/listr": "0.14.9",
|
|
83
|
-
"@types/
|
|
84
|
-
"@types/
|
|
85
|
-
"@types/
|
|
86
|
-
"@types/url-join": "4.0.
|
|
87
|
-
"@types/webrtc": "0.0.
|
|
88
|
-
"@types/ws": "
|
|
83
|
+
"@types/lodash": "^4.17.7",
|
|
84
|
+
"@types/node": "22.5.5",
|
|
85
|
+
"@types/nunjucks": "3.2.6",
|
|
86
|
+
"@types/url-join": "4.0.3",
|
|
87
|
+
"@types/webrtc": "0.0.43",
|
|
88
|
+
"@types/ws": "8.5.12",
|
|
89
89
|
"@typescript-eslint/eslint-plugin": "5.10.0",
|
|
90
90
|
"@typescript-eslint/parser": "5.10.0",
|
|
91
91
|
"eslint": "8.7.0",
|
|
@@ -94,19 +94,19 @@
|
|
|
94
94
|
"eslint-plugin-import": "2.25.4",
|
|
95
95
|
"hpagent": "^1.2.0",
|
|
96
96
|
"jest": "29.7.0",
|
|
97
|
-
"jest-extended": "^4.0.
|
|
98
|
-
"lint-staged": "15.2.
|
|
97
|
+
"jest-extended": "^4.0.2",
|
|
98
|
+
"lint-staged": "15.2.10",
|
|
99
99
|
"proxy": "^1.0.2",
|
|
100
|
-
"proxy-agent": "^6.3.
|
|
101
|
-
"release-it": "17.0
|
|
100
|
+
"proxy-agent": "^6.3.0",
|
|
101
|
+
"release-it": "17.6.0",
|
|
102
102
|
"shx": "0.3.4",
|
|
103
|
-
"ts-jest": "29.
|
|
104
|
-
"typescript": "
|
|
103
|
+
"ts-jest": "29.2.5",
|
|
104
|
+
"typescript": "5.6.2"
|
|
105
105
|
},
|
|
106
106
|
"lint-staged": {
|
|
107
107
|
"./**/*.{js,ts}": "eslint --cache --fix --max-warnings 0"
|
|
108
108
|
},
|
|
109
109
|
"optionalDependencies": {
|
|
110
|
-
"sharp": "^0.
|
|
110
|
+
"sharp": "^0.33.5"
|
|
111
111
|
}
|
|
112
112
|
}
|