askui 0.21.2 → 0.23.0

package/dist/cjs/execution/ui-control-client.d.ts

@@ -32,6 +32,7 @@ export declare class UiControlClient extends ApiCommands {
     private workspaceId;
     private executionRuntime;
     private stepReporter;
+    private aiElementArgs;
     private constructor();
     static build(clientArgs?: ClientArgs): Promise<UiControlClient>;
     /**
@@ -243,12 +244,6 @@ export declare class UiControlClient extends ApiCommands {
      * @param {string} label - The textfields label.
      */
     clickTextfieldNearestTo(label: string): Promise<void>;
-    /**
-     * Clicks an icon based on a description.
-     *
-     * @param {string} description
-     */
-    clickIcon(description: string): Promise<void>;
     /**
      * Wait until an AskUICommand does not fail.
      *

package/dist/cjs/execution/ui-control-client.js

@@ -18,11 +18,12 @@ const logger_1 = require("../lib/logger");
 const ui_control_client_dependency_builder_1 = require("./ui-control-client-dependency-builder");
 const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
 class UiControlClient extends dsl_1.ApiCommands {
-    constructor(workspaceId, executionRuntime, stepReporter) {
+    constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs) {
         super();
         this.workspaceId = workspaceId;
         this.executionRuntime = executionRuntime;
         this.stepReporter = stepReporter;
+        this.aiElementArgs = aiElementArgs;
         this.secretText = undefined;
     }
     static build() {
@@ -30,7 +31,7 @@ class UiControlClient extends dsl_1.ApiCommands {
             const builder = ui_control_client_dependency_builder_1.UiControlClientDependencyBuilder;
             const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
             const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
-            return new UiControlClient(workspaceId, executionRuntime, stepReporter);
+            return new UiControlClient(workspaceId, executionRuntime, stepReporter, clientArgsWithDefaults.aiElementArgs);
         });
     }
     /**
@@ -130,7 +131,7 @@ class UiControlClient extends dsl_1.ApiCommands {
                 return [];
             }
             // eslint-disable-next-line max-len
-            const workspaceAIElementCollection = yield ai_element_collection_1.AIElementCollection.collectForWorkspaceId(this.workspaceId);
+            const workspaceAIElementCollection = yield ai_element_collection_1.AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs);
             return workspaceAIElementCollection.getByNames(names);
         });
     }
@@ -409,16 +410,6 @@ class UiControlClient extends dsl_1.ApiCommands {
                 .exec();
         });
     }
-    /**
-     * Clicks an icon based on a description.
-     *
-     * @param {string} description
-     */
-    clickIcon(description) {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.click().icon().matching(description).exec();
-        });
-    }
     /**
      * Wait until an AskUICommand does not fail.
      *

package/dist/cjs/execution/ui-controller-client-interface.d.ts

@@ -4,6 +4,7 @@ import { ModelCompositionBranch } from './model-composition-branch';
 import { Reporter } from '../core/reporting';
 import { Context } from './context';
 import { RetryStrategy } from './retry-strategies/retry-strategy';
+import { AIElementArgs } from '../core/ai-element/ai-elements-args';
 /**
  * Context object to provide additional information about the context of (test) automation.
  *
@@ -47,6 +48,8 @@ export interface ContextArgs {
  * @property {(RetryStrategy | undefined)} [retryStrategy] - Default: `new LinearRetryStrategy()`.
  *    Strategy for retrying failed requests to the inference server. This can help manage transient
  *    errors or network issues, improving the reliability of interactions with the server.
+ * @property {AIElementArgs} [aiElementArgs] - Options for configuring how AI elements are
+ *   collected.
  */
 export interface ClientArgs {
     readonly uiControllerUrl?: string;
@@ -59,11 +62,13 @@ export interface ClientArgs {
     readonly context?: ContextArgs | undefined;
     readonly inferenceServerApiVersion?: string;
     readonly retryStrategy?: RetryStrategy;
+    readonly aiElementArgs?: AIElementArgs;
 }
 export interface ClientArgsWithDefaults extends ClientArgs {
     readonly uiControllerUrl: string;
     readonly inferenceServerUrl: string;
     readonly context: Context;
     readonly inferenceServerApiVersion: string;
+    readonly aiElementArgs: AIElementArgs;
     readonly retryStrategy?: RetryStrategy;
 }

package/dist/esm/core/ai-element/ai-element-collection.d.ts

@@ -1,10 +1,12 @@
 import { AIElement } from './ai-element';
 import { CustomElementJson } from '../model/custom-element-json';
+import { AIElementArgs } from './ai-elements-args';
 export declare class AIElementCollection {
     private elements;
     static AI_ELEMENT_FOLDER: string;
     constructor(elements: AIElement[]);
-    static collectForWorkspaceId(workspaceId: string | undefined): Promise<AIElementCollection>;
+    static collectAIElements(workspaceId: string | undefined, aiElementArgs: AIElementArgs): Promise<AIElementCollection>;
     getByName(name: string): CustomElementJson[];
     getByNames(names: string[]): CustomElementJson[];
+    private static CollectAiElementsFromLocation;
 }

package/dist/esm/core/ai-element/ai-element-collection.js

@@ -17,37 +17,37 @@ export class AIElementCollection {
     constructor(elements) {
         this.elements = elements;
     }
-    static collectForWorkspaceId(workspaceId) {
+    static collectAIElements(workspaceId, aiElementArgs) {
         return __awaiter(this, void 0, void 0, function* () {
-            logger.debug(`Collecting AIElements for workspace '${workspaceId}' ...`);
-            if (workspaceId === undefined) {
+            if (!workspaceId) {
                 throw new AIElementError("Value of 'workspaceId' must be defined.");
             }
             const workspaceAIElementFolder = path.join(AIElementCollection.AI_ELEMENT_FOLDER, workspaceId);
-            if (!(yield fs.pathExists(workspaceAIElementFolder))) {
-                throw new AIElementError(`Missing AIElement folder for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
-            }
-            const files = yield fs.readdir(workspaceAIElementFolder);
-            if (files.length === 0) {
-                throw new AIElementError(`'${workspaceAIElementFolder}' is empty. No AIElement files found for workspace '${workspaceId}'.`);
-            }
-            const aiElements = yield Promise.all(files
-                .filter((file) => path.extname(file) === '.json')
-                .map((file) => __awaiter(this, void 0, void 0, function* () {
-                const jsonFile = path.join(workspaceAIElementFolder, file);
-                const baseName = path.basename(jsonFile, '.json');
-                const pngFile = path.join(workspaceAIElementFolder, `${baseName}.png`);
-                if (yield fs.pathExists(pngFile)) {
-                    const metadata = JSON.parse(yield fs.readFile(jsonFile, 'utf-8'));
-                    return AIElement.fromJson(metadata, pngFile);
+            const aiElementsLocations = [
+                workspaceAIElementFolder,
+                ...aiElementArgs.additionalLocations.map((userPath) => path.resolve(userPath)),
+            ];
+            const aiElements = [];
+            yield Promise.all(aiElementsLocations.map((aiElementLocation) => __awaiter(this, void 0, void 0, function* () {
+                if (yield fs.pathExists(aiElementLocation)) {
+                    aiElements.push(...AIElementCollection.CollectAiElementsFromLocation(aiElementLocation));
+                }
+                else {
+                    const errorMessage = `AIElements location '${aiElementLocation}' does not exist.`;
+                    if (aiElementArgs.onLocationNotExist === 'error') {
+                        throw new AIElementError(errorMessage);
+                    }
+                    else if (aiElementArgs.onLocationNotExist === 'warn') {
+                        logger.warn(errorMessage);
+                    }
                 }
-                return null;
             })));
-            const validAIElements = aiElements.filter((element) => element !== null);
-            if (validAIElements.length === 0) {
-                throw new AIElementError(`No AIElement files found for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
+            if (aiElements.length === 0) {
+                const formattedLocations = aiElementsLocations.map((aiElementsLocation) => `"${aiElementsLocation}"`).join(', ');
+                const errorMessage = `No AIElements found in the following location${aiElementsLocations.length > 1 ? 's' : ''}: [${formattedLocations}].`;
+                throw new AIElementError(errorMessage);
             }
-            return new AIElementCollection(validAIElements);
+            return new AIElementCollection(aiElements);
         });
     }
     getByName(name) {
@@ -67,5 +67,28 @@ export class AIElementCollection {
         }
         return names.flatMap((name) => this.getByName(name));
     }
+    static CollectAiElementsFromLocation(aiElementLocation) {
+        const files = fs.readdirSync(aiElementLocation);
+        if (files.length === 0) {
+            return [];
+        }
+        const aiElements = files
+            .filter((file) => path.extname(file) === '.json')
+            .map((file) => {
+            const jsonFile = path.join(aiElementLocation, file);
+            const baseName = path.basename(jsonFile, '.json');
+            const pngFile = path.join(aiElementLocation, `${baseName}.png`);
+            if (fs.existsSync(pngFile)) {
+                const metadata = JSON.parse(fs.readFileSync(jsonFile, 'utf-8'));
+                return AIElement.fromJson(metadata, pngFile);
+            }
+            return null;
+        });
+        const validAIElements = aiElements.filter((element) => element !== null);
+        if (validAIElements.length === 0) {
+            logger.debug(`No valid AIElements found in '${aiElementLocation}'.`);
+        }
+        return validAIElements;
+    }
 }
 AIElementCollection.AI_ELEMENT_FOLDER = path.join(os.homedir(), '.askui', 'SnippingTool', 'AIElement');

package/dist/esm/core/ai-element/ai-elements-args.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Options for configuring how AI elements are collected.
+ *
+ * @interface AIElementArgs
+ *
+ * @property {string[]} additionalLocations - Additional directories to search for AI elements.
+ *  These directories will be checked for the presence of AI element files.
+ *
+ * @property {'Warn' | 'Error' | 'Ignore'} onLocationNotExist - Action to take when
+ *  one or more specified directories do not exist.
+ *      - `'warn'` → Logs a warning if a directory is missing.
+ *      - `'error'` → Throws an error if a directory is missing.
+ *      - `'igone'` → Skips missing directories without logging anything.
+ *  This action is triggered when specified directories do not exist,
+ *      not when AI elements within those directories are missing.
+ */
+export interface AIElementArgs {
+    additionalLocations: string[];
+    onLocationNotExist?: 'warn' | 'error' | 'ignore';
+}

package/dist/esm/core/ai-element/ai-elements-args.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/execution/dsl.d.ts

@@ -99,28 +99,6 @@ export declare class FluentFilters extends FluentBase {
      * @return {FluentFiltersOrRelations}
      */
     button(): FluentFiltersOrRelations;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelations}
-     */
-    row(index: number): FluentFiltersOrRelations;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelations}
-     */
-    column(index: number): FluentFiltersOrRelations;
-    /**
-     *
-     * @param {number} row_index - row index
-     * @param {number} column_index - column index
-     *
-     * @return {FluentFiltersOrRelations}
-     */
-    cell(row_index: number, column_index: number): FluentFiltersOrRelations;
     /**
      * Filters for a UI element 'table'.
      *
@@ -387,42 +365,6 @@ export declare class FluentFilters extends FluentBase {
      * @return {FluentFiltersOrRelations}
      */
     pta(text: string): FluentFiltersOrRelations;
-    /**
-     * Filters elements based on a textual description.
-     *
-     * **What Should I Write as Matching Text**
-     *
-     * The text description inside the `matching()` should describe the element visually.
-     * It understands color, some famous company/product names, general descriptions.
-     *
-     * **Important: _Matching only returns the best matching element when you use it with `get()`_**
-     *
-     * A bit of playing around to find a matching description is sometimes needed:
-     * For example, `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
-     * Generally, the more detail the better.
-     *
-     * We also recommend to not restrict the type of element by using the general selector `element()` as shown in the examples below.
-     *
-     * **Examples:**
-     * ```typescript
-     * // Select the black sneaker from a bunch of sneakers
-     * await aui.click().element().matching('a black sneaker shoe').exec();
-     *
-     * // Select an image that has text in it
-     * await aui.click().element().matching('has Burger King in it').exec();
-     * await aui.click().element().matching('has adidas in it').exec();
-     *
-     * // Target a logo/image by describing it
-     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
-     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
-     * await aui.click().element().matching('logo looking like a seashell').exec();
-     * ```
-     *
-     * @param {string} text - A description of the target element.
-     *
-     * @return {FluentFiltersOrRelations}
-     */
-    matching(text: string): FluentFiltersOrRelations;
 }
 export declare class FluentFiltersOrRelations extends FluentFilters {
     /**
@@ -933,28 +875,6 @@ export declare class FluentFiltersCondition extends FluentBase {
      * @return {FluentFiltersOrRelationsCondition}
      */
     button(): FluentFiltersOrRelationsCondition;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelationsCondition}
-     */
-    row(index: number): FluentFiltersOrRelationsCondition;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelationsCondition}
-     */
-    column(index: number): FluentFiltersOrRelationsCondition;
-    /**
-     *
-     * @param {number} row_index - row index
-     * @param {number} column_index - column index
-     *
-     * @return {FluentFiltersOrRelationsCondition}
-     */
-    cell(row_index: number, column_index: number): FluentFiltersOrRelationsCondition;
     /**
      * Filters for a UI element 'table'.
      *
@@ -1221,42 +1141,6 @@ export declare class FluentFiltersCondition extends FluentBase {
      * @return {FluentFiltersOrRelationsCondition}
      */
     pta(text: string): FluentFiltersOrRelationsCondition;
-    /**
-     * Filters elements based on a textual description.
-     *
-     * **What Should I Write as Matching Text**
-     *
-     * The text description inside the `matching()` should describe the element visually.
-     * It understands color, some famous company/product names, general descriptions.
-     *
-     * **Important: _Matching only returns the best matching element when you use it with `get()`_**
-     *
-     * A bit of playing around to find a matching description is sometimes needed:
-     * For example, `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
-     * Generally, the more detail the better.
-     *
-     * We also recommend to not restrict the type of element by using the general selector `element()` as shown in the examples below.
-     *
-     * **Examples:**
-     * ```typescript
-     * // Select the black sneaker from a bunch of sneakers
-     * await aui.click().element().matching('a black sneaker shoe').exec();
-     *
-     * // Select an image that has text in it
-     * await aui.click().element().matching('has Burger King in it').exec();
-     * await aui.click().element().matching('has adidas in it').exec();
-     *
-     * // Target a logo/image by describing it
-     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
-     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
-     * await aui.click().element().matching('logo looking like a seashell').exec();
-     * ```
-     *
-     * @param {string} text - A description of the target element.
-     *
-     * @return {FluentFiltersOrRelationsCondition}
-     */
-    matching(text: string): FluentFiltersOrRelationsCondition;
 }
 export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
     /**
@@ -2271,28 +2155,6 @@ export declare class FluentFiltersGetter extends FluentBase {
      * @return {FluentFiltersOrRelationsGetter}
      */
     button(): FluentFiltersOrRelationsGetter;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelationsGetter}
-     */
-    row(index: number): FluentFiltersOrRelationsGetter;
-    /**
-     *
-     * @param {number} index - element index
-     *
-     * @return {FluentFiltersOrRelationsGetter}
-     */
-    column(index: number): FluentFiltersOrRelationsGetter;
-    /**
-     *
-     * @param {number} row_index - row index
-     * @param {number} column_index - column index
-     *
-     * @return {FluentFiltersOrRelationsGetter}
-     */
-    cell(row_index: number, column_index: number): FluentFiltersOrRelationsGetter;
     /**
      * Filters for a UI element 'table'.
      *
@@ -2559,42 +2421,6 @@ export declare class FluentFiltersGetter extends FluentBase {
      * @return {FluentFiltersOrRelationsGetter}
      */
     pta(text: string): FluentFiltersOrRelationsGetter;
-    /**
-     * Filters elements based on a textual description.
-     *
-     * **What Should I Write as Matching Text**
-     *
-     * The text description inside the `matching()` should describe the element visually.
-     * It understands color, some famous company/product names, general descriptions.
-     *
-     * **Important: _Matching only returns the best matching element when you use it with `get()`_**
-     *
-     * A bit of playing around to find a matching description is sometimes needed:
-     * For example, `puzzle piece` can fail while `an icon showing a puzzle piece` might work.
-     * Generally, the more detail the better.
-     *
-     * We also recommend to not restrict the type of element by using the general selector `element()` as shown in the examples below.
-     *
-     * **Examples:**
-     * ```typescript
-     * // Select the black sneaker from a bunch of sneakers
-     * await aui.click().element().matching('a black sneaker shoe').exec();
-     *
-     * // Select an image that has text in it
-     * await aui.click().element().matching('has Burger King in it').exec();
-     * await aui.click().element().matching('has adidas in it').exec();
-     *
-     * // Target a logo/image by describing it
-     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
-     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
-     * await aui.click().element().matching('logo looking like a seashell').exec();
-     * ```
-     *
-     * @param {string} text - A description of the target element.
-     *
-     * @return {FluentFiltersOrRelationsGetter}
-     */
-    matching(text: string): FluentFiltersOrRelationsGetter;
 }
 export declare class FluentFiltersOrRelationsGetter extends FluentFiltersGetter {
     /**