@skyramp/skyramp 1.2.21 → 1.2.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/skyramp",
-  "version": "1.2.21",
+  "version": "1.2.22",
   "description": "module for leveraging skyramp cli functionality",
   "scripts": {
     "lint": "eslint 'src/**/*.js' 'src/**/*.ts' --fix",

package/src/classes/SkyrampClient.d.ts

@@ -251,7 +251,7 @@ export declare class SkyrampClient {
     negativeScenario: boolean
   ): Promise<string[]>;
 
-  sendRequest(options: SendRequestV2Options): Promise<void>;
+  sendRequest(options: SendRequestV2Options): Promise<ResponseV2>;
   deployDashboard(network: string): Promise<void>;
 
   generateRestTest(options: GenerateRestTestOptions): Promise<string>;

package/src/classes/SmartPlaywright.d.ts

@@ -1,2 +1,5 @@
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function newSkyrampPlaywrightPage(page: any): any;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function expect(obj: any): any;

package/src/classes/SmartPlaywright.js

@@ -1,5 +1,4 @@
 const { expect: playwrightExpect } = require('@playwright/test');
-/*
 const lib = require('../lib');
 const koffi = require('koffi');
 
@@ -9,6 +8,9 @@ const responseType = koffi.struct({
 });
 
 const improvePlaywrightSelectorWrapper = lib.func('improvePlywrightSelectorWrapper', responseType, ['string']);
+const hasJavascriptWrapper = lib.func('hasJavascript', 'bool', ['string']);
+
+const defaultWaitForTimeout = 1500
 
 async function improvePlaywrightSelectorWithLlm(originalSelector, errorMessage, domContext, pageTitle, pageURL) {
     const requestData = {
@@ -16,7 +18,8 @@ async function improvePlaywrightSelectorWithLlm(originalSelector, errorMessage,
         "error_message": errorMessage,
         "dom_context": domContext,
         "page_title": pageTitle,
-        "page_url": pageURL
+        "page_url": pageURL,
+        "language": "javascript"
     };
     const request = JSON.stringify(requestData);
 
@@ -39,7 +42,6 @@ async function improvePlaywrightSelectorWithLlm(originalSelector, errorMessage,
         });
     });
 }
-*/
 
 function debug(...args) {
     if (process.env.SKYRAMP_DEBUG == "true") {
@@ -47,6 +49,165 @@ function debug(...args) {
     }
 }
 
+function shouldAttemptImprovement(errorMessage, errorName) {
+    const improvementKeywords = [
+        "timeout",
+        "not found",
+        "no element",
+        "not visible",
+        "not attached",
+        "selector resolved to hidden",
+        "element is not enabled",
+        "strict mode violation"
+    ];
+
+    if (errorName != "") {
+        const playwrightErrorTypes = [
+            "TimeoutError",
+            "Error",
+            "LocatorAssertionError"
+        ];
+        if (playwrightErrorTypes.some(e => e == errorName)) {
+            return true;
+        }
+    }
+
+    let errorLower = errorMessage.toLowerCase();
+    return improvementKeywords.some(keyword => errorLower.includes(keyword));
+}
+
+// for debugging only
+/*
+const sampleSuggestions = [
+  {
+    selector: "page.getByText('K', { exact: true})",
+    confidence: 0.9,
+    reasoning: "The 'L' appears to be within a user profile button that shows 'K' as the user initial, using semantic role for better reliability"
+  },
+  {
+    selector: "page.locator('#onyx-user-dropdown')",
+    confidence: 0.85,
+    reasoning: 'Targets the specific user dropdown container with unique ID that contains the user initial'
+  },
+  {
+    selector: "page.locator('.bg-background-900').filter({ hasText: 'K' })",
+    confidence: 0.8,
+    reasoning: 'Targets the specific styled element containing the user initial with background class'
+  },
+  {
+    selector: "page.locator('div').filter({ hasText: /^K$/ }).nth(0)",
+    confidence: 0.75,
+    reasoning: "Uses regex to find exact 'K' text match and takes first occurrence"
+  },
+  {
+    selector: "page.getByText('kolja').locator('..').getByText('K', { exact: true })",
+    confidence: 0.7,
+    reasoning: "Navigates to parent of username text and finds the exact 'K' text within that context"
+  }
+]
+*/
+
+function parseErrorStack(ori) {
+    const stackLines = ori.split('\n');
+    let prev = ""
+    for (let idx = stackLines.length - 1; idx >= 0 ; idx --) {
+        // this could be improved
+        if (stackLines[idx].includes("@skyramp/skyramp")) {
+            break;
+        }
+        prev = stackLines[idx];
+    }
+    return prev
+}
+
+async function retryWithLLM(skyrampLocator, error) {
+    // when API_KEY is not given, we return right away
+    if (!process.env.API_KEY) {
+        throw error
+    }
+
+    let errorMessage = error.toString();
+    let errorType = typeof error;
+
+    if (!(skyrampLocator instanceof SkyrampPlaywrightLocator)) {
+        throw error
+    }
+
+    let locatorStr = skyrampLocator._locator.toString();
+
+    if (!shouldAttemptImprovement(errorMessage, errorType)) {
+        debug("cannot improve", error.name);
+        throw error
+    }
+
+    let pageTitle = await skyrampLocator.page().title();
+    let pageUrl = skyrampLocator.page().url();
+
+    debug("retry with LLM", skyrampLocator._locator);
+
+    let newMsg = `${errorMessage} (while using selector: ${locatorStr})`;
+
+    let pageContent = await skyrampLocator.page().content();
+
+    // for debugging only
+    // let suggestions = sampleSuggestions;
+
+    var suggestions;
+    try {
+        suggestions = await improvePlaywrightSelectorWithLlm(
+            locatorStr,
+            newMsg,
+            pageContent,
+            pageTitle,
+            pageUrl
+        );
+    } catch (error) {
+        debug("failed to consult with llm", error)
+        throw error
+    }
+
+    if (suggestions == null || suggestions.length == 0) {
+        debug("No LLM suggestoins available, failing with original error");
+        throw error;
+    }
+
+    suggestions.sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0))
+    debug(suggestions)
+
+    for (let suggestion of suggestions) {
+        debug(`try ${suggestion.selector} instead of ${skyrampLocator._locator}`);
+        try {
+            const newLocator = skyrampLocator.createLocatorFromString(suggestion.selector);
+            if (newLocator == null) {
+                continue
+            }
+
+            const locatorCount = await newLocator.count();
+
+            debug(` trying new Locator ${newLocator}, count = ${locatorCount}`);
+
+            if (locatorCount == 1) {
+                const func = newLocator[skyrampLocator.execFname];
+                return func.call(newLocator, skyrampLocator.execParam, skyrampLocator.execArgs).then(result => {
+                    console.log(`✅ SUCCESS! Used selector: ${newLocator} instead of ${skyrampLocator._locator}`);
+                    console.log(`  ${parseErrorStack(error.stack)}`);
+
+                    skyrampLocator.llmLocator = newLocator;
+                    skyrampLocator.locatorCount = 1;
+                    skyrampLocator._skyrampPage.addLLMChoices(skyrampLocator._locator, newLocator, error.stack);
+                    return result;
+                }).catch(error => {
+                    throw new Error(`retrying with LLM failed at ${skyrampLocator._locator} replacing ${newLocator}`, error);
+                });
+            }
+        } catch {
+            continue
+        }
+    }
+
+    throw new Error(`failed to find a working locator from LLM's suggestions`, error);
+}
+
 class SkyrampPlaywrightLocator {
     constructor(skyrampPage, locator, prevLocator, args, options, hydration) {
         this._skyrampPage = skyrampPage
@@ -84,49 +245,29 @@ class SkyrampPlaywrightLocator {
         return this._previousLocator && this._previousLocator.isHydration()
     }
 
-    shouldAttemptImprovement(errorMessage, errorType) {
-        const improvementKeywords = [
-            "timeout",
-            "not found",
-            "no element",
-            "not visible",
-            "not attached",
-            "selector resolved to hidden",
-            "element is not enabled",
-        ];
-
-        if (errorType != null) {
-            const playwrightErrorTypes = [
-                "TimeoutError",
-                "Error",
-                "LocatorAssertionError"
-            ];
-            if (playwrightErrorTypes.some(errType => errorType.name.endsWith(errType))) {
-                return true;
-            }
-        }
-
-        let errorLower = errorMessage.toLowerCase();
-        return improvementKeywords.some(keyword => errorLower.includes(keyword));
-    }
-
-    isSelectorMethod(methodName) {
-        // Check if a method uses selectors that can be improved.
-        const selectorMethods = [
-            'click', 'fill', 'type', 'press', 'check', 'uncheck', 'select_option',
-            'hover', 'focus', 'blur', 'scroll_into_view_if_needed', 'screenshot',
-            'text_content', 'inner_text', 'inner_html', 'get_attribute', 'is_visible',
-            'is_enabled', 'is_checked', 'is_disabled', 'is_editable', 'is_hidden'
-        ]
-        return selectorMethods.includes(methodName)
-    }
-
     async execute() {
-        debug(`execute ${ this._locator}.${this.execFname} with ${this.execParam} ${this.execArgs}`)
+        debug(`execute ${ this._locator}.${this.execFname} ${this.execParam ?? ''} ${this.execArgs ?? ''}`)
         const func = this._locator[this.execFname];
         return func.call(this._locator, this.execParam, this.execArgs);
     }
 
+    generateLLMErrors() {
+        let ret = "List of dynamic selectors that could be relevant to error:\n"
+
+        let choices = this._skyrampPage.getLLMChoices();
+        if (choices ==undefined || choices.length == 0) {
+            return ""
+        }
+        for (let i = 0 ; i < choices.length; i ++) {
+            const choice = choices[i];
+            ret += `${ i+1 }. original locator: ${choice.original}\n`
+            ret += `   selected locator: ${choice.new}\n`
+            ret += `   ${choice.at}\n`
+        }
+
+        return ret;
+    }
+
     // this is the function that does smart selector retry
     async SmartRetryWithFallback(fname, param, ...args) {
         this.execFname = fname;
@@ -137,6 +278,9 @@ class SkyrampPlaywrightLocator {
         debug(`handling ${ this._locator }.${ fname }, count = ${ locatorCount }`)
         this.locatorCount = locatorCount
 
+        let currentUrl = this._skyrampPage._page.url();
+        debug(`current url = ${currentUrl}`);
+
         // if locator exists, DOM is available
         if (locatorCount == 1) {
             // we try action, this will most likely succeed
@@ -144,65 +288,217 @@ class SkyrampPlaywrightLocator {
             return this.execute()
             .catch(error => {
                 if (error.name == "TimeoutError") {
-                    return this.execute().catch(error => {
-                        throw new Error("Potentially a hydration issue. Please add enough waitForTimeout()", error);
+                    debug(`locator ${this._locator} exists, but execution failed, wait a bit and retry`)
+                    this.wait(defaultWaitForTimeout)
+
+                    return this.execute().catch(newError => {
+                        debug(`second attempt on ${this._locator} failed`)
+                        debug(newError)
+                        if (this._skyrampPage.hasLLMChoices()) {
+                            newError.message += this.generateLLMErrors();
+                            throw newError;
+                        } else {
+                            throw new Error("Potentially a hydration issue. Please add enough waitForTimeout()", error);
+                        }
                     });
                 }
                 throw error;
             });
         } else if (locatorCount > 0) {
             // TODO
-            debug("multiple locators identified")
-            return this.execute();
+            debug(`multiple ${locatorCount} locators identified`)
+
+            if (locatorCount > 5) {
+                throw new Error(`${locatorCount} locators detected for ${this._locator}. Please add "data-testid" attribute for a more stable locator`);
+            }
+
+            return this.execute().then(result => {
+                return result;
+            }).catch(error => {
+                if (!process.env.API_KEY) {
+                    throw new Error(`${locatorCount} locators detected for ${this._locator}. Please add "data-testid" attribute for a more stable locator`, error)
+                }
+
+                console.log(`${locatorCount} locators found for ${this._locator}`)
+                // this is the palce where we consult with LLM
+                return retryWithLLM(this, error).then(result => {
+                    this.multiLocatorLLM = true;
+                    return result;
+                }).catch(newError => {
+                    if (this._skyrampPage.hasLLMChoices()) {
+                        newError.message += this.generateLLMErrors();
+                        throw newError;
+                    }
+                    throw new Error(`failed to find a good alternative for ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError);
+                });
+            });
         } else {
             // if locator does not exist, we need to consider two cases
             // one is if any actions that required hydration did not work
             // second, if locator id is not correct
-            let couldBeHydration = false
-            try {
-                // check if last step has potential hydration 
-                if (this._previousLocator && this._previousLocator.locatorCount == 0) {
-                    couldBeHydration = true
-                    debug(`previous action ${this._previousLocator._locator} is potentially associated with hydration`);
-                    // wait for a short time to finish hydration
-                    await this._skyrampPage._page.waitForTimeout(1500);
-                    // then re-execute the previous locator
-                    await this._previousLocator.execute();
-
+            // check if last step has potential hydration 
+            if (this._previousLocator && this._previousLocator.locatorCount == 0) {
+                debug(`previous action ${this._previousLocator._locator} is potentially associated with hydration`);
+                // wait for a short time to finish hydration
+                await this.wait(defaultWaitForTimeout);
+
+                const previousCount = await this._previousLocator.count();
+                debug(` re-execute the previous one ${this._previousLocator._locator}, previous Locator count = ${previousCount}`);
+
+                // re-execute the previous locator
+                await this._previousLocator.execute().catch(() => {
+                    // log the failure but continues to the current one
+                    debug(`failed to execute previous locator ${this._previousLocator._locator} again`);
+                });
+
+                try {
                     // then execute the current one
-                    return this.execute();
+                    return await this.execute();
+                } catch (error) {
+                    if (error.name == "TimeoutError") {
+                        debug(`${this._locator} failed at first try. attempting again with some timeout`);
+                        // wait for some time and re execute
+                        await this.wait(defaultWaitForTimeout);
+
+                        return this.execute().catch(newError => {
+                            if (!process.env.API_KEY) {
+                                throw new Error(`Cannot find locator ${this._locator} and likely a hydration issue on ${this._previousLocator._locator}. Please add enough waitForTimeout()`, newError);
+                            }
+                            // retry with llm
+                            return retryWithLLM(this, newError).then(result => {
+                                this.singleLocatorLLM = true;
+                                return result;
+                            }).catch(newError2 => {
+                                if (this._skyrampPage.hasLLMChoices()) {
+                                    newError2.message += this.generateLLMErrors();
+                                    throw newError2;
+                                }
+                                throw new Error(`Failed to find a good alternative of ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError2);
+                            });
+                        });
+                    }
+                    throw error;
+                }
+            } else {
+                if (this._previousLocator && this._previousLocator.multiLocatorLLM) {
+                    debug(`${this._locator} locator count is zero, but previous locator was selected by LLM from many candidates`);
+                } else if (this._previousLocator && this._previousLocator.singleLocatorLLM) {
+                    debug(`${this._locator} locator count is zero, but previous locator was selected by LLM from zero candidate`);
                 } else {
                     debug(`${this._locator} locator count is zero, but previous locator seems not related to hydration`);
-                    // previous action may not be associated with hydration
-                    // then we just try current locator. could be a locator with a wrong id
-                    // wait for a short time just in case
-                    await this._skyrampPage._page.waitForTimeout(1500);
-
-                    return await this.execute().then(result => {
-                        this.locatorCount = 1;
-                        return result
-                    }).catch(error => {
-                        throw error;
-                    });
                 }
-            } catch (error) {
-                if (couldBeHydration && error.name == "TimeoutError") {
-                    return this.execute()
-                        .catch(error => {
-                            throw new Error(`Potentially a hydration issue. Please add enough waitForTimeout() here or before ${this._previousLocator._locator}`, error);
+                // previous action may not be associated with hydration
+                // then we just try current locator. could be a locator with a wrong id
+                // wait for a short time just in case
+                await this.wait(defaultWaitForTimeout);
+
+                this.locatorCount = await this._locator.count();
+                debug(` after waiting locator count = ${this.locatorCount}, url = ${this._skyrampPage._page.url()}`);
+
+                try {
+                    return await this.execute();
+                } catch (error) {
+                    if (error.name == "TimeoutError") {
+                        debug(`${this._locator} failed at first try. attempting again with some timeout`);
+                        await this.wait(defaultWaitForTimeout);
+
+                        return this.execute().catch(newError => {
+                            if (newError.name == "TimeoutError") {
+                                if (!process.env.API_KEY) {
+                                    throw new Error(`Cannot find locator ${this._locator} and most likely not a hydration issue. Please add "data-testid" attribute for a more stable locator`, newError);
+                                }
+                                // retry with llm
+                                return retryWithLLM(this, newError).then(result => {
+                                    this.singleLocatorLLM = true;
+                                    return result;
+                                }).catch(newError2 => {
+                                    if (this._skyrampPage.hasLLMChoices()) {
+                                        newError2.message += this.generateLLMErrors();
+                                        throw newError2;
+                                    }
+                                    throw new Error(`Failed to find a good alternative of ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError2);
+                                });
+                            } else {
+                                throw newError;
+                            }
                         });
-                } else {
-                    debug(`${this._locator} failed at first try. attempting again with some timeout`);
-                    // wait for a short time before retry
-                    await this._skyrampPage._page.waitForTimeout(1500);
-                    // then re-execute the current locator
-                    // highly likely non-existent locator
-                    return this.execute().catch(error => {
-                        throw new Error(`Cannot find locator ${this._locator} and most likely not a hydration issue. Please add "data-testid" attribute for a more stable locator`, error);
-                    });
+                    }
+                    throw error;
+                }
+            }
+        }
+    }
+
+    parseFunctionChain(chainStr) {
+        // Remove any object prefix like "page."
+        chainStr = chainStr.replace(/^[a-zA-Z_][a-zA-Z0-9_]*\./, '');
+
+        // Match function calls: funcName(args)
+        const regex = /([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)/g;
+        let result = [];
+        let match;
+        while ((match = regex.exec(chainStr)) !== null) {
+            const funcName = match[1];
+            const argsStr = match[2].trim();
+            let args = [];
+            if (argsStr.length > 0) {
+                // Split arguments by comma, but keep objects and strings together
+                // This is a simple approach, for more complex cases use a parser
+                let argMatches = [];
+                let depth = 0, current = '', inString = false, stringChar = '';
+                for (let i = 0; i < argsStr.length; i++) {
+                    const c = argsStr[i];
+                    if ((c === "'" || c === '"') && !inString) {
+                        inString = true;
+                        stringChar = c;
+                        current += c;
+                    } else if (inString && c === stringChar) {
+                        inString = false;
+                        current += c;
+                    } else if (!inString && (c === '{' || c === '[')) {
+                        depth++;
+                        current += c;
+                    } else if (!inString && (c === '}' || c === ']')) {
+                        depth--;
+                        current += c;
+                    } else if (!inString && c === ',' && depth === 0) {
+                        argMatches.push(current.trim());
+                        current = '';
+                    } else {
+                        current += c;
+                    }
                 }
+                if (current.trim().length > 0) {
+                    argMatches.push(current.trim());
+                }
+                args = argMatches;
+            }
+            result.push({ function: funcName, arguments: args });
+        }
+        return result;
+    }
+
+    createLocatorFromString(suggested) {
+        suggested = suggested.trim();
+        if (suggested.startsWith("page.")) {
+            suggested = suggested.substring(5);
+        }
+
+        let chains = this.parseFunctionChain(suggested);
+
+        let cur = this.page();
+        // find locator per function chain
+        for (let i = 0; i < chains.length; i ++) {
+            const chain = chains[i];
+            let f = cur[chain.function];
+            let args = [];
+            for (const a of chain.arguments) {
+                let arg = Function('"use strict";return (' + a + ')')();
+                args.push(arg);
             }
+            cur = f.call(cur, ...args);
         }
+        return cur;
     }
 
     async click(...args) {
@@ -315,11 +611,17 @@ class SkyrampPlaywrightLocator {
     page() {
         return this._skyrampPage._page;
     }
+
+    async wait(t) {
+        debug(`wait for ${t}`);
+        return this._skyrampPage._page.waitForTimeout(t);
+    }
 }
 
 class SkyrampPlaywrightPage {
     constructor(page) {
         this._page = page;
+        this._curUrl = "";
         return new Proxy(this, {
             // The `get` trap is the key to forwarding.
             // This will foraward any methods not implemented in this struct
@@ -344,6 +646,10 @@ class SkyrampPlaywrightPage {
         });
     }
 
+    getCurUrl() {
+        return this._curUrl;
+    }
+
     pushLocator(locator) {
         if (this.locators == undefined ) {
             this.locators = [];
@@ -372,6 +678,7 @@ class SkyrampPlaywrightPage {
             debug(`handling ${originalLocator}`);
         }
         */
+
         let newLocator = new SkyrampPlaywrightLocator(this, originalLocator, prevLocator, [param], options, hydration);
         this.pushLocator(newLocator)
         return newLocator
@@ -416,6 +723,43 @@ class SkyrampPlaywrightPage {
         const originalLocator = this._page.getByAltText(alt, options);
         return this.newSkyrampPlaywrightLocator(originalLocator, alt, options);
     }
+
+    async goto(url, options) {
+        const result = await this._page.goto(url, options);
+        const content = await this._page.content();
+        if (hasJavascriptWrapper(content)) {
+            debug(`javascript download detected when visiting ${this._page.url()}`);
+            debug(`  wait for sometime for potential hydration`);
+            await this._page.waitForTimeout(defaultWaitForTimeout);
+        } else {
+            debug(`javascript not detected when visiting ${this._page.url()}`);
+        }
+        this._curUrl = this._page.url();
+        return result;
+    }
+
+    addLLMChoices(originalLocator, newLocator, stack) {
+        if (this.llmChoices == undefined) {
+            this.llmChoices = []
+        }
+
+        this.llmChoices.push({
+            "original": originalLocator.toString(),
+            "new": newLocator.toString(),
+            "at": parseErrorStack(stack),
+        });
+    }
+
+    getLLMChoices() {
+        return this.llmChoices
+    }
+
+    hasLLMChoices() {
+        if (this.llmChoices == undefined || this.llmChoices.length == 0) {
+            return false;
+        }
+        return true;
+    }
 }
 
 function newSkyrampPlaywrightPage(page) {