@skyramp/skyramp 1.2.22 → 1.2.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/skyramp",
-  "version": "1.2.22",
+  "version": "1.2.24",
   "description": "module for leveraging skyramp cli functionality",
   "scripts": {
     "lint": "eslint 'src/**/*.js' 'src/**/*.ts' --fix",

package/src/classes/SkyrampClient.d.ts

@@ -267,4 +267,12 @@ export declare class SkyrampClient {
     scenario: AsyncScenario | AsyncScenario[],
     options?: SendScenarioOptions
   ): Promise<AsyncTestStatus>;
+
+  /**
+   * Fetches OTP from Skyramp email
+   */
+  fetchLatestOtp(
+    email: string,
+    timestamp: Date
+  ): string;
 }

package/src/classes/SkyrampClient.js

@@ -3,6 +3,7 @@ const koffi = require('koffi');
 const TrafficConfig = require('./TrafficConfig');
 const RequestV2 = require('./RequestV2');
 const ResponseV2 = require('./ResponseV2');
+const utils = require('../utils');
 
 const workerInfoType = koffi.struct({
   container_name: 'char*',
@@ -99,6 +100,7 @@ class SkyrampClient {
    */
   constructor(kubeconfigPathOrOptions, clusterName, context, userToken, directory = process.cwd()) {
     this.local_image = false;
+    this.timestamp = Date.now();
     if (typeof kubeconfigPathOrOptions === 'object') {
       const options = kubeconfigPathOrOptions;
       this.workerNamespaces = [];
@@ -966,6 +968,41 @@ class SkyrampClient {
       );
     });
   }
+
+  async fetchLatestOtp(email, timestamp) {
+    var latestGetResponse
+    // if timestamp is not given,
+    // we use the time that this client was instantiated
+    if (timestamp == undefined || timestamp == null) {
+      timestamp = this.timestamp;
+    }
+
+    if (!email.includes("@otp.skyramp.dev")) {
+        throw new Error("non-Skyramp email is not supported")
+    }
+
+    const username = email.replace("@otp.skyramp.dev", "")
+
+    for (let i = 0; i < 10; i ++) {
+      // Execute Request
+      latestGetResponse = await this.sendRequest({
+          url: "https://tokenize.skyramp.dev",
+          path: `/msg/${username}/latest`,
+          method: "GET"
+      });
+
+      if (latestGetResponse.statusCode != 200) {
+          await new Promise(resolve => setTimeout(resolve, 1000));
+          continue
+      }
+
+      let codeTime = new Date(utils.getValue(latestGetResponse.responseBody, "timestamp"));
+      if (codeTime > timestamp) {
+        return utils.getValue(latestGetResponse.responseBody, "otp")
+      }
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+  }
 }
 
 module.exports = SkyrampClient;

package/src/classes/SmartPlaywright.js

@@ -143,8 +143,6 @@ async function retryWithLLM(skyrampLocator, error) {
     let pageTitle = await skyrampLocator.page().title();
     let pageUrl = skyrampLocator.page().url();
 
-    debug("retry with LLM", skyrampLocator._locator);
-
     let newMsg = `${errorMessage} (while using selector: ${locatorStr})`;
 
     let pageContent = await skyrampLocator.page().content();
@@ -175,7 +173,6 @@ async function retryWithLLM(skyrampLocator, error) {
     debug(suggestions)
 
     for (let suggestion of suggestions) {
-        debug(`try ${suggestion.selector} instead of ${skyrampLocator._locator}`);
         try {
             const newLocator = skyrampLocator.createLocatorFromString(suggestion.selector);
             if (newLocator == null) {
@@ -184,7 +181,7 @@ async function retryWithLLM(skyrampLocator, error) {
 
             const locatorCount = await newLocator.count();
 
-            debug(` trying new Locator ${newLocator}, count = ${locatorCount}`);
+            debug(`trying new Locator ${newLocator} instead of ${skyrampLocator._locator}, count = ${locatorCount}`);
 
             if (locatorCount == 1) {
                 const func = newLocator[skyrampLocator.execFname];
@@ -197,7 +194,8 @@ async function retryWithLLM(skyrampLocator, error) {
                     skyrampLocator._skyrampPage.addLLMChoices(skyrampLocator._locator, newLocator, error.stack);
                     return result;
                 }).catch(error => {
-                    throw new Error(`retrying with LLM failed at ${skyrampLocator._locator} replacing ${newLocator}`, error);
+                    // if it fails, move to the next one
+                    debug(`retrying with LLM failed at ${skyrampLocator._locator} replacing ${newLocator}`, error.name);
                 });
             }
         } catch {
@@ -205,7 +203,9 @@ async function retryWithLLM(skyrampLocator, error) {
         }
     }
 
-    throw new Error(`failed to find a working locator from LLM's suggestions`, error);
+    error.message = `Failed to find a good alternative for ${skyrampLocator._locator} with LLM.\n` +
+                    `Please add "data-testid" attribute for a more stable locator\n` + error.message;
+    throw error;
 }
 
 class SkyrampPlaywrightLocator {
@@ -246,7 +246,7 @@ class SkyrampPlaywrightLocator {
     }
 
     async execute() {
-        debug(`execute ${ this._locator}.${this.execFname} ${this.execParam ?? ''} ${this.execArgs ?? ''}`)
+        debug(`    execute ${ this._locator}.${this.execFname} ${this.execParam ?? ''} ${this.execArgs ?? ''}`)
         const func = this._locator[this.execFname];
         return func.call(this._locator, this.execParam, this.execArgs);
     }
@@ -268,69 +268,111 @@ class SkyrampPlaywrightLocator {
         return ret;
     }
 
-    // this is the function that does smart selector retry
+    wrapError(msg, error) {
+        let newMsg = msg;
+        if (this._skyrampPage.hasLLMChoices()) {
+            newMsg += this.generateLLMErrors();
+        }
+
+        error.message = error.message + "\n" + newMsg
+
+        return error
+    }
+
+    hydrationErrorMsg = "Potentially a hydration issue. Please add enough waitForTimeout()"
+
+    newPrevHyrdationErrorMsg() {
+        return `Cannot find locator ${this._locator} and likely a hydration issue on ${this._previousLocator._locator}.\n` +
+               `Please add enough waitForTimeout() on ${this._previoousLocator._locator}`;
+    }
+
+    newMultiLocatorErrorMsg() {
+        return `${this._locator} found ${this.locatorCount} locators. Please add "data-testid" attribute for a more stable locator`
+    }
+
+    async _retryWithLLM(error, msg1) {
+        // if API_KEY is not defined, throw an error here without trying
+        if (!process.env.API_KEY) {
+            error.message = msg1 + error.message;
+            throw error;
+        }
+
+        debug(`  try to get suggessions from LLM for ${this._locator}`);
+        return retryWithLLM(this, error).then(result => {
+            this.LLMselector = true;
+            return result;
+        }).catch(newError => {
+            throw this.wrapError("", newError)
+        });
+    }
+
     async SmartRetryWithFallback(fname, param, ...args) {
         this.execFname = fname;
         this.execParam = param;
         this.execArgs = args;
 
         let locatorCount = await this._locator.count();
-        debug(`handling ${ this._locator }.${ fname }, count = ${ locatorCount }`)
+        let currentUrl = this._skyrampPage._page.url();
+        debug(`handling ${ this._locator }.${ fname }, count = ${ locatorCount }, ${currentUrl}`);
         this.locatorCount = locatorCount
 
-        let currentUrl = this._skyrampPage._page.url();
-        debug(`current url = ${currentUrl}`);
+        //debug(`current url = ${currentUrl}`);
 
         // if locator exists, DOM is available
         if (locatorCount == 1) {
+            debug(`  single locator for ${this._locator} identified, ${currentUrl}`);
             // we try action, this will most likely succeed
             // if it fails, there could be potentially a hydration issue we can retry after a little wait time
-            return this.execute()
-            .catch(error => {
+            try {
+                return await this.execute().then(result => {
+                    return this._skyrampPage.checkNavigation(currentUrl, result);
+                });
+            } catch (error) {
+                debug(`  first attempt of ${this._locator} failed, ${error.name}`);
                 if (error.name == "TimeoutError") {
-                    debug(`locator ${this._locator} exists, but execution failed, wait a bit and retry`)
-                    this.wait(defaultWaitForTimeout)
+                    // this is likely a hydration issue
+                    // we wait a bit and retry
+                    debug(`  locator ${this._locator} exists, but execution failed, wait a bit and try from last step retry`);
+                    await this.wait(defaultWaitForTimeout);
+
+                    // Is this really necessary?
+                    await this.execute().catch(() => {
+                        debug(`  failed second time and execute previous locator ${this._previousLocator._locator} again`);
+                        return this._previousLocator.execute();
+                    }).catch(() => {
+                        debug(`  failed to execute previous locator ${this._previousLocator._locator} again, continue`);
+                    });
 
                     return this.execute().catch(newError => {
-                        debug(`second attempt on ${this._locator} failed`)
-                        debug(newError)
-                        if (this._skyrampPage.hasLLMChoices()) {
-                            newError.message += this.generateLLMErrors();
-                            throw newError;
-                        } else {
-                            throw new Error("Potentially a hydration issue. Please add enough waitForTimeout()", error);
+                        debug(`  second attempt on ${this._locator} failed ${newError.name}`);
+                        if (newError.name == "TimeoutError") {
+                            // this hadn't happened yet. we need to validate if this is indeed hydration case
+                            throw this.wrapError(this.hydrationErrorMsg, newError);
                         }
+                        if (newError.message.includes("strict mode violation")) {
+                            return this._retryWithLLM(newError, this.newMultiLocatorErrorMsg());
+                        }
+                        throw error;
                     });
                 }
+                if (error.message.includes("strict mode violation")) {
+                    return this._retryWithLLM(error, this.newMultiLocatorErrorMsg());
+                }
+                // we do not handle the rest of the error cases, but simply forward error
                 throw error;
-            });
+            }
         } else if (locatorCount > 0) {
-            // TODO
-            debug(`multiple ${locatorCount} locators identified`)
-
+            debug(`  multiple ${locatorCount} locators for ${this._locator} identified, ${currentUrl}`);
             if (locatorCount > 5) {
+                // we fail even without trying
                 throw new Error(`${locatorCount} locators detected for ${this._locator}. Please add "data-testid" attribute for a more stable locator`);
             }
 
+            // this will likely fail, but we try to get error message generated by playwright
             return this.execute().then(result => {
-                return result;
+                return this._skyrampPage.checkNavigation(currentUrl, result);
             }).catch(error => {
-                if (!process.env.API_KEY) {
-                    throw new Error(`${locatorCount} locators detected for ${this._locator}. Please add "data-testid" attribute for a more stable locator`, error)
-                }
-
-                console.log(`${locatorCount} locators found for ${this._locator}`)
-                // this is the palce where we consult with LLM
-                return retryWithLLM(this, error).then(result => {
-                    this.multiLocatorLLM = true;
-                    return result;
-                }).catch(newError => {
-                    if (this._skyrampPage.hasLLMChoices()) {
-                        newError.message += this.generateLLMErrors();
-                        throw newError;
-                    }
-                    throw new Error(`failed to find a good alternative for ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError);
-                });
+                return this._retryWithLLM(error, this.newMultiLocatorErrorMsg());
             });
         } else {
             // if locator does not exist, we need to consider two cases
@@ -338,54 +380,42 @@ class SkyrampPlaywrightLocator {
             // second, if locator id is not correct
             // check if last step has potential hydration 
             if (this._previousLocator && this._previousLocator.locatorCount == 0) {
-                debug(`previous action ${this._previousLocator._locator} is potentially associated with hydration`);
+                debug(`  previous action ${this._previousLocator._locator} is potentially associated with hydration, ${currentUrl}`);
                 // wait for a short time to finish hydration
                 await this.wait(defaultWaitForTimeout);
-
                 const previousCount = await this._previousLocator.count();
-                debug(` re-execute the previous one ${this._previousLocator._locator}, previous Locator count = ${previousCount}`);
-
+                debug(`  re-execute the previous one ${this._previousLocator._locator}, new locator count = ${previousCount}, ${this._skyrampPage._page.url()}`);
                 // re-execute the previous locator
                 await this._previousLocator.execute().catch(() => {
                     // log the failure but continues to the current one
-                    debug(`failed to execute previous locator ${this._previousLocator._locator} again`);
+                    debug(`  failed to execute previous locator ${this._previousLocator._locator} again, continue`);
                 });
 
                 try {
                     // then execute the current one
-                    return await this.execute();
+                    return await this.execute().then(result => {
+                        return this._skyrampPage.checkNavigation(currentUrl, result);
+                    });
                 } catch (error) {
                     if (error.name == "TimeoutError") {
-                        debug(`${this._locator} failed at first try. attempting again with some timeout`);
+                        debug(`  ${this._locator} failed at first try. attempting again with some timeout`);
                         // wait for some time and re execute
                         await this.wait(defaultWaitForTimeout);
-
                         return this.execute().catch(newError => {
-                            if (!process.env.API_KEY) {
-                                throw new Error(`Cannot find locator ${this._locator} and likely a hydration issue on ${this._previousLocator._locator}. Please add enough waitForTimeout()`, newError);
-                            }
-                            // retry with llm
-                            return retryWithLLM(this, newError).then(result => {
-                                this.singleLocatorLLM = true;
-                                return result;
-                            }).catch(newError2 => {
-                                if (this._skyrampPage.hasLLMChoices()) {
-                                    newError2.message += this.generateLLMErrors();
-                                    throw newError2;
-                                }
-                                throw new Error(`Failed to find a good alternative of ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError2);
-                            });
+                            return this._retryWithLLM(newError, this.newPrevHydrationErrorMsg());
                         });
                     }
+                    if (error.message.includes("strict mode violation")) {
+                        debug(`  a rare case when multiple locators are detected on ${this._locator}`);
+                        return this._retryWithLLM(error, this.newMultiLocatorErrorMsg());
+                    }
                     throw error;
                 }
             } else {
-                if (this._previousLocator && this._previousLocator.multiLocatorLLM) {
-                    debug(`${this._locator} locator count is zero, but previous locator was selected by LLM from many candidates`);
-                } else if (this._previousLocator && this._previousLocator.singleLocatorLLM) {
-                    debug(`${this._locator} locator count is zero, but previous locator was selected by LLM from zero candidate`);
+                if (this._previousLocator && this._previousLocator.LLMselector) {
+                    debug(`  ${this._locator} locator count is zero, but previous locator was selected by LLM`);
                 } else {
-                    debug(`${this._locator} locator count is zero, but previous locator seems not related to hydration`);
+                    debug(`  ${this._locator} locator count is zero, but previous locator seems not related to hydration`);
                 }
                 // previous action may not be associated with hydration
                 // then we just try current locator. could be a locator with a wrong id
@@ -393,36 +423,30 @@ class SkyrampPlaywrightLocator {
                 await this.wait(defaultWaitForTimeout);
 
                 this.locatorCount = await this._locator.count();
-                debug(` after waiting locator count = ${this.locatorCount}, url = ${this._skyrampPage._page.url()}`);
+                debug(`  after waiting locator ${this._locator} count = ${this.locatorCount}, ${this._skyrampPage._page.url()}`);
 
                 try {
-                    return await this.execute();
+                    return await this.execute().then(result => {
+                        return this._skyrampPage.checkNavigation(currentUrl, result);
+                    });
                 } catch (error) {
                     if (error.name == "TimeoutError") {
                         debug(`${this._locator} failed at first try. attempting again with some timeout`);
                         await this.wait(defaultWaitForTimeout);
-
                         return this.execute().catch(newError => {
                             if (newError.name == "TimeoutError") {
-                                if (!process.env.API_KEY) {
-                                    throw new Error(`Cannot find locator ${this._locator} and most likely not a hydration issue. Please add "data-testid" attribute for a more stable locator`, newError);
-                                }
-                                // retry with llm
-                                return retryWithLLM(this, newError).then(result => {
-                                    this.singleLocatorLLM = true;
-                                    return result;
-                                }).catch(newError2 => {
-                                    if (this._skyrampPage.hasLLMChoices()) {
-                                        newError2.message += this.generateLLMErrors();
-                                        throw newError2;
-                                    }
-                                    throw new Error(`Failed to find a good alternative of ${this._locator}. Please add "data-testid" attribute for a more stable locator`, newError2);
-                                });
-                            } else {
-                                throw newError;
+                                return this._retryWithLLM(newError, this.hydrationErrorMsg);
+                            }
+                            if (newError.message.includes("strict mode violation")) {
+                                return this._retryWithLLM(newError, this.newMultiLocatorErrorMsg());
                             }
+                            throw newError;
                         });
                     }
+                    // if multiple locator found
+                    if (this.locatorCount > 1 || error.message.includes("strict mode violation")) {
+                        return this._retryWithLLM(error, this.newMultiLocatorErrorMsg());
+                    }
                     throw error;
                 }
             }
@@ -613,7 +637,7 @@ class SkyrampPlaywrightLocator {
     }
 
     async wait(t) {
-        debug(`wait for ${t}`);
+        debug(`    wait for ${t}`);
         return this._skyrampPage._page.waitForTimeout(t);
     }
 }
@@ -760,6 +784,15 @@ class SkyrampPlaywrightPage {
         }
         return true;
     }
+
+    async checkNavigation(original, result) {
+        const newURL = this._page.url()
+        if (newURL != original) {
+            debug(`page navigation to ${newURL} detected, wait a bit`);
+            await this._page.waitForTimeout(1500);
+        }
+        return result;
+    }
 }
 
 function newSkyrampPlaywrightPage(page) {