@koalarx/scrapping 1.0.3

package/KoalaScrappingDom.js

@@ -0,0 +1,766 @@
+'use strict';
+var __awaiter =
+  (this && this.__awaiter) ||
+  function (thisArg, _arguments, P, generator) {
+    function adopt(value) {
+      return value instanceof P
+        ? value
+        : new P(function (resolve) {
+            resolve(value);
+          });
+    }
+    return new (P || (P = Promise))(function (resolve, reject) {
+      function fulfilled(value) {
+        try {
+          step(generator.next(value));
+        } catch (e) {
+          reject(e);
+        }
+      }
+      function rejected(value) {
+        try {
+          step(generator['throw'](value));
+        } catch (e) {
+          reject(e);
+        }
+      }
+      function step(result) {
+        result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
+      }
+      step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+  };
+var __importDefault =
+  (this && this.__importDefault) ||
+  function (mod) {
+    return mod && mod.__esModule ? mod : { default: mod };
+  };
+Object.defineProperty(exports, '__esModule', { value: true });
+exports.KoalaScrappingDom = void 0;
+const puppeteer_1 = __importDefault(require('puppeteer'));
+const BrowserEnum_1 = require('./enums/BrowserEnum');
+const delay_1 = require('@koalarx/utils/operators/delay');
+const string_1 = require('@koalarx/utils/operators/string');
+const html_table_to_json_1 = __importDefault(require('html-table-to-json'));
+class KoalaScrappingDom {
+  /**
+   * @param option | URl da página de início do processo
+   */
+  constructor(option) {
+    this.option = option;
+    this._offDialog = false;
+  }
+  closeDOM() {
+    return __awaiter(this, void 0, void 0, function* () {
+      if (this.browser.isConnected()) {
+        yield (0, delay_1.delay)(300);
+        yield this.browser.close();
+      }
+    });
+  }
+  openTab(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+      this.page = yield this.browser.newPage();
+      yield this.initObservableDialog();
+      yield this.goTo(url);
+    });
+  }
+  getLastPage() {
+    return __awaiter(this, void 0, void 0, function* () {
+      return (yield this.browser.pages())[(yield this.browser.pages()).length - 1];
+    });
+  }
+  offDialog() {
+    return __awaiter(this, void 0, void 0, function* () {
+      this._offDialog = true;
+    });
+  }
+  getDataFromTable(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.getTable(xPath).then((tableHtml) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const tmpResult = html_table_to_json_1.default.parse(tableHtml).results[0];
+          const result = [];
+          tmpResult.forEach((obj) => {
+            const newObj = {};
+            Object.keys(obj).forEach((key) => {
+              let value = obj[key];
+              if (value.indexOf('/', 2) >= 0 && value.indexOf('/', 5) >= 0 && value.length === 10) {
+                const arrValue = value.split('/');
+                value = `${arrValue[2]}-${arrValue[1]}-${arrValue[0]}`;
+              }
+              newObj[(0, string_1.toCamelCase)(key)] = value;
+            });
+            result.push(newObj);
+          });
+          return result;
+        }),
+      );
+    });
+  }
+  switchLastTab() {
+    return __awaiter(this, void 0, void 0, function* () {
+      this.page = yield this.getLastPage();
+      yield this.initObservableDialog();
+    });
+  }
+  closeTab() {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.close();
+      this.page = yield this.getLastPage();
+    });
+  }
+  initObservableDialog() {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.removeAllListeners('dialog');
+      this.page.on('dialog', (dialog) =>
+        __awaiter(this, void 0, void 0, function* () {
+          var _a;
+          this.mensagemAlert = yield dialog.message();
+          if (
+            ((_a = this.option.ignoredMessages) !== null && _a !== void 0 ? _a : []).find(
+              (message) => message === this.mensagemAlert,
+            )
+          ) {
+            yield dialog.accept();
+          } else {
+            if (!this._offDialog) {
+              yield dialog.dismiss();
+            }
+          }
+        }),
+      );
+    });
+  }
+  goTo(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.goto(url, { timeout: 0, waitUntil: 'load' }).catch((e) => {
+        throw e;
+      });
+    });
+  }
+  clearInput(xpath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page
+        .waitForXPath(xpath)
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            yield el.evaluate((input) => {
+              input.value = '';
+            });
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  pasteValueInField(xPath, keys, displayNone = false, hardValidation = false, valueFix = false) {
+    return __awaiter(this, void 0, void 0, function* () {
+      if (yield this.existElement(xPath)) {
+        yield this.page.waitForXPath(xPath).then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            yield el.evaluate(
+              (input, value, elDisplayNone) => {
+                input.maxLength = 600;
+                input.value = value;
+                if (elDisplayNone) {
+                  input.style.color = 'transparent';
+                }
+              },
+              keys,
+              displayNone,
+            );
+            if (valueFix) {
+              yield this.writeTextInField(xPath, '0');
+              yield this.keyboardPress(xPath, 'Backspace');
+            }
+            if (hardValidation) {
+              yield (0, delay_1.delay)(2000);
+              const current = yield this.getTextContentOnInputElement(xPath);
+              if (current !== keys) {
+                yield this.pasteValueInField(xPath, keys, displayNone, hardValidation);
+              }
+            }
+          }),
+        );
+      }
+    });
+  }
+  writeTextInField(xPath, value) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.focus();
+          yield el.type(value, { delay: 20 });
+        }),
+      );
+    });
+  }
+  authenticate() {
+    return __awaiter(this, void 0, void 0, function* () {
+      if (this.option.browser !== BrowserEnum_1.BrowserEnum.firefox) {
+        yield this.pasteValueInField(this.option.login.xPath, this.option.login.value, true, false);
+        yield this.pasteValueInField(this.option.password.xPath, this.option.password.value, true, false);
+      } else {
+        yield this.sendKeys(this.option.login.xPath, this.option.login.value, true, false);
+        yield this.sendKeys(this.option.password.xPath, this.option.password.value, true, false);
+      }
+    });
+  }
+  init(headless = true, devtools = false) {
+    var _a, _b, _c;
+    return __awaiter(this, void 0, void 0, function* () {
+      if (!this.browser) {
+        const args = [
+          '--autoplay-policy=user-gesture-required',
+          '--disable-background-networking',
+          '--disable-background-timer-throttling',
+          '--disable-backgrounding-occluded-windows',
+          '--disable-breakpad',
+          '--disable-client-side-phishing-detection',
+          '--disable-component-update',
+          '--disable-default-apps',
+          '--disable-dev-shm-usage',
+          '--disable-domain-reliability',
+          '--disable-extensions',
+          '--disable-features=AudioServiceOutOfProcess',
+          '--disable-hang-monitor',
+          '--disable-ipc-flooding-protection',
+          '--disable-notifications',
+          '--disable-offer-store-unmasked-wallet-cards',
+          '--disable-popup-blocking',
+          '--disable-print-preview',
+          '--disable-prompt-on-repost',
+          '--disable-renderer-backgrounding',
+          '--disable-setuid-sandbox',
+          '--disable-speech-api',
+          '--disable-sync',
+          '--hide-scrollbars',
+          '--ignore-gpu-blacklist',
+          '--metrics-recording-only',
+          '--mute-audio',
+          '--no-default-browser-check',
+          '--no-first-run',
+          '--no-pings',
+          '--no-sandbox',
+          '--no-zygote',
+          '--password-store=basic',
+          '--use-gl=swiftshader',
+          '--use-mock-keychain',
+          '--no-sandbox',
+          '--disable-setuid-sandbox',
+          '-wait-for-browser',
+        ];
+        if (!headless) args.push('--start-maximized');
+        const launchOptions = {
+          headless,
+          devtools,
+          timeout: 60000,
+          defaultViewport: null,
+          args,
+          slowMo:
+            (_b = (_a = this.option) === null || _a === void 0 ? void 0 : _a.slowMo) !== null && _b !== void 0 ? _b : 0,
+        };
+        if (!this.option.browser || this.option.browser === BrowserEnum_1.BrowserEnum.chromium) {
+          launchOptions.product = 'chrome';
+        } else {
+          throw new TypeError('No momento este navegador não é suportado.');
+        }
+        this.browser = yield puppeteer_1.default.launch(launchOptions);
+        this.page = yield this.getLastPage();
+        yield this.initObservableDialog();
+        yield this.goTo(this.option.url);
+        if (this.option.browser === BrowserEnum_1.BrowserEnum.firefox) {
+          yield this.page.click('body');
+        }
+        if (!headless && this.option.browser !== BrowserEnum_1.BrowserEnum.firefox) {
+          yield this.page.removeAllListeners('request');
+          yield this.page.setRequestInterception(
+            (_c = this.option.loadMinimalist) !== null && _c !== void 0 ? _c : false,
+          );
+          if (this.option.loadMinimalist) {
+            this.page.on('request', (req) => {
+              if (
+                req.resourceType() === 'stylesheet' ||
+                req.resourceType() === 'font' ||
+                req.resourceType() === 'image'
+              ) {
+                req.abort();
+              } else {
+                req.continue();
+              }
+            });
+          }
+        }
+      }
+    });
+  }
+  reloadPage() {
+    return __awaiter(this, void 0, void 0, function* () {
+      if (!this.option.blockReloadPage) {
+        yield this.page.reload({
+          timeout: 240000,
+        });
+      }
+    });
+  }
+  blockReloadPage(block) {
+    this.option.blockReloadPage = block;
+  }
+  waitForXpathIsNotVisible(xPath, waitFor = 2000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      let isVisible = true;
+      while (isVisible) {
+        yield (0, delay_1.delay)(waitFor);
+        isVisible = yield this.existElement(xPath);
+      }
+    });
+  }
+  displayNone(xpath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xpath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.evaluate((input) => {
+            input.style.display = 'none';
+          });
+        }),
+      );
+    });
+  }
+  displayVisible(xpath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xpath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.evaluate((input) => {
+            input.style.removeProperty('display');
+          });
+        }),
+      );
+    });
+  }
+  getTextContentOnElement(xPath, delayUntil = 1000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page
+        .waitForXPath(xPath, {
+          visible: true,
+          timeout: delayUntil,
+        })
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            return yield el
+              .evaluate((div) => {
+                return div.textContent;
+              })
+              .catch((e) => {
+                throw e;
+              });
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  getTextContentOnInputElement(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page
+        .waitForXPath(xPath, {
+          visible: true,
+          timeout: 1000,
+        })
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            return yield el
+              .evaluate((input) => {
+                return input.value;
+              })
+              .catch((e) => {
+                throw e;
+              });
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  existElementOnList(xPathList, containsMessageError = [], delayUntil = 5000) {
+    return new Promise((resolve) =>
+      __awaiter(this, void 0, void 0, function* () {
+        let elementExist = false;
+        for (const xPath of xPathList.values()) {
+          if (yield this.existElement(xPath, delayUntil)) {
+            const message = yield this.getTextContentOnElement(xPath);
+            if (containsMessageError.length === 0 || containsMessageError.indexOf(message) >= 0) {
+              elementExist = true;
+              break;
+            }
+          }
+        }
+        resolve(elementExist);
+      }),
+    );
+  }
+  existElement(xPath, delayUntil = 5000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield this.page
+            .waitForXPath(xPath, { timeout: delayUntil, visible: true })
+            .then(() =>
+              __awaiter(this, void 0, void 0, function* () {
+                return yield resolve(true);
+              }),
+            )
+            .catch(() =>
+              __awaiter(this, void 0, void 0, function* () {
+                return yield resolve(false);
+              }),
+            );
+        }),
+      );
+    });
+  }
+  waitForDialogAlert(ms = 5000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          let currentTimeOut = 0;
+          const timeout = setTimeout(
+            () =>
+              __awaiter(this, void 0, void 0, function* () {
+                currentTimeOut += 300;
+                if (this.mensagemAlert || currentTimeOut >= ms) {
+                  resolve(true);
+                  clearTimeout(timeout);
+                }
+              }),
+            300,
+          );
+        }),
+      );
+    });
+  }
+  waitForDialogAlert2(ms = 5000, xPathSuccess) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const val = 301;
+          let aux = 0;
+          while (aux <= ms) {
+            yield (0, delay_1.delay)(val);
+            if (xPathSuccess) {
+              if (!(yield this.existElement(xPathSuccess))) {
+                resolve(true);
+              }
+            }
+            if (this.mensagemAlert) {
+              aux = ms;
+              resolve(true);
+            } else {
+              aux += val;
+            }
+          }
+          resolve(false);
+        }),
+      );
+    });
+  }
+  alert(msg) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.evaluate((value) => {
+        alert(value);
+      }, msg);
+    });
+  }
+  scrollPageAxisXY(x = 0, y = 0) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.evaluate(() => {
+        scrollTo(x !== 0 ? x : 0, y !== 0 ? y : 0);
+      });
+    });
+  }
+  keyboardPress(xPath, optionKey) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.focus();
+          yield el.press(optionKey);
+        }),
+      );
+    });
+  }
+  selectorInputValue(
+    idSelector,
+    keys,
+    displayNone = false,
+    waitFor = 301,
+    hardValidation = true,
+    delayHardValidation = 2000,
+  ) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield (0, delay_1.delay)(waitFor);
+      yield this.page
+        .waitForSelector(idSelector, {
+          visible: true,
+          hidden: false,
+        })
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            yield el.select(keys);
+            if (hardValidation) {
+              yield (0, delay_1.delay)(delayHardValidation);
+              const current = yield el.evaluate((select) => {
+                return select.value;
+              });
+              if (current !== keys) {
+                yield this.selectorInputValue(idSelector, keys, displayNone, waitFor, hardValidation);
+              }
+            }
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  verifyFrameIsOpened(identifier) {
+    return new Promise((resolve) =>
+      __awaiter(this, void 0, void 0, function* () {
+        let frame = yield this.page.frames().find((f) => f.name() === identifier);
+        if (!frame) {
+          const element = yield this.page.$(identifier);
+          if (element) {
+            frame = yield element.contentFrame();
+          }
+        }
+        resolve(!!frame);
+      }),
+    );
+  }
+  click(xPath, waitFor = 301) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield (0, delay_1.delay)(waitFor);
+      if (yield this.existElement(xPath)) {
+        yield this.page.waitForXPath(xPath).then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            yield el.focus();
+            yield el.click({ delay: 20 });
+          }),
+        );
+      }
+    });
+  }
+  getTable(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          return el.evaluate((table) => {
+            return table.outerHTML;
+          });
+        }),
+      );
+    });
+  }
+  getValueCheckbox(xpath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page
+        .waitForXPath(xpath)
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            return yield el.evaluate((input) => {
+              return input.checked;
+            });
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  getHrefFromAnchor(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          return yield el.evaluate((anchor) => {
+            return anchor.href;
+          });
+        }),
+      );
+    });
+  }
+  getSrcFromImage(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          return yield el.evaluate((image) => {
+            return image.src;
+          });
+        }),
+      );
+    });
+  }
+  getTitleFromElement(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          return yield el.evaluate((image) => {
+            return image.title;
+          });
+        }),
+      );
+    });
+  }
+  sendKeys(xPath, keys, displayNone = false, hardValidation = false) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.focus();
+          if (this.option.browser === BrowserEnum_1.BrowserEnum.firefox) {
+            yield (0, delay_1.delay)(301);
+          }
+          yield el.type(keys, { delay: 20 });
+          yield el.evaluate((inp, elDisplayNone) => {
+            if (elDisplayNone) {
+              inp.style.color = 'transparent';
+            }
+          }, displayNone);
+          if (hardValidation) {
+            yield el.press('Tab');
+            yield (0, delay_1.delay)(2000);
+            yield el.focus();
+            const current = yield this.getTextContentOnInputElement(xPath);
+            if (current !== keys) {
+              yield this.clearInput(xPath);
+              yield this.sendKeys(xPath, keys, displayNone, hardValidation);
+            }
+          }
+        }),
+      );
+    });
+  }
+  getMessageAlert(xPathSuccess, waitDialog = 60000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.waitForDialogAlert2(waitDialog, xPathSuccess);
+      return new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const msg = this.mensagemAlert;
+          this.mensagemAlert = null;
+          resolve(msg);
+        }),
+      );
+    });
+  }
+  waitElementIsVisible(xPath, delayUntil = 600000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const ii = 301;
+          let aux = 0;
+          while (aux <= delayUntil) {
+            yield (0, delay_1.delay)(ii);
+            if (yield this.existElement(xPath, ii)) {
+              resolve(true);
+            } else {
+              aux += ii * 2;
+            }
+          }
+          resolve(false);
+        }),
+      );
+    });
+  }
+  checkOnCheckBox(xPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xPath).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.evaluate((checkBox) => {
+            checkBox.click();
+          });
+        }),
+      );
+    });
+  }
+  waitElementWhileIsVisible(xPath, delayUntil = 60000) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const ii = 301;
+          let aux = 0;
+          while (aux <= delayUntil) {
+            yield (0, delay_1.delay)(ii);
+            if (yield this.existElement(xPath, ii)) {
+              aux += ii * 2;
+            } else {
+              resolve(true);
+            }
+          }
+          resolve(false);
+        }),
+      );
+    });
+  }
+  selectorInputValueForXpath(xPathSelector, keys, displayNone = false, waitFor = 301) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield (0, delay_1.delay)(waitFor);
+      yield this.page
+        .waitForXPath(xPathSelector, {
+          visible: true,
+          hidden: false,
+        })
+        .then((el) =>
+          __awaiter(this, void 0, void 0, function* () {
+            yield el.evaluate(
+              (input, value, elDisplayNone) => {
+                input.value = value;
+                if (elDisplayNone) {
+                  input.style.color = 'transparent';
+                }
+              },
+              keys,
+              displayNone,
+            );
+          }),
+        )
+        .catch((e) => {
+          throw e;
+        });
+    });
+  }
+  getOptionOnSelect(idSelect) {
+    return __awaiter(this, void 0, void 0, function* () {
+      return new Promise((resolve) =>
+        __awaiter(this, void 0, void 0, function* () {
+          const options = yield this.page.evaluate((value) => {
+            const optionsResult = [];
+            const seletor = document.getElementById(value);
+            if (seletor) {
+              for (const option of seletor.options) {
+                optionsResult.push(option.value);
+              }
+              return optionsResult;
+            }
+          }, idSelect);
+          resolve(options !== null && options !== void 0 ? options : []);
+        }),
+      ).catch((e) => {
+        throw e;
+      });
+    });
+  }
+  clickOnMenuHover(xPathMenuUl, xPathClick) {
+    return __awaiter(this, void 0, void 0, function* () {
+      yield this.page.waitForXPath(xPathMenuUl).then((el) =>
+        __awaiter(this, void 0, void 0, function* () {
+          yield el.evaluate((input) => {
+            input.style.display = 'block';
+          });
+        }),
+      );
+      yield this.click(xPathClick);
+    });
+  }
+}
+exports.KoalaScrappingDom = KoalaScrappingDom;
+//# sourceMappingURL=KoalaScrappingDom.js.map