@koalarx/scrapping 2.0.0 → 2.0.2

package/bunfig.toml

@@ -1,7 +0,0 @@
-[test]
-root = "lib/test"
-timeoutMs = 60000
-preload = ["./lib/test/setup.ts"]
-
-[loader]
-".ts" = "tsx"

package/eslint.config.mts

@@ -1,83 +0,0 @@
-import js from '@eslint/js'
-import vitest from '@vitest/eslint-plugin'
-import globals from 'globals'
-import tseslint from 'typescript-eslint'
-import prettierConfig from 'eslint-config-prettier'
-import prettierPlugin from 'eslint-plugin-prettier'
-
-export default [
-  {
-    ignores: [
-      '**/*.json',
-      'node_modules',
-      'dist',
-      'coverage',
-      'prisma/generated',
-      '**/*.yaml',
-    ],
-  },
-  js.configs.recommended,
-  ...tseslint.configs.recommended,
-  prettierConfig,
-  {
-    files: ['src/**/*.ts'],
-    languageOptions: {
-      parser: tseslint.parser,
-      parserOptions: {
-        ecmaVersion: 'latest',
-        sourceType: 'module',
-      },
-      globals: globals.node,
-    },
-    plugins: {
-      '@typescript-eslint': tseslint.plugin,
-      prettier: prettierPlugin,
-    },
-    rules: {
-      'prettier/prettier': [
-        'warn',
-        {
-          printWidth: 80,
-          tabWidth: 2,
-          singleQuote: true,
-          trailingComma: 'all',
-          arrowParens: 'always',
-          semi: false,
-          endOfLine: 'lf',
-        },
-      ],
-      'no-unused-vars': 'off',
-      'no-useless-constructor': 'off',
-      'no-new': 'off',
-      'no-use-before-define': 'off',
-      '@typescript-eslint/no-unused-vars': 'warn',
-      '@typescript-eslint/no-explicit-any': 'off',
-      '@typescript-eslint/no-unsafe-function-type': 'off',
-      '@typescript-eslint/ban-ts-comment': 'off',
-      '@typescript-eslint/no-unsafe-assignment': 'off',
-      '@typescript-eslint/no-unsafe-member-access': 'off',
-      '@typescript-eslint/no-unsafe-call': 'off',
-      '@typescript-eslint/no-require-imports': 'off',
-      'lines-between-class-members': [
-        'warn',
-        'always',
-        { exceptAfterSingleLine: true },
-      ],
-    },
-  },
-  {
-    files: ['src/**/*.spec.ts', 'src/test/setup-e2e.ts'],
-    languageOptions: {
-      globals: {
-        ...vitest.environments.env.globals,
-      },
-    },
-    plugins: {
-      vitest,
-    },
-    rules: {
-      ...vitest.configs.recommended.rules,
-      'vitest/no-conditional-expect': 'off',
-    },
-  },
-]

package/lib/core/@types/browser-config.ts

@@ -1,8 +0,0 @@
-export interface BrowserConfig {
-  headless?: boolean
-  proxy?: string
-  minimalist?: boolean
-  slowMo?: number
-  downloadFolderPath?: string
-  screenshotFolderPath?: string
-}

package/lib/core/@types/dom-options.ts

@@ -1,4 +0,0 @@
-export interface DOMOptions {
-  downloadFolderPath?: string
-  screenshotFolderPath?: string
-}

package/lib/core/@types/get-datatable-options.ts

@@ -1,9 +0,0 @@
-export interface GetDatatableOptions {
-  withPagination?: {
-    nextButtonSelector: string
-  }
-  infiniteScroll?: {
-    scrollContainerSelector: string
-  }
-  limit?: number
-}

package/lib/core/Browser.ts

@@ -1,81 +0,0 @@
-import puppeteer, { Browser as PuppeteerBrowser } from 'puppeteer'
-import type { BrowserConfig } from './@types/browser-config'
-import { BROWSER_ARGS } from './constants/args'
-import { Page } from './Page'
-import { existsSync, mkdirSync } from 'node:fs'
-
-export class Browser {
-  private _browser?: PuppeteerBrowser
-  private _page?: Page
-
-  constructor(private config: BrowserConfig) {}
-
-  get page() {
-    if (!this._page) {
-      throw new Error('DOM is not lauched. Certificate you call init method.')
-    }
-
-    return this._page
-  }
-
-  async init() {
-    const { headless, proxy, minimalist, slowMo } = this.config
-
-    const downloadPath = this.config.downloadFolderPath ?? './downloads'
-
-    if (!existsSync(downloadPath)) {
-      mkdirSync(downloadPath)
-    }
-
-    const args = BROWSER_ARGS.concat(
-      proxy ? [`--proxy-server=${proxy}`] : [],
-    ).concat(headless ? [] : ['--start-maximized'])
-
-    const browser = await puppeteer.launch({
-      args,
-      headless,
-      defaultViewport: null,
-      slowMo,
-      downloadBehavior: {
-        policy: 'allow',
-        downloadPath,
-      },
-    })
-
-    const page = await browser.pages().then((pages) => pages[0]!)
-
-    if (minimalist) {
-      page.removeAllListeners('request')
-
-      await page.setRequestInterception(true)
-
-      page.on('request', (req) => {
-        if (
-          req.resourceType() === 'stylesheet' ||
-          req.resourceType() === 'font' ||
-          req.resourceType() === 'image'
-        ) {
-          req.abort()
-        } else {
-          req.continue()
-        }
-      })
-    }
-
-    this._browser = browser
-    this._page = new Page(page)
-
-    return this
-  }
-
-  async close() {
-    if (!this._browser) {
-      throw new Error(
-        'Browser is not lauched. Certificate you call init method.',
-      )
-    }
-
-    await this._page.close()
-    await this._browser.close()
-  }
-}

package/lib/core/Dom.ts

@@ -1,144 +0,0 @@
-import { delay, KlDate, toCamelCase } from '@koalarx/utils'
-import htmlTableToJson from 'html-table-to-json'
-import { existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs'
-import path from 'node:path'
-import type { KeyInput, Page } from 'puppeteer'
-import { Frame as PuppeteerFrame } from 'puppeteer'
-import type { DOMOptions } from './@types/dom-options'
-import type { GetDatatableOptions } from './@types/get-datatable-options'
-
-export class DOM {
-  constructor(
-    private readonly _page: Page | PuppeteerFrame,
-    private readonly _options?: DOMOptions,
-  ) {}
-
-  protected get page() {
-    return this._page instanceof PuppeteerFrame ? this._page.page() : this._page
-  }
-
-  async close() {
-    await this.page.close()
-  }
-
-  async screenshot() {
-    const folderPath = this._options?.screenshotFolderPath ?? './screenshots'
-
-    if (!existsSync(folderPath)) {
-      mkdirSync(folderPath)
-    }
-
-    const screenshotPath = path.join(
-      folderPath,
-      `${new KlDate().format('ddMMyyyyHHmmss')}.jpg`,
-    )
-
-    await this.page.screenshot({ path: screenshotPath })
-  }
-
-  async pressKey(key: KeyInput, combine?: KeyInput) {
-    if (combine) {
-      await this.page.keyboard.down(combine)
-    }
-
-    await this.page.keyboard.press(key)
-  }
-
-  async goTo(url: string) {
-    await this._page.goto(url, { waitUntil: 'networkidle2' })
-  }
-
-  async fill(selector: string, value: string) {
-    await this._page.locator(selector).scroll()
-    await this._page.focus(selector)
-    await this._page.locator(selector).fill(value)
-  }
-
-  async click(selector: string) {
-    await this._page.locator(selector).scroll()
-    await this._page.click(selector)
-  }
-
-  async focus(selector: string) {
-    await this._page.locator(selector).scroll()
-    await this._page.focus(selector)
-  }
-
-  async content(selector: string) {
-    await this._page.locator(selector).scroll()
-    return this._page.$$eval(selector, (elements) => {
-      return elements.map(
-        (el) => (el as HTMLElement).innerText.trim() as string,
-      )
-    })
-  }
-
-  async getDatatable<T = any>(
-    selector: string,
-    options?: GetDatatableOptions,
-  ): Promise<T[]> {
-    const table = await this._page.$eval(selector, (table) => table.outerHTML)
-    const tableData = htmlTableToJson.parse(table).results[0]
-    const result: T[] = tableData.map((row) => {
-      const rowData: Record<string, string> = {}
-
-      Object.keys(row).forEach((key) => {
-        const value = row[key]
-        const numberValue = Number(value)
-        rowData[toCamelCase(key)] = isNaN(numberValue) ? value : numberValue
-      })
-
-      return rowData as T
-    })
-
-    if (options?.withPagination) {
-      const nextButtonSelector = options.withPagination.nextButtonSelector
-      const nextButton = await this._page.$(nextButtonSelector)
-
-      if (
-        nextButton &&
-        !(await nextButton.evaluate(
-          (btn) =>
-            btn.classList.contains('disabled') || btn.hasAttribute('disabled'),
-        ))
-      ) {
-        await nextButton.click()
-        const nextPageData = await this.getDatatable(selector, options)
-        result.push(...nextPageData)
-      }
-    }
-
-    return result
-  }
-
-  async waitNavigation() {
-    await this._page
-      .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-      .catch(() => null)
-  }
-
-  async getDownloadedFiles() {
-    const downloadPath = this._options?.downloadFolderPath ?? './downloads'
-
-    if (existsSync(downloadPath)) {
-      let contentDir = []
-
-      do {
-        await delay(1000)
-        contentDir = readdirSync(downloadPath)
-      } while (
-        contentDir.filter((filepath) => filepath.indexOf('.crdownload') >= 0)
-          .length > 0
-      )
-
-      const files: Buffer[] = []
-
-      contentDir.forEach((filepath) =>
-        files.push(readFileSync(`${downloadPath}/${filepath}`)),
-      )
-      return files
-    }
-
-    return []
-  }
-}

package/lib/core/Frame.ts

@@ -1,8 +0,0 @@
-import { Frame as PuppeteerFrame } from 'puppeteer'
-import { DOM } from './Dom'
-
-export class Frame extends DOM {
-  constructor(frame: PuppeteerFrame) {
-    super(frame)
-  }
-}

package/lib/core/Page.ts

@@ -1,56 +0,0 @@
-import type { Frame as PuppeteerFrame, Page as PuppeteerPage } from 'puppeteer'
-import { DOM } from './Dom'
-import { Frame } from './Frame'
-
-export class Page extends DOM {
-  constructor(page: PuppeteerPage) {
-    super(page)
-  }
-
-  async getFrameByURL(url: string) {
-    const frames = this.page.frames()
-    const frame = frames.find((frame) => frame.url() === url)
-
-    if (!frame) {
-      throw new Error(`Frame with URL "${url}" not found`)
-    }
-
-    return new Frame(frame)
-  }
-
-  async getFrameByName(name: string) {
-    const frames = this.page.frames()
-
-    let frame: PuppeteerFrame | undefined
-
-    for (const f of frames) {
-      const frameElement = await f.frameElement()
-
-      if (!frameElement) {
-        continue
-      }
-
-      const frameName = await frameElement.evaluate((el) =>
-        el.getAttribute('name'),
-      )
-
-      if (frameName === name) {
-        frame = f
-        break
-      }
-
-      const frameId = await frameElement.evaluate((el) => el.getAttribute('id'))
-
-      if (frameId === name) {
-        frame = f
-        break
-      }
-    }
-
-    if (!frame) {
-      throw new Error(`Frame with name "${name}" not found`)
-    }
-
-    return new Frame(frame)
-  }
-}

package/lib/core/constants/args.ts

@@ -1,40 +0,0 @@
-import { KlArray } from '@koalarx/utils'
-
-export const BROWSER_ARGS = new KlArray([
-  '--autoplay-policy=user-gesture-required',
-  '--disable-background-networking',
-  '--disable-background-timer-throttling',
-  '--disable-backgrounding-occluded-windows',
-  '--disable-breakpad',
-  '--disable-client-side-phishing-detection',
-  '--disable-component-update',
-  '--disable-default-apps',
-  '--disable-dev-shm-usage',
-  '--disable-domain-reliability',
-  '--disable-features=AudioServiceOutOfProcess',
-  '--disable-hang-monitor',
-  '--disable-ipc-flooding-protection',
-  '--disable-notifications',
-  '--disable-offer-store-unmasked-wallet-cards',
-  '--disable-popup-blocking',
-  '--disable-print-preview',
-  '--disable-prompt-on-repost',
-  '--disable-renderer-backgrounding',
-  '--disable-setuid-sandbox',
-  '--disable-speech-api',
-  '--disable-sync',
-  '--hide-scrollbars',
-  '--ignore-gpu-blacklist',
-  '--metrics-recording-only',
-  '--mute-audio',
-  '--no-default-browser-check',
-  '--no-first-run',
-  '--no-pings',
-  '--no-sandbox',
-  '--no-zygote',
-  '--password-store=basic',
-  '--use-gl=swiftshader',
-  '--use-mock-keychain',
-  '--disable-setuid-sandbox',
-  '-wait-for-browser',
-])

package/lib/test/download-file.spec.ts

@@ -1,18 +0,0 @@
-import { rmSync } from 'node:fs'
-import { TestVars } from './vars'
-
-test('Test RPA download file', async () => {
-  const page = TestVars.page
-
-  await page.goTo('https://proof.ovh.net/files/')
-  await page.click(
-    '#main > table > tbody > tr:nth-child(2) > td:nth-child(1) > a',
-  )
-
-  const files = await page.getDownloadedFiles()
-
-  expect(files.length).toBeGreaterThan(0)
-  expect(files[0]).toBeInstanceOf(Buffer)
-
-  rmSync('downloads', { recursive: true, force: true })
-}, 15000)

package/lib/test/frame-interaction.spec.ts

@@ -1,22 +0,0 @@
-import { TestVars } from './vars'
-
-test('Frame Interaction', async () => {
-  const page = TestVars.page
-  const urlFrame = 'https://pt.wikipedia.org'
-
-  await page.goTo(`https://iframetester.com/?url=${urlFrame}`)
-
-  const frame = await page.getFrameByName('iframe-window')
-
-  await frame.click('#p-search > a')
-  await frame.fill(
-    '#searchform > div > div > div.cdx-text-input.cdx-text-input--has-start-icon.cdx-text-input--status-default.cdx-search-input__text-input > input',
-    'Einstein',
-  )
-  await frame.pressKey('Enter')
-
-  const content = await frame.content('#mw-content-text p')
-
-  expect(content).toBeInstanceOf(Array)
-  expect(content.length > 0).toBeTruthy()
-})

package/lib/test/get-datatable-with-paginator.spec.ts

@@ -1,31 +0,0 @@
-import { TestVars } from './vars'
-
-interface DatatableItem {
-  name: string
-  position: string
-  office: string
-  age: string
-  startDate: string
-  salary: string
-}
-
-test('Test RPA get datatable with paginator', async () => {
-  const page = TestVars.page
-
-  await page.goTo('https://datatables.net')
-  const data = await page.getDatatable<DatatableItem>('#example', {
-    withPagination: {
-      nextButtonSelector:
-        '#example_wrapper > div:nth-child(3) > div.dt-layout-cell.dt-layout-end > div > nav > button.dt-paging-button.next',
-    },
-  })
-
-  expect(data).toBeInstanceOf(Array)
-  expect(data.length).toBeGreaterThan(10)
-  expect(data[0]).toHaveProperty('name')
-  expect(data[0]).toHaveProperty('position')
-  expect(data[0]).toHaveProperty('office')
-  expect(data[0]).toHaveProperty('age')
-  expect(data[0]).toHaveProperty('startDate')
-  expect(data[0]).toHaveProperty('salary')
-})

package/lib/test/search-wikipidea.spec.ts

@@ -1,18 +0,0 @@
-import { TestVars } from './vars'
-
-test('Test RPA search on wikipedia', async () => {
-  const page = TestVars.page
-
-  await page.goTo('https://pt.wikipedia.org')
-  await page.click('#searchInput')
-  await page.fill(
-    '#searchform > div > div > div.cdx-text-input.cdx-text-input--has-start-icon.cdx-text-input--status-default.cdx-search-input__text-input > input',
-    'Einstein',
-  )
-  await page.pressKey('Enter')
-
-  const content = await page.content('#mw-content-text p')
-
-  expect(content).toBeInstanceOf(Array)
-  expect(content.length > 0).toBeTruthy()
-})

package/lib/test/setup.ts

@@ -1,14 +0,0 @@
-import { Browser } from '../core/Browser'
-import { TestVars } from './vars'
-
-beforeAll(async () => {
-  TestVars.browser = await new Browser({
-    minimalist: false,
-    headless: false,
-  }).init()
-  TestVars.page = TestVars.browser.page
-})
-
-afterAll(async () => {
-  await TestVars.browser.close()
-})

package/lib/test/vars.ts

@@ -1,7 +0,0 @@
-import type { Browser } from '../core/Browser'
-import type { Page } from '../core/Page'
-
-export class TestVars {
-  static browser: Browser
-  static page: Page
-}