@koalarx/scrapping 2.0.0 → 2.0.2

package/@types/browser-config.d.ts

@@ -0,0 +1,8 @@
+export interface BrowserConfig {
+    headless?: boolean;
+    proxy?: string;
+    minimalist?: boolean;
+    slowMo?: number;
+    downloadFolderPath?: string;
+    screenshotFolderPath?: string;
+}

package/@types/browser-config.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/@types/dom-options.d.ts

@@ -0,0 +1,4 @@
+export interface DOMOptions {
+    downloadFolderPath?: string;
+    screenshotFolderPath?: string;
+}

package/@types/dom-options.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/@types/get-datatable-options.d.ts

@@ -0,0 +1,9 @@
+export interface GetDatatableOptions {
+    withPagination?: {
+        nextButtonSelector: string;
+    };
+    infiniteScroll?: {
+        scrollContainerSelector: string;
+    };
+    limit?: number;
+}

package/@types/get-datatable-options.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/Browser.d.ts

@@ -0,0 +1,11 @@
+import type { BrowserConfig } from './@types/browser-config';
+import { Page } from './Page';
+export declare class Browser {
+    private config;
+    private _browser?;
+    private _page?;
+    constructor(config: BrowserConfig);
+    get page(): Page;
+    init(): Promise<this>;
+    close(): Promise<void>;
+}

package/Browser.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Browser = void 0;
+const puppeteer_1 = require("puppeteer");
+const args_1 = require("./constants/args");
+const Page_1 = require("./Page");
+const node_fs_1 = require("node:fs");
+class Browser {
+    config;
+    _browser;
+    _page;
+    constructor(config) {
+        this.config = config;
+    }
+    get page() {
+        if (!this._page) {
+            throw new Error('DOM is not lauched. Certificate you call init method.');
+        }
+        return this._page;
+    }
+    async init() {
+        const { headless, proxy, minimalist, slowMo } = this.config;
+        const downloadPath = this.config.downloadFolderPath ?? './downloads';
+        if (!(0, node_fs_1.existsSync)(downloadPath)) {
+            (0, node_fs_1.mkdirSync)(downloadPath);
+        }
+        const args = args_1.BROWSER_ARGS.concat(proxy ? [`--proxy-server=${proxy}`] : []).concat(headless ? [] : ['--start-maximized']);
+        const browser = await puppeteer_1.default.launch({
+            args,
+            headless,
+            defaultViewport: null,
+            slowMo,
+            downloadBehavior: {
+                policy: 'allow',
+                downloadPath,
+            },
+        });
+        const page = await browser.pages().then((pages) => pages[0]);
+        if (minimalist) {
+            page.removeAllListeners('request');
+            await page.setRequestInterception(true);
+            page.on('request', (req) => {
+                if (req.resourceType() === 'stylesheet' ||
+                    req.resourceType() === 'font' ||
+                    req.resourceType() === 'image') {
+                    req.abort();
+                }
+                else {
+                    req.continue();
+                }
+            });
+        }
+        this._browser = browser;
+        this._page = new Page_1.Page(page);
+        return this;
+    }
+    async close() {
+        if (!this._browser) {
+            throw new Error('Browser is not lauched. Certificate you call init method.');
+        }
+        await this._page.close();
+        await this._browser.close();
+    }
+}
+exports.Browser = Browser;

package/Dom.d.ts

@@ -0,0 +1,21 @@
+import type { KeyInput, Page } from 'puppeteer';
+import { Frame as PuppeteerFrame } from 'puppeteer';
+import type { DOMOptions } from './@types/dom-options';
+import type { GetDatatableOptions } from './@types/get-datatable-options';
+export declare class DOM {
+    private readonly _page;
+    private readonly _options?;
+    constructor(_page: Page | PuppeteerFrame, _options?: DOMOptions);
+    protected get page(): Page;
+    close(): Promise<void>;
+    screenshot(): Promise<void>;
+    pressKey(key: KeyInput, combine?: KeyInput): Promise<void>;
+    goTo(url: string): Promise<void>;
+    fill(selector: string, value: string): Promise<void>;
+    click(selector: string): Promise<void>;
+    focus(selector: string): Promise<void>;
+    content(selector: string): Promise<string[]>;
+    getDatatable<T = any>(selector: string, options?: GetDatatableOptions): Promise<T[]>;
+    waitNavigation(): Promise<void>;
+    getDownloadedFiles(): Promise<Buffer<ArrayBufferLike>[]>;
+}

package/Dom.js

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DOM = void 0;
+const utils_1 = require("@koalarx/utils");
+const html_table_to_json_1 = require("html-table-to-json");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const puppeteer_1 = require("puppeteer");
+class DOM {
+    _page;
+    _options;
+    constructor(_page, _options) {
+        this._page = _page;
+        this._options = _options;
+    }
+    get page() {
+        return this._page instanceof puppeteer_1.Frame ? this._page.page() : this._page;
+    }
+    async close() {
+        await this.page.close();
+    }
+    async screenshot() {
+        const folderPath = this._options?.screenshotFolderPath ?? './screenshots';
+        if (!(0, node_fs_1.existsSync)(folderPath)) {
+            (0, node_fs_1.mkdirSync)(folderPath);
+        }
+        const screenshotPath = node_path_1.default.join(folderPath, `${new utils_1.KlDate().format('ddMMyyyyHHmmss')}.jpg`);
+        await this.page.screenshot({ path: screenshotPath });
+    }
+    async pressKey(key, combine) {
+        if (combine) {
+            await this.page.keyboard.down(combine);
+        }
+        await this.page.keyboard.press(key);
+    }
+    async goTo(url) {
+        await this._page.goto(url, { waitUntil: 'networkidle2' });
+    }
+    async fill(selector, value) {
+        await this._page.locator(selector).scroll();
+        await this._page.focus(selector);
+        await this._page.locator(selector).fill(value);
+    }
+    async click(selector) {
+        await this._page.locator(selector).scroll();
+        await this._page.click(selector);
+    }
+    async focus(selector) {
+        await this._page.locator(selector).scroll();
+        await this._page.focus(selector);
+    }
+    async content(selector) {
+        await this._page.locator(selector).scroll();
+        return this._page.$$eval(selector, (elements) => {
+            return elements.map((el) => el.innerText.trim());
+        });
+    }
+    async getDatatable(selector, options) {
+        const table = await this._page.$eval(selector, (table) => table.outerHTML);
+        const tableData = html_table_to_json_1.default.parse(table).results[0];
+        const result = tableData.map((row) => {
+            const rowData = {};
+            Object.keys(row).forEach((key) => {
+                const value = row[key];
+                const numberValue = Number(value);
+                rowData[(0, utils_1.toCamelCase)(key)] = isNaN(numberValue) ? value : numberValue;
+            });
+            return rowData;
+        });
+        if (options?.withPagination) {
+            const nextButtonSelector = options.withPagination.nextButtonSelector;
+            const nextButton = await this._page.$(nextButtonSelector);
+            if (nextButton &&
+                !(await nextButton.evaluate((btn) => btn.classList.contains('disabled') || btn.hasAttribute('disabled')))) {
+                await nextButton.click();
+                const nextPageData = await this.getDatatable(selector, options);
+                result.push(...nextPageData);
+            }
+        }
+        return result;
+    }
+    async waitNavigation() {
+        await this._page
+            .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
+            .catch(() => null);
+    }
+    async getDownloadedFiles() {
+        const downloadPath = this._options?.downloadFolderPath ?? './downloads';
+        if ((0, node_fs_1.existsSync)(downloadPath)) {
+            let contentDir = [];
+            do {
+                await (0, utils_1.delay)(1000);
+                contentDir = (0, node_fs_1.readdirSync)(downloadPath);
+            } while (contentDir.filter((filepath) => filepath.indexOf('.crdownload') >= 0)
+                .length > 0);
+            const files = [];
+            contentDir.forEach((filepath) => files.push((0, node_fs_1.readFileSync)(`${downloadPath}/${filepath}`)));
+            return files;
+        }
+        return [];
+    }
+}
+exports.DOM = DOM;

package/Frame.d.ts

@@ -0,0 +1,5 @@
+import { Frame as PuppeteerFrame } from 'puppeteer';
+import { DOM } from './Dom';
+export declare class Frame extends DOM {
+    constructor(frame: PuppeteerFrame);
+}

package/Frame.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Frame = void 0;
+const Dom_1 = require("./Dom");
+class Frame extends Dom_1.DOM {
+    constructor(frame) {
+        super(frame);
+    }
+}
+exports.Frame = Frame;

package/Page.d.ts

@@ -0,0 +1,8 @@
+import type { Page as PuppeteerPage } from 'puppeteer';
+import { DOM } from './Dom';
+import { Frame } from './Frame';
+export declare class Page extends DOM {
+    constructor(page: PuppeteerPage);
+    getFrameByURL(url: string): Promise<Frame>;
+    getFrameByName(name: string): Promise<Frame>;
+}

package/Page.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Page = void 0;
+const Dom_1 = require("./Dom");
+const Frame_1 = require("./Frame");
+class Page extends Dom_1.DOM {
+    constructor(page) {
+        super(page);
+    }
+    async getFrameByURL(url) {
+        const frames = this.page.frames();
+        const frame = frames.find((frame) => frame.url() === url);
+        if (!frame) {
+            throw new Error(`Frame with URL "${url}" not found`);
+        }
+        return new Frame_1.Frame(frame);
+    }
+    async getFrameByName(name) {
+        const frames = this.page.frames();
+        let frame;
+        for (const f of frames) {
+            const frameElement = await f.frameElement();
+            if (!frameElement) {
+                continue;
+            }
+            const frameName = await frameElement.evaluate((el) => el.getAttribute('name'));
+            if (frameName === name) {
+                frame = f;
+                break;
+            }
+            const frameId = await frameElement.evaluate((el) => el.getAttribute('id'));
+            if (frameId === name) {
+                frame = f;
+                break;
+            }
+        }
+        if (!frame) {
+            throw new Error(`Frame with name "${name}" not found`);
+        }
+        return new Frame_1.Frame(frame);
+    }
+}
+exports.Page = Page;

package/constants/args.d.ts

@@ -0,0 +1,2 @@
+import { KlArray } from '@koalarx/utils';
+export declare const BROWSER_ARGS: KlArray<string>;

package/constants/args.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BROWSER_ARGS = void 0;
+const utils_1 = require("@koalarx/utils");
+exports.BROWSER_ARGS = new utils_1.KlArray([
+    '--autoplay-policy=user-gesture-required',
+    '--disable-background-networking',
+    '--disable-background-timer-throttling',
+    '--disable-backgrounding-occluded-windows',
+    '--disable-breakpad',
+    '--disable-client-side-phishing-detection',
+    '--disable-component-update',
+    '--disable-default-apps',
+    '--disable-dev-shm-usage',
+    '--disable-domain-reliability',
+    '--disable-features=AudioServiceOutOfProcess',
+    '--disable-hang-monitor',
+    '--disable-ipc-flooding-protection',
+    '--disable-notifications',
+    '--disable-offer-store-unmasked-wallet-cards',
+    '--disable-popup-blocking',
+    '--disable-print-preview',
+    '--disable-prompt-on-repost',
+    '--disable-renderer-backgrounding',
+    '--disable-setuid-sandbox',
+    '--disable-speech-api',
+    '--disable-sync',
+    '--hide-scrollbars',
+    '--ignore-gpu-blacklist',
+    '--metrics-recording-only',
+    '--mute-audio',
+    '--no-default-browser-check',
+    '--no-first-run',
+    '--no-pings',
+    '--no-sandbox',
+    '--no-zygote',
+    '--password-store=basic',
+    '--use-gl=swiftshader',
+    '--use-mock-keychain',
+    '--disable-setuid-sandbox',
+    '-wait-for-browser',
+]);

package/package.json

@@ -1,9 +1,13 @@
 {
-  "version": "2.0.0",
+  "version": "2.0.2",
   "name": "@koalarx/scrapping",
   "type": "module",
   "scripts": {
-    "build": "tsc -p tsconfig.build.json && cp README.md package.json dist"
+    "build": "tsc -p tsconfig.build.json && cp README.md package.json dist",
+    "prepare:hotfix": "bun pm version patch --no-git-tag-version",
+    "prepare:feature": "bun pm version minor --no-git-tag-version",
+    "prepare:release": "bun pm version major --no-git-tag-version",
+    "prepare:only-tag": "echo 'Creating tag without version bump'"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
@@ -24,4 +28,4 @@
     "html-table-to-json": "^1.0.0",
     "puppeteer": "^24.37.2"
   }
-}
+}

package/.github/workflows/npm-publish.yml

@@ -1,32 +0,0 @@
-name: NPM PUBLISH
-on:
-  push:
-    branches: [main]
-jobs:
-  build:
-    name: "npm publish library"
-    runs-on: ubuntu-latest
-    steps:
-      - name: 🚚 Get latest code
-        uses: actions/checkout@v3
-        with:
-          ref: main
-
-      - name: Setup Bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: latest
-
-      - name: 🔨 Install Dependencies
-        run: bun ci
-
-      - name: 🔨 Build Project
-        run: bun run build
-
-      - name: 🔑 Configure NPM
-        run: npm config set "//registry.npmjs.org/:_authToken" "${{ secrets.NPM_TOKEN }}"
-      
-      - name: 📚 Publish @koalarx/nest
-        run: npm publish
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.prettierrc.json

@@ -1,9 +0,0 @@
-{
-  "printWidth": 80,
-  "tabWidth": 2,
-  "singleQuote": true,
-  "trailingComma": "all",
-  "arrowParens": "always",
-  "semi": false,
-  "endOfLine": "lf"
-}

package/.vscode/launch.json

@@ -1,13 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "type": "bun",
-      "request": "launch",
-      "name": "Debug Bun File",
-      "program": "${file}", // This runs the currently active file
-      "cwd": "${workspaceFolder}",
-      "stopOnEntry": false // Set to true to pause on the first line
-    },
-  ]
-}

package/.vscode/mcp.json

@@ -1,10 +0,0 @@
-{
-  "servers": {
-    "koala-nest-docs": {
-      "command": "bunx",
-      "args": [
-        "@koalarx/mcp-server"
-      ]
-    }
-  }
-}

package/.vscode/settings.json

@@ -1,151 +0,0 @@
-{
-  "material-icon-theme.activeIconPack": "nest",
-  "editor.fontFamily": "Fira Code",
-  "editor.fontSize": 15,
-  "editor.fontLigatures": true,
-  "emmet.syntaxProfiles": {
-    "javascript": "jsx"
-  },
-  "workbench.startupEditor": "newUntitledFile",
-  "javascript.suggest.autoImports": true,
-  "javascript.updateImportsOnFileMove.enabled": "always",
-  "editor.rulers": [80, 120],
-  "extensions.ignoreRecommendations": true,
-  "typescript.tsserver.log": "off",
-  "files.associations": {
-    ".sequelizerc": "javascript",
-    ".stylelintrc": "json",
-    "*.tsx": "typescriptreact",
-    ".env.*": "dotenv",
-    ".prettierrc": "json"
-  },
-  "editor.parameterHints.enabled": false,
-  "editor.renderLineHighlight": "gutter",
-  "editor.lineHeight": 26,
-  "editor.suggestSelection": "first",
-  "explorer.confirmDelete": false,
-  "gitlens.codeLens.recentChange.enabled": false,
-  "terminal.integrated.showExitAlert": false,
-  "[prisma]": {
-    "editor.formatOnSave": true
-  },
-  "typescript.suggest.autoImports": true,
-  "terminal.integrated.env.osx": {
-    "FIG_NEW_SESSION": "1"
-  },
-  "workbench.editor.labelFormat": "short",
-  "editor.acceptSuggestionOnCommitCharacter": false,
-  "explorer.compactFolders": false,
-  "git.enableSmartCommit": true,
-  "editor.accessibilitySupport": "off",
-  "explorer.confirmDragAndDrop": false,
-  "terminal.integrated.fontSize": 14,
-  "editor.semanticHighlighting.enabled": false,
-  "breadcrumbs.enabled": true,
-  "gitlens.codeLens.authors.enabled": false,
-  "editor.tabSize": 2,
-  "files.exclude": {
-    "**/CVS": true,
-    "**/.DS_Store": true,
-    "**/.hg": true,
-    "**/.svn": true,
-    "**/.git": true
-  },
-  "gitlens.codeLens.enabled": false,
-  "workbench.iconTheme": "material-icon-theme",
-  "material-icon-theme.languages.associations": {
-    "dotenv": "tune"
-  },
-  "material-icon-theme.files.associations": {
-    "vitest.config.e2e.ts": "vitest",
-    "*.e2e-spec.ts": "test-js",
-    "*.dto.ts": "diff",
-    "*.handler.ts": "esbuild",
-    "*.request.ts": "log",
-    "*.schema.ts": "scheme",
-    "*.response.ts": "conduct",
-    "*.validator.ts": "commitlint"
-  },
-  "material-icon-theme.folders.associations": {
-    "adapters": "contract",
-    "grpc": "pipe",
-    "kube": "kubernetes",
-    "main": "lib",
-    "websockets": "pipe",
-    "implementations": "core",
-    "protos": "pipe",
-    "entities": "class",
-    "kafka": "pipe",
-    "use-cases": "functions",
-    "migrations": "tools",
-    "schemas": "class",
-    "useCases": "functions",
-    "eslint-config": "tools",
-    "typeorm": "database",
-    "_shared": "shared",
-    "mappers": "meta",
-    "fakes": "mock",
-    "modules": "components",
-    "subscribers": "messages",
-    "domain": "class",
-    "protocols": "contract",
-    "infra": "app",
-    "view-models": "views",
-    "presentation": "template",
-    "dtos": "typescript",
-    "http": "container",
-    "providers": "include",
-    "factories": "class",
-    "repositories": "mappings",
-    "filters": "pipe"
-  },
-  "[javascript]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode",
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-      "source.fixAll.eslint": "explicit"
-    }
-  },
-  "[typescript]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode",
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-      "source.fixAll.eslint": "explicit"
-    }
-  },
-  "prettier.documentSelectors": ["**/*.ts", "**/*.tsx", "**/*.js"],
-  "terminal.integrated.defaultProfile.windows": "Command Prompt",
-  "git.confirmSync": false,
-  "files.eol": "\n",
-  "workbench.productIconTheme": "Default",
-  "npm.scriptExplorerAction": "run",
-  "npm.packageManager": "npm",
-  "editor.wordWrap": "on",
-  "editor.wrappingIndent": "indent",
-  "chat.mcp.serverSampling": {
-    "globo-seguros-api/.vscode/mcp.json: koala-nest-docs": {
-      "allowedModels": [
-        "copilot/claude-sonnet-4.5",
-        "copilot/auto",
-        "copilot/claude-haiku-4.5",
-        "copilot/claude-opus-4.5",
-        "copilot/claude-sonnet-4",
-        "copilot/gemini-2.5-pro",
-        "copilot/gemini-3-flash-preview",
-        "copilot/gemini-3-pro-preview",
-        "copilot/gpt-4.1",
-        "copilot/gpt-4o",
-        "copilot/gpt-5",
-        "copilot/gpt-5-mini",
-        "copilot/gpt-5-codex",
-        "copilot/gpt-5.1",
-        "copilot/gpt-5.1-codex",
-        "copilot/gpt-5.1-codex-max",
-        "copilot/gpt-5.1-codex-mini",
-        "copilot/gpt-5.2",
-        "copilot/gpt-5.2-codex",
-        "copilot/grok-code-fast-1"
-      ]
-    }
-  }
-}

package/.vscode/tasks.json

@@ -1,17 +0,0 @@
-{
-  "version": "2.0.0",
-  "tasks": [
-    {
-      "label": "bun: start:debug",
-      "type": "shell",
-      "command": "bun",
-      "args": ["run", "start:debug"],
-      "group": {
-        "kind": "build",
-        "isDefault": true
-      },
-      "problemMatcher": [],
-      "isBackground": true
-    }
-  ]
-}

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Igor D. Rangel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.