@koalarx/scrapping 2.0.3 → 2.0.4

package/Dom.d.ts

@@ -15,7 +15,9 @@ export declare class DOM {
     click(selector: string): Promise<void>;
     focus(selector: string): Promise<void>;
     content(selector: string): Promise<string[]>;
+    getText(selector: string): Promise<string>;
     getDatatable<T = any>(selector: string, options?: GetDatatableOptions): Promise<T[]>;
     waitNavigation(): Promise<void>;
+    waitIsVisible(selector: string, limitTime?: number): Promise<void>;
     getDownloadedFiles(): Promise<Buffer<ArrayBufferLike>[]>;
 }

package/Dom.js

@@ -55,19 +55,27 @@ class DOM {
             return elements.map((el) => el.innerText.trim());
         });
     }
+    async getText(selector) {
+        await this._page.locator(selector).scroll();
+        return this._page.$eval(selector, (el) => el.innerText.trim());
+    }
     async getDatatable(selector, options) {
         const table = await this._page.$eval(selector, (table) => table.outerHTML);
         const tableData = html_table_to_json_1.default.parse(table).results[0];
-        const result = tableData.map((row) => {
-            const rowData = {};
-            Object.keys(row).forEach((key) => {
-                const value = row[key];
-                const numberValue = Number(value);
-                rowData[(0, utils_1.toCamelCase)(key)] = isNaN(numberValue) ? value : numberValue;
-            });
-            return rowData;
-        });
+        let result = [];
+        if (tableData.length === 0) {
+            return result;
+        }
         if (options?.withPagination) {
+            result.push(...tableData.map((row) => {
+                const rowData = {};
+                Object.keys(row).forEach((key) => {
+                    const value = row[key];
+                    const numberValue = Number(value);
+                    rowData[(0, utils_1.toCamelCase)(key)] = isNaN(numberValue) ? value : numberValue;
+                });
+                return rowData;
+            }));
             const nextButtonSelector = options.withPagination.nextButtonSelector;
             const nextButton = await this._page.$(nextButtonSelector);
             if (nextButton &&
@@ -77,6 +85,20 @@ class DOM {
                 result.push(...nextPageData);
             }
         }
+        else if (options?.infiniteScroll) {
+            let endOfPage = false;
+            do {
+                await this._page
+                    .locator(`body > my-app > table > tbody > tr:nth-child(${tableData.length})`)
+                    .scroll();
+                await (0, utils_1.delay)(1000);
+                const nextPageData = await this.getDatatable(selector, options);
+                result.push(...nextPageData);
+                if (nextPageData.length === tableData.length) {
+                    endOfPage = true;
+                }
+            } while (!endOfPage);
+        }
         return result;
     }
     async waitNavigation() {
@@ -84,6 +106,23 @@ class DOM {
             .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
             .catch(() => null);
     }
+    async waitIsVisible(selector, limitTime = 10000) {
+        let currentTime = 0;
+        let intervalTime = 500;
+        let element = null;
+        do {
+            element = await this._page
+                .locator(selector)
+                .wait()
+                .catch(() => null);
+            if (element) {
+                break;
+            }
+            await (0, utils_1.delay)(intervalTime);
+            currentTime += intervalTime;
+        } while (!element && currentTime < limitTime);
+        await this._page.locator(selector).scroll();
+    }
     async getDownloadedFiles() {
         const downloadPath = this._options?.downloadFolderPath ?? './downloads';
         if ((0, node_fs_1.existsSync)(downloadPath)) {

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.0.3",
+  "version": "2.0.4",
   "name": "@koalarx/scrapping",
   "type": "module",
   "scripts": {