app-manager-edge-worker 1.0.4 → 1.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/handlers/previewHandler.js

@@ -287,6 +287,142 @@ export async function handleInspectElement(payload) {
 
                 const text = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
 
+                // ─── Table Detection (US-01, US-02, US-03) ───
+                let tableDetection = null;
+                const tableEl = el.closest("table");
+                if (tableEl) {
+                    const rows = Array.from(tableEl.querySelectorAll("tr"));
+                    if (rows.length > 0) {
+                        let tableSelector = "table";
+                        if (tableEl.id) {
+                            tableSelector = `table#${tableEl.id}`;
+                        } else if (tableEl.className && typeof tableEl.className === "string") {
+                            const cls = tableEl.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c));
+                            if (cls.length > 0) tableSelector = `table.${cls.slice(0, 2).join(".")}`;
+                        }
+
+                        const theadThs = Array.from(tableEl.querySelectorAll("thead th, thead td"));
+                        const firstRowCells = Array.from(rows[0].querySelectorAll("th, td"));
+                        const headerCells = theadThs.length > 0 ? theadThs : firstRowCells;
+
+                        const columns = headerCells.map((c, colIdx) => {
+                            const rawHeader = (c.innerText?.trim() || c.textContent?.trim() || "").replace(/\s+/g, " ");
+                            const normalizedKey = rawHeader.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || `column_${colIdx + 1}`;
+                            return { key: normalizedKey, label: rawHeader || `Column ${colIdx + 1}`, columnIndex: colIdx, type: "string", sampleValues: [] };
+                        });
+
+                        const sampleRows = [];
+                        for (const dr of rows.slice(1, 6)) {
+                            const cells = Array.from(dr.querySelectorAll("td, th")).map(c => c.innerText?.trim() || c.textContent?.trim() || "");
+                            const rec = {};
+                            columns.forEach((col, idx) => {
+                                rec[col.key] = cells[idx] !== undefined ? cells[idx] : "";
+                            });
+                            sampleRows.push(rec);
+                        }
+
+                        tableDetection = {
+                            isTable: true,
+                            tableSelector,
+                            hasHeaders: true,
+                            headerRowIndex: 0,
+                            totalRows: rows.length,
+                            columns,
+                            sampleRows,
+                        };
+                    }
+                }
+
+                // ─── Repeating Structure Detection (US-04, US-05, US-23) ───
+                let repeatingDetection = null;
+                if (!tableEl) {
+                    let currR = el;
+                    let depth = 0;
+                    while (currR && currR !== document.body && depth < 5) {
+                        const parent = currR.parentElement;
+                        if (parent && parent !== document.documentElement) {
+                            const tag = currR.tagName.toLowerCase();
+                            const classes = (currR.className && typeof currR.className === "string")
+                                ? currR.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
+                                : [];
+                            let selCandidate = tag;
+                            if (classes.length > 0) selCandidate = `${tag}.${classes.slice(0, 2).join(".")}`;
+
+                            const siblings = Array.from(parent.children).filter(c => {
+                                if (c.tagName.toLowerCase() !== tag) return false;
+                                if (classes.length > 0 && c.classList) {
+                                    return classes.some(cls => c.classList.contains(cls));
+                                }
+                                return true;
+                            });
+
+                            if (siblings.length >= 2) {
+                                let parentSel = parent.tagName.toLowerCase();
+                                if (parent.id) parentSel = `#${parent.id}`;
+                                else if (parent.className && typeof parent.className === "string") {
+                                    const pCls = parent.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c));
+                                    if (pCls.length > 0) parentSel = `${parentSel}.${pCls.slice(0, 2).join(".")}`;
+                                }
+                                const containerSelector = `${parentSel} > ${selCandidate}`;
+
+                                const repCard = siblings[0];
+                                const suggestedFields = [];
+
+                                const heading = repCard.querySelector("h1, h2, h3, h4, h5, .title, [class*='title'], .name, [class*='name']");
+                                if (heading) {
+                                    const tagH = heading.tagName.toLowerCase();
+                                    const clsH = heading.className && typeof heading.className === "string" ? heading.className.split(/\s+/)[0] : "";
+                                    suggestedFields.push({ key: "title", label: "Title", relativeSelector: clsH ? `.${clsH}` : tagH, attribute: "text", type: "string" });
+                                }
+
+                                const priceEl = repCard.querySelector(".price, [class*='price'], [data-price]");
+                                if (priceEl) {
+                                    const clsP = priceEl.className && typeof priceEl.className === "string" ? priceEl.className.split(/\s+/)[0] : "";
+                                    suggestedFields.push({ key: "price", label: "Price", relativeSelector: clsP ? `.${clsP}` : "[class*='price']", attribute: "text", type: "number" });
+                                }
+
+                                const imgEl = repCard.querySelector("img");
+                                if (imgEl && imgEl.getAttribute("src")) {
+                                    suggestedFields.push({ key: "image", label: "Image URL", relativeSelector: "img", attribute: "src", type: "string" });
+                                }
+
+                                const linkEl = repCard.querySelector("a[href]") || (repCard.tagName.toLowerCase() === "a" ? repCard : null);
+                                if (linkEl && linkEl.getAttribute("href")) {
+                                    suggestedFields.push({ key: "url", label: "Link", relativeSelector: repCard.tagName.toLowerCase() === "a" ? "" : "a", attribute: "href", type: "string" });
+                                }
+
+                                const sampleItems = siblings.slice(0, 3).map((item) => {
+                                    const rec = {};
+                                    suggestedFields.forEach((sf) => {
+                                        let val = "";
+                                        if (sf.relativeSelector) {
+                                            const sub = item.querySelector(sf.relativeSelector);
+                                            if (sub) {
+                                                val = sf.attribute === "text" ? (sub.innerText?.trim() || sub.textContent?.trim()) : sub.getAttribute(sf.attribute);
+                                            }
+                                        } else {
+                                            val = sf.attribute === "text" ? (item.innerText?.trim() || item.textContent?.trim()) : item.getAttribute(sf.attribute);
+                                        }
+                                        rec[sf.key] = val || "";
+                                    });
+                                    return rec;
+                                });
+
+                                repeatingDetection = {
+                                    isRepeating: true,
+                                    containerSelector,
+                                    itemCount: siblings.length,
+                                    suggestedFields,
+                                    sampleItems,
+                                };
+                                break;
+                            }
+                        }
+                        currR = currR.parentElement;
+                        depth++;
+                    }
+                }
+
                 return {
                     found: true,
                     tagName: el.tagName.toLowerCase(),
@@ -298,6 +434,8 @@ export async function handleInspectElement(payload) {
                     candidates,
                     hierarchy,
                     boundingRect,
+                    tableDetection,
+                    repeatingDetection,
                 };
             }, x, y);
 

package/src/handlers/scrapeHandler.js

@@ -1,6 +1,7 @@
 import axios from "axios";
 import * as cheerio from "cheerio";
 import localBrowserManager from "../browser.js";
+import { extractSmartCheerioText, runTransformPipeline, castValueToType } from "./transformUtils.js";
 
 /**
  * Extracts a single field value from a Puppeteer page given a strategy.
@@ -100,7 +101,7 @@ function extractCheerioField($, strategy) {
             const el = $(strategy.value);
             if (!el || el.length === 0) return null;
 
-            if (attr === "text") return el.first().text().trim();
+            if (attr === "text") return extractSmartCheerioText($, el.first());
             if (attr === "html") return el.first().html();
             return el.first().attr(attr) || null;
         }
@@ -116,7 +117,7 @@ function extractCheerioField($, strategy) {
                 try {
                     const el = $(css);
                     if (el && el.length > 0) {
-                        if (attr === "text") return el.first().text().trim();
+                        if (attr === "text") return extractSmartCheerioText($, el.first());
                         if (attr === "html") return el.first().html();
                         return el.first().attr(attr) || null;
                     }
@@ -129,7 +130,7 @@ function extractCheerioField($, strategy) {
         if (type === "text-relative") {
             const anchor = strategy.anchor || strategy.value;
             const match = $(`:contains("${anchor}")`).last();
-            if (match && match.length > 0) return match.text().trim();
+            if (match && match.length > 0) return extractSmartCheerioText($, match);
         }
 
         return null;
@@ -138,6 +139,181 @@ function extractCheerioField($, strategy) {
     }
 }
 
+/**
+ * Extracts a table collection using Cheerio.
+ */
+function extractTableCollectionCheerio($, collectionDef) {
+    if (!collectionDef) return [];
+    const selector = collectionDef.containerSelector || collectionDef.tableSelector || "table";
+    const tableEl = $(selector).first();
+    if (!tableEl || tableEl.length === 0) return [];
+
+    const hasHeaders = collectionDef.hasHeaders !== false;
+    const headerRowIndex = typeof collectionDef.headerRowIndex === "number" ? collectionDef.headerRowIndex : 0;
+    const maxItems = collectionDef.maxItems || 100;
+    const columns = collectionDef.columns || [];
+
+    const rows = [];
+    tableEl.find("tr").each((rIdx, rowEl) => {
+        const $row = $(rowEl);
+        const cells = [];
+        $row.find("th, td").each((_, cellEl) => {
+            cells.push(extractSmartCheerioText($, $(cellEl)));
+        });
+
+        if (cells.length > 0) {
+            rows.push({ rowIndex: rIdx, cells, rowEl: $row });
+        }
+    });
+
+    if (rows.length === 0) return [];
+
+    let activeColumns = [...columns];
+    if (activeColumns.length === 0) {
+        if (hasHeaders && rows[headerRowIndex]) {
+            activeColumns = rows[headerRowIndex].cells.map((headerText, colIdx) => {
+                const normalizedKey =
+                    headerText
+                        .toLowerCase()
+                        .replace(/[^a-z0-9]+/g, "_")
+                        .replace(/^_+|_+$/g, "") || `column_${colIdx + 1}`;
+                return {
+                    key: normalizedKey,
+                    label: headerText || `Column ${colIdx + 1}`,
+                    columnIndex: colIdx,
+                    type: "string",
+                    excluded: false,
+                };
+            });
+        } else {
+            activeColumns = rows[0].cells.map((_, colIdx) => ({
+                key: `column_${colIdx + 1}`,
+                label: `Column ${colIdx + 1}`,
+                columnIndex: colIdx,
+                type: "string",
+                excluded: false,
+            }));
+        }
+    }
+
+    const startRowIdx = hasHeaders ? headerRowIndex + 1 : 0;
+    const dataRows = rows.slice(startRowIdx, startRowIdx + maxItems);
+    const results = [];
+
+    for (const { cells, rowEl } of dataRows) {
+        const record = {};
+        for (let colIdx = 0; colIdx < activeColumns.length; colIdx++) {
+            const col = activeColumns[colIdx];
+            if (col.excluded) continue;
+            let rawVal = null;
+            const colSelector = col.relativeSelector || col.selector;
+
+            if (colSelector) {
+                const subEl = rowEl.find(colSelector).first();
+                if (subEl && subEl.length > 0) {
+                    const attr = col.attribute || "text";
+                    rawVal = attr === "text" ? extractSmartCheerioText($, subEl) : subEl.attr(attr) || null;
+                }
+            } else if (typeof col.columnIndex === "number" && col.columnIndex < cells.length) {
+                rawVal = cells[col.columnIndex];
+            } else if (colIdx < cells.length) {
+                rawVal = cells[colIdx];
+            }
+
+            if ((rawVal === null || rawVal === undefined || rawVal === "") && col.defaultValue !== undefined) {
+                rawVal = col.defaultValue;
+            }
+
+            // Apply column transforms
+            if (col.transforms && col.transforms.length > 0) {
+                rawVal = runTransformPipeline(rawVal, col.transforms);
+            }
+
+            // Cast to target type
+            if (col.type) {
+                rawVal = castValueToType(rawVal, col.type);
+            }
+
+            record[col.key] = rawVal;
+        }
+        results.push(record);
+    }
+
+    return results;
+}
+
+/**
+ * Extracts repeating items/cards using Cheerio.
+ */
+function extractRepeatingCollectionCheerio($, collectionDef) {
+    if (!collectionDef) return [];
+
+    let items;
+    if (collectionDef.rowSelector) {
+        if (collectionDef.containerSelector) {
+            items = $(collectionDef.containerSelector).find(collectionDef.rowSelector);
+        }
+        if (!items || items.length === 0) {
+            items = $(collectionDef.rowSelector);
+        }
+    } else if (collectionDef.containerSelector) {
+        items = $(collectionDef.containerSelector);
+    } else {
+        return [];
+    }
+
+    if (!items || items.length === 0) return [];
+
+    const maxItems = collectionDef.maxItems || 100;
+    const columns = collectionDef.columns || [];
+    const results = [];
+
+    items.slice(0, maxItems).each((_, itemEl) => {
+        const $item = $(itemEl);
+        const record = {};
+
+        for (const col of columns) {
+            if (col.excluded) continue;
+            let rawVal = null;
+            const colSelector = col.relativeSelector || col.selector;
+
+            if (colSelector) {
+                const target = $item.find(colSelector).first();
+                if (target && target.length > 0) {
+                    const attr = col.attribute || "text";
+                    rawVal = attr === "text" ? extractSmartCheerioText($, target) : target.attr(attr) || null;
+                }
+            } else {
+                const attr = col.attribute || "text";
+                rawVal = attr === "text" ? extractSmartCheerioText($, $item) : $item.attr(attr) || null;
+            }
+
+            if ((rawVal === null || rawVal === undefined || rawVal === "") && col.defaultValue !== undefined) {
+                rawVal = col.defaultValue;
+            }
+
+            // Apply column transforms
+            if (col.transforms && col.transforms.length > 0) {
+                rawVal = runTransformPipeline(rawVal, col.transforms);
+            }
+
+            // Cast to target type
+            if (col.type) {
+                rawVal = castValueToType(rawVal, col.type);
+            }
+
+            record[col.key] = rawVal;
+        }
+
+        const hasValue = Object.values(record).some((v) => v !== null && v !== undefined && v !== "");
+        if (hasValue) {
+            results.push(record);
+        }
+    });
+
+    return results;
+}
+
 /**
  * Handles SCRAPE_CHROMIUM task via local stealth Puppeteer.
  */
@@ -146,6 +322,7 @@ export async function handleChromiumScrape(payload) {
     const {
         url,
         fieldMappings = [],
+        collections = [],
         navigation = {},
         waitFor = {},
         timeout = 30000,
@@ -257,6 +434,13 @@ export async function handleChromiumScrape(payload) {
                     }
                 }
 
+                if (mapping.transforms && mapping.transforms.length > 0) {
+                    extractedVal = runTransformPipeline(extractedVal, mapping.transforms);
+                }
+                if (mapping.type) {
+                    extractedVal = castValueToType(extractedVal, mapping.type);
+                }
+
                 raw[mapping.field] = extractedVal;
                 strategySummary[mapping.field] = {
                     extracted: extractedVal !== null,
@@ -302,6 +486,163 @@ export async function handleChromiumScrape(payload) {
             }
         }
 
+        // 5. Extract structured collections (Tables & Repeating items)
+        const extractedCollections = {};
+        if (Array.isArray(collections) && collections.length > 0) {
+            for (const coll of collections) {
+                if (!coll || !coll.name) continue;
+                try {
+                    if (coll.type === "TABLE") {
+                        extractedCollections[coll.name] = await page.evaluate((cDef) => {
+                            const sel = cDef.containerSelector || "table";
+                            const table = document.querySelector(sel);
+                            if (!table) return [];
+
+                            const hasHeaders = cDef.hasHeaders !== false;
+                            const headerRowIndex = typeof cDef.headerRowIndex === "number" ? cDef.headerRowIndex : 0;
+                            const maxItems = cDef.maxItems || 100;
+                            const columns = cDef.columns || [];
+
+                            const rowEls = Array.from(table.querySelectorAll("tr"));
+                            const rowsData = rowEls
+                                .map((r) => {
+                                    const cells = Array.from(r.querySelectorAll("th, td")).map(
+                                        (c) => c.innerText?.trim() || c.textContent?.trim() || ""
+                                    );
+                                    return { cells, el: r };
+                                })
+                                .filter((r) => r.cells.length > 0);
+
+                            if (rowsData.length === 0) return [];
+
+                            let activeColumns = [...columns];
+                            if (activeColumns.length === 0) {
+                                if (hasHeaders && rowsData[headerRowIndex]) {
+                                    activeColumns = rowsData[headerRowIndex].cells.map((h, idx) => {
+                                        const key =
+                                            h.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") ||
+                                            `col_${idx + 1}`;
+                                        return { key, label: h || `Column ${idx + 1}`, columnIndex: idx, excluded: false };
+                                    });
+                                } else {
+                                    activeColumns = rowsData[0].cells.map((_, idx) => ({
+                                        key: `col_${idx + 1}`,
+                                        label: `Column ${idx + 1}`,
+                                        columnIndex: idx,
+                                        excluded: false,
+                                    }));
+                                }
+                            }
+
+                            const startIdx = hasHeaders ? headerRowIndex + 1 : 0;
+                            const sliceRows = rowsData.slice(startIdx, startIdx + maxItems);
+                            const results = [];
+
+                            for (const row of sliceRows) {
+                                const rec = {};
+                                for (let colIdx = 0; colIdx < activeColumns.length; colIdx++) {
+                                    const col = activeColumns[colIdx];
+                                    if (col.excluded) continue;
+                                    let val = null;
+                                    const colSel = col.relativeSelector || col.selector;
+                                    if (colSel) {
+                                        const subEl = row.el.querySelector(colSel);
+                                        if (subEl) {
+                                            const attr = col.attribute || "text";
+                                            val = attr === "text" ? (subEl.innerText?.trim() || subEl.textContent?.trim()) : subEl.getAttribute(attr);
+                                        }
+                                    } else if (typeof col.columnIndex === "number" && col.columnIndex < row.cells.length) {
+                                        val = row.cells[col.columnIndex];
+                                    } else if (colIdx < row.cells.length) {
+                                        val = row.cells[colIdx];
+                                    }
+                                    if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                        val = col.defaultValue;
+                                    }
+                                    rec[col.key] = val;
+                                }
+                                results.push(rec);
+                            }
+                            return results;
+                        }, coll);
+                    } else {
+                        extractedCollections[coll.name] = await page.evaluate((cDef) => {
+                            let items = [];
+                            if (cDef.rowSelector) {
+                                if (cDef.containerSelector) {
+                                    const container = document.querySelector(cDef.containerSelector);
+                                    if (container) {
+                                        items = Array.from(container.querySelectorAll(cDef.rowSelector));
+                                    }
+                                }
+                                if (items.length === 0) {
+                                    items = Array.from(document.querySelectorAll(cDef.rowSelector));
+                                }
+                            } else if (cDef.containerSelector) {
+                                items = Array.from(document.querySelectorAll(cDef.containerSelector));
+                            }
+
+                            if (items.length === 0) return [];
+
+                            const maxItems = cDef.maxItems || 100;
+                            const columns = cDef.columns || [];
+                            const results = [];
+
+                            for (const item of items.slice(0, maxItems)) {
+                                const rec = {};
+                                for (const col of columns) {
+                                    if (col.excluded) continue;
+                                    let val = null;
+                                    const colSel = col.relativeSelector || col.selector;
+                                    if (colSel) {
+                                        const target = item.querySelector(colSel);
+                                        if (target) {
+                                            const attr = col.attribute || "text";
+                                            val = attr === "text" ? (target.innerText?.trim() || target.textContent?.trim()) : target.getAttribute(attr);
+                                        }
+                                    } else {
+                                        const attr = col.attribute || "text";
+                                        val = attr === "text" ? (item.innerText?.trim() || item.textContent?.trim()) : item.getAttribute(attr);
+                                    }
+                                    if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                        val = col.defaultValue;
+                                    }
+                                    rec[col.key] = val;
+                                }
+                                const hasVal = Object.values(rec).some((v) => v !== null && v !== undefined && v !== "");
+                                if (hasVal) results.push(rec);
+                            }
+                            return results;
+                        }, coll);
+                    }
+
+                    // Apply column transforms and type casting
+                    if (Array.isArray(extractedCollections[coll.name]) && coll.columns) {
+                        extractedCollections[coll.name] = extractedCollections[coll.name].map((rec) => {
+                            const transformed = { ...rec };
+                            for (const col of coll.columns) {
+                                if (col.excluded) continue;
+                                let val = transformed[col.key];
+                                if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                    val = col.defaultValue;
+                                }
+                                if (col.transforms && col.transforms.length > 0) {
+                                    val = runTransformPipeline(val, col.transforms);
+                                }
+                                if (col.type) {
+                                    val = castValueToType(val, col.type);
+                                }
+                                transformed[col.key] = val;
+                            }
+                            return transformed;
+                        });
+                    }
+                } catch {
+                    extractedCollections[coll.name] = [];
+                }
+            }
+        }
+
         const executionTimeMs = Date.now() - startTime;
         const hasExtractedValues = Object.values(raw).some(
             (v) => v !== null && v !== undefined && v !== ""
@@ -320,6 +661,7 @@ export async function handleChromiumScrape(payload) {
             raw,
             strategySummary,
             extractedData: raw,
+            collections: extractedCollections,
             html,
             executionTimeMs,
         };
@@ -336,6 +678,7 @@ export async function handleHttpScrape(payload) {
     const {
         url,
         fieldMappings = [],
+        collections = [],
         timeout = 15000,
         headers = {},
         extractSelectors = [],
@@ -353,7 +696,7 @@ export async function handleHttpScrape(payload) {
         },
     });
 
-    const pageHtml = typeof response.data === "string" ? response.data : "";
+    let pageHtml = typeof response.data === "string" ? response.data : Buffer.isBuffer(response.data) ? response.data.toString("utf-8") : "";
     const $ = cheerio.load(pageHtml);
     const title = $("title").first().text().trim() || "";
 
@@ -377,6 +720,13 @@ export async function handleHttpScrape(payload) {
                 }
             }
 
+            if (mapping.transforms && mapping.transforms.length > 0) {
+                extractedVal = runTransformPipeline(extractedVal, mapping.transforms);
+            }
+            if (mapping.type) {
+                extractedVal = castValueToType(extractedVal, mapping.type);
+            }
+
             raw[mapping.field] = extractedVal;
             strategySummary[mapping.field] = {
                 extracted: extractedVal !== null,
@@ -415,6 +765,23 @@ export async function handleHttpScrape(payload) {
         }
     }
 
+    // 3. Extract structured collections (Tables & Repeating items)
+    const extractedCollections = {};
+    if (Array.isArray(collections) && collections.length > 0) {
+        for (const coll of collections) {
+            if (!coll || !coll.name) continue;
+            try {
+                if (coll.type === "TABLE") {
+                    extractedCollections[coll.name] = extractTableCollectionCheerio($, coll);
+                } else {
+                    extractedCollections[coll.name] = extractRepeatingCollectionCheerio($, coll);
+                }
+            } catch {
+                extractedCollections[coll.name] = [];
+            }
+        }
+    }
+
     const executionTimeMs = Date.now() - startTime;
     const hasExtractedValues = Object.values(raw).some(
         (v) => v !== null && v !== undefined && v !== ""
@@ -427,6 +794,7 @@ export async function handleHttpScrape(payload) {
         raw,
         strategySummary,
         extractedData: raw,
+        collections: extractedCollections,
         html: includeHtml || (!hasExtractedValues && fieldMappings.length > 0) ? pageHtml : "",
         executionTimeMs,
     };

package/src/handlers/transformUtils.js

@@ -0,0 +1,340 @@
+/**
+ * Transformation and text extraction utilities for the Edge Worker.
+ */
+
+/**
+ * Safely extracts text from Cheerio elements while preserving block/boundary line breaks (\n).
+ * Prevents glued text like "678.0013.30 (2.00%)" caused by native $(el).text() concatenation.
+ *
+ * @param {cheerio.CheerioAPI} $
+ * @param {cheerio.Cheerio} el
+ * @returns {string}
+ */
+export function extractSmartCheerioText($, el) {
+    if (!el || el.length === 0) return "";
+    try {
+        const clone = $(el).clone();
+        clone.find("br").replaceWith("\n");
+        clone.find("div, p, span, small, button, li, h1, h2, h3, h4, h5, h6, tr, td, th, section, article, header, footer, nav, aside, dt, dd").before("\n").after("\n");
+        return clone
+            .text()
+            .split("\n")
+            .map((s) => s.trim())
+            .filter(Boolean)
+            .join("\n");
+    } catch {
+        return $(el).text().trim();
+    }
+}
+
+/**
+ * Executes a single transformation step safely on an input value.
+ *
+ * @param {*} value
+ * @param {Object} step
+ * @returns {*}
+ */
+export function applyTransformStep(value, step) {
+    if (!step) return value;
+
+    const op = (step.operation || step.op || "").toLowerCase();
+    if (!op) return value;
+
+    if (op === "default") {
+        if (value === null || value === undefined || value === "") {
+            const defVal = step.value !== undefined ? step.value : step.params?.value;
+            return defVal !== undefined ? defVal : null;
+        }
+        return value;
+    }
+
+    if (value === null || value === undefined) {
+        return value;
+    }
+
+    switch (op) {
+        case "trim": {
+            return typeof value === "string" ? value.trim() : String(value).trim();
+        }
+
+        case "lowercase": {
+            return typeof value === "string" ? value.toLowerCase() : String(value).toLowerCase();
+        }
+
+        case "uppercase": {
+            return typeof value === "string" ? value.toUpperCase() : String(value).toUpperCase();
+        }
+
+        case "remove": {
+            const toRemove = step.value !== undefined ? String(step.value) : step.params?.value !== undefined ? String(step.params.value) : "";
+            if (!toRemove) return value;
+            return String(value).split(toRemove).join("");
+        }
+
+        case "remove_prefix": {
+            const prefix = step.value !== undefined ? String(step.value) : step.params?.value !== undefined ? String(step.params.value) : "";
+            if (!prefix) return value;
+            const strVal = String(value).trim();
+            if (strVal.startsWith(prefix)) {
+                return strVal.slice(prefix.length).trim();
+            }
+            return strVal;
+        }
+
+        case "remove_suffix": {
+            const suffix = step.value !== undefined ? String(step.value) : step.params?.value !== undefined ? String(step.params.value) : "";
+            if (!suffix) return value;
+            const strVal = String(value).trim();
+            if (strVal.endsWith(suffix)) {
+                return strVal.slice(0, strVal.length - suffix.length).trim();
+            }
+            return strVal;
+        }
+
+        case "replace": {
+            const fromStr = step.from !== undefined ? String(step.from) : step.params?.from !== undefined ? String(step.params.from) : "";
+            const toStr = step.to !== undefined ? String(step.to) : step.params?.to !== undefined ? String(step.params.to) : "";
+            return String(value).split(fromStr).join(toStr);
+        }
+
+        case "regex_extract": {
+            const pattern = step.pattern || step.params?.pattern;
+            if (!pattern) return value;
+            try {
+                const regex = new RegExp(pattern);
+                const match = String(value).match(regex);
+                if (!match) return null;
+                return match[1] !== undefined ? match[1] : match[0];
+            } catch {
+                return value;
+            }
+        }
+
+        case "regex_replace": {
+            const pattern = step.pattern || step.params?.pattern;
+            if (!pattern) return value;
+            try {
+                const regex = new RegExp(pattern, "g");
+                const toStr = step.to !== undefined ? String(step.to) : step.params?.to !== undefined ? String(step.params.to) : "";
+                return String(value).replace(regex, toStr);
+            } catch {
+                return value;
+            }
+        }
+
+        case "split": {
+            let sep = step.separator !== undefined ? String(step.separator) : step.params?.separator !== undefined ? String(step.params.separator) : step.params?.delimiter !== undefined ? String(step.params.delimiter) : step.value !== undefined ? String(step.value) : "\n";
+            if (sep === "\\n") sep = "\n";
+            else if (sep === "\\t") sep = "\t";
+            const parts = String(value).split(sep).map((s) => s.trim());
+            const index = step.index !== undefined ? step.index : step.params?.index;
+            if (index !== undefined && index !== null) {
+                const idx = parseInt(index, 10);
+                if (isNaN(idx)) return null;
+                const targetIdx = idx < 0 ? parts.length + idx : idx;
+                return targetIdx >= 0 && targetIdx < parts.length ? parts[targetIdx] : null;
+            }
+            return parts;
+        }
+
+        case "first_line": {
+            const parts = String(value).split("\n").map((s) => s.trim()).filter(Boolean);
+            return parts.length > 0 ? parts[0] : "";
+        }
+
+        case "second_line": {
+            const parts = String(value).split("\n").map((s) => s.trim()).filter(Boolean);
+            return parts.length > 1 ? parts[1] : "";
+        }
+
+        case "line": {
+            const idx = parseInt(step.index ?? step.value ?? step.params?.index ?? step.params?.value ?? 0, 10);
+            const parts = String(value).split("\n").map((s) => s.trim()).filter(Boolean);
+            if (isNaN(idx)) return parts[0] || "";
+            const targetIdx = idx < 0 ? parts.length + idx : idx;
+            return targetIdx >= 0 && targetIdx < parts.length ? parts[targetIdx] : "";
+        }
+
+        case "contains":
+        case "includes":
+        case "has_badge": {
+            const searchStr = step.value !== undefined ? String(step.value) : step.pattern !== undefined ? String(step.pattern) : step.params?.badgeKeyword !== undefined ? String(step.params.badgeKeyword) : step.params?.value !== undefined ? String(step.params.value) : "";
+            if (!searchStr) return false;
+            const caseSensitive = step.caseSensitive === true || step.params?.caseSensitive === true;
+            const strVal = String(value);
+            return caseSensitive
+                ? strVal.includes(searchStr)
+                : strVal.toLowerCase().includes(searchStr.toLowerCase());
+        }
+
+        case "cast": {
+            const targetType = (step.targetType || step.params?.targetType || step.type || step.value || "string").toLowerCase();
+            return castValueToType(value, targetType);
+        }
+
+        case "join": {
+            const delimiter = step.delimiter !== undefined ? String(step.delimiter) : step.value !== undefined ? String(step.value) : ", ";
+            return Array.isArray(value) ? value.join(delimiter) : String(value);
+        }
+
+        case "currency_cleanup": {
+            if (typeof value === "number") return value;
+            const str = String(value).trim();
+            const cleaned = str
+                .replace(/[\u20B9$€£¥₹]/g, "")
+                .replace(/\b(USD|EUR|GBP|INR|CAD|AUD|JPY|CHF|AED|Rs\.?|Rupees?)\b/gi, "")
+                .replace(/\s+/g, " ")
+                .replace(/,/g, "")
+                .trim();
+            const numMatch = cleaned.match(/[-+]?[0-9]+(?:\.[0-9]+)?/);
+            return numMatch ? parseFloat(numMatch[0]) : null;
+        }
+
+        case "number":
+        case "decimal": {
+            if (typeof value === "number") return value;
+            const str = String(value).trim().replace(/,/g, "");
+            const numMatch = str.match(/[-+]?[0-9]+(?:\.[0-9]+)?/);
+            return numMatch ? parseFloat(numMatch[0]) : null;
+        }
+
+        case "integer": {
+            if (typeof value === "number") return Math.round(value);
+            const str = String(value).trim().replace(/,/g, "");
+            const numMatch = str.match(/[-+]?[0-9]+/);
+            return numMatch ? parseInt(numMatch[0], 10) : null;
+        }
+
+        case "parse_boolean":
+        case "boolean": {
+            if (typeof value === "boolean") return value;
+            const str = String(value).trim().toLowerCase();
+            if (["true", "yes", "y", "1", "in stock", "available", "on", "active"].includes(str)) return true;
+            if (["false", "no", "n", "0", "out of stock", "unavailable", "off", "inactive"].includes(str)) return false;
+            return Boolean(value);
+        }
+
+        case "has_value":
+        case "is_not_empty":
+        case "not_empty":
+        case "exists": {
+            if (value === null || value === undefined) return false;
+            const lineIdx = step.lineIndex !== undefined ? step.lineIndex : step.params?.lineIndex !== undefined ? step.params?.lineIndex : step.index !== undefined ? step.index : step.params?.index;
+            if (lineIdx !== undefined && lineIdx !== null && lineIdx !== "") {
+                const idx = parseInt(lineIdx, 10);
+                const parts = String(value).split("\n").map((s) => s.trim()).filter(Boolean);
+                const targetIdx = idx > 0 ? idx - 1 : idx; // 1-indexed line number
+                return targetIdx >= 0 && targetIdx < parts.length && parts[targetIdx].length > 0;
+            }
+            const str = String(value).trim();
+            return str.length > 0 && str.toLowerCase() !== "false" && str !== "0";
+        }
+
+        case "custom":
+        case "expression":
+        case "script": {
+            const code = step.code || step.expression || step.value || step.script || step.params?.code || step.params?.expression || step.params?.script;
+            if (!code || typeof code !== "string") return value;
+            try {
+                const trimmed = code.trim();
+                if (/\b(process|require|import|global|Buffer|__dirname|__filename|eval|Function)\b/.test(trimmed)) {
+                    return value;
+                }
+
+                // Auto-quote unquoted string after == or === or != or !== e.g. == Vol Shocker -> == "Vol Shocker"
+                let sanitizedCode = trimmed.replace(/([=!]==?\s*)([A-Za-z][A-Za-z0-9 _-]*[A-Za-z0-9])(?=\s*$|\s*[;)])/g, (match, op, str) => {
+                    const trimmedStr = str.trim();
+                    if (['true', 'false', 'null', 'undefined', 'value', 'raw', 'lines'].includes(trimmedStr)) return match;
+                    if (trimmedStr.startsWith('"') || trimmedStr.startsWith("'")) return match;
+                    return `${op}"${trimmedStr}"`;
+                });
+
+                const lines = String(value).split("\n").map((s) => s.trim()).filter(Boolean);
+                const line = (n) => {
+                    const idx = parseInt(n, 10);
+                    const targetIdx = idx > 0 ? idx - 1 : idx;
+                    return lines[targetIdx] || "";
+                };
+                const hasLine = (n) => {
+                    const idx = parseInt(n, 10);
+                    const targetIdx = idx > 0 ? idx - 1 : idx;
+                    return targetIdx >= 0 && targetIdx < lines.length && lines[targetIdx].length > 0;
+                };
+                const has = (k) => String(value).toLowerCase().includes(String(k).toLowerCase());
+                const contains = has;
+                const hasValue = (v = value) => v !== null && v !== undefined && String(v).trim().length > 0;
+                const notEmpty = hasValue;
+
+                const fn = sanitizedCode.includes("return ")
+                    ? new Function("value", "raw", "lines", "line", "hasLine", "has", "contains", "hasValue", "notEmpty", sanitizedCode)
+                    : new Function("value", "raw", "lines", "line", "hasLine", "has", "contains", "hasValue", "notEmpty", `return (${sanitizedCode});`);
+                const result = fn(value, value, lines, line, hasLine, has, contains, hasValue, notEmpty);
+                return result !== undefined ? result : value;
+            } catch {
+                return value;
+            }
+        }
+
+        default:
+            return value;
+    }
+}
+
+/**
+ * Runs an array of transform steps sequentially on a raw extracted value.
+ *
+ * @param {*} rawValue
+ * @param {Array<Object>} pipeline
+ * @returns {*}
+ */
+export function runTransformPipeline(rawValue, pipeline = []) {
+    if (!Array.isArray(pipeline) || pipeline.length === 0) {
+        return rawValue;
+    }
+
+    let current = rawValue;
+    for (const step of pipeline) {
+        current = applyTransformStep(current, step);
+    }
+    return current;
+}
+
+/**
+ * Casts a transformed value to the target data type.
+ *
+ * @param {*} val
+ * @param {string} type
+ * @returns {*}
+ */
+export function castValueToType(val, type = "string") {
+    if (val === null || val === undefined) return val;
+
+    switch (type) {
+        case "integer": {
+            if (typeof val === "number") return Math.round(val);
+            const str = String(val).replace(/,/g, "").trim();
+            const num = parseInt(str, 10);
+            return isNaN(num) ? null : num;
+        }
+        case "number":
+        case "decimal": {
+            if (typeof val === "number") return val;
+            const str = String(val).replace(/[\u20B9$€£¥₹]/g, "").replace(/,/g, "").trim();
+            const num = parseFloat(str);
+            return isNaN(num) ? null : num;
+        }
+        case "boolean": {
+            if (typeof val === "boolean") return val;
+            const str = String(val).trim().toLowerCase();
+            return ["true", "yes", "1", "in stock", "available"].includes(str);
+        }
+        case "date": {
+            if (val instanceof Date) return val.toISOString();
+            const parsed = Date.parse(String(val));
+            return !isNaN(parsed) ? new Date(parsed).toISOString() : String(val);
+        }
+        case "string":
+        default:
+            return typeof val === "object" ? JSON.stringify(val) : String(val);
+    }
+}