mx-cloud 0.0.12 → 0.0.13

package/build/browserSide/scraper.js

@@ -360,18 +360,169 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
     window.scrapeList = function (_a) {
         return __awaiter(this, arguments, void 0, function* ({ listSelector, fields, limit = 10 }) {
             // XPath evaluation functions
-            const evaluateXPath = (rootElement, xpath) => {
+            const queryInsideContext = (context, part) => {
                 try {
-                    const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
-                        ? rootElement
-                        : rootElement.ownerDocument;
-                    if (!ownerDoc)
-                        return null;
-                    const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
-                    return result.singleNodeValue;
+                    const { tagName, conditions } = parseXPathPart(part);
+                    const candidateElements = Array.from(context.querySelectorAll(tagName));
+                    if (candidateElements.length === 0) {
+                        return [];
+                    }
+                    const matchingElements = candidateElements.filter((el) => {
+                        return elementMatchesConditions(el, conditions);
+                    });
+                    return matchingElements;
                 }
-                catch (error) {
-                    console.warn("XPath evaluation failed:", xpath, error);
+                catch (err) {
+                    console.error("Error in queryInsideContext:", err);
+                    return [];
+                }
+            };
+            // Helper function to parse XPath part
+            const parseXPathPart = (part) => {
+                const tagMatch = part.match(/^([a-zA-Z0-9-]+)/);
+                const tagName = tagMatch ? tagMatch[1] : "*";
+                const conditionMatches = part.match(/\[([^\]]+)\]/g);
+                const conditions = conditionMatches
+                    ? conditionMatches.map((c) => c.slice(1, -1))
+                    : [];
+                return { tagName, conditions };
+            };
+            // Helper function to check if element matches all conditions
+            const elementMatchesConditions = (element, conditions) => {
+                for (const condition of conditions) {
+                    if (!elementMatchesCondition(element, condition)) {
+                        return false;
+                    }
+                }
+                return true;
+            };
+            // Helper function to check if element matches a single condition
+            const elementMatchesCondition = (element, condition) => {
+                var _a, _b;
+                condition = condition.trim();
+                if (/^\d+$/.test(condition)) {
+                    return true;
+                }
+                // Handle @attribute="value"
+                const attrMatch = condition.match(/^@([^=]+)=["']([^"']+)["']$/);
+                if (attrMatch) {
+                    const [, attr, value] = attrMatch;
+                    const elementValue = element.getAttribute(attr);
+                    return elementValue === value;
+                }
+                // Handle contains(@class, 'value')
+                const classContainsMatch = condition.match(/^contains\(@class,\s*["']([^"']+)["']\)$/);
+                if (classContainsMatch) {
+                    const className = classContainsMatch[1];
+                    return element.classList.contains(className);
+                }
+                // Handle contains(@attribute, 'value')
+                const attrContainsMatch = condition.match(/^contains\(@([^,]+),\s*["']([^"']+)["']\)$/);
+                if (attrContainsMatch) {
+                    const [, attr, value] = attrContainsMatch;
+                    const elementValue = element.getAttribute(attr) || "";
+                    return elementValue.includes(value);
+                }
+                // Handle text()="value"
+                const textMatch = condition.match(/^text\(\)=["']([^"']+)["']$/);
+                if (textMatch) {
+                    const expectedText = textMatch[1];
+                    const elementText = ((_a = element.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || "";
+                    return elementText === expectedText;
+                }
+                // Handle contains(text(), 'value')
+                const textContainsMatch = condition.match(/^contains\(text\(\),\s*["']([^"']+)["']\)$/);
+                if (textContainsMatch) {
+                    const expectedText = textContainsMatch[1];
+                    const elementText = ((_b = element.textContent) === null || _b === void 0 ? void 0 : _b.trim()) || "";
+                    return elementText.includes(expectedText);
+                }
+                // Handle count(*)=0 (element has no children)
+                if (condition === "count(*)=0") {
+                    return element.children.length === 0;
+                }
+                // Handle other count conditions
+                const countMatch = condition.match(/^count\(\*\)=(\d+)$/);
+                if (countMatch) {
+                    const expectedCount = parseInt(countMatch[1]);
+                    return element.children.length === expectedCount;
+                }
+                return true;
+            };
+            const evaluateXPath = (document, xpath, isShadow = false) => {
+                try {
+                    const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                    if (!isShadow) {
+                        if (result === null) {
+                            return null;
+                        }
+                        return result;
+                    }
+                    let cleanPath = xpath;
+                    let isIndexed = false;
+                    const indexedMatch = xpath.match(/^\((.*?)\)\[(\d+)\](.*)$/);
+                    if (indexedMatch) {
+                        cleanPath = indexedMatch[1] + indexedMatch[3];
+                        isIndexed = true;
+                    }
+                    const pathParts = cleanPath
+                        .replace(/^\/\//, "")
+                        .split("/")
+                        .map((p) => p.trim())
+                        .filter((p) => p.length > 0);
+                    let currentContexts = [document];
+                    for (let i = 0; i < pathParts.length; i++) {
+                        const part = pathParts[i];
+                        const nextContexts = [];
+                        for (const ctx of currentContexts) {
+                            const positionalMatch = part.match(/^([^[]+)\[(\d+)\]$/);
+                            let partWithoutPosition = part;
+                            let requestedPosition = null;
+                            if (positionalMatch) {
+                                partWithoutPosition = positionalMatch[1];
+                                requestedPosition = parseInt(positionalMatch[2]);
+                            }
+                            const matched = queryInsideContext(ctx, partWithoutPosition);
+                            let elementsToAdd = matched;
+                            if (requestedPosition !== null) {
+                                const index = requestedPosition - 1; // XPath is 1-based, arrays are 0-based
+                                if (index >= 0 && index < matched.length) {
+                                    elementsToAdd = [matched[index]];
+                                }
+                                else {
+                                    console.warn(`Position ${requestedPosition} out of range (${matched.length} elements found)`);
+                                    elementsToAdd = [];
+                                }
+                            }
+                            elementsToAdd.forEach((el) => {
+                                nextContexts.push(el);
+                                if (el.shadowRoot) {
+                                    nextContexts.push(el.shadowRoot);
+                                }
+                            });
+                        }
+                        if (nextContexts.length === 0) {
+                            return null;
+                        }
+                        currentContexts = nextContexts;
+                    }
+                    if (currentContexts.length > 0) {
+                        if (isIndexed && indexedMatch) {
+                            const requestedIndex = parseInt(indexedMatch[2]) - 1;
+                            if (requestedIndex >= 0 && requestedIndex < currentContexts.length) {
+                                return currentContexts[requestedIndex];
+                            }
+                            else {
+                                console.warn(`Requested index ${requestedIndex + 1} out of range (${currentContexts.length} elements found)`);
+                                return null;
+                            }
+                        }
+                        return currentContexts[0];
+                    }
+                    return null;
+                }
+                catch (err) {
+                    console.error("Critical XPath failure:", xpath, err);
                     return null;
                 }
             };
@@ -834,7 +985,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         if (isXPathSelector(field.selector)) {
                             // Create indexed absolute XPath
                             const indexedSelector = createIndexedXPath(field.selector, listSelector, containerIndex + 1);
-                            element = evaluateXPath(document, indexedSelector);
+                            element = evaluateXPath(document, indexedSelector, field.isShadow);
                         }
                         else {
                             // Fallback for CSS selectors within XPath containers

package/build/interpret.js

@@ -829,9 +829,9 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return allResults;
                             let loadMoreCounter = 0;
-                            let previousResultCount = allResults.length;
-                            let noNewItemsCounter = 0;
-                            const MAX_NO_NEW_ITEMS = 2;
+                            // let previousResultCount = allResults.length;
+                            // let noNewItemsCounter = 0;
+                            // const MAX_NO_NEW_ITEMS = 2;
                             while (true) {
                                 // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
@@ -888,20 +888,19 @@ class Interpreter extends events_1.EventEmitter {
                                 const heightChanged = currentHeight !== previousHeight;
                                 previousHeight = currentHeight;
                                 yield scrapeCurrentPage();
-                                const currentResultCount = allResults.length;
-                                const newItemsAdded = currentResultCount > previousResultCount;
-                                if (!newItemsAdded) {
-                                    noNewItemsCounter++;
-                                    debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
-                                    if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
-                                        debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
-                                        return allResults;
-                                    }
-                                }
-                                else {
-                                    noNewItemsCounter = 0;
-                                    previousResultCount = currentResultCount;
-                                }
+                                // const currentResultCount = allResults.length;
+                                // const newItemsAdded = currentResultCount > previousResultCount;
+                                // if (!newItemsAdded) {
+                                //   noNewItemsCounter++;
+                                //   debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
+                                //   if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
+                                //     debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
+                                //     return allResults;
+                                //   }
+                                // } else {
+                                //   noNewItemsCounter = 0;
+                                //   previousResultCount = currentResultCount;
+                                // }
                                 if (checkLimit())
                                     return allResults;
                                 if (!heightChanged) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",