mx-cloud 0.0.24 → 0.0.25

package/build/interpret.d.ts

@@ -38,6 +38,7 @@ interface InterpreterOptions {
     serializableCallback: (output: any) => (void | Promise<void>);
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
+    robotType?: 'extract' | 'scrape' | 'deep-extract';
     debugChannel: Partial<{
         activeId: (id: number) => void;
         debugMessage: (msg: string) => void;
@@ -63,6 +64,8 @@ export default class Interpreter extends EventEmitter {
     private screenshotCounter;
     private scrapeListCounter;
     private serializableDataByType;
+    private pendingDeepExtraction;
+    private isInDeepExtractionPhase;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     trackAutohealFailure(error: string): void;
     private applyAdBlocker;
@@ -148,6 +151,56 @@ export default class Interpreter extends EventEmitter {
      * @returns {Promise<WhereWhatPair>} - The potentially modified action
      */
     private validateAndFixSelectors;
+    /**
+     * Extracts URLs from the current page's list elements.
+     * Used during pagination to maintain sync between scraped results and extracted URLs.
+     *
+     * @param page - Playwright page object
+     * @param listSelector - The selector used to identify list elements
+     * @param limit - Maximum number of elements to process (should match number of scraped items)
+     * @returns Array of URL arrays, one per list element
+     */
+    private extractUrlsFromCurrentPage;
+    /**
+     * Builds a hierarchical deep extraction plan by analyzing the workflow structure.
+     * Identifies goto actions and determines what actions to execute at each level.
+     * Workflow is bottom-to-top, so we scan from end to start.
+     */
+    private buildDeepExtractionHierarchy;
+    /**
+     * Extracts hrefs directly from the page based on scrapeSchema selectors.
+     * Checks ALL selectors from the schema config - if they point to anchor elements, extract href.
+     * This is called after scrapeSchema executes to capture hrefs for deep extraction.
+     */
+    private extractHrefsFromPage;
+    /**
+     * Filters URLs for deep extraction based on the goto action pattern.
+     * This is called immediately after the first capture action (scrapeList).
+     * Returns the filtered URL mappings that should be processed after workflow completion.
+     * Each mapping maintains alignment with the original scrapeList index.
+     */
+    private filterDeepExtractionUrls;
+    /**
+     * Filters pre-extracted URLs for deep extraction based on the goto action pattern.
+     * This is used for paginated lists where URLs were extracted during pagination.
+     * Returns the filtered URL mappings that maintain alignment with scrapeList indices.
+     */
+    private filterDeepExtractionUrlsFromExtracted;
+    /**
+     * Helper function to check if a URL matches a goto pattern.
+     */
+    private matchesGotoPattern;
+    /**
+     * Executes hierarchical deep extraction by processing each level recursively.
+     * URLs are already stored in each hierarchy level's urlMappings during workflow execution.
+     */
+    private executeHierarchicalDeepExtraction;
+    /**
+     * Executes deep extraction for a single level.
+     * URLs are already extracted and stored in hierarchy during workflow execution.
+     * This function just navigates to URLs and executes the capture actions.
+     */
+    private executeDeepExtractionLevel;
     private runLoop;
     private ensureScriptsLoaded;
     /**

package/build/interpret.js

@@ -74,6 +74,8 @@ class Interpreter extends events_1.EventEmitter {
             scrapeList: {},
             scrapeSchema: {}
         };
+        this.pendingDeepExtraction = null;
+        this.isInDeepExtractionPhase = false;
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
         this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
@@ -338,7 +340,7 @@ class Interpreter extends events_1.EventEmitter {
    * @param page Playwright Page object
    * @param steps Array of actions.
    */
-    carryOutSteps(page, steps) {
+    carryOutSteps(page, steps, currentWorkflow) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
             // Check abort flag at start of execution
@@ -494,6 +496,84 @@ class Interpreter extends events_1.EventEmitter {
                         scrapeList: this.serializableDataByType.scrapeList,
                         scrapeSchema: this.serializableDataByType.scrapeSchema
                     });
+                    if (this.options.robotType === 'deep-extract' && !this.isInDeepExtractionPhase && this.initializedWorkflow) {
+                        if (!this.pendingDeepExtraction) {
+                            console.log('DEBUG: Building hierarchical deep extraction plan from scrapeSchema...');
+                            const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
+                            if (hierarchyData.length > 0) {
+                                const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
+                                this.log(`Root scrapeSchema will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
+                                // Extract URLs from schema fields
+                                const urls = yield this.extractHrefsFromPage(page, schema);
+                                this.log(`scrapeSchema extracted ${urls.length} URLs from field selectors`, logger_1.Level.LOG);
+                                // Filter URLs against pattern
+                                const rootUrlMappings = urls
+                                    .map((url, index) => ({
+                                    scrapeListIndex: index,
+                                    url: this.matchesGotoPattern(url, nextLevelGotoPattern) ? url : null
+                                }))
+                                    .filter(m => m.url !== null);
+                                this.log(`Matched ${rootUrlMappings.length} URLs against pattern ${nextLevelGotoPattern}`, logger_1.Level.LOG);
+                                this.pendingDeepExtraction = {
+                                    page,
+                                    hierarchy: hierarchyData.map((level, idx) => ({
+                                        gotoPattern: level.gotoPattern,
+                                        actionsToExecute: level.actionsToExecute,
+                                        urlMappings: idx === nextLevelIndex ? rootUrlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url })) : []
+                                    }))
+                                };
+                            }
+                            else {
+                                console.log('DEBUG: No goto actions found, deep extraction skipped');
+                            }
+                        }
+                        else {
+                            this.log(`[Deep Extract] scrapeSchema "${name}" extracting URLs during workflow execution`, logger_1.Level.LOG);
+                            const hierarchy = this.pendingDeepExtraction.hierarchy;
+                            if (hierarchy && hierarchy.length > 0) {
+                                let targetLevelIndex = -1;
+                                for (let i = hierarchy.length - 1; i >= 0; i--) {
+                                    if (hierarchy[i].urlMappings.length === 0) {
+                                        targetLevelIndex = i;
+                                        break;
+                                    }
+                                }
+                                if (targetLevelIndex >= 0) {
+                                    const targetGotoPattern = hierarchy[targetLevelIndex].gotoPattern;
+                                    this.log(`[Deep Extract] Storing URLs for level ${targetLevelIndex}, pattern: ${targetGotoPattern}`, logger_1.Level.LOG);
+                                    const urls = yield this.extractHrefsFromPage(page, schema);
+                                    this.log(`[Deep Extract] Extracted ${urls.length} URLs from scrapeSchema field selectors`, logger_1.Level.LOG);
+                                    const urlMappings = urls
+                                        .map((url, index) => ({
+                                        index,
+                                        url: this.matchesGotoPattern(url, targetGotoPattern) ? url : null
+                                    }))
+                                        .filter(m => m.url !== null);
+                                    if (hierarchy[targetLevelIndex].urlMappings.length > 0) {
+                                        const existingUrls = new Set(hierarchy[targetLevelIndex].urlMappings.map(m => m.url).filter(u => u !== null));
+                                        const newUrls = urlMappings.filter(m => m.url !== null && !existingUrls.has(m.url));
+                                        if (newUrls.length > 0) {
+                                            const startIndex = hierarchy[targetLevelIndex].urlMappings.length;
+                                            hierarchy[targetLevelIndex].urlMappings.push(...newUrls.map((m, idx) => ({ index: startIndex + idx, url: m.url })));
+                                            this.log(`[Deep Extract] Merged ${newUrls.length} new URLs from scrapeSchema`, logger_1.Level.LOG);
+                                        }
+                                    }
+                                    else {
+                                        hierarchy[targetLevelIndex].urlMappings = urlMappings;
+                                    }
+                                    this.log(`[Deep Extract] Stored ${urlMappings.length} matching URLs`, logger_1.Level.LOG);
+                                    if (urlMappings.length > 0) {
+                                        const sampleSize = Math.min(3, urlMappings.length);
+                                        this.log(`[Deep Extract] Sample URLs (showing ${sampleSize} of ${urlMappings.length}):`, logger_1.Level.LOG);
+                                        urlMappings.slice(0, sampleSize).forEach((mapping, idx) => {
+                                            this.log(`[Deep Extract]   ${idx + 1}. ${mapping.url}`, logger_1.Level.LOG);
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }),
                 scrapeList: (config_1, ...args_1) => __awaiter(this, [config_1, ...args_1], void 0, function* (config, actionName = "") {
                     var _a, _b;
@@ -528,16 +608,58 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         else {
                             paginationUsed = true;
-                            scrapeResults = yield this.handlePagination(page, config, actionName);
+                            const paginationResult = yield this.handlePagination(page, config, actionName);
+                            scrapeResults = paginationResult.results;
+                            const paginationUrls = paginationResult.urls;
+                            if (this.options.robotType === 'deep-extract' && this.initializedWorkflow && scrapeResults.length > 0) {
+                                if (!this.pendingDeepExtraction) {
+                                    console.log('DEBUG: Building hierarchical deep extraction plan from pagination...');
+                                    const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
+                                    if (hierarchyData.length > 0) {
+                                        const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                        const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
+                                        this.log(`Root scrapeList (pagination) will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
+                                        const rootUrlMappings = this.filterDeepExtractionUrlsFromExtracted(paginationUrls, scrapeResults, nextLevelGotoPattern);
+                                        this.pendingDeepExtraction = {
+                                            page,
+                                            hierarchy: hierarchyData.map((level, idx) => ({
+                                                gotoPattern: level.gotoPattern,
+                                                actionsToExecute: level.actionsToExecute,
+                                                urlMappings: idx === nextLevelIndex ? rootUrlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url })) : []
+                                            }))
+                                        };
+                                    }
+                                }
+                                else {
+                                    this.log(`[Deep Extract] scrapeList (pagination) "${actionName}" extracting URLs`, logger_1.Level.LOG);
+                                    const hierarchy = this.pendingDeepExtraction.hierarchy;
+                                    if (hierarchy && hierarchy.length > 0) {
+                                        const nextLevelIndex = hierarchy.length >= 3 ? hierarchy.length - 3 : 0;
+                                        if (nextLevelIndex >= 0 && hierarchy[nextLevelIndex]) {
+                                            const nextGotoPattern = hierarchy[nextLevelIndex].gotoPattern;
+                                            this.log(`[Deep Extract] Extracting URLs for pattern: ${nextGotoPattern}`, logger_1.Level.LOG);
+                                            const urlMappings = this.filterDeepExtractionUrlsFromExtracted(paginationUrls, scrapeResults, nextGotoPattern);
+                                            this.log(`[Deep Extract] Found ${urlMappings.filter(m => m.url !== null).length} matching URLs`, logger_1.Level.LOG);
+                                            const validUrls = urlMappings.filter(m => m.url !== null);
+                                            if (validUrls.length > 0) {
+                                                const sampleSize = Math.min(3, validUrls.length);
+                                                this.log(`[Deep Extract] Sample URLs (showing ${sampleSize} of ${validUrls.length}):`, logger_1.Level.LOG);
+                                                validUrls.slice(0, sampleSize).forEach((mapping, idx) => {
+                                                    this.log(`[Deep Extract]   ${idx + 1}. ${mapping.url}`, logger_1.Level.LOG);
+                                                });
+                                            }
+                                        }
+                                    }
+                                }
+                            }
                         }
                         if (!Array.isArray(scrapeResults)) {
                             scrapeResults = [];
                         }
-                        console.log(`ScrapeList completed with ${scrapeResults.length} results`);
                         if (!paginationUsed) {
                             const actionType = "scrapeList";
                             let name = actionName || "";
-                            if (!name || name.trim() === "") {
+                            if (!name || name.trim() === "" || this.isInDeepExtractionPhase) {
                                 this.scrapeListCounter++;
                                 name = `List ${this.scrapeListCounter}`;
                             }
@@ -551,6 +673,69 @@ class Interpreter extends events_1.EventEmitter {
                                 scrapeList: this.serializableDataByType.scrapeList,
                                 scrapeSchema: this.serializableDataByType.scrapeSchema
                             });
+                            console.log(`DEBUG: Checking deep extract condition: robotType=${this.options.robotType}, hasWorkflow=${!!currentWorkflow}, alreadyPending=${!!this.pendingDeepExtraction}`);
+                            if (this.options.robotType === 'deep-extract' && !this.isInDeepExtractionPhase && this.initializedWorkflow) {
+                                if (!this.pendingDeepExtraction) {
+                                    console.log('DEBUG: Building hierarchical deep extraction plan...');
+                                    const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
+                                    if (hierarchyData.length > 0) {
+                                        const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                        const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
+                                        this.log(`Root scrapeList will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
+                                        const rootUrlMappings = yield this.filterDeepExtractionUrls(page, config.listSelector, scrapeResults, nextLevelGotoPattern);
+                                        this.pendingDeepExtraction = {
+                                            page,
+                                            hierarchy: hierarchyData.map((level, idx) => ({
+                                                gotoPattern: level.gotoPattern,
+                                                actionsToExecute: level.actionsToExecute,
+                                                urlMappings: idx === nextLevelIndex ? rootUrlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url })) : []
+                                            }))
+                                        };
+                                    }
+                                    else {
+                                        console.log('DEBUG: No goto actions found, deep extraction skipped');
+                                    }
+                                }
+                                else {
+                                    this.log(`[Deep Extract] scrapeList "${name}" extracting URLs during workflow execution`, logger_1.Level.LOG);
+                                    const hierarchy = this.pendingDeepExtraction.hierarchy;
+                                    if (hierarchy && hierarchy.length > 0) {
+                                        let targetLevelIndex = -1;
+                                        for (let i = hierarchy.length - 1; i >= 0; i--) {
+                                            if (hierarchy[i].urlMappings.length === 0) {
+                                                targetLevelIndex = i;
+                                                break;
+                                            }
+                                        }
+                                        if (targetLevelIndex >= 0) {
+                                            const nextGotoPattern = hierarchy[targetLevelIndex].gotoPattern;
+                                            this.log(`[Deep Extract] Storing URLs for level ${targetLevelIndex}, pattern: ${nextGotoPattern}`, logger_1.Level.LOG);
+                                            const urlMappings = yield this.filterDeepExtractionUrls(page, config.listSelector, scrapeResults, nextGotoPattern);
+                                            if (hierarchy[targetLevelIndex].urlMappings.length > 0) {
+                                                const existingUrls = new Set(hierarchy[targetLevelIndex].urlMappings.map(m => m.url).filter(u => u !== null));
+                                                const newUrls = urlMappings.filter(m => m.url !== null && !existingUrls.has(m.url));
+                                                if (newUrls.length > 0) {
+                                                    const startIndex = hierarchy[targetLevelIndex].urlMappings.length;
+                                                    hierarchy[targetLevelIndex].urlMappings.push(...newUrls.map((m, idx) => ({ index: startIndex + idx, url: m.url })));
+                                                    this.log(`[Deep Extract] Merged ${newUrls.length} new URLs`, logger_1.Level.LOG);
+                                                }
+                                            }
+                                            else {
+                                                hierarchy[targetLevelIndex].urlMappings = urlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url }));
+                                            }
+                                            this.log(`[Deep Extract] Stored ${urlMappings.filter(m => m.url !== null).length} matching URLs`, logger_1.Level.LOG);
+                                            const validUrls = urlMappings.filter(m => m.url !== null);
+                                            if (validUrls.length > 0) {
+                                                const sampleSize = Math.min(3, validUrls.length);
+                                                this.log(`[Deep Extract] Sample URLs (showing ${sampleSize} of ${validUrls.length}):`, logger_1.Level.LOG);
+                                                validUrls.slice(0, sampleSize).forEach((mapping, idx) => {
+                                                    this.log(`[Deep Extract]   ${idx + 1}. ${mapping.url}`, logger_1.Level.LOG);
+                                                });
+                                            }
+                                        }
+                                    }
+                                }
+                            }
                         }
                     }
                     catch (error) {
@@ -731,11 +916,12 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, arguments, void 0, function* (page, config, providedActionName = "") {
             if (this.isAborted) {
                 this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
-                return [];
+                return { results: [], urls: [] };
             }
             const actionType = "scrapeList";
             let actionName = providedActionName || "";
-            if (!actionName || actionName.trim() === "") {
+            // During deep extraction, ALWAYS auto-increment to create separate lists for each URL
+            if (!actionName || actionName.trim() === "" || this.isInDeepExtractionPhase) {
                 this.scrapeListCounter++;
                 actionName = `List ${this.scrapeListCounter}`;
             }
@@ -747,6 +933,7 @@ class Interpreter extends events_1.EventEmitter {
                 this.serializableDataByType[actionType][actionName] = [];
             }
             let allResults = [];
+            let allUrls = []; // Track URLs alongside results for deep-extract
             let previousHeight = 0;
             let scrapedItems = new Set();
             let visitedUrls = new Set();
@@ -773,14 +960,22 @@ class Interpreter extends events_1.EventEmitter {
                     debugLog(`Page evaluation failed: ${error.message}`);
                     return;
                 }
-                const newResults = results.filter(item => {
+                // Extract URLs for ALL items BEFORE filtering duplicates
+                // This ensures URL indices match result indices
+                const allItemUrls = yield this.extractUrlsFromCurrentPage(page, config.listSelector, results.length);
+                // Filter results AND URLs together using the same uniqueness logic
+                const newResults = [];
+                const newUrls = [];
+                results.forEach((item, index) => {
                     const uniqueKey = JSON.stringify(item);
-                    if (scrapedItems.has(uniqueKey))
-                        return false;
-                    scrapedItems.add(uniqueKey);
-                    return true;
+                    if (!scrapedItems.has(uniqueKey)) {
+                        scrapedItems.add(uniqueKey);
+                        newResults.push(item);
+                        newUrls.push(allItemUrls[index] || []); // Add corresponding URLs
+                    }
                 });
                 allResults = allResults.concat(newResults);
+                allUrls = allUrls.concat(newUrls);
                 debugLog("Results collected:", allResults.length);
                 // Store in serializableDataByType and send structured callback
                 this.serializableDataByType[actionType][actionName] = [...allResults];
@@ -792,6 +987,7 @@ class Interpreter extends events_1.EventEmitter {
             const checkLimit = () => {
                 if (config.limit && allResults.length >= config.limit) {
                     allResults = allResults.slice(0, config.limit);
+                    allUrls = allUrls.slice(0, config.limit); // Also trim URLs to maintain sync
                     return true;
                 }
                 return false;
@@ -917,16 +1113,16 @@ class Interpreter extends events_1.EventEmitter {
                     // Check abort flag at start of each pagination iteration
                     if (this.isAborted) {
                         this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
-                        return allResults;
+                        return { results: allResults, urls: allUrls };
                     }
                     // Pagination circuit breakers
                     if (++paginationIterations > MAX_PAGINATION_ITERATIONS) {
                         debugLog(`Maximum pagination iterations reached (${MAX_PAGINATION_ITERATIONS}), stopping`);
-                        return allResults;
+                        return { results: allResults, urls: allUrls };
                     }
                     if (Date.now() - paginationStartTime > MAX_PAGINATION_TIME) {
                         debugLog('Maximum pagination time reached (10 minutes), stopping');
-                        return allResults;
+                        return { results: allResults, urls: allUrls };
                     }
                     // Add async yield every 5 iterations to prevent event loop blocking
                     if (paginationIterations % 5 === 0) {
@@ -937,7 +1133,7 @@ class Interpreter extends events_1.EventEmitter {
                             let previousResultCount = allResults.length;
                             yield scrapeCurrentPage();
                             if (checkLimit()) {
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             }
                             yield page.evaluate(() => {
                                 const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
@@ -951,14 +1147,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
                                 if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                             }
                             else {
                                 unchangedResultCounter = 0;
                             }
                             if (currentHeight === previousHeight) {
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             }
                             previousHeight = currentHeight;
                             break;
@@ -967,7 +1163,7 @@ class Interpreter extends events_1.EventEmitter {
                             let previousResultCount = allResults.length;
                             yield scrapeCurrentPage();
                             if (checkLimit()) {
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             }
                             yield page.evaluate(() => window.scrollTo(0, 0));
                             yield page.waitForTimeout(2000);
@@ -976,14 +1172,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
                                 if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                             }
                             else {
                                 unchangedResultCounter = 0;
                             }
                             if (currentTopHeight === 0) {
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             }
                             previousHeight = currentTopHeight;
                             break;
@@ -993,7 +1189,7 @@ class Interpreter extends events_1.EventEmitter {
                             visitedUrls.add(currentUrl);
                             yield scrapeCurrentPage();
                             if (checkLimit())
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             const { button, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
                             availableSelectors = updatedSelectors;
                             if (!button || !workingSelector) {
@@ -1009,7 +1205,7 @@ class Interpreter extends events_1.EventEmitter {
                                     }
                                 }));
                                 if (!success)
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 break;
                             }
                             let retryCount = 0;
@@ -1139,14 +1335,14 @@ class Interpreter extends events_1.EventEmitter {
                             }
                             if (!paginationSuccess) {
                                 debugLog(`Pagination failed after ${MAX_RETRIES} attempts`);
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             }
                             break;
                         }
                         case 'clickLoadMore': {
                             yield scrapeCurrentPage();
                             if (checkLimit())
-                                return allResults;
+                                return { results: allResults, urls: allUrls };
                             let loadMoreCounter = 0;
                             const MAX_LOAD_MORE_ITERATIONS = 100; // Prevent infinite load more
                             const loadMoreStartTime = Date.now();
@@ -1155,11 +1351,11 @@ class Interpreter extends events_1.EventEmitter {
                                 // Load more circuit breakers
                                 if (loadMoreCounter >= MAX_LOAD_MORE_ITERATIONS) {
                                     debugLog(`Maximum load more iterations reached (${MAX_LOAD_MORE_ITERATIONS}), stopping`);
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                                 if (Date.now() - loadMoreStartTime > MAX_LOAD_MORE_TIME) {
                                     debugLog('Maximum load more time reached (5 minutes), stopping');
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                                 // Add async yield every 3 iterations
                                 if (loadMoreCounter % 3 === 0 && loadMoreCounter > 0) {
@@ -1170,7 +1366,7 @@ class Interpreter extends events_1.EventEmitter {
                                 availableSelectors = updatedSelectors;
                                 if (!workingSelector || !loadMoreButton) {
                                     debugLog('No working Load More selector found after retries');
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                                 // Implement retry mechanism for clicking the button
                                 let retryCount = 0;
@@ -1210,7 +1406,7 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 if (!clickSuccess) {
                                     debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`);
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                                 // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
@@ -1239,16 +1435,16 @@ class Interpreter extends events_1.EventEmitter {
                                 //   previousResultCount = currentResultCount;
                                 // }
                                 if (checkLimit())
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 if (!heightChanged) {
                                     debugLog('No more items loaded after Load More');
-                                    return allResults;
+                                    return { results: allResults, urls: allUrls };
                                 }
                             }
                         }
                         default: {
                             yield scrapeCurrentPage();
-                            return allResults;
+                            return { results: allResults, urls: allUrls };
                         }
                     }
                     if (checkLimit())
@@ -1257,9 +1453,9 @@ class Interpreter extends events_1.EventEmitter {
             }
             catch (error) {
                 debugLog(`Fatal error: ${error.message}`);
-                return allResults;
+                return { results: allResults, urls: allUrls };
             }
-            return allResults;
+            return { results: allResults, urls: allUrls };
         });
     }
     getMatchingActionId(workflow, pageState, usedActions) {
@@ -2183,6 +2379,518 @@ class Interpreter extends events_1.EventEmitter {
             return modifiedAction;
         });
     }
+    /**
+     * Extracts URLs from the current page's list elements.
+     * Used during pagination to maintain sync between scraped results and extracted URLs.
+     *
+     * @param page - Playwright page object
+     * @param listSelector - The selector used to identify list elements
+     * @param limit - Maximum number of elements to process (should match number of scraped items)
+     * @returns Array of URL arrays, one per list element
+     */
+    extractUrlsFromCurrentPage(page, listSelector, limit) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const extractedUrls = yield page.evaluate(({ selector, limit }) => {
+                const urlsByElement = [];
+                let listElements = [];
+                if (selector.startsWith('//') || selector.startsWith('(//')) {
+                    const xpathResult = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                        const node = xpathResult.snapshotItem(i);
+                        if (node && node.nodeType === Node.ELEMENT_NODE) {
+                            listElements.push(node);
+                        }
+                    }
+                }
+                else {
+                    listElements = Array.from(document.querySelectorAll(selector));
+                }
+                // Extract URLs from the first 'limit' elements that match the selector
+                // The limit corresponds to the number of items that were scraped
+                const elementsToProcess = listElements.slice(0, limit);
+                elementsToProcess.forEach(element => {
+                    const urls = [];
+                    if (element.tagName === 'A' && element.href) {
+                        urls.push(element.href);
+                    }
+                    const anchors = element.querySelectorAll('a[href]');
+                    anchors.forEach(anchor => {
+                        const href = anchor.href;
+                        if (href && !urls.includes(href)) {
+                            urls.push(href);
+                        }
+                    });
+                    urlsByElement.push(urls);
+                });
+                return urlsByElement;
+            }, { selector: listSelector, limit });
+            return extractedUrls;
+        });
+    }
+    /**
+     * Builds a hierarchical deep extraction plan by analyzing the workflow structure.
+     * Identifies goto actions and determines what actions to execute at each level.
+     * Workflow is bottom-to-top, so we scan from end to start.
+     */
+    buildDeepExtractionHierarchy(currentWorkflow) {
+        var _a, _b;
+        const hierarchy = [];
+        // Find all goto action indices with their patterns
+        const gotoData = [];
+        currentWorkflow.forEach((pair, index) => {
+            var _a;
+            if (pair.what && pair.what.some(action => action.action === 'goto')) {
+                const gotoAction = pair.what.find(action => action.action === 'goto');
+                const pattern = (_a = gotoAction === null || gotoAction === void 0 ? void 0 : gotoAction.args) === null || _a === void 0 ? void 0 : _a[0];
+                if (pattern) {
+                    gotoData.push({ index, pattern: String(pattern) });
+                }
+            }
+        });
+        if (gotoData.length === 0) {
+            this.log('No goto actions found in workflow', logger_1.Level.WARN);
+            return [];
+        }
+        this.log(`Found ${gotoData.length} goto action(s) at indices: ${gotoData.map(g => g.index).join(', ')}`, logger_1.Level.LOG);
+        const uniqueGotos = [];
+        for (let i = 0; i < gotoData.length; i++) {
+            const current = gotoData[i];
+            const next = gotoData[i + 1];
+            if (next && current.pattern === next.pattern) {
+                this.log(`Skipping duplicate goto at index ${next.index} (same as ${current.index})`, logger_1.Level.LOG);
+                i++;
+            }
+            uniqueGotos.push(current);
+        }
+        this.log(`After deduplication: ${uniqueGotos.length} unique goto(s)`, logger_1.Level.LOG);
+        for (let i = 0; i < uniqueGotos.length; i++) {
+            const gotoIndex = uniqueGotos[i].index;
+            const gotoPattern = uniqueGotos[i].pattern;
+            const nextGotoIndex = i > 0 ? uniqueGotos[i - 1].index : 0;
+            let actionsToExecute = currentWorkflow.slice(nextGotoIndex, gotoIndex);
+            actionsToExecute = actionsToExecute.filter(pair => {
+                return !pair.what || !pair.what.some(action => action.action === 'goto');
+            });
+            const dataExtractionActions = actionsToExecute.filter(pair => {
+                return pair.what && pair.what.some(action => action.action === 'scrapeSchema' ||
+                    action.action === 'scrapeList' ||
+                    action.action === 'screenshot');
+            });
+            if (dataExtractionActions.length === 0) {
+                this.log(`No data extraction actions found between goto at ${gotoIndex} and next level`, logger_1.Level.WARN);
+                continue;
+            }
+            let sourceActionName = '';
+            let sourceActionType = 'scrapeList';
+            if (i === uniqueGotos.length - 1) {
+                const scrapeListBefore = currentWorkflow.slice(gotoIndex + 1).find(pair => pair.what && pair.what.some(action => action.action === 'scrapeList'));
+                if (scrapeListBefore) {
+                    const scrapeListAction = scrapeListBefore.what.find(action => action.action === 'scrapeList');
+                    sourceActionName = ((_b = (_a = scrapeListAction === null || scrapeListAction === void 0 ? void 0 : scrapeListAction.args) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.name) || (scrapeListAction === null || scrapeListAction === void 0 ? void 0 : scrapeListAction.name) || '';
+                    sourceActionType = 'scrapeList';
+                }
+            }
+            else {
+                sourceActionName = '';
+                sourceActionType = 'scrapeSchema';
+            }
+            hierarchy.push({
+                gotoActionIndex: gotoIndex,
+                gotoPattern: String(gotoPattern),
+                actionsToExecute: dataExtractionActions,
+                sourceActionName,
+                sourceActionType
+            });
+            this.log(`Level ${i}: goto at index ${gotoIndex}, pattern=${gotoPattern}, actions=${dataExtractionActions.length}`, logger_1.Level.LOG);
+        }
+        return hierarchy;
+    }
+    /**
+     * Extracts hrefs directly from the page based on scrapeSchema selectors.
+     * Checks ALL selectors from the schema config - if they point to anchor elements, extract href.
+     * This is called after scrapeSchema executes to capture hrefs for deep extraction.
+     */
+    extractHrefsFromPage(page, schemaConfig) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const fields = schemaConfig.fields || schemaConfig;
+                const selectors = [];
+                for (const [fieldName, fieldConfig] of Object.entries(fields)) {
+                    if (fieldConfig && typeof fieldConfig === 'object' && fieldConfig.selector) {
+                        selectors.push(String(fieldConfig.selector));
+                    }
+                }
+                if (selectors.length === 0) {
+                    return [];
+                }
+                const extractedUrls = yield page.evaluate((selectorList) => {
+                    const urls = [];
+                    for (const selector of selectorList) {
+                        if (!selector)
+                            continue;
+                        try {
+                            let elements = [];
+                            if (selector.startsWith('//') || selector.startsWith('(//')) {
+                                const xpathResult = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                                for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                                    const node = xpathResult.snapshotItem(i);
+                                    if (node && node.nodeType === Node.ELEMENT_NODE) {
+                                        elements.push(node);
+                                    }
+                                }
+                            }
+                            else {
+                                elements = Array.from(document.querySelectorAll(selector));
+                            }
+                            for (const element of elements) {
+                                if (element.tagName === 'A' && element.href) {
+                                    const href = element.href;
+                                    if (href && !urls.includes(href)) {
+                                        urls.push(href);
+                                    }
+                                }
+                            }
+                        }
+                        catch (error) {
+                            console.warn(`Failed to extract hrefs for selector ${selector}:`, error);
+                        }
+                    }
+                    return urls;
+                }, selectors);
+                this.log(`Extracted ${extractedUrls.length} hrefs from page for schema selectors`, logger_1.Level.LOG);
+                return extractedUrls;
+            }
+            catch (error) {
+                this.log(`Failed to extract hrefs from page: ${error.message}`, logger_1.Level.ERROR);
+                return [];
+            }
+        });
+    }
+    /**
+     * Filters URLs for deep extraction based on the goto action pattern.
+     * This is called immediately after the first capture action (scrapeList).
+     * Returns the filtered URL mappings that should be processed after workflow completion.
+     * Each mapping maintains alignment with the original scrapeList index.
+     */
+    filterDeepExtractionUrls(page, listSelector, scrapeResults, gotoTargetPattern) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                this.log(`Deep extraction: Filtering URLs from list structure (${scrapeResults.length} items)`, logger_1.Level.LOG);
+                const extractedUrls = yield page.evaluate(({ selector, limit }) => {
+                    const urlsByElement = [];
+                    let listElements = [];
+                    if (selector.startsWith('//') || selector.startsWith('(//')) {
+                        const xpathResult = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                        for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                            const node = xpathResult.snapshotItem(i);
+                            if (node && node.nodeType === Node.ELEMENT_NODE) {
+                                listElements.push(node);
+                            }
+                        }
+                    }
+                    else {
+                        listElements = Array.from(document.querySelectorAll(selector));
+                    }
+                    const elementsToProcess = listElements.slice(0, limit);
+                    elementsToProcess.forEach(element => {
+                        const urls = [];
+                        if (element.tagName === 'A' && element.href) {
+                            urls.push(element.href);
+                        }
+                        const anchors = element.querySelectorAll('a[href]');
+                        anchors.forEach(anchor => {
+                            const href = anchor.href;
+                            if (href && !urls.includes(href)) {
+                                urls.push(href);
+                            }
+                        });
+                        urlsByElement.push(urls);
+                    });
+                    return urlsByElement;
+                }, { selector: listSelector, limit: scrapeResults.length });
+                const totalUrlCount = extractedUrls.reduce((sum, urls) => sum + urls.length, 0);
+                this.log(`Extracted ${totalUrlCount} total URLs from ${scrapeResults.length} list items (avg ${(totalUrlCount / scrapeResults.length).toFixed(1)} URLs per item)`, logger_1.Level.LOG);
+                const getUrlPattern = (url) => {
+                    try {
+                        const urlObj = new URL(url);
+                        const pathname = urlObj.pathname.replace(/\/$/, '');
+                        const segments = pathname.split('/').filter(s => s.length > 0);
+                        return {
+                            origin: urlObj.origin,
+                            pathSegments: segments
+                        };
+                    }
+                    catch (_a) {
+                        return null;
+                    }
+                };
+                const targetPattern = getUrlPattern(String(gotoTargetPattern));
+                const targetNormalized = String(gotoTargetPattern).replace(/\/$/, '').toLowerCase();
+                if (!targetPattern) {
+                    this.log('Could not parse goto URL pattern, skipping deep extraction', logger_1.Level.WARN);
+                    return [];
+                }
+                this.log(`Target URL pattern: ${targetPattern.origin}/${targetPattern.pathSegments.join('/')}`, logger_1.Level.LOG);
+                const urlMappings = [];
+                extractedUrls.forEach((urlsFromElement, scrapeListIndex) => {
+                    let matchingUrl = null;
+                    for (const url of urlsFromElement) {
+                        const urlPattern = getUrlPattern(url);
+                        if (!urlPattern)
+                            continue;
+                        if (urlPattern.origin !== targetPattern.origin)
+                            continue;
+                        if (urlPattern.pathSegments.length !== targetPattern.pathSegments.length)
+                            continue;
+                        let pathMatches = true;
+                        for (let i = 0; i < targetPattern.pathSegments.length - 1; i++) {
+                            if (urlPattern.pathSegments[i] !== targetPattern.pathSegments[i]) {
+                                pathMatches = false;
+                                break;
+                            }
+                        }
+                        if (!pathMatches)
+                            continue;
+                        const urlNormalized = url.replace(/\/$/, '').toLowerCase();
+                        if (urlNormalized === targetNormalized) {
+                            this.log(`Excluding already-visited URL: ${url}`, logger_1.Level.LOG);
+                            continue;
+                        }
+                        matchingUrl = url;
+                        break;
+                    }
+                    urlMappings.push({
+                        scrapeListIndex,
+                        url: matchingUrl
+                    });
+                });
+                const matchedCount = urlMappings.filter(m => m.url !== null).length;
+                this.log(`Filtered to ${matchedCount} matching URLs for deep extraction (out of ${scrapeResults.length} total items)`, logger_1.Level.LOG);
+                if (matchedCount > 0) {
+                    const matchedMappings = urlMappings.filter(m => m.url !== null);
+                    const sampleSize = Math.min(5, matchedMappings.length);
+                    const sample = matchedMappings.slice(0, sampleSize);
+                    this.log(`Sample of matching URLs (showing ${sampleSize} of ${matchedMappings.length}):`, logger_1.Level.LOG);
+                    sample.forEach((mapping, idx) => {
+                        this.log(`  ${idx + 1}. [Index ${mapping.scrapeListIndex}] ${mapping.url}`, logger_1.Level.LOG);
+                    });
+                }
+                else {
+                    this.log('No matching URLs found. Check if extracted URLs match the pattern.', logger_1.Level.WARN);
+                }
+                return urlMappings;
+            }
+            catch (error) {
+                this.log(`URL filtering failed: ${error.message}`, logger_1.Level.ERROR);
+                return [];
+            }
+        });
+    }
+    /**
+     * Filters pre-extracted URLs for deep extraction based on the goto action pattern.
+     * This is used for paginated lists where URLs were extracted during pagination.
+     * Returns the filtered URL mappings that maintain alignment with scrapeList indices.
+     */
+    filterDeepExtractionUrlsFromExtracted(extractedUrls, scrapeResults, gotoTargetPattern) {
+        try {
+            const totalUrlCount = extractedUrls.reduce((sum, urls) => sum + urls.length, 0);
+            this.log(`Deep extraction: Filtering ${totalUrlCount} pre-extracted URLs from ${scrapeResults.length} items`, logger_1.Level.LOG);
+            const getUrlPattern = (url) => {
+                try {
+                    const urlObj = new URL(url);
+                    const pathname = urlObj.pathname.replace(/\/$/, '');
+                    const segments = pathname.split('/').filter(s => s.length > 0);
+                    return {
+                        origin: urlObj.origin,
+                        pathSegments: segments
+                    };
+                }
+                catch (_a) {
+                    return null;
+                }
+            };
+            const targetPattern = getUrlPattern(String(gotoTargetPattern));
+            const targetNormalized = String(gotoTargetPattern).replace(/\/$/, '').toLowerCase();
+            if (!targetPattern) {
+                this.log('Could not parse goto URL pattern, skipping deep extraction', logger_1.Level.WARN);
+                return [];
+            }
+            this.log(`Target URL pattern: ${targetPattern.origin}/${targetPattern.pathSegments.join('/')}`, logger_1.Level.LOG);
+            const urlMappings = [];
+            extractedUrls.forEach((urlsFromElement, scrapeListIndex) => {
+                let matchingUrl = null;
+                for (const url of urlsFromElement) {
+                    const urlPattern = getUrlPattern(url);
+                    if (!urlPattern)
+                        continue;
+                    if (urlPattern.origin !== targetPattern.origin)
+                        continue;
+                    if (urlPattern.pathSegments.length !== targetPattern.pathSegments.length)
+                        continue;
+                    let pathMatches = true;
+                    for (let i = 0; i < targetPattern.pathSegments.length - 1; i++) {
+                        if (urlPattern.pathSegments[i] !== targetPattern.pathSegments[i]) {
+                            pathMatches = false;
+                            break;
+                        }
+                    }
+                    if (!pathMatches)
+                        continue;
+                    const urlNormalized = url.replace(/\/$/, '').toLowerCase();
+                    if (urlNormalized === targetNormalized) {
+                        this.log(`Excluding already-visited URL: ${url}`, logger_1.Level.LOG);
+                        continue;
+                    }
+                    matchingUrl = url;
+                    break;
+                }
+                urlMappings.push({
+                    scrapeListIndex,
+                    url: matchingUrl
+                });
+            });
+            const matchedCount = urlMappings.filter(m => m.url !== null).length;
+            this.log(`Filtered to ${matchedCount} matching URLs for deep extraction (out of ${scrapeResults.length} total items)`, logger_1.Level.LOG);
+            if (matchedCount > 0) {
+                const matchedMappings = urlMappings.filter(m => m.url !== null);
+                const sampleSize = Math.min(5, matchedMappings.length);
+                const sample = matchedMappings.slice(0, sampleSize);
+                this.log(`Sample of matching URLs (showing ${sampleSize} of ${matchedMappings.length}):`, logger_1.Level.LOG);
+                sample.forEach((mapping, idx) => {
+                    this.log(`  ${idx + 1}. [Index ${mapping.scrapeListIndex}] ${mapping.url}`, logger_1.Level.LOG);
+                });
+            }
+            else {
+                this.log('No matching URLs found. Check if extracted URLs match the pattern.', logger_1.Level.WARN);
+            }
+            return urlMappings;
+        }
+        catch (error) {
+            this.log(`URL filtering failed: ${error.message}`, logger_1.Level.ERROR);
+            return [];
+        }
+    }
+    /**
+     * Helper function to check if a URL matches a goto pattern.
+     */
+    matchesGotoPattern(url, gotoPattern) {
+        try {
+            const getUrlPattern = (urlStr) => {
+                try {
+                    const urlObj = new URL(urlStr);
+                    const pathname = urlObj.pathname.replace(/\/$/, '');
+                    const segments = pathname.split('/').filter(s => s.length > 0);
+                    return { origin: urlObj.origin, pathSegments: segments };
+                }
+                catch (_a) {
+                    return null;
+                }
+            };
+            const urlPattern = getUrlPattern(url);
+            const targetPattern = getUrlPattern(gotoPattern);
+            const targetNormalized = gotoPattern.replace(/\/$/, '').toLowerCase();
+            const urlNormalized = url.replace(/\/$/, '').toLowerCase();
+            if (!urlPattern || !targetPattern)
+                return false;
+            if (urlPattern.origin !== targetPattern.origin)
+                return false;
+            if (urlPattern.pathSegments.length !== targetPattern.pathSegments.length)
+                return false;
+            if (urlNormalized === targetNormalized)
+                return false; // Skip exact matches
+            for (let i = 0; i < targetPattern.pathSegments.length - 1; i++) {
+                if (urlPattern.pathSegments[i] !== targetPattern.pathSegments[i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        catch (_a) {
+            return false;
+        }
+    }
+    /**
+     * Executes hierarchical deep extraction by processing each level recursively.
+     * URLs are already stored in each hierarchy level's urlMappings during workflow execution.
+     */
+    executeHierarchicalDeepExtraction(page, hierarchy) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                if (hierarchy.length === 0) {
+                    this.log('No hierarchy levels to process', logger_1.Level.LOG);
+                    return;
+                }
+                this.log(`\n=== Starting Hierarchical Deep Extraction (${hierarchy.length} level${hierarchy.length > 1 ? 's' : ''}) ===`, logger_1.Level.LOG);
+                this.isInDeepExtractionPhase = true;
+                const startLevel = hierarchy.length >= 2 ? hierarchy.length - 2 : hierarchy.length - 1;
+                for (let levelIndex = startLevel; levelIndex >= 0; levelIndex--) {
+                    const level = hierarchy[levelIndex];
+                    const currentLevelUrls = level.urlMappings;
+                    this.log(`\n=== Processing Deep Extraction Level ${startLevel - levelIndex + 1}/${startLevel + 1} ===`, logger_1.Level.LOG);
+                    this.log(`Goto pattern: ${level.gotoPattern}`, logger_1.Level.LOG);
+                    this.log(`Actions to execute: ${level.actionsToExecute.length}`, logger_1.Level.LOG);
+                    this.log(`URLs to process: ${currentLevelUrls.filter(m => m.url !== null).length}`, logger_1.Level.LOG);
+                    if (currentLevelUrls.length === 0 || currentLevelUrls.every(u => !u.url)) {
+                        this.log('No valid URLs at this level - stopping here', logger_1.Level.LOG);
+                        break;
+                    }
+                    yield this.executeDeepExtractionLevel(page, level, currentLevelUrls);
+                }
+                this.log('\n=== Hierarchical Deep Extraction Completed ===', logger_1.Level.LOG);
+            }
+            catch (error) {
+                this.log(`Hierarchical deep extraction failed: ${error.message}`, logger_1.Level.ERROR);
+            }
+            finally {
+                this.isInDeepExtractionPhase = false;
+            }
+        });
+    }
+    /**
+     * Executes deep extraction for a single level.
+     * URLs are already extracted and stored in hierarchy during workflow execution.
+     * This function just navigates to URLs and executes the capture actions.
+     */
+    executeDeepExtractionLevel(page, level, urlMappings) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const validMappings = urlMappings.filter(m => m.url !== null);
+                if (validMappings.length === 0) {
+                    this.log('No URLs to process for this level', logger_1.Level.LOG);
+                    return;
+                }
+                this.log(`Processing ${validMappings.length} URLs`, logger_1.Level.LOG);
+                for (const mapping of validMappings) {
+                    try {
+                        this.log(`[${mapping.index}] Navigating to: ${mapping.url}`, logger_1.Level.LOG);
+                        yield page.goto(mapping.url);
+                        yield page.waitForLoadState('networkidle', { timeout: 30000 });
+                        for (let i = level.actionsToExecute.length - 1; i >= 0; i--) {
+                            const actionPair = level.actionsToExecute[i];
+                            if (this.isAborted) {
+                                this.log('Workflow aborted during deep extraction', logger_1.Level.WARN);
+                                return;
+                            }
+                            const validatedAction = yield this.validateAndFixSelectors(page, actionPair);
+                            const filteredActions = validatedAction.what.filter(action => action.action === 'scrapeSchema' ||
+                                action.action === 'scrapeList' ||
+                                action.action === 'screenshot');
+                            if (filteredActions.length > 0) {
+                                yield this.carryOutSteps(page, filteredActions);
+                            }
+                        }
+                        this.log(`[${mapping.index}] Completed`, logger_1.Level.LOG);
+                    }
+                    catch (error) {
+                        this.log(`[${mapping.index}] Failed: ${error.message}`, logger_1.Level.ERROR);
+                    }
+                }
+            }
+            catch (error) {
+                this.log(`Level execution failed: ${error.message}`, logger_1.Level.ERROR);
+            }
+        });
+    }
     runLoop(p, workflow) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
@@ -2267,6 +2975,20 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 if (workflowCopy.length === 0) {
                     this.log('All actions completed. Workflow finished.', logger_1.Level.LOG);
+                    if (this.pendingDeepExtraction) {
+                        this.log('Starting deferred hierarchical deep extraction now that workflow has completed...', logger_1.Level.LOG);
+                        const { page, hierarchy } = this.pendingDeepExtraction;
+                        try {
+                            yield this.executeHierarchicalDeepExtraction(page, hierarchy);
+                            this.log('Hierarchical deep extraction completed successfully', logger_1.Level.LOG);
+                        }
+                        catch (error) {
+                            this.log(`Hierarchical deep extraction failed: ${error.message}`, logger_1.Level.ERROR);
+                        }
+                        finally {
+                            this.pendingDeepExtraction = null;
+                        }
+                    }
                     cleanup();
                     return;
                 }
@@ -2335,7 +3057,7 @@ class Interpreter extends events_1.EventEmitter {
                     try {
                         const validatedAction = yield this.validateAndFixSelectors(p, action);
                         console.log("Carrying out:", validatedAction.what);
-                        yield this.carryOutSteps(p, validatedAction.what);
+                        yield this.carryOutSteps(p, validatedAction.what, workflowCopy);
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",