mx-cloud 0.0.25 → 0.0.27

package/build/interpret.js

@@ -49,7 +49,6 @@ const adblocker_playwright_1 = require("@cliqz/adblocker-playwright");
 const cross_fetch_1 = __importDefault(require("cross-fetch"));
 const path_1 = __importDefault(require("path"));
 const events_1 = require("events");
-const logic_1 = require("./types/logic");
 const utils_1 = require("./utils/utils");
 const concurrency_1 = __importDefault(require("./utils/concurrency"));
 const preprocessor_1 = __importDefault(require("./preprocessor"));
@@ -72,10 +71,21 @@ class Interpreter extends events_1.EventEmitter {
         this.scrapeListCounter = 0;
         this.serializableDataByType = {
             scrapeList: {},
-            scrapeSchema: {}
+            scrapeSchema: {},
+            crawl: {},
+            search: {}
         };
         this.pendingDeepExtraction = null;
         this.isInDeepExtractionPhase = false;
+        this.deepExtractionStats = {
+            totalUrlsFound: 0,
+            matchedUrls: 0,
+            successfulExtractions: 0,
+            failedExtractions: 0,
+            skippedDueToPattern: 0
+        };
+        this.totalActions = 0;
+        this.executedActions = 0;
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
         this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
@@ -134,189 +144,12 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
-    // private getSelectors(workflow: Workflow, actionId: number): string[] {
-    //   const selectors: string[] = [];
-    //   // Validate actionId
-    //   if (actionId <= 0) {
-    //       console.log("No previous selectors to collect.");
-    //       return selectors; // Empty array as there are no previous steps
-    //   }
-    //   // Iterate from the start up to (but not including) actionId
-    //   for (let index = 0; index < actionId; index++) {
-    //       const currentSelectors = workflow[index]?.where?.selectors;
-    //       console.log(`Selectors at step ${index}:`, currentSelectors);
-    //       if (currentSelectors && currentSelectors.length > 0) {
-    //           currentSelectors.forEach((selector) => {
-    //               if (!selectors.includes(selector)) {
-    //                   selectors.push(selector); // Avoid duplicates
-    //               }
-    //           });
-    //       }
-    //   }
-    //   console.log("Collected Selectors:", selectors);
-    //   return selectors;
-    // }
-    getSelectors(workflow) {
-        var _a, _b;
-        const selectorsSet = new Set();
-        if (workflow.length === 0) {
-            return [];
-        }
-        for (let index = workflow.length - 1; index >= 0; index--) {
-            const currentSelectors = (_b = (_a = workflow[index]) === null || _a === void 0 ? void 0 : _a.where) === null || _b === void 0 ? void 0 : _b.selectors;
-            if (currentSelectors && currentSelectors.length > 0) {
-                currentSelectors.forEach((selector) => selectorsSet.add(selector));
-                return Array.from(selectorsSet);
-            }
-        }
-        return [];
-    }
-    /**
-      * Returns the context object from given Page and the current workflow.\
-      * \
-      * `workflow` is used for selector extraction - function searches for used selectors to
-      * look for later in the page's context.
-      * @param page Playwright Page object
-      * @param workflow Current **initialized** workflow (array of where-what pairs).
-      * @returns {PageState} State of the current page.
-      */
-    getState(page, workflowCopy, selectors) {
-        return __awaiter(this, void 0, void 0, function* () {
-            /**
-             * All the selectors present in the current Workflow
-             */
-            // const selectors = Preprocessor.extractSelectors(workflow);
-            // console.log("Current selectors:", selectors);
-            /**
-              * Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
-              * @param selector Selector to be queried
-              * @returns True if the targetted element is actionable, false otherwise.
-              */
-            // const actionable = async (selector: string): Promise<boolean> => {
-            //   try {
-            //     const proms = [
-            //       page.isEnabled(selector, { timeout: 10000 }),
-            //       page.isVisible(selector, { timeout: 10000 }),
-            //     ];
-            //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
-            //   } catch (e) {
-            //     // log(<Error>e, Level.ERROR);
-            //     return false;
-            //   }
-            // };
-            /**
-              * Object of selectors present in the current page.
-              */
-            // const presentSelectors: SelectorArray = await Promise.all(
-            //   selectors.map(async (selector) => {
-            //     if (await actionable(selector)) {
-            //       return [selector];
-            //     }
-            //     return [];
-            //   }),
-            // ).then((x) => x.flat());
-            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
-                try {
-                    yield page.waitForSelector(selector, { state: 'attached' });
-                    return [selector];
-                }
-                catch (e) {
-                    return [];
-                }
-            }))).then((x) => x.flat());
-            const action = workflowCopy[workflowCopy.length - 1];
-            // console.log("Next action:", action)
-            let url = page.url();
-            if (action && action.where.url !== url && action.where.url !== "about:blank") {
-                url = action.where.url;
-            }
-            return {
-                url,
-                cookies: (yield page.context().cookies([page.url()]))
-                    .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
-                selectors: presentSelectors,
-            };
-        });
-    }
-    /**
-     * Tests if the given action is applicable with the given context.
-     * @param where Tested *where* condition
-     * @param context Current browser context.
-     * @returns True if `where` is applicable in the given context, false otherwise
-     */
-    applicable(where, context, usedActions = []) {
-        /**
-         * Given two arbitrary objects, determines whether `subset` is a subset of `superset`.\
-         * \
-         * For every key in `subset`, there must be a corresponding key with equal scalar
-         * value in `superset`, or `inclusive(subset[key], superset[key])` must hold.
-         * @param subset Arbitrary non-cyclic JS object (where clause)
-         * @param superset Arbitrary non-cyclic JS object (browser context)
-         * @returns `true` if `subset <= superset`, `false` otherwise.
-         */
-        const inclusive = (subset, superset) => (Object.entries(subset).every(([key, value]) => {
-            /**
-             * Arrays are compared without order (are transformed into objects before comparison).
-             */
-            const parsedValue = Array.isArray(value) ? (0, utils_1.arrayToObject)(value) : value;
-            const parsedSuperset = {};
-            parsedSuperset[key] = Array.isArray(superset[key])
-                ? (0, utils_1.arrayToObject)(superset[key])
-                : superset[key];
-            if ((key === 'url' || key === 'selectors') &&
-                Array.isArray(value) && Array.isArray(superset[key]) &&
-                value.length === 0 && superset[key].length === 0) {
-                return true;
-            }
-            if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) {
-                return value.some(selector => superset[key].includes(selector));
-            }
-            // Every `subset` key must exist in the `superset` and
-            // have the same value (strict equality), or subset[key] <= superset[key]
-            return parsedSuperset[key]
-                && ((parsedSuperset[key] === parsedValue)
-                    || ((parsedValue).constructor.name === 'RegExp' && parsedValue.test(parsedSuperset[key]))
-                    || ((parsedValue).constructor.name !== 'RegExp'
-                        && typeof parsedValue === 'object' && inclusive(parsedValue, parsedSuperset[key])));
-        }));
-        // Every value in the "where" object should be compliant to the current state.
-        return Object.entries(where).every(([key, value]) => {
-            if (logic_1.operators.includes(key)) {
-                const array = Array.isArray(value)
-                    ? value
-                    : Object.entries(value).map((a) => Object.fromEntries([a]));
-                // every condition is treated as a single context
-                switch (key) {
-                    case '$and':
-                        return array === null || array === void 0 ? void 0 : array.every((x) => this.applicable(x, context));
-                    case '$or':
-                        return array === null || array === void 0 ? void 0 : array.some((x) => this.applicable(x, context));
-                    case '$not':
-                        return !this.applicable(value, context); // $not should be a unary operator
-                    default:
-                        throw new Error('Undefined logic operator.');
-                }
-            }
-            else if (logic_1.meta.includes(key)) {
-                const testRegexString = (x) => {
-                    if (typeof value === 'string') {
-                        return x === value;
-                    }
-                    return value.test(x);
-                };
-                switch (key) {
-                    case '$before':
-                        return !usedActions.find(testRegexString);
-                    case '$after':
-                        return !!usedActions.find(testRegexString);
-                    default:
-                        throw new Error('Undefined meta operator.');
-                }
-            }
-            else {
-                // Current key is a base condition (url, cookies, selectors)
-                return inclusive({ [key]: value }, context);
-            }
+    callWithTimeout(callback_1) {
+        return __awaiter(this, arguments, void 0, function* (callback, timeoutMs = 30000, operationName = 'callback') {
+            return Promise.race([
+                Promise.resolve(callback()),
+                new Promise((_, reject) => setTimeout(() => reject(new Error(`${operationName} timeout after ${timeoutMs}ms`)), timeoutMs))
+            ]);
         });
     }
     /**
@@ -363,25 +196,20 @@ class Interpreter extends events_1.EventEmitter {
                         this.options.debugChannel.setActionType("screenshot");
                     }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
-                    // Prefer explicit nameOverride (from workflow step.name or computed action name)
-                    // If nameOverride is provided (non-empty) use it *as-is*.
-                    // Only use counter-appended name when no nameOverride is available.
                     const explicitName = (typeof nameOverride === 'string' && nameOverride.trim().length > 0) ? nameOverride.trim() : null;
                     let screenshotName;
                     if (explicitName) {
                         screenshotName = explicitName;
                     }
                     else {
-                        // If no explicit name, produce a readable generated name with a counter
                         this.screenshotCounter += 1;
                         screenshotName = `Screenshot ${this.screenshotCounter}`;
                     }
-                    // Pass structured metadata (name included) to binaryCallback
-                    yield this.options.binaryCallback({
+                    yield this.callWithTimeout(() => this.options.binaryCallback({
                         name: screenshotName,
                         data: screenshotBuffer,
                         mimeType: "image/png",
-                    }, "image/png");
+                    }, "image/png"), 30000, 'binaryCallback (screenshot)');
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
@@ -430,7 +258,7 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
-                    yield this.options.serializableCallback(scrapeResults);
+                    yield this.callWithTimeout(() => this.options.serializableCallback(scrapeResults), 30000, 'serializableCallback (scrape)');
                 }),
                 scrapeSchema: (schema_1, ...args_1) => __awaiter(this, [schema_1, ...args_1], void 0, function* (schema, actionName = "") {
                     var _a;
@@ -442,7 +270,7 @@ class Interpreter extends events_1.EventEmitter {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
                     if (this.options.mode && this.options.mode === 'editor') {
-                        yield this.options.serializableCallback({});
+                        yield this.callWithTimeout(() => this.options.serializableCallback({}), 30000, 'serializableCallback (scrapeSchema editor mode)');
                         return;
                     }
                     yield this.ensureScriptsLoaded(page);
@@ -492,22 +320,32 @@ class Interpreter extends events_1.EventEmitter {
                         this.serializableDataByType[actionType][name] = [];
                     }
                     this.serializableDataByType[actionType][name] = [...this.cumulativeResults];
-                    yield this.options.serializableCallback({
+                    yield this.callWithTimeout(() => this.options.serializableCallback({
                         scrapeList: this.serializableDataByType.scrapeList,
                         scrapeSchema: this.serializableDataByType.scrapeSchema
-                    });
+                    }), 30000, 'serializableCallback (scrapeSchema)');
+                    const MAX_CUMULATIVE_RESULTS = 1000;
+                    if (this.cumulativeResults.length > MAX_CUMULATIVE_RESULTS) {
+                        this.cumulativeResults = this.cumulativeResults.slice(-500);
+                    }
+                    const MAX_STORED_SCHEMAS = 50;
+                    const schemaKeys = Object.keys(this.serializableDataByType[actionType]);
+                    if (schemaKeys.length > MAX_STORED_SCHEMAS) {
+                        const sortedKeys = schemaKeys.sort();
+                        const keysToRemove = sortedKeys.slice(0, schemaKeys.length - MAX_STORED_SCHEMAS);
+                        keysToRemove.forEach(key => {
+                            delete this.serializableDataByType[actionType][key];
+                        });
+                    }
                     if (this.options.robotType === 'deep-extract' && !this.isInDeepExtractionPhase && this.initializedWorkflow) {
                         if (!this.pendingDeepExtraction) {
-                            console.log('DEBUG: Building hierarchical deep extraction plan from scrapeSchema...');
                             const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
                             if (hierarchyData.length > 0) {
-                                const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                const nextLevelIndex = 0;
                                 const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
                                 this.log(`Root scrapeSchema will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
-                                // Extract URLs from schema fields
                                 const urls = yield this.extractHrefsFromPage(page, schema);
                                 this.log(`scrapeSchema extracted ${urls.length} URLs from field selectors`, logger_1.Level.LOG);
-                                // Filter URLs against pattern
                                 const rootUrlMappings = urls
                                     .map((url, index) => ({
                                     scrapeListIndex: index,
@@ -524,9 +362,6 @@ class Interpreter extends events_1.EventEmitter {
                                     }))
                                 };
                             }
-                            else {
-                                console.log('DEBUG: No goto actions found, deep extraction skipped');
-                            }
                         }
                         else {
                             this.log(`[Deep Extract] scrapeSchema "${name}" extracting URLs during workflow execution`, logger_1.Level.LOG);
@@ -585,7 +420,7 @@ class Interpreter extends events_1.EventEmitter {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
                     if (this.options.mode && this.options.mode === 'editor') {
-                        yield this.options.serializableCallback({});
+                        yield this.callWithTimeout(() => this.options.serializableCallback({}), 30000, 'serializableCallback (scrapeList editor mode)');
                         return;
                     }
                     try {
@@ -613,10 +448,9 @@ class Interpreter extends events_1.EventEmitter {
                             const paginationUrls = paginationResult.urls;
                             if (this.options.robotType === 'deep-extract' && this.initializedWorkflow && scrapeResults.length > 0) {
                                 if (!this.pendingDeepExtraction) {
-                                    console.log('DEBUG: Building hierarchical deep extraction plan from pagination...');
                                     const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
                                     if (hierarchyData.length > 0) {
-                                        const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                        const nextLevelIndex = 0;
                                         const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
                                         this.log(`Root scrapeList (pagination) will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
                                         const rootUrlMappings = this.filterDeepExtractionUrlsFromExtracted(paginationUrls, scrapeResults, nextLevelGotoPattern);
@@ -625,6 +459,9 @@ class Interpreter extends events_1.EventEmitter {
                                             hierarchy: hierarchyData.map((level, idx) => ({
                                                 gotoPattern: level.gotoPattern,
                                                 actionsToExecute: level.actionsToExecute,
+                                                sourceActionName: level.sourceActionName,
+                                                sourceActionType: level.sourceActionType,
+                                                deepExtractionLimit: level.deepExtractionLimit,
                                                 urlMappings: idx === nextLevelIndex ? rootUrlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url })) : []
                                             }))
                                         };
@@ -634,12 +471,30 @@ class Interpreter extends events_1.EventEmitter {
                                     this.log(`[Deep Extract] scrapeList (pagination) "${actionName}" extracting URLs`, logger_1.Level.LOG);
                                     const hierarchy = this.pendingDeepExtraction.hierarchy;
                                     if (hierarchy && hierarchy.length > 0) {
-                                        const nextLevelIndex = hierarchy.length >= 3 ? hierarchy.length - 3 : 0;
-                                        if (nextLevelIndex >= 0 && hierarchy[nextLevelIndex]) {
-                                            const nextGotoPattern = hierarchy[nextLevelIndex].gotoPattern;
+                                        let targetLevelIndex = -1;
+                                        for (let i = hierarchy.length - 1; i >= 0; i--) {
+                                            if (hierarchy[i].urlMappings.length === 0) {
+                                                targetLevelIndex = i;
+                                                break;
+                                            }
+                                        }
+                                        if (targetLevelIndex >= 0) {
+                                            const nextGotoPattern = hierarchy[targetLevelIndex].gotoPattern;
                                             this.log(`[Deep Extract] Extracting URLs for pattern: ${nextGotoPattern}`, logger_1.Level.LOG);
                                             const urlMappings = this.filterDeepExtractionUrlsFromExtracted(paginationUrls, scrapeResults, nextGotoPattern);
                                             this.log(`[Deep Extract] Found ${urlMappings.filter(m => m.url !== null).length} matching URLs`, logger_1.Level.LOG);
+                                            if (hierarchy[targetLevelIndex].urlMappings.length > 0) {
+                                                const existingUrls = new Set(hierarchy[targetLevelIndex].urlMappings.map(m => m.url).filter(u => u !== null));
+                                                const newUrls = urlMappings.filter(m => m.url !== null && !existingUrls.has(m.url));
+                                                if (newUrls.length > 0) {
+                                                    const startIndex = hierarchy[targetLevelIndex].urlMappings.length;
+                                                    hierarchy[targetLevelIndex].urlMappings.push(...newUrls.map((m, idx) => ({ index: startIndex + idx, url: m.url })));
+                                                    this.log(`[Deep Extract] Merged ${newUrls.length} new URLs from pagination`, logger_1.Level.LOG);
+                                                }
+                                            }
+                                            else {
+                                                hierarchy[targetLevelIndex].urlMappings = urlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url }));
+                                            }
                                             const validUrls = urlMappings.filter(m => m.url !== null);
                                             if (validUrls.length > 0) {
                                                 const sampleSize = Math.min(3, validUrls.length);
@@ -669,17 +524,24 @@ class Interpreter extends events_1.EventEmitter {
                                 this.serializableDataByType[actionType][name] = [];
                             }
                             this.serializableDataByType[actionType][name].push(...scrapeResults);
-                            yield this.options.serializableCallback({
+                            yield this.callWithTimeout(() => this.options.serializableCallback({
                                 scrapeList: this.serializableDataByType.scrapeList,
                                 scrapeSchema: this.serializableDataByType.scrapeSchema
-                            });
-                            console.log(`DEBUG: Checking deep extract condition: robotType=${this.options.robotType}, hasWorkflow=${!!currentWorkflow}, alreadyPending=${!!this.pendingDeepExtraction}`);
+                            }), 30000, 'serializableCallback (scrapeList)');
+                            const MAX_STORED_LISTS = 50;
+                            const listKeys = Object.keys(this.serializableDataByType[actionType]);
+                            if (listKeys.length > MAX_STORED_LISTS) {
+                                const sortedKeys = listKeys.sort();
+                                const keysToRemove = sortedKeys.slice(0, listKeys.length - MAX_STORED_LISTS);
+                                keysToRemove.forEach(key => {
+                                    delete this.serializableDataByType[actionType][key];
+                                });
+                            }
                             if (this.options.robotType === 'deep-extract' && !this.isInDeepExtractionPhase && this.initializedWorkflow) {
                                 if (!this.pendingDeepExtraction) {
-                                    console.log('DEBUG: Building hierarchical deep extraction plan...');
                                     const hierarchyData = this.buildDeepExtractionHierarchy(this.initializedWorkflow);
                                     if (hierarchyData.length > 0) {
-                                        const nextLevelIndex = hierarchyData.length >= 2 ? hierarchyData.length - 2 : hierarchyData.length - 1;
+                                        const nextLevelIndex = 0;
                                         const nextLevelGotoPattern = hierarchyData[nextLevelIndex].gotoPattern;
                                         this.log(`Root scrapeList will extract URLs matching pattern: ${nextLevelGotoPattern}`, logger_1.Level.LOG);
                                         const rootUrlMappings = yield this.filterDeepExtractionUrls(page, config.listSelector, scrapeResults, nextLevelGotoPattern);
@@ -688,13 +550,13 @@ class Interpreter extends events_1.EventEmitter {
                                             hierarchy: hierarchyData.map((level, idx) => ({
                                                 gotoPattern: level.gotoPattern,
                                                 actionsToExecute: level.actionsToExecute,
+                                                sourceActionName: level.sourceActionName,
+                                                sourceActionType: level.sourceActionType,
+                                                deepExtractionLimit: level.deepExtractionLimit,
                                                 urlMappings: idx === nextLevelIndex ? rootUrlMappings.map(m => ({ index: m.scrapeListIndex, url: m.url })) : []
                                             }))
                                         };
                                     }
-                                    else {
-                                        console.log('DEBUG: No goto actions found, deep extraction skipped');
-                                    }
                                 }
                                 else {
                                     this.log(`[Deep Extract] scrapeList "${name}" extracting URLs during workflow execution`, logger_1.Level.LOG);
@@ -767,7 +629,7 @@ class Interpreter extends events_1.EventEmitter {
                     const scrapeResults = yield page.evaluate((listSelector) => {
                         return window.scrapeListAuto(listSelector);
                     }, config.listSelector);
-                    yield this.options.serializableCallback(scrapeResults);
+                    yield this.callWithTimeout(() => this.options.serializableCallback(scrapeResults), 30000, 'serializableCallback (scrapeListAuto)');
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
@@ -805,11 +667,622 @@ class Interpreter extends events_1.EventEmitter {
                         this.emit('flag', page, res);
                     });
                 }),
+                crawl: (crawlConfig) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping crawl', logger_1.Level.WARN);
+                        return;
+                    }
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('crawl');
+                    }
+                    this.log('Starting crawl operation', logger_1.Level.LOG);
+                    try {
+                        // Get current page URL and log it
+                        const currentUrl = page.url();
+                        this.log(`Current page URL: ${currentUrl}`, logger_1.Level.LOG);
+                        // If page is on about:blank or empty, we need to wait for navigation
+                        if (!currentUrl || currentUrl === 'about:blank' || currentUrl === '') {
+                            this.log('Page not yet navigated, waiting for navigation...', logger_1.Level.WARN);
+                            yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                        }
+                        const baseUrl = page.url();
+                        this.log(`Using base URL for crawl: ${baseUrl}`, logger_1.Level.LOG);
+                        const parsedBase = new URL(baseUrl);
+                        const baseDomain = parsedBase.hostname;
+                        let discoveredUrls = [];
+                        // Step 1: Sitemap discovery using XMLHttpRequest to avoid polyfills
+                        if (crawlConfig.useSitemap) {
+                            this.log('Fetching sitemap URLs...', logger_1.Level.LOG);
+                            try {
+                                const sitemapUrl = `${parsedBase.protocol}//${parsedBase.host}/sitemap.xml`;
+                                // Use XMLHttpRequest instead of fetch to avoid polyfills
+                                const sitemapUrls = yield page.evaluate((url) => {
+                                    return new Promise((resolve) => {
+                                        const xhr = new XMLHttpRequest();
+                                        xhr.open('GET', url, true);
+                                        xhr.onload = function () {
+                                            if (xhr.status === 200) {
+                                                const text = xhr.responseText;
+                                                const locMatches = text.match(/<loc>(.*?)<\/loc>/g) || [];
+                                                const urls = locMatches.map(match => match.replace(/<\/?loc>/g, ''));
+                                                resolve(urls);
+                                            }
+                                            else {
+                                                resolve([]);
+                                            }
+                                        };
+                                        xhr.onerror = function () {
+                                            resolve([]);
+                                        };
+                                        xhr.send();
+                                    });
+                                }, sitemapUrl);
+                                if (sitemapUrls.length > 0) {
+                                    const nestedSitemaps = sitemapUrls.filter(url => url.endsWith('/sitemap') || url.endsWith('sitemap.xml') || url.includes('/sitemap/'));
+                                    const regularUrls = sitemapUrls.filter(url => !url.endsWith('/sitemap') && !url.endsWith('sitemap.xml') && !url.includes('/sitemap/'));
+                                    discoveredUrls.push(...regularUrls);
+                                    this.log(`Found ${regularUrls.length} regular URLs from main sitemap`, logger_1.Level.LOG);
+                                    for (const nestedUrl of nestedSitemaps.slice(0, 10)) {
+                                        try {
+                                            this.log(`Fetching nested sitemap: ${nestedUrl}`, logger_1.Level.LOG);
+                                            const nestedUrls = yield page.evaluate((url) => {
+                                                return new Promise((resolve) => {
+                                                    const xhr = new XMLHttpRequest();
+                                                    xhr.open('GET', url, true);
+                                                    xhr.onload = function () {
+                                                        if (xhr.status === 200) {
+                                                            const text = xhr.responseText;
+                                                            const locMatches = text.match(/<loc>(.*?)<\/loc>/g) || [];
+                                                            const urls = locMatches.map(match => match.replace(/<\/?loc>/g, ''));
+                                                            resolve(urls);
+                                                        }
+                                                        else {
+                                                            resolve([]);
+                                                        }
+                                                    };
+                                                    xhr.onerror = function () {
+                                                        resolve([]);
+                                                    };
+                                                    xhr.send();
+                                                });
+                                            }, nestedUrl);
+                                            if (nestedUrls.length > 0) {
+                                                discoveredUrls.push(...nestedUrls);
+                                                this.log(`Found ${nestedUrls.length} URLs from nested sitemap ${nestedUrl}`, logger_1.Level.LOG);
+                                            }
+                                        }
+                                        catch (error) {
+                                            this.log(`Failed to fetch nested sitemap ${nestedUrl}: ${error.message}`, logger_1.Level.WARN);
+                                        }
+                                    }
+                                    this.log(`Total URLs from all sitemaps: ${discoveredUrls.length}`, logger_1.Level.LOG);
+                                }
+                                else {
+                                    this.log('No URLs found in sitemap or sitemap not available', logger_1.Level.WARN);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Sitemap fetch failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (crawlConfig.followLinks) {
+                            this.log('Extracting links from current page...', logger_1.Level.LOG);
+                            try {
+                                yield page.waitForLoadState('load', { timeout: 15000 }).catch(() => { });
+                                yield page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
+                                    this.log('Network did not become idle, continuing anyway', logger_1.Level.WARN);
+                                });
+                                yield new Promise(resolve => setTimeout(resolve, 5000));
+                                const anchorCount = yield page.evaluate(() => {
+                                    return document.querySelectorAll('a').length;
+                                });
+                                this.log(`Page has ${anchorCount} total anchor tags`, logger_1.Level.LOG);
+                                const pageLinks = yield page.evaluate(() => {
+                                    const links = [];
+                                    const allAnchors = document.querySelectorAll('a');
+                                    console.log('Total anchors found:', allAnchors.length);
+                                    for (let i = 0; i < allAnchors.length; i++) {
+                                        const anchor = allAnchors[i];
+                                        const href = anchor.getAttribute('href');
+                                        const fullHref = anchor.href;
+                                        if (fullHref && (fullHref.startsWith('http://') || fullHref.startsWith('https://'))) {
+                                            links.push(fullHref);
+                                        }
+                                    }
+                                    console.log('Links extracted:', links.length);
+                                    return links;
+                                });
+                                discoveredUrls.push(...pageLinks);
+                                this.log(`Found ${pageLinks.length} links from page`, logger_1.Level.LOG);
+                            }
+                            catch (error) {
+                                this.log(`Link extraction failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        const filteredUrls = discoveredUrls.filter(url => {
+                            try {
+                                const urlObj = new URL(url);
+                                if (crawlConfig.mode === 'domain') {
+                                    if (urlObj.hostname !== baseDomain)
+                                        return false;
+                                }
+                                else if (crawlConfig.mode === 'subdomain') {
+                                    if (!urlObj.hostname.endsWith(baseDomain) && urlObj.hostname !== baseDomain)
+                                        return false;
+                                }
+                                else if (crawlConfig.mode === 'path') {
+                                    if (urlObj.hostname !== baseDomain || !urlObj.pathname.startsWith(parsedBase.pathname))
+                                        return false;
+                                }
+                                if (crawlConfig.includePaths && crawlConfig.includePaths.length > 0) {
+                                    const matches = crawlConfig.includePaths.some(pattern => {
+                                        const regex = new RegExp(pattern);
+                                        return regex.test(url);
+                                    });
+                                    if (!matches)
+                                        return false;
+                                }
+                                if (crawlConfig.excludePaths && crawlConfig.excludePaths.length > 0) {
+                                    const matches = crawlConfig.excludePaths.some(pattern => {
+                                        const regex = new RegExp(pattern);
+                                        return regex.test(url);
+                                    });
+                                    if (matches)
+                                        return false;
+                                }
+                                return true;
+                            }
+                            catch (error) {
+                                return false;
+                            }
+                        });
+                        const uniqueUrls = Array.from(new Set(filteredUrls.map(url => {
+                            return url.replace(/#.*$/, '').replace(/\/$/, '');
+                        })));
+                        const basePathname = parsedBase.pathname;
+                        const prioritizedUrls = uniqueUrls.sort((a, b) => {
+                            try {
+                                const aUrl = new URL(a);
+                                const bUrl = new URL(b);
+                                const aMatchesBase = aUrl.pathname.startsWith(basePathname);
+                                const bMatchesBase = bUrl.pathname.startsWith(basePathname);
+                                if (aMatchesBase && !bMatchesBase)
+                                    return -1;
+                                if (!aMatchesBase && bMatchesBase)
+                                    return 1;
+                                return 0;
+                            }
+                            catch (error) {
+                                return 0;
+                            }
+                        });
+                        const finalUrls = prioritizedUrls.slice(0, crawlConfig.limit);
+                        this.log(`Crawl discovered ${finalUrls.length} URLs (from ${discoveredUrls.length} total)`, logger_1.Level.LOG);
+                        this.log(`Starting to scrape content from ${finalUrls.length} discovered URLs...`, logger_1.Level.LOG);
+                        const crawlResults = [];
+                        for (let i = 0; i < finalUrls.length; i++) {
+                            const url = finalUrls[i];
+                            try {
+                                this.log(`[${i + 1}/${finalUrls.length}] Scraping: ${url}`, logger_1.Level.LOG);
+                                yield page.goto(url, {
+                                    waitUntil: 'domcontentloaded',
+                                    timeout: 30000
+                                }).catch(() => {
+                                    this.log(`Failed to navigate to ${url}, skipping...`, logger_1.Level.WARN);
+                                });
+                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                const pageData = yield page.evaluate(() => {
+                                    var _a, _b;
+                                    const getMeta = (name) => {
+                                        const meta = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+                                        return (meta === null || meta === void 0 ? void 0 : meta.getAttribute('content')) || '';
+                                    };
+                                    const getAllMeta = () => {
+                                        const metadata = {};
+                                        const metaTags = document.querySelectorAll('meta');
+                                        metaTags.forEach(tag => {
+                                            const name = tag.getAttribute('name') || tag.getAttribute('property');
+                                            const content = tag.getAttribute('content');
+                                            if (name && content) {
+                                                metadata[name] = content;
+                                            }
+                                        });
+                                        return metadata;
+                                    };
+                                    const title = document.title || '';
+                                    const bodyText = ((_a = document.body) === null || _a === void 0 ? void 0 : _a.innerText) || '';
+                                    const elementsWithMxId = document.querySelectorAll('[data-mx-id]');
+                                    elementsWithMxId.forEach(el => el.removeAttribute('data-mx-id'));
+                                    const html = document.documentElement.outerHTML;
+                                    const links = Array.from(document.querySelectorAll('a')).map(a => a.href);
+                                    const allMetadata = getAllMeta();
+                                    return {
+                                        title,
+                                        description: getMeta('description'),
+                                        text: bodyText,
+                                        html: html,
+                                        links: links,
+                                        wordCount: bodyText.split(/\s+/).filter(w => w.length > 0).length,
+                                        metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
+                                    };
+                                });
+                                crawlResults.push({
+                                    metadata: Object.assign(Object.assign({}, pageData.metadata), { url: url, sourceURL: url }),
+                                    html: pageData.html,
+                                    text: pageData.text,
+                                    links: pageData.links,
+                                    wordCount: pageData.wordCount,
+                                    scrapedAt: new Date().toISOString()
+                                });
+                                this.log(`✓ Scraped ${url} (${pageData.wordCount} words)`, logger_1.Level.LOG);
+                            }
+                            catch (error) {
+                                this.log(`Failed to scrape ${url}: ${error.message}`, logger_1.Level.WARN);
+                                crawlResults.push({
+                                    url: url,
+                                    error: error.message,
+                                    scrapedAt: new Date().toISOString()
+                                });
+                            }
+                        }
+                        this.log(`Successfully scraped ${crawlResults.length} pages`, logger_1.Level.LOG);
+                        const actionType = "crawl";
+                        const actionName = "Crawl Results";
+                        if (!this.serializableDataByType[actionType]) {
+                            this.serializableDataByType[actionType] = {};
+                        }
+                        if (!this.serializableDataByType[actionType][actionName]) {
+                            this.serializableDataByType[actionType][actionName] = [];
+                        }
+                        this.serializableDataByType[actionType][actionName] = crawlResults;
+                        yield this.options.serializableCallback({
+                            scrapeList: this.serializableDataByType.scrapeList || {},
+                            scrapeSchema: this.serializableDataByType.scrapeSchema || {},
+                            crawl: this.serializableDataByType.crawl || {}
+                        });
+                    }
+                    catch (error) {
+                        this.log(`Crawl action failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`Crawl execution error: ${error.message}`);
+                    }
+                }),
+                search: (searchConfig) => __awaiter(this, void 0, void 0, function* () {
+                    var _a, _b;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping search', logger_1.Level.WARN);
+                        return;
+                    }
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('search');
+                    }
+                    searchConfig.provider = 'duckduckgo';
+                    this.log(`Performing DuckDuckGo search for: ${searchConfig.query}`, logger_1.Level.LOG);
+                    try {
+                        let searchUrl = `https://duckduckgo.com/?q=${encodeURIComponent(searchConfig.query)}`;
+                        if ((_b = searchConfig.filters) === null || _b === void 0 ? void 0 : _b.timeRange) {
+                            const timeMap = {
+                                'day': 'd',
+                                'week': 'w',
+                                'month': 'm',
+                                'year': 'y'
+                            };
+                            searchUrl += `&df=${timeMap[searchConfig.filters.timeRange]}`;
+                        }
+                        const initialDelay = 500 + Math.random() * 1000;
+                        yield new Promise(resolve => setTimeout(resolve, initialDelay));
+                        yield page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+                        yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => {
+                            this.log('Load state timeout, continuing anyway', logger_1.Level.WARN);
+                        });
+                        const pageLoadDelay = 2000 + Math.random() * 1500;
+                        yield new Promise(resolve => setTimeout(resolve, pageLoadDelay));
+                        let searchResults = [];
+                        let retryCount = 0;
+                        const maxRetries = 2;
+                        while (searchResults.length === 0 && retryCount <= maxRetries) {
+                            if (retryCount > 0) {
+                                this.log(`Retry attempt ${retryCount}/${maxRetries} for DuckDuckGo search...`, logger_1.Level.LOG);
+                                const retryDelay = 1000 * Math.pow(2, retryCount) + Math.random() * 1000;
+                                yield new Promise(resolve => setTimeout(resolve, retryDelay));
+                            }
+                            this.log('Attempting to extract DuckDuckGo search results...', logger_1.Level.LOG);
+                            yield page.waitForSelector('[data-testid="result"], .result', { timeout: 5000 }).catch(() => {
+                                this.log('DuckDuckGo results not found on initial wait', logger_1.Level.WARN);
+                            });
+                            let currentResultCount = 0;
+                            const maxLoadAttempts = Math.ceil(searchConfig.limit / 10) * 2;
+                            let loadAttempts = 0;
+                            let noNewResultsCount = 0;
+                            while (currentResultCount < searchConfig.limit && loadAttempts < maxLoadAttempts && noNewResultsCount < 3) {
+                                const previousCount = currentResultCount;
+                                currentResultCount = yield page.evaluate(() => {
+                                    const selectors = [
+                                        '[data-testid="result"]',
+                                        'article[data-testid="result"]',
+                                        'li[data-layout="organic"]',
+                                        '.result',
+                                        'article[data-testid]'
+                                    ];
+                                    for (const selector of selectors) {
+                                        const elements = document.querySelectorAll(selector);
+                                        if (elements.length > 0) {
+                                            return elements.length;
+                                        }
+                                    }
+                                    return 0;
+                                });
+                                if (currentResultCount >= searchConfig.limit) {
+                                    this.log(`Reached desired result count: ${currentResultCount}`, logger_1.Level.LOG);
+                                    break;
+                                }
+                                if (currentResultCount === previousCount) {
+                                    noNewResultsCount++;
+                                    this.log(`No new results after load more (attempt ${noNewResultsCount}/3)`, logger_1.Level.WARN);
+                                    if (noNewResultsCount >= 3)
+                                        break;
+                                }
+                                else {
+                                    noNewResultsCount = 0;
+                                    this.log(`Current results count: ${currentResultCount}/${searchConfig.limit}`, logger_1.Level.LOG);
+                                }
+                                yield page.evaluate(() => {
+                                    window.scrollTo(0, document.body.scrollHeight);
+                                });
+                                yield new Promise(resolve => setTimeout(resolve, 800));
+                                const loadMoreClicked = yield page.evaluate(() => {
+                                    const selectors = [
+                                        '#more-results',
+                                        'button:has-text("More results")',
+                                        'button:has-text("more results")',
+                                        'button[id*="more"]',
+                                        'button:has-text("Load more")'
+                                    ];
+                                    for (const selector of selectors) {
+                                        try {
+                                            const button = document.querySelector(selector);
+                                            if (button && button.offsetParent !== null) {
+                                                button.click();
+                                                console.log(`Clicked load more button with selector: ${selector}`);
+                                                return true;
+                                            }
+                                        }
+                                        catch (e) {
+                                            continue;
+                                        }
+                                    }
+                                    return false;
+                                });
+                                if (loadMoreClicked) {
+                                    this.log('Clicked "More results" button', logger_1.Level.LOG);
+                                    yield new Promise(resolve => setTimeout(resolve, 1500 + Math.random() * 1000));
+                                }
+                                else {
+                                    this.log('No "More results" button found, results may be limited', logger_1.Level.WARN);
+                                    break;
+                                }
+                                loadAttempts++;
+                            }
+                            this.log(`Finished pagination. Total results available: ${currentResultCount}`, logger_1.Level.LOG);
+                            searchResults = yield page.evaluate((limit) => {
+                                const results = [];
+                                const cleanDescription = (text) => {
+                                    if (!text)
+                                        return '';
+                                    let cleaned = text.replace(/^\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago\s*/i, '');
+                                    cleaned = cleaned.replace(/^[A-Z][a-z]{2}\s+\d{1,2},?\s+\d{4}\s*[—\-]\s*/i, '');
+                                    cleaned = cleaned.replace(/^\d{4}-\d{2}-\d{2}\s*[—\-]\s*/i, '');
+                                    cleaned = cleaned.trim().replace(/\s+/g, ' ');
+                                    return cleaned;
+                                };
+                                const selectors = [
+                                    '[data-testid="result"]',
+                                    'article[data-testid="result"]',
+                                    'li[data-layout="organic"]',
+                                    '.result',
+                                    'article[data-testid]'
+                                ];
+                                let allElements = [];
+                                for (const selector of selectors) {
+                                    const elements = Array.from(document.querySelectorAll(selector));
+                                    if (elements.length > 0) {
+                                        console.log(`Found ${elements.length} DDG elements with: ${selector}`);
+                                        allElements = elements;
+                                        break;
+                                    }
+                                }
+                                for (let i = 0; i < Math.min(allElements.length, limit); i++) {
+                                    const element = allElements[i];
+                                    const titleEl = element.querySelector('h2, [data-testid="result-title-a"], h3, [data-testid="result-title"]');
+                                    let linkEl = titleEl === null || titleEl === void 0 ? void 0 : titleEl.querySelector('a[href]');
+                                    if (!linkEl) {
+                                        linkEl = element.querySelector('a[href]');
+                                    }
+                                    if (!linkEl || !linkEl.href)
+                                        continue;
+                                    let actualUrl = linkEl.href;
+                                    if (actualUrl.includes('uddg=')) {
+                                        try {
+                                            const urlParams = new URLSearchParams(actualUrl.split('?')[1]);
+                                            const uddgUrl = urlParams.get('uddg');
+                                            if (uddgUrl) {
+                                                actualUrl = decodeURIComponent(uddgUrl);
+                                            }
+                                        }
+                                        catch (e) {
+                                            console.log('Failed to parse uddg parameter:', e);
+                                        }
+                                    }
+                                    if (actualUrl.includes('duckduckgo.com')) {
+                                        console.log(`Skipping DDG internal URL: ${actualUrl}`);
+                                        continue;
+                                    }
+                                    const descEl = element.querySelector('[data-result="snippet"], .result__snippet, [data-testid="result-snippet"]');
+                                    if (titleEl && titleEl.textContent && actualUrl) {
+                                        const rawDescription = ((descEl === null || descEl === void 0 ? void 0 : descEl.textContent) || '').trim();
+                                        const cleanedDescription = cleanDescription(rawDescription);
+                                        results.push({
+                                            url: actualUrl,
+                                            title: titleEl.textContent.trim(),
+                                            description: cleanedDescription,
+                                            position: results.length + 1
+                                        });
+                                    }
+                                }
+                                console.log(`Extracted ${results.length} DuckDuckGo search results`);
+                                return results;
+                            }, searchConfig.limit);
+                            if (searchResults.length === 0) {
+                                this.log(`No DuckDuckGo results found (attempt ${retryCount + 1}/${maxRetries + 1})`, logger_1.Level.WARN);
+                                retryCount++;
+                            }
+                            else {
+                                this.log(`Successfully extracted ${searchResults.length} results`, logger_1.Level.LOG);
+                                break;
+                            }
+                        }
+                        this.log(`Search found ${searchResults.length} results`, logger_1.Level.LOG);
+                        if (searchConfig.mode === 'discover') {
+                            const actionType = "search";
+                            const actionName = "Search Results";
+                            if (!this.serializableDataByType[actionType]) {
+                                this.serializableDataByType[actionType] = {};
+                            }
+                            if (!this.serializableDataByType[actionType][actionName]) {
+                                this.serializableDataByType[actionType][actionName] = {};
+                            }
+                            const searchData = {
+                                query: searchConfig.query,
+                                provider: searchConfig.provider,
+                                filters: searchConfig.filters || {},
+                                resultsCount: searchResults.length,
+                                results: searchResults,
+                                searchedAt: new Date().toISOString()
+                            };
+                            this.serializableDataByType[actionType][actionName] = searchData;
+                            yield this.options.serializableCallback({
+                                scrapeList: this.serializableDataByType.scrapeList || {},
+                                scrapeSchema: this.serializableDataByType.scrapeSchema || {},
+                                crawl: this.serializableDataByType.crawl || {},
+                                search: this.serializableDataByType.search || {}
+                            });
+                            this.log(`Search completed in discover mode with ${searchResults.length} results`, logger_1.Level.LOG);
+                            return;
+                        }
+                        this.log(`Starting to scrape content from ${searchResults.length} search results...`, logger_1.Level.LOG);
+                        const scrapedResults = [];
+                        for (let i = 0; i < searchResults.length; i++) {
+                            const result = searchResults[i];
+                            try {
+                                this.log(`[${i + 1}/${searchResults.length}] Scraping: ${result.url}`, logger_1.Level.LOG);
+                                yield page.goto(result.url, {
+                                    waitUntil: 'domcontentloaded',
+                                    timeout: 30000
+                                }).catch(() => {
+                                    this.log(`Failed to navigate to ${result.url}, skipping...`, logger_1.Level.WARN);
+                                });
+                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                const pageData = yield page.evaluate(() => {
+                                    var _a, _b;
+                                    const getMeta = (name) => {
+                                        const meta = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+                                        return (meta === null || meta === void 0 ? void 0 : meta.getAttribute('content')) || '';
+                                    };
+                                    const getAllMeta = () => {
+                                        const metadata = {};
+                                        const metaTags = document.querySelectorAll('meta');
+                                        metaTags.forEach(tag => {
+                                            const name = tag.getAttribute('name') || tag.getAttribute('property');
+                                            const content = tag.getAttribute('content');
+                                            if (name && content) {
+                                                metadata[name] = content;
+                                            }
+                                        });
+                                        return metadata;
+                                    };
+                                    const title = document.title || '';
+                                    const bodyText = ((_a = document.body) === null || _a === void 0 ? void 0 : _a.innerText) || '';
+                                    const elementsWithMxId = document.querySelectorAll('[data-mx-id]');
+                                    elementsWithMxId.forEach(el => el.removeAttribute('data-mx-id'));
+                                    const html = document.documentElement.outerHTML;
+                                    const links = Array.from(document.querySelectorAll('a')).map(a => a.href);
+                                    const allMetadata = getAllMeta();
+                                    return {
+                                        title,
+                                        description: getMeta('description'),
+                                        text: bodyText,
+                                        html: html,
+                                        links: links,
+                                        wordCount: bodyText.split(/\s+/).filter(w => w.length > 0).length,
+                                        metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
+                                    };
+                                });
+                                scrapedResults.push({
+                                    searchResult: {
+                                        query: searchConfig.query,
+                                        position: result.position,
+                                        searchTitle: result.title,
+                                        searchDescription: result.description,
+                                    },
+                                    metadata: Object.assign(Object.assign({}, pageData.metadata), { url: result.url, sourceURL: result.url }),
+                                    html: pageData.html,
+                                    text: pageData.text,
+                                    links: pageData.links,
+                                    wordCount: pageData.wordCount,
+                                    scrapedAt: new Date().toISOString()
+                                });
+                                this.log(`✓ Scraped ${result.url} (${pageData.wordCount} words)`, logger_1.Level.LOG);
+                            }
+                            catch (error) {
+                                this.log(`Failed to scrape ${result.url}: ${error.message}`, logger_1.Level.WARN);
+                                scrapedResults.push({
+                                    searchResult: {
+                                        query: searchConfig.query,
+                                        position: result.position,
+                                        searchTitle: result.title,
+                                        searchDescription: result.description,
+                                    },
+                                    url: result.url,
+                                    error: error.message,
+                                    scrapedAt: new Date().toISOString()
+                                });
+                            }
+                        }
+                        this.log(`Successfully scraped ${scrapedResults.length} search results`, logger_1.Level.LOG);
+                        const actionType = "search";
+                        const actionName = "Search Results";
+                        if (!this.serializableDataByType[actionType]) {
+                            this.serializableDataByType[actionType] = {};
+                        }
+                        if (!this.serializableDataByType[actionType][actionName]) {
+                            this.serializableDataByType[actionType][actionName] = {};
+                        }
+                        const searchData = {
+                            query: searchConfig.query,
+                            provider: searchConfig.provider,
+                            filters: searchConfig.filters || {},
+                            mode: searchConfig.mode,
+                            resultsCount: scrapedResults.length,
+                            results: scrapedResults,
+                            searchedAt: new Date().toISOString()
+                        };
+                        this.serializableDataByType[actionType][actionName] = searchData;
+                        yield this.options.serializableCallback({
+                            scrapeList: this.serializableDataByType.scrapeList || {},
+                            scrapeSchema: this.serializableDataByType.scrapeSchema || {},
+                            crawl: this.serializableDataByType.crawl || {},
+                            search: this.serializableDataByType.search || {}
+                        });
+                    }
+                    catch (error) {
+                        this.log(`Search action failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`Search execution error: ${error.message}`);
+                    }
+                }),
             };
             const executeAction = (invokee, methodName, args) => __awaiter(this, void 0, void 0, function* () {
                 console.log("Executing action:", methodName, args);
                 if (methodName === 'press' || methodName === 'type') {
-                    // Extract only the first two arguments for these methods
                     const limitedArgs = Array.isArray(args) ? args.slice(0, 2) : [args];
                     yield invokee[methodName](...limitedArgs);
                     return;
@@ -822,7 +1295,6 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             for (const step of steps) {
-                // Check abort flag before each step
                 if (this.isAborted) {
                     this.log('Workflow aborted during step execution', logger_1.Level.WARN);
                     return;
@@ -844,7 +1316,6 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 try {
                     if (step.action in wawActions) {
-                        // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
                         const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
                         if (step.action === 'screenshot') {
                             yield wawActions.screenshot(...(params !== null && params !== void 0 ? params : []), stepName !== null && stepName !== void 0 ? stepName : undefined);
@@ -861,7 +1332,6 @@ class Interpreter extends events_1.EventEmitter {
                         if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                             this.options.debugChannel.setActionType(String(step.action));
                         }
-                        // Implements the dot notation for the "method name" in the workflow
                         const levels = String(step.action).split('.');
                         const methodName = levels[levels.length - 1];
                         let invokee = page;
@@ -870,7 +1340,6 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         if (methodName === 'waitForLoadState') {
                             try {
-                                // Add timeout if not already specified
                                 let args = step.args;
                                 if (Array.isArray(args) && args.length === 1) {
                                     args = [args[0], { timeout: 30000 }];
@@ -894,7 +1363,7 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 catch (error) {
                                     this.log(`Click action failed for selector ${(_b = step.args) === null || _b === void 0 ? void 0 : _b[0]}: ${error.message}`, logger_1.Level.WARN);
-                                    continue; // Skip to next action
+                                    continue;
                                 }
                             }
                         }
@@ -905,7 +1374,6 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 catch (error) {
                     this.log(`Action ${String(step.action)} failed: ${error.message}`, logger_1.Level.WARN);
-                    // Continue to next action instead of breaking
                     continue;
                 }
                 yield new Promise((res) => { setTimeout(res, 500); });
@@ -920,12 +1388,10 @@ class Interpreter extends events_1.EventEmitter {
             }
             const actionType = "scrapeList";
             let actionName = providedActionName || "";
-            // During deep extraction, ALWAYS auto-increment to create separate lists for each URL
             if (!actionName || actionName.trim() === "" || this.isInDeepExtractionPhase) {
                 this.scrapeListCounter++;
                 actionName = `List ${this.scrapeListCounter}`;
             }
-            // Initialize storage for this action
             if (!this.serializableDataByType[actionType]) {
                 this.serializableDataByType[actionType] = {};
             }
@@ -933,23 +1399,21 @@ class Interpreter extends events_1.EventEmitter {
                 this.serializableDataByType[actionType][actionName] = [];
             }
             let allResults = [];
-            let allUrls = []; // Track URLs alongside results for deep-extract
+            let allUrls = [];
             let previousHeight = 0;
             let scrapedItems = new Set();
             let visitedUrls = new Set();
             const MAX_RETRIES = 3;
-            const RETRY_DELAY = 1000; // 1 second delay between retries
+            const RETRY_DELAY = 1000;
             const MAX_UNCHANGED_RESULTS = 5;
             const debugLog = (message, ...args) => {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
             const scrapeCurrentPage = () => __awaiter(this, void 0, void 0, function* () {
-                // Check abort flag before scraping current page
                 if (this.isAborted) {
                     debugLog("Workflow aborted, stopping scrapeCurrentPage");
                     return;
                 }
-                // Add timeout to prevent hanging on page evaluation
                 const evaluationPromise = page.evaluate((cfg) => window.scrapeList(cfg), config);
                 const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 10000));
                 let results;
@@ -960,10 +1424,7 @@ class Interpreter extends events_1.EventEmitter {
                     debugLog(`Page evaluation failed: ${error.message}`);
                     return;
                 }
-                // Extract URLs for ALL items BEFORE filtering duplicates
-                // This ensures URL indices match result indices
                 const allItemUrls = yield this.extractUrlsFromCurrentPage(page, config.listSelector, results.length);
-                // Filter results AND URLs together using the same uniqueness logic
                 const newResults = [];
                 const newUrls = [];
                 results.forEach((item, index) => {
@@ -971,28 +1432,35 @@ class Interpreter extends events_1.EventEmitter {
                     if (!scrapedItems.has(uniqueKey)) {
                         scrapedItems.add(uniqueKey);
                         newResults.push(item);
-                        newUrls.push(allItemUrls[index] || []); // Add corresponding URLs
+                        newUrls.push(allItemUrls[index] || []);
                     }
                 });
                 allResults = allResults.concat(newResults);
                 allUrls = allUrls.concat(newUrls);
                 debugLog("Results collected:", allResults.length);
-                // Store in serializableDataByType and send structured callback
                 this.serializableDataByType[actionType][actionName] = [...allResults];
-                yield this.options.serializableCallback({
+                yield this.callWithTimeout(() => this.options.serializableCallback({
                     scrapeList: this.serializableDataByType.scrapeList,
                     scrapeSchema: this.serializableDataByType.scrapeSchema
-                });
+                }), 30000, 'serializableCallback (pagination scrapeList)');
+                const MAX_STORED_LISTS = 50;
+                const listKeys = Object.keys(this.serializableDataByType[actionType]);
+                if (listKeys.length > MAX_STORED_LISTS) {
+                    const sortedKeys = listKeys.sort();
+                    const keysToRemove = sortedKeys.slice(0, listKeys.length - MAX_STORED_LISTS);
+                    keysToRemove.forEach(key => {
+                        delete this.serializableDataByType[actionType][key];
+                    });
+                }
             });
             const checkLimit = () => {
                 if (config.limit && allResults.length >= config.limit) {
                     allResults = allResults.slice(0, config.limit);
-                    allUrls = allUrls.slice(0, config.limit); // Also trim URLs to maintain sync
+                    allUrls = allUrls.slice(0, config.limit);
                     return true;
                 }
                 return false;
             };
-            // Helper function to detect if a selector is XPath
             const isXPathSelector = (selector) => {
                 return selector.startsWith('//') ||
                     selector.startsWith('/') ||
@@ -1004,11 +1472,9 @@ class Interpreter extends events_1.EventEmitter {
                     selector.includes(' and ') ||
                     selector.includes(' or ');
             };
-            // Helper function to wait for selector (CSS or XPath)
             const waitForSelectorUniversal = (selector_2, ...args_1) => __awaiter(this, [selector_2, ...args_1], void 0, function* (selector, options = {}) {
                 try {
                     if (isXPathSelector(selector)) {
-                        // Use XPath locator
                         const locator = page.locator(`xpath=${selector}`);
                         yield locator.waitFor({
                             state: 'attached',
@@ -1017,7 +1483,6 @@ class Interpreter extends events_1.EventEmitter {
                         return yield locator.elementHandle();
                     }
                     else {
-                        // Use CSS selector
                         return yield page.waitForSelector(selector, {
                             state: 'attached',
                             timeout: options.timeout || 10000
@@ -1028,13 +1493,11 @@ class Interpreter extends events_1.EventEmitter {
                     return null;
                 }
             });
-            // Enhanced button finder with retry mechanism for both CSS and XPath selectors
             const findWorkingButton = (selectors) => __awaiter(this, void 0, void 0, function* () {
                 const startTime = Date.now();
                 const MAX_BUTTON_SEARCH_TIME = 15000;
                 let updatedSelectors = [...selectors];
                 for (let i = 0; i < selectors.length; i++) {
-                    // Check overall timeout
                     if (Date.now() - startTime > MAX_BUTTON_SEARCH_TIME) {
                         debugLog(`Button search timeout reached (${MAX_BUTTON_SEARCH_TIME}ms), aborting`);
                         break;
@@ -1044,7 +1507,6 @@ class Interpreter extends events_1.EventEmitter {
                     let selectorSuccess = false;
                     while (retryCount < MAX_RETRIES && !selectorSuccess) {
                         try {
-                            // Reduce timeout to prevent hanging on slow selectors
                             const button = yield waitForSelectorUniversal(selector, { timeout: 2000 });
                             if (button) {
                                 debugLog('Found working selector:', selector);
@@ -1055,7 +1517,6 @@ class Interpreter extends events_1.EventEmitter {
                                 };
                             }
                             else {
-                                // Treat null result as failed attempt
                                 retryCount++;
                                 debugLog(`Selector "${selector}" not found: attempt ${retryCount}/${MAX_RETRIES}`);
                                 if (retryCount < MAX_RETRIES) {
@@ -1064,7 +1525,7 @@ class Interpreter extends events_1.EventEmitter {
                                 else {
                                     debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
                                     updatedSelectors = updatedSelectors.filter(s => s !== selector);
-                                    selectorSuccess = true; // Exit retry loop for this selector
+                                    selectorSuccess = true;
                                 }
                             }
                         }
@@ -1077,7 +1538,7 @@ class Interpreter extends events_1.EventEmitter {
                             else {
                                 debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
                                 updatedSelectors = updatedSelectors.filter(s => s !== selector);
-                                selectorSuccess = true; // Exit retry loop for this selector
+                                selectorSuccess = true;
                             }
                         }
                     }
@@ -1105,17 +1566,15 @@ class Interpreter extends events_1.EventEmitter {
             let availableSelectors = config.pagination.selector.split(',');
             let unchangedResultCounter = 0;
             let paginationIterations = 0;
-            const MAX_PAGINATION_ITERATIONS = 100; // Prevent infinite pagination
+            const MAX_PAGINATION_ITERATIONS = 100;
             const paginationStartTime = Date.now();
-            const MAX_PAGINATION_TIME = 30 * 60 * 1000; // 30 minutes max for pagination
+            const MAX_PAGINATION_TIME = 30 * 60 * 1000;
             try {
                 while (true) {
-                    // Check abort flag at start of each pagination iteration
                     if (this.isAborted) {
                         this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
                         return { results: allResults, urls: allUrls };
                     }
-                    // Pagination circuit breakers
                     if (++paginationIterations > MAX_PAGINATION_ITERATIONS) {
                         debugLog(`Maximum pagination iterations reached (${MAX_PAGINATION_ITERATIONS}), stopping`);
                         return { results: allResults, urls: allUrls };
@@ -1124,7 +1583,6 @@ class Interpreter extends events_1.EventEmitter {
                         debugLog('Maximum pagination time reached (10 minutes), stopping');
                         return { results: allResults, urls: allUrls };
                     }
-                    // Add async yield every 5 iterations to prevent event loop blocking
                     if (paginationIterations % 5 === 0) {
                         yield new Promise(resolve => setImmediate(resolve));
                     }
@@ -1144,12 +1602,20 @@ class Interpreter extends events_1.EventEmitter {
                                 return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
                             });
                             const currentResultCount = allResults.length;
-                            if (currentResultCount === previousResultCount) {
+                            const newItemsFound = currentResultCount - previousResultCount;
+                            if (newItemsFound === 0) {
                                 unchangedResultCounter++;
                                 if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
                                     return { results: allResults, urls: allUrls };
                                 }
                             }
+                            else if (newItemsFound < 3) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS * 2) {
+                                    debugLog('Very slow pagination detected (< 3 items/scroll), stopping');
+                                    return { results: allResults, urls: allUrls };
+                                }
+                            }
                             else {
                                 unchangedResultCounter = 0;
                             }
@@ -1169,12 +1635,20 @@ class Interpreter extends events_1.EventEmitter {
                             yield page.waitForTimeout(2000);
                             const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
                             const currentResultCount = allResults.length;
-                            if (currentResultCount === previousResultCount) {
+                            const newItemsFound = currentResultCount - previousResultCount;
+                            if (newItemsFound === 0) {
                                 unchangedResultCounter++;
                                 if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
                                     return { results: allResults, urls: allUrls };
                                 }
                             }
+                            else if (newItemsFound < 3) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS * 2) {
+                                    debugLog('Very slow pagination detected (< 3 items/scroll), stopping');
+                                    return { results: allResults, urls: allUrls };
+                                }
+                            }
                             else {
                                 unchangedResultCounter = 0;
                             }
@@ -1191,9 +1665,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return { results: allResults, urls: allUrls };
                             const { button, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
-                            availableSelectors = updatedSelectors;
+                            if (workingSelector) {
+                                config.pagination.selector = workingSelector;
+                                availableSelectors = [workingSelector];
+                            }
+                            else {
+                                availableSelectors = updatedSelectors;
+                            }
                             if (!button || !workingSelector) {
-                                // Final retry for navigation when no selectors work
                                 const success = yield retryOperation(() => __awaiter(this, void 0, void 0, function* () {
                                     try {
                                         yield page.evaluate(() => window.history.forward());
@@ -1210,7 +1689,6 @@ class Interpreter extends events_1.EventEmitter {
                             }
                             let retryCount = 0;
                             let paginationSuccess = false;
-                            // Capture basic content signature before click - with XPath support
                             const captureContentSignature = () => __awaiter(this, void 0, void 0, function* () {
                                 return yield page.evaluate((listSelector) => {
                                     const isXPath = (selector) => {
@@ -1219,7 +1697,6 @@ class Interpreter extends events_1.EventEmitter {
                                     let items = [];
                                     if (isXPath(listSelector)) {
                                         try {
-                                            // Use XPath to find elements
                                             const xpathResult = document.evaluate(listSelector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
                                             items = [];
                                             for (let i = 0; i < xpathResult.snapshotLength; i++) {
@@ -1231,7 +1708,6 @@ class Interpreter extends events_1.EventEmitter {
                                         }
                                         catch (xpathError) {
                                             console.warn('XPath evaluation failed, trying CSS selector as fallback:', xpathError);
-                                            // Fallback to CSS selector
                                             try {
                                                 items = document.querySelectorAll(listSelector);
                                             }
@@ -1243,7 +1719,6 @@ class Interpreter extends events_1.EventEmitter {
                                     }
                                     else {
                                         try {
-                                            // Use CSS selector
                                             items = document.querySelectorAll(listSelector);
                                         }
                                         catch (cssError) {
@@ -1344,11 +1819,10 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return { results: allResults, urls: allUrls };
                             let loadMoreCounter = 0;
-                            const MAX_LOAD_MORE_ITERATIONS = 100; // Prevent infinite load more
+                            const MAX_LOAD_MORE_ITERATIONS = 100;
                             const loadMoreStartTime = Date.now();
-                            const MAX_LOAD_MORE_TIME = 30 * 60 * 1000; // 5 minutes max for load more
+                            const MAX_LOAD_MORE_TIME = 30 * 60 * 1000;
                             while (true) {
-                                // Load more circuit breakers
                                 if (loadMoreCounter >= MAX_LOAD_MORE_ITERATIONS) {
                                     debugLog(`Maximum load more iterations reached (${MAX_LOAD_MORE_ITERATIONS}), stopping`);
                                     return { results: allResults, urls: allUrls };
@@ -1357,18 +1831,21 @@ class Interpreter extends events_1.EventEmitter {
                                     debugLog('Maximum load more time reached (5 minutes), stopping');
                                     return { results: allResults, urls: allUrls };
                                 }
-                                // Add async yield every 3 iterations
                                 if (loadMoreCounter % 3 === 0 && loadMoreCounter > 0) {
                                     yield new Promise(resolve => setImmediate(resolve));
                                 }
-                                // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
-                                availableSelectors = updatedSelectors;
+                                if (workingSelector) {
+                                    config.pagination.selector = workingSelector;
+                                    availableSelectors = [workingSelector];
+                                }
+                                else {
+                                    availableSelectors = updatedSelectors;
+                                }
                                 if (!workingSelector || !loadMoreButton) {
                                     debugLog('No working Load More selector found after retries');
                                     return { results: allResults, urls: allUrls };
                                 }
-                                // Implement retry mechanism for clicking the button
                                 let retryCount = 0;
                                 let clickSuccess = false;
                                 while (retryCount < MAX_RETRIES && !clickSuccess) {
@@ -1379,14 +1856,13 @@ class Interpreter extends events_1.EventEmitter {
                                         }
                                         catch (error) {
                                             debugLog(`Regular click failed on attempt ${retryCount + 1}. Trying DispatchEvent`);
-                                            // If regular click fails, try dispatchEvent
                                             try {
                                                 yield loadMoreButton.dispatchEvent('click');
                                                 clickSuccess = true;
                                             }
                                             catch (dispatchError) {
                                                 debugLog(`DispatchEvent failed on attempt ${retryCount + 1}.`);
-                                                throw dispatchError; // Propagate error to trigger retry
+                                                throw dispatchError;
                                             }
                                         }
                                         if (clickSuccess) {
@@ -1408,7 +1884,6 @@ class Interpreter extends events_1.EventEmitter {
                                     debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`);
                                     return { results: allResults, urls: allUrls };
                                 }
-                                // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
                                 yield page.evaluate(() => {
                                     const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
@@ -1458,644 +1933,15 @@ class Interpreter extends events_1.EventEmitter {
             return { results: allResults, urls: allUrls };
         });
     }
-    getMatchingActionId(workflow, pageState, usedActions) {
-        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
-            const step = workflow[actionId];
-            const isApplicable = this.applicable(step.where, pageState, usedActions);
-            console.log("-------------------------------------------------------------");
-            console.log(`Where:`, step.where);
-            console.log(`Page state:`, pageState);
-            console.log(`Match result: ${isApplicable}`);
-            console.log("-------------------------------------------------------------");
-            if (isApplicable) {
-                return actionId;
-            }
-        }
-    }
-    removeShadowSelectors(workflow) {
-        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
-            const step = workflow[actionId];
-            // Check if step has where and selectors
-            if (step.where && Array.isArray(step.where.selectors)) {
-                // Filter out selectors that contain ">>"
-                step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
-            }
-        }
-        return workflow;
-    }
     removeSpecialSelectors(workflow) {
         for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
             const step = workflow[actionId];
             if (step.where && Array.isArray(step.where.selectors)) {
-                // Filter out if selector has EITHER ":>>" OR ">>"
                 step.where.selectors = step.where.selectors.filter(selector => !(selector.includes(':>>') || selector.includes('>>')));
             }
         }
         return workflow;
     }
-    generatePageNodeInformation(page_1) {
-        return __awaiter(this, arguments, void 0, function* (page, selector = 'body') {
-            yield page.waitForTimeout(100);
-            const nodeInfo = yield page.evaluate((sel) => {
-                const serialize = (node) => {
-                    var _a, _b;
-                    // Handle text nodes
-                    if (node.nodeType === Node.TEXT_NODE) {
-                        // Get coordinates from parent element for text nodes
-                        let coordinates;
-                        if (node.parentElement) {
-                            const rect = node.parentElement.getBoundingClientRect();
-                            coordinates = {
-                                x: rect.x,
-                                y: rect.y + window.scrollY,
-                            };
-                        }
-                        return {
-                            type: 'TEXT_NODE',
-                            textContent: ((_a = node.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || '',
-                            nodeType: node.nodeType,
-                            coordinates
-                        };
-                    }
-                    // Handle element nodes
-                    const element = node;
-                    const attributes = {};
-                    for (const attr of Array.from(element.attributes || [])) {
-                        attributes[attr.name] = attr.value;
-                    }
-                    // Get coordinates for element nodes
-                    let coordinates;
-                    if (node.nodeType === Node.ELEMENT_NODE) {
-                        const rect = element.getBoundingClientRect();
-                        coordinates = {
-                            x: rect.left + window.scrollX,
-                            y: rect.top + window.scrollY
-                        };
-                    }
-                    // Get all child nodes (including text nodes)
-                    const children = Array.from(node.childNodes)
-                        .map(child => serialize(child))
-                        .filter(child => {
-                        return !(child.type === 'TEXT_NODE' && child.textContent === '');
-                    });
-                    return {
-                        type: 'ELEMENT_NODE',
-                        tagName: ((_b = element.tagName) === null || _b === void 0 ? void 0 : _b.toLowerCase()) || '',
-                        nodeType: node.nodeType,
-                        attributes,
-                        coordinates,
-                        children
-                    };
-                };
-                const rootElement = document.querySelector(sel);
-                if (!rootElement) {
-                    throw new Error(`Element not found: ${sel}`);
-                }
-                const exactRect = rootElement.getBoundingClientRect();
-                const exactCoordinates = {
-                    x: exactRect.left + window.scrollX,
-                    y: exactRect.top + window.scrollY
-                };
-                const nodeInfo = serialize(rootElement);
-                nodeInfo.coordinates = exactCoordinates;
-                return nodeInfo;
-            }, selector);
-            return nodeInfo;
-        });
-    }
-    detectElementChanges(page, schema) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            const currentDomTree = yield this.generatePageNodeInformation(page, 'body');
-            const changes = [];
-            const isScrapeList = 'listSelector' in schema;
-            const findMatchingElement = (field, currentTree, targetInfo) => {
-                if (currentTree.type !== 'ELEMENT_NODE')
-                    return { element: null, confidence: 0 };
-                const directMatchScore = () => {
-                    let score = 0;
-                    const weights = {
-                        tag: 0.25,
-                        attributes: 0.35,
-                        content: 0.25,
-                        structure: 0.15
-                    };
-                    if ('tagName' in targetInfo &&
-                        currentTree.tagName === targetInfo.tagName) {
-                        score += weights.tag;
-                    }
-                    const targetAttrs = 'attributes' in targetInfo ? targetInfo.attributes || {} : {};
-                    const currentAttrs = currentTree.attributes || {};
-                    const targetClasses = (targetAttrs.class || '').split(/\s+/).filter(Boolean);
-                    const currentClasses = (currentAttrs.class || '').split(/\s+/).filter(Boolean);
-                    let classScore = 0;
-                    if (targetClasses.length === 0 && currentClasses.length === 0) {
-                        classScore = 1; // Both have no classes
-                    }
-                    else if (targetClasses.length > 0 && currentClasses.length > 0) {
-                        const commonClasses = targetClasses.filter(c => currentClasses.includes(c));
-                        if (commonClasses.length > 0) {
-                            classScore = commonClasses.length / Math.max(targetClasses.length, 1);
-                        }
-                    }
-                    let idScore = 0;
-                    if (targetAttrs.id && currentAttrs.id) {
-                        idScore = targetAttrs.id === currentAttrs.id ? 1 : 0;
-                    }
-                    else if (!targetAttrs.id && !currentAttrs.id) {
-                        idScore = 1; // Both have no id
-                    }
-                    const otherAttrs = Object.keys(Object.assign(Object.assign({}, targetAttrs), currentAttrs))
-                        .filter(key => key !== 'class' && key !== 'id');
-                    let otherAttrsScore = 0;
-                    if (otherAttrs.length > 0) {
-                        let matches = 0;
-                        for (const key of otherAttrs) {
-                            if (targetAttrs[key] === currentAttrs[key]) {
-                                matches++;
-                            }
-                            else if (key in targetAttrs && key in currentAttrs) {
-                                matches += 0.5;
-                            }
-                        }
-                        otherAttrsScore = matches / otherAttrs.length;
-                    }
-                    else {
-                        otherAttrsScore = 1;
-                    }
-                    const attributeWeights = { class: 0.5, id: 0.3, other: 0.2 };
-                    const attrScore = (classScore * attributeWeights.class) +
-                        (idScore * attributeWeights.id) +
-                        (otherAttrsScore * attributeWeights.other);
-                    score += weights.attributes * attrScore;
-                    if ('children' in targetInfo) {
-                        const targetText = targetInfo.children
-                            .filter(child => child.type === 'TEXT_NODE')
-                            .map(child => child.textContent)
-                            .join(' ')
-                            .trim();
-                        const currentText = currentTree.children
-                            .filter(child => child.type === 'TEXT_NODE')
-                            .map(child => child.textContent)
-                            .join(' ')
-                            .trim();
-                        let contentScore = 0;
-                        if (targetText === currentText) {
-                            contentScore = 1;
-                        }
-                        else if (!targetText && !currentText) {
-                            contentScore = 1;
-                        }
-                        else if (targetText && currentText) {
-                            if (targetText.includes(currentText) || currentText.includes(targetText)) {
-                                const ratio = Math.min(targetText.length, currentText.length) /
-                                    Math.max(targetText.length, currentText.length);
-                                contentScore = 0.7 + (ratio * 0.3);
-                            }
-                            else {
-                                const targetWords = targetText.toLowerCase().split(/\s+/).filter(Boolean);
-                                const currentWords = currentText.toLowerCase().split(/\s+/).filter(Boolean);
-                                if (targetWords.length > 0 && currentWords.length > 0) {
-                                    const commonWords = targetWords.filter(word => currentWords.includes(word));
-                                    if (commonWords.length > 0) {
-                                        contentScore = commonWords.length / Math.max(targetWords.length, currentWords.length);
-                                    }
-                                }
-                            }
-                            const numericPattern = /^\s*\$?[\d.,]+\s*$/;
-                            if (numericPattern.test(targetText) && numericPattern.test(currentText)) {
-                                contentScore = Math.max(contentScore, 0.8);
-                            }
-                        }
-                        score += weights.content * contentScore;
-                    }
-                    if ('children' in targetInfo) {
-                        const targetElementChildren = targetInfo.children
-                            .filter(child => child.type === 'ELEMENT_NODE');
-                        const currentElementChildren = currentTree.children
-                            .filter(child => child.type === 'ELEMENT_NODE');
-                        let structureScore = 0;
-                        if (targetElementChildren.length === 0 && currentElementChildren.length === 0) {
-                            structureScore = 1;
-                        }
-                        else if (targetElementChildren.length > 0 && currentElementChildren.length > 0) {
-                            const targetChildTags = targetElementChildren
-                                .map(child => child.tagName.toLowerCase());
-                            const currentChildTags = currentElementChildren
-                                .map(child => child.tagName.toLowerCase());
-                            const tagMatches = targetChildTags.filter(tag => currentChildTags.includes(tag)).length;
-                            const tagCountDiff = Math.abs(targetChildTags.length - currentChildTags.length);
-                            const maxTags = Math.max(targetChildTags.length, currentChildTags.length);
-                            if (maxTags > 0) {
-                                structureScore =
-                                    (0.7 * (tagMatches / maxTags)) +
-                                        (0.3 * (1 - (tagCountDiff / maxTags)));
-                            }
-                        }
-                        score += weights.structure * structureScore;
-                    }
-                    return score;
-                };
-                const score = directMatchScore();
-                if (field === 'listSelector') {
-                    if (score >= 0.85) {
-                        return { element: currentTree, confidence: score };
-                    }
-                }
-                else {
-                    if (score >= 0.7) {
-                        return { element: currentTree, confidence: score };
-                    }
-                }
-                let bestMatch = { element: null, confidence: 0 };
-                for (const child of currentTree.children) {
-                    if (child.type === 'ELEMENT_NODE') {
-                        const childMatch = findMatchingElement(field, child, targetInfo);
-                        if (childMatch.confidence > bestMatch.confidence) {
-                            bestMatch = childMatch;
-                        }
-                    }
-                }
-                return bestMatch;
-            };
-            const findMatchingParentElement = (field, currentDomTree, parentNodeInfo) => {
-                if (currentDomTree.type !== 'ELEMENT_NODE') {
-                    return { element: null, confidence: 0, changes: [] };
-                }
-                const matchesParentElement = (element) => {
-                    if (element.type !== 'ELEMENT_NODE' || parentNodeInfo.type !== 'ELEMENT_NODE') {
-                        return 0;
-                    }
-                    let score = 0;
-                    const weights = {
-                        tagName: 0.3,
-                        structure: 0.3,
-                        children: 0.4
-                    };
-                    if (element.tagName === parentNodeInfo.tagName) {
-                        score += weights.tagName;
-                    }
-                    const targetChildren = parentNodeInfo.children || [];
-                    const currentChildren = element.children || [];
-                    const targetChildTags = targetChildren
-                        .filter(child => child.type === 'ELEMENT_NODE')
-                        .map(child => child.tagName);
-                    const currentChildTags = currentChildren
-                        .filter(child => child.type === 'ELEMENT_NODE')
-                        .map(child => child.tagName);
-                    let matchingChildren = 0;
-                    for (const tag of targetChildTags) {
-                        if (currentChildTags.includes(tag)) {
-                            matchingChildren++;
-                        }
-                    }
-                    if (targetChildTags.length > 0) {
-                        score += weights.structure * (matchingChildren / targetChildTags.length);
-                    }
-                    const targetChildElements = targetChildren.filter(child => child.type === 'ELEMENT_NODE');
-                    const currentChildElements = currentChildren.filter(child => child.type === 'ELEMENT_NODE');
-                    let childContentScore = 0;
-                    let matchedChildrenCount = 0;
-                    for (const targetChild of targetChildElements) {
-                        for (const currentChild of currentChildElements) {
-                            if (targetChild.tagName === currentChild.tagName &&
-                                contentSimilarity(targetChild, currentChild) > 0.7) {
-                                matchedChildrenCount++;
-                                break;
-                            }
-                        }
-                    }
-                    if (targetChildElements.length > 0) {
-                        childContentScore = matchedChildrenCount / targetChildElements.length;
-                        score += weights.children * childContentScore;
-                    }
-                    return score;
-                };
-                const contentSimilarity = (element1, element2) => {
-                    var _a, _b;
-                    if (element1.type !== 'ELEMENT_NODE' || element2.type !== 'ELEMENT_NODE') {
-                        return 0;
-                    }
-                    const text1 = ((_a = element1.children) === null || _a === void 0 ? void 0 : _a.filter(child => child.type === 'TEXT_NODE').map(child => child.textContent).join(' ').trim()) || '';
-                    const text2 = ((_b = element2.children) === null || _b === void 0 ? void 0 : _b.filter(child => child.type === 'TEXT_NODE').map(child => child.textContent).join(' ').trim()) || '';
-                    if (!text1 && !text2)
-                        return 1;
-                    if (!text1 || !text2)
-                        return 0;
-                    if (text1 === text2)
-                        return 1;
-                    if (text1.includes(text2) || text2.includes(text1)) {
-                        return 0.8;
-                    }
-                    const words1 = text1.toLowerCase().split(/\s+/).filter(Boolean);
-                    const words2 = text2.toLowerCase().split(/\s+/).filter(Boolean);
-                    let commonCount = 0;
-                    for (const word of words1) {
-                        if (words2.includes(word)) {
-                            commonCount++;
-                        }
-                    }
-                    return commonCount / Math.max(words1.length, words2.length);
-                };
-                const detectAttributeChanges = (element) => {
-                    const changes = [];
-                    if (element.type !== 'ELEMENT_NODE' ||
-                        parentNodeInfo.type !== 'ELEMENT_NODE') {
-                        return changes;
-                    }
-                    const originalAttrs = parentNodeInfo.attributes || {};
-                    const currentAttrs = element.attributes || {};
-                    for (const key of Object.keys(Object.assign(Object.assign({}, originalAttrs), currentAttrs))) {
-                        if (originalAttrs[key] !== currentAttrs[key]) {
-                            changes.push({
-                                attribute: key,
-                                originalValue: originalAttrs[key] || '',
-                                currentValue: currentAttrs[key] || ''
-                            });
-                        }
-                    }
-                    return changes;
-                };
-                const matchScore = matchesParentElement(currentDomTree);
-                if (field === 'listSelector') {
-                    if (matchScore >= 0.85) {
-                        const changes = detectAttributeChanges(currentDomTree);
-                        return {
-                            element: currentDomTree,
-                            confidence: matchScore,
-                            changes
-                        };
-                    }
-                }
-                else {
-                    if (matchScore >= 0.7) {
-                        const changes = detectAttributeChanges(currentDomTree);
-                        return {
-                            element: currentDomTree,
-                            confidence: matchScore,
-                            changes
-                        };
-                    }
-                }
-                let bestMatch = { element: null, confidence: 0, changes: [] };
-                for (const child of currentDomTree.children) {
-                    if (child.type === 'ELEMENT_NODE') {
-                        const childResult = findMatchingParentElement(field, child, parentNodeInfo);
-                        if (childResult.confidence > bestMatch.confidence) {
-                            bestMatch = childResult;
-                        }
-                    }
-                }
-                return bestMatch;
-            };
-            const processElement = (field, elementConfig) => __awaiter(this, void 0, void 0, function* () {
-                const checkChanges = (originalNode, currentNode, element, isParent = false) => {
-                    const originalAttrs = 'attributes' in originalNode ? originalNode.attributes : {};
-                    const currentAttrs = 'attributes' in currentNode ? currentNode.attributes : {};
-                    if ((Object.keys(originalAttrs).length > 0 && Object.keys(currentAttrs).length === 0) ||
-                        Object.keys(Object.assign(Object.assign({}, originalAttrs), currentAttrs)).some(key => {
-                            if (key === 'class') {
-                                const originalClasses = (originalAttrs[key] || '').split(/\s+/).filter(Boolean);
-                                const currentClasses = (currentAttrs[key] || '').split(/\s+/).filter(Boolean);
-                                return JSON.stringify(originalClasses.sort()) !== JSON.stringify(currentClasses.sort());
-                            }
-                            return originalAttrs[key] !== currentAttrs[key];
-                        })) {
-                        changes.push({
-                            type: 'ATTRIBUTES_CHANGED',
-                            field,
-                            originalState: originalAttrs,
-                            currentState: currentAttrs,
-                            coordinates: currentNode.coordinates,
-                            element: element.asElement(),
-                            isParent,
-                            confidence,
-                        });
-                    }
-                    if ('tagName' in originalNode && originalNode.tagName && currentNode.tagName &&
-                        originalNode.tagName.toLowerCase() !== currentNode.tagName.toLowerCase()) {
-                        changes.push({
-                            type: 'TAG_CHANGED',
-                            field,
-                            originalState: originalNode.tagName,
-                            currentState: currentNode.tagName,
-                            coordinates: currentNode.coordinates,
-                            element: element.asElement(),
-                            isParent,
-                            confidence,
-                        });
-                    }
-                };
-                const { element: matchedElement, confidence } = findMatchingElement(field, currentDomTree, elementConfig.nodeInfo);
-                if (matchedElement) {
-                    const actualElement = yield page.evaluateHandle((element) => {
-                        function findExactElement(elementInfo) {
-                            const candidatesAtCoords = document.elementsFromPoint(elementInfo.coordinates.x, elementInfo.coordinates.y);
-                            const getNormalizedText = (el) => {
-                                var _a;
-                                if (el.nodeType === Node.TEXT_NODE)
-                                    return ((_a = el.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || '';
-                                const textNodes = Array.from(el.childNodes)
-                                    .filter((node) => node.nodeType === Node.TEXT_NODE);
-                                return textNodes.map((node) => { var _a; return (_a = node.textContent) === null || _a === void 0 ? void 0 : _a.trim(); }).join('').trim();
-                            };
-                            const targetText = elementInfo.children
-                                .filter(child => child.type === 'TEXT_NODE')
-                                .map(child => child.textContent)
-                                .join('')
-                                .trim();
-                            const exactMatch = Array.from(candidatesAtCoords).find(candidate => {
-                                if (candidate.tagName.toLowerCase() !== elementInfo.tagName.toLowerCase()) {
-                                    return false;
-                                }
-                                const candidateText = getNormalizedText(candidate);
-                                const textMatch = targetText === candidateText;
-                                if (Object.keys(elementInfo.attributes || {}).length > 0) {
-                                    const attrsMatch = Object.entries(elementInfo.attributes).every(([key, value]) => {
-                                        if (key === 'class') {
-                                            const elementClasses = (value || '').split(/\s+/).filter(Boolean);
-                                            const candidateClasses = (candidate.getAttribute('class') || '')
-                                                .split(/\s+/)
-                                                .filter(Boolean);
-                                            return JSON.stringify(elementClasses.sort()) ===
-                                                JSON.stringify(candidateClasses.sort());
-                                        }
-                                        return candidate.getAttribute(key) === value;
-                                    });
-                                    return textMatch && attrsMatch;
-                                }
-                                const elementChildTags = elementInfo.children
-                                    .filter(child => child.type === 'ELEMENT_NODE')
-                                    .map(child => child.tagName.toLowerCase());
-                                const candidateChildTags = Array.from(candidate.children)
-                                    .map(child => child.tagName.toLowerCase());
-                                const structureMatch = JSON.stringify(elementChildTags) ===
-                                    JSON.stringify(candidateChildTags);
-                                return textMatch && structureMatch;
-                            });
-                            if (exactMatch)
-                                return exactMatch;
-                            const allCandidates = document.getElementsByTagName(elementInfo.tagName);
-                            return Array.from(allCandidates).find(candidate => {
-                                const candidateText = getNormalizedText(candidate);
-                                const textMatch = targetText === candidateText;
-                                const rect = candidate.getBoundingClientRect();
-                                const coordsMatch = Math.abs(rect.x - elementInfo.coordinates.x) < 5 &&
-                                    Math.abs(rect.y - elementInfo.coordinates.y) < 5;
-                                return textMatch && coordsMatch;
-                            });
-                        }
-                        return findExactElement(element);
-                    }, matchedElement);
-                    if (actualElement) {
-                        checkChanges(elementConfig.nodeInfo, matchedElement, actualElement);
-                    }
-                    if ('parent' in elementConfig.nodeInfo && elementConfig.nodeInfo.parent) {
-                        const { element: matchedParentElement, confidence: parentConfidence } = findMatchingParentElement(field, currentDomTree, elementConfig.nodeInfo.parent);
-                        if (matchedParentElement && parentConfidence >= 0.5) {
-                            checkChanges(elementConfig.nodeInfo.parent, matchedParentElement, actualElement, true);
-                        }
-                    }
-                }
-            });
-            if (isScrapeList) {
-                if (schema.listSelectorInfo) {
-                    yield processElement('listSelector', schema.listSelectorInfo);
-                }
-                if (schema.fields) {
-                    for (const [fieldName, fieldConfig] of Object.entries(schema.fields)) {
-                        yield processElement(`fields.${fieldName}`, {
-                            nodeInfo: fieldConfig.nodeInfo,
-                            coordinates: fieldConfig.coordinates,
-                            tag: fieldConfig.tag,
-                        });
-                    }
-                }
-                if ((_a = schema.pagination) === null || _a === void 0 ? void 0 : _a.nodeInfo) {
-                    yield processElement('pagination', {
-                        nodeInfo: schema.pagination.nodeInfo,
-                        coordinates: schema.pagination.coordinates,
-                    });
-                }
-            }
-            else {
-                const scrapeSchema = schema;
-                for (const [field, config] of Object.entries(scrapeSchema)) {
-                    yield processElement(field, {
-                        nodeInfo: config.nodeInfo,
-                        coordinates: config.coordinates,
-                        tag: config.tag,
-                    });
-                }
-            }
-            return changes;
-        });
-    }
-    validateWorkflowAction(page, action) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            const modifiedAction = JSON.parse(JSON.stringify(action));
-            const actionIndex = modifiedAction.what.findIndex(item => item.action === "scrapeSchema" || item.action === "scrapeList");
-            if (actionIndex !== -1) {
-                const schema = modifiedAction.what[actionIndex].args[0];
-                const isScrapeList = modifiedAction.what[actionIndex].action === "scrapeList";
-                try {
-                    const changes = yield this.detectElementChanges(page, schema);
-                    const uniqueChanges = {};
-                    changes.forEach(change => {
-                        if (!uniqueChanges[change.field] || change.type === 'TAG_CHANGED') {
-                            uniqueChanges[change.field] = change;
-                        }
-                    });
-                    // console.log("Unique changes detected:", uniqueChanges);
-                    console.log("Changes detected:", changes);
-                    for (const fieldName in uniqueChanges) {
-                        let schemaField = isScrapeList ?
-                            fieldName === 'listSelector' ? schema[fieldName] :
-                                fieldName.startsWith('fields.') ? schema.fields[fieldName.split('.')[1]] :
-                                    fieldName === 'pagination' ? schema.pagination : null
-                            : schema[fieldName];
-                        const change = uniqueChanges[fieldName];
-                        if (change.element && schemaField) {
-                            try {
-                                let newSelectors;
-                                if (fieldName === 'listSelector') {
-                                    newSelectors = yield (0, selector_1.generateNonUniqueSelectors)(page, change.element, '');
-                                }
-                                else if (isScrapeList && fieldName.startsWith('fields.')) {
-                                    newSelectors = yield (0, selector_1.generateNonUniqueSelectors)(page, change.element, schema.listSelector);
-                                }
-                                else {
-                                    newSelectors = yield (0, selector_1.generateSelectors)(page, change.element);
-                                    if (fieldName === 'pagination') {
-                                        let chainedSelectors = [
-                                            (_a = newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.iframeSelector) === null || _a === void 0 ? void 0 : _a.full,
-                                            (_b = newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.shadowSelector) === null || _b === void 0 ? void 0 : _b.full,
-                                            newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.testIdSelector,
-                                            newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.id,
-                                            newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.hrefSelector,
-                                            newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.accessibilitySelector,
-                                            newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.attrSelector,
-                                        ]
-                                            .filter(selector => selector !== null && selector !== undefined)
-                                            .join(',');
-                                        newSelectors = chainedSelectors;
-                                    }
-                                }
-                                if (!newSelectors) {
-                                    throw new Error('Failed to generate new selectors');
-                                }
-                                const bestSelector = fieldName !== 'pagination' ? yield (0, utils_1.getBestSelector)({
-                                    selectors: newSelectors,
-                                    tagName: change.type === 'TAG_CHANGED' ?
-                                        change.currentState :
-                                        schemaField.tag
-                                }) : newSelectors;
-                                if (!bestSelector) {
-                                    throw new Error('Failed to determine best selector');
-                                }
-                                // Update selectors
-                                let previousSelector;
-                                if (fieldName === 'listSelector') {
-                                    previousSelector = schema.listSelector;
-                                    schema.listSelector = bestSelector;
-                                }
-                                else if (isScrapeList && fieldName.startsWith('fields.')) {
-                                    const fieldKey = fieldName.split('.')[1];
-                                    previousSelector = schema.fields[fieldKey].selector;
-                                    schema.fields[fieldKey].selector = bestSelector;
-                                }
-                                else if (fieldName === 'pagination') {
-                                    previousSelector = schema.pagination.selector;
-                                    schema.pagination.selector = bestSelector;
-                                }
-                                else {
-                                    previousSelector = schema[fieldName].selector;
-                                    schema[fieldName].selector = bestSelector;
-                                }
-                                console.log(`Updated ${fieldName} from ${previousSelector} to ${bestSelector}`);
-                                if (modifiedAction.where.selectors) {
-                                    const selectorIndex = modifiedAction.where.selectors.findIndex(s => s.includes(previousSelector));
-                                    if (selectorIndex !== -1) {
-                                        modifiedAction.where.selectors[selectorIndex] = bestSelector;
-                                    }
-                                }
-                            }
-                            catch (error) {
-                                console.error(`Auto-heal failed for field ${fieldName}:`, error);
-                                this.trackAutohealFailure(`Failed to auto-heal field ${fieldName}: ${error.message}`);
-                            }
-                        }
-                    }
-                }
-                catch (error) {
-                    console.error('Complete auto-heal failure:', error);
-                    this.trackAutohealFailure(`Complete auto-heal failure: ${error.message}`);
-                }
-            }
-            return modifiedAction;
-        });
-    }
     /**
      * Test if a selector is working on the current page
      * @param {Page} page - Playwright page object
@@ -2116,7 +1962,6 @@ class Interpreter extends events_1.EventEmitter {
                     selector.includes('@id=');
                 let count = 0;
                 if (isXPath) {
-                    // Add timeout to prevent XPath hanging
                     const locator = page.locator(`xpath=${selector}`);
                     const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('XPath timeout')), 5000));
                     try {
@@ -2126,12 +1971,10 @@ class Interpreter extends events_1.EventEmitter {
                         ]);
                     }
                     catch (error) {
-                        // XPath timed out or failed
                         return false;
                     }
                 }
                 else {
-                    // Add timeout to CSS selector operations
                     try {
                         const elementsPromise = page.$$(selector);
                         const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('CSS selector timeout')), 5000));
@@ -2142,15 +1985,12 @@ class Interpreter extends events_1.EventEmitter {
                         count = elements ? elements.length : 0;
                     }
                     catch (error) {
-                        // CSS selector timed out or failed
                         return false;
                     }
                 }
-                // For list selectors, we need multiple elements
                 if (isListSelector) {
                     return count >= 2;
                 }
-                // For field selectors, we need at least one element
                 return count >= 1;
             }
             catch (error) {
@@ -2170,12 +2010,10 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, arguments, void 0, function* (page, fallbackSelector, isListSelector = false, listContext = '', isPagination = false) {
             var _a, _b;
             try {
-                // First check if fallback selector works
                 const fallbackWorks = yield this.testSelectorWorks(page, fallbackSelector, isListSelector);
                 if (!fallbackWorks) {
                     return null;
                 }
-                // Get element using fallback selector
                 const isXPath = fallbackSelector.startsWith('//') ||
                     fallbackSelector.startsWith('/') ||
                     fallbackSelector.includes('contains(@');
@@ -2189,7 +2027,6 @@ class Interpreter extends events_1.EventEmitter {
                 if (!element) {
                     return null;
                 }
-                // Generate new selectors
                 let newSelectors;
                 if (isListSelector) {
                     return yield (0, selector_1.generateListSelectorFromFallback)(page, fallbackSelector);
@@ -2200,7 +2037,6 @@ class Interpreter extends events_1.EventEmitter {
                 else {
                     newSelectors = yield (0, selector_1.generateFieldSelectorFromFallback)(page, fallbackSelector);
                     if (isPagination) {
-                        // For pagination, chain selectors in priority order
                         let chainedSelectors = [
                             (_a = newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.iframeSelector) === null || _a === void 0 ? void 0 : _a.full,
                             (_b = newSelectors === null || newSelectors === void 0 ? void 0 : newSelectors.shadowSelector) === null || _b === void 0 ? void 0 : _b.full,
@@ -2215,7 +2051,6 @@ class Interpreter extends events_1.EventEmitter {
                         return chainedSelectors;
                     }
                     else {
-                        // For non-pagination, use getBestSelector
                         const tagName = yield element.evaluate(el => el.tagName.toLowerCase());
                         return yield (0, utils_1.getBestSelector)({
                             selectors: newSelectors,
@@ -2240,7 +2075,6 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             let hasChanges = false;
             try {
-                // Validate listSelector
                 const listSelectorWorks = yield this.testSelectorWorks(page, scrapeListConfig.listSelector, true);
                 if (!listSelectorWorks && scrapeListConfig.listFallbackSelector) {
                     console.log(`ListSelector "${scrapeListConfig.listSelector}" not working, trying fallback...`);
@@ -2251,11 +2085,17 @@ class Interpreter extends events_1.EventEmitter {
                         hasChanges = true;
                     }
                 }
-                // Validate field selectors
                 if (scrapeListConfig.fields) {
-                    for (const [fieldName, fieldConfig] of Object.entries(scrapeListConfig.fields)) {
-                        const fieldSelectorWorks = yield this.testSelectorWorks(page, fieldConfig.selector, false);
-                        if (!fieldSelectorWorks && fieldConfig.fallbackSelector) {
+                    const fieldEntries = Object.entries(scrapeListConfig.fields);
+                    const selectorTests = yield Promise.all(fieldEntries.map((_a) => __awaiter(this, [_a], void 0, function* ([fieldName, fieldConfig]) {
+                        return ({
+                            fieldName,
+                            fieldConfig,
+                            works: yield this.testSelectorWorks(page, fieldConfig.selector, false)
+                        });
+                    })));
+                    for (const { fieldName, fieldConfig, works } of selectorTests) {
+                        if (!works && fieldConfig.fallbackSelector) {
                             console.log(`Field selector "${fieldConfig.selector}" for ${fieldName} not working, trying fallback...`);
                             const newFieldSelector = yield this.generateSelectorFromFallback(page, fieldConfig.fallbackSelector, false, scrapeListConfig.listSelector);
                             if (newFieldSelector) {
@@ -2263,14 +2103,20 @@ class Interpreter extends events_1.EventEmitter {
                                 fieldConfig.selector = newFieldSelector;
                                 hasChanges = true;
                             }
+                            else {
+                                this.log(`WARNING: Failed to autoheal field "${fieldName}" - selector broken and fallback failed. Data for this field may be incomplete.`, logger_1.Level.WARN);
+                                this.trackAutohealFailure(`Field "${fieldName}" autoheal failed - both primary selector and fallback selector failed`);
+                            }
+                        }
+                        else if (!works) {
+                            this.log(`WARNING: Field "${fieldName}" selector not working and no fallback available. Data for this field may be incomplete.`, logger_1.Level.WARN);
+                            this.trackAutohealFailure(`Field "${fieldName}" selector broken with no fallback defined`);
                         }
                     }
                 }
-                // Validate pagination selector if it exists and is not empty
                 if (scrapeListConfig.pagination &&
                     scrapeListConfig.pagination.selector &&
                     scrapeListConfig.pagination.selector.trim() !== '') {
-                    // Handle comma-separated pagination selectors
                     const paginationSelectors = scrapeListConfig.pagination.selector.split(',').map(s => s.trim());
                     let workingSelector = null;
                     for (const selector of paginationSelectors) {
@@ -2311,19 +2157,34 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             let hasChanges = false;
             try {
-                for (const [fieldName, fieldConfig] of Object.entries(scrapeSchemaConfig)) {
-                    if (fieldConfig.selector) {
-                        const selectorWorks = yield this.testSelectorWorks(page, fieldConfig.selector, false);
-                        if (!selectorWorks && fieldConfig.fallbackSelector) {
-                            console.log(`Schema field selector "${fieldConfig.selector}" for ${fieldName} not working, trying fallback...`);
-                            const newSelector = yield this.generateSelectorFromFallback(page, fieldConfig.fallbackSelector, false);
-                            if (newSelector) {
-                                console.log(`Updated schema field selector for ${fieldName}: ${fieldConfig.selector} -> ${newSelector}`);
-                                fieldConfig.selector = newSelector;
-                                hasChanges = true;
-                            }
+                const schemaEntries = Object.entries(scrapeSchemaConfig);
+                const selectorTests = yield Promise.all(schemaEntries
+                    .filter(([_, fieldConfig]) => fieldConfig.selector)
+                    .map((_a) => __awaiter(this, [_a], void 0, function* ([fieldName, fieldConfig]) {
+                    return ({
+                        fieldName,
+                        fieldConfig,
+                        works: yield this.testSelectorWorks(page, fieldConfig.selector, false)
+                    });
+                })));
+                for (const { fieldName, fieldConfig, works } of selectorTests) {
+                    if (!works && fieldConfig.fallbackSelector) {
+                        console.log(`Schema field selector "${fieldConfig.selector}" for ${fieldName} not working, trying fallback...`);
+                        const newSelector = yield this.generateSelectorFromFallback(page, fieldConfig.fallbackSelector, false);
+                        if (newSelector) {
+                            console.log(`Updated schema field selector for ${fieldName}: ${fieldConfig.selector} -> ${newSelector}`);
+                            fieldConfig.selector = newSelector;
+                            hasChanges = true;
+                        }
+                        else {
+                            this.log(`WARNING: Failed to autoheal schema field "${fieldName}" - selector broken and fallback failed. Data for this field may be incomplete.`, logger_1.Level.WARN);
+                            this.trackAutohealFailure(`Schema field "${fieldName}" autoheal failed - both primary selector and fallback selector failed`);
                         }
                     }
+                    else if (!works) {
+                        this.log(`WARNING: Schema field "${fieldName}" selector not working and no fallback available. Data for this field may be incomplete.`, logger_1.Level.WARN);
+                        this.trackAutohealFailure(`Schema field "${fieldName}" selector broken with no fallback defined`);
+                    }
                 }
             }
             catch (error) {
@@ -2343,10 +2204,8 @@ class Interpreter extends events_1.EventEmitter {
             const modifiedAction = JSON.parse(JSON.stringify(action));
             let totalChanges = 0;
             try {
-                // Process each action in the 'what' array
                 for (let i = 0; i < modifiedAction.what.length; i++) {
                     const whatAction = modifiedAction.what[i];
-                    // Handle scrapeList actions
                     if (whatAction.action === 'scrapeList' && whatAction.args && whatAction.args[0]) {
                         console.log(`Validating scrapeList action...`);
                         const hasChanges = yield this.validateScrapeListAction(whatAction.args[0], page);
@@ -2355,7 +2214,6 @@ class Interpreter extends events_1.EventEmitter {
                             console.log(`Fixed scrapeList selectors`);
                         }
                     }
-                    // Handle scrapeSchema actions
                     if (whatAction.action === 'scrapeSchema' && whatAction.args && whatAction.args[0]) {
                         console.log(`Validating scrapeSchema action...`);
                         const hasChanges = yield this.validateScrapeSchemaAction(whatAction.args[0], page);
@@ -2405,8 +2263,6 @@ class Interpreter extends events_1.EventEmitter {
                 else {
                     listElements = Array.from(document.querySelectorAll(selector));
                 }
-                // Extract URLs from the first 'limit' elements that match the selector
-                // The limit corresponds to the number of items that were scraped
                 const elementsToProcess = listElements.slice(0, limit);
                 elementsToProcess.forEach(element => {
                     const urls = [];
@@ -2433,9 +2289,8 @@ class Interpreter extends events_1.EventEmitter {
      * Workflow is bottom-to-top, so we scan from end to start.
      */
     buildDeepExtractionHierarchy(currentWorkflow) {
-        var _a, _b;
+        var _a, _b, _c;
         const hierarchy = [];
-        // Find all goto action indices with their patterns
         const gotoData = [];
         currentWorkflow.forEach((pair, index) => {
             var _a;
@@ -2482,12 +2337,16 @@ class Interpreter extends events_1.EventEmitter {
             }
             let sourceActionName = '';
             let sourceActionType = 'scrapeList';
+            let deepExtractionLimit = undefined;
             if (i === uniqueGotos.length - 1) {
-                const scrapeListBefore = currentWorkflow.slice(gotoIndex + 1).find(pair => pair.what && pair.what.some(action => action.action === 'scrapeList'));
+                const scrapeListBefore = currentWorkflow.slice(0, gotoIndex).reverse().find(pair => pair.what && pair.what.some(action => action.action === 'scrapeList'));
                 if (scrapeListBefore) {
                     const scrapeListAction = scrapeListBefore.what.find(action => action.action === 'scrapeList');
                     sourceActionName = ((_b = (_a = scrapeListAction === null || scrapeListAction === void 0 ? void 0 : scrapeListAction.args) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.name) || (scrapeListAction === null || scrapeListAction === void 0 ? void 0 : scrapeListAction.name) || '';
                     sourceActionType = 'scrapeList';
+                    if (((_c = scrapeListAction === null || scrapeListAction === void 0 ? void 0 : scrapeListAction.args) === null || _c === void 0 ? void 0 : _c[0]) && typeof scrapeListAction.args[0] === 'object') {
+                        deepExtractionLimit = scrapeListAction.args[0].deepExtractionLimit;
+                    }
                 }
             }
             else {
@@ -2499,9 +2358,10 @@ class Interpreter extends events_1.EventEmitter {
                 gotoPattern: String(gotoPattern),
                 actionsToExecute: dataExtractionActions,
                 sourceActionName,
-                sourceActionType
+                sourceActionType,
+                deepExtractionLimit
             });
-            this.log(`Level ${i}: goto at index ${gotoIndex}, pattern=${gotoPattern}, actions=${dataExtractionActions.length}`, logger_1.Level.LOG);
+            this.log(`Level ${i}: goto at index ${gotoIndex}, pattern=${gotoPattern}, actions=${dataExtractionActions.length}${deepExtractionLimit ? `, deepLimit=${deepExtractionLimit}` : ''}`, logger_1.Level.LOG);
         }
         return hierarchy;
     }
@@ -2663,9 +2523,11 @@ class Interpreter extends events_1.EventEmitter {
                         scrapeListIndex,
                         url: matchingUrl
                     });
+                    if (!matchingUrl) {
+                        this.deepExtractionStats.skippedDueToPattern++;
+                    }
                 });
                 const matchedCount = urlMappings.filter(m => m.url !== null).length;
-                this.log(`Filtered to ${matchedCount} matching URLs for deep extraction (out of ${scrapeResults.length} total items)`, logger_1.Level.LOG);
                 if (matchedCount > 0) {
                     const matchedMappings = urlMappings.filter(m => m.url !== null);
                     const sampleSize = Math.min(5, matchedMappings.length);
@@ -2727,20 +2589,17 @@ class Interpreter extends events_1.EventEmitter {
                         continue;
                     if (urlPattern.pathSegments.length !== targetPattern.pathSegments.length)
                         continue;
-                    let pathMatches = true;
-                    for (let i = 0; i < targetPattern.pathSegments.length - 1; i++) {
-                        if (urlPattern.pathSegments[i] !== targetPattern.pathSegments[i]) {
-                            pathMatches = false;
-                            break;
-                        }
-                    }
-                    if (!pathMatches)
-                        continue;
                     const urlNormalized = url.replace(/\/$/, '').toLowerCase();
                     if (urlNormalized === targetNormalized) {
                         this.log(`Excluding already-visited URL: ${url}`, logger_1.Level.LOG);
                         continue;
                     }
+                    const matched = this.matchesGotoPattern(url, String(gotoTargetPattern));
+                    if (!matched) {
+                        this.log(`    ❌ Pattern mismatch for: ${url}`, logger_1.Level.LOG);
+                        continue;
+                    }
+                    this.log(`    ✅ MATCHED: ${url}`, logger_1.Level.LOG);
                     matchingUrl = url;
                     break;
                 }
@@ -2748,9 +2607,11 @@ class Interpreter extends events_1.EventEmitter {
                     scrapeListIndex,
                     url: matchingUrl
                 });
+                if (!matchingUrl) {
+                    this.deepExtractionStats.skippedDueToPattern++;
+                }
             });
-            const matchedCount = urlMappings.filter(m => m.url !== null).length;
-            this.log(`Filtered to ${matchedCount} matching URLs for deep extraction (out of ${scrapeResults.length} total items)`, logger_1.Level.LOG);
+            const matchedCount = extractedUrls.filter(urls => urls.length > 0).length;
             if (matchedCount > 0) {
                 const matchedMappings = urlMappings.filter(m => m.url !== null);
                 const sampleSize = Math.min(5, matchedMappings.length);
@@ -2773,6 +2634,16 @@ class Interpreter extends events_1.EventEmitter {
     /**
      * Helper function to check if a URL matches a goto pattern.
      */
+    /**
+     * Generic pattern matching for deep extraction URLs.
+     * Works across any website by analyzing URL structure rather than relying on keywords.
+     *
+     * Strategy:
+     * 1. Match URLs with same origin and path length
+     * 2. Identify "structural" segments (numbers, short words, etc.) that should match exactly
+     * 3. Allow other segments to vary (dynamic content like IDs, slugs, names)
+     * 4. Skip exact matches to avoid duplicates
+     */
     matchesGotoPattern(url, gotoPattern) {
         try {
             const getUrlPattern = (urlStr) => {
@@ -2797,11 +2668,42 @@ class Interpreter extends events_1.EventEmitter {
             if (urlPattern.pathSegments.length !== targetPattern.pathSegments.length)
                 return false;
             if (urlNormalized === targetNormalized)
-                return false; // Skip exact matches
-            for (let i = 0; i < targetPattern.pathSegments.length - 1; i++) {
-                if (urlPattern.pathSegments[i] !== targetPattern.pathSegments[i]) {
+                return false;
+            /**
+             * Generic heuristic to identify "structural" segments that should match exactly.
+             * These are segments that define the URL structure, not the content.
+             */
+            const isStructuralSegment = (segment, index, totalSegments) => {
+                const normalized = segment.toLowerCase();
+                if (/^\d+$/.test(normalized)) {
+                    return true;
+                }
+                if (normalized.length >= 2 && normalized.length <= 5 && /^[a-z0-9-]+$/.test(normalized)) {
+                    return true;
+                }
+                const wordCount = normalized.split(/[-_]/).length;
+                if (wordCount === 2 && normalized.length <= 15) {
+                    return true;
+                }
+                if (/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(normalized)) {
+                    return false;
+                }
+                if (normalized.length > 20 && /^[a-z0-9-]+$/.test(normalized)) {
                     return false;
                 }
+                if (normalized.length >= 6 && normalized.length <= 20) {
+                    return false;
+                }
+                return index < Math.ceil(totalSegments / 2);
+            };
+            for (let i = 0; i < targetPattern.pathSegments.length; i++) {
+                const targetSegment = targetPattern.pathSegments[i];
+                const urlSegment = urlPattern.pathSegments[i];
+                if (isStructuralSegment(targetSegment, i, targetPattern.pathSegments.length)) {
+                    if (targetSegment.toLowerCase() !== urlSegment.toLowerCase()) {
+                        return false;
+                    }
+                }
             }
             return true;
         }
@@ -2822,24 +2724,52 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 this.log(`\n=== Starting Hierarchical Deep Extraction (${hierarchy.length} level${hierarchy.length > 1 ? 's' : ''}) ===`, logger_1.Level.LOG);
                 this.isInDeepExtractionPhase = true;
-                const startLevel = hierarchy.length >= 2 ? hierarchy.length - 2 : hierarchy.length - 1;
-                for (let levelIndex = startLevel; levelIndex >= 0; levelIndex--) {
+                const startLevel = 0;
+                for (let levelIndex = startLevel; levelIndex < hierarchy.length; levelIndex++) {
                     const level = hierarchy[levelIndex];
                     const currentLevelUrls = level.urlMappings;
-                    this.log(`\n=== Processing Deep Extraction Level ${startLevel - levelIndex + 1}/${startLevel + 1} ===`, logger_1.Level.LOG);
+                    const parentLevel = levelIndex + 1 < hierarchy.length ? hierarchy[levelIndex + 1] : null;
+                    const effectiveLimit = (parentLevel === null || parentLevel === void 0 ? void 0 : parentLevel.deepExtractionLimit) || level.deepExtractionLimit;
+                    this.log(`\n=== Processing Deep Extraction Level ${levelIndex + 1}/${hierarchy.length} ===`, logger_1.Level.LOG);
                     this.log(`Goto pattern: ${level.gotoPattern}`, logger_1.Level.LOG);
                     this.log(`Actions to execute: ${level.actionsToExecute.length}`, logger_1.Level.LOG);
                     this.log(`URLs to process: ${currentLevelUrls.filter(m => m.url !== null).length}`, logger_1.Level.LOG);
+                    if (effectiveLimit) {
+                        this.log(`Deep extraction limit: ${effectiveLimit}`, logger_1.Level.LOG);
+                    }
                     if (currentLevelUrls.length === 0 || currentLevelUrls.every(u => !u.url)) {
                         this.log('No valid URLs at this level - stopping here', logger_1.Level.LOG);
                         break;
                     }
-                    yield this.executeDeepExtractionLevel(page, level, currentLevelUrls);
+                    yield this.executeDeepExtractionLevel(page, Object.assign(Object.assign({}, level), { deepExtractionLimit: effectiveLimit }), currentLevelUrls);
                 }
                 this.log('\n=== Hierarchical Deep Extraction Completed ===', logger_1.Level.LOG);
+                if (this.deepExtractionStats.totalUrlsFound > 0) {
+                    yield this.callWithTimeout(() => this.options.serializableCallback({
+                        deepExtractionStats: {
+                            totalUrlsFound: this.deepExtractionStats.totalUrlsFound,
+                            successfulExtractions: this.deepExtractionStats.successfulExtractions,
+                            failedExtractions: this.deepExtractionStats.failedExtractions
+                        }
+                    }), 30000, 'serializableCallback (deep extraction stats)');
+                }
             }
             catch (error) {
                 this.log(`Hierarchical deep extraction failed: ${error.message}`, logger_1.Level.ERROR);
+                if (this.deepExtractionStats.totalUrlsFound > 0) {
+                    try {
+                        yield this.callWithTimeout(() => this.options.serializableCallback({
+                            deepExtractionStats: {
+                                totalUrlsFound: this.deepExtractionStats.totalUrlsFound,
+                                successfulExtractions: this.deepExtractionStats.successfulExtractions,
+                                failedExtractions: this.deepExtractionStats.failedExtractions
+                            }
+                        }), 30000, 'serializableCallback (deep extraction stats error)');
+                    }
+                    catch (callbackError) {
+                        this.log(`Failed to send stats on error: ${callbackError.message}`, logger_1.Level.ERROR);
+                    }
+                }
             }
             finally {
                 this.isInDeepExtractionPhase = false;
@@ -2854,17 +2784,54 @@ class Interpreter extends events_1.EventEmitter {
     executeDeepExtractionLevel(page, level, urlMappings) {
         return __awaiter(this, void 0, void 0, function* () {
             try {
-                const validMappings = urlMappings.filter(m => m.url !== null);
+                let validMappings = urlMappings.filter(m => m.url !== null);
+                const deepExtractionLimit = level.deepExtractionLimit;
+                if (deepExtractionLimit && validMappings.length > deepExtractionLimit) {
+                    this.log(`Found deepExtractionLimit: ${deepExtractionLimit} from parent action`, logger_1.Level.LOG);
+                    this.log(`Deep extraction limit applied: ${validMappings.length} URLs found, limiting to ${deepExtractionLimit}`, logger_1.Level.LOG);
+                    validMappings = validMappings.slice(0, deepExtractionLimit);
+                }
+                else if (deepExtractionLimit) {
+                    this.log(`Deep extraction limit: ${deepExtractionLimit} URLs configured (found ${validMappings.length} URLs, all within limit)`, logger_1.Level.LOG);
+                }
+                else {
+                    this.log(`No deep extraction limit configured, processing all ${validMappings.length} URLs`, logger_1.Level.LOG);
+                }
                 if (validMappings.length === 0) {
                     this.log('No URLs to process for this level', logger_1.Level.LOG);
                     return;
                 }
-                this.log(`Processing ${validMappings.length} URLs`, logger_1.Level.LOG);
+                const isFirstLevel = this.deepExtractionStats.totalUrlsFound === 0;
+                if (isFirstLevel) {
+                    this.deepExtractionStats.totalUrlsFound = validMappings.length;
+                }
+                else {
+                    this.deepExtractionStats.totalUrlsFound += validMappings.length;
+                }
+                this.log(`Processing ${validMappings.length} URLs for deep extraction`, logger_1.Level.LOG);
                 for (const mapping of validMappings) {
                     try {
                         this.log(`[${mapping.index}] Navigating to: ${mapping.url}`, logger_1.Level.LOG);
                         yield page.goto(mapping.url);
                         yield page.waitForLoadState('networkidle', { timeout: 30000 });
+                        const getTotalItems = () => {
+                            let total = 0;
+                            for (const key in this.serializableDataByType.scrapeSchema || {}) {
+                                const schemaData = this.serializableDataByType.scrapeSchema[key];
+                                if (Array.isArray(schemaData)) {
+                                    total += schemaData.length;
+                                }
+                                else if (schemaData && typeof schemaData === 'object') {
+                                    total += Object.keys(schemaData).length > 0 ? 1 : 0;
+                                }
+                            }
+                            for (const key in this.serializableDataByType.scrapeList || {}) {
+                                const listData = this.serializableDataByType.scrapeList[key];
+                                total += Array.isArray(listData) ? listData.length : 0;
+                            }
+                            return total;
+                        };
+                        const itemCountBefore = getTotalItems();
                         for (let i = level.actionsToExecute.length - 1; i >= 0; i--) {
                             const actionPair = level.actionsToExecute[i];
                             if (this.isAborted) {
@@ -2879,10 +2846,34 @@ class Interpreter extends events_1.EventEmitter {
                                 yield this.carryOutSteps(page, filteredActions);
                             }
                         }
-                        this.log(`[${mapping.index}] Completed`, logger_1.Level.LOG);
+                        const itemCountAfter = getTotalItems();
+                        const dataWasExtracted = itemCountAfter > itemCountBefore;
+                        if (dataWasExtracted) {
+                            this.log(`[${mapping.index}] Completed - Data extracted successfully (${itemCountAfter - itemCountBefore} items)`, logger_1.Level.LOG);
+                            this.deepExtractionStats.successfulExtractions++;
+                        }
+                        else {
+                            this.log(`[${mapping.index}] Completed - No data extracted`, logger_1.Level.WARN);
+                            this.deepExtractionStats.failedExtractions++;
+                        }
+                        yield this.callWithTimeout(() => this.options.serializableCallback({
+                            deepExtractionStats: {
+                                totalUrlsFound: this.deepExtractionStats.totalUrlsFound,
+                                successfulExtractions: this.deepExtractionStats.successfulExtractions,
+                                failedExtractions: this.deepExtractionStats.failedExtractions
+                            }
+                        }), 30000, 'serializableCallback (deep extraction item stats)');
                     }
                     catch (error) {
                         this.log(`[${mapping.index}] Failed: ${error.message}`, logger_1.Level.ERROR);
+                        this.deepExtractionStats.failedExtractions++;
+                        yield this.callWithTimeout(() => this.options.serializableCallback({
+                            deepExtractionStats: {
+                                totalUrlsFound: this.deepExtractionStats.totalUrlsFound,
+                                successfulExtractions: this.deepExtractionStats.successfulExtractions,
+                                failedExtractions: this.deepExtractionStats.failedExtractions
+                            }
+                        }), 30000, 'serializableCallback (deep extraction failed item stats)');
                     }
                 }
             }
@@ -2903,10 +2894,7 @@ class Interpreter extends events_1.EventEmitter {
             catch (error) {
                 this.log(`Failed to apply ad-blocker: ${error.message}`, logger_1.Level.ERROR);
             }
-            const usedActions = [];
-            let selectors = [];
             let lastAction = null;
-            let actionId = -1;
             let repeatCount = 0;
             /**
             *  Enables the interpreter functionality for popup windows.
@@ -2919,12 +2907,11 @@ class Interpreter extends events_1.EventEmitter {
             p.on('popup', popupHandler);
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             let loopIterations = 0;
-            const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
+            const MAX_LOOP_ITERATIONS = 1000;
             let consecutiveFailures = 0;
             const MAX_CONSECUTIVE_FAILURES = 10;
             const startTime = Date.now();
-            const MAX_EXECUTION_TIME = 30 * 60 * 1000; // 30 minutes max
-            // Cleanup function to remove popup listener
+            const MAX_EXECUTION_TIME = 30 * 60 * 1000;
             const cleanup = () => {
                 try {
                     if (!p.isClosed()) {
@@ -2935,32 +2922,26 @@ class Interpreter extends events_1.EventEmitter {
                 }
             };
             while (true) {
-                // Multiple circuit breakers to prevent infinite loops
                 if (++loopIterations > MAX_LOOP_ITERATIONS) {
                     this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
                     cleanup();
                     return;
                 }
-                // Time-based circuit breaker
                 if (Date.now() - startTime > MAX_EXECUTION_TIME) {
                     this.log('Maximum execution time reached (30 minutes), terminating workflow', logger_1.Level.ERROR);
                     cleanup();
                     return;
                 }
-                // Failure-based circuit breaker
                 if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
                     this.log('Too many consecutive failures, terminating to prevent hang', logger_1.Level.ERROR);
                     cleanup();
                     return;
                 }
-                // Check abort flag immediately
                 if (this.isAborted) {
                     this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
                     cleanup();
                     return;
                 }
-                // Checks whether the page was closed from outside,
-                //  or the workflow execution has been stopped via `interpreter.stop()`
                 if (p.isClosed() || !this.stopper) {
                     cleanup();
                     return;
@@ -2992,55 +2973,12 @@ class Interpreter extends events_1.EventEmitter {
                     cleanup();
                     return;
                 }
-                // const newSelectors = this.getSelectors(workflowCopy);
-                //   newSelectors.forEach(selector => {
-                //     if (!selectors.includes(selector)) {
-                //       selectors.push(selector);
-                //     }
-                // });
-                // let pageState = {};
-                // let getStateTest = "Hello";
-                // try {
-                //   pageState = await this.getState(p, workflowCopy, selectors);
-                //   selectors = [];
-                //   console.log("Empty selectors:", selectors)
-                // } catch (e: any) {
-                //   this.log('The browser has been closed.');
-                //   return;
-                // }
-                // if (this.options.debug) {
-                //   this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
-                // }
-                // const actionId = workflow.findIndex((step) => {
-                //   const isApplicable = this.applicable(step.where, pageState, usedActions);
-                //   console.log("-------------------------------------------------------------");
-                //   console.log(`Where:`, step.where);
-                //   console.log(`Page state:`, pageState);
-                //   console.log(`Match result: ${isApplicable}`);
-                //   console.log("-------------------------------------------------------------");
-                //   return isApplicable;
-                // });
-                // if (workflowCopy[0]) {
-                //   if (workflowCopy[0].what[1].action === 'scrapeSchema') {
-                //     const schema = workflowCopy[0].what[1].args[0];
-                //     await p.goto(workflowCopy[0].where.url.toString())
-                //     await p.waitForLoadState();
-                //     const changes = await this.detectElementChanges(p, schema);
-                //     console.log("Page URL: ", workflowCopy[0].where.url.toString());
-                //     console.log("SCHEMA CHANGES:", changes);
-                //   }
-                // }
-                // actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
-                // if (actionId !== -1 && workflowCopy[actionId]) {
-                //   workflowCopy[actionId] = await this.validateWorkflowAction(p, workflowCopy[actionId]);
-                // }
                 const actionId = workflowCopy.length - 1;
                 const action = workflowCopy[actionId];
                 console.log("MATCHED ACTION:", action);
                 console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
-                if (action) { // action is matched
-                    // Check abort flag before executing action
+                if (action) {
                     if (this.isAborted) {
                         this.log('Workflow aborted before action execution', logger_1.Level.WARN);
                         return;
@@ -3054,27 +2992,43 @@ class Interpreter extends events_1.EventEmitter {
                         return;
                     }
                     lastAction = action;
-                    try {
-                        const validatedAction = yield this.validateAndFixSelectors(p, action);
-                        console.log("Carrying out:", validatedAction.what);
-                        yield this.carryOutSteps(p, validatedAction.what, workflowCopy);
-                        usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
-                        workflowCopy.splice(actionId, 1);
-                        console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // Reset counters on successful action (but keep some history to prevent infinite resets)
-                        loopIterations = Math.max(0, loopIterations - 10);
-                        consecutiveFailures = 0;
-                        // Add async yield to prevent event loop blocking
-                        if (loopIterations % 10 === 0) {
-                            yield new Promise(resolve => setImmediate(resolve));
-                        }
-                    }
-                    catch (e) {
-                        this.log(e, logger_1.Level.ERROR);
-                        consecutiveFailures++;
-                        // Add delay on failures to prevent tight error loops
-                        yield new Promise(resolve => setTimeout(resolve, Math.min(1000, consecutiveFailures * 200)));
-                        // Don't crash on individual action failures - continue with next iteration
+                    const MAX_ACTION_RETRIES = 3;
+                    let actionRetries = 0;
+                    let actionSucceeded = false;
+                    while (actionRetries < MAX_ACTION_RETRIES && !actionSucceeded) {
+                        try {
+                            const validatedAction = yield this.validateAndFixSelectors(p, action);
+                            console.log("Carrying out:", validatedAction.what);
+                            yield this.carryOutSteps(p, validatedAction.what, workflowCopy);
+                            workflowCopy.splice(actionId, 1);
+                            console.log(`Action with ID ${action.id} removed from the workflow copy.`);
+                            this.executedActions++;
+                            const percentage = Math.round((this.executedActions / this.totalActions) * 100);
+                            if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.progressUpdate) {
+                                this.options.debugChannel.progressUpdate(this.executedActions, this.totalActions, percentage);
+                            }
+                            actionSucceeded = true;
+                            consecutiveFailures = 0;
+                            loopIterations = Math.max(0, loopIterations - 10);
+                            if (loopIterations % 10 === 0) {
+                                yield new Promise(resolve => setImmediate(resolve));
+                            }
+                        }
+                        catch (e) {
+                            actionRetries++;
+                            this.log(e, logger_1.Level.ERROR);
+                            if (actionRetries < MAX_ACTION_RETRIES) {
+                                this.log(`Retrying action (attempt ${actionRetries + 1}/${MAX_ACTION_RETRIES})`, logger_1.Level.WARN);
+                                yield new Promise(resolve => setTimeout(resolve, 1000 * actionRetries));
+                            }
+                            else {
+                                this.log(`Action failed after ${MAX_ACTION_RETRIES} retries`, logger_1.Level.ERROR);
+                                consecutiveFailures++;
+                                yield new Promise(resolve => setTimeout(resolve, Math.min(1000, consecutiveFailures * 200)));
+                            }
+                        }
+                    }
+                    if (!actionSucceeded) {
                         continue;
                     }
                 }
@@ -3088,8 +3042,8 @@ class Interpreter extends events_1.EventEmitter {
     }
     ensureScriptsLoaded(page) {
         return __awaiter(this, void 0, void 0, function* () {
+            let scriptsLoaded = false;
             try {
-                // Add timeout to prevent hanging on script evaluation
                 const evaluationPromise = page.evaluate(() => typeof window.scrape === 'function' &&
                     typeof window.scrapeSchema === 'function' &&
                     typeof window.scrapeList === 'function' &&
@@ -3097,22 +3051,21 @@ class Interpreter extends events_1.EventEmitter {
                     typeof window.scrollDown === 'function' &&
                     typeof window.scrollUp === 'function');
                 const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Script check timeout')), 3000));
-                const isScriptLoaded = yield Promise.race([
+                scriptsLoaded = yield Promise.race([
                     evaluationPromise,
                     timeoutPromise
                 ]);
-                if (!isScriptLoaded) {
-                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
-                }
             }
             catch (error) {
-                // If script check fails, try to add the script anyway
-                this.log(`Script check failed, adding script anyway: ${error.message}`, logger_1.Level.WARN);
+                this.log(`Script check failed or timed out: ${error.message}`, logger_1.Level.WARN);
+                scriptsLoaded = false;
+            }
+            if (!scriptsLoaded) {
                 try {
                     yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
                 }
                 catch (scriptError) {
-                    this.log(`Failed to add script: ${scriptError.message}`, logger_1.Level.ERROR);
+                    this.log(`Failed to add scraper script: ${scriptError.message}`, logger_1.Level.ERROR);
                 }
             }
         });
@@ -3127,10 +3080,10 @@ class Interpreter extends events_1.EventEmitter {
      */
     run(page, params) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             this.log('Starting the workflow.', logger_1.Level.LOG);
             const context = page.context();
             page.setDefaultNavigationTimeout(100000);
-            // Check proxy settings from context options
             const contextOptions = context._options;
             const hasProxy = !!(contextOptions === null || contextOptions === void 0 ? void 0 : contextOptions.proxy);
             this.log(`Proxy settings: ${hasProxy ? `Proxy is configured...` : 'No proxy configured...'}`);
@@ -3146,6 +3099,11 @@ class Interpreter extends events_1.EventEmitter {
              * `this.workflow` with the parameters initialized.
              */
             this.initializedWorkflow = preprocessor_1.default.initWorkflow(this.workflow, params);
+            this.totalActions = this.initializedWorkflow.length;
+            this.executedActions = 0;
+            if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.progressUpdate) {
+                this.options.debugChannel.progressUpdate(0, this.totalActions, 0);
+            }
             yield this.ensureScriptsLoaded(page);
             this.stopper = () => {
                 this.stopper = null;
@@ -3173,7 +3131,6 @@ class Interpreter extends events_1.EventEmitter {
     cleanup() {
         return __awaiter(this, void 0, void 0, function* () {
             try {
-                // Stop any running workflows first
                 if (this.stopper) {
                     try {
                         yield this.stop();
@@ -3182,7 +3139,6 @@ class Interpreter extends events_1.EventEmitter {
                         this.log(`Error stopping workflow during cleanup: ${error.message}`, logger_1.Level.WARN);
                     }
                 }
-                // Clear ad-blocker resources
                 if (this.blocker) {
                     try {
                         this.blocker = null;
@@ -3192,12 +3148,10 @@ class Interpreter extends events_1.EventEmitter {
                         this.log(`Error cleaning up ad-blocker: ${error.message}`, logger_1.Level.WARN);
                     }
                 }
-                // Clear accumulated data to free memory
                 this.cumulativeResults = [];
                 this.autohealFailures = [];
                 this.namedResults = {};
                 this.serializableDataByType = { scrapeList: {}, scrapeSchema: {} };
-                // Reset state
                 this.isAborted = false;
                 this.initializedWorkflow = null;
                 this.log('Interpreter cleanup completed', logger_1.Level.DEBUG);