maxun-core 0.0.31 → 0.0.33

package/build/interpret.d.ts

@@ -37,6 +37,7 @@ interface InterpreterOptions {
     serializableCallback: (output: any) => (void | Promise<void>);
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
+    type?: 'extract' | 'scrape' | 'crawl' | 'search';
     debugChannel: Partial<{
         activeId: (id: number) => void;
         debugMessage: (msg: string) => void;
@@ -55,6 +56,7 @@ export default class Interpreter extends EventEmitter {
     private concurrency;
     private stopper;
     private isAborted;
+    private visualRenderRequired;
     private log;
     private cumulativeResults;
     private namedResults;
@@ -90,6 +92,43 @@ export default class Interpreter extends EventEmitter {
      * @returns True if `where` is applicable in the given context, false otherwise
      */
     private applicable;
+    /**
+     * Returns the optimal Playwright `waitUntil` navigation strategy based on
+     * whether the current operation requires visual rendering.
+     *
+     * - `'networkidle'`            — used when screenshots are requested; waits for all
+     *                         sub-resources so the page renders correctly.
+     * - `'domcontentloaded'` — used for all DOM-only operations (scraping, crawling,
+     *                         extraction, search); skips stylesheet/image loading for
+     *                         maximum speed.
+     *
+     * @param blockOverride Pass `true` when the caller will take a screenshot
+     *                          or requires styled layout. Defaults to `false`.
+     */
+    private getNavigationWaitStrategy;
+    /**
+     * Returns true if any step in the given `what` block requires a fully
+     * rendered page.
+     */
+    private blockNeedsVisualRender;
+    /**
+     * Returns true if any of the remaining blocks in the workflow require a visual render
+     * before the next page navigation.
+     */
+    private remainingWorkflowNeedsVisualRender;
+    /**
+     * Helper to wait for a "Network Quiet Window" (no meaningful activity for X ms).
+     */
+    private waitForNetworkQuiet;
+    /**
+     * Scans the remaining workflow to find the next meaningful extraction selector.
+     */
+    private getUpcomingExtractionSelector;
+    /**
+     * Function to wait for images to load.
+     */
+    private waitForImagesLoaded;
+    private waitForDynamicStability;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -35,7 +35,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const path_1 = __importDefault(require("path"));
+const path = __importStar(require("path"));
 const events_1 = require("events");
 const logic_1 = require("./types/logic");
 const utils_1 = require("./utils/utils");
@@ -51,6 +51,7 @@ class Interpreter extends events_1.EventEmitter {
         super();
         this.stopper = null;
         this.isAborted = false;
+        this.visualRenderRequired = false;
         // private blocker: PlaywrightBlocker | null = null;
         this.cumulativeResults = [];
         this.namedResults = {};
@@ -69,6 +70,7 @@ class Interpreter extends events_1.EventEmitter {
         this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
                 (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN);
             }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
+        this.visualRenderRequired = ((options === null || options === void 0 ? void 0 : options.type) === 'extract');
         this.concurrency = new concurrency_1.default(this.options.maxConcurrency);
         this.log = (...args) => (0, logger_1.default)(...args);
         const error = preprocessor_1.default.validateWorkflow(workflow);
@@ -290,6 +292,167 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    /**
+     * Returns the optimal Playwright `waitUntil` navigation strategy based on
+     * whether the current operation requires visual rendering.
+     *
+     * - `'networkidle'`            — used when screenshots are requested; waits for all
+     *                         sub-resources so the page renders correctly.
+     * - `'domcontentloaded'` — used for all DOM-only operations (scraping, crawling,
+     *                         extraction, search); skips stylesheet/image loading for
+     *                         maximum speed.
+     *
+     * @param blockOverride Pass `true` when the caller will take a screenshot
+     *                          or requires styled layout. Defaults to `false`.
+     */
+    getNavigationWaitStrategy(blockOverride) {
+        const finalRequirement = blockOverride !== null && blockOverride !== void 0 ? blockOverride : this.visualRenderRequired;
+        return finalRequirement ? 'networkidle' : 'domcontentloaded';
+    }
+    /**
+     * Returns true if any step in the given `what` block requires a fully
+     * rendered page.
+     */
+    blockNeedsVisualRender(steps) {
+        return steps.some((s) => {
+            var _a, _b;
+            if (s.action === 'screenshot')
+                return true;
+            if (s.action === 'scrapeList' || s.action === 'scrapeSchema')
+                return true;
+            const firstArg = Array.isArray(s.args) ? s.args[0] : s.args;
+            if (!firstArg || typeof firstArg !== 'object')
+                return false;
+            if (s.action === 'scrape') {
+                const formats = (_a = firstArg.formats) !== null && _a !== void 0 ? _a : [];
+                const heavyFormats = ['markdown', 'html', 'text', 'screenshot-visible', 'screenshot-full'];
+                return formats.some((f) => heavyFormats.includes(f));
+            }
+            if (s.action === 'crawl' || s.action === 'search') {
+                const outputFormats = (_b = firstArg.outputFormats) !== null && _b !== void 0 ? _b : [];
+                const heavyFormats = ['markdown', 'html', 'text', 'screenshot-visible', 'screenshot-full'];
+                return outputFormats.some((f) => heavyFormats.includes(f));
+            }
+            return false;
+        });
+    }
+    /**
+     * Returns true if any of the remaining blocks in the workflow require a visual render
+     * before the next page navigation.
+     */
+    remainingWorkflowNeedsVisualRender(remainingWorkflow) {
+        if (!remainingWorkflow || remainingWorkflow.length === 0)
+            return false;
+        for (let i = remainingWorkflow.length - 1; i >= 0; i--) {
+            const pair = remainingWorkflow[i];
+            if (this.blockNeedsVisualRender(pair.what))
+                return true;
+            if (pair.what.some(s => s.action === 'goto'))
+                return false;
+        }
+        return false;
+    }
+    /**
+     * Helper to wait for a "Network Quiet Window" (no meaningful activity for X ms).
+     */
+    waitForNetworkQuiet(page, timeout = 4000, quietWindow = 600) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let lastRequestTime = Date.now();
+            const onRequest = () => { lastRequestTime = Date.now(); };
+            page.on('request', onRequest);
+            page.on('requestfinished', onRequest);
+            page.on('requestfailed', onRequest);
+            try {
+                const checkInterval = 100;
+                const start = Date.now();
+                while (Date.now() - start < timeout) {
+                    if (Date.now() - lastRequestTime > quietWindow)
+                        return;
+                    yield new Promise(r => setTimeout(r, checkInterval));
+                }
+            }
+            finally {
+                page.off('request', onRequest);
+                page.off('requestfinished', onRequest);
+                page.off('requestfailed', onRequest);
+            }
+        });
+    }
+    /**
+     * Scans the remaining workflow to find the next meaningful extraction selector.
+     */
+    getUpcomingExtractionSelector(remainingWorkflow) {
+        if (!remainingWorkflow || remainingWorkflow.length === 0)
+            return null;
+        for (let i = remainingWorkflow.length - 1; i >= 0; i--) {
+            const pair = remainingWorkflow[i];
+            for (const s of pair.what) {
+                if (s.action === 'goto')
+                    return null;
+                if (s.action === 'scrapeList' || s.action === 'scrapeSchema') {
+                    const firstArg = Array.isArray(s.args) ? s.args[0] : s.args;
+                    if (firstArg === null || firstArg === void 0 ? void 0 : firstArg.listSelector)
+                        return firstArg.listSelector;
+                    if (firstArg === null || firstArg === void 0 ? void 0 : firstArg.fields) {
+                        const firstField = Object.values(firstArg.fields)[0];
+                        if (firstField === null || firstField === void 0 ? void 0 : firstField.selector)
+                            return firstField.selector;
+                    }
+                    if (firstArg === null || firstArg === void 0 ? void 0 : firstArg.selector)
+                        return firstArg.selector;
+                }
+            }
+        }
+        return null;
+    }
+    /**
+     * Function to wait for images to load.
+     */
+    waitForImagesLoaded(page) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield page.waitForFunction(() => Array.from(document.images).every(img => img.complete), { timeout: 5000 }).catch(() => { });
+        });
+    }
+    waitForDynamicStability(page, upcomingWorkflow = []) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const targetSelector = this.getUpcomingExtractionSelector(upcomingWorkflow);
+                const signals = [
+                    this.waitForNetworkQuiet(page, 10000, 1000),
+                    page.evaluate(() => __awaiter(this, void 0, void 0, function* () {
+                        let lastLen = 0;
+                        let stableIterations = 0;
+                        for (let i = 0; i < 60; i++) {
+                            const currentLen = document.body.innerText.length;
+                            if (currentLen > 200 && currentLen === lastLen) {
+                                stableIterations++;
+                            }
+                            else {
+                                stableIterations = 0;
+                            }
+                            if (stableIterations >= 8)
+                                return true;
+                            lastLen = currentLen;
+                            yield new Promise(r => setTimeout(r, 100));
+                        }
+                        return false;
+                    })).catch(() => { }),
+                    new Promise(resolve => setTimeout(resolve, 10000))
+                ];
+                if (targetSelector) {
+                    const found = yield page.waitForSelector(targetSelector, { timeout: 8000 }).catch(() => null);
+                    if (found) {
+                        yield new Promise(resolve => setTimeout(resolve, 1000));
+                        return;
+                    }
+                }
+                yield Promise.race(signals);
+                yield new Promise(resolve => setTimeout(resolve, 1500));
+            }
+            catch (e) {
+            }
+        });
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -299,7 +462,7 @@ class Interpreter extends events_1.EventEmitter {
    * @param page Playwright Page object
    * @param steps Array of actions.
    */
-    carryOutSteps(page, steps) {
+    carryOutSteps(page, steps, currentWorkflow) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
             if (this.isAborted) {
@@ -320,6 +483,7 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.setActionType) {
                         this.options.debugChannel.setActionType("screenshot");
                     }
+                    yield this.waitForImagesLoaded(page);
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
                     const explicitName = (typeof nameOverride === 'string' && nameOverride.trim().length > 0) ? nameOverride.trim() : null;
                     let screenshotName;
@@ -352,8 +516,7 @@ class Interpreter extends events_1.EventEmitter {
                             let newPage = null;
                             try {
                                 newPage = yield context.newPage();
-                                yield newPage.goto(link);
-                                yield newPage.waitForLoadState('networkidle');
+                                yield newPage.goto(link, { waitUntil: this.getNavigationWaitStrategy() });
                                 yield this.runLoop(newPage, this.initializedWorkflow);
                             }
                             catch (e) {
@@ -381,6 +544,10 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_d = this.options.debugChannel) === null || _d === void 0 ? void 0 : _d.setActionType) {
                         this.options.debugChannel.setActionType('scrape');
                     }
+                    yield this.waitForDynamicStability(page, [{
+                            action: 'scrape',
+                            args: [selector]
+                        }]);
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
                     yield this.options.serializableCallback(scrapeResults);
@@ -394,12 +561,22 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_e = this.options.debugChannel) === null || _e === void 0 ? void 0 : _e.setActionType) {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
+                    yield this.waitForDynamicStability(page, [{
+                            action: 'scrapeSchema',
+                            args: [schema]
+                        }]);
                     if (this.options.mode && this.options.mode === 'editor') {
                         yield this.options.serializableCallback({});
                         return;
                     }
                     yield this.ensureScriptsLoaded(page);
-                    const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
+                    const normalizedSchema = Object.fromEntries(Object.entries(schema).map(([key, value]) => [
+                        key,
+                        typeof value === 'string'
+                            ? { selector: value, tag: '', attribute: 'innerText', shadow: '' }
+                            : value,
+                    ]));
+                    const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), normalizedSchema);
                     if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
                         this.cumulativeResults = [];
                     }
@@ -446,10 +623,10 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     this.serializableDataByType[actionType][name] = [...this.cumulativeResults];
                     yield this.options.serializableCallback({
-                        scrapeList: this.serializableDataByType.scrapeList,
-                        scrapeSchema: this.serializableDataByType.scrapeSchema,
-                        crawl: this.serializableDataByType.crawl || {},
-                        search: this.serializableDataByType.search || {}
+                        scrapeList: this.serializableDataByType['scrapeList'],
+                        scrapeSchema: this.serializableDataByType['scrapeSchema'],
+                        crawl: this.serializableDataByType['crawl'] || {},
+                        search: this.serializableDataByType['search'] || {}
                     });
                 }),
                 scrapeList: (config, actionName = "") => __awaiter(this, void 0, void 0, function* () {
@@ -505,8 +682,8 @@ class Interpreter extends events_1.EventEmitter {
                             }
                             this.serializableDataByType[actionType][name].push(...scrapeResults);
                             yield this.options.serializableCallback({
-                                scrapeList: this.serializableDataByType.scrapeList,
-                                scrapeSchema: this.serializableDataByType.scrapeSchema
+                                scrapeList: this.serializableDataByType['scrapeList'],
+                                scrapeSchema: this.serializableDataByType['scrapeSchema']
                             });
                         }
                     }
@@ -525,8 +702,8 @@ class Interpreter extends events_1.EventEmitter {
                             this.serializableDataByType[actionType] = {};
                         this.serializableDataByType[actionType][name] = [];
                         yield this.options.serializableCallback({
-                            scrapeList: this.serializableDataByType.scrapeList,
-                            scrapeSchema: this.serializableDataByType.scrapeSchema
+                            scrapeList: this.serializableDataByType['scrapeList'],
+                            scrapeSchema: this.serializableDataByType['scrapeSchema']
                         });
                     }
                 }),
@@ -765,8 +942,7 @@ class Interpreter extends events_1.EventEmitter {
                         };
                         const extractLinksFromPage = () => __awaiter(this, void 0, void 0, function* () {
                             try {
-                                yield page.waitForLoadState('load', { timeout: 15000 }).catch(() => { });
-                                yield page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
+                                yield page.waitForLoadState(this.getNavigationWaitStrategy(), { timeout: 15000 }).catch(() => { });
                                 yield new Promise(resolve => setTimeout(resolve, 1000));
                                 const pageLinks = yield page.evaluate(() => {
                                     const links = [];
@@ -937,12 +1113,12 @@ class Interpreter extends events_1.EventEmitter {
                                     yield new Promise(resolve => setTimeout(resolve, robotRules.crawlDelay));
                                 }
                                 yield page.goto(url, {
-                                    waitUntil: 'domcontentloaded',
+                                    waitUntil: this.getNavigationWaitStrategy(),
                                     timeout: 30000
                                 }).catch((err) => {
                                     throw new Error(`Navigation failed: ${err.message}`);
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                yield this.waitForDynamicStability(page, currentWorkflow || []);
                                 const pageResult = yield scrapePageContent(url);
                                 pageResult.metadata.depth = depth;
                                 crawlResults.push(pageResult);
@@ -989,10 +1165,10 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         this.serializableDataByType[actionType][actionName] = crawlResults;
                         yield this.options.serializableCallback({
-                            scrapeList: this.serializableDataByType.scrapeList || {},
-                            scrapeSchema: this.serializableDataByType.scrapeSchema || {},
-                            crawl: this.serializableDataByType.crawl || {},
-                            search: this.serializableDataByType.search || {}
+                            scrapeList: this.serializableDataByType['scrapeList'] || {},
+                            scrapeSchema: this.serializableDataByType['scrapeSchema'] || {},
+                            crawl: this.serializableDataByType['crawl'] || {},
+                            search: this.serializableDataByType['search'] || {}
                         });
                     }
                     catch (error) {
@@ -1025,7 +1201,7 @@ class Interpreter extends events_1.EventEmitter {
                         const initialDelay = 500 + Math.random() * 1000;
                         yield new Promise(resolve => setTimeout(resolve, initialDelay));
                         yield page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
-                        yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => {
+                        yield page.waitForLoadState(this.getNavigationWaitStrategy(), { timeout: 15000 }).catch(() => {
                             this.log('Load state timeout, continuing anyway', logger_1.Level.WARN);
                         });
                         const pageLoadDelay = 2000 + Math.random() * 1500;
@@ -1214,10 +1390,10 @@ class Interpreter extends events_1.EventEmitter {
                             };
                             this.serializableDataByType[actionType][actionName] = searchData;
                             yield this.options.serializableCallback({
-                                scrapeList: this.serializableDataByType.scrapeList || {},
-                                scrapeSchema: this.serializableDataByType.scrapeSchema || {},
-                                crawl: this.serializableDataByType.crawl || {},
-                                search: this.serializableDataByType.search || {}
+                                scrapeList: this.serializableDataByType['scrapeList'] || {},
+                                scrapeSchema: this.serializableDataByType['scrapeSchema'] || {},
+                                crawl: this.serializableDataByType['crawl'] || {},
+                                search: this.serializableDataByType['search'] || {}
                             });
                             this.log(`Search completed in discover mode with ${searchResults.length} results`, logger_1.Level.LOG);
                             return;
@@ -1229,12 +1405,12 @@ class Interpreter extends events_1.EventEmitter {
                             try {
                                 this.log(`[${i + 1}/${searchResults.length}] Scraping: ${result.url}`, logger_1.Level.LOG);
                                 yield page.goto(result.url, {
-                                    waitUntil: 'domcontentloaded',
+                                    waitUntil: this.getNavigationWaitStrategy(),
                                     timeout: 30000
                                 }).catch(() => {
                                     this.log(`Failed to navigate to ${result.url}, skipping...`, logger_1.Level.WARN);
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                yield this.waitForDynamicStability(page, currentWorkflow || []);
                                 const pageData = yield page.evaluate(() => {
                                     var _a, _b;
                                     const getMeta = (name) => {
@@ -1321,10 +1497,10 @@ class Interpreter extends events_1.EventEmitter {
                         };
                         this.serializableDataByType[actionType][actionName] = searchData;
                         yield this.options.serializableCallback({
-                            scrapeList: this.serializableDataByType.scrapeList || {},
-                            scrapeSchema: this.serializableDataByType.scrapeSchema || {},
-                            crawl: this.serializableDataByType.crawl || {},
-                            search: this.serializableDataByType.search || {}
+                            scrapeList: this.serializableDataByType['scrapeList'] || {},
+                            scrapeSchema: this.serializableDataByType['scrapeSchema'] || {},
+                            crawl: this.serializableDataByType['crawl'] || {},
+                            search: this.serializableDataByType['search'] || {}
                         });
                     }
                     catch (error) {
@@ -1402,19 +1578,52 @@ class Interpreter extends events_1.EventEmitter {
                     for (const level of levels.splice(0, levels.length - 1)) {
                         invokee = invokee[level];
                     }
-                    if (methodName === 'waitForLoadState') {
+                    if (methodName === 'goto') {
+                        try {
+                            const gotoArgs = step.args || [];
+                            const url = gotoArgs[0];
+                            const existingOpts = (typeof gotoArgs[1] === 'object' && gotoArgs[1] !== null)
+                                ? Object.assign({}, gotoArgs[1]) : {};
+                            const requestedWait = existingOpts.waitUntil;
+                            const remaining = (currentWorkflow || []).slice(0, -1);
+                            const needsDataSoon = this.blockNeedsVisualRender(steps) || this.remainingWorkflowNeedsVisualRender(remaining);
+                            if (!requestedWait || requestedWait === 'networkidle' || requestedWait === 'load') {
+                                existingOpts.waitUntil = 'domcontentloaded';
+                                this.log(`goto: navigation speed-optimized to 'domcontentloaded' + surgical-ready midground`, logger_1.Level.LOG);
+                            }
+                            if (!existingOpts.timeout)
+                                existingOpts.timeout = 15000;
+                            yield executeAction(invokee, methodName, [url, existingOpts]);
+                            if (needsDataSoon) {
+                                yield this.waitForDynamicStability(page, (currentWorkflow || []).slice(0, -1));
+                            }
+                        }
+                        catch (error) {
+                            this.log(`goto failed: ${error.message}`, logger_1.Level.WARN);
+                        }
+                    }
+                    else if (methodName === 'waitForLoadState') {
                         try {
                             let args = step.args;
-                            if (Array.isArray(args) && args.length === 1) {
-                                args = [args[0], { timeout: 30000 }];
+                            if (!Array.isArray(args)) {
+                                args = [args];
                             }
-                            else if (!Array.isArray(args)) {
-                                args = [args, { timeout: 30000 }];
+                            const requestedState = args[0];
+                            const remaining = (currentWorkflow || []).slice(0, -1);
+                            const needsDataSoon = this.blockNeedsVisualRender(steps) || this.remainingWorkflowNeedsVisualRender(remaining);
+                            const optimalState = (requestedState === 'networkidle' || requestedState === 'load')
+                                ? 'domcontentloaded'
+                                : requestedState;
+                            this.log(`waitForLoadState: workflow requested '${requestedState}', using 'domcontentloaded' + surgical-ready midground`, logger_1.Level.LOG);
+                            args = [optimalState, { timeout: 15000 }];
+                            yield executeAction(invokee, methodName, args);
+                            if (needsDataSoon) {
+                                yield this.waitForDynamicStability(page, (currentWorkflow || []).slice(0, -1));
                             }
-                            yield executeAction(invokee, methodName, step.args);
                         }
                         catch (error) {
-                            yield executeAction(invokee, methodName, 'domcontentloaded');
+                            yield executeAction(invokee, methodName, ['domcontentloaded', { timeout: 10000 }]);
+                            yield this.waitForDynamicStability(page, (currentWorkflow || []).slice(0, -1));
                         }
                     }
                     else if (methodName === 'click') {
@@ -1423,7 +1632,8 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         catch (error) {
                             try {
-                                yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                                const clickArgs = Array.isArray(step.args) ? step.args : [step.args];
+                                yield executeAction(invokee, methodName, [clickArgs[0], { force: true }]);
                             }
                             catch (error) {
                                 this.log(`Click action failed: ${error.message}`, logger_1.Level.WARN);
@@ -1479,6 +1689,10 @@ class Interpreter extends events_1.EventEmitter {
                     debugLog("Workflow aborted, stopping scrapeCurrentPage");
                     return;
                 }
+                yield this.waitForDynamicStability(page, [{
+                        action: 'scrapeList',
+                        args: [config]
+                    }]);
                 const evaluationPromise = page.evaluate((cfg) => window.scrapeList(cfg), config);
                 const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 10000));
                 let results;
@@ -1509,10 +1723,10 @@ class Interpreter extends events_1.EventEmitter {
                 allResults = allResults.concat(itemsToAdd);
                 this.serializableDataByType[actionType][actionName] = [...allResults];
                 yield this.options.serializableCallback({
-                    scrapeList: this.serializableDataByType.scrapeList,
-                    scrapeSchema: this.serializableDataByType.scrapeSchema,
-                    crawl: this.serializableDataByType.crawl || {},
-                    search: this.serializableDataByType.search || {}
+                    scrapeList: this.serializableDataByType['scrapeList'],
+                    scrapeSchema: this.serializableDataByType['scrapeSchema'],
+                    crawl: this.serializableDataByType['crawl'] || {},
+                    search: this.serializableDataByType['search'] || {}
                 });
             });
             const checkLimit = () => {
@@ -1839,7 +2053,7 @@ class Interpreter extends events_1.EventEmitter {
                                             }
                                         }
                                     }
-                                    yield page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => { });
+                                    yield page.waitForLoadState(this.getNavigationWaitStrategy(), { timeout: 15000 }).catch(() => { });
                                     if (!paginationSuccess) {
                                         const newUrl = page.url();
                                         const afterSignature = yield captureContentSignature();
@@ -2022,7 +2236,7 @@ class Interpreter extends events_1.EventEmitter {
         return workflow;
     }
     runLoop(p, workflow) {
-        var _a, _b, _c;
+        var _a, _b, _c, _d, _e;
         return __awaiter(this, void 0, void 0, function* () {
             if (this.isAborted) {
                 this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
@@ -2133,7 +2347,11 @@ class Interpreter extends events_1.EventEmitter {
                     repeatCount = action === lastAction ? repeatCount + 1 : 0;
                     console.log("REPEAT COUNT", repeatCount);
                     if (this.options.maxRepeats && repeatCount > this.options.maxRepeats) {
-                        return;
+                        const failedAction = ((_c = (_b = action === null || action === void 0 ? void 0 : action.what) === null || _b === void 0 ? void 0 : _b.find((w) => (w === null || w === void 0 ? void 0 : w.action) !== 'flag')) === null || _c === void 0 ? void 0 : _c.action) || 'unknown';
+                        const maxRepeats = this.options.maxRepeats;
+                        this.log(`Action ${String(failedAction)} exceeded max retries (${maxRepeats})`, logger_1.Level.ERROR);
+                        cleanup();
+                        throw new Error(`Action ${String(failedAction)} exceeded max retries (${maxRepeats})`);
                     }
                     lastAction = action;
                     if (this.isAborted) {
@@ -2142,13 +2360,13 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     try {
                         console.log("Carrying out:", action.what);
-                        yield this.carryOutSteps(p, action.what);
-                        usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
+                        yield this.carryOutSteps(p, action.what, workflowCopy);
+                        usedActions.push((_d = action.id) !== null && _d !== void 0 ? _d : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
                         this.executedActions++;
                         const percentage = Math.round((this.executedActions / this.totalActions) * 100);
-                        if ((_c = this.options.debugChannel) === null || _c === void 0 ? void 0 : _c.progressUpdate) {
+                        if ((_e = this.options.debugChannel) === null || _e === void 0 ? void 0 : _e.progressUpdate) {
                             this.options.debugChannel.progressUpdate(this.executedActions, this.totalActions, percentage);
                         }
                         // const newSelectors = this.getPreviousSelectors(workflow, actionId);
@@ -2190,13 +2408,13 @@ class Interpreter extends events_1.EventEmitter {
                     timeoutPromise
                 ]);
                 if (!isScriptLoaded) {
-                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                    yield page.addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
                 }
             }
             catch (error) {
                 this.log(`Script check failed, adding script anyway: ${error.message}`, logger_1.Level.WARN);
                 try {
-                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                    yield page.addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
                 }
                 catch (scriptError) {
                     this.log(`Failed to add script: ${scriptError.message}`, logger_1.Level.ERROR);

package/build/utils/concurrency.d.ts

@@ -15,10 +15,14 @@ export default class Concurrency {
     */
     private jobQueue;
     /**
-    * "Resolve" callbacks of the waitForCompletion() promises.
+    * Resolve/reject callbacks of the waitForCompletion() promises.
     */
     private waiting;
     /**
+    * First worker error captured during current execution wave.
+    */
+    private firstError;
+    /**
     * Constructs a new instance of concurrency manager.
     * @param {number} maxConcurrency Maximum number of workers running in parallel.
     */

package/build/utils/concurrency.js

@@ -22,9 +22,13 @@ class Concurrency {
         */
         this.jobQueue = [];
         /**
-        * "Resolve" callbacks of the waitForCompletion() promises.
+        * Resolve/reject callbacks of the waitForCompletion() promises.
         */
         this.waiting = [];
+        /**
+        * First worker error captured during current execution wave.
+        */
+        this.firstError = null;
         this.maxConcurrency = maxConcurrency;
     }
     /**
@@ -38,7 +42,11 @@ class Concurrency {
                 // console.debug("Job finished, running the next waiting job...");
                 this.runNextJob();
             }).catch((error) => {
-                console.error(`Job failed with error: ${error.message}`);
+                const normalizedError = error instanceof Error ? error : new Error(String(error));
+                console.error(`Job failed with error: ${normalizedError.message}`);
+                if (!this.firstError) {
+                    this.firstError = normalizedError;
+                }
                 // Continue processing other jobs even if one fails
                 this.runNextJob();
             });
@@ -48,7 +56,18 @@ class Concurrency {
             this.activeWorkers -= 1;
             if (this.activeWorkers === 0) {
                 // console.debug("This concurrency manager is idle!");
-                this.waiting.forEach((x) => x());
+                const pending = [...this.waiting];
+                this.waiting = [];
+                const pendingError = this.firstError;
+                this.firstError = null;
+                pending.forEach(({ resolve, reject }) => {
+                    if (pendingError) {
+                        reject(pendingError);
+                    }
+                    else {
+                        resolve();
+                    }
+                });
             }
         }
     }
@@ -77,8 +96,8 @@ class Concurrency {
     * @returns Promise, resolved after there is no running/waiting worker.
     */
     waitForCompletion() {
-        return new Promise((res) => {
-            this.waiting.push(res);
+        return new Promise((resolve, reject) => {
+            this.waiting.push({ resolve, reject });
         });
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.31",
+  "version": "0.0.33",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",