maxun-core 0.0.1

package/build/interpret.js

@@ -0,0 +1,555 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const adblocker_playwright_1 = require("@cliqz/adblocker-playwright");
+const cross_fetch_1 = __importDefault(require("cross-fetch"));
+const path_1 = __importDefault(require("path"));
+const events_1 = require("events");
+const logic_1 = require("./types/logic");
+const utils_1 = require("./utils/utils");
+const concurrency_1 = __importDefault(require("./utils/concurrency"));
+const preprocessor_1 = __importDefault(require("./preprocessor"));
+const logger_1 = __importStar(require("./utils/logger"));
+/**
+ * Class for running the Smart Workflows.
+ */
+class Interpreter extends events_1.EventEmitter {
+    constructor(workflow, options) {
+        var _a;
+        super();
+        this.stopper = null;
+        this.blocker = null;
+        this.workflow = workflow.workflow;
+        this.initializedWorkflow = null;
+        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
+        this.concurrency = new concurrency_1.default(this.options.maxConcurrency);
+        this.log = (...args) => (0, logger_1.default)(...args);
+        const error = preprocessor_1.default.validateWorkflow(workflow);
+        if (error) {
+            throw (error);
+        }
+        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.debugMessage) {
+            const oldLog = this.log;
+            // @ts-ignore
+            this.log = (...args) => {
+                if (args[1] !== logger_1.Level.LOG) {
+                    this.options.debugChannel.debugMessage(typeof args[0] === 'string' ? args[0] : args[0].message);
+                }
+                oldLog(...args);
+            };
+        }
+        adblocker_playwright_1.PlaywrightBlocker.fromPrebuiltAdsAndTracking(cross_fetch_1.default).then(blocker => {
+            this.blocker = blocker;
+        }).catch(err => {
+            this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
+        });
+    }
+    applyAdBlocker(page) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.blocker) {
+                yield this.blocker.enableBlockingInPage(page);
+            }
+        });
+    }
+    disableAdBlocker(page) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.blocker) {
+                yield this.blocker.disableBlockingInPage(page);
+            }
+        });
+    }
+    /**
+      * Returns the context object from given Page and the current workflow.\
+      * \
+      * `workflow` is used for selector extraction - function searches for used selectors to
+      * look for later in the page's context.
+      * @param page Playwright Page object
+      * @param workflow Current **initialized** workflow (array of where-what pairs).
+      * @returns {PageState} State of the current page.
+      */
+    getState(page, workflow) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield page.setViewportSize({ width: 900, height: 400 });
+            /**
+             * All the selectors present in the current Workflow
+             */
+            const selectors = preprocessor_1.default.extractSelectors(workflow);
+            /**
+              * Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
+              * @param selector Selector to be queried
+              * @returns True if the targetted element is actionable, false otherwise.
+              */
+            const actionable = (selector) => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    const proms = [
+                        page.isEnabled(selector, { timeout: 500 }),
+                        page.isVisible(selector, { timeout: 500 }),
+                    ];
+                    return yield Promise.all(proms).then((bools) => bools.every((x) => x));
+                }
+                catch (e) {
+                    // log(<Error>e, Level.ERROR);
+                    return false;
+                }
+            });
+            /**
+              * Object of selectors present in the current page.
+              */
+            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
+                if (yield actionable(selector)) {
+                    return [selector];
+                }
+                return [];
+            }))).then((x) => x.flat());
+            return {
+                url: page.url(),
+                cookies: (yield page.context().cookies([page.url()]))
+                    .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
+                selectors: presentSelectors,
+            };
+        });
+    }
+    /**
+     * Tests if the given action is applicable with the given context.
+     * @param where Tested *where* condition
+     * @param context Current browser context.
+     * @returns True if `where` is applicable in the given context, false otherwise
+     */
+    applicable(where, context, usedActions = []) {
+        /**
+         * Given two arbitrary objects, determines whether `subset` is a subset of `superset`.\
+         * \
+         * For every key in `subset`, there must be a corresponding key with equal scalar
+         * value in `superset`, or `inclusive(subset[key], superset[key])` must hold.
+         * @param subset Arbitrary non-cyclic JS object (where clause)
+         * @param superset Arbitrary non-cyclic JS object (browser context)
+         * @returns `true` if `subset <= superset`, `false` otherwise.
+         */
+        const inclusive = (subset, superset) => (Object.entries(subset).every(([key, value]) => {
+            /**
+             * Arrays are compared without order (are transformed into objects before comparison).
+             */
+            const parsedValue = Array.isArray(value) ? (0, utils_1.arrayToObject)(value) : value;
+            const parsedSuperset = {};
+            parsedSuperset[key] = Array.isArray(superset[key])
+                ? (0, utils_1.arrayToObject)(superset[key])
+                : superset[key];
+            // Every `subset` key must exist in the `superset` and
+            // have the same value (strict equality), or subset[key] <= superset[key]
+            return parsedSuperset[key]
+                && ((parsedSuperset[key] === parsedValue)
+                    || ((parsedValue).constructor.name === 'RegExp' && parsedValue.test(parsedSuperset[key]))
+                    || ((parsedValue).constructor.name !== 'RegExp'
+                        && typeof parsedValue === 'object' && inclusive(parsedValue, parsedSuperset[key])));
+        }));
+        // Every value in the "where" object should be compliant to the current state.
+        return Object.entries(where).every(([key, value]) => {
+            if (logic_1.operators.includes(key)) {
+                const array = Array.isArray(value)
+                    ? value
+                    : Object.entries(value).map((a) => Object.fromEntries([a]));
+                // every condition is treated as a single context
+                switch (key) {
+                    case '$and':
+                        return array === null || array === void 0 ? void 0 : array.every((x) => this.applicable(x, context));
+                    case '$or':
+                        return array === null || array === void 0 ? void 0 : array.some((x) => this.applicable(x, context));
+                    case '$not':
+                        return !this.applicable(value, context); // $not should be a unary operator
+                    default:
+                        throw new Error('Undefined logic operator.');
+                }
+            }
+            else if (logic_1.meta.includes(key)) {
+                const testRegexString = (x) => {
+                    if (typeof value === 'string') {
+                        return x === value;
+                    }
+                    return value.test(x);
+                };
+                switch (key) {
+                    case '$before':
+                        return !usedActions.find(testRegexString);
+                    case '$after':
+                        return !!usedActions.find(testRegexString);
+                    default:
+                        throw new Error('Undefined meta operator.');
+                }
+            }
+            else {
+                // Current key is a base condition (url, cookies, selectors)
+                return inclusive({ [key]: value }, context);
+            }
+        });
+    }
+    /**
+   * Given a Playwright's page object and a "declarative" list of actions, this function
+   * calls all mentioned functions on the Page object.\
+   * \
+   * Manipulates the iterator indexes (experimental feature, likely to be removed in
+   * the following versions of maxun-core)
+   * @param page Playwright Page object
+   * @param steps Array of actions.
+   */
+    carryOutSteps(page, steps) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            /**
+             * Defines overloaded (or added) methods/actions usable in the workflow.
+             * If a method overloads any existing method of the Page class, it accepts the same set
+             * of parameters *(but can override some!)*\
+             * \
+             * Also, following piece of code defines functions to be run in the browser's context.
+             * Beware of false linter errors - here, we know better!
+             */
+            const wawActions = {
+                screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                    const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
+                    yield this.options.binaryCallback(screenshotBuffer, 'image/png');
+                }),
+                enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    const links = yield page.locator(selector)
+                        .evaluateAll(
+                    // @ts-ignore
+                    (elements) => elements.map((a) => a.href).filter((x) => x));
+                    const context = page.context();
+                    for (const link of links) {
+                        // eslint-disable-next-line
+                        this.concurrency.addJob(() => __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const newPage = yield context.newPage();
+                                yield newPage.setViewportSize({ width: 900, height: 400 });
+                                yield newPage.goto(link);
+                                yield newPage.waitForLoadState('networkidle');
+                                yield this.runLoop(newPage, this.initializedWorkflow);
+                            }
+                            catch (e) {
+                                // `runLoop` uses soft mode, so it recovers from it's own exceptions
+                                // but newPage(), goto() and waitForLoadState() don't (and will kill
+                                // the interpreter by throwing).
+                                this.log(e, logger_1.Level.ERROR);
+                            }
+                        }));
+                    }
+                    yield page.close();
+                }),
+                scrape: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.ensureScriptsLoaded(page);
+                    const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
+                    yield this.options.serializableCallback(scrapeResults);
+                }),
+                scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.ensureScriptsLoaded(page);
+                    const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
+                    yield this.options.serializableCallback(scrapeResult);
+                }),
+                scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.ensureScriptsLoaded(page);
+                    if (!config.pagination) {
+                        const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                        yield this.options.serializableCallback(scrapeResults);
+                    }
+                    else {
+                        const scrapeResults = yield this.handlePagination(page, config);
+                        yield this.options.serializableCallback(scrapeResults);
+                    }
+                }),
+                scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.ensureScriptsLoaded(page);
+                    const scrapeResults = yield page.evaluate((listSelector) => {
+                        return window.scrapeListAuto(listSelector);
+                    }, config.listSelector);
+                    yield this.options.serializableCallback(scrapeResults);
+                }),
+                scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
+                    yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
+                        for (let i = 1; i <= (pagesInternal !== null && pagesInternal !== void 0 ? pagesInternal : 1); i += 1) {
+                            // @ts-ignore
+                            window.scrollTo(0, window.scrollY + window.innerHeight);
+                        }
+                    }), pages !== null && pages !== void 0 ? pages : 1);
+                }),
+                script: (code) => __awaiter(this, void 0, void 0, function* () {
+                    const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
+                    const x = new AsyncFunction('page', 'log', code);
+                    yield x(page, this.log);
+                }),
+                flag: () => __awaiter(this, void 0, void 0, function* () {
+                    return new Promise((res) => {
+                        this.emit('flag', page, res);
+                    });
+                }),
+            };
+            for (const step of steps) {
+                this.log(`Launching ${step.action}`, logger_1.Level.LOG);
+                if (step.action in wawActions) {
+                    // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
+                    const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
+                    yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
+                }
+                else {
+                    // Implements the dot notation for the "method name" in the workflow
+                    const levels = step.action.split('.');
+                    const methodName = levels[levels.length - 1];
+                    let invokee = page;
+                    for (const level of levels.splice(0, levels.length - 1)) {
+                        invokee = invokee[level];
+                    }
+                    if (!step.args || Array.isArray(step.args)) {
+                        yield invokee[methodName](...((_a = step.args) !== null && _a !== void 0 ? _a : []));
+                    }
+                    else {
+                        yield invokee[methodName](step.args);
+                    }
+                }
+                yield new Promise((res) => { setTimeout(res, 500); });
+            }
+        });
+    }
+    handlePagination(page, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let allResults = [];
+            let previousHeight = 0;
+            // track unique items per page to avoid re-scraping
+            let scrapedItems = new Set();
+            while (true) {
+                switch (config.pagination.type) {
+                    case 'scrollDown':
+                        yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                        yield page.waitForTimeout(2000);
+                        const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                        if (currentHeight === previousHeight) {
+                            const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                            allResults = allResults.concat(finalResults);
+                            return allResults;
+                        }
+                        previousHeight = currentHeight;
+                        break;
+                    case 'scrollUp':
+                        yield page.evaluate(() => window.scrollTo(0, 0));
+                        yield page.waitForTimeout(2000);
+                        const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
+                        if (currentTopHeight === 0) {
+                            const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                            allResults = allResults.concat(finalResults);
+                            return allResults;
+                        }
+                        previousHeight = currentTopHeight;
+                        break;
+                    case 'clickNext':
+                        const pageResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                        // Filter out already scraped items
+                        const newResults = pageResults.filter(item => {
+                            const uniqueKey = JSON.stringify(item);
+                            if (scrapedItems.has(uniqueKey))
+                                return false; // Ignore if already scraped
+                            scrapedItems.add(uniqueKey); // Mark as scraped
+                            return true;
+                        });
+                        allResults = allResults.concat(newResults);
+                        if (config.limit && allResults.length >= config.limit) {
+                            return allResults.slice(0, config.limit);
+                        }
+                        const nextButton = yield page.$(config.pagination.selector);
+                        if (!nextButton) {
+                            return allResults; // No more pages to scrape
+                        }
+                        yield Promise.all([
+                            nextButton.click(),
+                            page.waitForNavigation({ waitUntil: 'networkidle' })
+                        ]);
+                        yield page.waitForTimeout(1000);
+                        break;
+                    case 'clickLoadMore':
+                        while (true) {
+                            const loadMoreButton = yield page.$(config.pagination.selector);
+                            if (!loadMoreButton) {
+                                // No more "Load More" button, so scrape the remaining items
+                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                                allResults = allResults.concat(finalResults);
+                                return allResults;
+                            }
+                            // Click the 'Load More' button to load additional items
+                            yield loadMoreButton.click();
+                            yield page.waitForTimeout(2000); // Wait for new items to load
+                            // After clicking 'Load More', scroll down to load more items
+                            yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            yield page.waitForTimeout(2000);
+                            // Check if more items are available
+                            const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            if (currentHeight === previousHeight) {
+                                // No more items loaded, return the scraped results
+                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                                allResults = allResults.concat(finalResults);
+                                return allResults;
+                            }
+                            previousHeight = currentHeight;
+                            if (config.limit && allResults.length >= config.limit) {
+                                // If limit is set and reached, return the limited results
+                                allResults = allResults.slice(0, config.limit);
+                                break;
+                            }
+                        }
+                        break;
+                    default:
+                        const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                        allResults = allResults.concat(results);
+                        return allResults;
+                }
+                if (config.limit && allResults.length >= config.limit) {
+                    allResults = allResults.slice(0, config.limit);
+                    break;
+                }
+            }
+            return allResults;
+        });
+    }
+    runLoop(p, workflow) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            // apply ad-blocker to the current page
+            yield this.applyAdBlocker(p);
+            const usedActions = [];
+            let lastAction = null;
+            let repeatCount = 0;
+            /**
+            *  Enables the interpreter functionality for popup windows.
+            * User-requested concurrency should be entirely managed by the concurrency manager,
+            * e.g. via `enqueueLinks`.
+            */
+            p.on('popup', (popup) => {
+                this.concurrency.addJob(() => this.runLoop(popup, workflow));
+            });
+            /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
+            while (true) {
+                // Checks whether the page was closed from outside,
+                //  or the workflow execution has been stopped via `interpreter.stop()`
+                if (p.isClosed() || !this.stopper) {
+                    return;
+                }
+                try {
+                    yield p.waitForLoadState();
+                }
+                catch (e) {
+                    yield p.close();
+                    return;
+                }
+                let pageState = {};
+                try {
+                    pageState = yield this.getState(p, workflow);
+                }
+                catch (e) {
+                    this.log('The browser has been closed.');
+                    return;
+                }
+                if (this.options.debug) {
+                    this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
+                }
+                const actionId = workflow.findIndex((step) => this.applicable(step.where, pageState, usedActions));
+                const action = workflow[actionId];
+                this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
+                if (action) { // action is matched
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
+                        this.options.debugChannel.activeId(actionId);
+                    }
+                    repeatCount = action === lastAction ? repeatCount + 1 : 0;
+                    if (this.options.maxRepeats && repeatCount >= this.options.maxRepeats) {
+                        return;
+                    }
+                    lastAction = action;
+                    try {
+                        yield this.carryOutSteps(p, action.what);
+                        usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
+                    }
+                    catch (e) {
+                        this.log(e, logger_1.Level.ERROR);
+                    }
+                }
+                else {
+                    //await this.disableAdBlocker(p);
+                    return;
+                }
+            }
+        });
+    }
+    ensureScriptsLoaded(page) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const isScriptLoaded = yield page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function' && typeof window.scrapeListAuto === 'function' && typeof window.scrollDown === 'function' && typeof window.scrollUp === 'function');
+            if (!isScriptLoaded) {
+                yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+            }
+        });
+    }
+    /**
+     * Spawns a browser context and runs given workflow.
+     * \
+     * Resolves after the playback is finished.
+     * @param {Page} [page] Page to run the workflow on.
+     * @param {ParamType} params Workflow specific, set of parameters
+     *  for the `{$param: nameofparam}` fields.
+     */
+    run(page, params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.stopper) {
+                throw new Error('This Interpreter is already running a workflow. To run another workflow, please, spawn another Interpreter.');
+            }
+            /**
+             * `this.workflow` with the parameters initialized.
+             */
+            this.initializedWorkflow = preprocessor_1.default.initWorkflow(this.workflow, params);
+            yield this.ensureScriptsLoaded(page);
+            this.stopper = () => {
+                this.stopper = null;
+            };
+            this.concurrency.addJob(() => this.runLoop(page, this.initializedWorkflow));
+            yield this.concurrency.waitForCompletion();
+            this.stopper = null;
+        });
+    }
+    stop() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.stopper) {
+                yield this.stopper();
+                this.stopper = null;
+            }
+            else {
+                throw new Error('Cannot stop, there is no running workflow!');
+            }
+        });
+    }
+}
+exports.default = Interpreter;

package/build/preprocessor.d.ts

@@ -0,0 +1,24 @@
+import { Workflow, WorkflowFile, ParamType, SelectorArray } from './types/workflow';
+/**
+* Class for static processing the workflow files/objects.
+*/
+export default class Preprocessor {
+    static validateWorkflow(workflow: WorkflowFile): any;
+    /**
+    * Extracts parameter names from the workflow.
+    * @param {WorkflowFile} workflow The given workflow
+    * @returns {String[]} List of parameters' names.
+    */
+    static getParams(workflow: WorkflowFile): string[];
+    /**
+    * List all the selectors used in the given workflow (only literal "selector"
+    * field in WHERE clauses so far)
+    */
+    static extractSelectors(workflow: Workflow): SelectorArray;
+    /**
+    * Recursively crawl `object` and initializes params - replaces the `{$param : paramName}` objects
+    * with the defined value.
+    * @returns {Workflow} Copy of the given workflow, modified (the initial workflow is left untouched).
+    */
+    static initWorkflow(workflow: Workflow, params?: ParamType): Workflow;
+}