maxun-core 0.0.1

package/build/preprocessor.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const joi_1 = __importDefault(require("joi"));
+const logic_1 = require("./types/logic");
+/**
+* Class for static processing the workflow files/objects.
+*/
+class Preprocessor {
+    static validateWorkflow(workflow) {
+        const regex = joi_1.default.object({
+            $regex: joi_1.default.string().required(),
+        });
+        const whereSchema = joi_1.default.object({
+            url: [joi_1.default.string().uri(), regex],
+            selectors: joi_1.default.array().items(joi_1.default.string()),
+            cookies: joi_1.default.object({}).pattern(joi_1.default.string(), joi_1.default.string()),
+            $after: [joi_1.default.string(), regex],
+            $before: [joi_1.default.string(), regex],
+            $and: joi_1.default.array().items(joi_1.default.link('#whereSchema')),
+            $or: joi_1.default.array().items(joi_1.default.link('#whereSchema')),
+            $not: joi_1.default.link('#whereSchema'),
+        }).id('whereSchema');
+        const schema = joi_1.default.object({
+            meta: joi_1.default.object({
+                name: joi_1.default.string(),
+                desc: joi_1.default.string(),
+            }),
+            workflow: joi_1.default.array().items(joi_1.default.object({
+                id: joi_1.default.string(),
+                where: whereSchema.required(),
+                what: joi_1.default.array().items({
+                    action: joi_1.default.string().required(),
+                    args: joi_1.default.array().items(joi_1.default.any()),
+                }).required(),
+            })).required(),
+        });
+        const { error } = schema.validate(workflow);
+        return error;
+    }
+    /**
+    * Extracts parameter names from the workflow.
+    * @param {WorkflowFile} workflow The given workflow
+    * @returns {String[]} List of parameters' names.
+    */
+    static getParams(workflow) {
+        const getParamsRecurse = (object) => {
+            if (typeof object === 'object') {
+                // Recursion base case
+                if (object.$param) {
+                    return [object.$param];
+                }
+                // Recursion general case
+                return Object.values(object)
+                    .reduce((p, v) => [...p, ...getParamsRecurse(v)], []);
+            }
+            return [];
+        };
+        return getParamsRecurse(workflow.workflow);
+    }
+    /**
+    * List all the selectors used in the given workflow (only literal "selector"
+    * field in WHERE clauses so far)
+    */
+    // TODO : add recursive selector search (also in click/fill etc. events?)
+    static extractSelectors(workflow) {
+        /**
+    * Given a Where condition, this function extracts
+    * all the existing selectors from it (recursively).
+    */
+        const selectorsFromCondition = (where) => {
+            var _a;
+            // the `selectors` field is either on the top level
+            let out = (_a = where.selectors) !== null && _a !== void 0 ? _a : [];
+            if (!Array.isArray(out)) {
+                out = [out];
+            }
+            // or nested in the "operator" array
+            logic_1.operators.forEach((op) => {
+                let condWhere = where[op];
+                if (condWhere) {
+                    condWhere = Array.isArray(condWhere) ? condWhere : [condWhere];
+                    (condWhere).forEach((subWhere) => {
+                        out = [...out, ...selectorsFromCondition(subWhere)];
+                    });
+                }
+            });
+            return out;
+        };
+        // Iterate through all the steps and extract the selectors from all of them.
+        return workflow.reduce((p, step) => [
+            ...p,
+            ...selectorsFromCondition(step.where).filter((x) => !p.includes(x)),
+        ], []);
+    }
+    /**
+    * Recursively crawl `object` and initializes params - replaces the `{$param : paramName}` objects
+    * with the defined value.
+    * @returns {Workflow} Copy of the given workflow, modified (the initial workflow is left untouched).
+    */
+    static initWorkflow(workflow, params) {
+        const paramNames = this.getParams({ workflow });
+        if (Object.keys(params !== null && params !== void 0 ? params : {}).sort().join(',') !== paramNames.sort().join(',')) {
+            throw new Error(`Provided parameters do not match the workflow parameters
+      provided: ${Object.keys(params !== null && params !== void 0 ? params : {}).sort().join(',')},
+      expected: ${paramNames.sort().join(',')}
+      `);
+        }
+        /**
+         * A recursive method for initializing special `{key: value}` syntax objects in the workflow.
+         * @param object Workflow to initialize (or a part of it).
+         * @param k key to look for ($regex, $param)
+         * @param f function mutating the special `{}` syntax into
+         *            its true representation (RegExp...)
+         * @returns Updated object
+         */
+        const initSpecialRecurse = (object, k, f) => {
+            if (!object || typeof object !== 'object') {
+                return object;
+            }
+            const out = object;
+            // for every key (child) of the object
+            Object.keys(object).forEach((key) => {
+                // if the field has only one key, which is `k`
+                if (Object.keys(object[key]).length === 1 && object[key][k]) {
+                    // process the current special tag (init param, hydrate regex...)
+                    out[key] = f(object[key][k]);
+                }
+                else {
+                    initSpecialRecurse(object[key], k, f);
+                }
+            });
+            return out;
+        };
+        // TODO: do better deep copy, this is hideous.
+        let workflowCopy = JSON.parse(JSON.stringify(workflow));
+        if (params) {
+            workflowCopy = initSpecialRecurse(workflowCopy, '$param', (paramName) => {
+                if (params && params[paramName]) {
+                    return params[paramName];
+                }
+                throw new SyntaxError(`Unspecified parameter found ${paramName}.`);
+            });
+        }
+        workflowCopy = initSpecialRecurse(workflowCopy, '$regex', (regex) => new RegExp(regex));
+        return workflowCopy;
+    }
+}
+exports.default = Preprocessor;

package/build/types/logic.d.ts

@@ -0,0 +1,4 @@
+export declare const unaryOperators: readonly ["$not"];
+export declare const naryOperators: readonly ["$and", "$or"];
+export declare const operators: readonly ["$not", "$and", "$or"];
+export declare const meta: readonly ["$before", "$after"];

package/build/types/logic.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.meta = exports.operators = exports.naryOperators = exports.unaryOperators = void 0;
+exports.unaryOperators = ['$not'];
+exports.naryOperators = ['$and', '$or'];
+exports.operators = [...exports.unaryOperators, ...exports.naryOperators];
+exports.meta = ['$before', '$after'];

package/build/types/workflow.d.ts

@@ -0,0 +1,47 @@
+import { Page } from 'playwright';
+import { naryOperators, unaryOperators, operators, meta } from './logic';
+export type Operator = typeof operators[number];
+export type UnaryOperator = typeof unaryOperators[number];
+export type NAryOperator = typeof naryOperators[number];
+export type Meta = typeof meta[number];
+export type SelectorArray = string[];
+type RegexableString = string | {
+    '$regex': string;
+};
+type BaseConditions = {
+    'url': RegexableString;
+    'cookies': Record<string, RegexableString>;
+    'selectors': SelectorArray;
+} & Record<Meta, RegexableString>;
+export type Where = Partial<{
+    [key in NAryOperator]: Where[];
+}> & // either a logic operator (arity N)
+Partial<{
+    [key in UnaryOperator]: Where;
+}> & // or an unary operator
+Partial<BaseConditions>;
+type MethodNames<T> = {
+    [K in keyof T]: T[K] extends Function ? K : never;
+}[keyof T];
+export type CustomFunctions = 'scrape' | 'scrapeSchema' | 'scroll' | 'screenshot' | 'script' | 'enqueueLinks' | 'flag' | 'scrapeList' | 'scrapeListAuto';
+export type What = {
+    action: MethodNames<Page> | CustomFunctions;
+    args?: any[];
+};
+export type PageState = Partial<BaseConditions>;
+export type ParamType = Record<string, any>;
+export type MetaData = {
+    name?: string;
+    desc?: string;
+};
+export interface WhereWhatPair {
+    id?: string;
+    where: Where;
+    what: What[];
+}
+export type Workflow = WhereWhatPair[];
+export type WorkflowFile = {
+    meta?: MetaData;
+    workflow: Workflow;
+};
+export {};

package/build/types/workflow.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/utils/concurrency.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Concurrency class for running concurrent tasks while managing a limited amount of resources.
+ */
+export default class Concurrency {
+    /**
+       * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+       */
+    maxConcurrency: number;
+    /**
+       * Number of currently active workers.
+       */
+    activeWorkers: number;
+    /**
+       * Queue of jobs waiting to be completed.
+       */
+    private jobQueue;
+    /**
+       * "Resolve" callbacks of the waitForCompletion() promises.
+       */
+    private waiting;
+    /**
+       * Constructs a new instance of concurrency manager.
+       * @param {number} maxConcurrency Maximum number of workers running in parallel.
+       */
+    constructor(maxConcurrency: number);
+    /**
+       * Takes a waiting job out of the queue and runs it.
+       */
+    private runNextJob;
+    /**
+       * Pass a job (a time-demanding async function) to the concurrency manager. \
+       * The time of the job's execution depends on the concurrency manager itself
+       * (given a generous enough `maxConcurrency` value, it might be immediate,
+       * but this is not guaranteed).
+       * @param worker Async function to be executed (job to be processed).
+       */
+    addJob(job: () => Promise<any>): void;
+    /**
+       * Waits until there is no running nor waiting job. \
+       * If the concurrency manager is idle at the time of calling this function,
+       * it waits until at least one job is compeleted (can be "presubscribed").
+       * @returns Promise, resolved after there is no running/waiting worker.
+       */
+    waitForCompletion(): Promise<void>;
+}

package/build/utils/concurrency.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Concurrency class for running concurrent tasks while managing a limited amount of resources.
+ */
+class Concurrency {
+    /**
+       * Constructs a new instance of concurrency manager.
+       * @param {number} maxConcurrency Maximum number of workers running in parallel.
+       */
+    constructor(maxConcurrency) {
+        /**
+           * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+           */
+        this.maxConcurrency = 1;
+        /**
+           * Number of currently active workers.
+           */
+        this.activeWorkers = 0;
+        /**
+           * Queue of jobs waiting to be completed.
+           */
+        this.jobQueue = [];
+        /**
+           * "Resolve" callbacks of the waitForCompletion() promises.
+           */
+        this.waiting = [];
+        this.maxConcurrency = maxConcurrency;
+    }
+    /**
+       * Takes a waiting job out of the queue and runs it.
+       */
+    runNextJob() {
+        const job = this.jobQueue.pop();
+        if (job) {
+            // console.debug("Running a job...");
+            job().then(() => {
+                // console.debug("Job finished, running the next waiting job...");
+                this.runNextJob();
+            });
+        }
+        else {
+            // console.debug("No waiting job found!");
+            this.activeWorkers -= 1;
+            if (this.activeWorkers === 0) {
+                // console.debug("This concurrency manager is idle!");
+                this.waiting.forEach((x) => x());
+            }
+        }
+    }
+    /**
+       * Pass a job (a time-demanding async function) to the concurrency manager. \
+       * The time of the job's execution depends on the concurrency manager itself
+       * (given a generous enough `maxConcurrency` value, it might be immediate,
+       * but this is not guaranteed).
+       * @param worker Async function to be executed (job to be processed).
+       */
+    addJob(job) {
+        // console.debug("Adding a worker!");
+        this.jobQueue.push(job);
+        if (!this.maxConcurrency || this.activeWorkers < this.maxConcurrency) {
+            this.runNextJob();
+            this.activeWorkers += 1;
+        }
+        else {
+            // console.debug("No capacity to run a worker now, waiting!");
+        }
+    }
+    /**
+       * Waits until there is no running nor waiting job. \
+       * If the concurrency manager is idle at the time of calling this function,
+       * it waits until at least one job is compeleted (can be "presubscribed").
+       * @returns Promise, resolved after there is no running/waiting worker.
+       */
+    waitForCompletion() {
+        return new Promise((res) => {
+            this.waiting.push(res);
+        });
+    }
+}
+exports.default = Concurrency;

package/build/utils/logger.d.ts

@@ -0,0 +1,9 @@
+export declare enum Level {
+    DATE = 36,
+    LOG = 0,
+    WARN = 93,
+    ERROR = 31,
+    DEBUG = 95,
+    RESET = 0
+}
+export default function logger(message: string | Error, level?: (Level.LOG | Level.WARN | Level.ERROR | Level.DEBUG)): void;

package/build/utils/logger.js

@@ -0,0 +1,31 @@
+"use strict";
+/*
+* Logger class for more detailed and comprehensible logs (with colors and timestamps)
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Level = void 0;
+var Level;
+(function (Level) {
+    Level[Level["DATE"] = 36] = "DATE";
+    Level[Level["LOG"] = 0] = "LOG";
+    Level[Level["WARN"] = 93] = "WARN";
+    Level[Level["ERROR"] = 31] = "ERROR";
+    Level[Level["DEBUG"] = 95] = "DEBUG";
+    Level[Level["RESET"] = 0] = "RESET";
+})(Level = exports.Level || (exports.Level = {}));
+function logger(message, level = Level.LOG) {
+    let m = message;
+    if (message.constructor.name.includes('Error') && typeof message !== 'string') {
+        m = (message).message;
+    }
+    process.stdout.write(`\x1b[${Level.DATE}m[${(new Date()).toLocaleString()}]\x1b[0m `);
+    process.stdout.write(`\x1b[${level}m`);
+    if (level === Level.ERROR || level === Level.WARN) {
+        process.stderr.write(m);
+    }
+    else {
+        process.stdout.write(m);
+    }
+    process.stdout.write(`\x1b[${Level.RESET}m\n`);
+}
+exports.default = logger;

package/build/utils/utils.d.ts

@@ -0,0 +1,8 @@
+/**
+ * ESLint rule in case there is only one util function
+ * (it still does not represent the "utils" file)
+*/
+/**
+ * Converts an array of scalars to an object with **items** of the array **for keys**.
+ */
+export declare function arrayToObject(array: any[]): any;

package/build/utils/utils.js

@@ -0,0 +1,15 @@
+"use strict";
+/**
+ * ESLint rule in case there is only one util function
+ * (it still does not represent the "utils" file)
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.arrayToObject = void 0;
+/* eslint-disable import/prefer-default-export */
+/**
+ * Converts an array of scalars to an object with **items** of the array **for keys**.
+ */
+function arrayToObject(array) {
+    return array.reduce((p, x) => (Object.assign(Object.assign({}, p), { [x]: [] })), {});
+}
+exports.arrayToObject = arrayToObject;

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "maxun-core",
+  "version": "0.0.1",
+  "description": "Core package for Maxun, responsible for data extraction",
+  "main": "build/index.js",
+  "typings": "build/index.d.ts",
+  "scripts": {
+    "test": "jest",
+    "build": "npm run clean && tsc",
+    "lint": "eslint .",
+    "clean": "rimraf ./build"
+  },
+  "files": [
+    "build/*"
+  ],
+  "keywords": [
+    "maxun",
+    "no-code scraping",
+    "web",
+    "automation",
+    "workflow",
+    "data extraction",
+    "scraping"
+  ],
+  "author": "Maxun",
+  "license": "AGPL-3.0-or-later",
+  "dependencies": {
+    "@cliqz/adblocker-playwright": "^1.31.3",
+    "cross-fetch": "^4.0.0",
+    "joi": "^17.6.0",
+    "playwright": "^1.20.1",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  }
+}