maxun-core 0.0.1 → 0.0.2

package/build/interpret.d.ts

@@ -2,6 +2,7 @@
 import { Page } from 'playwright';
 import { EventEmitter } from 'events';
 import { WorkflowFile, ParamType } from './types/workflow';
+import { ProxyConfig } from './proxy';
 /**
  * Defines optional intepreter options (passed in constructor)
  */
@@ -15,6 +16,8 @@ interface InterpreterOptions {
         activeId: Function;
         debugMessage: Function;
     }>;
+    proxy?: ProxyConfig | null;
+    onProxyError?: (error: Error, proxy: ProxyConfig) => Promise<ProxyConfig | null>;
 }
 /**
  * Class for running the Smart Workflows.
@@ -27,7 +30,13 @@ export default class Interpreter extends EventEmitter {
     private stopper;
     private log;
     private blocker;
+    private browser;
+    private contexts;
+    private currentProxy;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
+    updateProxy(proxyConfig: ProxyConfig | null): void;
+    private createProxyContext;
+    private createProxyPage;
     private applyAdBlocker;
     private disableAdBlocker;
     /**

package/build/interpret.js

@@ -53,9 +53,16 @@ class Interpreter extends events_1.EventEmitter {
         super();
         this.stopper = null;
         this.blocker = null;
+        this.browser = null;
+        this.contexts = [];
+        this.currentProxy = null;
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
-        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
+        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {}, proxy: null, onProxyError: (error, proxy) => __awaiter(this, void 0, void 0, function* () {
+                this.log(`Proxy error: ${error.message}`, logger_1.Level.ERROR);
+                return null;
+            }) }, options);
+        this.currentProxy = this.options.proxy;
         this.concurrency = new concurrency_1.default(this.options.maxConcurrency);
         this.log = (...args) => (0, logger_1.default)(...args);
         const error = preprocessor_1.default.validateWorkflow(workflow);
@@ -78,6 +85,42 @@ class Interpreter extends events_1.EventEmitter {
             this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
         });
     }
+    updateProxy(proxyConfig) {
+        this.currentProxy = proxyConfig;
+        this.log(`Proxy configuration updated`, logger_1.Level.LOG);
+    }
+    createProxyContext(browser) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.currentProxy) {
+                return browser.newContext();
+            }
+            try {
+                const context = yield browser.newContext({
+                    proxy: this.currentProxy
+                });
+                this.contexts.push(context);
+                return context;
+            }
+            catch (error) {
+                if (this.options.onProxyError) {
+                    const newProxy = yield this.options.onProxyError(error, this.currentProxy);
+                    if (newProxy) {
+                        this.currentProxy = newProxy;
+                        return this.createProxyContext(browser);
+                    }
+                }
+                throw error;
+            }
+        });
+    }
+    // create a new page with proxy
+    createProxyPage(context) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const page = yield context.newPage();
+            yield page.setViewportSize({ width: 900, height: 400 });
+            return page;
+        });
+    }
     applyAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
@@ -246,12 +289,12 @@ class Interpreter extends events_1.EventEmitter {
                         .evaluateAll(
                     // @ts-ignore
                     (elements) => elements.map((a) => a.href).filter((x) => x));
-                    const context = page.context();
+                    const context = yield this.createProxyContext(page.context().browser());
                     for (const link of links) {
                         // eslint-disable-next-line
                         this.concurrency.addJob(() => __awaiter(this, void 0, void 0, function* () {
                             try {
-                                const newPage = yield context.newPage();
+                                const newPage = yield this.createProxyPage(context);
                                 yield newPage.setViewportSize({ width: 900, height: 400 });
                                 yield newPage.goto(link);
                                 yield newPage.waitForLoadState('networkidle');
@@ -531,12 +574,20 @@ class Interpreter extends events_1.EventEmitter {
              * `this.workflow` with the parameters initialized.
              */
             this.initializedWorkflow = preprocessor_1.default.initWorkflow(this.workflow, params);
-            yield this.ensureScriptsLoaded(page);
+            // Create a new context with proxy configuration
+            const context = yield this.createProxyContext(page.context().browser());
+            // Create a new page with proxy
+            const proxyPage = yield this.createProxyPage(context);
+            // Copy over the current page's URL and state
+            yield proxyPage.goto(page.url());
+            yield this.ensureScriptsLoaded(proxyPage);
             this.stopper = () => {
                 this.stopper = null;
             };
-            this.concurrency.addJob(() => this.runLoop(page, this.initializedWorkflow));
+            this.concurrency.addJob(() => this.runLoop(proxyPage, this.initializedWorkflow));
             yield this.concurrency.waitForCompletion();
+            yield Promise.all(this.contexts.map(ctx => ctx.close()));
+            this.contexts = [];
             this.stopper = null;
         });
     }

package/build/proxy.d.ts

@@ -0,0 +1,5 @@
+export interface ProxyConfig {
+    server: string;
+    username?: string;
+    password?: string;
+}

package/build/proxy.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",