maxun-core 0.0.2 → 0.0.4

package/build/browserSide/scraper.js

@@ -192,7 +192,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 return undefined;
             switch (attribute) {
                 case 'href':
-                    return elem.getAttribute('href');
+                    const relativeHref = elem.getAttribute('href'); // Get the href attribute
+                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null; // Convert to full URL
                 case 'src':
                     return elem.getAttribute('src');
                 case 'innerText':

package/build/interpret.d.ts

@@ -2,7 +2,6 @@
 import { Page } from 'playwright';
 import { EventEmitter } from 'events';
 import { WorkflowFile, ParamType } from './types/workflow';
-import { ProxyConfig } from './proxy';
 /**
  * Defines optional intepreter options (passed in constructor)
  */
@@ -16,8 +15,6 @@ interface InterpreterOptions {
         activeId: Function;
         debugMessage: Function;
     }>;
-    proxy?: ProxyConfig | null;
-    onProxyError?: (error: Error, proxy: ProxyConfig) => Promise<ProxyConfig | null>;
 }
 /**
  * Class for running the Smart Workflows.
@@ -30,13 +27,7 @@ export default class Interpreter extends EventEmitter {
     private stopper;
     private log;
     private blocker;
-    private browser;
-    private contexts;
-    private currentProxy;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
-    updateProxy(proxyConfig: ProxyConfig | null): void;
-    private createProxyContext;
-    private createProxyPage;
     private applyAdBlocker;
     private disableAdBlocker;
     /**

package/build/interpret.js

@@ -53,16 +53,9 @@ class Interpreter extends events_1.EventEmitter {
         super();
         this.stopper = null;
         this.blocker = null;
-        this.browser = null;
-        this.contexts = [];
-        this.currentProxy = null;
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
-        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {}, proxy: null, onProxyError: (error, proxy) => __awaiter(this, void 0, void 0, function* () {
-                this.log(`Proxy error: ${error.message}`, logger_1.Level.ERROR);
-                return null;
-            }) }, options);
-        this.currentProxy = this.options.proxy;
+        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
         this.concurrency = new concurrency_1.default(this.options.maxConcurrency);
         this.log = (...args) => (0, logger_1.default)(...args);
         const error = preprocessor_1.default.validateWorkflow(workflow);
@@ -85,42 +78,6 @@ class Interpreter extends events_1.EventEmitter {
             this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
         });
     }
-    updateProxy(proxyConfig) {
-        this.currentProxy = proxyConfig;
-        this.log(`Proxy configuration updated`, logger_1.Level.LOG);
-    }
-    createProxyContext(browser) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.currentProxy) {
-                return browser.newContext();
-            }
-            try {
-                const context = yield browser.newContext({
-                    proxy: this.currentProxy
-                });
-                this.contexts.push(context);
-                return context;
-            }
-            catch (error) {
-                if (this.options.onProxyError) {
-                    const newProxy = yield this.options.onProxyError(error, this.currentProxy);
-                    if (newProxy) {
-                        this.currentProxy = newProxy;
-                        return this.createProxyContext(browser);
-                    }
-                }
-                throw error;
-            }
-        });
-    }
-    // create a new page with proxy
-    createProxyPage(context) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const page = yield context.newPage();
-            yield page.setViewportSize({ width: 900, height: 400 });
-            return page;
-        });
-    }
     applyAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
@@ -146,7 +103,6 @@ class Interpreter extends events_1.EventEmitter {
       */
     getState(page, workflow) {
         return __awaiter(this, void 0, void 0, function* () {
-            yield page.setViewportSize({ width: 900, height: 400 });
             /**
              * All the selectors present in the current Workflow
              */
@@ -289,13 +245,12 @@ class Interpreter extends events_1.EventEmitter {
                         .evaluateAll(
                     // @ts-ignore
                     (elements) => elements.map((a) => a.href).filter((x) => x));
-                    const context = yield this.createProxyContext(page.context().browser());
+                    const context = page.context();
                     for (const link of links) {
                         // eslint-disable-next-line
                         this.concurrency.addJob(() => __awaiter(this, void 0, void 0, function* () {
                             try {
-                                const newPage = yield this.createProxyPage(context);
-                                yield newPage.setViewportSize({ width: 900, height: 400 });
+                                const newPage = yield context.newPage();
                                 yield newPage.goto(link);
                                 yield newPage.waitForLoadState('networkidle');
                                 yield this.runLoop(newPage, this.initializedWorkflow);
@@ -567,6 +522,17 @@ class Interpreter extends events_1.EventEmitter {
      */
     run(page, params) {
         return __awaiter(this, void 0, void 0, function* () {
+            this.log('Starting the workflow.', logger_1.Level.LOG);
+            const context = page.context();
+            // Check proxy settings from context options
+            const contextOptions = context._options;
+            const hasProxy = !!(contextOptions === null || contextOptions === void 0 ? void 0 : contextOptions.proxy);
+            this.log(`Proxy settings: ${hasProxy ? `Proxy is configured...` : 'No proxy configured...'}`);
+            if (hasProxy) {
+                if (contextOptions.proxy.username) {
+                    this.log(`Proxy authenticated...`);
+                }
+            }
             if (this.stopper) {
                 throw new Error('This Interpreter is already running a workflow. To run another workflow, please, spawn another Interpreter.');
             }
@@ -574,20 +540,12 @@ class Interpreter extends events_1.EventEmitter {
              * `this.workflow` with the parameters initialized.
              */
             this.initializedWorkflow = preprocessor_1.default.initWorkflow(this.workflow, params);
-            // Create a new context with proxy configuration
-            const context = yield this.createProxyContext(page.context().browser());
-            // Create a new page with proxy
-            const proxyPage = yield this.createProxyPage(context);
-            // Copy over the current page's URL and state
-            yield proxyPage.goto(page.url());
-            yield this.ensureScriptsLoaded(proxyPage);
+            yield this.ensureScriptsLoaded(page);
             this.stopper = () => {
                 this.stopper = null;
             };
-            this.concurrency.addJob(() => this.runLoop(proxyPage, this.initializedWorkflow));
+            this.concurrency.addJob(() => this.runLoop(page, this.initializedWorkflow));
             yield this.concurrency.waitForCompletion();
-            yield Promise.all(this.contexts.map(ctx => ctx.close()));
-            this.contexts = [];
             this.stopper = null;
         });
     }

package/build/utils/concurrency.d.ts

@@ -38,7 +38,7 @@ export default class Concurrency {
     /**
        * Waits until there is no running nor waiting job. \
        * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is compeleted (can be "presubscribed").
+       * it waits until at least one job is completed (can be "presubscribed").
        * @returns Promise, resolved after there is no running/waiting worker.
        */
     waitForCompletion(): Promise<void>;

package/build/utils/concurrency.js

@@ -69,7 +69,7 @@ class Concurrency {
     /**
        * Waits until there is no running nor waiting job. \
        * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is compeleted (can be "presubscribed").
+       * it waits until at least one job is completed (can be "presubscribed").
        * @returns Promise, resolved after there is no running/waiting worker.
        */
     waitForCompletion() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
@@ -20,7 +20,12 @@
     "automation",
     "workflow",
     "data extraction",
-    "scraping"
+    "scraping",
+    "web scraper",
+    "web scraping",
+    "data scraping",
+    "no-code web scraper",
+    "no-code web scraping"
   ],
   "author": "Maxun",
   "license": "AGPL-3.0-or-later",

package/build/proxy.d.ts

@@ -1,5 +0,0 @@
-export interface ProxyConfig {
-    server: string;
-    username?: string;
-    password?: string;
-}

package/build/proxy.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });