bdy 1.14.6-dev → 1.14.7-beta

package/distTs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "bdy",
   "preferGlobal": false,
-  "version": "1.14.6-dev",
+  "version": "1.14.7-beta",
   "type": "commonjs",
   "license": "MIT",
   "scripts": {

package/distTs/src/agent/manager.js

@@ -206,6 +206,10 @@ class AgentManagerClass {
             return;
         }
         logger_1.default.setDebug(!!data.debug);
+        logger_1.default.info(`MANAGER REQUEST: change debug to ${!!data.debug}`);
+        this.serverOutput(res, {
+            success: true,
+        });
     }
     async processTunnelStop(req, res) {
         if (!this.agent) {

package/distTs/src/command/vt/scrape.js

@@ -0,0 +1,193 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../../utils");
+const commander_1 = require("commander");
+const texts_1 = require("../../texts");
+const validation_1 = require("../../visualTest/validation");
+const output_1 = __importDefault(require("../../output"));
+const requests_1 = require("../../visualTest/requests");
+const zod_1 = require("zod");
+const node_zlib_1 = require("node:zlib");
+const tar_stream_1 = __importDefault(require("tar-stream"));
+const promises_1 = require("node:stream/promises");
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const promises_2 = require("node:fs/promises");
+const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
+commandScrape.argument('<url>', texts_1.OPTION_SCRAPE_URL);
+commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
+commandScrape.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAPE_OUTPUT_TYPE)
+    .choices(['jpeg', 'png', 'md', 'html'])
+    .makeOptionMandatory());
+commandScrape.option('--quality <quality>', texts_1.OPTION_SCRAPE_QUALITY);
+commandScrape.option('--fullPage', texts_1.OPTION_SCRAPE_FULL_PAGE, false);
+commandScrape.option('--cssSelector <selector>', texts_1.OPTION_SCRAPE_CSS_SELECTOR);
+commandScrape.option('--xpathSelector <selector>', texts_1.OPTION_SCRAPE_XPATH_SELECTOR);
+commandScrape.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAPE_BROWSER)
+    .choices(['chrome', 'firefox', 'safari'])
+    .default('chrome'));
+commandScrape.option('--viewport <viewport>', texts_1.OPTION_SCRAPE_VIEWPORT, '1920x1080');
+commandScrape.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAPE_DEVICE_PIXEL_RATIO, '1');
+commandScrape.option('--waitForElement <selector>', texts_1.OPTION_SCRAPE_WAIT_FOR_ELEMENT);
+commandScrape.option('--darkMode', texts_1.OPTION_SCRAPE_DARK_MODE, false);
+commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
+commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
+commandScrape.action(async (inputUrl, options) => {
+    if (!(0, validation_1.checkToken)()) {
+        output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
+    }
+    const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
+    try {
+        const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
+        output_1.default.normal('Starting scrape session');
+        const status = await watchSessionStatus(buildId);
+        if (!status.ok) {
+            output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
+        }
+        output_1.default.normal('Downloading scrape package');
+        const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
+        const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
+        const unpack = tar_stream_1.default.extract();
+        unpack.on('entry', async (header, stream, next) => {
+            const currentDir = process.cwd();
+            const preparedOutputDir = outputDir.startsWith('.')
+                ? node_path_1.default.join(currentDir, outputDir)
+                : outputDir;
+            const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
+            try {
+                if (header.type === 'file') {
+                    await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
+                    const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
+                    await (0, promises_1.pipeline)(stream, fileWriteStream);
+                    next();
+                }
+                else {
+                    stream.resume();
+                    next();
+                }
+            }
+            catch (entryError) {
+                output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
+                next(entryError);
+            }
+        });
+        await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
+        output_1.default.exitSuccess('Downloading scrape package finished');
+    }
+    catch (error) {
+        output_1.default.exitError(`${error}`);
+    }
+});
+function validateInputAndOptions(input, options) {
+    const urlSchema = zod_1.z.string().url();
+    const optionsSchema = zod_1.z.object({
+        follow: zod_1.z.boolean(),
+        outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
+        quality: zod_1.z.coerce.number().min(1).max(100).optional(),
+        outputDir: zod_1.z.string().default('.'),
+        fullPage: zod_1.z.boolean().optional(),
+        cssSelector: zod_1.z.string().optional(),
+        xpathSelector: zod_1.z.string().optional(),
+        browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
+        viewport: zod_1.z
+            .string()
+            .refine((value) => {
+            const [width, height] = value.split('x');
+            return (width &&
+                height &&
+                !isNaN(Number(width)) &&
+                !isNaN(Number(height)) &&
+                Number(width) > 0 &&
+                Number(height) > 0);
+        }, 'Invalid viewport format, example: 1920x1080')
+            .transform((value) => {
+            const [width, height] = value.split('x');
+            return {
+                width: Number(width),
+                height: Number(height),
+            };
+        }),
+        devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
+        darkMode: zod_1.z.boolean(),
+        delay: zod_1.z.coerce.number().min(0).max(10000),
+        waitForElement: zod_1.z.string().optional(),
+    });
+    try {
+        const url = urlSchema.parse(input);
+        const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
+        if (typeof quality === 'number' && outputType !== 'jpeg') {
+            output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
+        }
+        if (cssSelector && xpathSelector) {
+            output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
+        }
+        return {
+            url,
+            follow,
+            outputType,
+            quality,
+            outputDir,
+            fullPage,
+            cssSelector,
+            xpathSelector,
+            browser,
+            viewport,
+            devicePixelRatio,
+            darkMode,
+            delay,
+            waitForElement,
+        };
+    }
+    catch (error) {
+        if (error instanceof zod_1.ZodError) {
+            output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
+        }
+        else {
+            throw error;
+        }
+    }
+}
+async function watchSessionStatus(buildId) {
+    return new Promise((resolve) => {
+        const eventSource = (0, requests_1.connectToScrapSession)(buildId);
+        eventSource.addEventListener('SESSION_STATUS', (event) => {
+            const data = JSON.parse(event.data);
+            if (data.status === 'GATHER_URLS_COMPLETED') {
+                output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
+            }
+            else if (data.status === 'GATHER_URLS_FAILED') {
+                output_1.default.error('Gathering URLs failed');
+            }
+            else if (data.status === 'SCRAPE_URL_COMPLETED') {
+                output_1.default.normal(`Scraping ${data.text} completed`);
+            }
+            else if (data.status === 'SCRAPE_URL_FAILED') {
+                output_1.default.error(`Scraping ${data.text} failed`);
+            }
+            else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
+                output_1.default.normal('Package created');
+            }
+            else if (data.status === 'CREATE_PACKAGE_FAILED') {
+                output_1.default.error('Package creation failed');
+            }
+            else if (data.status === 'FINISHED') {
+                eventSource.close();
+                output_1.default.normal('Scrape session finished');
+                resolve({ ok: true });
+            }
+        });
+        eventSource.addEventListener('error', (event) => {
+            if (event.code) {
+                eventSource.close();
+                if (event.code === 410) {
+                    output_1.default.normal('Scrape session finished');
+                }
+                resolve({ ok: event.code === 410, error: event.code });
+            }
+        });
+    });
+}
+exports.default = commandScrape;

package/distTs/src/logger.js

@@ -35,6 +35,7 @@ class Logger {
     }
     setDebug(debug) {
         this.debugOn = debug;
+        this.getPino().level = this.debugOn ? 'debug' : 'info';
     }
     getPino() {
         if (!this.p) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "bdy",
   "preferGlobal": false,
-  "version": "1.14.6-dev",
+  "version": "1.14.7-beta",
   "type": "commonjs",
   "license": "MIT",
   "scripts": {