bdy 1.18.1-dev → 1.18.3-dev

package/distTs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "bdy",
   "preferGlobal": false,
-  "version": "1.18.1-dev",
+  "version": "1.18.3-dev",
   "type": "commonjs",
   "license": "MIT",
   "scripts": {

package/distTs/src/command/scrape/validation.js

@@ -0,0 +1,157 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateInputAndOptions = validateInputAndOptions;
+const zod_1 = require("zod");
+const output_1 = __importDefault(require("../../output"));
+const validation_1 = require("../vt/shared/validation");
+const urlSchema = zod_1.z.string().url().optional();
+const browserSchema = zod_1.z.enum(['chrome', 'firefox', 'safari']);
+const browsersListSchema = zod_1.z
+    .string()
+    .transform((value) => value
+    .split(',')
+    .map((browser) => browser.trim().toLowerCase())
+    .filter((browser) => browser.length > 0))
+    .refine((browsers) => browsers.length > 0, {
+    message: 'Invalid browsers list. Supported values: chrome,firefox,safari',
+})
+    .pipe(zod_1.z.array(browserSchema))
+    .transform((browsers) => Array.from(new Set(browsers.map((browser) => browser === 'chrome'
+    ? 'CHROMIUM'
+    : browser === 'firefox'
+        ? 'FIREFOX'
+        : 'WEBKIT'))));
+const optionsSchema = zod_1.z.object({
+    follow: zod_1.z.boolean(),
+    respectRobots: zod_1.z.boolean(),
+    outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']).optional(),
+    outputTypes: zod_1.z.string().optional(),
+    quality: zod_1.z.coerce.number().min(1).max(100).optional(),
+    outputDir: zod_1.z.string().default('.'),
+    fullPage: zod_1.z.boolean().optional(),
+    cssSelector: zod_1.z.string().optional(),
+    xpathSelector: zod_1.z.string().optional(),
+    colorScheme: zod_1.z.enum(['LIGHT', 'DARK', 'LIGHT_AND_DARK']).optional(),
+    browsers: browsersListSchema.optional(),
+    devices: zod_1.z.string().optional(),
+    delay: zod_1.z.coerce.number().min(0).max(10000),
+    waitFor: validation_1.waitForSchema,
+    cookie: validation_1.cookieSchema,
+    header: validation_1.headerSchema,
+    localStorage: zod_1.z
+        .array(zod_1.z.string().regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
+        message: "LocalStorage option must follow pattern '[scope::]key=value' (scope is optional)",
+    }))
+        .optional()
+        .transform((value) => value?.map((v) => {
+        const { scope, key, value } = (0, validation_1.parseScopedKeyValue)(v);
+        return { scope, key, value };
+    })),
+});
+function validateInputAndOptions(input, options) {
+    try {
+        const url = urlSchema.parse(input);
+        const { follow, respectRobots, outputType, outputTypes: rawOutputTypes, quality, outputDir, fullPage, cssSelector, xpathSelector, colorScheme, browsers: parsedBrowsers, devices: rawDevices, delay, waitFor, cookie, header, localStorage, } = optionsSchema.parse(options);
+        let parsedOutputTypes;
+        if (rawOutputTypes) {
+            try {
+                const outputTypeEntrySchema = zod_1.z.array(zod_1.z
+                    .object({
+                    type: zod_1.z.string().transform((v) => v.toUpperCase()),
+                    selector: zod_1.z
+                        .object({
+                        type: zod_1.z.enum(['CSS', 'XPATH']).optional(),
+                        value: zod_1.z.string().optional(),
+                    })
+                        .optional(),
+                    quality: zod_1.z.number().min(1).max(100).optional(),
+                })
+                    .transform((data) => ({
+                    ...data,
+                    type: data.type,
+                })));
+                parsedOutputTypes = outputTypeEntrySchema.parse(JSON.parse(rawOutputTypes));
+            }
+            catch {
+                output_1.default.exitError("Invalid --outputTypes value. Use JSON array, e.g. --outputTypes '[{\"type\":\"png\"},{\"type\":\"jpeg\",\"quality\":80}]'");
+            }
+        }
+        else if (outputType) {
+            if (typeof quality === 'number' && outputType !== 'jpeg') {
+                output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
+            }
+            if (cssSelector && xpathSelector) {
+                output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
+            }
+            const entry = {
+                type: outputType.toUpperCase(),
+            };
+            if (cssSelector) {
+                entry.selector = { type: 'CSS', value: cssSelector };
+            }
+            else if (xpathSelector) {
+                entry.selector = { type: 'XPATH', value: xpathSelector };
+            }
+            if (typeof quality === 'number') {
+                entry.quality = quality;
+            }
+            parsedOutputTypes = [entry];
+        }
+        let parsedDevices;
+        if (rawDevices) {
+            try {
+                const viewportSchema = zod_1.z
+                    .object({
+                    width: zod_1.z.number().positive(),
+                    height: zod_1.z.number().positive(),
+                })
+                    .strict();
+                const deviceSchema = zod_1.z.array(zod_1.z
+                    .object({
+                    viewport: viewportSchema,
+                    screen: viewportSchema.optional(),
+                    devicePixelRatio: zod_1.z.number().positive().optional(),
+                    isMobile: zod_1.z.boolean().optional(),
+                })
+                    .strict()
+                    .transform((data) => ({
+                    viewport: data.viewport,
+                    screen: data.screen ?? data.viewport,
+                    devicePixelRatio: data.devicePixelRatio ?? 1,
+                    isMobile: data.isMobile ?? false,
+                })));
+                parsedDevices = deviceSchema.parse(JSON.parse(rawDevices));
+            }
+            catch {
+                output_1.default.exitError('Invalid --devices value. Use JSON array, e.g. --devices \'[{"viewport":{"width":1920,"height":1080}}]\'');
+            }
+        }
+        return {
+            url,
+            follow,
+            respectRobots,
+            outputTypes: parsedOutputTypes,
+            outputDir,
+            fullPage,
+            colorScheme,
+            browsers: parsedBrowsers,
+            devices: parsedDevices,
+            cookies: cookie,
+            requestHeaders: header,
+            delay,
+            waitForSelectors: waitFor,
+            localStorage,
+        };
+    }
+    catch (error) {
+        if (error instanceof zod_1.ZodError) {
+            output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
+        }
+        else {
+            throw error;
+        }
+    }
+}

package/distTs/src/command/scrape.js

@@ -0,0 +1,143 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const commander_1 = require("commander");
+const texts_1 = require("../texts");
+const validation_1 = require("../visualTest/validation");
+const output_1 = __importDefault(require("../output"));
+const requests_1 = require("../visualTest/requests");
+const node_zlib_1 = require("node:zlib");
+const tar_stream_1 = __importDefault(require("tar-stream"));
+const promises_1 = require("node:stream/promises");
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const promises_2 = require("node:fs/promises");
+const context_1 = require("../visualTest/context");
+const ci_info_1 = require("@buddy-works/ci-info");
+const validation_2 = require("./scrape/validation");
+const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
+commandScrape.argument('[url]', texts_1.OPTION_SCRAPE_URL);
+commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
+commandScrape.option('--respectRobots', texts_1.OPTION_COMPARE_RESPECT_ROBOTS, false);
+commandScrape.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAPE_OUTPUT_TYPE).choices([
+    'jpeg',
+    'png',
+    'md',
+    'html',
+]));
+commandScrape.option('--outputTypes <json>', texts_1.OPTION_SCRAPE_OUTPUT_TYPES);
+commandScrape.option('--quality <quality>', texts_1.OPTION_SCRAPE_QUALITY);
+commandScrape.option('--fullPage', texts_1.OPTION_SCRAPE_FULL_PAGE, false);
+commandScrape.option('--cssSelector <selector>', texts_1.OPTION_SCRAPE_CSS_SELECTOR);
+commandScrape.option('--xpathSelector <selector>', texts_1.OPTION_SCRAPE_XPATH_SELECTOR);
+commandScrape.addOption(new commander_1.Option('--colorScheme <scheme>', texts_1.OPTION_SCRAPE_COLOR_SCHEME).choices([
+    'LIGHT',
+    'DARK',
+    'LIGHT_AND_DARK',
+]));
+commandScrape.option('--browsers <browsers>', texts_1.OPTION_SCRAPE_BROWSERS);
+commandScrape.option('--devices <devices>', texts_1.OPTION_SCRAPE_DEVICES);
+commandScrape.option('--waitFor <waitFors...>', texts_1.OPTION_COMPARE_WAIT_FOR);
+commandScrape.option('--cookie <cookies...>', texts_1.OPTION_COMPARE_COOKIE);
+commandScrape.option('--header <headers...>', texts_1.OPTION_COMPARE_HEADER);
+commandScrape.option('--localStorage <items...>', texts_1.OPTION_SCRAPE_LOCAL_STORAGE);
+commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
+commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
+commandScrape.action(async (inputUrl, options) => {
+    if (!(0, validation_1.checkToken)('scrape')) {
+        output_1.default.exitError(texts_1.ERR_MISSING_SCRAPE_TOKEN);
+    }
+    const { url, follow, respectRobots, outputTypes, outputDir, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage, } = (0, validation_2.validateInputAndOptions)(inputUrl, options);
+    try {
+        const ciAndGitInfo = await (0, ci_info_1.getCiAndGitInfo)({});
+        (0, context_1.setCiAndCommitInfo)(ciAndGitInfo);
+        const { buildId } = await (0, requests_1.sendScrap)(url, follow, respectRobots, outputTypes, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage);
+        const status = await watchSessionStatus(buildId);
+        if (!status.ok) {
+            output_1.default.exitError(`Scrape session failed: ${status.error}`);
+        }
+        output_1.default.normal('Downloading scrape package');
+        const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
+        const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
+        const unpack = tar_stream_1.default.extract();
+        unpack.on('entry', async (header, stream, next) => {
+            const currentDir = process.cwd();
+            const preparedOutputDir = outputDir.startsWith('.')
+                ? node_path_1.default.join(currentDir, outputDir)
+                : outputDir;
+            const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
+            try {
+                if (header.type === 'file') {
+                    await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
+                    const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
+                    await (0, promises_1.pipeline)(stream, fileWriteStream);
+                    next();
+                }
+                else {
+                    stream.resume();
+                    next();
+                }
+            }
+            catch (entryError) {
+                output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
+                next(entryError);
+            }
+        });
+        await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
+        output_1.default.exitSuccess('Downloading scrape package finished');
+    }
+    catch (error) {
+        output_1.default.exitError(`${error}`);
+    }
+});
+async function watchSessionStatus(buildId) {
+    return new Promise((resolve) => {
+        const eventSource = (0, requests_1.connectToScrapSession)(buildId);
+        eventSource.addEventListener('SESSION_STATUS', (event) => {
+            const data = JSON.parse(event.data);
+            if (data.status === 'STARTED') {
+                output_1.default.normal('Scrape session started');
+            }
+            else if (data.status === 'GATHER_URLS_COMPLETED') {
+                output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
+            }
+            else if (data.status === 'GATHER_URLS_FAILED') {
+                output_1.default.error('Gathering URLs failed');
+            }
+            else if (data.status === 'SCRAPE_URL_COMPLETED') {
+                output_1.default.normal(`Scraping ${data.text} completed`);
+            }
+            else if (data.status === 'SCRAPE_URL_FAILED') {
+                output_1.default.error(`Scraping ${data.text} failed`);
+            }
+            else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
+                output_1.default.normal('Package created');
+            }
+            else if (data.status === 'CREATE_PACKAGE_FAILED') {
+                output_1.default.error('Package creation failed');
+            }
+            else if (data.status === 'ERROR') {
+                eventSource.close();
+                resolve({ ok: false, error: data.text });
+            }
+            else if (data.status === 'FINISHED') {
+                eventSource.close();
+                output_1.default.normal('Scrape session finished');
+                resolve({ ok: true });
+            }
+        });
+        eventSource.addEventListener('error', (event) => {
+            if (event.code) {
+                eventSource.close();
+                if (event.code === 410) {
+                    output_1.default.normal('Scrape session finished');
+                }
+                resolve({ ok: event.code === 410, error: event.code });
+            }
+        });
+    });
+}
+exports.default = commandScrape;

package/distTs/src/command/vt/compare.js

@@ -59,6 +59,7 @@ commandVtCompare.action(async (options) => {
         output_1.default.normal(`List of urls:\n${filteredUrls.join('\n')}`);
     }
     const ciAndGitInfo = await (0, ci_info_1.getCiAndGitInfo)({});
+    output_1.default.normal((0, ci_info_1.formattedCiInfo)(ciAndGitInfo));
     (0, context_1.setCiAndCommitInfo)(ciAndGitInfo);
     try {
         const { message } = await (0, requests_1.sendCompareLinks)(filteredUrls, validatedOptions, sitemapSource);

package/distTs/src/command/vt/shared/validation.js

@@ -0,0 +1,118 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.waitForSchema = exports.headerSchema = exports.cookieSchema = exports.DEFAULT_SCOPE = void 0;
+exports.parseScopedSelector = parseScopedSelector;
+exports.parseScopedKeyValue = parseScopedKeyValue;
+const zod_1 = require("zod");
+const output_1 = __importDefault(require("../../../output"));
+exports.DEFAULT_SCOPE = '**';
+function parseScopedSelector(value) {
+    return value?.map((v) => {
+        let scope, type, selectorValue;
+        if (v.includes('::CSS=') || v.includes('::XPATH=')) {
+            const [scopePart, ...rest] = v.split('::');
+            const [typePart, ...valuePart] = rest.join('::').split('=');
+            type = typePart;
+            selectorValue = valuePart.join('=');
+            scope = scopePart;
+        }
+        else {
+            const [typePart, ...valuePart] = v.split('=');
+            type = typePart;
+            selectorValue = valuePart.join('=');
+            scope = exports.DEFAULT_SCOPE;
+        }
+        return { scope, type, value: selectorValue };
+    });
+}
+function parseScopedKeyValue(rawValue) {
+    let scope = exports.DEFAULT_SCOPE;
+    let keyValue = rawValue;
+    const scopeSeparatorIndex = rawValue.indexOf('::');
+    if (scopeSeparatorIndex >= 0) {
+        scope = rawValue.slice(0, scopeSeparatorIndex).trim();
+        keyValue = rawValue.slice(scopeSeparatorIndex + 2).trim();
+    }
+    const equalSignIndex = keyValue.indexOf('=');
+    if (equalSignIndex <= 0) {
+        output_1.default.exitError("Option value must follow pattern '[scope::]key=value' (scope is optional)");
+    }
+    const key = keyValue.slice(0, equalSignIndex).trim();
+    const value = keyValue.slice(equalSignIndex + 1).trim();
+    if (!key) {
+        output_1.default.exitError('Option key cannot be empty');
+    }
+    return { scope, key, value };
+}
+exports.cookieSchema = zod_1.z
+    .array(zod_1.z
+    .string()
+    .max(4096, {
+    message: 'Cookie must be less than 4096 characters',
+})
+    .regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
+    message: "Cookie option must follow pattern '[scope::]key=value[;attribute]' (scope is optional)",
+}))
+    .optional()
+    .transform((value) => value?.map((v) => {
+    let scope = exports.DEFAULT_SCOPE;
+    let cookieValue = v;
+    if (v.includes('::')) {
+        const [scopePart, valuePart] = v.split('::');
+        scope = scopePart.trim();
+        cookieValue = valuePart.trim();
+    }
+    const cookieParts = cookieValue.split(';').map((part) => part.trim());
+    const mainPart = cookieParts[0];
+    const firstEqualSignIndex = mainPart.indexOf('=');
+    const key = mainPart.slice(0, firstEqualSignIndex).trim();
+    const value = mainPart.slice(firstEqualSignIndex + 1).trim();
+    const cookie = {
+        scope,
+        key,
+        value,
+        httpOnly: false,
+        secure: false,
+    };
+    for (let i = 1; i < cookieParts.length; i++) {
+        const part = cookieParts[i].toLowerCase();
+        if (part === 'httponly') {
+            cookie.httpOnly = true;
+        }
+        else if (part === 'secure') {
+            cookie.secure = true;
+        }
+        else if (part.startsWith('domain=')) {
+            cookie.domain = part.substring(7);
+        }
+        else if (part.startsWith('path=')) {
+            cookie.path = part.substring(5);
+        }
+        else if (part.startsWith('samesite=')) {
+            const sameSiteValue = part.substring(9);
+            if (['strict', 'lax', 'none'].includes(sameSiteValue)) {
+                cookie.sameSite = (sameSiteValue.charAt(0).toUpperCase() +
+                    sameSiteValue.slice(1));
+            }
+        }
+    }
+    return cookie;
+}));
+exports.headerSchema = zod_1.z
+    .array(zod_1.z.string().regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
+    message: "Header option must follow pattern '[scope::]key=value' (scope is optional)",
+}))
+    .optional()
+    .transform((value) => value?.map((v) => {
+    const { scope, key, value } = parseScopedKeyValue(v);
+    return { scope, key, value };
+}));
+exports.waitForSchema = zod_1.z
+    .array(zod_1.z.string().regex(/^(?:([^:]+)::)?(?:(CSS|XPATH))=(.+)$/, {
+    message: "WaitFor option must follow pattern '[scope::]type=value' where type must be CSS or XPATH (scope is optional)",
+}))
+    .optional()
+    .transform(parseScopedSelector);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "bdy",
   "preferGlobal": false,
-  "version": "1.18.1-dev",
+  "version": "1.18.3-dev",
   "type": "commonjs",
   "license": "MIT",
   "scripts": {