bdy 1.19.2-dev-pipeline → 1.19.2-dev

package/distTs/src/command/vt/scrape.js

@@ -0,0 +1,193 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../../utils");
+const commander_1 = require("commander");
+const texts_1 = require("../../texts");
+const validation_1 = require("../../visualTest/validation");
+const output_1 = __importDefault(require("../../output"));
+const requests_1 = require("../../visualTest/requests");
+const zod_1 = require("zod");
+const node_zlib_1 = require("node:zlib");
+const tar_stream_1 = __importDefault(require("tar-stream"));
+const promises_1 = require("node:stream/promises");
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const promises_2 = require("node:fs/promises");
+const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
+commandScrape.argument('<url>', texts_1.OPTION_SCRAPE_URL);
+commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
+commandScrape.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAPE_OUTPUT_TYPE)
+    .choices(['jpeg', 'png', 'md', 'html'])
+    .makeOptionMandatory());
+commandScrape.option('--quality <quality>', texts_1.OPTION_SCRAPE_QUALITY);
+commandScrape.option('--fullPage', texts_1.OPTION_SCRAPE_FULL_PAGE, false);
+commandScrape.option('--cssSelector <selector>', texts_1.OPTION_SCRAPE_CSS_SELECTOR);
+commandScrape.option('--xpathSelector <selector>', texts_1.OPTION_SCRAPE_XPATH_SELECTOR);
+commandScrape.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAPE_BROWSER)
+    .choices(['chrome', 'firefox', 'safari'])
+    .default('chrome'));
+commandScrape.option('--viewport <viewport>', texts_1.OPTION_SCRAPE_VIEWPORT, '1920x1080');
+commandScrape.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAPE_DEVICE_PIXEL_RATIO, '1');
+commandScrape.option('--waitForElement <selector>', texts_1.OPTION_SCRAPE_WAIT_FOR_ELEMENT);
+commandScrape.option('--darkMode', texts_1.OPTION_SCRAPE_DARK_MODE, false);
+commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
+commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
+commandScrape.action(async (inputUrl, options) => {
+    if (!(0, validation_1.checkToken)()) {
+        output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
+    }
+    const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
+    try {
+        const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
+        output_1.default.normal('Starting scrape session');
+        const status = await watchSessionStatus(buildId);
+        if (!status.ok) {
+            output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
+        }
+        output_1.default.normal('Downloading scrape package');
+        const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
+        const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
+        const unpack = tar_stream_1.default.extract();
+        unpack.on('entry', async (header, stream, next) => {
+            const currentDir = process.cwd();
+            const preparedOutputDir = outputDir.startsWith('.')
+                ? node_path_1.default.join(currentDir, outputDir)
+                : outputDir;
+            const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
+            try {
+                if (header.type === 'file') {
+                    await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
+                    const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
+                    await (0, promises_1.pipeline)(stream, fileWriteStream);
+                    next();
+                }
+                else {
+                    stream.resume();
+                    next();
+                }
+            }
+            catch (entryError) {
+                output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
+                next(entryError);
+            }
+        });
+        await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
+        output_1.default.exitSuccess('Downloading scrape package finished');
+    }
+    catch (error) {
+        output_1.default.exitError(`${error}`);
+    }
+});
+function validateInputAndOptions(input, options) {
+    const urlSchema = zod_1.z.string().url();
+    const optionsSchema = zod_1.z.object({
+        follow: zod_1.z.boolean(),
+        outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
+        quality: zod_1.z.coerce.number().min(1).max(100).optional(),
+        outputDir: zod_1.z.string().default('.'),
+        fullPage: zod_1.z.boolean().optional(),
+        cssSelector: zod_1.z.string().optional(),
+        xpathSelector: zod_1.z.string().optional(),
+        browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
+        viewport: zod_1.z
+            .string()
+            .refine((value) => {
+            const [width, height] = value.split('x');
+            return (width &&
+                height &&
+                !isNaN(Number(width)) &&
+                !isNaN(Number(height)) &&
+                Number(width) > 0 &&
+                Number(height) > 0);
+        }, 'Invalid viewport format, example: 1920x1080')
+            .transform((value) => {
+            const [width, height] = value.split('x');
+            return {
+                width: Number(width),
+                height: Number(height),
+            };
+        }),
+        devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
+        darkMode: zod_1.z.boolean(),
+        delay: zod_1.z.coerce.number().min(0).max(10000),
+        waitForElement: zod_1.z.string().optional(),
+    });
+    try {
+        const url = urlSchema.parse(input);
+        const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
+        if (typeof quality === 'number' && outputType !== 'jpeg') {
+            output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
+        }
+        if (cssSelector && xpathSelector) {
+            output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
+        }
+        return {
+            url,
+            follow,
+            outputType,
+            quality,
+            outputDir,
+            fullPage,
+            cssSelector,
+            xpathSelector,
+            browser,
+            viewport,
+            devicePixelRatio,
+            darkMode,
+            delay,
+            waitForElement,
+        };
+    }
+    catch (error) {
+        if (error instanceof zod_1.ZodError) {
+            output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
+        }
+        else {
+            throw error;
+        }
+    }
+}
+async function watchSessionStatus(buildId) {
+    return new Promise((resolve) => {
+        const eventSource = (0, requests_1.connectToScrapSession)(buildId);
+        eventSource.addEventListener('SESSION_STATUS', (event) => {
+            const data = JSON.parse(event.data);
+            if (data.status === 'GATHER_URLS_COMPLETED') {
+                output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
+            }
+            else if (data.status === 'GATHER_URLS_FAILED') {
+                output_1.default.error('Gathering URLs failed');
+            }
+            else if (data.status === 'SCRAPE_URL_COMPLETED') {
+                output_1.default.normal(`Scraping ${data.text} completed`);
+            }
+            else if (data.status === 'SCRAPE_URL_FAILED') {
+                output_1.default.error(`Scraping ${data.text} failed`);
+            }
+            else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
+                output_1.default.normal('Package created');
+            }
+            else if (data.status === 'CREATE_PACKAGE_FAILED') {
+                output_1.default.error('Package creation failed');
+            }
+            else if (data.status === 'FINISHED') {
+                eventSource.close();
+                output_1.default.normal('Scrape session finished');
+                resolve({ ok: true });
+            }
+        });
+        eventSource.addEventListener('error', (event) => {
+            if (event.code) {
+                eventSource.close();
+                if (event.code === 410) {
+                    output_1.default.normal('Scrape session finished');
+                }
+                resolve({ ok: event.code === 410, error: event.code });
+            }
+        });
+    });
+}
+exports.default = commandScrape;

package/distTs/src/input.js

@@ -334,8 +334,8 @@ class Input {
     }
     static restApiTokenClient(allowNoToken = false, api = '', region = '', token = '') {
         const ApiClient = require('./api/client').default;
-        const baseUrl = this.restApiBaseUrl(api, region);
         const { token: t, refreshToken, clientId, clientToken, clientSecret, } = this.restApiToken(allowNoToken, token);
+        const baseUrl = this.restApiBaseUrl(api, region, t);
         return new ApiClient(baseUrl, t, refreshToken, clientId, clientSecret, clientToken);
     }
     static artifactType(type) {
@@ -350,17 +350,6 @@ class Input {
             return utils_1.ARTIFACT_SCOPE.PROJECT;
         return utils_1.ARTIFACT_SCOPE.WORKSPACE;
     }
-    static pipelineRunId(runId) {
-        const r = parseInt(runId, 10);
-        if (!isNaN(r))
-            return r;
-        output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_WRONG_RUN_ID);
-    }
-    static pipelineRunActionId(runActionId) {
-        if (runActionId)
-            return runActionId;
-        output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_WRONG_RUN_ACTION_ID);
-    }
     static pipelineRunPriority(priority) {
         if (!priority)
             return null;
@@ -442,8 +431,26 @@ class Input {
             output_1.default.exitError(texts_1.ERR_REST_API_REGION);
         }
     }
-    static restApiBaseUrl(api = '', region = '') {
-        let u = api || process.env.BUDDY_API_ENDPOINT;
+    static restApiBaseUrlFromToken(token) {
+        // bdy_sb_us_aHR0cHM6Ly9hcGkuc2Q1LmNvbQ_f92ml8f1p27nzo45806vczm6c78d0
+        const s = (token || '').split('_');
+        if (s.length > 1) {
+            const base = s[s.length - 2];
+            try {
+                return Buffer.from(base, 'base64').toString('utf8');
+            }
+            catch {
+                // do nothing
+            }
+        }
+        return null;
+    }
+    static restApiBaseUrl(api = '', region = '', token = '') {
+        let u = api;
+        if (!u)
+            u = this.restApiBaseUrlFromToken(token);
+        if (!u)
+            u = process.env.BUDDY_API_ENDPOINT || null;
         if (!u)
             u = cfg_1.default.getBaseUrl();
         if (!u) {
@@ -468,6 +475,16 @@ class Input {
             output_1.default.exitError(texts_1.ERR_REST_API_URL);
         }
     }
+    static pipelineRunWaitTime(time) {
+        let t = Number.parseInt(time, 10);
+        if (!t)
+            t = 1440;
+        else if (t < 0)
+            t = 1;
+        else if (t > 1440)
+            t = 1440;
+        return t;
+    }
     static resolvePath(path) {
         const p = (0, node_path_1.resolve)(path);
         const exists = node_fs_1.default.existsSync(p);

package/distTs/src/output.js

@@ -18,69 +18,39 @@ class Output {
     static getCyanColor(text) {
         return chalk_1.default.hex('#00C9FF')(text);
     }
-    static getTermKitCyanColor(text) {
-        return `^C${text}^:`;
-    }
     static getBlueColor(text) {
         return chalk_1.default.hex('#1A86FD')(text);
     }
-    static getTermKitBlueColor(text) {
-        return `^B${text}^:`;
-    }
-    static getPurpleColor(text) {
-        return chalk_1.default.hex('#A06BC4')(text);
-    }
-    static getTermKitPurpleColor(text) {
-        return `^M${text}^:`;
+    static getLightColor(text) {
+        return chalk_1.default.hex('#05BBFF')(text);
     }
     static getMutedColor(text) {
         return chalk_1.default.hex('#8B8D98')(text);
     }
-    static getTermKitMutedColor(text) {
-        return `^w${text}^:`;
-    }
     static getDimColor(text) {
         return chalk_1.default.hex('#555869')(text);
     }
-    static getTermKitDimColor(text) {
-        return `^K${text}^:`;
-    }
     static getArgColor(text) {
         return chalk_1.default.hex('#5A8AB5')(text);
     }
-    static getTermKitArgColor(text) {
-        return `^B${text}^:`;
-    }
     static getLabelColor(text) {
         return chalk_1.default.hex('#D1D5DB')(text);
     }
-    static getTermKitLabelColor(text) {
-        return `^W${text}^:`;
-    }
     static getGreenColor(text) {
-        return chalk_1.default.greenBright(text);
-    }
-    static getTermKitGreenColor(text) {
-        return `^G${text}^:`;
+        return chalk_1.default.green(text);
     }
     static getYellowColor(text) {
         return chalk_1.default.yellow(text);
     }
-    static getTermKitYellowColor(text) {
-        return `^Y${text}^:`;
-    }
     static getRedColor(text) {
-        return chalk_1.default.redBright(text);
-    }
-    static getTermKitRedColor(text) {
-        return `^R${text}^:`;
+        return chalk_1.default.red(text);
     }
     static getInquirerTheme() {
         return {
             prefix: this.getCyanColor('?'),
             style: {
                 highlight: (text) => this.getCyanColor(text),
-                answer: (text) => this.getCyanColor(text),
+                answer: (text) => this.getLightColor(text),
                 message: (text) => this.getLabelColor(text),
                 help: (text) => this.getDimColor(text),
                 description: (text) => this.getArgColor(text),
@@ -114,6 +84,9 @@ class Output {
     static newline() {
         terminal('\n');
     }
+    static light(txt, newLine = true) {
+        this.normal(this.getLightColor(txt), newLine);
+    }
     static muted(txt, newLine = true) {
         this.normal(this.getMutedColor(txt), newLine);
     }
@@ -276,16 +249,11 @@ class Output {
     static async confirm(message = 'Are you sure?') {
         if (this.isStdInTTY()) {
             const prompts = require('@inquirer/prompts');
-            try {
-                return await prompts.confirm({
-                    message,
-                    theme: this.getInquirerTheme(),
-                    default: false,
-                });
-            }
-            catch {
-                this.exitError(texts_1.ERR_CANCELLED);
-            }
+            return await prompts.confirm({
+                message,
+                theme: this.getInquirerTheme(),
+                default: false,
+            });
         }
         this.cyan('? ', false);
         this.label(message, false);
@@ -295,19 +263,12 @@ class Output {
     static async inputString(message, def, required, signal) {
         if (this.isStdInTTY()) {
             const prompts = require('@inquirer/prompts');
-            try {
-                return await prompts.input({
-                    required,
-                    message,
-                    default: def,
-                    theme: this.getInquirerTheme(),
-                }, { signal });
-            }
-            catch (err) {
-                if (err.name === 'AbortPromptError')
-                    throw err;
-                this.exitError(texts_1.ERR_CANCELLED);
-            }
+            return await prompts.input({
+                required,
+                message,
+                default: def,
+                theme: this.getInquirerTheme(),
+            }, { signal });
         }
         this.cyan('? ', false);
         this.normal(message, false);
@@ -321,17 +282,12 @@ class Output {
     static async inputMenuAdv(message, items) {
         const prompts = require('@inquirer/prompts');
         if (this.isStdInTTY()) {
-            try {
-                return await prompts.select({
-                    message,
-                    theme: this.getInquirerTheme(),
-                    loop: false,
-                    choices: items,
-                });
-            }
-            catch {
-                this.exitError(texts_1.ERR_CANCELLED);
-            }
+            return await prompts.select({
+                message,
+                theme: this.getInquirerTheme(),
+                loop: false,
+                choices: items,
+            });
         }
         else {
             this.cyan('? ', false);
@@ -381,17 +337,12 @@ class Output {
     static async inputMenu(message, items) {
         if (this.isStdInTTY()) {
             const prompts = require('@inquirer/prompts');
-            try {
-                return await prompts.select({
-                    message,
-                    theme: this.getInquirerTheme(),
-                    loop: false,
-                    choices: items.map((item, index) => ({ value: index, name: item })),
-                });
-            }
-            catch {
-                this.exitError(texts_1.ERR_CANCELLED);
-            }
+            return await prompts.select({
+                message,
+                theme: this.getInquirerTheme(),
+                loop: false,
+                choices: items.map((item, index) => ({ value: index, name: item })),
+            });
         }
         else {
             this.cyan('? ', false);
@@ -432,72 +383,6 @@ class Output {
         }
         return `${normalized}/api`;
     }
-    static createScreenBuffer(onKey, onResize) {
-        const viewPort = new termkit_no_lazy_require_1.default.ScreenBuffer({
-            dst: terminal,
-            width: terminal.width,
-            height: terminal.height,
-            x: 1,
-            y: 1,
-        });
-        terminal.grabInput();
-        terminal.hideCursor();
-        terminal.fullscreen(true);
-        const exit = () => {
-            terminal.fullscreen(false);
-            terminal.hideCursor(false);
-            terminal.grabInput(false);
-            process.exit();
-        };
-        terminal.on('key', (name) => {
-            const shouldExit = onKey(name);
-            if (shouldExit)
-                exit();
-        });
-        terminal.on('resize', () => {
-            viewPort.resize({
-                width: termkit_no_lazy_require_1.default.terminal.width,
-                height: termkit_no_lazy_require_1.default.terminal.height,
-                x: 1,
-                y: 1,
-            });
-            onResize();
-        });
-        let y = 1;
-        const put = (str) => {
-            viewPort.put({
-                x: 1,
-                y,
-                markup: true,
-            }, str);
-            y += 1;
-            return y;
-        };
-        const blank = () => {
-            y += 1;
-            return y;
-        };
-        const reset = () => {
-            y = 1;
-            viewPort.fill({
-                char: ' ',
-            });
-        };
-        const apply = () => {
-            viewPort.draw();
-        };
-        const getY = () => {
-            return y;
-        };
-        return {
-            put,
-            reset,
-            blank,
-            apply,
-            getY,
-            exit,
-        };
-    }
     static async selectRegion() {
         const index = await Output.inputMenu(texts_1.TXT_LOGIN_SELECT_REGION, [
             'US (default)',
@@ -513,9 +398,6 @@ class Output {
             return utils_1.REST_API_REGION.AS;
         return utils_1.REST_API_REGION.ONPREM;
     }
-    static getTerminal() {
-        return terminal;
-    }
     static exitError(err) {
         this.killSpinner();
         terminal.fullscreen(false);