bdy 1.23.7-dev-target-commands → 1.23.7-dev

package/distTs/src/command/vt/scrape.js

@@ -0,0 +1,193 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../../utils");
+const commander_1 = require("commander");
+const texts_1 = require("../../texts");
+const validation_1 = require("../../visualTest/validation");
+const output_1 = __importDefault(require("../../output"));
+const requests_1 = require("../../visualTest/requests");
+const zod_1 = require("zod");
+const node_zlib_1 = require("node:zlib");
+const tar_stream_1 = __importDefault(require("tar-stream"));
+const promises_1 = require("node:stream/promises");
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const promises_2 = require("node:fs/promises");
+const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
+commandScrape.argument('<url>', texts_1.OPTION_SCRAPE_URL);
+commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
+commandScrape.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAPE_OUTPUT_TYPE)
+    .choices(['jpeg', 'png', 'md', 'html'])
+    .makeOptionMandatory());
+commandScrape.option('--quality <quality>', texts_1.OPTION_SCRAPE_QUALITY);
+commandScrape.option('--fullPage', texts_1.OPTION_SCRAPE_FULL_PAGE, false);
+commandScrape.option('--cssSelector <selector>', texts_1.OPTION_SCRAPE_CSS_SELECTOR);
+commandScrape.option('--xpathSelector <selector>', texts_1.OPTION_SCRAPE_XPATH_SELECTOR);
+commandScrape.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAPE_BROWSER)
+    .choices(['chrome', 'firefox', 'safari'])
+    .default('chrome'));
+commandScrape.option('--viewport <viewport>', texts_1.OPTION_SCRAPE_VIEWPORT, '1920x1080');
+commandScrape.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAPE_DEVICE_PIXEL_RATIO, '1');
+commandScrape.option('--waitForElement <selector>', texts_1.OPTION_SCRAPE_WAIT_FOR_ELEMENT);
+commandScrape.option('--darkMode', texts_1.OPTION_SCRAPE_DARK_MODE, false);
+commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
+commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
+commandScrape.action(async (inputUrl, options) => {
+    if (!(0, validation_1.checkToken)()) {
+        output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
+    }
+    const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
+    try {
+        const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
+        output_1.default.normal('Starting scrape session');
+        const status = await watchSessionStatus(buildId);
+        if (!status.ok) {
+            output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
+        }
+        output_1.default.normal('Downloading scrape package');
+        const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
+        const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
+        const unpack = tar_stream_1.default.extract();
+        unpack.on('entry', async (header, stream, next) => {
+            const currentDir = process.cwd();
+            const preparedOutputDir = outputDir.startsWith('.')
+                ? node_path_1.default.join(currentDir, outputDir)
+                : outputDir;
+            const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
+            try {
+                if (header.type === 'file') {
+                    await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
+                    const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
+                    await (0, promises_1.pipeline)(stream, fileWriteStream);
+                    next();
+                }
+                else {
+                    stream.resume();
+                    next();
+                }
+            }
+            catch (entryError) {
+                output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
+                next(entryError);
+            }
+        });
+        await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
+        output_1.default.exitSuccess('Downloading scrape package finished');
+    }
+    catch (error) {
+        output_1.default.exitError(`${error}`);
+    }
+});
+function validateInputAndOptions(input, options) {
+    const urlSchema = zod_1.z.string().url();
+    const optionsSchema = zod_1.z.object({
+        follow: zod_1.z.boolean(),
+        outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
+        quality: zod_1.z.coerce.number().min(1).max(100).optional(),
+        outputDir: zod_1.z.string().default('.'),
+        fullPage: zod_1.z.boolean().optional(),
+        cssSelector: zod_1.z.string().optional(),
+        xpathSelector: zod_1.z.string().optional(),
+        browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
+        viewport: zod_1.z
+            .string()
+            .refine((value) => {
+            const [width, height] = value.split('x');
+            return (width &&
+                height &&
+                !isNaN(Number(width)) &&
+                !isNaN(Number(height)) &&
+                Number(width) > 0 &&
+                Number(height) > 0);
+        }, 'Invalid viewport format, example: 1920x1080')
+            .transform((value) => {
+            const [width, height] = value.split('x');
+            return {
+                width: Number(width),
+                height: Number(height),
+            };
+        }),
+        devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
+        darkMode: zod_1.z.boolean(),
+        delay: zod_1.z.coerce.number().min(0).max(10000),
+        waitForElement: zod_1.z.string().optional(),
+    });
+    try {
+        const url = urlSchema.parse(input);
+        const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
+        if (typeof quality === 'number' && outputType !== 'jpeg') {
+            output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
+        }
+        if (cssSelector && xpathSelector) {
+            output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
+        }
+        return {
+            url,
+            follow,
+            outputType,
+            quality,
+            outputDir,
+            fullPage,
+            cssSelector,
+            xpathSelector,
+            browser,
+            viewport,
+            devicePixelRatio,
+            darkMode,
+            delay,
+            waitForElement,
+        };
+    }
+    catch (error) {
+        if (error instanceof zod_1.ZodError) {
+            output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
+        }
+        else {
+            throw error;
+        }
+    }
+}
+async function watchSessionStatus(buildId) {
+    return new Promise((resolve) => {
+        const eventSource = (0, requests_1.connectToScrapSession)(buildId);
+        eventSource.addEventListener('SESSION_STATUS', (event) => {
+            const data = JSON.parse(event.data);
+            if (data.status === 'GATHER_URLS_COMPLETED') {
+                output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
+            }
+            else if (data.status === 'GATHER_URLS_FAILED') {
+                output_1.default.error('Gathering URLs failed');
+            }
+            else if (data.status === 'SCRAPE_URL_COMPLETED') {
+                output_1.default.normal(`Scraping ${data.text} completed`);
+            }
+            else if (data.status === 'SCRAPE_URL_FAILED') {
+                output_1.default.error(`Scraping ${data.text} failed`);
+            }
+            else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
+                output_1.default.normal('Package created');
+            }
+            else if (data.status === 'CREATE_PACKAGE_FAILED') {
+                output_1.default.error('Package creation failed');
+            }
+            else if (data.status === 'FINISHED') {
+                eventSource.close();
+                output_1.default.normal('Scrape session finished');
+                resolve({ ok: true });
+            }
+        });
+        eventSource.addEventListener('error', (event) => {
+            if (event.code) {
+                eventSource.close();
+                if (event.code === 410) {
+                    output_1.default.normal('Scrape session finished');
+                }
+                resolve({ ok: event.code === 410, error: event.code });
+            }
+        });
+    });
+}
+exports.default = commandScrape;

package/distTs/src/input.js

@@ -1087,54 +1087,6 @@ class Input {
             return distro_1.SCOPE.PROJECT;
         return distro_1.SCOPE.WORKSPACE;
     }
-    static targetScope(project, pipeline, environment, scope) {
-        Input.targetLine(project, pipeline, environment);
-        if (scope) {
-            scope = scope.toUpperCase();
-            if (scope === utils_1.TARGET_SCOPE.WORKSPACE) {
-                return scope;
-            }
-            if (scope === utils_1.TARGET_SCOPE.PROJECT) {
-                if (!project)
-                    output_1.default.exitError(texts_1.ERR_COMMAND_SCOPE_NO_PROJECT);
-                return scope;
-            }
-            if (scope === utils_1.TARGET_SCOPE.PIPELINE) {
-                if (!pipeline)
-                    output_1.default.exitError(texts_1.ERR_TARGET_SCOPE_NO_PIPELINE);
-                return scope;
-            }
-            if (scope === utils_1.TARGET_SCOPE.ENVIRONMENT) {
-                if (!environment)
-                    output_1.default.exitError(texts_1.ERR_TARGET_SCOPE_NO_ENVIRONMENT);
-                return scope;
-            }
-            return output_1.default.exitError(texts_1.ERR_TARGET_SCOPE);
-        }
-        if (pipeline)
-            return utils_1.TARGET_SCOPE.PIPELINE;
-        if (environment)
-            return utils_1.TARGET_SCOPE.ENVIRONMENT;
-        if (project)
-            return utils_1.TARGET_SCOPE.PROJECT;
-        return utils_1.TARGET_SCOPE.WORKSPACE;
-    }
-    // Line context for resolving a target identifier via /identifiers:
-    // params describe one line from workspace down (ws -> project -> pipeline
-    // or environment), searched from the deepest scope up.
-    static targetLine(project, pipeline, environment) {
-        if (pipeline && !project) {
-            output_1.default.exitError(texts_1.ERR_TARGET_PIPELINE_REQUIRES_PROJECT);
-        }
-        if (pipeline && environment) {
-            output_1.default.exitError(texts_1.ERR_TARGET_PIPELINE_ENV_CONFLICT);
-        }
-        return {
-            project: project || undefined,
-            pipeline,
-            environment,
-        };
-    }
     static restApiProject(project, allowNull) {
         const ProjectCfg = require('./project/cfg').default;
         let p = process.env.BUDDY_PROJECT;
@@ -1155,12 +1107,6 @@ class Input {
             output_1.default.exitError(texts_1.ERR_SANDBOX_BOOT_SCRIPT_NOT_EXISTS);
         return node_fs_1.default.readFileSync(p, 'utf8');
     }
-    static tags(tags) {
-        return tags
-            .split(',')
-            .map((t) => t.trim())
-            .filter(Boolean);
-    }
     static restApiYaml(yaml) {
         if (!yaml)
             output_1.default.exitError(texts_1.ERR_REST_API_YAML);

package/distTs/src/logger.js

@@ -24,6 +24,9 @@ class Logger {
         try {
             const s = fs_1.default.statSync(this.logPath);
             if (s.size > 5242880) {
+                // No rights to re-apply on the copy: copyFileSync takes the source's mode with it, so
+                // the rotated log is as open as the one it came from - 0666 in a home directory, 0640
+                // in the system one - and stays writable by whoever rotates it next
                 fs_1.default.copyFileSync(this.logPath, this.log1Path);
                 fs_1.default.truncateSync(this.logPath);
             }
@@ -36,17 +39,46 @@ class Logger {
         this.debugOn = debug;
         this.getPino().level = this.debugOn ? 'debug' : 'info';
     }
+    // Which directory the log landed in decides both of these. In a home directory the 0700 on
+    // the directory is what keeps it private, so the file stays world-writable on purpose: it is
+    // the one thing in there a root run creates fresh in someone else's home, and a mode that
+    // locked that account out of its own log would take down every later command it runs. The
+    // system directory is world-traversable by design, so there the mode is the only thing
+    // between the log and every local account - and its owner is the service user, never
+    // whoever's HOME a root process happens to have inherited.
+    applyLogRights(p) {
+        try {
+            // Trailing separator, so a sibling directory cannot pass for the home one
+            const home = (0, utils_1.getHomeDirectory)() + path_1.sep;
+            const inHome = p.startsWith(home);
+            fs_1.default.chmodSync(p, inHome ? 0o666 : 0o640);
+            if (inHome)
+                (0, utils_1.chownToHomeOwner)(p);
+        }
+        catch {
+            // do nothing, a log we cannot re-mode still works for the account that opened it
+        }
+    }
     getPino() {
         if (!this.p) {
             this.logPath = (0, path_1.resolve)(this.rootPath, 'cli.log');
             this.log1Path = (0, path_1.resolve)(this.rootPath, 'cli.1.log');
-            this.logStream = fs_1.default.openSync(this.logPath, 'w');
             try {
-                fs_1.default.chmodSync(this.logPath, 0o666);
+                this.logStream = fs_1.default.openSync(this.logPath, 'w');
             }
             catch {
-                // do nothing
+                // A log we cannot open is not a reason to fail the command - a read-only home, a full
+                // disk or a file left behind by another account all end up here
+                this.logStream = null;
+                this.logPath = null;
+                this.log1Path = null;
             }
+            // Deliberately not inside the try above: the stream is already open and usable, and a
+            // mode or an owner that will not go on is no reason to throw it away and log nowhere.
+            // Resolving the home directory can itself fail - a service account whose home was
+            // removed - and the log that just opened is very often the system one.
+            if (this.logPath)
+                this.applyLogRights(this.logPath);
             const pino = require('pino');
             this.p = pino({
                 level: this.debugOn ? 'debug' : 'info',
@@ -57,7 +89,11 @@ class Logger {
                         level: label.toUpperCase(),
                     }),
                 },
-            }, pino.destination(this.logStream));
+            }, 
+            // Not stdout as a fallback: it is machine-read by `--format json` and the mcp command
+            this.logStream === null
+                ? { write: () => { } }
+                : pino.destination(this.logStream));
             this.checkLogSize();
             this.ts = setInterval(() => {
                 this.checkLogSize();