bdy 1.18.8-dev → 1.18.9-dev-commands-changes-869c3r8yn
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/command/api/request.js +25 -3
- package/distTs/src/command/crawl/validation.js +151 -0
- package/distTs/src/command/crawl.js +144 -0
- package/distTs/src/command/login.js +1 -2
- package/distTs/src/command/scrape/validation.js +21 -15
- package/distTs/src/command/scrape.js +14 -15
- package/distTs/src/command/tests/capture/validation.js +59 -0
- package/distTs/src/command/tests/capture.js +100 -0
- package/distTs/src/command/tests/unit/upload.js +86 -0
- package/distTs/src/command/tests/unit.js +11 -0
- package/distTs/src/command/tests/visual/session/close.js +27 -0
- package/distTs/src/command/tests/visual/session/create.js +82 -0
- package/distTs/src/command/tests/visual/session.js +13 -0
- package/distTs/src/command/tests/visual/setup.js +20 -0
- package/distTs/src/command/tests/visual/shared/validation.js +118 -0
- package/distTs/src/command/tests/visual/upload.js +138 -0
- package/distTs/src/command/tests/visual.js +15 -0
- package/distTs/src/command/tests.js +15 -0
- package/distTs/src/command/ut/upload.js +48 -16
- package/distTs/src/command/vt/close.js +5 -5
- package/distTs/src/command/vt/compare.js +15 -18
- package/distTs/src/command/vt/exec.js +28 -28
- package/distTs/src/command/vt/installBrowser.js +2 -2
- package/distTs/src/command/vt/storybook.js +14 -14
- package/distTs/src/index.js +4 -6
- package/distTs/src/output.js +7 -2
- package/distTs/src/texts.js +39 -37
- package/distTs/src/types/crawl.js +2 -0
- package/distTs/src/visualTest/requests.js +1 -1
- package/package.json +1 -1
- package/distTs/src/command/project/get.js +0 -18
- package/distTs/src/command/project/set.js +0 -31
- package/distTs/src/command/sandbox/get/yaml.js +0 -30
- package/distTs/src/command/vt/scrape.js +0 -193
package/distTs/package.json
CHANGED
|
@@ -77,12 +77,34 @@ const prepareMultipartBody = async (params) => {
|
|
|
77
77
|
}
|
|
78
78
|
return form;
|
|
79
79
|
};
|
|
80
|
+
const parseUrl = (url) => {
|
|
81
|
+
const query = {};
|
|
82
|
+
let path = url;
|
|
83
|
+
let host = '';
|
|
84
|
+
try {
|
|
85
|
+
const u = new URL(url, 'https://example.com');
|
|
86
|
+
if (u.host !== 'example.com')
|
|
87
|
+
host = u.host;
|
|
88
|
+
path = decodeURIComponent(u.pathname);
|
|
89
|
+
u.searchParams.forEach((v, k) => {
|
|
90
|
+
query[k] = v;
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// do nothing
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
host,
|
|
98
|
+
path,
|
|
99
|
+
query,
|
|
100
|
+
};
|
|
101
|
+
};
|
|
80
102
|
const request = async (method, url, options) => {
|
|
81
103
|
output_1.default.handleSignals();
|
|
104
|
+
const { query, path, host } = parseUrl(url);
|
|
82
105
|
const workspace = input_1.default.restApiWorkspace(options.workspace, true);
|
|
83
106
|
const project = input_1.default.restApiProject(options.project, true);
|
|
84
|
-
const client = input_1.default.restApiTokenClient();
|
|
85
|
-
const query = {};
|
|
107
|
+
const client = input_1.default.restApiTokenClient(false, host);
|
|
86
108
|
if (options.query) {
|
|
87
109
|
options.query.forEach((q) => {
|
|
88
110
|
const s = q.split(':');
|
|
@@ -92,7 +114,7 @@ const request = async (method, url, options) => {
|
|
|
92
114
|
query[s[0]] = s[1];
|
|
93
115
|
});
|
|
94
116
|
}
|
|
95
|
-
let preparedUrl =
|
|
117
|
+
let preparedUrl = path;
|
|
96
118
|
if (workspace) {
|
|
97
119
|
preparedUrl = preparedUrl.replaceAll('{workspace_domain}', encodeURIComponent(workspace));
|
|
98
120
|
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.validateInputAndOptions = validateInputAndOptions;
|
|
7
|
+
const zod_1 = require("zod");
|
|
8
|
+
const output_1 = __importDefault(require("../../output"));
|
|
9
|
+
const validation_1 = require("../tests/visual/shared/validation");
|
|
10
|
+
const urlSchema = zod_1.z.string().url().optional();
|
|
11
|
+
const browserSchema = zod_1.z.enum(['chrome', 'firefox', 'safari']);
|
|
12
|
+
const browsersListSchema = zod_1.z
|
|
13
|
+
.string()
|
|
14
|
+
.transform((value) => value
|
|
15
|
+
.split(',')
|
|
16
|
+
.map((browser) => browser.trim().toLowerCase())
|
|
17
|
+
.filter((browser) => browser.length > 0))
|
|
18
|
+
.refine((browsers) => browsers.length > 0, {
|
|
19
|
+
message: 'Invalid browsers list. Supported values: chrome,firefox,safari',
|
|
20
|
+
})
|
|
21
|
+
.pipe(zod_1.z.array(browserSchema))
|
|
22
|
+
.transform((browsers) => Array.from(new Set(browsers.map((browser) => browser === 'chrome'
|
|
23
|
+
? 'CHROMIUM'
|
|
24
|
+
: browser === 'firefox'
|
|
25
|
+
? 'FIREFOX'
|
|
26
|
+
: 'WEBKIT'))));
|
|
27
|
+
const optionsSchema = zod_1.z.object({
|
|
28
|
+
follow: zod_1.z.boolean(),
|
|
29
|
+
respectRobots: zod_1.z.boolean(),
|
|
30
|
+
outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']).optional(),
|
|
31
|
+
outputTypes: zod_1.z.string().optional(),
|
|
32
|
+
quality: zod_1.z.coerce.number().min(1).max(100).optional(),
|
|
33
|
+
outputDir: zod_1.z.string().default('.'),
|
|
34
|
+
fullPage: zod_1.z.boolean().optional(),
|
|
35
|
+
cssSelector: zod_1.z.string().optional(),
|
|
36
|
+
xpathSelector: zod_1.z.string().optional(),
|
|
37
|
+
colorScheme: zod_1.z.enum(['LIGHT', 'DARK', 'LIGHT_AND_DARK']).optional(),
|
|
38
|
+
browsers: browsersListSchema.optional(),
|
|
39
|
+
devices: zod_1.z.string().optional(),
|
|
40
|
+
delay: zod_1.z.coerce.number().min(0).max(10000),
|
|
41
|
+
waitFor: validation_1.waitForSchema,
|
|
42
|
+
cookie: validation_1.cookieSchema,
|
|
43
|
+
header: validation_1.headerSchema,
|
|
44
|
+
localStorage: zod_1.z
|
|
45
|
+
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
|
|
46
|
+
message: "LocalStorage option must follow pattern '[scope::]key=value' (scope is optional)",
|
|
47
|
+
}))
|
|
48
|
+
.optional()
|
|
49
|
+
.transform((value) => value?.map((v) => {
|
|
50
|
+
const { scope, key, value } = (0, validation_1.parseScopedKeyValue)(v);
|
|
51
|
+
return { scope, key, value };
|
|
52
|
+
})),
|
|
53
|
+
});
|
|
54
|
+
function validateInputAndOptions(input, options) {
|
|
55
|
+
try {
|
|
56
|
+
const url = urlSchema.parse(input);
|
|
57
|
+
const { follow, respectRobots, outputType, outputTypes: rawOutputTypes, quality, outputDir, fullPage, cssSelector, xpathSelector, colorScheme, browsers: parsedBrowsers, devices: rawDevices, delay, waitFor, cookie, header, localStorage, } = optionsSchema.parse(options);
|
|
58
|
+
let parsedOutputTypes;
|
|
59
|
+
if (rawOutputTypes) {
|
|
60
|
+
try {
|
|
61
|
+
const outputTypeEntrySchema = zod_1.z.array(zod_1.z
|
|
62
|
+
.object({
|
|
63
|
+
type: zod_1.z.string().transform((v) => v.toUpperCase()),
|
|
64
|
+
selector: zod_1.z
|
|
65
|
+
.object({
|
|
66
|
+
type: zod_1.z.enum(['CSS', 'XPATH']).optional(),
|
|
67
|
+
value: zod_1.z.string().optional(),
|
|
68
|
+
})
|
|
69
|
+
.optional(),
|
|
70
|
+
quality: zod_1.z.number().min(1).max(100).optional(),
|
|
71
|
+
})
|
|
72
|
+
.transform((data) => ({
|
|
73
|
+
...data,
|
|
74
|
+
type: data.type,
|
|
75
|
+
})));
|
|
76
|
+
parsedOutputTypes = outputTypeEntrySchema.parse(JSON.parse(rawOutputTypes));
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
output_1.default.exitError("Invalid --outputTypes value. Use JSON array, e.g. --outputTypes '[{\"type\":\"png\"},{\"type\":\"jpeg\",\"quality\":80}]'");
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
else if (outputType) {
|
|
83
|
+
if (typeof quality === 'number' && outputType !== 'jpeg') {
|
|
84
|
+
output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
|
|
85
|
+
}
|
|
86
|
+
if (cssSelector && xpathSelector) {
|
|
87
|
+
output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
|
|
88
|
+
}
|
|
89
|
+
const entry = {
|
|
90
|
+
type: outputType.toUpperCase(),
|
|
91
|
+
};
|
|
92
|
+
if (cssSelector) {
|
|
93
|
+
entry.selector = { type: 'CSS', value: cssSelector };
|
|
94
|
+
}
|
|
95
|
+
else if (xpathSelector) {
|
|
96
|
+
entry.selector = { type: 'XPATH', value: xpathSelector };
|
|
97
|
+
}
|
|
98
|
+
if (typeof quality === 'number') {
|
|
99
|
+
entry.quality = quality;
|
|
100
|
+
}
|
|
101
|
+
parsedOutputTypes = [entry];
|
|
102
|
+
}
|
|
103
|
+
let parsedDevices;
|
|
104
|
+
if (rawDevices) {
|
|
105
|
+
try {
|
|
106
|
+
const dimensionSchema = zod_1.z
|
|
107
|
+
.string()
|
|
108
|
+
.regex(/^\d+x\d+$/, 'Must be in format "widthxheight"')
|
|
109
|
+
.transform((val) => {
|
|
110
|
+
const [width, height] = val.split('x').map(Number);
|
|
111
|
+
return { width, height };
|
|
112
|
+
});
|
|
113
|
+
const deviceSchema = zod_1.z.array(zod_1.z.object({
|
|
114
|
+
name: zod_1.z.string().optional(),
|
|
115
|
+
viewport: dimensionSchema,
|
|
116
|
+
screen: dimensionSchema,
|
|
117
|
+
devicePixelRatio: zod_1.z.number().positive(),
|
|
118
|
+
isMobile: zod_1.z.boolean(),
|
|
119
|
+
}));
|
|
120
|
+
parsedDevices = deviceSchema.parse(JSON.parse(rawDevices));
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
output_1.default.exitError('Invalid --devices value. Use JSON array, e.g. --devices \'[{"viewport":"1920x1080","screen":"1920x1080","devicePixelRatio":1,"isMobile":false}]\'');
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
url,
|
|
128
|
+
follow,
|
|
129
|
+
respectRobots,
|
|
130
|
+
outputTypes: parsedOutputTypes,
|
|
131
|
+
outputDir,
|
|
132
|
+
fullPage,
|
|
133
|
+
colorScheme,
|
|
134
|
+
browsers: parsedBrowsers,
|
|
135
|
+
devices: parsedDevices,
|
|
136
|
+
cookies: cookie,
|
|
137
|
+
requestHeaders: header,
|
|
138
|
+
delay,
|
|
139
|
+
waitForSelectors: waitFor,
|
|
140
|
+
localStorage,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
if (error instanceof zod_1.ZodError) {
|
|
145
|
+
output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
throw error;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../utils");
|
|
7
|
+
const commander_1 = require("commander");
|
|
8
|
+
const texts_1 = require("../texts");
|
|
9
|
+
const output_1 = __importDefault(require("../output"));
|
|
10
|
+
const node_zlib_1 = require("node:zlib");
|
|
11
|
+
const promises_1 = require("node:stream/promises");
|
|
12
|
+
const node_fs_1 = require("node:fs");
|
|
13
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
14
|
+
const promises_2 = require("node:fs/promises");
|
|
15
|
+
const commandCrawl = (0, utils_1.newCommand)('crawl', texts_1.DESC_COMMAND_CRAWL);
|
|
16
|
+
commandCrawl.argument('[url]', texts_1.OPTION_CRAWL_URL);
|
|
17
|
+
commandCrawl.option('--follow', texts_1.OPTION_CRAWL_FOLLOW, false);
|
|
18
|
+
commandCrawl.option('--respectRobots', texts_1.OPTION_COMPARE_RESPECT_ROBOTS, false);
|
|
19
|
+
commandCrawl.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_CRAWL_OUTPUT_TYPE).choices([
|
|
20
|
+
'jpeg',
|
|
21
|
+
'png',
|
|
22
|
+
'md',
|
|
23
|
+
'html',
|
|
24
|
+
]));
|
|
25
|
+
commandCrawl.option('--outputTypes <json>', texts_1.OPTION_CRAWL_OUTPUT_TYPES);
|
|
26
|
+
commandCrawl.option('--quality <quality>', texts_1.OPTION_CRAWL_QUALITY);
|
|
27
|
+
commandCrawl.option('--fullPage', texts_1.OPTION_CRAWL_FULL_PAGE, false);
|
|
28
|
+
commandCrawl.option('--cssSelector <selector>', texts_1.OPTION_CRAWL_CSS_SELECTOR);
|
|
29
|
+
commandCrawl.option('--xpathSelector <selector>', texts_1.OPTION_CRAWL_XPATH_SELECTOR);
|
|
30
|
+
commandCrawl.addOption(new commander_1.Option('--colorScheme <scheme>', texts_1.OPTION_CRAWL_COLOR_SCHEME).choices([
|
|
31
|
+
'LIGHT',
|
|
32
|
+
'DARK',
|
|
33
|
+
'LIGHT_AND_DARK',
|
|
34
|
+
]));
|
|
35
|
+
commandCrawl.option('--browsers <browsers>', texts_1.OPTION_CRAWL_BROWSERS);
|
|
36
|
+
commandCrawl.option('--devices <devices>', texts_1.OPTION_CRAWL_DEVICES);
|
|
37
|
+
commandCrawl.option('--waitFor <waitFors...>', texts_1.OPTION_COMPARE_WAIT_FOR);
|
|
38
|
+
commandCrawl.option('--cookie <cookies...>', texts_1.OPTION_COMPARE_COOKIE);
|
|
39
|
+
commandCrawl.option('--header <headers...>', texts_1.OPTION_COMPARE_HEADER);
|
|
40
|
+
commandCrawl.option('--localStorage <items...>', texts_1.OPTION_CRAWL_LOCAL_STORAGE);
|
|
41
|
+
commandCrawl.option('--delay <delay>', texts_1.OPTION_CRAWL_DELAY, '0');
|
|
42
|
+
commandCrawl.option('--outputDir <dir>', texts_1.OPTION_CRAWL_OUTPUT_DIR, '.');
|
|
43
|
+
commandCrawl.action(async (inputUrl, options) => {
|
|
44
|
+
const { checkToken } = require('../visualTest/validation');
|
|
45
|
+
const { downloadScrapPackage, sendScrap } = require('../visualTest/requests');
|
|
46
|
+
const { setCiAndCommitInfo } = require('../visualTest/context');
|
|
47
|
+
const { validateInputAndOptions } = require('./crawl/validation');
|
|
48
|
+
const { getCiAndGitInfo } = require('@buddy-works/ci-info');
|
|
49
|
+
const tar = require('tar-stream');
|
|
50
|
+
if (!checkToken('scrape')) {
|
|
51
|
+
output_1.default.exitError(texts_1.ERR_MISSING_CRAWL_TOKEN);
|
|
52
|
+
}
|
|
53
|
+
const { url, follow, respectRobots, outputTypes, outputDir, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage, } = validateInputAndOptions(inputUrl, options);
|
|
54
|
+
try {
|
|
55
|
+
const ciAndGitInfo = await getCiAndGitInfo({});
|
|
56
|
+
setCiAndCommitInfo(ciAndGitInfo);
|
|
57
|
+
const { buildId } = await sendScrap(url, follow, respectRobots, outputTypes, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage);
|
|
58
|
+
const status = await watchSessionStatus(buildId);
|
|
59
|
+
if (!status.ok) {
|
|
60
|
+
output_1.default.exitError(`Crawl session failed: ${status.error}`);
|
|
61
|
+
}
|
|
62
|
+
output_1.default.normal('Downloading crawl package');
|
|
63
|
+
const scrapPackageStream = await downloadScrapPackage(buildId);
|
|
64
|
+
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
65
|
+
const unpack = tar.extract();
|
|
66
|
+
unpack.on('entry', async (header, stream, next) => {
|
|
67
|
+
const currentDir = process.cwd();
|
|
68
|
+
const preparedOutputDir = outputDir.startsWith('.')
|
|
69
|
+
? node_path_1.default.join(currentDir, outputDir)
|
|
70
|
+
: outputDir;
|
|
71
|
+
const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
|
|
72
|
+
try {
|
|
73
|
+
if (header.type === 'file') {
|
|
74
|
+
await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
|
|
75
|
+
const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
|
|
76
|
+
await (0, promises_1.pipeline)(stream, fileWriteStream);
|
|
77
|
+
next();
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
stream.resume();
|
|
81
|
+
next();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (entryError) {
|
|
85
|
+
output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
|
|
86
|
+
next(entryError);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
|
|
90
|
+
output_1.default.exitSuccess('Downloading crawl package finished');
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
output_1.default.exitError(`${error}`);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
async function watchSessionStatus(buildId) {
|
|
97
|
+
const { connectToScrapSession } = require('../visualTest/requests');
|
|
98
|
+
return new Promise((resolve) => {
|
|
99
|
+
const eventSource = connectToScrapSession(buildId);
|
|
100
|
+
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
101
|
+
const data = JSON.parse(event.data);
|
|
102
|
+
if (data.status === 'STARTED') {
|
|
103
|
+
output_1.default.normal('Crawl session started');
|
|
104
|
+
}
|
|
105
|
+
else if (data.status === 'GATHER_URLS_COMPLETED') {
|
|
106
|
+
output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
|
|
107
|
+
}
|
|
108
|
+
else if (data.status === 'GATHER_URLS_FAILED') {
|
|
109
|
+
output_1.default.error('Gathering URLs failed');
|
|
110
|
+
}
|
|
111
|
+
else if (data.status === 'SCRAPE_URL_COMPLETED') {
|
|
112
|
+
output_1.default.normal(`Crawling ${data.text} completed`);
|
|
113
|
+
}
|
|
114
|
+
else if (data.status === 'SCRAPE_URL_FAILED') {
|
|
115
|
+
output_1.default.error(`Crawling ${data.text} failed`);
|
|
116
|
+
}
|
|
117
|
+
else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
|
|
118
|
+
output_1.default.normal('Package created');
|
|
119
|
+
}
|
|
120
|
+
else if (data.status === 'CREATE_PACKAGE_FAILED') {
|
|
121
|
+
output_1.default.error('Package creation failed');
|
|
122
|
+
}
|
|
123
|
+
else if (data.status === 'ERROR') {
|
|
124
|
+
eventSource.close();
|
|
125
|
+
resolve({ ok: false, error: data.text });
|
|
126
|
+
}
|
|
127
|
+
else if (data.status === 'FINISHED') {
|
|
128
|
+
eventSource.close();
|
|
129
|
+
output_1.default.normal('Crawl session finished');
|
|
130
|
+
resolve({ ok: true });
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
eventSource.addEventListener('error', (event) => {
|
|
134
|
+
if (event.code) {
|
|
135
|
+
eventSource.close();
|
|
136
|
+
if (event.code === 410) {
|
|
137
|
+
output_1.default.normal('Crawl session finished');
|
|
138
|
+
}
|
|
139
|
+
resolve({ ok: event.code === 410, error: event.code });
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
exports.default = commandCrawl;
|
|
@@ -40,8 +40,7 @@ async function oauthServer(api, clientId, clientSecret) {
|
|
|
40
40
|
const urlCode = url.searchParams.get('code');
|
|
41
41
|
if (!urlCode) {
|
|
42
42
|
res.end(texts_1.ERR_LOGIN_HTTP_FAILED);
|
|
43
|
-
|
|
44
|
-
output_1.default.exitError(texts_1.ERR_LOGIN_HTTP_FAILED);
|
|
43
|
+
return;
|
|
45
44
|
}
|
|
46
45
|
const client = new ApiClient(new URL(`https://${api}`));
|
|
47
46
|
try {
|
|
@@ -103,24 +103,30 @@ function validateInputAndOptions(input, options) {
|
|
|
103
103
|
let parsedDevices;
|
|
104
104
|
if (rawDevices) {
|
|
105
105
|
try {
|
|
106
|
-
const
|
|
107
|
-
.
|
|
108
|
-
.
|
|
109
|
-
.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
106
|
+
const viewportSchema = zod_1.z
|
|
107
|
+
.object({
|
|
108
|
+
width: zod_1.z.number().positive(),
|
|
109
|
+
height: zod_1.z.number().positive(),
|
|
110
|
+
})
|
|
111
|
+
.strict();
|
|
112
|
+
const deviceSchema = zod_1.z.array(zod_1.z
|
|
113
|
+
.object({
|
|
114
|
+
viewport: viewportSchema,
|
|
115
|
+
screen: viewportSchema.optional(),
|
|
116
|
+
devicePixelRatio: zod_1.z.number().positive().optional(),
|
|
117
|
+
isMobile: zod_1.z.boolean().optional(),
|
|
118
|
+
})
|
|
119
|
+
.strict()
|
|
120
|
+
.transform((data) => ({
|
|
121
|
+
viewport: data.viewport,
|
|
122
|
+
screen: data.screen ?? data.viewport,
|
|
123
|
+
devicePixelRatio: data.devicePixelRatio ?? 1,
|
|
124
|
+
isMobile: data.isMobile ?? false,
|
|
125
|
+
})));
|
|
120
126
|
parsedDevices = deviceSchema.parse(JSON.parse(rawDevices));
|
|
121
127
|
}
|
|
122
128
|
catch {
|
|
123
|
-
output_1.default.exitError('Invalid --devices value. Use JSON array, e.g. --devices \'[{"viewport":"
|
|
129
|
+
output_1.default.exitError('Invalid --devices value. Use JSON array, e.g. --devices \'[{"viewport":{"width":1920,"height":1080}}]\'');
|
|
124
130
|
}
|
|
125
131
|
}
|
|
126
132
|
return {
|
|
@@ -6,12 +6,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
const utils_1 = require("../utils");
|
|
7
7
|
const commander_1 = require("commander");
|
|
8
8
|
const texts_1 = require("../texts");
|
|
9
|
+
const validation_1 = require("../visualTest/validation");
|
|
9
10
|
const output_1 = __importDefault(require("../output"));
|
|
11
|
+
const requests_1 = require("../visualTest/requests");
|
|
10
12
|
const node_zlib_1 = require("node:zlib");
|
|
13
|
+
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
11
14
|
const promises_1 = require("node:stream/promises");
|
|
12
15
|
const node_fs_1 = require("node:fs");
|
|
13
16
|
const node_path_1 = __importDefault(require("node:path"));
|
|
14
17
|
const promises_2 = require("node:fs/promises");
|
|
18
|
+
const context_1 = require("../visualTest/context");
|
|
19
|
+
const ci_info_1 = require("@buddy-works/ci-info");
|
|
20
|
+
const validation_2 = require("./scrape/validation");
|
|
15
21
|
const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
|
|
16
22
|
commandScrape.argument('[url]', texts_1.OPTION_SCRAPE_URL);
|
|
17
23
|
commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
|
|
@@ -41,28 +47,22 @@ commandScrape.option('--localStorage <items...>', texts_1.OPTION_SCRAPE_LOCAL_ST
|
|
|
41
47
|
commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
|
|
42
48
|
commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
|
|
43
49
|
commandScrape.action(async (inputUrl, options) => {
|
|
44
|
-
|
|
45
|
-
const { downloadScrapPackage, sendScrap } = require('../visualTest/requests');
|
|
46
|
-
const { setCiAndCommitInfo } = require('../visualTest/context');
|
|
47
|
-
const { validateInputAndOptions } = require('./scrape/validation');
|
|
48
|
-
const { getCiAndGitInfo } = require('@buddy-works/ci-info');
|
|
49
|
-
const tar = require('tar-stream');
|
|
50
|
-
if (!checkToken('scrape')) {
|
|
50
|
+
if (!(0, validation_1.checkToken)('scrape')) {
|
|
51
51
|
output_1.default.exitError(texts_1.ERR_MISSING_SCRAPE_TOKEN);
|
|
52
52
|
}
|
|
53
|
-
const { url, follow, respectRobots, outputTypes, outputDir, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage, } = validateInputAndOptions(inputUrl, options);
|
|
53
|
+
const { url, follow, respectRobots, outputTypes, outputDir, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage, } = (0, validation_2.validateInputAndOptions)(inputUrl, options);
|
|
54
54
|
try {
|
|
55
|
-
const ciAndGitInfo = await getCiAndGitInfo({});
|
|
56
|
-
setCiAndCommitInfo(ciAndGitInfo);
|
|
57
|
-
const { buildId } = await sendScrap(url, follow, respectRobots, outputTypes, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage);
|
|
55
|
+
const ciAndGitInfo = await (0, ci_info_1.getCiAndGitInfo)({});
|
|
56
|
+
(0, context_1.setCiAndCommitInfo)(ciAndGitInfo);
|
|
57
|
+
const { buildId } = await (0, requests_1.sendScrap)(url, follow, respectRobots, outputTypes, fullPage, colorScheme, browsers, devices, cookies, requestHeaders, delay, waitForSelectors, localStorage);
|
|
58
58
|
const status = await watchSessionStatus(buildId);
|
|
59
59
|
if (!status.ok) {
|
|
60
60
|
output_1.default.exitError(`Scrape session failed: ${status.error}`);
|
|
61
61
|
}
|
|
62
62
|
output_1.default.normal('Downloading scrape package');
|
|
63
|
-
const scrapPackageStream = await downloadScrapPackage(buildId);
|
|
63
|
+
const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
|
|
64
64
|
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
65
|
-
const unpack =
|
|
65
|
+
const unpack = tar_stream_1.default.extract();
|
|
66
66
|
unpack.on('entry', async (header, stream, next) => {
|
|
67
67
|
const currentDir = process.cwd();
|
|
68
68
|
const preparedOutputDir = outputDir.startsWith('.')
|
|
@@ -94,9 +94,8 @@ commandScrape.action(async (inputUrl, options) => {
|
|
|
94
94
|
}
|
|
95
95
|
});
|
|
96
96
|
async function watchSessionStatus(buildId) {
|
|
97
|
-
const { connectToScrapSession } = require('../visualTest/requests');
|
|
98
97
|
return new Promise((resolve) => {
|
|
99
|
-
const eventSource = connectToScrapSession(buildId);
|
|
98
|
+
const eventSource = (0, requests_1.connectToScrapSession)(buildId);
|
|
100
99
|
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
101
100
|
const data = JSON.parse(event.data);
|
|
102
101
|
if (data.status === 'STARTED') {
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.validateOptions = validateOptions;
|
|
7
|
+
exports.checkIfMinimalOptionsAreProvided = checkIfMinimalOptionsAreProvided;
|
|
8
|
+
const zod_1 = require("zod");
|
|
9
|
+
const output_1 = __importDefault(require("../../../output"));
|
|
10
|
+
const validation_1 = require("../visual/shared/validation");
|
|
11
|
+
const optionsSchema = zod_1.z.object({
|
|
12
|
+
urls: zod_1.z.string().optional(),
|
|
13
|
+
sitemap: zod_1.z.string().optional(),
|
|
14
|
+
urlsFile: zod_1.z.string().optional(),
|
|
15
|
+
follow: zod_1.z.boolean(),
|
|
16
|
+
respectRobots: zod_1.z.boolean(),
|
|
17
|
+
ignore: zod_1.z
|
|
18
|
+
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?(?:(CSS|XPATH))=(.+)$/, {
|
|
19
|
+
message: "Ignore option must follow pattern '[scope::]type=value' where type must be CSS or XPATH (scope is optional)",
|
|
20
|
+
}))
|
|
21
|
+
.optional()
|
|
22
|
+
.transform(validation_1.parseScopedSelector),
|
|
23
|
+
cookie: validation_1.cookieSchema,
|
|
24
|
+
header: validation_1.headerSchema,
|
|
25
|
+
delay: zod_1.z
|
|
26
|
+
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?(\d+)$/, {
|
|
27
|
+
message: "Delay option must follow pattern '[scope::]milliseconds' (scope is optional)",
|
|
28
|
+
}))
|
|
29
|
+
.optional()
|
|
30
|
+
.transform((value) => value?.map((v) => {
|
|
31
|
+
if (v.includes('::')) {
|
|
32
|
+
const [scope, milliseconds] = v.split('::');
|
|
33
|
+
return { scope: scope.trim(), value: Number(milliseconds) };
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
return { scope: validation_1.DEFAULT_SCOPE, value: Number(v) };
|
|
37
|
+
}
|
|
38
|
+
})),
|
|
39
|
+
waitFor: validation_1.waitForSchema,
|
|
40
|
+
ignoreUrls: zod_1.z.array(zod_1.z.string()).optional(),
|
|
41
|
+
dryRun: zod_1.z.boolean().optional(),
|
|
42
|
+
});
|
|
43
|
+
function validateOptions(options) {
|
|
44
|
+
try {
|
|
45
|
+
const validatedOptions = optionsSchema.parse(options);
|
|
46
|
+
return validatedOptions;
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (error instanceof zod_1.ZodError) {
|
|
50
|
+
output_1.default.exitError(error.errors.map((e) => e.message).join(', '));
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function checkIfMinimalOptionsAreProvided(options) {
|
|
58
|
+
return !!options.urls || !!options.sitemap || !!options.urlsFile;
|
|
59
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../../utils");
|
|
7
|
+
const texts_1 = require("../../texts");
|
|
8
|
+
const output_1 = __importDefault(require("../../output"));
|
|
9
|
+
const node_fs_1 = require("node:fs");
|
|
10
|
+
const commandCapture = (0, utils_1.newCommand)('capture', texts_1.DESC_COMMAND_CAPTURE);
|
|
11
|
+
commandCapture.option('--urls <urls>', texts_1.OPTION_COMPARE_URLS);
|
|
12
|
+
commandCapture.option('--sitemap <sitemap>', texts_1.OPTION_COMPARE_SITEMAP);
|
|
13
|
+
commandCapture.option('--urlsFile <urlsFile>', texts_1.OPTION_COMPARE_URLS_FILE);
|
|
14
|
+
commandCapture.option('--dryRun', texts_1.OPTION_COMPARE_DRY_RUN);
|
|
15
|
+
commandCapture.option('--follow', texts_1.OPTION_COMPARE_FOLLOW, false);
|
|
16
|
+
commandCapture.option('--respectRobots', texts_1.OPTION_COMPARE_RESPECT_ROBOTS, false);
|
|
17
|
+
commandCapture.option('--ignoreUrls <ignoreUrls...>', texts_1.OPTION_COMPARE_IGNORE_URLS);
|
|
18
|
+
commandCapture.option('--ignore <ignores...>', texts_1.OPTION_COMPARE_IGNORE);
|
|
19
|
+
commandCapture.option('--cookie <cookies...>', texts_1.OPTION_COMPARE_COOKIE);
|
|
20
|
+
commandCapture.option('--header <headers...>', texts_1.OPTION_COMPARE_HEADER);
|
|
21
|
+
commandCapture.option('--delay <delays...>', texts_1.OPTION_COMPARE_DELAY);
|
|
22
|
+
commandCapture.option('--waitFor <waitFors...>', texts_1.OPTION_COMPARE_WAIT_FOR);
|
|
23
|
+
commandCapture.action(async (options) => {
|
|
24
|
+
const { getCiAndGitInfo, formattedCiInfo } = require('@buddy-works/ci-info');
|
|
25
|
+
const { checkToken } = require('../../visualTest/validation');
|
|
26
|
+
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
27
|
+
const { sendCompareLinks } = require('../../visualTest/requests');
|
|
28
|
+
const { checkIfMinimalOptionsAreProvided, validateOptions } = require('./capture/validation');
|
|
29
|
+
const { setCiAndCommitInfo } = require('../../visualTest/context');
|
|
30
|
+
const validatedOptions = validateOptions(options);
|
|
31
|
+
if (!checkToken('vt')) {
|
|
32
|
+
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
33
|
+
}
|
|
34
|
+
if (!checkIfMinimalOptionsAreProvided(validatedOptions)) {
|
|
35
|
+
output_1.default.exitError(texts_1.ERR_MISSING_URLS);
|
|
36
|
+
}
|
|
37
|
+
let urls = [];
|
|
38
|
+
let sitemapSource;
|
|
39
|
+
if (validatedOptions.urls) {
|
|
40
|
+
const urlsList = getUrlsFromUrlOption(validatedOptions.urls);
|
|
41
|
+
urls = urls.concat(urlsList);
|
|
42
|
+
}
|
|
43
|
+
if (validatedOptions.urlsFile) {
|
|
44
|
+
const urlsList = getUrlsFromUrlFile(validatedOptions.urlsFile);
|
|
45
|
+
urls = urls.concat(urlsList);
|
|
46
|
+
}
|
|
47
|
+
if (validatedOptions.sitemap) {
|
|
48
|
+
sitemapSource = addProtocolIfMissing(validatedOptions.sitemap);
|
|
49
|
+
}
|
|
50
|
+
const { filteredUrls, duplicates } = filterDuplicates(urls);
|
|
51
|
+
if (duplicates.length > 0) {
|
|
52
|
+
output_1.default.normal(`Detected ${duplicates.length} duplicated urls:`);
|
|
53
|
+
output_1.default.normal(duplicates.join('\n'));
|
|
54
|
+
}
|
|
55
|
+
if (validatedOptions.dryRun) {
|
|
56
|
+
output_1.default.exitSuccess(`List of urls:\n${filteredUrls.join('\n')}`);
|
|
57
|
+
}
|
|
58
|
+
else if (filteredUrls.length > 1) {
|
|
59
|
+
output_1.default.normal(`List of urls:\n${filteredUrls.join('\n')}`);
|
|
60
|
+
}
|
|
61
|
+
const ciAndGitInfo = await getCiAndGitInfo({});
|
|
62
|
+
output_1.default.normal(formattedCiInfo(ciAndGitInfo));
|
|
63
|
+
setCiAndCommitInfo(ciAndGitInfo);
|
|
64
|
+
try {
|
|
65
|
+
const { message } = await sendCompareLinks(filteredUrls, validatedOptions, sitemapSource);
|
|
66
|
+
output_1.default.exitSuccess(message);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
output_1.default.exitError(`${error}`);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
function getUrlsFromUrlOption(urls) {
|
|
73
|
+
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
74
|
+
return urls.split(',').map((url) => addProtocolIfMissing(url).trim());
|
|
75
|
+
}
|
|
76
|
+
function getUrlsFromUrlFile(urlsFile) {
|
|
77
|
+
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
78
|
+
const urlsFromFile = (0, node_fs_1.readFileSync)(urlsFile, 'utf-8');
|
|
79
|
+
return urlsFromFile
|
|
80
|
+
.split('\n')
|
|
81
|
+
.filter((url) => url.trim().length > 0)
|
|
82
|
+
.map((url) => addProtocolIfMissing(url).trim());
|
|
83
|
+
}
|
|
84
|
+
function filterDuplicates(urls) {
|
|
85
|
+
const seen = new Set();
|
|
86
|
+
const duplicates = new Set();
|
|
87
|
+
for (const url of urls) {
|
|
88
|
+
if (seen.has(url)) {
|
|
89
|
+
duplicates.add(url);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
seen.add(url);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
filteredUrls: Array.from(seen),
|
|
97
|
+
duplicates: Array.from(duplicates),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
exports.default = commandCapture;
|