bdy 1.25.1-master → 1.25.2-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +2 -6
- package/distTs/src/command/api/info.js +224 -255
- package/distTs/src/command/api/list.js +14 -17
- package/distTs/src/command/yaml/actions/detect.js +11 -2
- package/distTs/src/command/yaml/actions/info.js +22 -12
- package/distTs/src/command/yaml/actions/list.js +11 -20
- package/distTs/src/command/yaml/actions/schema.js +43 -87
- package/distTs/src/command/yaml/pipeline.js +9 -25
- package/distTs/src/command/yaml/render.js +83 -71
- package/distTs/src/command/yaml/schemaUtils.js +12 -50
- package/distTs/src/command/yaml/validate.js +32 -27
- package/distTs/src/diskCache.js +0 -1
- package/distTs/src/openapi.js +16 -68
- package/distTs/src/texts.js +7 -9
- package/package.json +2 -6
- package/distTs/README.md +0 -181
- package/distTs/src/autocomplete.js +0 -90
- package/distTs/src/command/autocomplete/install.js +0 -16
- package/distTs/src/command/autocomplete/uninstall.js +0 -16
- package/distTs/src/command/autocomplete.js +0 -14
- package/distTs/src/command/environment/resolve.js +0 -47
- package/distTs/src/command/package/create.js +0 -64
- package/distTs/src/command/package/delete.js +0 -38
- package/distTs/src/command/package/docker/login.js +0 -26
- package/distTs/src/command/package/docker.js +0 -11
- package/distTs/src/command/package/download.js +0 -138
- package/distTs/src/command/package/get.js +0 -36
- package/distTs/src/command/package/list.js +0 -31
- package/distTs/src/command/package/publish.js +0 -241
- package/distTs/src/command/package/version/delete.js +0 -39
- package/distTs/src/command/package/version/get.js +0 -45
- package/distTs/src/command/package/version/list.js +0 -47
- package/distTs/src/command/package/version.js +0 -16
- package/distTs/src/command/package.js +0 -26
- package/distTs/src/command/project/get.js +0 -18
- package/distTs/src/command/project/set.js +0 -31
- package/distTs/src/command/sandbox/get/yaml.js +0 -30
- package/distTs/src/command/scrape/validation.js +0 -154
- package/distTs/src/command/scrape.js +0 -144
- package/distTs/src/command/ut/upload.js +0 -86
- package/distTs/src/command/ut.js +0 -12
- package/distTs/src/command/vt/close.js +0 -27
- package/distTs/src/command/vt/compare/validation.js +0 -46
- package/distTs/src/command/vt/compare.js +0 -100
- package/distTs/src/command/vt/exec.js +0 -82
- package/distTs/src/command/vt/installBrowser.js +0 -20
- package/distTs/src/command/vt/scrape.js +0 -193
- package/distTs/src/command/vt/shared/validation.js +0 -145
- package/distTs/src/command/vt/storybook.js +0 -138
- package/distTs/src/command/vt.js +0 -20
- package/distTs/src/main.js +0 -190
- package/distTs/src/types/options.js +0 -2
- package/distTs/src/types/scrape.js +0 -2
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const output_1 = __importDefault(require("../../output"));
|
|
9
|
-
const commandVtClose = (0, utils_1.newCommand)('close', texts_1.DESC_COMMAND_VT_CLOSE);
|
|
10
|
-
commandVtClose.action(async () => {
|
|
11
|
-
const { checkBuildId, checkToken } = require('../../visualTest/validation');
|
|
12
|
-
const { closeSession } = require('../../visualTest/requests');
|
|
13
|
-
if (!checkToken('vt')) {
|
|
14
|
-
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
15
|
-
}
|
|
16
|
-
if (!checkBuildId()) {
|
|
17
|
-
output_1.default.exitError(texts_1.ERR_MISSING_BUILD_ID);
|
|
18
|
-
}
|
|
19
|
-
try {
|
|
20
|
-
const { message } = await closeSession();
|
|
21
|
-
output_1.default.exitNormal(message);
|
|
22
|
-
}
|
|
23
|
-
catch (error) {
|
|
24
|
-
output_1.default.exitError(`${error}`);
|
|
25
|
-
}
|
|
26
|
-
});
|
|
27
|
-
exports.default = commandVtClose;
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.validateOptions = validateOptions;
|
|
7
|
-
exports.checkIfMinimalOptionsAreProvided = checkIfMinimalOptionsAreProvided;
|
|
8
|
-
const zod_1 = require("zod");
|
|
9
|
-
const output_1 = __importDefault(require("../../../output"));
|
|
10
|
-
const validation_1 = require("../shared/validation");
|
|
11
|
-
const optionsSchema = zod_1.z.object({
|
|
12
|
-
urls: zod_1.z.string().optional(),
|
|
13
|
-
sitemap: zod_1.z.string().optional(),
|
|
14
|
-
urlsFile: zod_1.z.string().optional(),
|
|
15
|
-
follow: zod_1.z.boolean(),
|
|
16
|
-
respectRobots: zod_1.z.boolean(),
|
|
17
|
-
ignore: zod_1.z
|
|
18
|
-
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?(?:(CSS|XPATH))=(.+)$/, {
|
|
19
|
-
message: "Ignore option must follow pattern '[scope::]type=value' where type must be CSS or XPATH (scope is optional)",
|
|
20
|
-
}))
|
|
21
|
-
.optional()
|
|
22
|
-
.transform(validation_1.parseScopedSelector),
|
|
23
|
-
cookie: validation_1.cookieSchema,
|
|
24
|
-
header: validation_1.headerSchema,
|
|
25
|
-
delay: validation_1.delaySchema,
|
|
26
|
-
waitFor: validation_1.waitForSchema,
|
|
27
|
-
ignoreUrls: zod_1.z.array(zod_1.z.string()).optional(),
|
|
28
|
-
dryRun: zod_1.z.boolean().optional(),
|
|
29
|
-
});
|
|
30
|
-
function validateOptions(options) {
|
|
31
|
-
try {
|
|
32
|
-
const validatedOptions = optionsSchema.parse(options);
|
|
33
|
-
return validatedOptions;
|
|
34
|
-
}
|
|
35
|
-
catch (error) {
|
|
36
|
-
if (error instanceof zod_1.ZodError) {
|
|
37
|
-
output_1.default.exitError(error.errors.map((e) => e.message).join(', '));
|
|
38
|
-
}
|
|
39
|
-
else {
|
|
40
|
-
throw error;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
function checkIfMinimalOptionsAreProvided(options) {
|
|
45
|
-
return !!options.urls || !!options.sitemap || !!options.urlsFile;
|
|
46
|
-
}
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const output_1 = __importDefault(require("../../output"));
|
|
9
|
-
const node_fs_1 = require("node:fs");
|
|
10
|
-
const commandVtCompare = (0, utils_1.newCommand)('compare', texts_1.DESC_COMMAND_VT_COMPARE);
|
|
11
|
-
commandVtCompare.option('--urls <urls>', texts_1.OPTION_COMPARE_URLS);
|
|
12
|
-
commandVtCompare.option('--sitemap <sitemap>', texts_1.OPTION_COMPARE_SITEMAP);
|
|
13
|
-
commandVtCompare.option('--urlsFile <urlsFile>', texts_1.OPTION_COMPARE_URLS_FILE);
|
|
14
|
-
commandVtCompare.option('--dryRun', texts_1.OPTION_COMPARE_DRY_RUN);
|
|
15
|
-
commandVtCompare.option('--follow', texts_1.OPTION_COMPARE_FOLLOW, false);
|
|
16
|
-
commandVtCompare.option('--respectRobots', texts_1.OPTION_COMPARE_RESPECT_ROBOTS, false);
|
|
17
|
-
commandVtCompare.option('--ignoreUrls <ignoreUrls...>', texts_1.OPTION_COMPARE_IGNORE_URLS);
|
|
18
|
-
commandVtCompare.option('--ignore <ignores...>', texts_1.OPTION_COMPARE_IGNORE);
|
|
19
|
-
commandVtCompare.option('--cookie <cookies...>', texts_1.OPTION_COMPARE_COOKIE);
|
|
20
|
-
commandVtCompare.option('--header <headers...>', texts_1.OPTION_COMPARE_HEADER);
|
|
21
|
-
commandVtCompare.option('--delay <delays...>', texts_1.OPTION_COMPARE_DELAY);
|
|
22
|
-
commandVtCompare.option('--waitFor <waitFors...>', texts_1.OPTION_COMPARE_WAIT_FOR);
|
|
23
|
-
commandVtCompare.action(async (options) => {
|
|
24
|
-
const { getCiAndGitInfo, formattedCiInfo } = require('@buddy-works/ci-info');
|
|
25
|
-
const { checkToken } = require('../../visualTest/validation');
|
|
26
|
-
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
27
|
-
const { sendCompareLinks } = require('../../visualTest/requests');
|
|
28
|
-
const { checkIfMinimalOptionsAreProvided, validateOptions } = require('./compare/validation');
|
|
29
|
-
const { setCiAndCommitInfo } = require('../../visualTest/context');
|
|
30
|
-
const validatedOptions = validateOptions(options);
|
|
31
|
-
if (!checkToken('vt')) {
|
|
32
|
-
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
33
|
-
}
|
|
34
|
-
if (!checkIfMinimalOptionsAreProvided(validatedOptions)) {
|
|
35
|
-
output_1.default.exitError(texts_1.ERR_MISSING_URLS);
|
|
36
|
-
}
|
|
37
|
-
let urls = [];
|
|
38
|
-
let sitemapSource;
|
|
39
|
-
if (validatedOptions.urls) {
|
|
40
|
-
const urlsList = getUrlsFromUrlOption(validatedOptions.urls);
|
|
41
|
-
urls = urls.concat(urlsList);
|
|
42
|
-
}
|
|
43
|
-
if (validatedOptions.urlsFile) {
|
|
44
|
-
const urlsList = getUrlsFromUrlFile(validatedOptions.urlsFile);
|
|
45
|
-
urls = urls.concat(urlsList);
|
|
46
|
-
}
|
|
47
|
-
if (validatedOptions.sitemap) {
|
|
48
|
-
sitemapSource = addProtocolIfMissing(validatedOptions.sitemap);
|
|
49
|
-
}
|
|
50
|
-
const { filteredUrls, duplicates } = filterDuplicates(urls);
|
|
51
|
-
if (duplicates.length > 0) {
|
|
52
|
-
output_1.default.normal(`Detected ${duplicates.length} duplicated urls:`);
|
|
53
|
-
output_1.default.normal(duplicates.join('\n'));
|
|
54
|
-
}
|
|
55
|
-
if (validatedOptions.dryRun) {
|
|
56
|
-
output_1.default.exitSuccess(`List of urls:\n${filteredUrls.join('\n')}`);
|
|
57
|
-
}
|
|
58
|
-
else if (filteredUrls.length > 1) {
|
|
59
|
-
output_1.default.normal(`List of urls:\n${filteredUrls.join('\n')}`);
|
|
60
|
-
}
|
|
61
|
-
const ciAndGitInfo = await getCiAndGitInfo({});
|
|
62
|
-
output_1.default.normal(formattedCiInfo(ciAndGitInfo));
|
|
63
|
-
setCiAndCommitInfo(ciAndGitInfo);
|
|
64
|
-
try {
|
|
65
|
-
const { message } = await sendCompareLinks(filteredUrls, validatedOptions, sitemapSource);
|
|
66
|
-
output_1.default.exitSuccess(message);
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
output_1.default.exitError(`${error}`);
|
|
70
|
-
}
|
|
71
|
-
});
|
|
72
|
-
function getUrlsFromUrlOption(urls) {
|
|
73
|
-
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
74
|
-
return urls.split(',').map((url) => addProtocolIfMissing(url).trim());
|
|
75
|
-
}
|
|
76
|
-
function getUrlsFromUrlFile(urlsFile) {
|
|
77
|
-
const { addProtocolIfMissing } = require('../../visualTest/linkUtils');
|
|
78
|
-
const urlsFromFile = (0, node_fs_1.readFileSync)(urlsFile, 'utf-8');
|
|
79
|
-
return urlsFromFile
|
|
80
|
-
.split('\n')
|
|
81
|
-
.filter((url) => url.trim().length > 0)
|
|
82
|
-
.map((url) => addProtocolIfMissing(url).trim());
|
|
83
|
-
}
|
|
84
|
-
function filterDuplicates(urls) {
|
|
85
|
-
const seen = new Set();
|
|
86
|
-
const duplicates = new Set();
|
|
87
|
-
for (const url of urls) {
|
|
88
|
-
if (seen.has(url)) {
|
|
89
|
-
duplicates.add(url);
|
|
90
|
-
}
|
|
91
|
-
else {
|
|
92
|
-
seen.add(url);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
return {
|
|
96
|
-
filteredUrls: Array.from(seen),
|
|
97
|
-
duplicates: Array.from(duplicates),
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
exports.default = commandVtCompare;
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const output_1 = __importDefault(require("../../output"));
|
|
9
|
-
const commandVtExec = (0, utils_1.newCommand)('exec', texts_1.DESC_COMMAND_VT_EXEC);
|
|
10
|
-
commandVtExec.argument('<command>', texts_1.OPTION_EXEC_COMMAND);
|
|
11
|
-
commandVtExec.option('--skipDiscovery', texts_1.OPTION_EXEC_SKIP_DISCOVERY);
|
|
12
|
-
commandVtExec.option('--oneByOne', texts_1.OPTION_EXEC_ONE_BY_ONE);
|
|
13
|
-
commandVtExec.option('--parallel', texts_1.OPTION_EXEC_PARALLEL);
|
|
14
|
-
commandVtExec.action(async (command, options) => {
|
|
15
|
-
const which = require('which');
|
|
16
|
-
const { getCiAndGitInfo, formattedCiInfo } = require('@buddy-works/ci-info');
|
|
17
|
-
const { checkParallel, checkToken } = require('../../visualTest/validation');
|
|
18
|
-
const { getDefaultSettings } = require('../../visualTest/requests');
|
|
19
|
-
const { debug, setBrowserPath, setCiAndCommitInfo, setExecOptions } = require('../../visualTest/context');
|
|
20
|
-
const { createServer } = require('../../visualTest/server');
|
|
21
|
-
const { finishProcessingSnapshots, setDefaultSettings, showSessionLink } = require('../../visualTest/snapshots');
|
|
22
|
-
const { testExec } = require('../../visualTest/exec');
|
|
23
|
-
const { getBrowserPath } = require('../../visualTest/browser');
|
|
24
|
-
setExecOptions(options);
|
|
25
|
-
try {
|
|
26
|
-
const browserPath = await getBrowserPath();
|
|
27
|
-
setBrowserPath(browserPath);
|
|
28
|
-
}
|
|
29
|
-
catch (error) {
|
|
30
|
-
output_1.default.exitError(`${error}`);
|
|
31
|
-
}
|
|
32
|
-
if (!checkToken('vt')) {
|
|
33
|
-
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
34
|
-
}
|
|
35
|
-
if (!checkParallel()) {
|
|
36
|
-
output_1.default.exitError(texts_1.ERR_MISSING_BUILD_ID);
|
|
37
|
-
}
|
|
38
|
-
const app = await createServer();
|
|
39
|
-
const [mainCommand, ...mainCommandArguments] = command.split(' ');
|
|
40
|
-
output_1.default.normal((0, texts_1.LOG_RUNNING_EXEC_COMMAND)(`${mainCommand} ${[...mainCommandArguments].join(' ')}`));
|
|
41
|
-
const resolved = await which(mainCommand, { nothrow: true });
|
|
42
|
-
if (!resolved) {
|
|
43
|
-
output_1.default.exitError((0, texts_1.ERR_MISSING_EXEC_COMMAND)(`${mainCommand} ${[...mainCommandArguments].join(' ')}`));
|
|
44
|
-
}
|
|
45
|
-
try {
|
|
46
|
-
let t1, t11;
|
|
47
|
-
if (debug) {
|
|
48
|
-
t1 = performance.now();
|
|
49
|
-
}
|
|
50
|
-
const defaultSettings = await getDefaultSettings();
|
|
51
|
-
setDefaultSettings(defaultSettings);
|
|
52
|
-
const ciAndGitInfo = await getCiAndGitInfo({
|
|
53
|
-
baseBranch: defaultSettings.baseBranch,
|
|
54
|
-
logger: output_1.default.warning,
|
|
55
|
-
});
|
|
56
|
-
setCiAndCommitInfo(ciAndGitInfo);
|
|
57
|
-
output_1.default.normal(formattedCiInfo(ciAndGitInfo));
|
|
58
|
-
if (debug) {
|
|
59
|
-
t11 = performance.now();
|
|
60
|
-
}
|
|
61
|
-
const spawnedProcessExitCode = await testExec(mainCommand, [
|
|
62
|
-
...mainCommandArguments,
|
|
63
|
-
]);
|
|
64
|
-
if (debug && t11) {
|
|
65
|
-
const t22 = performance.now();
|
|
66
|
-
output_1.default.normal((0, texts_1.DEBUG_EXEC_TEST_COMMAND)(t22 - t11));
|
|
67
|
-
}
|
|
68
|
-
await app.close();
|
|
69
|
-
await finishProcessingSnapshots(spawnedProcessExitCode);
|
|
70
|
-
showSessionLink();
|
|
71
|
-
if (debug && t1) {
|
|
72
|
-
const t2 = performance.now();
|
|
73
|
-
output_1.default.normal((0, texts_1.DEBUG_EXEC_COMMAND)(t2 - t1));
|
|
74
|
-
}
|
|
75
|
-
process.exit(spawnedProcessExitCode);
|
|
76
|
-
}
|
|
77
|
-
catch (error) {
|
|
78
|
-
await app.close();
|
|
79
|
-
output_1.default.exitError(`${error}`);
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
exports.default = commandVtExec;
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const output_1 = __importDefault(require("../../output"));
|
|
9
|
-
const commandVtInstallBrowser = (0, utils_1.newCommand)('install-browser', texts_1.DESC_COMMAND_VT_INSTALL_BROWSER);
|
|
10
|
-
commandVtInstallBrowser.action(async () => {
|
|
11
|
-
try {
|
|
12
|
-
const { installBrowser } = require('../../visualTest/browser');
|
|
13
|
-
await installBrowser();
|
|
14
|
-
output_1.default.exitNormal('');
|
|
15
|
-
}
|
|
16
|
-
catch (error) {
|
|
17
|
-
output_1.default.exitError(`${error}`);
|
|
18
|
-
}
|
|
19
|
-
});
|
|
20
|
-
exports.default = commandVtInstallBrowser;
|
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const commander_1 = require("commander");
|
|
8
|
-
const texts_1 = require("../../texts");
|
|
9
|
-
const validation_1 = require("../../visualTest/validation");
|
|
10
|
-
const output_1 = __importDefault(require("../../output"));
|
|
11
|
-
const requests_1 = require("../../visualTest/requests");
|
|
12
|
-
const zod_1 = require("zod");
|
|
13
|
-
const node_zlib_1 = require("node:zlib");
|
|
14
|
-
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
15
|
-
const promises_1 = require("node:stream/promises");
|
|
16
|
-
const node_fs_1 = require("node:fs");
|
|
17
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
-
const promises_2 = require("node:fs/promises");
|
|
19
|
-
const commandScrape = (0, utils_1.newCommand)('scrape', texts_1.DESC_COMMAND_VT_SCRAPE);
|
|
20
|
-
commandScrape.argument('<url>', texts_1.OPTION_SCRAPE_URL);
|
|
21
|
-
commandScrape.option('--follow', texts_1.OPTION_SCRAPE_FOLLOW, false);
|
|
22
|
-
commandScrape.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAPE_OUTPUT_TYPE)
|
|
23
|
-
.choices(['jpeg', 'png', 'md', 'html'])
|
|
24
|
-
.makeOptionMandatory());
|
|
25
|
-
commandScrape.option('--quality <quality>', texts_1.OPTION_SCRAPE_QUALITY);
|
|
26
|
-
commandScrape.option('--fullPage', texts_1.OPTION_SCRAPE_FULL_PAGE, false);
|
|
27
|
-
commandScrape.option('--cssSelector <selector>', texts_1.OPTION_SCRAPE_CSS_SELECTOR);
|
|
28
|
-
commandScrape.option('--xpathSelector <selector>', texts_1.OPTION_SCRAPE_XPATH_SELECTOR);
|
|
29
|
-
commandScrape.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAPE_BROWSER)
|
|
30
|
-
.choices(['chrome', 'firefox', 'safari'])
|
|
31
|
-
.default('chrome'));
|
|
32
|
-
commandScrape.option('--viewport <viewport>', texts_1.OPTION_SCRAPE_VIEWPORT, '1920x1080');
|
|
33
|
-
commandScrape.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAPE_DEVICE_PIXEL_RATIO, '1');
|
|
34
|
-
commandScrape.option('--waitForElement <selector>', texts_1.OPTION_SCRAPE_WAIT_FOR_ELEMENT);
|
|
35
|
-
commandScrape.option('--darkMode', texts_1.OPTION_SCRAPE_DARK_MODE, false);
|
|
36
|
-
commandScrape.option('--delay <delay>', texts_1.OPTION_SCRAPE_DELAY, '0');
|
|
37
|
-
commandScrape.option('--outputDir <dir>', texts_1.OPTION_SCRAPE_OUTPUT_DIR, '.');
|
|
38
|
-
commandScrape.action(async (inputUrl, options) => {
|
|
39
|
-
if (!(0, validation_1.checkToken)()) {
|
|
40
|
-
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
41
|
-
}
|
|
42
|
-
const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
|
|
43
|
-
try {
|
|
44
|
-
const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
|
|
45
|
-
output_1.default.normal('Starting scrape session');
|
|
46
|
-
const status = await watchSessionStatus(buildId);
|
|
47
|
-
if (!status.ok) {
|
|
48
|
-
output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
|
|
49
|
-
}
|
|
50
|
-
output_1.default.normal('Downloading scrape package');
|
|
51
|
-
const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
|
|
52
|
-
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
53
|
-
const unpack = tar_stream_1.default.extract();
|
|
54
|
-
unpack.on('entry', async (header, stream, next) => {
|
|
55
|
-
const currentDir = process.cwd();
|
|
56
|
-
const preparedOutputDir = outputDir.startsWith('.')
|
|
57
|
-
? node_path_1.default.join(currentDir, outputDir)
|
|
58
|
-
: outputDir;
|
|
59
|
-
const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
|
|
60
|
-
try {
|
|
61
|
-
if (header.type === 'file') {
|
|
62
|
-
await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
|
|
63
|
-
const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
|
|
64
|
-
await (0, promises_1.pipeline)(stream, fileWriteStream);
|
|
65
|
-
next();
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
stream.resume();
|
|
69
|
-
next();
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
catch (entryError) {
|
|
73
|
-
output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
|
|
74
|
-
next(entryError);
|
|
75
|
-
}
|
|
76
|
-
});
|
|
77
|
-
await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
|
|
78
|
-
output_1.default.exitSuccess('Downloading scrape package finished');
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
output_1.default.exitError(`${error}`);
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
function validateInputAndOptions(input, options) {
|
|
85
|
-
const urlSchema = zod_1.z.string().url();
|
|
86
|
-
const optionsSchema = zod_1.z.object({
|
|
87
|
-
follow: zod_1.z.boolean(),
|
|
88
|
-
outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
|
|
89
|
-
quality: zod_1.z.coerce.number().min(1).max(100).optional(),
|
|
90
|
-
outputDir: zod_1.z.string().default('.'),
|
|
91
|
-
fullPage: zod_1.z.boolean().optional(),
|
|
92
|
-
cssSelector: zod_1.z.string().optional(),
|
|
93
|
-
xpathSelector: zod_1.z.string().optional(),
|
|
94
|
-
browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
|
|
95
|
-
viewport: zod_1.z
|
|
96
|
-
.string()
|
|
97
|
-
.refine((value) => {
|
|
98
|
-
const [width, height] = value.split('x');
|
|
99
|
-
return (width &&
|
|
100
|
-
height &&
|
|
101
|
-
!isNaN(Number(width)) &&
|
|
102
|
-
!isNaN(Number(height)) &&
|
|
103
|
-
Number(width) > 0 &&
|
|
104
|
-
Number(height) > 0);
|
|
105
|
-
}, 'Invalid viewport format, example: 1920x1080')
|
|
106
|
-
.transform((value) => {
|
|
107
|
-
const [width, height] = value.split('x');
|
|
108
|
-
return {
|
|
109
|
-
width: Number(width),
|
|
110
|
-
height: Number(height),
|
|
111
|
-
};
|
|
112
|
-
}),
|
|
113
|
-
devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
|
|
114
|
-
darkMode: zod_1.z.boolean(),
|
|
115
|
-
delay: zod_1.z.coerce.number().min(0).max(10000),
|
|
116
|
-
waitForElement: zod_1.z.string().optional(),
|
|
117
|
-
});
|
|
118
|
-
try {
|
|
119
|
-
const url = urlSchema.parse(input);
|
|
120
|
-
const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
|
|
121
|
-
if (typeof quality === 'number' && outputType !== 'jpeg') {
|
|
122
|
-
output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
|
|
123
|
-
}
|
|
124
|
-
if (cssSelector && xpathSelector) {
|
|
125
|
-
output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
|
|
126
|
-
}
|
|
127
|
-
return {
|
|
128
|
-
url,
|
|
129
|
-
follow,
|
|
130
|
-
outputType,
|
|
131
|
-
quality,
|
|
132
|
-
outputDir,
|
|
133
|
-
fullPage,
|
|
134
|
-
cssSelector,
|
|
135
|
-
xpathSelector,
|
|
136
|
-
browser,
|
|
137
|
-
viewport,
|
|
138
|
-
devicePixelRatio,
|
|
139
|
-
darkMode,
|
|
140
|
-
delay,
|
|
141
|
-
waitForElement,
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
catch (error) {
|
|
145
|
-
if (error instanceof zod_1.ZodError) {
|
|
146
|
-
output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
|
|
147
|
-
}
|
|
148
|
-
else {
|
|
149
|
-
throw error;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
async function watchSessionStatus(buildId) {
|
|
154
|
-
return new Promise((resolve) => {
|
|
155
|
-
const eventSource = (0, requests_1.connectToScrapSession)(buildId);
|
|
156
|
-
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
157
|
-
const data = JSON.parse(event.data);
|
|
158
|
-
if (data.status === 'GATHER_URLS_COMPLETED') {
|
|
159
|
-
output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
|
|
160
|
-
}
|
|
161
|
-
else if (data.status === 'GATHER_URLS_FAILED') {
|
|
162
|
-
output_1.default.error('Gathering URLs failed');
|
|
163
|
-
}
|
|
164
|
-
else if (data.status === 'SCRAPE_URL_COMPLETED') {
|
|
165
|
-
output_1.default.normal(`Scraping ${data.text} completed`);
|
|
166
|
-
}
|
|
167
|
-
else if (data.status === 'SCRAPE_URL_FAILED') {
|
|
168
|
-
output_1.default.error(`Scraping ${data.text} failed`);
|
|
169
|
-
}
|
|
170
|
-
else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
|
|
171
|
-
output_1.default.normal('Package created');
|
|
172
|
-
}
|
|
173
|
-
else if (data.status === 'CREATE_PACKAGE_FAILED') {
|
|
174
|
-
output_1.default.error('Package creation failed');
|
|
175
|
-
}
|
|
176
|
-
else if (data.status === 'FINISHED') {
|
|
177
|
-
eventSource.close();
|
|
178
|
-
output_1.default.normal('Scrape session finished');
|
|
179
|
-
resolve({ ok: true });
|
|
180
|
-
}
|
|
181
|
-
});
|
|
182
|
-
eventSource.addEventListener('error', (event) => {
|
|
183
|
-
if (event.code) {
|
|
184
|
-
eventSource.close();
|
|
185
|
-
if (event.code === 410) {
|
|
186
|
-
output_1.default.normal('Scrape session finished');
|
|
187
|
-
}
|
|
188
|
-
resolve({ ok: event.code === 410, error: event.code });
|
|
189
|
-
}
|
|
190
|
-
});
|
|
191
|
-
});
|
|
192
|
-
}
|
|
193
|
-
exports.default = commandScrape;
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.waitForSchema = exports.delaySchema = exports.headerSchema = exports.cookieSchema = exports.DEFAULT_SCOPE = void 0;
|
|
7
|
-
exports.parseScopedSelector = parseScopedSelector;
|
|
8
|
-
exports.parseScopedKeyValue = parseScopedKeyValue;
|
|
9
|
-
const zod_1 = require("zod");
|
|
10
|
-
const output_1 = __importDefault(require("../../../output"));
|
|
11
|
-
exports.DEFAULT_SCOPE = '**';
|
|
12
|
-
function parseScopedSelector(value) {
|
|
13
|
-
return value?.map((v) => {
|
|
14
|
-
let scope, type, selectorValue;
|
|
15
|
-
if (v.includes('::CSS=') || v.includes('::XPATH=')) {
|
|
16
|
-
const [scopePart, ...rest] = v.split('::');
|
|
17
|
-
const [typePart, ...valuePart] = rest.join('::').split('=');
|
|
18
|
-
type = typePart;
|
|
19
|
-
selectorValue = valuePart.join('=');
|
|
20
|
-
scope = scopePart;
|
|
21
|
-
}
|
|
22
|
-
else {
|
|
23
|
-
const [typePart, ...valuePart] = v.split('=');
|
|
24
|
-
type = typePart;
|
|
25
|
-
selectorValue = valuePart.join('=');
|
|
26
|
-
scope = exports.DEFAULT_SCOPE;
|
|
27
|
-
}
|
|
28
|
-
return { scope, type, value: selectorValue };
|
|
29
|
-
});
|
|
30
|
-
}
|
|
31
|
-
function parseScopedKeyValue(rawValue) {
|
|
32
|
-
let scope = exports.DEFAULT_SCOPE;
|
|
33
|
-
let keyValue = rawValue;
|
|
34
|
-
const scopeSeparatorIndex = rawValue.indexOf('::');
|
|
35
|
-
if (scopeSeparatorIndex >= 0) {
|
|
36
|
-
scope = rawValue.slice(0, scopeSeparatorIndex).trim();
|
|
37
|
-
keyValue = rawValue.slice(scopeSeparatorIndex + 2).trim();
|
|
38
|
-
}
|
|
39
|
-
const equalSignIndex = keyValue.indexOf('=');
|
|
40
|
-
if (equalSignIndex <= 0) {
|
|
41
|
-
output_1.default.exitError("Option value must follow pattern '[scope::]key=value' (scope is optional)");
|
|
42
|
-
}
|
|
43
|
-
const key = keyValue.slice(0, equalSignIndex).trim();
|
|
44
|
-
const value = keyValue.slice(equalSignIndex + 1).trim();
|
|
45
|
-
if (!key) {
|
|
46
|
-
output_1.default.exitError('Option key cannot be empty');
|
|
47
|
-
}
|
|
48
|
-
return { scope, key, value };
|
|
49
|
-
}
|
|
50
|
-
exports.cookieSchema = zod_1.z
|
|
51
|
-
.array(zod_1.z
|
|
52
|
-
.string()
|
|
53
|
-
.max(4096, {
|
|
54
|
-
message: 'Cookie must be less than 4096 characters',
|
|
55
|
-
})
|
|
56
|
-
.regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
|
|
57
|
-
message: "Cookie option must follow pattern '[scope::]key=value[;attribute]' (scope is optional)",
|
|
58
|
-
}))
|
|
59
|
-
.optional()
|
|
60
|
-
.transform((value) => value?.map((v) => {
|
|
61
|
-
let scope = exports.DEFAULT_SCOPE;
|
|
62
|
-
let cookieValue = v;
|
|
63
|
-
if (v.includes('::')) {
|
|
64
|
-
const [scopePart, valuePart] = v.split('::');
|
|
65
|
-
scope = scopePart.trim();
|
|
66
|
-
cookieValue = valuePart.trim();
|
|
67
|
-
}
|
|
68
|
-
const cookieParts = cookieValue.split(';').map((part) => part.trim());
|
|
69
|
-
const mainPart = cookieParts[0];
|
|
70
|
-
const firstEqualSignIndex = mainPart.indexOf('=');
|
|
71
|
-
const key = mainPart.slice(0, firstEqualSignIndex).trim();
|
|
72
|
-
const value = mainPart.slice(firstEqualSignIndex + 1).trim();
|
|
73
|
-
const cookie = {
|
|
74
|
-
scope,
|
|
75
|
-
key,
|
|
76
|
-
value,
|
|
77
|
-
httpOnly: false,
|
|
78
|
-
secure: false,
|
|
79
|
-
};
|
|
80
|
-
for (let i = 1; i < cookieParts.length; i++) {
|
|
81
|
-
const part = cookieParts[i].toLowerCase();
|
|
82
|
-
if (part === 'httponly') {
|
|
83
|
-
cookie.httpOnly = true;
|
|
84
|
-
}
|
|
85
|
-
else if (part === 'secure') {
|
|
86
|
-
cookie.secure = true;
|
|
87
|
-
}
|
|
88
|
-
else if (part.startsWith('domain=')) {
|
|
89
|
-
cookie.domain = part.substring(7);
|
|
90
|
-
}
|
|
91
|
-
else if (part.startsWith('path=')) {
|
|
92
|
-
cookie.path = part.substring(5);
|
|
93
|
-
}
|
|
94
|
-
else if (part.startsWith('samesite=')) {
|
|
95
|
-
const sameSiteValue = part.substring(9);
|
|
96
|
-
if (['strict', 'lax', 'none'].includes(sameSiteValue)) {
|
|
97
|
-
cookie.sameSite = (sameSiteValue.charAt(0).toUpperCase() +
|
|
98
|
-
sameSiteValue.slice(1));
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
return cookie;
|
|
103
|
-
}));
|
|
104
|
-
exports.headerSchema = zod_1.z
|
|
105
|
-
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?([^=]+)=(.*)$/, {
|
|
106
|
-
message: "Header option must follow pattern '[scope::]key=value' (scope is optional)",
|
|
107
|
-
}))
|
|
108
|
-
.optional()
|
|
109
|
-
.transform((value) => value?.map((v) => {
|
|
110
|
-
const { scope, key, value } = parseScopedKeyValue(v);
|
|
111
|
-
return { scope, key, value };
|
|
112
|
-
}));
|
|
113
|
-
exports.delaySchema = zod_1.z
|
|
114
|
-
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?(\d+)$/, {
|
|
115
|
-
message: "Delay option must follow pattern '[scope::]milliseconds' (scope is optional)",
|
|
116
|
-
}))
|
|
117
|
-
.optional()
|
|
118
|
-
.transform((value) => value?.map((v) => {
|
|
119
|
-
let scope = exports.DEFAULT_SCOPE;
|
|
120
|
-
let milliseconds;
|
|
121
|
-
if (v.includes('::')) {
|
|
122
|
-
const [scopePart, msPart] = v.split('::');
|
|
123
|
-
scope = scopePart.trim();
|
|
124
|
-
milliseconds = Number(msPart);
|
|
125
|
-
}
|
|
126
|
-
else {
|
|
127
|
-
milliseconds = Number(v);
|
|
128
|
-
}
|
|
129
|
-
if (milliseconds < 1 || milliseconds > 60000) {
|
|
130
|
-
throw new zod_1.z.ZodError([
|
|
131
|
-
{
|
|
132
|
-
code: 'custom',
|
|
133
|
-
message: 'Delay must be between 1 and 60000 milliseconds',
|
|
134
|
-
path: [],
|
|
135
|
-
},
|
|
136
|
-
]);
|
|
137
|
-
}
|
|
138
|
-
return { scope, value: milliseconds };
|
|
139
|
-
}));
|
|
140
|
-
exports.waitForSchema = zod_1.z
|
|
141
|
-
.array(zod_1.z.string().regex(/^(?:([^:]+)::)?(?:(CSS|XPATH))=(.+)$/, {
|
|
142
|
-
message: "WaitFor option must follow pattern '[scope::]type=value' where type must be CSS or XPATH (scope is optional)",
|
|
143
|
-
}))
|
|
144
|
-
.optional()
|
|
145
|
-
.transform(parseScopedSelector);
|