bdy 1.9.38 → 1.9.40-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/distTs/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bdy",
|
|
3
3
|
"preferGlobal": false,
|
|
4
|
-
"version": "1.9.
|
|
4
|
+
"version": "1.9.40-dev",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"scripts": {
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"commander": "12.1.0",
|
|
27
27
|
"content-disposition": "0.5.4",
|
|
28
28
|
"cross-spawn": "7.0.6",
|
|
29
|
+
"eventsource": "4.0.0",
|
|
29
30
|
"fastify": "4.28.1",
|
|
30
31
|
"isbinaryfile": "5.0.2",
|
|
31
32
|
"jsonwebtoken": "9.0.2",
|
|
@@ -41,6 +42,7 @@
|
|
|
41
42
|
"range-parser": "1.2.1",
|
|
42
43
|
"socket.io-client": "4.7.5",
|
|
43
44
|
"ssh2": "1.15.0",
|
|
45
|
+
"tar-stream": "3.1.7",
|
|
44
46
|
"terminal-kit": "3.1.1",
|
|
45
47
|
"undici": "6.21.2",
|
|
46
48
|
"uuid": "10.0.0",
|
|
@@ -58,6 +60,7 @@
|
|
|
58
60
|
"@types/cross-spawn": "6.0.6",
|
|
59
61
|
"@types/eslint__js": "8.42.3",
|
|
60
62
|
"@types/ssh2": "1.15.1",
|
|
63
|
+
"@types/tar-stream": "3.1.3",
|
|
61
64
|
"@types/terminal-kit": "2.5.6",
|
|
62
65
|
"@types/uuid": "10.0.0",
|
|
63
66
|
"@types/which": "3.0.4",
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../../utils");
|
|
7
|
+
const commander_1 = require("commander");
|
|
8
|
+
const texts_js_1 = require("../../texts.js");
|
|
9
|
+
const validation_1 = require("../../visualTest/validation");
|
|
10
|
+
const output_1 = __importDefault(require("../../output"));
|
|
11
|
+
const requests_1 = require("../../visualTest/requests");
|
|
12
|
+
const zod_1 = require("zod");
|
|
13
|
+
const node_zlib_1 = require("node:zlib");
|
|
14
|
+
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
15
|
+
const promises_1 = require("node:stream/promises");
|
|
16
|
+
const node_fs_1 = require("node:fs");
|
|
17
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
+
const promises_2 = require("node:fs/promises");
|
|
19
|
+
const commandScrap = (0, utils_1.newCommand)('scrap', texts_js_1.DESC_COMMAND_VT_SCRAP);
|
|
20
|
+
commandScrap.argument('<url>', texts_js_1.OPTION_SCRAP_URL);
|
|
21
|
+
commandScrap.option('--follow', texts_js_1.OPTION_SCRAP_FOLLOW, false);
|
|
22
|
+
commandScrap.addOption(new commander_1.Option('--outputType <type>', texts_js_1.OPTION_SCRAP_OUTPUT_TYPE)
|
|
23
|
+
.choices(['jpeg', 'png', 'md', 'html'])
|
|
24
|
+
.makeOptionMandatory());
|
|
25
|
+
commandScrap.option('--quality <quality>', texts_js_1.OPTION_SCRAP_QUALITY);
|
|
26
|
+
commandScrap.option('--fullPage', texts_js_1.OPTION_SCRAP_FULL_PAGE, false);
|
|
27
|
+
commandScrap.option('--cssSelector <selector>', texts_js_1.OPTION_SCRAP_CSS_SELECTOR);
|
|
28
|
+
commandScrap.option('--xpathSelector <selector>', texts_js_1.OPTION_SCRAP_XPATH_SELECTOR);
|
|
29
|
+
commandScrap.option('--outputDir <dir>', texts_js_1.OPTION_SCRAP_OUTPUT_DIR, '.');
|
|
30
|
+
commandScrap.action(async (inputUrl, options) => {
|
|
31
|
+
if (!(0, validation_1.checkToken)()) {
|
|
32
|
+
output_1.default.exitError(texts_js_1.ERR_MISSING_TOKEN);
|
|
33
|
+
}
|
|
34
|
+
const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, } = validateInputAndOptions(inputUrl, options);
|
|
35
|
+
try {
|
|
36
|
+
const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector);
|
|
37
|
+
output_1.default.normal('Starting scrap session');
|
|
38
|
+
const status = await watchSessionStatus(buildId);
|
|
39
|
+
if (!status.ok) {
|
|
40
|
+
output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
|
|
41
|
+
}
|
|
42
|
+
output_1.default.normal('Downloading scrap package');
|
|
43
|
+
const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
|
|
44
|
+
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
45
|
+
const unpack = tar_stream_1.default.extract();
|
|
46
|
+
unpack.on('entry', async (header, stream, next) => {
|
|
47
|
+
const currentDir = process.cwd();
|
|
48
|
+
const preparedOutputDir = outputDir.startsWith('.')
|
|
49
|
+
? node_path_1.default.join(currentDir, outputDir)
|
|
50
|
+
: outputDir;
|
|
51
|
+
const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
|
|
52
|
+
try {
|
|
53
|
+
if (header.type === 'file') {
|
|
54
|
+
await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
|
|
55
|
+
const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
|
|
56
|
+
await (0, promises_1.pipeline)(stream, fileWriteStream);
|
|
57
|
+
next();
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
stream.resume();
|
|
61
|
+
next();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch (entryError) {
|
|
65
|
+
output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
|
|
66
|
+
next(entryError);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
|
|
70
|
+
output_1.default.exitSuccess('Downloading scrap package finished');
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
output_1.default.exitError(`${error}`);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
function validateInputAndOptions(input, options) {
|
|
77
|
+
const urlSchema = zod_1.z.string().url();
|
|
78
|
+
const optionsSchema = zod_1.z.object({
|
|
79
|
+
follow: zod_1.z.boolean(),
|
|
80
|
+
outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
|
|
81
|
+
quality: zod_1.z.coerce.number().min(1).max(100).optional(),
|
|
82
|
+
outputDir: zod_1.z.string().default('.'),
|
|
83
|
+
fullPage: zod_1.z.boolean().optional(),
|
|
84
|
+
cssSelector: zod_1.z.string().optional(),
|
|
85
|
+
xpathSelector: zod_1.z.string().optional(),
|
|
86
|
+
});
|
|
87
|
+
try {
|
|
88
|
+
const url = urlSchema.parse(input);
|
|
89
|
+
const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, } = optionsSchema.parse(options);
|
|
90
|
+
if (typeof quality === 'number' && outputType !== 'jpeg') {
|
|
91
|
+
output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
|
|
92
|
+
}
|
|
93
|
+
if (cssSelector && xpathSelector) {
|
|
94
|
+
output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
url,
|
|
98
|
+
follow,
|
|
99
|
+
outputType,
|
|
100
|
+
quality,
|
|
101
|
+
outputDir,
|
|
102
|
+
fullPage,
|
|
103
|
+
cssSelector,
|
|
104
|
+
xpathSelector,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
if (error instanceof zod_1.ZodError) {
|
|
109
|
+
output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
async function watchSessionStatus(buildId) {
|
|
117
|
+
return new Promise((resolve) => {
|
|
118
|
+
const eventSource = (0, requests_1.connectToScrapSession)(buildId);
|
|
119
|
+
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
120
|
+
const data = JSON.parse(event.data);
|
|
121
|
+
if (data.status === 'GATHER_URLS_COMPLETED') {
|
|
122
|
+
output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
|
|
123
|
+
}
|
|
124
|
+
else if (data.status === 'GATHER_URLS_FAILED') {
|
|
125
|
+
output_1.default.error('Gathering URLs failed');
|
|
126
|
+
}
|
|
127
|
+
else if (data.status === 'SCRAPE_URL_COMPLETED') {
|
|
128
|
+
output_1.default.normal(`Scraping ${data.text} completed`);
|
|
129
|
+
}
|
|
130
|
+
else if (data.status === 'SCRAPE_URL_FAILED') {
|
|
131
|
+
output_1.default.error(`Scraping ${data.text} failed`);
|
|
132
|
+
}
|
|
133
|
+
else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
|
|
134
|
+
output_1.default.normal('Package created');
|
|
135
|
+
}
|
|
136
|
+
else if (data.status === 'CREATE_PACKAGE_FAILED') {
|
|
137
|
+
output_1.default.error('Package creation failed');
|
|
138
|
+
}
|
|
139
|
+
else if (data.status === 'FINISHED') {
|
|
140
|
+
eventSource.close();
|
|
141
|
+
output_1.default.normal('Scrap session finished');
|
|
142
|
+
resolve({ ok: true });
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
eventSource.addEventListener('error', (event) => {
|
|
146
|
+
if (event.code) {
|
|
147
|
+
eventSource.close();
|
|
148
|
+
if (event.code === 410) {
|
|
149
|
+
output_1.default.normal('Scrap session finished');
|
|
150
|
+
}
|
|
151
|
+
resolve({ ok: event.code === 410, error: event.code });
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
exports.default = commandScrap;
|
package/distTs/src/command/vt.js
CHANGED
|
@@ -9,9 +9,11 @@ const close_1 = __importDefault(require("./vt/close"));
|
|
|
9
9
|
const storybook_1 = __importDefault(require("./vt/storybook"));
|
|
10
10
|
const exec_1 = __importDefault(require("./vt/exec"));
|
|
11
11
|
const installBrowser_1 = __importDefault(require("./vt/installBrowser"));
|
|
12
|
+
const scrap_js_1 = __importDefault(require("./vt/scrap.js"));
|
|
12
13
|
const commandVt = (0, utils_1.newCommand)('vt', texts_js_1.DESC_COMMAND_VT);
|
|
13
14
|
commandVt.addCommand(close_1.default);
|
|
14
15
|
commandVt.addCommand(storybook_1.default);
|
|
15
16
|
commandVt.addCommand(exec_1.default);
|
|
16
17
|
commandVt.addCommand(installBrowser_1.default);
|
|
18
|
+
commandVt.addCommand(scrap_js_1.default);
|
|
17
19
|
exports.default = commandVt;
|
package/distTs/src/texts.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ERR_SWW = exports.ERR_NOT_FOUND = exports.ERR_FAILED_TO_CONNECT_TO_AGENT = exports.ERR_TUNNEL_REMOVED = exports.ERR_TUNNELS_DISABLED = exports.ERR_AGENT_LIMIT_REACHED = exports.ERR_TUNNEL_TARGET_INVALID = exports.ERR_TUNNEL_LIMIT_REACHED = exports.ERR_DOMAIN_RESTRICTED = exports.ERR_SUBDOMAIN_INVALID = exports.ERR_SUBDOMAIN_TAKEN = exports.ERR_AGENT_REMOVED = exports.ERR_FAILED_TO_CONNECT = exports.ERR_TUNNEL_ALREADY_EXISTS = exports.ERR_AGENT_NOT_SUPPORTED = exports.ERR_AGENT_ADMIN_RIGHTS = exports.ERR_AGENT_ENABLE = exports.ERR_SWW_AGENT_UPDATING = exports.ERR_SWW_AGENT_DISABLING = exports.ERR_SWW_AGENT_ENABLING = exports.ERR_AGENT_NOT_FOUND = exports.ERR_AGENT_NOT_RUNNING = exports.ERR_AGENT_NOT_ENABLED = exports.ERR_TUNNEL_NOT_FOUND = exports.ERR_WHITELIST_IS_NOT_VALID = exports.ERR_USER_AGENT_IS_NOT_VALID = exports.ERR_BA_IS_NOT_VALID = exports.ERR_BA_LOGIN_NOT_PROVIDED = exports.ERR_BA_PASSWORD_NOT_PROVIDED = exports.ERR_CB_THRESHOLD_IS_NOT_VALID = exports.ERR_DOMAIN_IS_NOT_VALID = exports.ERR_CERT_PATH_IS_NOT_VALID = exports.ERR_KEY_PATH_IS_NOT_VALID = exports.ERR_CA_PATH_IS_NOT_VALID = exports.ERR_WRONG_CA = exports.ERR_WRONG_KEY_CERT = exports.ERR_NAME_WITHOUT_ASTERISK = exports.ERR_PORT_IS_NOT_VALID = exports.ERR_REGION_IS_NOT_VALID = exports.ERR_PATH_IS_NOT_DIRECTORY = exports.ERR_DIRECTORY_DOES_NOT_EXISTS = exports.ERR_SUBDOMAIN_IS_TOO_LONG = exports.ERR_SUBDOMAIN_IS_TOO_SHORT = exports.ERR_SUBDOMAIN_IS_NOT_VALID = exports.ERR_TERMINATE_IS_NOT_VALID = exports.ERR_TIMEOUT_IS_NOT_VALID = exports.ERR_TYPE_IS_NOT_VALID = exports.ERR_TARGET_IS_NOT_VALID = exports.ERR_SAVING_AGENT_CONFIG = exports.ERR_AGENT_NOT_REGISTERED = void 0;
|
|
4
|
-
exports.
|
|
5
|
-
exports.
|
|
6
|
-
exports.
|
|
7
|
-
exports.
|
|
8
|
-
exports.DEBUG_WAIT_FOR_IDLE_TIMEOUT = exports.DEBUG_WAIT_FOR_IDLE = exports.DEBUG_RESOURCE_DISCOVERY_TIMEOUT = exports.DEBUG_AUTO_WIDTH = exports.DEBUG_AUTO_SCROLL = exports.DEBUG_RESOURCE_SCRAPPING_URL = exports.DEBUG_SNAPSHOT_PROCESSING = exports.DEBUG_SNAPSHOTS_PROCESSING = exports.DEBUG_EXEC_COMMAND = exports.DEBUG_EXEC_TEST_COMMAND = exports.LOG_INSTALLED_BROWSER = exports.LOG_SESSION_LINK = exports.LOG_SENDING_DATA = exports.LOG_SENDING_REQUEST = exports.LOG_PROCESSING_SNAPSHOTS = exports.LOG_RUNNING_EXEC_COMMAND = exports.LOG_TUNNEL_SSH_STREAM = exports.LOG_TUNNEL_TLS_AGENT_STREAM = exports.LOG_TUNNEL_TLS_REGION_STREAM = exports.LOG_TUNNEL_TLS_TARGET_STREAM = exports.LOG_TUNNEL_HTTP2_STREAM = exports.LOG_TUNNEL_HTTP1_STREAM = exports.LOG_TUNNEL_TCP_STREAM = exports.LOG_TUNNEL_HTTP_WRONG_USER_AGENTS = exports.LOG_TUNNEL_HTTP_CIRCUIT_BREAKER_OPEN = exports.LOG_TUNNEL_HTTP_RATE_LIMIT = exports.LOG_TUNNEL_HTTP_WRON_AUTH = exports.LOG_TUNNEL_IDENTIFIED = exports.LOG_TUNNEL_DISCONNECTED = exports.LOG_TUNNEL_FAILED = exports.LOG_TUNNEL_CONNECTED = exports.LOG_AGENT_STARTED = exports.LOG_AGENT_SERVER_STARTED = void 0;
|
|
4
|
+
exports.TXT_AGENT_TARGET_ENABLED = exports.TXT_AGENT_IS_DISABLED = exports.TXT_AGENT_IS_ENABLED_AND_HAVE_TROUBLES = exports.TXT_AGENT_IS_ENABLED_AND_INITIALIZING = exports.TXT_AGENT_IS_ENABLED_AND_STOPPED = exports.TXT_AGENT_IS_ENABLED_AND_STARTED = exports.TXT_AGENT_RESTARTED = exports.TXT_AGENT_STARTED = exports.TXT_AGENT_STOPPED = exports.WARN_BROWSER_VERSION = exports.ERR_NO_SNAPSHOTS_TO_SEND = exports.ERR_INVALID_SNAPSHOT = exports.ERR_GETTING_COMMIT_DETAILS = exports.ERR_GETTING_BASE_COMMIT = exports.ERR_HEAD_BRANCH_NOT_DEFINED = exports.ERR_BASE_BRANCH_NOT_DEFINED = exports.ERR_INVALID_COMMIT_HASH = exports.ERR_GETTING_COMMIT_HASH = exports.ERR_INVALID_BRANCH_NAME = exports.ERR_GETTING_BRANCH_NAME = exports.ERR_MISSING_HEAD_COMMIT_IN_FILE = exports.ERR_READING_FILE_WITH_HEAD_COMMIT = exports.ERR_MISSING_FILE_WITH_HEAD_COMMIT = exports.ERR_GITHUB_EVENT_PATH_NOT_FOUND = exports.ERR_TEST_EXECUTION = exports.ERR_INVALID_JSON = exports.ERR_INVALID_DOWNLOAD_RESPONSE = exports.ERR_INVALID_SCRAP_RESPONSE = exports.ERR_INVALID_STORYBOOK_RESPONSE = exports.ERR_INVALID_DEFAULT_SETTINGS_RESPONSE = exports.ERR_INVALID_CLOSE_SESSION_RESPONSE = exports.ERR_INVALID_SNAPSHOTS_RESPONSE = exports.ERR_INVALID_SNAPSHOT_RESPONSE = exports.ERR_RESOURCE_NOT_FOUND = exports.ERR_MISSING_EXEC_COMMAND = exports.ERR_PARSING_STORIES = exports.ERR_UNSUPPORTED_STORYBOOK = exports.ERR_MISSING_STORYBOOK_INDEX_FILE = exports.ERR_WRONG_STORYBOOK_DIRECTORY = exports.ERR_MISSING_BUILD_ID = exports.ERR_MISSING_TOKEN = exports.ERR_CONFIG_CORRUPTED = exports.ERR_WRONG_TOKEN = exports.ERR_TOKEN_NOT_PROVIDED = exports.ERR_CANT_CREATE_DIR_IN_HOME = exports.ERR_CONNECTION_ERROR = exports.ERR_CONNECTION_TIMEOUT = exports.ERR_WRONG_STREAM = exports.ERR_WRONG_HANDSHAKE = exports.ERR_FETCH_VERSION = void 0;
|
|
5
|
+
exports.DESC_COMMAND_AGENT_VERSION = exports.DESC_COMMAND_AGENT_UPDATE = exports.DESC_COMMAND_AGENT_TARGET = exports.DESC_COMMAND_AGENT_TUNNEL = exports.DESC_COMMAND_AGENT_STOP = exports.DESC_COMMAND_AGENT_TARGET_DISABLE = exports.DESC_COMMAND_AGENT_TARGET_ENABLE = exports.DESC_COMMAND_AGENT_TARGET_STATUS = exports.DESC_COMMAND_AGENT_STATUS = exports.DESC_COMMAND_AGENT_RESTART = exports.DESC_COMMAND_AGENT_START = exports.DESC_COMMAND_AGENT_INSTALL = exports.DESC_COMMAND_AGENT_UNINSTALL = exports.DESC_COMMAND_AGENT_TUNNEL_REMOVE = exports.DESC_COMMAND_AGENT_TUNNEL_STATUS = exports.DESC_COMMAND_AGENT_TUNNEL_LIST = exports.DESC_COMMAND_CONFIG_SET = exports.DESC_COMMAND_CONFIG_REMOVE = exports.DESC_COMMAND_CONFIG_GET = exports.DESC_COMMAND_CONFIG_ADD = exports.DESC_COMMAND_CONFIG_SET_WHITELIST = exports.DESC_COMMAND_CONFIG_SET_TOKEN = exports.DESC_COMMAND_CONFIG_SET_TIMEOUT = exports.DESC_COMMAND_CONFIG_SET_REGION = exports.DESC_COMMAND_CONFIG_REMOVE_TUNNEL = exports.DESC_COMMAND_CONFIG_GET_WHITELIST = exports.DESC_COMMAND_CONFIG_GET_TUNNELS = exports.DESC_COMMAND_CONFIG_GET_TUNNEL = exports.DESC_COMMAND_CONFIG_GET_TOKEN = exports.DESC_COMMAND_CONFIG_GET_TIMEOUT = exports.DESC_COMMAND_CONFIG_GET_REGION = exports.DESC_COMMAND_CONFIG_ADD_TLS = exports.DESC_COMMAND_CONFIG_ADD_TCP = exports.DESC_COMMAND_CONFIG_ADD_HTTP = exports.AGENT_FETCH_RETRY = exports.NO_TUNNELS_STARTED = exports.TXT_TUNNEL_ADDED = exports.TXT_TUNNEL_REMOVED = exports.TXT_REGION_SAVED = exports.TXT_TIMEOUT_SAVED = exports.TXT_TOKEN_REMOVED = exports.TXT_TOKEN_SAVED = exports.TXT_WHITELIST_SAVED = exports.TXT_TUNNEL_STOPPED = exports.TXT_TUNNEL_STARTED = exports.TXT_AGENT_DISABLED = exports.TXT_AGENT_ALREADY_ENABLED = exports.TXT_AGENT_UPDATED = exports.TXT_AGENT_ENABLED = exports.TXT_AGENT_TARGET_DISABLED = void 0;
|
|
6
|
+
exports.OPTION_AGENT_ID = exports.OPTION_ID = exports.OPTION_NAME = exports.OPTION_TARGET = exports.OPTION_TLS_TERMINATE = exports.OPTION_TLS_CA = exports.OPTION_TLS_CERT = exports.OPTION_TLS_KEY = exports.OPTION_HTTP_CIRCUIT_BREAKER = exports.OPTION_HTTP_COMPRESSION = exports.OPTION_HTTP_2 = exports.OPTION_HTTP_VERIFY = exports.OPTION_HTTP_LOG = exports.OPTION_HTTP_AUTH = exports.OPTION_HTTP_HOST = exports.OPTION_FORCE = exports.OPTION_TOKEN = exports.OPTION_TIMEOUT = exports.OPTION_DOMAIN = exports.OPTION_FOLLOW = exports.OPTION_SUBDOMAIN = exports.OPTION_SERVE = exports.OPTION_HEADER_USER_AGENT = exports.OPTION_RESPONSE_HEADER = exports.OPTION_HEADER = exports.OPTION_WHITELIST = exports.OPTION_DEFAULT_REGION = exports.OPTION_REGION = exports.TXT_CI_INFO = exports.TXT_STORIES_AMOUNT = exports.TXT_OPENING_TUNNEL = exports.TXT_UPDATING_AGENT = exports.TXT_ENABLING_AGENT = exports.TXT_DISABLING_AGENT = exports.TXT_NEW_AGENT_VERSION = exports.TXT_NEW_CLI_VERSION = exports.TXT_NEW_CLI_DOCKER_VERSION = exports.DESC_COMMAND_VT_INSTALL_BROWSER = exports.DESC_COMMAND_VT_EXEC = exports.DESC_COMMAND_VT_SCRAP = exports.DESC_COMMAND_VT_STORYBOOK = exports.DESC_COMMAND_VT_CLOSE = exports.DESC_COMMAND_VT = exports.DESC_PROGRAM = exports.DESC_COMMAND_TLS = exports.DESC_COMMAND_TCP = exports.DESC_COMMAND_START = exports.DESC_COMMAND_AGENT = exports.DESC_COMMAND_HTTP = exports.DESC_COMMAND_CONFIG = void 0;
|
|
7
|
+
exports.LOG_STARTING_TUNNEL = exports.LOG_ENABLING_AGENT_TARGET = exports.LOG_DISABLING_AGENT_TARGET = exports.LOG_REMOVING_TUNNEL = exports.LOG_TUNNEL_REGISTERED = exports.LOG_ERROR_WHILE_REFRESHING_AGENT = exports.LOG_REGISTERING_TUNNEL = exports.LOG_GETTING_AGENT = exports.LOG_UNREGISTERING_AGENT = exports.LOG_REGION_DETECTED = exports.LOG_AGENT_REGISTERED = exports.LOG_SOCKET_DISCONNECTED = exports.LOG_SOCKET_CONNECTED = exports.LOG_AGENT_NSSM_CLEARING = exports.LOG_AGENT_NSSM_EXTRACTING = exports.LOG_AGENT_NSSM_DOWNLOADING = exports.LOG_AGENT_ENABLED = exports.LOG_AGENT_STARTING_SYSTEM = exports.LOG_AGENT_STOPPING_SYSTEM = exports.LOG_AGENT_ENABLING_SYSTEM = exports.LOG_AGENT_SYSTEM_SERVICE_CONFIG = exports.LOG_AGENT_EXTRACTING_ARCHIVE = exports.LOG_AGENT_DOWNLOADING_ARCHIVE = exports.LOG_AGENT_SYSTEM_DIR = exports.LOG_ERROR_SAVING_AGENT_LOCAL_CONFIG = exports.LOG_ERROR_SAVING_AGENT_SYSTEM_CONFIG = exports.LOG_ERROR_SAVING_AGENT_CONFIG = exports.LOG_SAVING_AGENT_LOCAL_CONFIG = exports.LOG_SAVING_AGENT_SYSTEM_CONFIG = exports.LOG_SAVING_AGENT_CONFIG = exports.LOG_REGISTERING_AGENT = exports.OPTION_SCRAP_OUTPUT_DIR = exports.OPTION_SCRAP_XPATH_SELECTOR = exports.OPTION_SCRAP_CSS_SELECTOR = exports.OPTION_SCRAP_FULL_PAGE = exports.OPTION_SCRAP_QUALITY = exports.OPTION_SCRAP_OUTPUT_TYPE = exports.OPTION_SCRAP_FOLLOW = exports.OPTION_SCRAP_URL = exports.OPTION_EXEC_PARALLEL = exports.OPTION_EXEC_ONE_BY_ONE = exports.OPTION_EXEC_SKIP_DISCOVERY = exports.OPTION_EXEC_COMMAND = exports.OPTION_AGENT_DEBUG = exports.OPTION_AGENT_PORT = exports.OPTION_AGENT_TARGET = exports.OPTION_PASS = exports.OPTION_USER = exports.OPTION_AGENT_TOKEN = exports.OPTION_AGENT_START = void 0;
|
|
8
|
+
exports.DEBUG_WAIT_FOR_IDLE_TIMEOUT = exports.DEBUG_WAIT_FOR_IDLE = exports.DEBUG_RESOURCE_DISCOVERY_TIMEOUT = exports.DEBUG_AUTO_WIDTH = exports.DEBUG_AUTO_SCROLL = exports.DEBUG_RESOURCE_SCRAPPING_URL = exports.DEBUG_SNAPSHOT_PROCESSING = exports.DEBUG_SNAPSHOTS_PROCESSING = exports.DEBUG_EXEC_COMMAND = exports.DEBUG_EXEC_TEST_COMMAND = exports.LOG_INSTALLED_BROWSER = exports.LOG_SESSION_LINK = exports.LOG_SENDING_DATA = exports.LOG_SENDING_REQUEST = exports.LOG_PROCESSING_SNAPSHOTS = exports.LOG_RUNNING_EXEC_COMMAND = exports.LOG_TUNNEL_SSH_STREAM = exports.LOG_TUNNEL_TLS_AGENT_STREAM = exports.LOG_TUNNEL_TLS_REGION_STREAM = exports.LOG_TUNNEL_TLS_TARGET_STREAM = exports.LOG_TUNNEL_HTTP2_STREAM = exports.LOG_TUNNEL_HTTP1_STREAM = exports.LOG_TUNNEL_TCP_STREAM = exports.LOG_TUNNEL_HTTP_WRONG_USER_AGENTS = exports.LOG_TUNNEL_HTTP_CIRCUIT_BREAKER_OPEN = exports.LOG_TUNNEL_HTTP_RATE_LIMIT = exports.LOG_TUNNEL_HTTP_WRON_AUTH = exports.LOG_TUNNEL_IDENTIFIED = exports.LOG_TUNNEL_DISCONNECTED = exports.LOG_TUNNEL_FAILED = exports.LOG_TUNNEL_CONNECTED = exports.LOG_AGENT_STARTED = exports.LOG_AGENT_SERVER_STARTED = exports.LOG_ERROR_STARTING_AGENT_SERVER = exports.LOG_REQUEST = exports.LOG_SSH_CONNECTION = exports.LOG_WRONG_STREAM = exports.LOG_DETECTED_STREAM = exports.LOG_HTTP2_REQUEST = exports.LOG_HTTP2_CONNECTION = exports.LOG_HTTP1_REQUEST = exports.LOG_HTTP1_CONNECTION = exports.LOG_ERROR = exports.LOG_STOPPING_TUNNEL = void 0;
|
|
9
9
|
const utils_1 = require("./utils");
|
|
10
10
|
exports.ERR_AGENT_NOT_REGISTERED = 'Agent not registered. Exiting.';
|
|
11
11
|
exports.ERR_SAVING_AGENT_CONFIG = 'Failed saving agent config. Exiting.';
|
|
@@ -104,6 +104,8 @@ exports.ERR_INVALID_SNAPSHOTS_RESPONSE = `Invalid send snapshots response`;
|
|
|
104
104
|
exports.ERR_INVALID_CLOSE_SESSION_RESPONSE = `Invalid close session response`;
|
|
105
105
|
exports.ERR_INVALID_DEFAULT_SETTINGS_RESPONSE = `Invalid default settings response`;
|
|
106
106
|
exports.ERR_INVALID_STORYBOOK_RESPONSE = `Invalid send storybook response`;
|
|
107
|
+
exports.ERR_INVALID_SCRAP_RESPONSE = `Invalid send scrap response`;
|
|
108
|
+
exports.ERR_INVALID_DOWNLOAD_RESPONSE = `Invalid download response`;
|
|
107
109
|
exports.ERR_INVALID_JSON = `Invalid JSON`;
|
|
108
110
|
exports.ERR_TEST_EXECUTION = `Test process exited with error`;
|
|
109
111
|
exports.ERR_GITHUB_EVENT_PATH_NOT_FOUND = `GITHUB_EVENT_PATH is not defined`;
|
|
@@ -200,6 +202,7 @@ exports.DESC_PROGRAM = 'Buddy exposes local networked services behinds NATs and
|
|
|
200
202
|
exports.DESC_COMMAND_VT = 'Commands to interact with the visual test service';
|
|
201
203
|
exports.DESC_COMMAND_VT_CLOSE = 'Close visual test session.';
|
|
202
204
|
exports.DESC_COMMAND_VT_STORYBOOK = 'Create visual test session from storybook';
|
|
205
|
+
exports.DESC_COMMAND_VT_SCRAP = 'Scrap website';
|
|
203
206
|
exports.DESC_COMMAND_VT_EXEC = 'Run tests which use visual service plugin';
|
|
204
207
|
exports.DESC_COMMAND_VT_INSTALL_BROWSER = 'Install browser for visual test';
|
|
205
208
|
const TXT_NEW_CLI_DOCKER_VERSION = (version) => `A new version of bdy Docker image is available! (${version})\n\n`;
|
|
@@ -254,6 +257,14 @@ exports.OPTION_EXEC_COMMAND = 'Test runner command';
|
|
|
254
257
|
exports.OPTION_EXEC_SKIP_DISCOVERY = 'Skip resource discovery';
|
|
255
258
|
exports.OPTION_EXEC_ONE_BY_ONE = 'Send snapshots one by one';
|
|
256
259
|
exports.OPTION_EXEC_PARALLEL = 'Send parts of session from different processes';
|
|
260
|
+
exports.OPTION_SCRAP_URL = 'URL to scrape';
|
|
261
|
+
exports.OPTION_SCRAP_FOLLOW = 'Scrape all subviews of the page';
|
|
262
|
+
exports.OPTION_SCRAP_OUTPUT_TYPE = 'Output type';
|
|
263
|
+
exports.OPTION_SCRAP_QUALITY = 'Quality of the png output (default: "100")';
|
|
264
|
+
exports.OPTION_SCRAP_FULL_PAGE = 'Scrap full page';
|
|
265
|
+
exports.OPTION_SCRAP_CSS_SELECTOR = 'CSS selector to scrape';
|
|
266
|
+
exports.OPTION_SCRAP_XPATH_SELECTOR = 'XPath selector to scrape';
|
|
267
|
+
exports.OPTION_SCRAP_OUTPUT_DIR = 'Output directory';
|
|
257
268
|
exports.LOG_REGISTERING_AGENT = 'Registering agent...';
|
|
258
269
|
exports.LOG_SAVING_AGENT_CONFIG = 'Saving agent config...';
|
|
259
270
|
exports.LOG_SAVING_AGENT_SYSTEM_CONFIG = 'Saving agent system config...';
|
|
@@ -8,12 +8,16 @@ exports.sendSnapshots = sendSnapshots;
|
|
|
8
8
|
exports.closeSession = closeSession;
|
|
9
9
|
exports.getDefaultSettings = getDefaultSettings;
|
|
10
10
|
exports.sendStorybook = sendStorybook;
|
|
11
|
+
exports.sendScrap = sendScrap;
|
|
12
|
+
exports.downloadScrapPackage = downloadScrapPackage;
|
|
13
|
+
exports.connectToScrapSession = connectToScrapSession;
|
|
11
14
|
const context_js_1 = require("./context.js");
|
|
12
15
|
const undici_1 = require("undici");
|
|
13
16
|
const uuid_1 = require("uuid");
|
|
14
17
|
const node_fs_1 = require("node:fs");
|
|
15
18
|
const output_1 = __importDefault(require("../output"));
|
|
16
19
|
const texts_1 = require("../texts");
|
|
20
|
+
const eventsource_1 = require("eventsource");
|
|
17
21
|
const serviceUrl = process.env.SNAPSHOTS_SERVICE_URL || 'http://localhost:3000';
|
|
18
22
|
function prepareSnapshotInternal(snapshot) {
|
|
19
23
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
@@ -180,6 +184,42 @@ async function sendStorybook(snapshots, filePaths) {
|
|
|
180
184
|
}
|
|
181
185
|
return message;
|
|
182
186
|
}
|
|
187
|
+
async function sendScrap(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector) {
|
|
188
|
+
const payload = {
|
|
189
|
+
token: context_js_1.token,
|
|
190
|
+
url,
|
|
191
|
+
follow,
|
|
192
|
+
outputType,
|
|
193
|
+
quality,
|
|
194
|
+
fullPage,
|
|
195
|
+
cssSelector,
|
|
196
|
+
xpathSelector,
|
|
197
|
+
};
|
|
198
|
+
const [message, response] = await sendRequest({
|
|
199
|
+
url: '/scrap',
|
|
200
|
+
payload,
|
|
201
|
+
});
|
|
202
|
+
if (message) {
|
|
203
|
+
throw new Error(message);
|
|
204
|
+
}
|
|
205
|
+
if (!response) {
|
|
206
|
+
throw new Error(texts_1.ERR_INVALID_SCRAP_RESPONSE);
|
|
207
|
+
}
|
|
208
|
+
return response;
|
|
209
|
+
}
|
|
210
|
+
async function downloadScrapPackage(buildId) {
|
|
211
|
+
const [message, response] = await sendRequest({
|
|
212
|
+
url: '/download',
|
|
213
|
+
payload: { token: context_js_1.token, buildId },
|
|
214
|
+
});
|
|
215
|
+
if (message) {
|
|
216
|
+
throw new Error(message);
|
|
217
|
+
}
|
|
218
|
+
if (!response) {
|
|
219
|
+
throw new Error(texts_1.ERR_INVALID_DOWNLOAD_RESPONSE);
|
|
220
|
+
}
|
|
221
|
+
return response;
|
|
222
|
+
}
|
|
183
223
|
async function sendRequest({ url, payload, multipart, }) {
|
|
184
224
|
output_1.default.debug((0, texts_1.LOG_SENDING_REQUEST)(`${serviceUrl}${url}`));
|
|
185
225
|
const init = {
|
|
@@ -206,6 +246,24 @@ async function sendRequest({ url, payload, multipart, }) {
|
|
|
206
246
|
throw new Error(texts_1.ERR_INVALID_JSON);
|
|
207
247
|
}
|
|
208
248
|
}
|
|
249
|
+
if (contentType && contentType.includes('application/octet-stream')) {
|
|
250
|
+
const buffer = response.body;
|
|
251
|
+
return [undefined, buffer];
|
|
252
|
+
}
|
|
209
253
|
const text = await response.text();
|
|
210
254
|
return [text, undefined];
|
|
211
255
|
}
|
|
256
|
+
function connectToScrapSession(buildId) {
|
|
257
|
+
return new eventsource_1.EventSource(`${serviceUrl}/sse`, {
|
|
258
|
+
fetch: (url, options) => {
|
|
259
|
+
return (0, undici_1.fetch)(url, {
|
|
260
|
+
...options,
|
|
261
|
+
headers: {
|
|
262
|
+
...options?.headers,
|
|
263
|
+
'x-token': context_js_1.token,
|
|
264
|
+
'x-build-id': buildId,
|
|
265
|
+
},
|
|
266
|
+
});
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bdy",
|
|
3
3
|
"preferGlobal": false,
|
|
4
|
-
"version": "1.9.
|
|
4
|
+
"version": "1.9.40-dev",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"scripts": {
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"commander": "12.1.0",
|
|
27
27
|
"content-disposition": "0.5.4",
|
|
28
28
|
"cross-spawn": "7.0.6",
|
|
29
|
+
"eventsource": "4.0.0",
|
|
29
30
|
"fastify": "4.28.1",
|
|
30
31
|
"isbinaryfile": "5.0.2",
|
|
31
32
|
"jsonwebtoken": "9.0.2",
|
|
@@ -41,6 +42,7 @@
|
|
|
41
42
|
"range-parser": "1.2.1",
|
|
42
43
|
"socket.io-client": "4.7.5",
|
|
43
44
|
"ssh2": "1.15.0",
|
|
45
|
+
"tar-stream": "3.1.7",
|
|
44
46
|
"terminal-kit": "3.1.1",
|
|
45
47
|
"undici": "6.21.2",
|
|
46
48
|
"uuid": "10.0.0",
|
|
@@ -58,6 +60,7 @@
|
|
|
58
60
|
"@types/cross-spawn": "6.0.6",
|
|
59
61
|
"@types/eslint__js": "8.42.3",
|
|
60
62
|
"@types/ssh2": "1.15.1",
|
|
63
|
+
"@types/tar-stream": "3.1.3",
|
|
61
64
|
"@types/terminal-kit": "2.5.6",
|
|
62
65
|
"@types/uuid": "10.0.0",
|
|
63
66
|
"@types/which": "3.0.4",
|