bdy 1.16.17-dev → 1.16.18-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/command/vt/compare.js +2 -2
- package/package.json +1 -1
- package/distTs/src/command/agent/down.js +0 -22
- package/distTs/src/command/agent/standalone/kill.js +0 -22
- package/distTs/src/command/agent/standalone.js +0 -136
- package/distTs/src/command/agent/up.js +0 -134
- package/distTs/src/command/vt/scrap.js +0 -193
package/distTs/package.json
CHANGED
|
@@ -40,11 +40,11 @@ commandVtCompare.action(async (options) => {
|
|
|
40
40
|
const urlsList = getUrlsFromUrlOption(validatedOptions.urls);
|
|
41
41
|
urls = urls.concat(urlsList);
|
|
42
42
|
}
|
|
43
|
-
|
|
43
|
+
if (validatedOptions.urlsFile) {
|
|
44
44
|
const urlsList = getUrlsFromUrlFile(validatedOptions.urlsFile);
|
|
45
45
|
urls = urls.concat(urlsList);
|
|
46
46
|
}
|
|
47
|
-
|
|
47
|
+
if (validatedOptions.sitemap) {
|
|
48
48
|
sitemapSource = (0, linkUtils_1.addProtocolIfMissing)(validatedOptions.sitemap);
|
|
49
49
|
}
|
|
50
50
|
const { filteredUrls, duplicates } = filterDuplicates(urls);
|
package/package.json
CHANGED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const manager_1 = __importDefault(require("../../agent/manager"));
|
|
9
|
-
const output_1 = __importDefault(require("../../output"));
|
|
10
|
-
const commandAgentDown = (0, utils_1.newCommand)('down', texts_1.DESC_COMMAND_AGENT_STANDALONE_KILL);
|
|
11
|
-
commandAgentDown.action(async () => {
|
|
12
|
-
const hasAdminRights = await manager_1.default.system.hasAdminRights();
|
|
13
|
-
if (!hasAdminRights) {
|
|
14
|
-
output_1.default.exitError(texts_1.ERR_AGENT_ADMIN_RIGHTS);
|
|
15
|
-
}
|
|
16
|
-
if (!manager_1.default.system.isStandaloneProcRunning()) {
|
|
17
|
-
output_1.default.exitError(texts_1.ERR_AGENT_NOT_RUNNING);
|
|
18
|
-
}
|
|
19
|
-
manager_1.default.system.killStandaloneProc();
|
|
20
|
-
output_1.default.exitSuccess(texts_1.TXT_AGENT_STANDALONE_STOPPED);
|
|
21
|
-
});
|
|
22
|
-
exports.default = commandAgentDown;
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../../utils");
|
|
7
|
-
const texts_1 = require("../../../texts");
|
|
8
|
-
const manager_1 = __importDefault(require("../../../agent/manager"));
|
|
9
|
-
const output_1 = __importDefault(require("../../../output"));
|
|
10
|
-
const commandAgentStandaloneKill = (0, utils_1.newCommand)('kill', texts_1.DESC_COMMAND_AGENT_STANDALONE_KILL);
|
|
11
|
-
commandAgentStandaloneKill.action(async () => {
|
|
12
|
-
const hasAdminRights = await manager_1.default.system.hasAdminRights();
|
|
13
|
-
if (!hasAdminRights) {
|
|
14
|
-
output_1.default.exitError(texts_1.ERR_AGENT_ADMIN_RIGHTS);
|
|
15
|
-
}
|
|
16
|
-
if (!manager_1.default.system.isStandaloneProcRunning()) {
|
|
17
|
-
output_1.default.exitError(texts_1.ERR_AGENT_NOT_RUNNING);
|
|
18
|
-
}
|
|
19
|
-
manager_1.default.system.killStandaloneProc();
|
|
20
|
-
output_1.default.exitSuccess(texts_1.TXT_AGENT_STANDALONE_STOPPED);
|
|
21
|
-
});
|
|
22
|
-
exports.default = commandAgentStandaloneKill;
|
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const manager_1 = __importDefault(require("../../agent/manager"));
|
|
9
|
-
const output_1 = __importDefault(require("../../output"));
|
|
10
|
-
const buddy_1 = __importDefault(require("../../tunnel/api/buddy"));
|
|
11
|
-
const logger_1 = __importDefault(require("../../logger"));
|
|
12
|
-
const cfg_1 = __importDefault(require("../../tunnel/cfg"));
|
|
13
|
-
const input_1 = __importDefault(require("../../input"));
|
|
14
|
-
const kill_1 = __importDefault(require("./standalone/kill"));
|
|
15
|
-
const commandAgentStandalone = (0, utils_1.newCommand)('standalone', texts_1.DESC_COMMAND_AGENT_STANDALONE);
|
|
16
|
-
commandAgentStandalone.option('-i, --id <id>', texts_1.OPTION_AGENT_ID);
|
|
17
|
-
commandAgentStandalone.option('-t, --token <token>', texts_1.OPTION_AGENT_TOKEN);
|
|
18
|
-
commandAgentStandalone.option('-p, --port <port>', texts_1.OPTION_AGENT_PORT, '7456');
|
|
19
|
-
commandAgentStandalone.option('--target', texts_1.OPTION_AGENT_TARGET);
|
|
20
|
-
commandAgentStandalone.addCommand(kill_1.default);
|
|
21
|
-
commandAgentStandalone.action(async (options) => {
|
|
22
|
-
const hasAdminRights = await manager_1.default.system.hasAdminRights();
|
|
23
|
-
if (!hasAdminRights) {
|
|
24
|
-
output_1.default.exitError(texts_1.ERR_AGENT_ADMIN_RIGHTS);
|
|
25
|
-
}
|
|
26
|
-
try {
|
|
27
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKING);
|
|
28
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKING, false);
|
|
29
|
-
manager_1.default.system.killStandaloneProc();
|
|
30
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKED);
|
|
31
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKED);
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_ERROR);
|
|
35
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_ERROR);
|
|
36
|
-
logger_1.default.error(err);
|
|
37
|
-
output_1.default.exitError(err);
|
|
38
|
-
}
|
|
39
|
-
const isEnabled = await manager_1.default.system.isEnabled();
|
|
40
|
-
if (isEnabled) {
|
|
41
|
-
output_1.default.exitError(texts_1.ERR_AGENT_STANDALONE_SERVICE_INSTALLED);
|
|
42
|
-
}
|
|
43
|
-
let id;
|
|
44
|
-
let host;
|
|
45
|
-
let token;
|
|
46
|
-
let port;
|
|
47
|
-
try {
|
|
48
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_CHECKING);
|
|
49
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_CHECKING, false);
|
|
50
|
-
const json = manager_1.default.system.loadSystemConfig();
|
|
51
|
-
id = json.id;
|
|
52
|
-
host = json.host;
|
|
53
|
-
token = json.token;
|
|
54
|
-
port = json.port;
|
|
55
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_FOUND);
|
|
56
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_FOUND);
|
|
57
|
-
}
|
|
58
|
-
catch {
|
|
59
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_NOT_FOUND);
|
|
60
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_NOT_FOUND);
|
|
61
|
-
}
|
|
62
|
-
if (!port) {
|
|
63
|
-
port = input_1.default.port(options.port);
|
|
64
|
-
}
|
|
65
|
-
if (!id && options.id) {
|
|
66
|
-
id = options.id;
|
|
67
|
-
logger_1.default.info((0, texts_1.TXT_AGENT_STANDALONE_USING_AGENT)(id));
|
|
68
|
-
output_1.default.normal((0, texts_1.TXT_AGENT_STANDALONE_USING_AGENT)(id));
|
|
69
|
-
}
|
|
70
|
-
if (!token) {
|
|
71
|
-
token = options.token;
|
|
72
|
-
}
|
|
73
|
-
if (!id) {
|
|
74
|
-
if (token)
|
|
75
|
-
cfg_1.default.setToken(token);
|
|
76
|
-
host = cfg_1.default.getTokenHost();
|
|
77
|
-
try {
|
|
78
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERING, false);
|
|
79
|
-
const agent = await buddy_1.default.register(true, !!options.target, host, cfg_1.default.getToken(), (region) => {
|
|
80
|
-
cfg_1.default.setRegionIfNotSet(region);
|
|
81
|
-
});
|
|
82
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERED);
|
|
83
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERED);
|
|
84
|
-
id = agent.id;
|
|
85
|
-
token = agent.token;
|
|
86
|
-
agent.destroy();
|
|
87
|
-
}
|
|
88
|
-
catch (err) {
|
|
89
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTER_ERROR);
|
|
90
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTER_ERROR);
|
|
91
|
-
logger_1.default.error(err);
|
|
92
|
-
output_1.default.exitError(err);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
else {
|
|
96
|
-
if (!host) {
|
|
97
|
-
host = cfg_1.default.getTokenHost();
|
|
98
|
-
}
|
|
99
|
-
try {
|
|
100
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHING);
|
|
101
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHING, false);
|
|
102
|
-
const agent = await buddy_1.default.fetchAgent(id, host, token);
|
|
103
|
-
agent.destroy();
|
|
104
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHED);
|
|
105
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHED);
|
|
106
|
-
}
|
|
107
|
-
catch (err) {
|
|
108
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCH_ERROR);
|
|
109
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCH_ERROR);
|
|
110
|
-
logger_1.default.error(err);
|
|
111
|
-
output_1.default.exitError(err);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
let saved = manager_1.default.system.saveSystemConfig(id, host, token, port);
|
|
115
|
-
if (!saved) {
|
|
116
|
-
output_1.default.exitError(texts_1.ERR_SWW_AGENT_ENABLING);
|
|
117
|
-
}
|
|
118
|
-
saved = manager_1.default.system.saveStandaloneProcConfig();
|
|
119
|
-
if (!saved) {
|
|
120
|
-
output_1.default.exitError(texts_1.ERR_SWW_AGENT_ENABLING);
|
|
121
|
-
}
|
|
122
|
-
logger_1.default.changeRootPath(manager_1.default.system.getNewAgentConfigDir());
|
|
123
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_STARTED);
|
|
124
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_STARTED);
|
|
125
|
-
process.on('SIGINT', onExit);
|
|
126
|
-
process.on('SIGTERM', onExit);
|
|
127
|
-
process.on('SIGQUIT', onExit);
|
|
128
|
-
manager_1.default.start(id, host, token, port, true, true);
|
|
129
|
-
});
|
|
130
|
-
const onExit = () => {
|
|
131
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_EXITING);
|
|
132
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_EXITING);
|
|
133
|
-
manager_1.default.system.clearStandaloneProcConfig();
|
|
134
|
-
process.exit(0);
|
|
135
|
-
};
|
|
136
|
-
exports.default = commandAgentStandalone;
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const texts_1 = require("../../texts");
|
|
8
|
-
const manager_1 = __importDefault(require("../../agent/manager"));
|
|
9
|
-
const output_1 = __importDefault(require("../../output"));
|
|
10
|
-
const buddy_1 = __importDefault(require("../../tunnel/api/buddy"));
|
|
11
|
-
const logger_1 = __importDefault(require("../../logger"));
|
|
12
|
-
const cfg_1 = __importDefault(require("../../tunnel/cfg"));
|
|
13
|
-
const input_1 = __importDefault(require("../../input"));
|
|
14
|
-
const commandAgentUp = (0, utils_1.newCommand)('up', texts_1.DESC_COMMAND_AGENT_STANDALONE);
|
|
15
|
-
commandAgentUp.option('-i, --id <id>', texts_1.OPTION_AGENT_ID);
|
|
16
|
-
commandAgentUp.option('-t, --token <token>', texts_1.OPTION_AGENT_TOKEN);
|
|
17
|
-
commandAgentUp.option('-p, --port <port>', texts_1.OPTION_AGENT_PORT, '7456');
|
|
18
|
-
commandAgentUp.option('--target', texts_1.OPTION_AGENT_TARGET);
|
|
19
|
-
commandAgentUp.action(async (options) => {
|
|
20
|
-
const hasAdminRights = await manager_1.default.system.hasAdminRights();
|
|
21
|
-
if (!hasAdminRights) {
|
|
22
|
-
output_1.default.exitError(texts_1.ERR_AGENT_ADMIN_RIGHTS);
|
|
23
|
-
}
|
|
24
|
-
try {
|
|
25
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKING);
|
|
26
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKING, false);
|
|
27
|
-
manager_1.default.system.killStandaloneProc();
|
|
28
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKED);
|
|
29
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_CHECKED);
|
|
30
|
-
}
|
|
31
|
-
catch (err) {
|
|
32
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_PROC_ERROR);
|
|
33
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_PROC_ERROR);
|
|
34
|
-
logger_1.default.error(err);
|
|
35
|
-
output_1.default.exitError(err);
|
|
36
|
-
}
|
|
37
|
-
const isEnabled = await manager_1.default.system.isEnabled();
|
|
38
|
-
if (isEnabled) {
|
|
39
|
-
output_1.default.exitError(texts_1.ERR_AGENT_STANDALONE_SERVICE_INSTALLED);
|
|
40
|
-
}
|
|
41
|
-
let id;
|
|
42
|
-
let host;
|
|
43
|
-
let token;
|
|
44
|
-
let port;
|
|
45
|
-
try {
|
|
46
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_CHECKING);
|
|
47
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_CHECKING, false);
|
|
48
|
-
const json = manager_1.default.system.loadSystemConfig();
|
|
49
|
-
id = json.id;
|
|
50
|
-
host = json.host;
|
|
51
|
-
token = json.token;
|
|
52
|
-
port = json.port;
|
|
53
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_FOUND);
|
|
54
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_FOUND);
|
|
55
|
-
}
|
|
56
|
-
catch {
|
|
57
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_CONFIG_NOT_FOUND);
|
|
58
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_CONFIG_NOT_FOUND);
|
|
59
|
-
}
|
|
60
|
-
if (!port) {
|
|
61
|
-
port = input_1.default.port(options.port);
|
|
62
|
-
}
|
|
63
|
-
if (!id && options.id) {
|
|
64
|
-
id = options.id;
|
|
65
|
-
logger_1.default.info((0, texts_1.TXT_AGENT_STANDALONE_USING_AGENT)(id));
|
|
66
|
-
output_1.default.normal((0, texts_1.TXT_AGENT_STANDALONE_USING_AGENT)(id));
|
|
67
|
-
}
|
|
68
|
-
if (!token) {
|
|
69
|
-
token = options.token;
|
|
70
|
-
}
|
|
71
|
-
if (!id) {
|
|
72
|
-
if (token)
|
|
73
|
-
cfg_1.default.setToken(token);
|
|
74
|
-
host = cfg_1.default.getTokenHost();
|
|
75
|
-
try {
|
|
76
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERING, false);
|
|
77
|
-
const agent = await buddy_1.default.register(true, !!options.target, host, cfg_1.default.getToken(), (region) => {
|
|
78
|
-
cfg_1.default.setRegionIfNotSet(region);
|
|
79
|
-
});
|
|
80
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERED);
|
|
81
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTERED);
|
|
82
|
-
id = agent.id;
|
|
83
|
-
token = agent.token;
|
|
84
|
-
agent.destroy();
|
|
85
|
-
}
|
|
86
|
-
catch (err) {
|
|
87
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTER_ERROR);
|
|
88
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_REGISTER_ERROR);
|
|
89
|
-
logger_1.default.error(err);
|
|
90
|
-
output_1.default.exitError(err);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
if (!host) {
|
|
95
|
-
host = cfg_1.default.getTokenHost();
|
|
96
|
-
}
|
|
97
|
-
try {
|
|
98
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHING);
|
|
99
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHING, false);
|
|
100
|
-
const agent = await buddy_1.default.fetchAgent(id, host, token);
|
|
101
|
-
agent.destroy();
|
|
102
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHED);
|
|
103
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCHED);
|
|
104
|
-
}
|
|
105
|
-
catch (err) {
|
|
106
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCH_ERROR);
|
|
107
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_AGENT_FETCH_ERROR);
|
|
108
|
-
logger_1.default.error(err);
|
|
109
|
-
output_1.default.exitError(err);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
let saved = manager_1.default.system.saveSystemConfig(id, host, token, port);
|
|
113
|
-
if (!saved) {
|
|
114
|
-
output_1.default.exitError(texts_1.ERR_SWW_AGENT_ENABLING);
|
|
115
|
-
}
|
|
116
|
-
saved = manager_1.default.system.saveStandaloneProcConfig();
|
|
117
|
-
if (!saved) {
|
|
118
|
-
output_1.default.exitError(texts_1.ERR_SWW_AGENT_ENABLING);
|
|
119
|
-
}
|
|
120
|
-
logger_1.default.changeRootPath(manager_1.default.system.getNewAgentConfigDir());
|
|
121
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_STARTED);
|
|
122
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_STARTED);
|
|
123
|
-
process.on('SIGINT', onExit);
|
|
124
|
-
process.on('SIGTERM', onExit);
|
|
125
|
-
process.on('SIGQUIT', onExit);
|
|
126
|
-
manager_1.default.start(id, host, token, port, true, true);
|
|
127
|
-
});
|
|
128
|
-
const onExit = () => {
|
|
129
|
-
logger_1.default.info(texts_1.TXT_AGENT_STANDALONE_EXITING);
|
|
130
|
-
output_1.default.normal(texts_1.TXT_AGENT_STANDALONE_EXITING);
|
|
131
|
-
manager_1.default.system.clearStandaloneProcConfig();
|
|
132
|
-
process.exit(0);
|
|
133
|
-
};
|
|
134
|
-
exports.default = commandAgentUp;
|
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const utils_1 = require("../../utils");
|
|
7
|
-
const commander_1 = require("commander");
|
|
8
|
-
const texts_1 = require("../../texts");
|
|
9
|
-
const validation_1 = require("../../visualTest/validation");
|
|
10
|
-
const output_1 = __importDefault(require("../../output"));
|
|
11
|
-
const requests_1 = require("../../visualTest/requests");
|
|
12
|
-
const zod_1 = require("zod");
|
|
13
|
-
const node_zlib_1 = require("node:zlib");
|
|
14
|
-
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
15
|
-
const promises_1 = require("node:stream/promises");
|
|
16
|
-
const node_fs_1 = require("node:fs");
|
|
17
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
-
const promises_2 = require("node:fs/promises");
|
|
19
|
-
const commandScrap = (0, utils_1.newCommand)('scrap', texts_1.DESC_COMMAND_VT_SCRAP);
|
|
20
|
-
commandScrap.argument('<url>', texts_1.OPTION_SCRAP_URL);
|
|
21
|
-
commandScrap.option('--follow', texts_1.OPTION_SCRAP_FOLLOW, false);
|
|
22
|
-
commandScrap.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAP_OUTPUT_TYPE)
|
|
23
|
-
.choices(['jpeg', 'png', 'md', 'html'])
|
|
24
|
-
.makeOptionMandatory());
|
|
25
|
-
commandScrap.option('--quality <quality>', texts_1.OPTION_SCRAP_QUALITY);
|
|
26
|
-
commandScrap.option('--fullPage', texts_1.OPTION_SCRAP_FULL_PAGE, false);
|
|
27
|
-
commandScrap.option('--cssSelector <selector>', texts_1.OPTION_SCRAP_CSS_SELECTOR);
|
|
28
|
-
commandScrap.option('--xpathSelector <selector>', texts_1.OPTION_SCRAP_XPATH_SELECTOR);
|
|
29
|
-
commandScrap.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAP_BROWSER)
|
|
30
|
-
.choices(['chrome', 'firefox', 'safari'])
|
|
31
|
-
.default('chrome'));
|
|
32
|
-
commandScrap.option('--viewport <viewport>', texts_1.OPTION_SCRAP_VIEWPORT, '1920x1080');
|
|
33
|
-
commandScrap.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAP_DEVICE_PIXEL_RATIO, '1');
|
|
34
|
-
commandScrap.option('--waitForElement <selector>', texts_1.OPTION_SCRAP_WAIT_FOR_ELEMENT);
|
|
35
|
-
commandScrap.option('--darkMode', texts_1.OPTION_SCRAP_DARK_MODE, false);
|
|
36
|
-
commandScrap.option('--delay <delay>', texts_1.OPTION_SCRAP_DELAY, '0');
|
|
37
|
-
commandScrap.option('--outputDir <dir>', texts_1.OPTION_SCRAP_OUTPUT_DIR, '.');
|
|
38
|
-
commandScrap.action(async (inputUrl, options) => {
|
|
39
|
-
if (!(0, validation_1.checkToken)()) {
|
|
40
|
-
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
41
|
-
}
|
|
42
|
-
const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
|
|
43
|
-
try {
|
|
44
|
-
const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
|
|
45
|
-
output_1.default.normal('Starting scrap session');
|
|
46
|
-
const status = await watchSessionStatus(buildId);
|
|
47
|
-
if (!status.ok) {
|
|
48
|
-
output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
|
|
49
|
-
}
|
|
50
|
-
output_1.default.normal('Downloading scrap package');
|
|
51
|
-
const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
|
|
52
|
-
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
53
|
-
const unpack = tar_stream_1.default.extract();
|
|
54
|
-
unpack.on('entry', async (header, stream, next) => {
|
|
55
|
-
const currentDir = process.cwd();
|
|
56
|
-
const preparedOutputDir = outputDir.startsWith('.')
|
|
57
|
-
? node_path_1.default.join(currentDir, outputDir)
|
|
58
|
-
: outputDir;
|
|
59
|
-
const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
|
|
60
|
-
try {
|
|
61
|
-
if (header.type === 'file') {
|
|
62
|
-
await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
|
|
63
|
-
const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
|
|
64
|
-
await (0, promises_1.pipeline)(stream, fileWriteStream);
|
|
65
|
-
next();
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
stream.resume();
|
|
69
|
-
next();
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
catch (entryError) {
|
|
73
|
-
output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
|
|
74
|
-
next(entryError);
|
|
75
|
-
}
|
|
76
|
-
});
|
|
77
|
-
await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
|
|
78
|
-
output_1.default.exitSuccess('Downloading scrap package finished');
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
output_1.default.exitError(`${error}`);
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
function validateInputAndOptions(input, options) {
|
|
85
|
-
const urlSchema = zod_1.z.string().url();
|
|
86
|
-
const optionsSchema = zod_1.z.object({
|
|
87
|
-
follow: zod_1.z.boolean(),
|
|
88
|
-
outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
|
|
89
|
-
quality: zod_1.z.coerce.number().min(1).max(100).optional(),
|
|
90
|
-
outputDir: zod_1.z.string().default('.'),
|
|
91
|
-
fullPage: zod_1.z.boolean().optional(),
|
|
92
|
-
cssSelector: zod_1.z.string().optional(),
|
|
93
|
-
xpathSelector: zod_1.z.string().optional(),
|
|
94
|
-
browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
|
|
95
|
-
viewport: zod_1.z
|
|
96
|
-
.string()
|
|
97
|
-
.refine((value) => {
|
|
98
|
-
const [width, height] = value.split('x');
|
|
99
|
-
return (width &&
|
|
100
|
-
height &&
|
|
101
|
-
!isNaN(Number(width)) &&
|
|
102
|
-
!isNaN(Number(height)) &&
|
|
103
|
-
Number(width) > 0 &&
|
|
104
|
-
Number(height) > 0);
|
|
105
|
-
}, 'Invalid viewport format, example: 1920x1080')
|
|
106
|
-
.transform((value) => {
|
|
107
|
-
const [width, height] = value.split('x');
|
|
108
|
-
return {
|
|
109
|
-
width: Number(width),
|
|
110
|
-
height: Number(height),
|
|
111
|
-
};
|
|
112
|
-
}),
|
|
113
|
-
devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
|
|
114
|
-
darkMode: zod_1.z.boolean(),
|
|
115
|
-
delay: zod_1.z.coerce.number().min(0).max(10000),
|
|
116
|
-
waitForElement: zod_1.z.string().optional(),
|
|
117
|
-
});
|
|
118
|
-
try {
|
|
119
|
-
const url = urlSchema.parse(input);
|
|
120
|
-
const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
|
|
121
|
-
if (typeof quality === 'number' && outputType !== 'jpeg') {
|
|
122
|
-
output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
|
|
123
|
-
}
|
|
124
|
-
if (cssSelector && xpathSelector) {
|
|
125
|
-
output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
|
|
126
|
-
}
|
|
127
|
-
return {
|
|
128
|
-
url,
|
|
129
|
-
follow,
|
|
130
|
-
outputType,
|
|
131
|
-
quality,
|
|
132
|
-
outputDir,
|
|
133
|
-
fullPage,
|
|
134
|
-
cssSelector,
|
|
135
|
-
xpathSelector,
|
|
136
|
-
browser,
|
|
137
|
-
viewport,
|
|
138
|
-
devicePixelRatio,
|
|
139
|
-
darkMode,
|
|
140
|
-
delay,
|
|
141
|
-
waitForElement,
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
catch (error) {
|
|
145
|
-
if (error instanceof zod_1.ZodError) {
|
|
146
|
-
output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
|
|
147
|
-
}
|
|
148
|
-
else {
|
|
149
|
-
throw error;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
async function watchSessionStatus(buildId) {
|
|
154
|
-
return new Promise((resolve) => {
|
|
155
|
-
const eventSource = (0, requests_1.connectToScrapSession)(buildId);
|
|
156
|
-
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
157
|
-
const data = JSON.parse(event.data);
|
|
158
|
-
if (data.status === 'GATHER_URLS_COMPLETED') {
|
|
159
|
-
output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
|
|
160
|
-
}
|
|
161
|
-
else if (data.status === 'GATHER_URLS_FAILED') {
|
|
162
|
-
output_1.default.error('Gathering URLs failed');
|
|
163
|
-
}
|
|
164
|
-
else if (data.status === 'SCRAPE_URL_COMPLETED') {
|
|
165
|
-
output_1.default.normal(`Scraping ${data.text} completed`);
|
|
166
|
-
}
|
|
167
|
-
else if (data.status === 'SCRAPE_URL_FAILED') {
|
|
168
|
-
output_1.default.error(`Scraping ${data.text} failed`);
|
|
169
|
-
}
|
|
170
|
-
else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
|
|
171
|
-
output_1.default.normal('Package created');
|
|
172
|
-
}
|
|
173
|
-
else if (data.status === 'CREATE_PACKAGE_FAILED') {
|
|
174
|
-
output_1.default.error('Package creation failed');
|
|
175
|
-
}
|
|
176
|
-
else if (data.status === 'FINISHED') {
|
|
177
|
-
eventSource.close();
|
|
178
|
-
output_1.default.normal('Scrap session finished');
|
|
179
|
-
resolve({ ok: true });
|
|
180
|
-
}
|
|
181
|
-
});
|
|
182
|
-
eventSource.addEventListener('error', (event) => {
|
|
183
|
-
if (event.code) {
|
|
184
|
-
eventSource.close();
|
|
185
|
-
if (event.code === 410) {
|
|
186
|
-
output_1.default.normal('Scrap session finished');
|
|
187
|
-
}
|
|
188
|
-
resolve({ ok: event.code === 410, error: event.code });
|
|
189
|
-
}
|
|
190
|
-
});
|
|
191
|
-
});
|
|
192
|
-
}
|
|
193
|
-
exports.default = commandScrap;
|