bdy 1.14.10-dev-package → 1.14.11-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/agent/manager.js +8 -2
- package/distTs/src/agent/system.js +26 -4
- package/distTs/src/api/client.js +25 -102
- package/distTs/src/command/agent/install.js +18 -1
- package/distTs/src/command/agent/run.js +1 -1
- package/distTs/src/command/agent/uninstall.js +6 -0
- package/distTs/src/command/pipeline/run.js +3 -3
- package/distTs/src/command/vt/scrap.js +193 -0
- package/distTs/src/index.js +0 -2
- package/distTs/src/input.js +1 -19
- package/distTs/src/output.js +2 -5
- package/distTs/src/texts.js +12 -60
- package/distTs/src/tunnel/server/ssh.js +3 -0
- package/package.json +1 -1
- package/distTs/src/command/package/download.js +0 -259
- package/distTs/src/command/package/publish.js +0 -229
- package/distTs/src/command/package.js +0 -14
package/distTs/package.json
CHANGED
|
@@ -273,6 +273,12 @@ class AgentManagerClass {
|
|
|
273
273
|
catch {
|
|
274
274
|
// do nothing
|
|
275
275
|
}
|
|
276
|
+
try {
|
|
277
|
+
this.system.clearNewAgentFiles();
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
// do nothing
|
|
281
|
+
}
|
|
276
282
|
output_1.default.exitError(txt);
|
|
277
283
|
}
|
|
278
284
|
async agentStop() {
|
|
@@ -491,7 +497,7 @@ class AgentManagerClass {
|
|
|
491
497
|
if (!forceParam) {
|
|
492
498
|
try {
|
|
493
499
|
// load from config
|
|
494
|
-
const json = this.system.
|
|
500
|
+
const json = this.system.loadNewAgentConfig();
|
|
495
501
|
this.shouldStart = !!json.shouldStart;
|
|
496
502
|
this.sshHostKey = json.sshHostKey;
|
|
497
503
|
}
|
|
@@ -508,7 +514,7 @@ class AgentManagerClass {
|
|
|
508
514
|
}
|
|
509
515
|
try {
|
|
510
516
|
// resave config
|
|
511
|
-
this.system.
|
|
517
|
+
this.system.saveNewAgentConfig(this.shouldStart, this.sshHostKey || '');
|
|
512
518
|
return true;
|
|
513
519
|
}
|
|
514
520
|
catch {
|
|
@@ -35,9 +35,15 @@ class AgentSystem {
|
|
|
35
35
|
getSystemConfigDir() {
|
|
36
36
|
return '';
|
|
37
37
|
}
|
|
38
|
+
getNewAgentConfigDir() {
|
|
39
|
+
return this.getSystemConfigDir();
|
|
40
|
+
}
|
|
38
41
|
getAgentConfigDir() {
|
|
39
42
|
return (0, utils_1.getHomeDirectory)();
|
|
40
43
|
}
|
|
44
|
+
getNewAgentConfigPath() {
|
|
45
|
+
return path_1.default.join(this.getNewAgentConfigDir(), 'srv.json');
|
|
46
|
+
}
|
|
41
47
|
getAgentConfigPath() {
|
|
42
48
|
return path_1.default.join(this.getAgentConfigDir(), 'agent.json');
|
|
43
49
|
}
|
|
@@ -78,8 +84,15 @@ class AgentSystem {
|
|
|
78
84
|
const str = (0, fs_1.readFileSync)(this.getSystemConfigPath(), 'utf-8');
|
|
79
85
|
return JSON.parse(str);
|
|
80
86
|
}
|
|
81
|
-
|
|
82
|
-
const
|
|
87
|
+
loadNewAgentConfig() {
|
|
88
|
+
const exists = (0, fs_1.existsSync)(this.getAgentConfigPath());
|
|
89
|
+
if (exists) {
|
|
90
|
+
(0, fs_1.cpSync)(this.getAgentConfigPath(), this.getNewAgentConfigPath(), {
|
|
91
|
+
force: true,
|
|
92
|
+
});
|
|
93
|
+
this.clearAgentFiles();
|
|
94
|
+
}
|
|
95
|
+
const str = (0, fs_1.readFileSync)(this.getNewAgentConfigPath(), 'utf-8');
|
|
83
96
|
return JSON.parse(str);
|
|
84
97
|
}
|
|
85
98
|
clearSystemFiles() {
|
|
@@ -89,6 +102,9 @@ class AgentSystem {
|
|
|
89
102
|
clearAgentFiles() {
|
|
90
103
|
(0, fs_1.rmSync)(this.getAgentConfigPath(), { force: true });
|
|
91
104
|
}
|
|
105
|
+
clearNewAgentFiles() {
|
|
106
|
+
(0, fs_1.rmSync)(this.getNewAgentConfigPath(), { force: true });
|
|
107
|
+
}
|
|
92
108
|
ensureSystemConfigDir() {
|
|
93
109
|
(0, fs_1.mkdirSync)(this.getSystemConfigDir(), { recursive: true });
|
|
94
110
|
}
|
|
@@ -160,10 +176,16 @@ class AgentSystem {
|
|
|
160
176
|
return false;
|
|
161
177
|
}
|
|
162
178
|
}
|
|
163
|
-
|
|
179
|
+
saveNewAgentConfig(shouldStart, sshHostKey) {
|
|
180
|
+
try {
|
|
181
|
+
this.clearAgentFiles();
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
// do nothing
|
|
185
|
+
}
|
|
164
186
|
try {
|
|
165
187
|
logger_1.default.info(texts_1.LOG_SAVING_AGENT_LOCAL_CONFIG);
|
|
166
|
-
(0, fs_1.writeFileSync)(this.
|
|
188
|
+
(0, fs_1.writeFileSync)(this.getNewAgentConfigPath(), JSON.stringify({
|
|
167
189
|
shouldStart,
|
|
168
190
|
sshHostKey,
|
|
169
191
|
}), 'utf-8');
|
package/distTs/src/api/client.js
CHANGED
|
@@ -19,12 +19,26 @@ class ApiClient {
|
|
|
19
19
|
},
|
|
20
20
|
});
|
|
21
21
|
}
|
|
22
|
-
async
|
|
22
|
+
async request(method, path, body, parseResponseBody = false) {
|
|
23
|
+
const headers = {
|
|
24
|
+
authorization: `Bearer ${this.token}`,
|
|
25
|
+
};
|
|
26
|
+
let bodyParsed = undefined;
|
|
27
|
+
if (body) {
|
|
28
|
+
headers['content-type'] = 'application/json; charset=utf-8';
|
|
29
|
+
bodyParsed = JSON.stringify(body);
|
|
30
|
+
}
|
|
31
|
+
const opts = {
|
|
32
|
+
method,
|
|
33
|
+
path,
|
|
34
|
+
headers,
|
|
35
|
+
body: bodyParsed,
|
|
36
|
+
};
|
|
23
37
|
let status;
|
|
24
38
|
let responseBody;
|
|
25
|
-
logger_1.default.debug(`API CLIENT: ${
|
|
26
|
-
logger_1.default.debug(
|
|
27
|
-
logger_1.default.debug(
|
|
39
|
+
logger_1.default.debug(`API CLIENT: ${method} ${this.baseUrl.protocol}//${this.baseUrl.host}${path}`);
|
|
40
|
+
logger_1.default.debug(headers);
|
|
41
|
+
logger_1.default.debug(body);
|
|
28
42
|
try {
|
|
29
43
|
const r = await this.client.request(opts);
|
|
30
44
|
status = r.statusCode;
|
|
@@ -40,7 +54,11 @@ class ApiClient {
|
|
|
40
54
|
await responseBody.dump();
|
|
41
55
|
throw new Error(texts_1.ERR_REST_API_WRONG_TOKEN);
|
|
42
56
|
}
|
|
43
|
-
if (
|
|
57
|
+
if (status === 403) {
|
|
58
|
+
await responseBody.dump();
|
|
59
|
+
throw new Error(texts_1.ERR_REST_API_RATE_LIMIT);
|
|
60
|
+
}
|
|
61
|
+
if ([400, 404].includes(status)) {
|
|
44
62
|
let json;
|
|
45
63
|
try {
|
|
46
64
|
json = await responseBody.json();
|
|
@@ -61,7 +79,7 @@ class ApiClient {
|
|
|
61
79
|
else
|
|
62
80
|
throw new Error(texts_1.ERR_REST_API_GENERAL_ERROR);
|
|
63
81
|
}
|
|
64
|
-
if (
|
|
82
|
+
if (status === 200) {
|
|
65
83
|
if (parseResponseBody) {
|
|
66
84
|
try {
|
|
67
85
|
const b = await responseBody.json();
|
|
@@ -75,9 +93,6 @@ class ApiClient {
|
|
|
75
93
|
throw new Error(texts_1.ERR_REST_API_GENERAL_ERROR);
|
|
76
94
|
}
|
|
77
95
|
}
|
|
78
|
-
else if (returnRawBody) {
|
|
79
|
-
return responseBody;
|
|
80
|
-
}
|
|
81
96
|
else {
|
|
82
97
|
await responseBody.dump();
|
|
83
98
|
return null;
|
|
@@ -88,59 +103,8 @@ class ApiClient {
|
|
|
88
103
|
throw new Error(texts_1.ERR_REST_API_GENERAL_ERROR);
|
|
89
104
|
}
|
|
90
105
|
}
|
|
91
|
-
async requestMultipart(path, body, parseResponseBody = false) {
|
|
92
|
-
const headers = {
|
|
93
|
-
authorization: `Bearer ${this.token}`,
|
|
94
|
-
};
|
|
95
|
-
const opts = {
|
|
96
|
-
method: 'POST',
|
|
97
|
-
path,
|
|
98
|
-
headers,
|
|
99
|
-
body,
|
|
100
|
-
};
|
|
101
|
-
return await this._request(opts, parseResponseBody);
|
|
102
|
-
}
|
|
103
|
-
async request(method, path, body, parseResponseBody = false, returnRawBody = false) {
|
|
104
|
-
const headers = {
|
|
105
|
-
authorization: `Bearer ${this.token}`,
|
|
106
|
-
};
|
|
107
|
-
let bodyParsed = undefined;
|
|
108
|
-
if (body) {
|
|
109
|
-
headers['content-type'] = 'application/json; charset=utf-8';
|
|
110
|
-
bodyParsed = JSON.stringify(body);
|
|
111
|
-
}
|
|
112
|
-
const opts = {
|
|
113
|
-
method,
|
|
114
|
-
path,
|
|
115
|
-
headers,
|
|
116
|
-
body: bodyParsed,
|
|
117
|
-
};
|
|
118
|
-
return await this._request(opts, parseResponseBody, returnRawBody);
|
|
119
|
-
}
|
|
120
|
-
async getResourceByIdentifier(workspace, params) {
|
|
121
|
-
let query = '';
|
|
122
|
-
Object.keys(params).forEach((key) => {
|
|
123
|
-
if (!query)
|
|
124
|
-
query += '?';
|
|
125
|
-
else
|
|
126
|
-
query += '&';
|
|
127
|
-
query += encodeURIComponent(key) + '=' + encodeURIComponent(params[key]);
|
|
128
|
-
});
|
|
129
|
-
return await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/identifiers${query}`, null, true);
|
|
130
|
-
}
|
|
131
106
|
async getPipelineByIdentifier(workspace, project, identifier) {
|
|
132
|
-
return await this.
|
|
133
|
-
}
|
|
134
|
-
async getPackageVersionByIdentifier(workspace, project, pkg, version) {
|
|
135
|
-
const opts = {
|
|
136
|
-
package: pkg,
|
|
137
|
-
package_version: version
|
|
138
|
-
};
|
|
139
|
-
if (project)
|
|
140
|
-
opts.project = project;
|
|
141
|
-
if (version)
|
|
142
|
-
opts.package_version = version;
|
|
143
|
-
return await this.getResourceByIdentifier(workspace, opts);
|
|
107
|
+
return await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/identifiers?project=${encodeURIComponent(project)}&pipeline=${encodeURIComponent(identifier)}`, null, true);
|
|
144
108
|
}
|
|
145
109
|
async getPipelineRun(workspace, project, pipelineId, executionId) {
|
|
146
110
|
return await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/projects/${encodeURIComponent(project)}/pipelines/${encodeURIComponent(pipelineId)}/executions/${encodeURIComponent(executionId)}`, null, true);
|
|
@@ -148,46 +112,5 @@ class ApiClient {
|
|
|
148
112
|
async postPipelineRun(workspace, project, pipelineId, body) {
|
|
149
113
|
return await this.request('POST', `/workspaces/${encodeURIComponent(workspace)}/projects/${encodeURIComponent(project)}/pipelines/${encodeURIComponent(pipelineId)}/executions`, body, true);
|
|
150
114
|
}
|
|
151
|
-
async getPackageVersion(workspace, pkgId, versionId) {
|
|
152
|
-
return await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/packages/${encodeURIComponent(pkgId)}/versions/${encodeURIComponent(versionId)}`, null, true);
|
|
153
|
-
}
|
|
154
|
-
async getPackageLatest(workspace, pkgId) {
|
|
155
|
-
const res = await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/packages/${encodeURIComponent(pkgId)}/versions?page=1&per_page=1`, null, true);
|
|
156
|
-
if (res && res.versions && res.versions.length > 0) {
|
|
157
|
-
return res.versions[0];
|
|
158
|
-
}
|
|
159
|
-
return null;
|
|
160
|
-
}
|
|
161
|
-
async downloadPackageVersion(workspace, pkgId, versionId) {
|
|
162
|
-
return await this.request('GET', `/workspaces/${encodeURIComponent(workspace)}/packages/${encodeURIComponent(pkgId)}/versions/${encodeURIComponent(versionId)}/download`, null, false, true);
|
|
163
|
-
}
|
|
164
|
-
async postPackageVersion(workspace, pkgId, version) {
|
|
165
|
-
return await this.request('POST', `/workspaces/${encodeURIComponent(workspace)}/packages/${encodeURIComponent(pkgId)}/versions`, {
|
|
166
|
-
version
|
|
167
|
-
}, true);
|
|
168
|
-
}
|
|
169
|
-
async postPackageVersionZip(workspace, pkgId, versionId, file) {
|
|
170
|
-
const form = new undici_1.FormData();
|
|
171
|
-
form.append('file', file);
|
|
172
|
-
return await this.requestMultipart(`/workspaces/${encodeURIComponent(workspace)}/packages/${encodeURIComponent(pkgId)}/versions/${versionId}/upload`, form, true);
|
|
173
|
-
}
|
|
174
|
-
async postPackage(workspace, project, identifier) {
|
|
175
|
-
const body = {
|
|
176
|
-
name: identifier,
|
|
177
|
-
identifier,
|
|
178
|
-
type: 'FILE',
|
|
179
|
-
scope: 'WORKSPACE',
|
|
180
|
-
authorization: {
|
|
181
|
-
type: 'BUDDY'
|
|
182
|
-
}
|
|
183
|
-
};
|
|
184
|
-
if (project) {
|
|
185
|
-
body.project = {
|
|
186
|
-
name: project
|
|
187
|
-
};
|
|
188
|
-
body.scope = 'PROJECT';
|
|
189
|
-
}
|
|
190
|
-
return await this.request('POST', `/workspaces/${encodeURIComponent(workspace)}/packages`, body, true);
|
|
191
|
-
}
|
|
192
115
|
}
|
|
193
116
|
exports.default = ApiClient;
|
|
@@ -37,6 +37,12 @@ const removeAllAndExit = async (txt, id, host, token) => {
|
|
|
37
37
|
catch {
|
|
38
38
|
// do nothing
|
|
39
39
|
}
|
|
40
|
+
try {
|
|
41
|
+
manager_1.default.system.clearNewAgentFiles();
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// do nothing
|
|
45
|
+
}
|
|
40
46
|
output_1.default.exitError(txt);
|
|
41
47
|
};
|
|
42
48
|
const commandAgentInstall = (0, utils_1.newCommand)('install', texts_1.DESC_COMMAND_AGENT_INSTALL);
|
|
@@ -95,7 +101,18 @@ commandAgentInstall.action(async (options) => {
|
|
|
95
101
|
if (!saved) {
|
|
96
102
|
await removeAllAndExit(texts_1.ERR_SWW_AGENT_ENABLING, id, host, token);
|
|
97
103
|
}
|
|
98
|
-
|
|
104
|
+
try {
|
|
105
|
+
manager_1.default.system.clearAgentFiles();
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
// do nothing
|
|
109
|
+
}
|
|
110
|
+
try {
|
|
111
|
+
manager_1.default.system.clearNewAgentFiles();
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
// do nothing
|
|
115
|
+
}
|
|
99
116
|
await output_1.default.spinner(texts_1.TXT_ENABLING_AGENT);
|
|
100
117
|
if (process.env.DEBUG === '1') {
|
|
101
118
|
manager_1.default.start(id, host, token, port, !!options.start);
|
|
@@ -13,7 +13,7 @@ commandAgentRun.option('--token <token>');
|
|
|
13
13
|
commandAgentRun.option('--port <port>');
|
|
14
14
|
commandAgentRun.option('--start <start>');
|
|
15
15
|
commandAgentRun.action(async (options) => {
|
|
16
|
-
logger_1.default.changeRootPath(manager_1.default.system.
|
|
16
|
+
logger_1.default.changeRootPath(manager_1.default.system.getNewAgentConfigDir());
|
|
17
17
|
manager_1.default.start(options.id, options.host, options.token, parseInt(options.port, 10), String(options.start) === 'true');
|
|
18
18
|
});
|
|
19
19
|
exports.default = commandAgentRun;
|
|
@@ -39,6 +39,12 @@ commandAgentUninstall.action(async () => {
|
|
|
39
39
|
catch {
|
|
40
40
|
// do nothing
|
|
41
41
|
}
|
|
42
|
+
try {
|
|
43
|
+
manager_1.default.system.clearNewAgentFiles();
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// do nothing
|
|
47
|
+
}
|
|
42
48
|
output_1.default.exitSuccess(texts_1.TXT_AGENT_DISABLED);
|
|
43
49
|
});
|
|
44
50
|
exports.default = commandAgentUninstall;
|
|
@@ -38,13 +38,13 @@ commandPipelineRun.action(async (identifier, options) => {
|
|
|
38
38
|
const client = new client_1.default(baseUrl, token);
|
|
39
39
|
const data = await client.getPipelineByIdentifier(workspace, project, identifier);
|
|
40
40
|
if (!data || !data.domain) {
|
|
41
|
-
output_1.default.exitError(texts_1.
|
|
41
|
+
output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_WORKSPACE_NOT_FOUND);
|
|
42
42
|
}
|
|
43
43
|
if (!data.project_identifier) {
|
|
44
|
-
output_1.default.exitError(texts_1.
|
|
44
|
+
output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_PROJECT_NOT_FOUND);
|
|
45
45
|
}
|
|
46
46
|
if (!data.pipeline_id) {
|
|
47
|
-
output_1.default.exitError(texts_1.
|
|
47
|
+
output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_NOT_FOUND);
|
|
48
48
|
}
|
|
49
49
|
const body = {};
|
|
50
50
|
if (options.branch) {
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../../utils");
|
|
7
|
+
const commander_1 = require("commander");
|
|
8
|
+
const texts_1 = require("../../texts");
|
|
9
|
+
const validation_1 = require("../../visualTest/validation");
|
|
10
|
+
const output_1 = __importDefault(require("../../output"));
|
|
11
|
+
const requests_1 = require("../../visualTest/requests");
|
|
12
|
+
const zod_1 = require("zod");
|
|
13
|
+
const node_zlib_1 = require("node:zlib");
|
|
14
|
+
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
15
|
+
const promises_1 = require("node:stream/promises");
|
|
16
|
+
const node_fs_1 = require("node:fs");
|
|
17
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
+
const promises_2 = require("node:fs/promises");
|
|
19
|
+
const commandScrap = (0, utils_1.newCommand)('scrap', texts_1.DESC_COMMAND_VT_SCRAP);
|
|
20
|
+
commandScrap.argument('<url>', texts_1.OPTION_SCRAP_URL);
|
|
21
|
+
commandScrap.option('--follow', texts_1.OPTION_SCRAP_FOLLOW, false);
|
|
22
|
+
commandScrap.addOption(new commander_1.Option('--outputType <type>', texts_1.OPTION_SCRAP_OUTPUT_TYPE)
|
|
23
|
+
.choices(['jpeg', 'png', 'md', 'html'])
|
|
24
|
+
.makeOptionMandatory());
|
|
25
|
+
commandScrap.option('--quality <quality>', texts_1.OPTION_SCRAP_QUALITY);
|
|
26
|
+
commandScrap.option('--fullPage', texts_1.OPTION_SCRAP_FULL_PAGE, false);
|
|
27
|
+
commandScrap.option('--cssSelector <selector>', texts_1.OPTION_SCRAP_CSS_SELECTOR);
|
|
28
|
+
commandScrap.option('--xpathSelector <selector>', texts_1.OPTION_SCRAP_XPATH_SELECTOR);
|
|
29
|
+
commandScrap.addOption(new commander_1.Option('--browser <browser>', texts_1.OPTION_SCRAP_BROWSER)
|
|
30
|
+
.choices(['chrome', 'firefox', 'safari'])
|
|
31
|
+
.default('chrome'));
|
|
32
|
+
commandScrap.option('--viewport <viewport>', texts_1.OPTION_SCRAP_VIEWPORT, '1920x1080');
|
|
33
|
+
commandScrap.option('--devicePixelRatio <ratio>', texts_1.OPTION_SCRAP_DEVICE_PIXEL_RATIO, '1');
|
|
34
|
+
commandScrap.option('--waitForElement <selector>', texts_1.OPTION_SCRAP_WAIT_FOR_ELEMENT);
|
|
35
|
+
commandScrap.option('--darkMode', texts_1.OPTION_SCRAP_DARK_MODE, false);
|
|
36
|
+
commandScrap.option('--delay <delay>', texts_1.OPTION_SCRAP_DELAY, '0');
|
|
37
|
+
commandScrap.option('--outputDir <dir>', texts_1.OPTION_SCRAP_OUTPUT_DIR, '.');
|
|
38
|
+
commandScrap.action(async (inputUrl, options) => {
|
|
39
|
+
if (!(0, validation_1.checkToken)()) {
|
|
40
|
+
output_1.default.exitError(texts_1.ERR_MISSING_VT_TOKEN);
|
|
41
|
+
}
|
|
42
|
+
const { url, follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = validateInputAndOptions(inputUrl, options);
|
|
43
|
+
try {
|
|
44
|
+
const { buildId } = await (0, requests_1.sendScrap)(url, outputType, follow, quality, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement);
|
|
45
|
+
output_1.default.normal('Starting scrap session');
|
|
46
|
+
const status = await watchSessionStatus(buildId);
|
|
47
|
+
if (!status.ok) {
|
|
48
|
+
output_1.default.exitError(`Unexpected error while watching session status: ${status.error}`);
|
|
49
|
+
}
|
|
50
|
+
output_1.default.normal('Downloading scrap package');
|
|
51
|
+
const scrapPackageStream = await (0, requests_1.downloadScrapPackage)(buildId);
|
|
52
|
+
const brotliDecompressor = (0, node_zlib_1.createBrotliDecompress)();
|
|
53
|
+
const unpack = tar_stream_1.default.extract();
|
|
54
|
+
unpack.on('entry', async (header, stream, next) => {
|
|
55
|
+
const currentDir = process.cwd();
|
|
56
|
+
const preparedOutputDir = outputDir.startsWith('.')
|
|
57
|
+
? node_path_1.default.join(currentDir, outputDir)
|
|
58
|
+
: outputDir;
|
|
59
|
+
const newFilePath = node_path_1.default.join(preparedOutputDir, header.name);
|
|
60
|
+
try {
|
|
61
|
+
if (header.type === 'file') {
|
|
62
|
+
await (0, promises_2.mkdir)(node_path_1.default.dirname(newFilePath), { recursive: true });
|
|
63
|
+
const fileWriteStream = (0, node_fs_1.createWriteStream)(newFilePath);
|
|
64
|
+
await (0, promises_1.pipeline)(stream, fileWriteStream);
|
|
65
|
+
next();
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
stream.resume();
|
|
69
|
+
next();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
catch (entryError) {
|
|
73
|
+
output_1.default.error(`Error processing entry ${header.name}: ${entryError}`);
|
|
74
|
+
next(entryError);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
await (0, promises_1.pipeline)(scrapPackageStream, brotliDecompressor, unpack);
|
|
78
|
+
output_1.default.exitSuccess('Downloading scrap package finished');
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
output_1.default.exitError(`${error}`);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
function validateInputAndOptions(input, options) {
|
|
85
|
+
const urlSchema = zod_1.z.string().url();
|
|
86
|
+
const optionsSchema = zod_1.z.object({
|
|
87
|
+
follow: zod_1.z.boolean(),
|
|
88
|
+
outputType: zod_1.z.enum(['jpeg', 'png', 'md', 'html']),
|
|
89
|
+
quality: zod_1.z.coerce.number().min(1).max(100).optional(),
|
|
90
|
+
outputDir: zod_1.z.string().default('.'),
|
|
91
|
+
fullPage: zod_1.z.boolean().optional(),
|
|
92
|
+
cssSelector: zod_1.z.string().optional(),
|
|
93
|
+
xpathSelector: zod_1.z.string().optional(),
|
|
94
|
+
browser: zod_1.z.enum(['chrome', 'firefox', 'safari']),
|
|
95
|
+
viewport: zod_1.z
|
|
96
|
+
.string()
|
|
97
|
+
.refine((value) => {
|
|
98
|
+
const [width, height] = value.split('x');
|
|
99
|
+
return (width &&
|
|
100
|
+
height &&
|
|
101
|
+
!isNaN(Number(width)) &&
|
|
102
|
+
!isNaN(Number(height)) &&
|
|
103
|
+
Number(width) > 0 &&
|
|
104
|
+
Number(height) > 0);
|
|
105
|
+
}, 'Invalid viewport format, example: 1920x1080')
|
|
106
|
+
.transform((value) => {
|
|
107
|
+
const [width, height] = value.split('x');
|
|
108
|
+
return {
|
|
109
|
+
width: Number(width),
|
|
110
|
+
height: Number(height),
|
|
111
|
+
};
|
|
112
|
+
}),
|
|
113
|
+
devicePixelRatio: zod_1.z.coerce.number().min(1).max(4),
|
|
114
|
+
darkMode: zod_1.z.boolean(),
|
|
115
|
+
delay: zod_1.z.coerce.number().min(0).max(10000),
|
|
116
|
+
waitForElement: zod_1.z.string().optional(),
|
|
117
|
+
});
|
|
118
|
+
try {
|
|
119
|
+
const url = urlSchema.parse(input);
|
|
120
|
+
const { follow, outputType, quality, outputDir, fullPage, cssSelector, xpathSelector, browser, viewport, devicePixelRatio, darkMode, delay, waitForElement, } = optionsSchema.parse(options);
|
|
121
|
+
if (typeof quality === 'number' && outputType !== 'jpeg') {
|
|
122
|
+
output_1.default.exitError('Quality is only supported for jpeg output type, use --outputType jpeg');
|
|
123
|
+
}
|
|
124
|
+
if (cssSelector && xpathSelector) {
|
|
125
|
+
output_1.default.exitError('Only one of --cssSelector or --xpathSelector can be used');
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
url,
|
|
129
|
+
follow,
|
|
130
|
+
outputType,
|
|
131
|
+
quality,
|
|
132
|
+
outputDir,
|
|
133
|
+
fullPage,
|
|
134
|
+
cssSelector,
|
|
135
|
+
xpathSelector,
|
|
136
|
+
browser,
|
|
137
|
+
viewport,
|
|
138
|
+
devicePixelRatio,
|
|
139
|
+
darkMode,
|
|
140
|
+
delay,
|
|
141
|
+
waitForElement,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (error instanceof zod_1.ZodError) {
|
|
146
|
+
output_1.default.exitError(error.errors.map((e) => `${e.path}: ${e.message}`).join(', '));
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
throw error;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async function watchSessionStatus(buildId) {
|
|
154
|
+
return new Promise((resolve) => {
|
|
155
|
+
const eventSource = (0, requests_1.connectToScrapSession)(buildId);
|
|
156
|
+
eventSource.addEventListener('SESSION_STATUS', (event) => {
|
|
157
|
+
const data = JSON.parse(event.data);
|
|
158
|
+
if (data.status === 'GATHER_URLS_COMPLETED') {
|
|
159
|
+
output_1.default.normal(`Gathering URLs completed, found ${data.text} URLs`);
|
|
160
|
+
}
|
|
161
|
+
else if (data.status === 'GATHER_URLS_FAILED') {
|
|
162
|
+
output_1.default.error('Gathering URLs failed');
|
|
163
|
+
}
|
|
164
|
+
else if (data.status === 'SCRAPE_URL_COMPLETED') {
|
|
165
|
+
output_1.default.normal(`Scraping ${data.text} completed`);
|
|
166
|
+
}
|
|
167
|
+
else if (data.status === 'SCRAPE_URL_FAILED') {
|
|
168
|
+
output_1.default.error(`Scraping ${data.text} failed`);
|
|
169
|
+
}
|
|
170
|
+
else if (data.status === 'CREATE_PACKAGE_COMPLETED') {
|
|
171
|
+
output_1.default.normal('Package created');
|
|
172
|
+
}
|
|
173
|
+
else if (data.status === 'CREATE_PACKAGE_FAILED') {
|
|
174
|
+
output_1.default.error('Package creation failed');
|
|
175
|
+
}
|
|
176
|
+
else if (data.status === 'FINISHED') {
|
|
177
|
+
eventSource.close();
|
|
178
|
+
output_1.default.normal('Scrap session finished');
|
|
179
|
+
resolve({ ok: true });
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
eventSource.addEventListener('error', (event) => {
|
|
183
|
+
if (event.code) {
|
|
184
|
+
eventSource.close();
|
|
185
|
+
if (event.code === 410) {
|
|
186
|
+
output_1.default.normal('Scrap session finished');
|
|
187
|
+
}
|
|
188
|
+
resolve({ ok: event.code === 410, error: event.code });
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
exports.default = commandScrap;
|
package/distTs/src/index.js
CHANGED
|
@@ -15,7 +15,6 @@ const vt_1 = __importDefault(require("./command/vt"));
|
|
|
15
15
|
const ut_1 = __importDefault(require("./command/ut"));
|
|
16
16
|
const tunnel_1 = __importDefault(require("./command/tunnel"));
|
|
17
17
|
const pipeline_1 = __importDefault(require("./command/pipeline"));
|
|
18
|
-
const package_1 = __importDefault(require("./command/package"));
|
|
19
18
|
stream_1.default.setDefaultHighWaterMark(false, 67108864);
|
|
20
19
|
process.title = 'bdy';
|
|
21
20
|
process.on('uncaughtException', (err) => {
|
|
@@ -34,5 +33,4 @@ program.addCommand(version_1.default);
|
|
|
34
33
|
program.addCommand(vt_1.default);
|
|
35
34
|
program.addCommand(ut_1.default);
|
|
36
35
|
program.addCommand(pipeline_1.default);
|
|
37
|
-
program.addCommand(package_1.default);
|
|
38
36
|
program.parse();
|
package/distTs/src/input.js
CHANGED
|
@@ -45,8 +45,6 @@ const crypto_1 = __importDefault(require("crypto"));
|
|
|
45
45
|
const utils_1 = require("./utils");
|
|
46
46
|
const texts_1 = require("./texts");
|
|
47
47
|
const tunnel_1 = require("./types/tunnel");
|
|
48
|
-
const uuid_1 = require("uuid");
|
|
49
|
-
const node_path_1 = require("node:path");
|
|
50
48
|
class Input {
|
|
51
49
|
static timeout(timeout) {
|
|
52
50
|
const t = parseInt(timeout, 10);
|
|
@@ -341,14 +339,6 @@ class Input {
|
|
|
341
339
|
t = 1440;
|
|
342
340
|
return t;
|
|
343
341
|
}
|
|
344
|
-
static resolvePath(path) {
|
|
345
|
-
const p = (0, node_path_1.resolve)(path);
|
|
346
|
-
const exists = fs_1.default.existsSync(p);
|
|
347
|
-
if (!exists) {
|
|
348
|
-
output_1.default.exitError(texts_1.ERR_PATH_NOT_EXISTS);
|
|
349
|
-
}
|
|
350
|
-
return p;
|
|
351
|
-
}
|
|
352
342
|
static restApiWorkspace(workspace) {
|
|
353
343
|
let w = process.env.BUDDY_WORKSPACE;
|
|
354
344
|
if (workspace)
|
|
@@ -358,23 +348,15 @@ class Input {
|
|
|
358
348
|
}
|
|
359
349
|
return w;
|
|
360
350
|
}
|
|
361
|
-
static restApiProject(project
|
|
351
|
+
static restApiProject(project) {
|
|
362
352
|
let p = process.env.BUDDY_PROJECT;
|
|
363
353
|
if (project)
|
|
364
354
|
p = project;
|
|
365
355
|
if (!p) {
|
|
366
|
-
if (allowNull)
|
|
367
|
-
return null;
|
|
368
356
|
output_1.default.exitError(texts_1.ERR_REST_API_PROJECT);
|
|
369
357
|
}
|
|
370
358
|
return p;
|
|
371
359
|
}
|
|
372
|
-
static restApiPackageVersion(version) {
|
|
373
|
-
let v = version;
|
|
374
|
-
if (!v)
|
|
375
|
-
v = (0, uuid_1.v4)();
|
|
376
|
-
return v;
|
|
377
|
-
}
|
|
378
360
|
static name(name) {
|
|
379
361
|
if (name.includes('*')) {
|
|
380
362
|
output_1.default.exitError(texts_1.ERR_NAME_WITHOUT_ASTERISK);
|
package/distTs/src/output.js
CHANGED
|
@@ -29,11 +29,8 @@ class Output {
|
|
|
29
29
|
static newline() {
|
|
30
30
|
terminal('\n');
|
|
31
31
|
}
|
|
32
|
-
static normal(txt
|
|
33
|
-
|
|
34
|
-
if (newLine)
|
|
35
|
-
msg += '\n';
|
|
36
|
-
terminal(msg);
|
|
32
|
+
static normal(txt) {
|
|
33
|
+
terminal(`${txt}\n`);
|
|
37
34
|
}
|
|
38
35
|
static warning(txt) {
|
|
39
36
|
terminal.yellow(`${txt}\n`);
|