bdy 1.19.0-dev → 1.19.0-dev-pipeline
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/api/client.js +61 -2
- package/distTs/src/command/crawl/link.js +61 -0
- package/distTs/src/command/crawl/run.js +147 -0
- package/distTs/src/command/crawl/validation.js +154 -0
- package/distTs/src/command/crawl.js +13 -0
- package/distTs/src/command/pipeline/run/apply.js +62 -0
- package/distTs/src/command/pipeline/run/approve.js +62 -0
- package/distTs/src/command/pipeline/run/cancel.js +36 -0
- package/distTs/src/command/pipeline/run/list.js +52 -0
- package/distTs/src/command/pipeline/run/logs.js +37 -0
- package/distTs/src/command/pipeline/run/retry.js +36 -0
- package/distTs/src/command/pipeline/run/start.js +96 -0
- package/distTs/src/command/pipeline/run/status.js +35 -0
- package/distTs/src/command/pipeline/run.js +22 -125
- package/distTs/src/command/project/link.js +11 -11
- package/distTs/src/command/tests/capture/validation.js +46 -0
- package/distTs/src/command/tests/capture.js +103 -0
- package/distTs/src/command/tests/unit/link.js +61 -0
- package/distTs/src/command/tests/unit/upload.js +91 -0
- package/distTs/src/command/tests/unit.js +13 -0
- package/distTs/src/command/tests/visual/link.js +61 -0
- package/distTs/src/command/tests/visual/session/close.js +32 -0
- package/distTs/src/command/tests/visual/session/create.js +86 -0
- package/distTs/src/command/tests/visual/session.js +13 -0
- package/distTs/src/command/tests/visual/setup.js +20 -0
- package/distTs/src/command/tests/visual/shared/validation.js +145 -0
- package/distTs/src/command/tests/visual/upload.js +141 -0
- package/distTs/src/command/tests/visual.js +17 -0
- package/distTs/src/command/tests.js +15 -0
- package/distTs/src/crawl/requests.js +141 -0
- package/distTs/src/input.js +11 -10
- package/distTs/src/output/pipeline.js +1507 -0
- package/distTs/src/output.js +149 -31
- package/distTs/src/texts.js +55 -33
- package/distTs/src/tunnel/output/interactive/tunnel.js +2 -2
- package/distTs/src/types/crawl.js +2 -0
- package/distTs/src/types/pipeline.js +424 -0
- package/distTs/src/unitTest/context.js +26 -0
- package/package.json +1 -1
- package/distTs/src/command/project/get.js +0 -18
- package/distTs/src/command/project/set.js +0 -31
- package/distTs/src/command/sandbox/get/yaml.js +0 -30
- package/distTs/src/command/vt/scrape.js +0 -193
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../../utils");
|
|
7
|
+
const texts_1 = require("../../texts");
|
|
8
|
+
const setup_1 = __importDefault(require("./visual/setup"));
|
|
9
|
+
const upload_1 = __importDefault(require("./visual/upload"));
|
|
10
|
+
const session_1 = __importDefault(require("./visual/session"));
|
|
11
|
+
const link_1 = __importDefault(require("./visual/link"));
|
|
12
|
+
const commandVisual = (0, utils_1.newCommand)('visual', texts_1.DESC_COMMAND_VISUAL);
|
|
13
|
+
commandVisual.addCommand(setup_1.default);
|
|
14
|
+
commandVisual.addCommand(upload_1.default);
|
|
15
|
+
commandVisual.addCommand(session_1.default);
|
|
16
|
+
commandVisual.addCommand(link_1.default);
|
|
17
|
+
exports.default = commandVisual;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const utils_1 = require("../utils");
|
|
7
|
+
const texts_1 = require("../texts");
|
|
8
|
+
const unit_1 = __importDefault(require("./tests/unit"));
|
|
9
|
+
const visual_1 = __importDefault(require("./tests/visual"));
|
|
10
|
+
const capture_1 = __importDefault(require("./tests/capture"));
|
|
11
|
+
const commandTests = (0, utils_1.newCommand)('tests', texts_1.DESC_COMMAND_TESTS);
|
|
12
|
+
commandTests.addCommand(unit_1.default);
|
|
13
|
+
commandTests.addCommand(visual_1.default);
|
|
14
|
+
commandTests.addCommand(capture_1.default);
|
|
15
|
+
exports.default = commandTests;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.sendCrawl = sendCrawl;
|
|
7
|
+
exports.downloadCrawlPackage = downloadCrawlPackage;
|
|
8
|
+
exports.connectToCrawlSession = connectToCrawlSession;
|
|
9
|
+
const context_1 = require("../visualTest/context");
|
|
10
|
+
const undici_1 = require("undici");
|
|
11
|
+
const output_1 = __importDefault(require("../output"));
|
|
12
|
+
const texts_1 = require("../texts");
|
|
13
|
+
const eventsource_1 = require("eventsource");
|
|
14
|
+
const customServiceUrl = process.env.BUDDY_CRAWL_SERVICE_URL;
|
|
15
|
+
function checkIfIsDevToken(token) {
|
|
16
|
+
const tokenParts = token.split('_');
|
|
17
|
+
return tokenParts.length === 5;
|
|
18
|
+
}
|
|
19
|
+
function getServiceUrl(token) {
|
|
20
|
+
if (customServiceUrl) {
|
|
21
|
+
return customServiceUrl;
|
|
22
|
+
}
|
|
23
|
+
const tokenParts = token.split('_');
|
|
24
|
+
const isDevToken = checkIfIsDevToken(token);
|
|
25
|
+
const devServiceUrl = Buffer.from(tokenParts[3], 'base64url').toString('utf-8');
|
|
26
|
+
if (isDevToken) {
|
|
27
|
+
if (devServiceUrl.includes('local.io')) {
|
|
28
|
+
throw new Error('Use BUDDY_CRAWL_SERVICE_URL to set the service URL in this environment');
|
|
29
|
+
}
|
|
30
|
+
return devServiceUrl;
|
|
31
|
+
}
|
|
32
|
+
if (token.startsWith('bud_crawl_eu')) {
|
|
33
|
+
return 'https://vt.eu.buddy.works';
|
|
34
|
+
}
|
|
35
|
+
if (token.startsWith('bud_crawl_asia')) {
|
|
36
|
+
return 'https://vt.asia.buddy.works';
|
|
37
|
+
}
|
|
38
|
+
return 'https://vt.buddy.works';
|
|
39
|
+
}
|
|
40
|
+
async function sendCrawl(ctx, url, follow, respectRobots, outputTypes, colorScheme, browsers, devices, cookies, requestHeaders, delays, waitForSelectors, localStorage) {
|
|
41
|
+
const payload = {
|
|
42
|
+
url,
|
|
43
|
+
follow,
|
|
44
|
+
respectRobots,
|
|
45
|
+
outputTypes,
|
|
46
|
+
colorScheme,
|
|
47
|
+
browsers,
|
|
48
|
+
devices,
|
|
49
|
+
cookies,
|
|
50
|
+
requestHeaders,
|
|
51
|
+
delays,
|
|
52
|
+
waitForSelectors,
|
|
53
|
+
localStorage,
|
|
54
|
+
pipelineId: ctx.pipelineId,
|
|
55
|
+
pipelineName: ctx.pipelineName,
|
|
56
|
+
executionId: ctx.executionId,
|
|
57
|
+
actionId: ctx.actionId,
|
|
58
|
+
invokerId: ctx.invokerId,
|
|
59
|
+
ci: ctx.ci,
|
|
60
|
+
executionUrl: ctx.executionUrl,
|
|
61
|
+
};
|
|
62
|
+
const [message, response] = await sendRequest({
|
|
63
|
+
path: '/crawl',
|
|
64
|
+
token: ctx.token,
|
|
65
|
+
payload,
|
|
66
|
+
});
|
|
67
|
+
if (message) {
|
|
68
|
+
throw new Error(message);
|
|
69
|
+
}
|
|
70
|
+
if (!response) {
|
|
71
|
+
throw new Error(texts_1.ERR_INVALID_CRAWL_RESPONSE);
|
|
72
|
+
}
|
|
73
|
+
return response;
|
|
74
|
+
}
|
|
75
|
+
async function downloadCrawlPackage(ctx, buildId) {
|
|
76
|
+
const [message, response] = await sendRequest({
|
|
77
|
+
path: '/download',
|
|
78
|
+
token: ctx.token,
|
|
79
|
+
payload: { token: ctx.token, buildId },
|
|
80
|
+
});
|
|
81
|
+
if (message) {
|
|
82
|
+
throw new Error(message);
|
|
83
|
+
}
|
|
84
|
+
if (!response) {
|
|
85
|
+
throw new Error(texts_1.ERR_INVALID_DOWNLOAD_RESPONSE);
|
|
86
|
+
}
|
|
87
|
+
return response;
|
|
88
|
+
}
|
|
89
|
+
function connectToCrawlSession(ctx, buildId) {
|
|
90
|
+
return new eventsource_1.EventSource(`${getServiceUrl(ctx.token)}/sse`, {
|
|
91
|
+
fetch: (url, options) => {
|
|
92
|
+
return (0, undici_1.fetch)(url, {
|
|
93
|
+
...options,
|
|
94
|
+
headers: {
|
|
95
|
+
...options?.headers,
|
|
96
|
+
'x-token': ctx.token,
|
|
97
|
+
'x-cli-version': context_1.cliVersion,
|
|
98
|
+
'x-build-id': buildId,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
async function sendRequest({ path, token, payload, }) {
|
|
105
|
+
const fullUrl = new URL(path, getServiceUrl(token));
|
|
106
|
+
output_1.default.debug((0, texts_1.LOG_SENDING_REQUEST)(fullUrl.toString()));
|
|
107
|
+
const init = {
|
|
108
|
+
method: 'GET',
|
|
109
|
+
redirect: 'follow',
|
|
110
|
+
headers: {
|
|
111
|
+
'X-CLI-VERSION': context_1.cliVersion,
|
|
112
|
+
'X-TOKEN': token,
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
if (payload) {
|
|
116
|
+
init.method = 'POST';
|
|
117
|
+
init.headers = {
|
|
118
|
+
'Content-Type': 'application/json',
|
|
119
|
+
'X-TOKEN': token,
|
|
120
|
+
'X-CLI-VERSION': context_1.cliVersion,
|
|
121
|
+
};
|
|
122
|
+
init.body = JSON.stringify(payload);
|
|
123
|
+
}
|
|
124
|
+
const response = await (0, undici_1.fetch)(fullUrl, init);
|
|
125
|
+
const contentType = response.headers.get('content-type');
|
|
126
|
+
if (contentType && contentType.includes('application/json')) {
|
|
127
|
+
try {
|
|
128
|
+
const json = (await response.json());
|
|
129
|
+
return [undefined, json];
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
throw new Error(texts_1.ERR_INVALID_JSON);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (contentType && contentType.includes('application/octet-stream')) {
|
|
136
|
+
const buffer = response.body;
|
|
137
|
+
return [undefined, buffer];
|
|
138
|
+
}
|
|
139
|
+
const text = await response.text();
|
|
140
|
+
return [text, undefined];
|
|
141
|
+
}
|
package/distTs/src/input.js
CHANGED
|
@@ -350,6 +350,17 @@ class Input {
|
|
|
350
350
|
return utils_1.ARTIFACT_SCOPE.PROJECT;
|
|
351
351
|
return utils_1.ARTIFACT_SCOPE.WORKSPACE;
|
|
352
352
|
}
|
|
353
|
+
static pipelineRunId(runId) {
|
|
354
|
+
const r = parseInt(runId, 10);
|
|
355
|
+
if (!isNaN(r))
|
|
356
|
+
return r;
|
|
357
|
+
output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_WRONG_RUN_ID);
|
|
358
|
+
}
|
|
359
|
+
static pipelineRunActionId(runActionId) {
|
|
360
|
+
if (runActionId)
|
|
361
|
+
return runActionId;
|
|
362
|
+
output_1.default.exitError(texts_1.ERR_RUN_PIPELINE_WRONG_RUN_ACTION_ID);
|
|
363
|
+
}
|
|
353
364
|
static pipelineRunPriority(priority) {
|
|
354
365
|
if (!priority)
|
|
355
366
|
return null;
|
|
@@ -457,16 +468,6 @@ class Input {
|
|
|
457
468
|
output_1.default.exitError(texts_1.ERR_REST_API_URL);
|
|
458
469
|
}
|
|
459
470
|
}
|
|
460
|
-
static pipelineRunWaitTime(time) {
|
|
461
|
-
let t = Number.parseInt(time, 10);
|
|
462
|
-
if (!t)
|
|
463
|
-
t = 1440;
|
|
464
|
-
else if (t < 0)
|
|
465
|
-
t = 1;
|
|
466
|
-
else if (t > 1440)
|
|
467
|
-
t = 1440;
|
|
468
|
-
return t;
|
|
469
|
-
}
|
|
470
471
|
static resolvePath(path) {
|
|
471
472
|
const p = (0, node_path_1.resolve)(path);
|
|
472
473
|
const exists = node_fs_1.default.existsSync(p);
|