bdy 1.20.2-dev → 1.20.3-master
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/api/client.js +0 -83
- package/distTs/src/command/login.js +6 -0
- package/distTs/src/command/whoami.js +0 -12
- package/distTs/src/index.js +6 -4
- package/distTs/src/input.js +20 -83
- package/distTs/src/project/cfg.js +0 -39
- package/distTs/src/texts.js +41 -67
- package/distTs/src/unitTest/requests.js +10 -23
- package/distTs/src/visualTest/context.js +31 -42
- package/distTs/src/visualTest/requests.js +139 -39
- package/distTs/src/visualTest/resources.js +38 -40
- package/distTs/src/visualTest/server.js +2 -2
- package/distTs/src/visualTest/snapshots.js +17 -18
- package/distTs/src/visualTest/validation.js +10 -2
- package/package.json +1 -1
|
@@ -9,11 +9,15 @@ exports.closeSession = closeSession;
|
|
|
9
9
|
exports.getDefaultSettings = getDefaultSettings;
|
|
10
10
|
exports.sendStorybook = sendStorybook;
|
|
11
11
|
exports.sendCompareLinks = sendCompareLinks;
|
|
12
|
+
exports.sendScrap = sendScrap;
|
|
13
|
+
exports.downloadScrapPackage = downloadScrapPackage;
|
|
14
|
+
exports.connectToScrapSession = connectToScrapSession;
|
|
12
15
|
const context_1 = require("./context");
|
|
13
16
|
const undici_1 = require("undici");
|
|
14
17
|
const uuid_1 = require("uuid");
|
|
15
18
|
const output_1 = __importDefault(require("../output"));
|
|
16
19
|
const texts_1 = require("../texts");
|
|
20
|
+
const eventsource_1 = require("eventsource");
|
|
17
21
|
const customServiceUrl = process.env.BUDDY_VT_SERVICE_URL;
|
|
18
22
|
function checkIfIsDevToken(token) {
|
|
19
23
|
const tokenParts = token.split('_');
|
|
@@ -32,32 +36,14 @@ function getServiceUrl(token) {
|
|
|
32
36
|
}
|
|
33
37
|
return devServiceUrl;
|
|
34
38
|
}
|
|
35
|
-
if (token.startsWith('bud_vt_eu') || token.startsWith('
|
|
39
|
+
if (token.startsWith('bud_vt_eu') || token.startsWith('bud_scrap_eu')) {
|
|
36
40
|
return 'https://vt.eu.buddy.works';
|
|
37
41
|
}
|
|
38
|
-
if (token.startsWith('bud_vt_asia') || token.startsWith('
|
|
42
|
+
if (token.startsWith('bud_vt_asia') || token.startsWith('bud_scrap_asia')) {
|
|
39
43
|
return 'https://vt.asia.buddy.works';
|
|
40
44
|
}
|
|
41
45
|
return 'https://vt.buddy.works';
|
|
42
46
|
}
|
|
43
|
-
function getCiPayload(ctx) {
|
|
44
|
-
return {
|
|
45
|
-
ci: ctx.ci,
|
|
46
|
-
branch: ctx.branch,
|
|
47
|
-
tag: ctx.tag,
|
|
48
|
-
pullRequestNumber: ctx.pullRequestNumber,
|
|
49
|
-
commit: ctx.commit,
|
|
50
|
-
baseCommit: ctx.baseCommit,
|
|
51
|
-
pipelineId: ctx.pipelineId,
|
|
52
|
-
pipelineName: ctx.pipelineName,
|
|
53
|
-
actionId: ctx.actionId,
|
|
54
|
-
executionId: ctx.executionId,
|
|
55
|
-
invokerId: ctx.invokerId,
|
|
56
|
-
commitDetails: ctx.commitDetails,
|
|
57
|
-
cliId: context_1.cliId,
|
|
58
|
-
executionUrl: ctx.executionUrl,
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
47
|
function prepareSnapshotInternal(snapshot) {
|
|
62
48
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
63
49
|
const { resourceDiscoveryTimeout, cookies, ...filteredSnapshot } = snapshot;
|
|
@@ -95,20 +81,33 @@ function createFormData(info, files) {
|
|
|
95
81
|
}
|
|
96
82
|
return formData;
|
|
97
83
|
}
|
|
98
|
-
async function sendSnapshot(
|
|
84
|
+
async function sendSnapshot(snapshot, firstSnapshot) {
|
|
99
85
|
const files = [];
|
|
100
86
|
const preparedSnapshot = prepareSnapshot(snapshot, files);
|
|
101
87
|
const info = {
|
|
102
88
|
snapshot: preparedSnapshot,
|
|
103
89
|
buildId: context_1.buildId,
|
|
104
|
-
parallel:
|
|
105
|
-
|
|
90
|
+
parallel: context_1.parallel,
|
|
91
|
+
ci: context_1.ci,
|
|
92
|
+
branch: context_1.branch,
|
|
93
|
+
tag: context_1.tag,
|
|
94
|
+
pullRequestNumber: context_1.pullRequestNumber,
|
|
95
|
+
commit: context_1.commit,
|
|
96
|
+
baseCommit: context_1.baseCommit,
|
|
97
|
+
pipelineId: context_1.pipelineId,
|
|
98
|
+
pipelineName: context_1.pipelineName,
|
|
99
|
+
actionId: context_1.actionId,
|
|
100
|
+
executionId: context_1.executionId,
|
|
101
|
+
invokerId: context_1.invokerId,
|
|
102
|
+
commitDetails: context_1.commitDetails,
|
|
103
|
+
cliId: context_1.cliId,
|
|
104
|
+
executionUrl: context_1.executionUrl,
|
|
106
105
|
firstSnapshot,
|
|
107
106
|
};
|
|
108
107
|
const formData = createFormData(info, files);
|
|
109
108
|
const [message, response] = await sendRequest({
|
|
110
109
|
path: '/snapshot',
|
|
111
|
-
token:
|
|
110
|
+
token: context_1.vtToken,
|
|
112
111
|
payload: formData,
|
|
113
112
|
multipart: true,
|
|
114
113
|
});
|
|
@@ -120,20 +119,33 @@ async function sendSnapshot(ctx, snapshot, firstSnapshot) {
|
|
|
120
119
|
}
|
|
121
120
|
return response;
|
|
122
121
|
}
|
|
123
|
-
async function sendSnapshots(
|
|
122
|
+
async function sendSnapshots(snapshots) {
|
|
124
123
|
const files = [];
|
|
125
124
|
const preparedSnapshots = snapshots.map((snapshot) => prepareSnapshot(snapshot, files));
|
|
126
125
|
const info = {
|
|
127
126
|
snapshots: preparedSnapshots,
|
|
128
127
|
buildId: context_1.buildId,
|
|
129
|
-
parallel:
|
|
130
|
-
|
|
128
|
+
parallel: context_1.parallel,
|
|
129
|
+
ci: context_1.ci,
|
|
130
|
+
branch: context_1.branch,
|
|
131
|
+
tag: context_1.tag,
|
|
132
|
+
pullRequestNumber: context_1.pullRequestNumber,
|
|
133
|
+
commit: context_1.commit,
|
|
134
|
+
baseCommit: context_1.baseCommit,
|
|
135
|
+
pipelineId: context_1.pipelineId,
|
|
136
|
+
pipelineName: context_1.pipelineName,
|
|
137
|
+
actionId: context_1.actionId,
|
|
138
|
+
executionId: context_1.executionId,
|
|
139
|
+
invokerId: context_1.invokerId,
|
|
140
|
+
commitDetails: context_1.commitDetails,
|
|
141
|
+
cliId: context_1.cliId,
|
|
142
|
+
executionUrl: context_1.executionUrl,
|
|
131
143
|
firstSnapshot: true,
|
|
132
144
|
};
|
|
133
145
|
const formData = createFormData(info, files);
|
|
134
146
|
const [message, response] = await sendRequest({
|
|
135
147
|
path: '/snapshots',
|
|
136
|
-
token:
|
|
148
|
+
token: context_1.vtToken,
|
|
137
149
|
payload: formData,
|
|
138
150
|
multipart: true,
|
|
139
151
|
});
|
|
@@ -145,14 +157,14 @@ async function sendSnapshots(ctx, snapshots) {
|
|
|
145
157
|
}
|
|
146
158
|
return response;
|
|
147
159
|
}
|
|
148
|
-
async function closeSession(
|
|
160
|
+
async function closeSession() {
|
|
149
161
|
const payload = {
|
|
150
|
-
token:
|
|
162
|
+
token: context_1.vtToken,
|
|
151
163
|
buildId: context_1.buildId,
|
|
152
164
|
};
|
|
153
165
|
const [message, response] = await sendRequest({
|
|
154
166
|
path: '/close',
|
|
155
|
-
token:
|
|
167
|
+
token: context_1.vtToken,
|
|
156
168
|
payload,
|
|
157
169
|
});
|
|
158
170
|
if (message) {
|
|
@@ -163,10 +175,10 @@ async function closeSession(ctx) {
|
|
|
163
175
|
}
|
|
164
176
|
return response;
|
|
165
177
|
}
|
|
166
|
-
async function getDefaultSettings(
|
|
178
|
+
async function getDefaultSettings() {
|
|
167
179
|
const [message, response] = await sendRequest({
|
|
168
180
|
path: `/defaultSettings`,
|
|
169
|
-
token:
|
|
181
|
+
token: context_1.vtToken,
|
|
170
182
|
});
|
|
171
183
|
if (message) {
|
|
172
184
|
throw new Error(message);
|
|
@@ -176,11 +188,24 @@ async function getDefaultSettings(ctx) {
|
|
|
176
188
|
}
|
|
177
189
|
return response;
|
|
178
190
|
}
|
|
179
|
-
async function sendStorybook(
|
|
191
|
+
async function sendStorybook(snapshots, compressedStorybookDirectory) {
|
|
180
192
|
const info = {
|
|
181
193
|
snapshots,
|
|
182
194
|
buildId: context_1.buildId,
|
|
183
|
-
|
|
195
|
+
ci: context_1.ci,
|
|
196
|
+
branch: context_1.branch,
|
|
197
|
+
tag: context_1.tag,
|
|
198
|
+
pullRequestNumber: context_1.pullRequestNumber,
|
|
199
|
+
commit: context_1.commit,
|
|
200
|
+
baseCommit: context_1.baseCommit,
|
|
201
|
+
pipelineId: context_1.pipelineId,
|
|
202
|
+
pipelineName: context_1.pipelineName,
|
|
203
|
+
actionId: context_1.actionId,
|
|
204
|
+
executionId: context_1.executionId,
|
|
205
|
+
invokerId: context_1.invokerId,
|
|
206
|
+
commitDetails: context_1.commitDetails,
|
|
207
|
+
cliId: context_1.cliId,
|
|
208
|
+
executionUrl: context_1.executionUrl,
|
|
184
209
|
};
|
|
185
210
|
const files = [
|
|
186
211
|
{
|
|
@@ -191,7 +216,7 @@ async function sendStorybook(ctx, snapshots, compressedStorybookDirectory) {
|
|
|
191
216
|
const formData = createFormData(info, files);
|
|
192
217
|
const [message, response] = await sendRequest({
|
|
193
218
|
path: '/storybook',
|
|
194
|
-
token:
|
|
219
|
+
token: context_1.vtToken,
|
|
195
220
|
payload: formData,
|
|
196
221
|
multipart: true,
|
|
197
222
|
});
|
|
@@ -203,7 +228,7 @@ async function sendStorybook(ctx, snapshots, compressedStorybookDirectory) {
|
|
|
203
228
|
}
|
|
204
229
|
return response;
|
|
205
230
|
}
|
|
206
|
-
async function sendCompareLinks(
|
|
231
|
+
async function sendCompareLinks(urls, validatedOptions, sitemapSource) {
|
|
207
232
|
const info = {
|
|
208
233
|
urls,
|
|
209
234
|
sitemapSource,
|
|
@@ -216,11 +241,22 @@ async function sendCompareLinks(ctx, urls, validatedOptions, sitemapSource) {
|
|
|
216
241
|
respectRobots: validatedOptions.respectRobots,
|
|
217
242
|
ignoreUrls: validatedOptions.ignoreUrls ?? [],
|
|
218
243
|
buildId: context_1.buildId,
|
|
219
|
-
|
|
244
|
+
ci: context_1.ci,
|
|
245
|
+
branch: context_1.branch,
|
|
246
|
+
tag: context_1.tag,
|
|
247
|
+
commit: context_1.commit,
|
|
248
|
+
baseCommit: context_1.baseCommit,
|
|
249
|
+
pipelineId: context_1.pipelineId,
|
|
250
|
+
pipelineName: context_1.pipelineName,
|
|
251
|
+
actionId: context_1.actionId,
|
|
252
|
+
executionId: context_1.executionId,
|
|
253
|
+
invokerId: context_1.invokerId,
|
|
254
|
+
commitDetails: context_1.commitDetails,
|
|
255
|
+
cliId: context_1.cliId,
|
|
220
256
|
};
|
|
221
257
|
const [message, response] = await sendRequest({
|
|
222
258
|
path: '/compareLinks',
|
|
223
|
-
token:
|
|
259
|
+
token: context_1.vtToken,
|
|
224
260
|
payload: info,
|
|
225
261
|
});
|
|
226
262
|
if (message) {
|
|
@@ -231,6 +267,55 @@ async function sendCompareLinks(ctx, urls, validatedOptions, sitemapSource) {
|
|
|
231
267
|
}
|
|
232
268
|
return response;
|
|
233
269
|
}
|
|
270
|
+
async function sendScrap(url, follow, respectRobots, outputTypes, colorScheme, browsers, devices, cookies, requestHeaders, delays, waitForSelectors, localStorage) {
|
|
271
|
+
const payload = {
|
|
272
|
+
url,
|
|
273
|
+
follow,
|
|
274
|
+
respectRobots,
|
|
275
|
+
outputTypes,
|
|
276
|
+
colorScheme,
|
|
277
|
+
browsers,
|
|
278
|
+
devices,
|
|
279
|
+
cookies,
|
|
280
|
+
requestHeaders,
|
|
281
|
+
delays,
|
|
282
|
+
waitForSelectors,
|
|
283
|
+
localStorage,
|
|
284
|
+
pipelineId: context_1.pipelineId,
|
|
285
|
+
pipelineName: context_1.pipelineName,
|
|
286
|
+
executionId: context_1.executionId,
|
|
287
|
+
actionId: context_1.actionId,
|
|
288
|
+
invokerId: context_1.invokerId,
|
|
289
|
+
ci: context_1.ci,
|
|
290
|
+
executionUrl: context_1.executionUrl,
|
|
291
|
+
};
|
|
292
|
+
const [message, response] = await sendRequest({
|
|
293
|
+
path: '/scrap',
|
|
294
|
+
token: context_1.scrapeToken,
|
|
295
|
+
payload,
|
|
296
|
+
});
|
|
297
|
+
if (message) {
|
|
298
|
+
throw new Error(message);
|
|
299
|
+
}
|
|
300
|
+
if (!response) {
|
|
301
|
+
throw new Error(texts_1.ERR_INVALID_SCRAPE_RESPONSE);
|
|
302
|
+
}
|
|
303
|
+
return response;
|
|
304
|
+
}
|
|
305
|
+
async function downloadScrapPackage(buildId) {
|
|
306
|
+
const [message, response] = await sendRequest({
|
|
307
|
+
path: '/download',
|
|
308
|
+
token: context_1.scrapeToken,
|
|
309
|
+
payload: { token: context_1.scrapeToken, buildId },
|
|
310
|
+
});
|
|
311
|
+
if (message) {
|
|
312
|
+
throw new Error(message);
|
|
313
|
+
}
|
|
314
|
+
if (!response) {
|
|
315
|
+
throw new Error(texts_1.ERR_INVALID_DOWNLOAD_RESPONSE);
|
|
316
|
+
}
|
|
317
|
+
return response;
|
|
318
|
+
}
|
|
234
319
|
async function sendRequest({ path, token, payload, multipart, }) {
|
|
235
320
|
const fullUrl = new URL(path, getServiceUrl(token));
|
|
236
321
|
output_1.default.debug((0, texts_1.LOG_SENDING_REQUEST)(fullUrl.toString()));
|
|
@@ -281,3 +366,18 @@ async function sendRequest({ path, token, payload, multipart, }) {
|
|
|
281
366
|
const text = await response.text();
|
|
282
367
|
return [text, undefined];
|
|
283
368
|
}
|
|
369
|
+
function connectToScrapSession(buildId) {
|
|
370
|
+
return new eventsource_1.EventSource(`${getServiceUrl(context_1.scrapeToken)}/sse`, {
|
|
371
|
+
fetch: (url, options) => {
|
|
372
|
+
return (0, undici_1.fetch)(url, {
|
|
373
|
+
...options,
|
|
374
|
+
headers: {
|
|
375
|
+
...options?.headers,
|
|
376
|
+
'x-token': context_1.scrapeToken,
|
|
377
|
+
'x-cli-version': context_1.cliVersion,
|
|
378
|
+
'x-build-id': buildId,
|
|
379
|
+
},
|
|
380
|
+
});
|
|
381
|
+
},
|
|
382
|
+
});
|
|
383
|
+
}
|
|
@@ -42,65 +42,63 @@ const context_1 = require("./context");
|
|
|
42
42
|
const output_1 = __importDefault(require("../output"));
|
|
43
43
|
const texts_1 = require("../texts");
|
|
44
44
|
const utils_1 = require("../utils");
|
|
45
|
-
const
|
|
46
|
-
const
|
|
45
|
+
const scrapedUrls = new Map();
|
|
46
|
+
const scrapedResources = new Map();
|
|
47
47
|
let showedBrowserVersionWarning = false;
|
|
48
|
-
async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, cookies = [],
|
|
49
|
-
const
|
|
50
|
-
const missingDevices =
|
|
51
|
-
? devices.filter(({ viewport }) => !
|
|
48
|
+
async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, cookies = [], }) {
|
|
49
|
+
const scrapedUrl = scrapedUrls.get(url);
|
|
50
|
+
const missingDevices = scrapedUrl
|
|
51
|
+
? devices.filter(({ viewport }) => !scrapedUrl.widths.includes(viewport.width))
|
|
52
52
|
: [];
|
|
53
|
-
if (
|
|
53
|
+
if (scrapedUrl && missingDevices.length === 0) {
|
|
54
54
|
return getResourceUrlsData({
|
|
55
|
-
duplicatedResourcesUrls:
|
|
55
|
+
duplicatedResourcesUrls: scrapedUrl.resources,
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
|
-
else if (
|
|
59
|
-
const {
|
|
58
|
+
else if (scrapedUrl && missingDevices.length > 0) {
|
|
59
|
+
const { scrapedResourcesUrls, duplicatedResourcesUrls } = await output_1.default.debugAction((0, texts_1.DEBUG_RESOURCE_SCRAPPING_URL)(url), scrapeResources({
|
|
60
60
|
url,
|
|
61
61
|
devices: missingDevices,
|
|
62
62
|
resourceDiscoveryTimeout,
|
|
63
63
|
cookies,
|
|
64
|
-
browserPath,
|
|
65
64
|
}));
|
|
66
|
-
const
|
|
67
|
-
|
|
65
|
+
const duplicatedScrapedResourceUrls = [...scrapedUrl.resources];
|
|
66
|
+
scrapedUrls.set(url, {
|
|
68
67
|
widths: [
|
|
69
|
-
...
|
|
68
|
+
...scrapedUrl.widths,
|
|
70
69
|
...missingDevices.map(({ viewport }) => viewport.width),
|
|
71
70
|
],
|
|
72
71
|
resources: [
|
|
73
72
|
...new Set([
|
|
74
|
-
...
|
|
75
|
-
...
|
|
73
|
+
...scrapedUrl.resources,
|
|
74
|
+
...scrapedResourcesUrls,
|
|
76
75
|
...duplicatedResourcesUrls,
|
|
77
76
|
]),
|
|
78
77
|
],
|
|
79
78
|
});
|
|
80
79
|
return getResourceUrlsData({
|
|
81
|
-
resourcesUrls:
|
|
80
|
+
resourcesUrls: scrapedResourcesUrls,
|
|
82
81
|
duplicatedResourcesUrls: [
|
|
83
82
|
...new Set([
|
|
84
83
|
...duplicatedResourcesUrls,
|
|
85
|
-
...
|
|
84
|
+
...duplicatedScrapedResourceUrls,
|
|
86
85
|
]),
|
|
87
86
|
],
|
|
88
87
|
});
|
|
89
88
|
}
|
|
90
89
|
else {
|
|
91
|
-
const {
|
|
90
|
+
const { scrapedResourcesUrls, duplicatedResourcesUrls } = await output_1.default.debugAction((0, texts_1.DEBUG_RESOURCE_SCRAPPING_URL)(url), scrapeResources({
|
|
92
91
|
url,
|
|
93
92
|
devices,
|
|
94
93
|
resourceDiscoveryTimeout,
|
|
95
94
|
cookies,
|
|
96
|
-
browserPath,
|
|
97
95
|
}));
|
|
98
|
-
|
|
96
|
+
scrapedUrls.set(url, {
|
|
99
97
|
widths: devices.map(({ viewport }) => viewport.width),
|
|
100
|
-
resources: [...
|
|
98
|
+
resources: [...scrapedResourcesUrls, ...duplicatedResourcesUrls],
|
|
101
99
|
});
|
|
102
100
|
return getResourceUrlsData({
|
|
103
|
-
resourcesUrls:
|
|
101
|
+
resourcesUrls: scrapedResourcesUrls,
|
|
104
102
|
duplicatedResourcesUrls,
|
|
105
103
|
});
|
|
106
104
|
}
|
|
@@ -108,13 +106,13 @@ async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, co
|
|
|
108
106
|
function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [], }) {
|
|
109
107
|
const resources = [];
|
|
110
108
|
for (const resourceUrl of resourcesUrls) {
|
|
111
|
-
const
|
|
112
|
-
if (
|
|
109
|
+
const scrapedResource = scrapedResources.get(resourceUrl);
|
|
110
|
+
if (scrapedResource) {
|
|
113
111
|
resources.push({
|
|
114
112
|
url: resourceUrl,
|
|
115
|
-
status:
|
|
116
|
-
headers:
|
|
117
|
-
body:
|
|
113
|
+
status: scrapedResource.status,
|
|
114
|
+
headers: scrapedResource.headers,
|
|
115
|
+
body: scrapedResource.body,
|
|
118
116
|
});
|
|
119
117
|
}
|
|
120
118
|
else {
|
|
@@ -122,12 +120,12 @@ function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [],
|
|
|
122
120
|
}
|
|
123
121
|
}
|
|
124
122
|
for (const resourceUrl of duplicatedResourcesUrls) {
|
|
125
|
-
const
|
|
126
|
-
if (
|
|
123
|
+
const scrapedResource = scrapedResources.get(resourceUrl);
|
|
124
|
+
if (scrapedResource) {
|
|
127
125
|
resources.push({
|
|
128
126
|
url: resourceUrl,
|
|
129
|
-
status:
|
|
130
|
-
headers:
|
|
127
|
+
status: scrapedResource.status,
|
|
128
|
+
headers: scrapedResource.headers,
|
|
131
129
|
duplicate: true,
|
|
132
130
|
});
|
|
133
131
|
}
|
|
@@ -137,15 +135,15 @@ function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [],
|
|
|
137
135
|
}
|
|
138
136
|
return resources;
|
|
139
137
|
}
|
|
140
|
-
async function
|
|
141
|
-
const
|
|
138
|
+
async function scrapeResources({ url, devices, resourceDiscoveryTimeout, cookies = [], }) {
|
|
139
|
+
const scrapedResourcesUrls = [];
|
|
142
140
|
const duplicatedResourcesUrls = [];
|
|
143
141
|
const launchOptions = {
|
|
144
142
|
headless: 'shell',
|
|
145
143
|
args: ['--no-sandbox'],
|
|
146
144
|
acceptInsecureCerts: true,
|
|
147
145
|
timeout: 120_000,
|
|
148
|
-
executablePath: browserPath,
|
|
146
|
+
executablePath: context_1.browserPath,
|
|
149
147
|
};
|
|
150
148
|
const browser = await puppeteer_core_1.default.launch(launchOptions);
|
|
151
149
|
if (!showedBrowserVersionWarning) {
|
|
@@ -174,8 +172,8 @@ async function crawlResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
174
172
|
page.on('response', async (response) => {
|
|
175
173
|
const request = response.request();
|
|
176
174
|
const requestUrl = request.url();
|
|
177
|
-
const isCollected =
|
|
178
|
-
if (isCollected && !
|
|
175
|
+
const isCollected = scrapedResources.has(requestUrl);
|
|
176
|
+
if (isCollected && !scrapedResourcesUrls.includes(requestUrl)) {
|
|
179
177
|
duplicatedResourcesUrls.push(requestUrl);
|
|
180
178
|
}
|
|
181
179
|
else {
|
|
@@ -195,8 +193,8 @@ async function crawlResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
195
193
|
responseError = error;
|
|
196
194
|
}
|
|
197
195
|
if (!responseError && body) {
|
|
198
|
-
|
|
199
|
-
|
|
196
|
+
scrapedResourcesUrls.push(requestUrl);
|
|
197
|
+
scrapedResources.set(requestUrl, {
|
|
200
198
|
status,
|
|
201
199
|
headers: { 'content-type': contentType },
|
|
202
200
|
body,
|
|
@@ -231,7 +229,7 @@ async function crawlResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
231
229
|
}
|
|
232
230
|
}
|
|
233
231
|
await browser.close();
|
|
234
|
-
return {
|
|
232
|
+
return { scrapedResourcesUrls, duplicatedResourcesUrls };
|
|
235
233
|
}
|
|
236
234
|
async function autoScroll(page) {
|
|
237
235
|
await page.evaluate(async () => {
|
|
@@ -81,7 +81,7 @@ function prepareSnapshotPlugin(data) {
|
|
|
81
81
|
}
|
|
82
82
|
return undefined;
|
|
83
83
|
}
|
|
84
|
-
async function createServer(
|
|
84
|
+
async function createServer() {
|
|
85
85
|
const app = (0, fastify_1.default)({
|
|
86
86
|
// logger: true,
|
|
87
87
|
bodyLimit: 50 * 1024 * 1024, //50 MB,
|
|
@@ -95,7 +95,7 @@ async function createServer(ctx) {
|
|
|
95
95
|
const data = request.body;
|
|
96
96
|
const preparedSnapshot = prepareSnapshotPlugin(data);
|
|
97
97
|
if (preparedSnapshot) {
|
|
98
|
-
(0, snapshots_1.addSnapshot)(
|
|
98
|
+
(0, snapshots_1.addSnapshot)(preparedSnapshot);
|
|
99
99
|
reply.send('Snapshot received');
|
|
100
100
|
}
|
|
101
101
|
else {
|
|
@@ -20,13 +20,13 @@ let sessionLink = '';
|
|
|
20
20
|
function setDefaultSettings(defaultSettings) {
|
|
21
21
|
defaultDevices = defaultSettings.defaultDevices;
|
|
22
22
|
}
|
|
23
|
-
function addSnapshot(
|
|
23
|
+
function addSnapshot(snapshot) {
|
|
24
24
|
const preparedSnapshot = {
|
|
25
25
|
...snapshot,
|
|
26
26
|
devices: snapshot.devices.length > 0 ? snapshot.devices : defaultDevices,
|
|
27
27
|
};
|
|
28
|
-
if (
|
|
29
|
-
(0, queue_1.addToQueue)(() => processSnapshot(
|
|
28
|
+
if (context_1.oneByOne) {
|
|
29
|
+
(0, queue_1.addToQueue)(() => processSnapshot(preparedSnapshot, firstSnapshot));
|
|
30
30
|
}
|
|
31
31
|
else {
|
|
32
32
|
snapshots.push(preparedSnapshot);
|
|
@@ -35,12 +35,12 @@ function addSnapshot(ctx, snapshot) {
|
|
|
35
35
|
firstSnapshot = false;
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
|
-
async function finishProcessingSnapshots(
|
|
39
|
-
if (
|
|
38
|
+
async function finishProcessingSnapshots(spawnedProcessExitCode) {
|
|
39
|
+
if (context_1.oneByOne) {
|
|
40
40
|
return await (0, queue_1.isQueueEmpty)();
|
|
41
41
|
}
|
|
42
42
|
if (spawnedProcessExitCode === 0) {
|
|
43
|
-
return await processSnapshots(
|
|
43
|
+
return await processSnapshots();
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
function showSessionLink() {
|
|
@@ -48,17 +48,17 @@ function showSessionLink() {
|
|
|
48
48
|
output_1.default.normal((0, texts_1.LOG_SESSION_LINK)(sessionLink));
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
|
-
async function processSnapshot(
|
|
51
|
+
async function processSnapshot(snapshot, firstSnapshot) {
|
|
52
52
|
let t1;
|
|
53
53
|
if (context_1.debug) {
|
|
54
54
|
t1 = performance.now();
|
|
55
55
|
}
|
|
56
|
-
const preparedSnapshot =
|
|
56
|
+
const preparedSnapshot = context_1.skipDiscovery
|
|
57
57
|
? snapshot
|
|
58
|
-
: await getSnapshotWithResources(
|
|
58
|
+
: await getSnapshotWithResources(snapshot);
|
|
59
59
|
try {
|
|
60
60
|
output_1.default.normal(texts_1.LOG_SENDING_DATA);
|
|
61
|
-
const { message } = await (0, requests_1.sendSnapshot)(
|
|
61
|
+
const { message } = await (0, requests_1.sendSnapshot)(preparedSnapshot, firstSnapshot);
|
|
62
62
|
sessionLink = message;
|
|
63
63
|
}
|
|
64
64
|
catch (error) {
|
|
@@ -69,22 +69,22 @@ async function processSnapshot(ctx, snapshot, firstSnapshot) {
|
|
|
69
69
|
output_1.default.normal((0, texts_1.DEBUG_SNAPSHOT_PROCESSING)(snapshot.name, t2 - t1));
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
|
-
async function processSnapshots(
|
|
72
|
+
async function processSnapshots() {
|
|
73
73
|
let t1;
|
|
74
74
|
if (context_1.debug) {
|
|
75
75
|
t1 = performance.now();
|
|
76
76
|
}
|
|
77
77
|
output_1.default.normal((0, texts_1.LOG_PROCESSING_SNAPSHOTS)(snapshots.length));
|
|
78
|
-
const preparedSnapshots =
|
|
78
|
+
const preparedSnapshots = context_1.skipDiscovery
|
|
79
79
|
? snapshots
|
|
80
|
-
: await getSnapshotsWithResources(
|
|
80
|
+
: await getSnapshotsWithResources(snapshots);
|
|
81
81
|
if (preparedSnapshots.length === 0) {
|
|
82
82
|
output_1.default.error(texts_1.ERR_NO_SNAPSHOTS_TO_SEND);
|
|
83
83
|
}
|
|
84
84
|
else {
|
|
85
85
|
try {
|
|
86
86
|
output_1.default.normal(texts_1.LOG_SENDING_DATA);
|
|
87
|
-
const { message } = await (0, requests_1.sendSnapshots)(
|
|
87
|
+
const { message } = await (0, requests_1.sendSnapshots)(preparedSnapshots);
|
|
88
88
|
sessionLink = message;
|
|
89
89
|
}
|
|
90
90
|
catch (error) {
|
|
@@ -96,14 +96,13 @@ async function processSnapshots(ctx) {
|
|
|
96
96
|
output_1.default.normal((0, texts_1.DEBUG_SNAPSHOTS_PROCESSING)(t2 - t1));
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
async function getSnapshotWithResources(
|
|
99
|
+
async function getSnapshotWithResources(snapshot) {
|
|
100
100
|
const { url, devices, resourceDiscoveryTimeout, cookies } = snapshot;
|
|
101
101
|
const resources = await (0, resources_1.collectResources)({
|
|
102
102
|
url,
|
|
103
103
|
devices,
|
|
104
104
|
resourceDiscoveryTimeout,
|
|
105
105
|
cookies,
|
|
106
|
-
browserPath: ctx.browserPath,
|
|
107
106
|
});
|
|
108
107
|
if (context_1.debug) {
|
|
109
108
|
output_1.default.normal(`Collected resources for snapshot: ${snapshot.name}`);
|
|
@@ -116,11 +115,11 @@ async function getSnapshotWithResources(ctx, snapshot) {
|
|
|
116
115
|
resources: [...snapshot.resources, ...resources],
|
|
117
116
|
};
|
|
118
117
|
}
|
|
119
|
-
async function getSnapshotsWithResources(
|
|
118
|
+
async function getSnapshotsWithResources(snapshots) {
|
|
120
119
|
const snapshotsWithResources = [];
|
|
121
120
|
const prepareSnapshots = filterDuplicates(snapshots);
|
|
122
121
|
for (const snapshot of prepareSnapshots) {
|
|
123
|
-
const snapshotWithResources = await getSnapshotWithResources(
|
|
122
|
+
const snapshotWithResources = await getSnapshotWithResources(snapshot);
|
|
124
123
|
snapshotsWithResources.push(snapshotWithResources);
|
|
125
124
|
}
|
|
126
125
|
return snapshotsWithResources;
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.checkToken = checkToken;
|
|
3
4
|
exports.checkBuildId = checkBuildId;
|
|
4
5
|
exports.checkParallel = checkParallel;
|
|
6
|
+
const context_1 = require("./context");
|
|
7
|
+
function checkToken(type) {
|
|
8
|
+
if (type === 'scrape') {
|
|
9
|
+
return !!context_1.scrapeToken;
|
|
10
|
+
}
|
|
11
|
+
return !!context_1.vtToken;
|
|
12
|
+
}
|
|
5
13
|
function checkBuildId() {
|
|
6
14
|
return !!process.env.SNAPSHOTS_BUILD_ID;
|
|
7
15
|
}
|
|
8
|
-
function checkParallel(
|
|
9
|
-
if (
|
|
16
|
+
function checkParallel() {
|
|
17
|
+
if (context_1.parallel) {
|
|
10
18
|
return checkBuildId();
|
|
11
19
|
}
|
|
12
20
|
return true;
|