bdy 1.20.0 → 1.20.2-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/distTs/package.json +1 -1
- package/distTs/src/api/client.js +83 -0
- package/distTs/src/command/crawl/link.js +61 -0
- package/distTs/src/command/crawl/run.js +147 -0
- package/distTs/src/command/crawl/validation.js +154 -0
- package/distTs/src/command/crawl.js +13 -0
- package/distTs/src/command/tests/capture/validation.js +46 -0
- package/distTs/src/command/tests/capture.js +103 -0
- package/distTs/src/command/tests/unit/link.js +61 -0
- package/distTs/src/command/tests/unit/upload.js +91 -0
- package/distTs/src/command/tests/unit.js +13 -0
- package/distTs/src/command/tests/visual/link.js +61 -0
- package/distTs/src/command/tests/visual/session/close.js +32 -0
- package/distTs/src/command/tests/visual/session/create.js +86 -0
- package/distTs/src/command/tests/visual/session.js +13 -0
- package/distTs/src/command/tests/visual/setup.js +20 -0
- package/distTs/src/command/tests/visual/shared/validation.js +145 -0
- package/distTs/src/command/tests/visual/upload.js +141 -0
- package/distTs/src/command/tests/visual.js +17 -0
- package/distTs/src/command/tests.js +15 -0
- package/distTs/src/command/whoami.js +12 -0
- package/distTs/src/crawl/requests.js +141 -0
- package/distTs/src/index.js +4 -6
- package/distTs/src/input.js +78 -0
- package/distTs/src/project/cfg.js +39 -0
- package/distTs/src/texts.js +67 -41
- package/distTs/src/types/crawl.js +2 -0
- package/distTs/src/unitTest/context.js +26 -0
- package/distTs/src/unitTest/requests.js +23 -10
- package/distTs/src/visualTest/context.js +42 -31
- package/distTs/src/visualTest/requests.js +39 -139
- package/distTs/src/visualTest/resources.js +40 -38
- package/distTs/src/visualTest/server.js +2 -2
- package/distTs/src/visualTest/snapshots.js +18 -17
- package/distTs/src/visualTest/validation.js +2 -10
- package/package.json +1 -1
|
@@ -9,15 +9,11 @@ exports.closeSession = closeSession;
|
|
|
9
9
|
exports.getDefaultSettings = getDefaultSettings;
|
|
10
10
|
exports.sendStorybook = sendStorybook;
|
|
11
11
|
exports.sendCompareLinks = sendCompareLinks;
|
|
12
|
-
exports.sendScrap = sendScrap;
|
|
13
|
-
exports.downloadScrapPackage = downloadScrapPackage;
|
|
14
|
-
exports.connectToScrapSession = connectToScrapSession;
|
|
15
12
|
const context_1 = require("./context");
|
|
16
13
|
const undici_1 = require("undici");
|
|
17
14
|
const uuid_1 = require("uuid");
|
|
18
15
|
const output_1 = __importDefault(require("../output"));
|
|
19
16
|
const texts_1 = require("../texts");
|
|
20
|
-
const eventsource_1 = require("eventsource");
|
|
21
17
|
const customServiceUrl = process.env.BUDDY_VT_SERVICE_URL;
|
|
22
18
|
function checkIfIsDevToken(token) {
|
|
23
19
|
const tokenParts = token.split('_');
|
|
@@ -36,14 +32,32 @@ function getServiceUrl(token) {
|
|
|
36
32
|
}
|
|
37
33
|
return devServiceUrl;
|
|
38
34
|
}
|
|
39
|
-
if (token.startsWith('bud_vt_eu') || token.startsWith('
|
|
35
|
+
if (token.startsWith('bud_vt_eu') || token.startsWith('bud_crawl_eu')) {
|
|
40
36
|
return 'https://vt.eu.buddy.works';
|
|
41
37
|
}
|
|
42
|
-
if (token.startsWith('bud_vt_asia') || token.startsWith('
|
|
38
|
+
if (token.startsWith('bud_vt_asia') || token.startsWith('bud_crawl_asia')) {
|
|
43
39
|
return 'https://vt.asia.buddy.works';
|
|
44
40
|
}
|
|
45
41
|
return 'https://vt.buddy.works';
|
|
46
42
|
}
|
|
43
|
+
function getCiPayload(ctx) {
|
|
44
|
+
return {
|
|
45
|
+
ci: ctx.ci,
|
|
46
|
+
branch: ctx.branch,
|
|
47
|
+
tag: ctx.tag,
|
|
48
|
+
pullRequestNumber: ctx.pullRequestNumber,
|
|
49
|
+
commit: ctx.commit,
|
|
50
|
+
baseCommit: ctx.baseCommit,
|
|
51
|
+
pipelineId: ctx.pipelineId,
|
|
52
|
+
pipelineName: ctx.pipelineName,
|
|
53
|
+
actionId: ctx.actionId,
|
|
54
|
+
executionId: ctx.executionId,
|
|
55
|
+
invokerId: ctx.invokerId,
|
|
56
|
+
commitDetails: ctx.commitDetails,
|
|
57
|
+
cliId: context_1.cliId,
|
|
58
|
+
executionUrl: ctx.executionUrl,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
47
61
|
function prepareSnapshotInternal(snapshot) {
|
|
48
62
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
49
63
|
const { resourceDiscoveryTimeout, cookies, ...filteredSnapshot } = snapshot;
|
|
@@ -81,33 +95,20 @@ function createFormData(info, files) {
|
|
|
81
95
|
}
|
|
82
96
|
return formData;
|
|
83
97
|
}
|
|
84
|
-
async function sendSnapshot(snapshot, firstSnapshot) {
|
|
98
|
+
async function sendSnapshot(ctx, snapshot, firstSnapshot) {
|
|
85
99
|
const files = [];
|
|
86
100
|
const preparedSnapshot = prepareSnapshot(snapshot, files);
|
|
87
101
|
const info = {
|
|
88
102
|
snapshot: preparedSnapshot,
|
|
89
103
|
buildId: context_1.buildId,
|
|
90
|
-
parallel:
|
|
91
|
-
|
|
92
|
-
branch: context_1.branch,
|
|
93
|
-
tag: context_1.tag,
|
|
94
|
-
pullRequestNumber: context_1.pullRequestNumber,
|
|
95
|
-
commit: context_1.commit,
|
|
96
|
-
baseCommit: context_1.baseCommit,
|
|
97
|
-
pipelineId: context_1.pipelineId,
|
|
98
|
-
pipelineName: context_1.pipelineName,
|
|
99
|
-
actionId: context_1.actionId,
|
|
100
|
-
executionId: context_1.executionId,
|
|
101
|
-
invokerId: context_1.invokerId,
|
|
102
|
-
commitDetails: context_1.commitDetails,
|
|
103
|
-
cliId: context_1.cliId,
|
|
104
|
-
executionUrl: context_1.executionUrl,
|
|
104
|
+
parallel: ctx.parallel,
|
|
105
|
+
...getCiPayload(ctx),
|
|
105
106
|
firstSnapshot,
|
|
106
107
|
};
|
|
107
108
|
const formData = createFormData(info, files);
|
|
108
109
|
const [message, response] = await sendRequest({
|
|
109
110
|
path: '/snapshot',
|
|
110
|
-
token:
|
|
111
|
+
token: ctx.token,
|
|
111
112
|
payload: formData,
|
|
112
113
|
multipart: true,
|
|
113
114
|
});
|
|
@@ -119,33 +120,20 @@ async function sendSnapshot(snapshot, firstSnapshot) {
|
|
|
119
120
|
}
|
|
120
121
|
return response;
|
|
121
122
|
}
|
|
122
|
-
async function sendSnapshots(snapshots) {
|
|
123
|
+
async function sendSnapshots(ctx, snapshots) {
|
|
123
124
|
const files = [];
|
|
124
125
|
const preparedSnapshots = snapshots.map((snapshot) => prepareSnapshot(snapshot, files));
|
|
125
126
|
const info = {
|
|
126
127
|
snapshots: preparedSnapshots,
|
|
127
128
|
buildId: context_1.buildId,
|
|
128
|
-
parallel:
|
|
129
|
-
|
|
130
|
-
branch: context_1.branch,
|
|
131
|
-
tag: context_1.tag,
|
|
132
|
-
pullRequestNumber: context_1.pullRequestNumber,
|
|
133
|
-
commit: context_1.commit,
|
|
134
|
-
baseCommit: context_1.baseCommit,
|
|
135
|
-
pipelineId: context_1.pipelineId,
|
|
136
|
-
pipelineName: context_1.pipelineName,
|
|
137
|
-
actionId: context_1.actionId,
|
|
138
|
-
executionId: context_1.executionId,
|
|
139
|
-
invokerId: context_1.invokerId,
|
|
140
|
-
commitDetails: context_1.commitDetails,
|
|
141
|
-
cliId: context_1.cliId,
|
|
142
|
-
executionUrl: context_1.executionUrl,
|
|
129
|
+
parallel: ctx.parallel,
|
|
130
|
+
...getCiPayload(ctx),
|
|
143
131
|
firstSnapshot: true,
|
|
144
132
|
};
|
|
145
133
|
const formData = createFormData(info, files);
|
|
146
134
|
const [message, response] = await sendRequest({
|
|
147
135
|
path: '/snapshots',
|
|
148
|
-
token:
|
|
136
|
+
token: ctx.token,
|
|
149
137
|
payload: formData,
|
|
150
138
|
multipart: true,
|
|
151
139
|
});
|
|
@@ -157,14 +145,14 @@ async function sendSnapshots(snapshots) {
|
|
|
157
145
|
}
|
|
158
146
|
return response;
|
|
159
147
|
}
|
|
160
|
-
async function closeSession() {
|
|
148
|
+
async function closeSession(ctx) {
|
|
161
149
|
const payload = {
|
|
162
|
-
token:
|
|
150
|
+
token: ctx.token,
|
|
163
151
|
buildId: context_1.buildId,
|
|
164
152
|
};
|
|
165
153
|
const [message, response] = await sendRequest({
|
|
166
154
|
path: '/close',
|
|
167
|
-
token:
|
|
155
|
+
token: ctx.token,
|
|
168
156
|
payload,
|
|
169
157
|
});
|
|
170
158
|
if (message) {
|
|
@@ -175,10 +163,10 @@ async function closeSession() {
|
|
|
175
163
|
}
|
|
176
164
|
return response;
|
|
177
165
|
}
|
|
178
|
-
async function getDefaultSettings() {
|
|
166
|
+
async function getDefaultSettings(ctx) {
|
|
179
167
|
const [message, response] = await sendRequest({
|
|
180
168
|
path: `/defaultSettings`,
|
|
181
|
-
token:
|
|
169
|
+
token: ctx.token,
|
|
182
170
|
});
|
|
183
171
|
if (message) {
|
|
184
172
|
throw new Error(message);
|
|
@@ -188,24 +176,11 @@ async function getDefaultSettings() {
|
|
|
188
176
|
}
|
|
189
177
|
return response;
|
|
190
178
|
}
|
|
191
|
-
async function sendStorybook(snapshots, compressedStorybookDirectory) {
|
|
179
|
+
async function sendStorybook(ctx, snapshots, compressedStorybookDirectory) {
|
|
192
180
|
const info = {
|
|
193
181
|
snapshots,
|
|
194
182
|
buildId: context_1.buildId,
|
|
195
|
-
|
|
196
|
-
branch: context_1.branch,
|
|
197
|
-
tag: context_1.tag,
|
|
198
|
-
pullRequestNumber: context_1.pullRequestNumber,
|
|
199
|
-
commit: context_1.commit,
|
|
200
|
-
baseCommit: context_1.baseCommit,
|
|
201
|
-
pipelineId: context_1.pipelineId,
|
|
202
|
-
pipelineName: context_1.pipelineName,
|
|
203
|
-
actionId: context_1.actionId,
|
|
204
|
-
executionId: context_1.executionId,
|
|
205
|
-
invokerId: context_1.invokerId,
|
|
206
|
-
commitDetails: context_1.commitDetails,
|
|
207
|
-
cliId: context_1.cliId,
|
|
208
|
-
executionUrl: context_1.executionUrl,
|
|
183
|
+
...getCiPayload(ctx),
|
|
209
184
|
};
|
|
210
185
|
const files = [
|
|
211
186
|
{
|
|
@@ -216,7 +191,7 @@ async function sendStorybook(snapshots, compressedStorybookDirectory) {
|
|
|
216
191
|
const formData = createFormData(info, files);
|
|
217
192
|
const [message, response] = await sendRequest({
|
|
218
193
|
path: '/storybook',
|
|
219
|
-
token:
|
|
194
|
+
token: ctx.token,
|
|
220
195
|
payload: formData,
|
|
221
196
|
multipart: true,
|
|
222
197
|
});
|
|
@@ -228,7 +203,7 @@ async function sendStorybook(snapshots, compressedStorybookDirectory) {
|
|
|
228
203
|
}
|
|
229
204
|
return response;
|
|
230
205
|
}
|
|
231
|
-
async function sendCompareLinks(urls, validatedOptions, sitemapSource) {
|
|
206
|
+
async function sendCompareLinks(ctx, urls, validatedOptions, sitemapSource) {
|
|
232
207
|
const info = {
|
|
233
208
|
urls,
|
|
234
209
|
sitemapSource,
|
|
@@ -241,22 +216,11 @@ async function sendCompareLinks(urls, validatedOptions, sitemapSource) {
|
|
|
241
216
|
respectRobots: validatedOptions.respectRobots,
|
|
242
217
|
ignoreUrls: validatedOptions.ignoreUrls ?? [],
|
|
243
218
|
buildId: context_1.buildId,
|
|
244
|
-
|
|
245
|
-
branch: context_1.branch,
|
|
246
|
-
tag: context_1.tag,
|
|
247
|
-
commit: context_1.commit,
|
|
248
|
-
baseCommit: context_1.baseCommit,
|
|
249
|
-
pipelineId: context_1.pipelineId,
|
|
250
|
-
pipelineName: context_1.pipelineName,
|
|
251
|
-
actionId: context_1.actionId,
|
|
252
|
-
executionId: context_1.executionId,
|
|
253
|
-
invokerId: context_1.invokerId,
|
|
254
|
-
commitDetails: context_1.commitDetails,
|
|
255
|
-
cliId: context_1.cliId,
|
|
219
|
+
...getCiPayload(ctx),
|
|
256
220
|
};
|
|
257
221
|
const [message, response] = await sendRequest({
|
|
258
222
|
path: '/compareLinks',
|
|
259
|
-
token:
|
|
223
|
+
token: ctx.token,
|
|
260
224
|
payload: info,
|
|
261
225
|
});
|
|
262
226
|
if (message) {
|
|
@@ -267,55 +231,6 @@ async function sendCompareLinks(urls, validatedOptions, sitemapSource) {
|
|
|
267
231
|
}
|
|
268
232
|
return response;
|
|
269
233
|
}
|
|
270
|
-
async function sendScrap(url, follow, respectRobots, outputTypes, colorScheme, browsers, devices, cookies, requestHeaders, delays, waitForSelectors, localStorage) {
|
|
271
|
-
const payload = {
|
|
272
|
-
url,
|
|
273
|
-
follow,
|
|
274
|
-
respectRobots,
|
|
275
|
-
outputTypes,
|
|
276
|
-
colorScheme,
|
|
277
|
-
browsers,
|
|
278
|
-
devices,
|
|
279
|
-
cookies,
|
|
280
|
-
requestHeaders,
|
|
281
|
-
delays,
|
|
282
|
-
waitForSelectors,
|
|
283
|
-
localStorage,
|
|
284
|
-
pipelineId: context_1.pipelineId,
|
|
285
|
-
pipelineName: context_1.pipelineName,
|
|
286
|
-
executionId: context_1.executionId,
|
|
287
|
-
actionId: context_1.actionId,
|
|
288
|
-
invokerId: context_1.invokerId,
|
|
289
|
-
ci: context_1.ci,
|
|
290
|
-
executionUrl: context_1.executionUrl,
|
|
291
|
-
};
|
|
292
|
-
const [message, response] = await sendRequest({
|
|
293
|
-
path: '/scrap',
|
|
294
|
-
token: context_1.scrapeToken,
|
|
295
|
-
payload,
|
|
296
|
-
});
|
|
297
|
-
if (message) {
|
|
298
|
-
throw new Error(message);
|
|
299
|
-
}
|
|
300
|
-
if (!response) {
|
|
301
|
-
throw new Error(texts_1.ERR_INVALID_SCRAPE_RESPONSE);
|
|
302
|
-
}
|
|
303
|
-
return response;
|
|
304
|
-
}
|
|
305
|
-
async function downloadScrapPackage(buildId) {
|
|
306
|
-
const [message, response] = await sendRequest({
|
|
307
|
-
path: '/download',
|
|
308
|
-
token: context_1.scrapeToken,
|
|
309
|
-
payload: { token: context_1.scrapeToken, buildId },
|
|
310
|
-
});
|
|
311
|
-
if (message) {
|
|
312
|
-
throw new Error(message);
|
|
313
|
-
}
|
|
314
|
-
if (!response) {
|
|
315
|
-
throw new Error(texts_1.ERR_INVALID_DOWNLOAD_RESPONSE);
|
|
316
|
-
}
|
|
317
|
-
return response;
|
|
318
|
-
}
|
|
319
234
|
async function sendRequest({ path, token, payload, multipart, }) {
|
|
320
235
|
const fullUrl = new URL(path, getServiceUrl(token));
|
|
321
236
|
output_1.default.debug((0, texts_1.LOG_SENDING_REQUEST)(fullUrl.toString()));
|
|
@@ -366,18 +281,3 @@ async function sendRequest({ path, token, payload, multipart, }) {
|
|
|
366
281
|
const text = await response.text();
|
|
367
282
|
return [text, undefined];
|
|
368
283
|
}
|
|
369
|
-
function connectToScrapSession(buildId) {
|
|
370
|
-
return new eventsource_1.EventSource(`${getServiceUrl(context_1.scrapeToken)}/sse`, {
|
|
371
|
-
fetch: (url, options) => {
|
|
372
|
-
return (0, undici_1.fetch)(url, {
|
|
373
|
-
...options,
|
|
374
|
-
headers: {
|
|
375
|
-
...options?.headers,
|
|
376
|
-
'x-token': context_1.scrapeToken,
|
|
377
|
-
'x-cli-version': context_1.cliVersion,
|
|
378
|
-
'x-build-id': buildId,
|
|
379
|
-
},
|
|
380
|
-
});
|
|
381
|
-
},
|
|
382
|
-
});
|
|
383
|
-
}
|
|
@@ -42,63 +42,65 @@ const context_1 = require("./context");
|
|
|
42
42
|
const output_1 = __importDefault(require("../output"));
|
|
43
43
|
const texts_1 = require("../texts");
|
|
44
44
|
const utils_1 = require("../utils");
|
|
45
|
-
const
|
|
46
|
-
const
|
|
45
|
+
const crawledUrls = new Map();
|
|
46
|
+
const crawledResources = new Map();
|
|
47
47
|
let showedBrowserVersionWarning = false;
|
|
48
|
-
async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, cookies = [], }) {
|
|
49
|
-
const
|
|
50
|
-
const missingDevices =
|
|
51
|
-
? devices.filter(({ viewport }) => !
|
|
48
|
+
async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, cookies = [], browserPath, }) {
|
|
49
|
+
const crawledUrl = crawledUrls.get(url);
|
|
50
|
+
const missingDevices = crawledUrl
|
|
51
|
+
? devices.filter(({ viewport }) => !crawledUrl.widths.includes(viewport.width))
|
|
52
52
|
: [];
|
|
53
|
-
if (
|
|
53
|
+
if (crawledUrl && missingDevices.length === 0) {
|
|
54
54
|
return getResourceUrlsData({
|
|
55
|
-
duplicatedResourcesUrls:
|
|
55
|
+
duplicatedResourcesUrls: crawledUrl.resources,
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
|
-
else if (
|
|
59
|
-
const {
|
|
58
|
+
else if (crawledUrl && missingDevices.length > 0) {
|
|
59
|
+
const { crawledResourcesUrls, duplicatedResourcesUrls } = await output_1.default.debugAction((0, texts_1.DEBUG_RESOURCE_CRAWLING_URL)(url), crawlResources({
|
|
60
60
|
url,
|
|
61
61
|
devices: missingDevices,
|
|
62
62
|
resourceDiscoveryTimeout,
|
|
63
63
|
cookies,
|
|
64
|
+
browserPath,
|
|
64
65
|
}));
|
|
65
|
-
const
|
|
66
|
-
|
|
66
|
+
const duplicatedCrawledResourceUrls = [...crawledUrl.resources];
|
|
67
|
+
crawledUrls.set(url, {
|
|
67
68
|
widths: [
|
|
68
|
-
...
|
|
69
|
+
...crawledUrl.widths,
|
|
69
70
|
...missingDevices.map(({ viewport }) => viewport.width),
|
|
70
71
|
],
|
|
71
72
|
resources: [
|
|
72
73
|
...new Set([
|
|
73
|
-
...
|
|
74
|
-
...
|
|
74
|
+
...crawledUrl.resources,
|
|
75
|
+
...crawledResourcesUrls,
|
|
75
76
|
...duplicatedResourcesUrls,
|
|
76
77
|
]),
|
|
77
78
|
],
|
|
78
79
|
});
|
|
79
80
|
return getResourceUrlsData({
|
|
80
|
-
resourcesUrls:
|
|
81
|
+
resourcesUrls: crawledResourcesUrls,
|
|
81
82
|
duplicatedResourcesUrls: [
|
|
82
83
|
...new Set([
|
|
83
84
|
...duplicatedResourcesUrls,
|
|
84
|
-
...
|
|
85
|
+
...duplicatedCrawledResourceUrls,
|
|
85
86
|
]),
|
|
86
87
|
],
|
|
87
88
|
});
|
|
88
89
|
}
|
|
89
90
|
else {
|
|
90
|
-
const {
|
|
91
|
+
const { crawledResourcesUrls, duplicatedResourcesUrls } = await output_1.default.debugAction((0, texts_1.DEBUG_RESOURCE_CRAWLING_URL)(url), crawlResources({
|
|
91
92
|
url,
|
|
92
93
|
devices,
|
|
93
94
|
resourceDiscoveryTimeout,
|
|
94
95
|
cookies,
|
|
96
|
+
browserPath,
|
|
95
97
|
}));
|
|
96
|
-
|
|
98
|
+
crawledUrls.set(url, {
|
|
97
99
|
widths: devices.map(({ viewport }) => viewport.width),
|
|
98
|
-
resources: [...
|
|
100
|
+
resources: [...crawledResourcesUrls, ...duplicatedResourcesUrls],
|
|
99
101
|
});
|
|
100
102
|
return getResourceUrlsData({
|
|
101
|
-
resourcesUrls:
|
|
103
|
+
resourcesUrls: crawledResourcesUrls,
|
|
102
104
|
duplicatedResourcesUrls,
|
|
103
105
|
});
|
|
104
106
|
}
|
|
@@ -106,13 +108,13 @@ async function collectResources({ url, devices, resourceDiscoveryTimeout = 0, co
|
|
|
106
108
|
function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [], }) {
|
|
107
109
|
const resources = [];
|
|
108
110
|
for (const resourceUrl of resourcesUrls) {
|
|
109
|
-
const
|
|
110
|
-
if (
|
|
111
|
+
const crawledResource = crawledResources.get(resourceUrl);
|
|
112
|
+
if (crawledResource) {
|
|
111
113
|
resources.push({
|
|
112
114
|
url: resourceUrl,
|
|
113
|
-
status:
|
|
114
|
-
headers:
|
|
115
|
-
body:
|
|
115
|
+
status: crawledResource.status,
|
|
116
|
+
headers: crawledResource.headers,
|
|
117
|
+
body: crawledResource.body,
|
|
116
118
|
});
|
|
117
119
|
}
|
|
118
120
|
else {
|
|
@@ -120,12 +122,12 @@ function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [],
|
|
|
120
122
|
}
|
|
121
123
|
}
|
|
122
124
|
for (const resourceUrl of duplicatedResourcesUrls) {
|
|
123
|
-
const
|
|
124
|
-
if (
|
|
125
|
+
const crawledResource = crawledResources.get(resourceUrl);
|
|
126
|
+
if (crawledResource) {
|
|
125
127
|
resources.push({
|
|
126
128
|
url: resourceUrl,
|
|
127
|
-
status:
|
|
128
|
-
headers:
|
|
129
|
+
status: crawledResource.status,
|
|
130
|
+
headers: crawledResource.headers,
|
|
129
131
|
duplicate: true,
|
|
130
132
|
});
|
|
131
133
|
}
|
|
@@ -135,15 +137,15 @@ function getResourceUrlsData({ resourcesUrls = [], duplicatedResourcesUrls = [],
|
|
|
135
137
|
}
|
|
136
138
|
return resources;
|
|
137
139
|
}
|
|
138
|
-
async function
|
|
139
|
-
const
|
|
140
|
+
async function crawlResources({ url, devices, resourceDiscoveryTimeout, cookies = [], browserPath, }) {
|
|
141
|
+
const crawledResourcesUrls = [];
|
|
140
142
|
const duplicatedResourcesUrls = [];
|
|
141
143
|
const launchOptions = {
|
|
142
144
|
headless: 'shell',
|
|
143
145
|
args: ['--no-sandbox'],
|
|
144
146
|
acceptInsecureCerts: true,
|
|
145
147
|
timeout: 120_000,
|
|
146
|
-
executablePath:
|
|
148
|
+
executablePath: browserPath,
|
|
147
149
|
};
|
|
148
150
|
const browser = await puppeteer_core_1.default.launch(launchOptions);
|
|
149
151
|
if (!showedBrowserVersionWarning) {
|
|
@@ -172,8 +174,8 @@ async function scrapeResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
172
174
|
page.on('response', async (response) => {
|
|
173
175
|
const request = response.request();
|
|
174
176
|
const requestUrl = request.url();
|
|
175
|
-
const isCollected =
|
|
176
|
-
if (isCollected && !
|
|
177
|
+
const isCollected = crawledResources.has(requestUrl);
|
|
178
|
+
if (isCollected && !crawledResourcesUrls.includes(requestUrl)) {
|
|
177
179
|
duplicatedResourcesUrls.push(requestUrl);
|
|
178
180
|
}
|
|
179
181
|
else {
|
|
@@ -193,8 +195,8 @@ async function scrapeResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
193
195
|
responseError = error;
|
|
194
196
|
}
|
|
195
197
|
if (!responseError && body) {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
+
crawledResourcesUrls.push(requestUrl);
|
|
199
|
+
crawledResources.set(requestUrl, {
|
|
198
200
|
status,
|
|
199
201
|
headers: { 'content-type': contentType },
|
|
200
202
|
body,
|
|
@@ -229,7 +231,7 @@ async function scrapeResources({ url, devices, resourceDiscoveryTimeout, cookies
|
|
|
229
231
|
}
|
|
230
232
|
}
|
|
231
233
|
await browser.close();
|
|
232
|
-
return {
|
|
234
|
+
return { crawledResourcesUrls, duplicatedResourcesUrls };
|
|
233
235
|
}
|
|
234
236
|
async function autoScroll(page) {
|
|
235
237
|
await page.evaluate(async () => {
|
|
@@ -81,7 +81,7 @@ function prepareSnapshotPlugin(data) {
|
|
|
81
81
|
}
|
|
82
82
|
return undefined;
|
|
83
83
|
}
|
|
84
|
-
async function createServer() {
|
|
84
|
+
async function createServer(ctx) {
|
|
85
85
|
const app = (0, fastify_1.default)({
|
|
86
86
|
// logger: true,
|
|
87
87
|
bodyLimit: 50 * 1024 * 1024, //50 MB,
|
|
@@ -95,7 +95,7 @@ async function createServer() {
|
|
|
95
95
|
const data = request.body;
|
|
96
96
|
const preparedSnapshot = prepareSnapshotPlugin(data);
|
|
97
97
|
if (preparedSnapshot) {
|
|
98
|
-
(0, snapshots_1.addSnapshot)(preparedSnapshot);
|
|
98
|
+
(0, snapshots_1.addSnapshot)(ctx, preparedSnapshot);
|
|
99
99
|
reply.send('Snapshot received');
|
|
100
100
|
}
|
|
101
101
|
else {
|
|
@@ -20,13 +20,13 @@ let sessionLink = '';
|
|
|
20
20
|
function setDefaultSettings(defaultSettings) {
|
|
21
21
|
defaultDevices = defaultSettings.defaultDevices;
|
|
22
22
|
}
|
|
23
|
-
function addSnapshot(snapshot) {
|
|
23
|
+
function addSnapshot(ctx, snapshot) {
|
|
24
24
|
const preparedSnapshot = {
|
|
25
25
|
...snapshot,
|
|
26
26
|
devices: snapshot.devices.length > 0 ? snapshot.devices : defaultDevices,
|
|
27
27
|
};
|
|
28
|
-
if (
|
|
29
|
-
(0, queue_1.addToQueue)(() => processSnapshot(preparedSnapshot, firstSnapshot));
|
|
28
|
+
if (ctx.oneByOne) {
|
|
29
|
+
(0, queue_1.addToQueue)(() => processSnapshot(ctx, preparedSnapshot, firstSnapshot));
|
|
30
30
|
}
|
|
31
31
|
else {
|
|
32
32
|
snapshots.push(preparedSnapshot);
|
|
@@ -35,12 +35,12 @@ function addSnapshot(snapshot) {
|
|
|
35
35
|
firstSnapshot = false;
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
|
-
async function finishProcessingSnapshots(spawnedProcessExitCode) {
|
|
39
|
-
if (
|
|
38
|
+
async function finishProcessingSnapshots(ctx, spawnedProcessExitCode) {
|
|
39
|
+
if (ctx.oneByOne) {
|
|
40
40
|
return await (0, queue_1.isQueueEmpty)();
|
|
41
41
|
}
|
|
42
42
|
if (spawnedProcessExitCode === 0) {
|
|
43
|
-
return await processSnapshots();
|
|
43
|
+
return await processSnapshots(ctx);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
function showSessionLink() {
|
|
@@ -48,17 +48,17 @@ function showSessionLink() {
|
|
|
48
48
|
output_1.default.normal((0, texts_1.LOG_SESSION_LINK)(sessionLink));
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
|
-
async function processSnapshot(snapshot, firstSnapshot) {
|
|
51
|
+
async function processSnapshot(ctx, snapshot, firstSnapshot) {
|
|
52
52
|
let t1;
|
|
53
53
|
if (context_1.debug) {
|
|
54
54
|
t1 = performance.now();
|
|
55
55
|
}
|
|
56
|
-
const preparedSnapshot =
|
|
56
|
+
const preparedSnapshot = ctx.skipDiscovery
|
|
57
57
|
? snapshot
|
|
58
|
-
: await getSnapshotWithResources(snapshot);
|
|
58
|
+
: await getSnapshotWithResources(ctx, snapshot);
|
|
59
59
|
try {
|
|
60
60
|
output_1.default.normal(texts_1.LOG_SENDING_DATA);
|
|
61
|
-
const { message } = await (0, requests_1.sendSnapshot)(preparedSnapshot, firstSnapshot);
|
|
61
|
+
const { message } = await (0, requests_1.sendSnapshot)(ctx, preparedSnapshot, firstSnapshot);
|
|
62
62
|
sessionLink = message;
|
|
63
63
|
}
|
|
64
64
|
catch (error) {
|
|
@@ -69,22 +69,22 @@ async function processSnapshot(snapshot, firstSnapshot) {
|
|
|
69
69
|
output_1.default.normal((0, texts_1.DEBUG_SNAPSHOT_PROCESSING)(snapshot.name, t2 - t1));
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
|
-
async function processSnapshots() {
|
|
72
|
+
async function processSnapshots(ctx) {
|
|
73
73
|
let t1;
|
|
74
74
|
if (context_1.debug) {
|
|
75
75
|
t1 = performance.now();
|
|
76
76
|
}
|
|
77
77
|
output_1.default.normal((0, texts_1.LOG_PROCESSING_SNAPSHOTS)(snapshots.length));
|
|
78
|
-
const preparedSnapshots =
|
|
78
|
+
const preparedSnapshots = ctx.skipDiscovery
|
|
79
79
|
? snapshots
|
|
80
|
-
: await getSnapshotsWithResources(snapshots);
|
|
80
|
+
: await getSnapshotsWithResources(ctx, snapshots);
|
|
81
81
|
if (preparedSnapshots.length === 0) {
|
|
82
82
|
output_1.default.error(texts_1.ERR_NO_SNAPSHOTS_TO_SEND);
|
|
83
83
|
}
|
|
84
84
|
else {
|
|
85
85
|
try {
|
|
86
86
|
output_1.default.normal(texts_1.LOG_SENDING_DATA);
|
|
87
|
-
const { message } = await (0, requests_1.sendSnapshots)(preparedSnapshots);
|
|
87
|
+
const { message } = await (0, requests_1.sendSnapshots)(ctx, preparedSnapshots);
|
|
88
88
|
sessionLink = message;
|
|
89
89
|
}
|
|
90
90
|
catch (error) {
|
|
@@ -96,13 +96,14 @@ async function processSnapshots() {
|
|
|
96
96
|
output_1.default.normal((0, texts_1.DEBUG_SNAPSHOTS_PROCESSING)(t2 - t1));
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
async function getSnapshotWithResources(snapshot) {
|
|
99
|
+
async function getSnapshotWithResources(ctx, snapshot) {
|
|
100
100
|
const { url, devices, resourceDiscoveryTimeout, cookies } = snapshot;
|
|
101
101
|
const resources = await (0, resources_1.collectResources)({
|
|
102
102
|
url,
|
|
103
103
|
devices,
|
|
104
104
|
resourceDiscoveryTimeout,
|
|
105
105
|
cookies,
|
|
106
|
+
browserPath: ctx.browserPath,
|
|
106
107
|
});
|
|
107
108
|
if (context_1.debug) {
|
|
108
109
|
output_1.default.normal(`Collected resources for snapshot: ${snapshot.name}`);
|
|
@@ -115,11 +116,11 @@ async function getSnapshotWithResources(snapshot) {
|
|
|
115
116
|
resources: [...snapshot.resources, ...resources],
|
|
116
117
|
};
|
|
117
118
|
}
|
|
118
|
-
async function getSnapshotsWithResources(snapshots) {
|
|
119
|
+
async function getSnapshotsWithResources(ctx, snapshots) {
|
|
119
120
|
const snapshotsWithResources = [];
|
|
120
121
|
const prepareSnapshots = filterDuplicates(snapshots);
|
|
121
122
|
for (const snapshot of prepareSnapshots) {
|
|
122
|
-
const snapshotWithResources = await getSnapshotWithResources(snapshot);
|
|
123
|
+
const snapshotWithResources = await getSnapshotWithResources(ctx, snapshot);
|
|
123
124
|
snapshotsWithResources.push(snapshotWithResources);
|
|
124
125
|
}
|
|
125
126
|
return snapshotsWithResources;
|
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.checkToken = checkToken;
|
|
4
3
|
exports.checkBuildId = checkBuildId;
|
|
5
4
|
exports.checkParallel = checkParallel;
|
|
6
|
-
const context_1 = require("./context");
|
|
7
|
-
function checkToken(type) {
|
|
8
|
-
if (type === 'scrape') {
|
|
9
|
-
return !!context_1.scrapeToken;
|
|
10
|
-
}
|
|
11
|
-
return !!context_1.vtToken;
|
|
12
|
-
}
|
|
13
5
|
function checkBuildId() {
|
|
14
6
|
return !!process.env.SNAPSHOTS_BUILD_ID;
|
|
15
7
|
}
|
|
16
|
-
function checkParallel() {
|
|
17
|
-
if (
|
|
8
|
+
function checkParallel(ctx) {
|
|
9
|
+
if (ctx.parallel) {
|
|
18
10
|
return checkBuildId();
|
|
19
11
|
}
|
|
20
12
|
return true;
|