@alwaysmeticulous/cli 2.3.4 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/command-utils/common-options.d.ts +110 -0
- package/dist/command-utils/common-options.js +87 -0
- package/dist/command-utils/common-types.d.ts +20 -0
- package/dist/command-utils/common-types.js +2 -0
- package/dist/commands/create-test/create-test.command.d.ts +2 -2
- package/dist/commands/create-test/create-test.command.js +24 -40
- package/dist/commands/record/record.command.d.ts +9 -9
- package/dist/commands/replay/replay.command.d.ts +27 -26
- package/dist/commands/replay/replay.command.js +95 -76
- package/dist/commands/run-all-tests/run-all-tests.command.d.ts +11 -18
- package/dist/commands/run-all-tests/run-all-tests.command.js +40 -79
- package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts +4 -4
- package/dist/commands/screenshot-diff/screenshot-diff.command.js +15 -19
- package/dist/config/config.js +3 -4
- package/dist/config/config.types.d.ts +7 -8
- package/dist/deflake-tests/deflake-tests.handler.d.ts +12 -3
- package/dist/deflake-tests/deflake-tests.handler.js +42 -6
- package/dist/image/diff.utils.d.ts +8 -1
- package/dist/image/diff.utils.js +1 -5
- package/dist/parallel-tests/messages.types.d.ts +3 -18
- package/dist/parallel-tests/parallel-tests.handler.d.ts +9 -14
- package/dist/parallel-tests/parallel-tests.handler.js +11 -15
- package/dist/parallel-tests/task.handler.js +2 -26
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/utils/api-token.utils.d.ts +1 -1
- package/dist/utils/api-token.utils.js +1 -1
- package/dist/utils/config.utils.d.ts +6 -1
- package/dist/utils/config.utils.js +17 -8
- package/package.json +6 -6
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.replay = exports.replayCommandHandler = void 0;
|
|
6
|
+
exports.replay = exports.rawReplayCommandHandler = exports.replayCommandHandler = void 0;
|
|
7
7
|
const common_1 = require("@alwaysmeticulous/common");
|
|
8
8
|
const promises_1 = require("fs/promises");
|
|
9
9
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
@@ -13,17 +13,18 @@ const client_1 = require("../../api/client");
|
|
|
13
13
|
const replay_api_1 = require("../../api/replay.api");
|
|
14
14
|
const upload_1 = require("../../api/upload");
|
|
15
15
|
const archive_1 = require("../../archive/archive");
|
|
16
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
16
17
|
const local_data_utils_1 = require("../../local-data/local-data.utils");
|
|
17
18
|
const replay_assets_1 = require("../../local-data/replay-assets");
|
|
18
19
|
const replays_1 = require("../../local-data/replays");
|
|
20
|
+
const serve_assets_from_simulation_1 = require("../../local-data/serve-assets-from-simulation");
|
|
19
21
|
const sessions_1 = require("../../local-data/sessions");
|
|
20
22
|
const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
|
|
23
|
+
const config_utils_1 = require("../../utils/config.utils");
|
|
21
24
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
22
25
|
const version_utils_1 = require("../../utils/version.utils");
|
|
23
26
|
const screenshot_diff_command_1 = require("../screenshot-diff/screenshot-diff.command");
|
|
24
|
-
const
|
|
25
|
-
const serve_assets_from_simulation_1 = require("../../local-data/serve-assets-from-simulation");
|
|
26
|
-
const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId: baseReplayId_, diffThreshold, diffPixelThreshold, save, exitOnMismatch, padTime, shiftTime, networkStubbing, moveBeforeClick, cookies, cookiesFile, accelerate, maxDurationMs, maxEventCount, }) => {
|
|
27
|
+
const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, }) => {
|
|
27
28
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
28
29
|
const client = (0, client_1.createClient)({ apiToken });
|
|
29
30
|
// 1. Check session files
|
|
@@ -34,9 +35,7 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
34
35
|
logger.debug(`Commit: ${commitSha}`);
|
|
35
36
|
const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
|
|
36
37
|
// 3. If simulationIdForAssets specified then download assets & spin up local server
|
|
37
|
-
const
|
|
38
|
-
? await (0, serve_assets_from_simulation_1.serveAssetsFromSimulation)(client, simulationIdForAssets)
|
|
39
|
-
: undefined;
|
|
38
|
+
const { appUrl, closeServer } = await serveOrGetAppUrl(client, replayTarget);
|
|
40
39
|
// 4. Load replay assets
|
|
41
40
|
const browserUserInteractions = await (0, replay_assets_1.fetchAsset)("replay/v2/snippet-user-interactions.bundle.js");
|
|
42
41
|
const browserPlayback = await (0, replay_assets_1.fetchAsset)("replay/v2/snippet-playback.bundle.js");
|
|
@@ -65,16 +64,14 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
65
64
|
const tempDir = await (0, promises_1.mkdtemp)((0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "replays", tempDirName));
|
|
66
65
|
// 6. Create and save replay parameters
|
|
67
66
|
const replayEventsParams = {
|
|
68
|
-
appUrl:
|
|
67
|
+
appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
|
|
68
|
+
replayExecutionOptions: executionOptions,
|
|
69
69
|
browser: null,
|
|
70
70
|
outputDir: tempDir,
|
|
71
71
|
session,
|
|
72
72
|
sessionData,
|
|
73
73
|
recordingId: "manual-replay",
|
|
74
74
|
meticulousSha: "meticulousSha",
|
|
75
|
-
headless: headless || false,
|
|
76
|
-
devTools: devTools || false,
|
|
77
|
-
bypassCSP: bypassCSP || false,
|
|
78
75
|
dependencies: {
|
|
79
76
|
browserUserInteractions: {
|
|
80
77
|
key: "browserUserInteractions",
|
|
@@ -101,23 +98,14 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
101
98
|
location: nodeUserInteractions,
|
|
102
99
|
},
|
|
103
100
|
},
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
screenshot: screenshot,
|
|
107
|
-
screenshotSelector: screenshotSelector || "",
|
|
108
|
-
networkStubbing,
|
|
109
|
-
moveBeforeClick: moveBeforeClick || false,
|
|
110
|
-
cookies: cookies || null,
|
|
111
|
-
cookiesFile: cookiesFile || "",
|
|
112
|
-
accelerate,
|
|
113
|
-
...(maxDurationMs != null ? { maxDurationMs } : {}),
|
|
114
|
-
...(maxEventCount != null ? { maxEventCount } : {}),
|
|
101
|
+
screenshottingOptions,
|
|
102
|
+
cookiesFile: cookiesFile !== null && cookiesFile !== void 0 ? cookiesFile : null,
|
|
115
103
|
};
|
|
116
104
|
await (0, promises_1.writeFile)((0, path_1.join)(tempDir, "replayEventsParams.json"), JSON.stringify(replayEventsParams));
|
|
117
105
|
// 7. Perform replay
|
|
118
106
|
const startTime = luxon_1.DateTime.utc();
|
|
119
107
|
await replayEvents(replayEventsParams);
|
|
120
|
-
|
|
108
|
+
closeServer === null || closeServer === void 0 ? void 0 : closeServer();
|
|
121
109
|
const endTime = luxon_1.DateTime.utc();
|
|
122
110
|
logger.info(`Simulation time: ${endTime.diff(startTime).as("seconds")} seconds`);
|
|
123
111
|
logger.info("Sending simulation results to Meticulous");
|
|
@@ -157,7 +145,7 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
157
145
|
logger.info("=======");
|
|
158
146
|
// 12. Diff against base replay screenshot if one is provided
|
|
159
147
|
const baseReplayId = baseReplayId_ || "";
|
|
160
|
-
if (
|
|
148
|
+
if (screenshottingOptions.enabled && baseReplayId) {
|
|
161
149
|
logger.info(`Diffing screenshots against replay ${baseReplayId}`);
|
|
162
150
|
await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
|
|
163
151
|
await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
|
|
@@ -169,14 +157,13 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
169
157
|
headReplayId: replay.id,
|
|
170
158
|
baseScreenshotsDir: baseReplayScreenshotsDir,
|
|
171
159
|
headScreenshotsDir: headReplayScreenshotsDir,
|
|
172
|
-
|
|
173
|
-
pixelThreshold: diffPixelThreshold,
|
|
160
|
+
diffOptions: screenshottingOptions.diffOptions,
|
|
174
161
|
exitOnMismatch: !!exitOnMismatch,
|
|
175
162
|
});
|
|
176
163
|
}
|
|
177
164
|
// 13. Add test case to meticulous.json if --save option is passed
|
|
178
165
|
if (save) {
|
|
179
|
-
if (!
|
|
166
|
+
if (!screenshottingOptions.enabled) {
|
|
180
167
|
logger.error("Warning: saving a new test case without screenshot enabled.");
|
|
181
168
|
}
|
|
182
169
|
await (0, config_utils_1.addTestCase)({
|
|
@@ -189,20 +176,82 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
189
176
|
return replay;
|
|
190
177
|
};
|
|
191
178
|
exports.replayCommandHandler = replayCommandHandler;
|
|
192
|
-
const
|
|
193
|
-
|
|
179
|
+
const serveOrGetAppUrl = async (client, replayTarget) => {
|
|
180
|
+
if (replayTarget.type === "snapshotted-assets") {
|
|
181
|
+
const server = await (0, serve_assets_from_simulation_1.serveAssetsFromSimulation)(client, replayTarget.simulationIdForAssets);
|
|
182
|
+
return {
|
|
183
|
+
appUrl: server.url,
|
|
184
|
+
closeServer: server.closeServer,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
if (replayTarget.type === "url") {
|
|
188
|
+
return { appUrl: replayTarget.appUrl };
|
|
189
|
+
}
|
|
190
|
+
if (replayTarget.type === "original-recorded-url") {
|
|
191
|
+
return {};
|
|
192
|
+
}
|
|
193
|
+
return unknownReplayTargetType(replayTarget);
|
|
194
|
+
};
|
|
195
|
+
const unknownReplayTargetType = (replayTarget) => {
|
|
196
|
+
throw new Error(`Unknown type of replay target: ${JSON.stringify(replayTarget)}`);
|
|
197
|
+
};
|
|
198
|
+
const rawReplayCommandHandler = ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId, diffThreshold, diffPixelThreshold, save, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, accelerate, maxDurationMs, maxEventCount, storyboard, }) => {
|
|
199
|
+
const executionOptions = {
|
|
200
|
+
headless,
|
|
201
|
+
devTools,
|
|
202
|
+
bypassCSP,
|
|
203
|
+
padTime,
|
|
204
|
+
shiftTime,
|
|
205
|
+
networkStubbing,
|
|
206
|
+
accelerate,
|
|
207
|
+
moveBeforeClick,
|
|
208
|
+
maxDurationMs: maxDurationMs !== null && maxDurationMs !== void 0 ? maxDurationMs : null,
|
|
209
|
+
maxEventCount: maxEventCount !== null && maxEventCount !== void 0 ? maxEventCount : null,
|
|
210
|
+
};
|
|
211
|
+
const storyboardOptions = storyboard
|
|
212
|
+
? { enabled: true }
|
|
213
|
+
: { enabled: false };
|
|
214
|
+
const screenshottingOptions = screenshot
|
|
215
|
+
? {
|
|
216
|
+
enabled: true,
|
|
217
|
+
screenshotSelector: screenshotSelector !== null && screenshotSelector !== void 0 ? screenshotSelector : null,
|
|
218
|
+
diffOptions: { diffPixelThreshold, diffThreshold },
|
|
219
|
+
storyboardOptions,
|
|
220
|
+
}
|
|
221
|
+
: { enabled: false };
|
|
222
|
+
return (0, exports.replayCommandHandler)({
|
|
223
|
+
replayTarget: getReplayTarget({
|
|
224
|
+
appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
|
|
225
|
+
simulationIdForAssets: simulationIdForAssets !== null && simulationIdForAssets !== void 0 ? simulationIdForAssets : null,
|
|
226
|
+
}),
|
|
227
|
+
executionOptions,
|
|
228
|
+
screenshottingOptions,
|
|
229
|
+
apiToken,
|
|
230
|
+
commitSha,
|
|
231
|
+
cookiesFile,
|
|
232
|
+
sessionId,
|
|
233
|
+
baseSimulationId,
|
|
234
|
+
save,
|
|
235
|
+
exitOnMismatch: true,
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
exports.rawReplayCommandHandler = rawReplayCommandHandler;
|
|
239
|
+
const getReplayTarget = ({ appUrl, simulationIdForAssets, }) => {
|
|
240
|
+
if (simulationIdForAssets) {
|
|
241
|
+
return { type: "snapshotted-assets", simulationIdForAssets };
|
|
242
|
+
}
|
|
243
|
+
if (appUrl) {
|
|
244
|
+
return { type: "url", appUrl };
|
|
245
|
+
}
|
|
246
|
+
return { type: "original-recorded-url" };
|
|
194
247
|
};
|
|
195
248
|
exports.replay = {
|
|
196
249
|
command: "simulate",
|
|
197
250
|
aliases: ["replay"],
|
|
198
251
|
describe: "Simulate (replay) a recorded session",
|
|
199
252
|
builder: {
|
|
200
|
-
apiToken:
|
|
201
|
-
|
|
202
|
-
},
|
|
203
|
-
commitSha: {
|
|
204
|
-
string: true,
|
|
205
|
-
},
|
|
253
|
+
apiToken: common_options_1.OPTIONS.apiToken,
|
|
254
|
+
commitSha: common_options_1.OPTIONS.commitSha,
|
|
206
255
|
sessionId: {
|
|
207
256
|
string: true,
|
|
208
257
|
demandOption: true,
|
|
@@ -216,18 +265,6 @@ exports.replay = {
|
|
|
216
265
|
conflicts: "appUrl",
|
|
217
266
|
description: "If present will run the session against a local server serving up previously snapshotted assets (HTML, JS, CSS etc.) from the specified prior simulation, instead of against a URL. An alternative to specifying an app URL.",
|
|
218
267
|
},
|
|
219
|
-
headless: {
|
|
220
|
-
boolean: true,
|
|
221
|
-
description: "Start browser in headless mode",
|
|
222
|
-
},
|
|
223
|
-
devTools: {
|
|
224
|
-
boolean: true,
|
|
225
|
-
description: "Open Chrome Dev Tools",
|
|
226
|
-
},
|
|
227
|
-
bypassCSP: {
|
|
228
|
-
boolean: true,
|
|
229
|
-
description: "Enables bypass CSP in the browser",
|
|
230
|
-
},
|
|
231
268
|
screenshot: {
|
|
232
269
|
boolean: true,
|
|
233
270
|
description: "Take a screenshot at the end of simulation",
|
|
@@ -242,46 +279,21 @@ exports.replay = {
|
|
|
242
279
|
description: "Base simulation id to diff the final state screenshot against",
|
|
243
280
|
alias: "baseReplayId",
|
|
244
281
|
},
|
|
245
|
-
diffThreshold: {
|
|
246
|
-
number: true,
|
|
247
|
-
description: "Acceptable maximum proportion of changed pixels, between 0 and 1. If this proportion is exceeded then the test will fail.",
|
|
248
|
-
},
|
|
249
|
-
diffPixelThreshold: {
|
|
250
|
-
number: true,
|
|
251
|
-
description: "A number between 0 and 1. Color/brightness differences in individual pixels will be ignored if the difference is less than this threshold. A value of 1.0 would accept any difference in color, while a value of 0.0 would accept no difference in color.",
|
|
252
|
-
},
|
|
253
282
|
save: {
|
|
254
283
|
boolean: true,
|
|
255
284
|
description: "Adds the simulation to the list of test cases in meticulous.json",
|
|
256
285
|
},
|
|
257
|
-
padTime: {
|
|
258
|
-
boolean: true,
|
|
259
|
-
description: "Pad simulation time according to recording duration. Please note this option will be ignored if running with the '--accelerate' option.",
|
|
260
|
-
default: true,
|
|
261
|
-
},
|
|
262
|
-
shiftTime: {
|
|
263
|
-
boolean: true,
|
|
264
|
-
description: "Shift time during simulation to be set as the recording time",
|
|
265
|
-
default: true,
|
|
266
|
-
},
|
|
267
|
-
networkStubbing: {
|
|
268
|
-
boolean: true,
|
|
269
|
-
description: "Stub network requests during simulation",
|
|
270
|
-
default: true,
|
|
271
|
-
},
|
|
272
286
|
moveBeforeClick: {
|
|
273
287
|
boolean: true,
|
|
274
288
|
description: "Simulate mouse movement before clicking",
|
|
289
|
+
default: false,
|
|
275
290
|
},
|
|
276
291
|
cookiesFile: {
|
|
277
292
|
string: true,
|
|
278
293
|
description: "Path to cookies to inject before simulation",
|
|
279
294
|
},
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
description: "Fast forward through any pauses to replay as fast as possible. Warning: this option is experimental and may be deprecated",
|
|
283
|
-
default: false,
|
|
284
|
-
},
|
|
295
|
+
...common_options_1.COMMON_REPLAY_OPTIONS,
|
|
296
|
+
...common_options_1.SCREENSHOT_DIFF_OPTIONS,
|
|
285
297
|
maxDurationMs: {
|
|
286
298
|
number: true,
|
|
287
299
|
description: "Maximum duration (in milliseconds) the simulation will run",
|
|
@@ -290,6 +302,13 @@ exports.replay = {
|
|
|
290
302
|
number: true,
|
|
291
303
|
description: "Maximum number of events the simulation will run",
|
|
292
304
|
},
|
|
305
|
+
storyboard: {
|
|
306
|
+
boolean: true,
|
|
307
|
+
description: "Take a storyboard of screenshots during simulation",
|
|
308
|
+
default: false,
|
|
309
|
+
},
|
|
293
310
|
},
|
|
294
|
-
handler: (0, sentry_utils_1.wrapHandler)(
|
|
311
|
+
handler: (0, sentry_utils_1.wrapHandler)(async (options) => {
|
|
312
|
+
await (0, exports.rawReplayCommandHandler)(options);
|
|
313
|
+
}),
|
|
295
314
|
};
|
|
@@ -1,24 +1,17 @@
|
|
|
1
|
+
import { ReplayExecutionOptions } from "@alwaysmeticulous/common";
|
|
1
2
|
import { CommandModule } from "yargs";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
diffPixelThreshold?: number | null | undefined;
|
|
12
|
-
padTime: boolean;
|
|
13
|
-
shiftTime: boolean;
|
|
14
|
-
networkStubbing: boolean;
|
|
15
|
-
githubSummary?: boolean | null | undefined;
|
|
16
|
-
parallelize?: boolean | null | undefined;
|
|
17
|
-
parallelTasks?: number | null | undefined;
|
|
3
|
+
import { ScreenshotDiffOptions } from "../../command-utils/common-types";
|
|
4
|
+
interface Options extends ScreenshotDiffOptions, ReplayExecutionOptions {
|
|
5
|
+
apiToken?: string;
|
|
6
|
+
commitSha?: string;
|
|
7
|
+
appUrl?: string;
|
|
8
|
+
useAssetsSnapshottedInBaseSimulation: boolean;
|
|
9
|
+
githubSummary?: boolean;
|
|
10
|
+
parallelize?: boolean;
|
|
11
|
+
parallelTasks?: number | null;
|
|
18
12
|
deflake: boolean;
|
|
19
13
|
useCache: boolean;
|
|
20
|
-
testsFile?: string
|
|
21
|
-
accelerate: boolean;
|
|
14
|
+
testsFile?: string;
|
|
22
15
|
}
|
|
23
16
|
export declare const runAllTests: CommandModule<unknown, Options>;
|
|
24
17
|
export {};
|
|
@@ -8,6 +8,7 @@ const common_1 = require("@alwaysmeticulous/common");
|
|
|
8
8
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
9
9
|
const client_1 = require("../../api/client");
|
|
10
10
|
const test_run_api_1 = require("../../api/test-run.api");
|
|
11
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
11
12
|
const config_1 = require("../../config/config");
|
|
12
13
|
const deflake_tests_handler_1 = require("../../deflake-tests/deflake-tests.handler");
|
|
13
14
|
const parallel_tests_handler_1 = require("../../parallel-tests/parallel-tests.handler");
|
|
@@ -18,6 +19,24 @@ const run_all_tests_utils_1 = require("../../utils/run-all-tests.utils");
|
|
|
18
19
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
19
20
|
const version_utils_1 = require("../../utils/version.utils");
|
|
20
21
|
const handler = async ({ apiToken, commitSha: commitSha_, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks, deflake, useCache, testsFile, accelerate, }) => {
|
|
22
|
+
const executionOptions = {
|
|
23
|
+
headless,
|
|
24
|
+
devTools,
|
|
25
|
+
bypassCSP,
|
|
26
|
+
padTime,
|
|
27
|
+
shiftTime,
|
|
28
|
+
networkStubbing,
|
|
29
|
+
accelerate,
|
|
30
|
+
moveBeforeClick: false,
|
|
31
|
+
maxDurationMs: null,
|
|
32
|
+
maxEventCount: null, // we don't expose this option
|
|
33
|
+
};
|
|
34
|
+
const screenshottingOptions = {
|
|
35
|
+
enabled: true,
|
|
36
|
+
screenshotSelector: null,
|
|
37
|
+
diffOptions: { diffPixelThreshold, diffThreshold },
|
|
38
|
+
storyboardOptions: { enabled: false }, // we don't expose this option
|
|
39
|
+
};
|
|
21
40
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
22
41
|
const client = (0, client_1.createClient)({ apiToken });
|
|
23
42
|
const config = await (0, config_1.readConfig)(testsFile || undefined);
|
|
@@ -47,51 +66,33 @@ const handler = async ({ apiToken, commitSha: commitSha_, appUrl, useAssetsSnaps
|
|
|
47
66
|
config,
|
|
48
67
|
client,
|
|
49
68
|
testRun,
|
|
50
|
-
|
|
69
|
+
executionOptions,
|
|
70
|
+
screenshottingOptions,
|
|
71
|
+
apiToken: apiToken !== null && apiToken !== void 0 ? apiToken : null,
|
|
51
72
|
commitSha,
|
|
52
|
-
appUrl,
|
|
73
|
+
appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
|
|
53
74
|
useAssetsSnapshottedInBaseSimulation,
|
|
54
|
-
|
|
55
|
-
devTools,
|
|
56
|
-
bypassCSP,
|
|
57
|
-
diffThreshold,
|
|
58
|
-
diffPixelThreshold,
|
|
59
|
-
padTime,
|
|
60
|
-
shiftTime,
|
|
61
|
-
networkStubbing,
|
|
62
|
-
parallelTasks,
|
|
75
|
+
parallelTasks: parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : null,
|
|
63
76
|
deflake,
|
|
64
77
|
cachedTestRunResults,
|
|
65
|
-
accelerate,
|
|
66
78
|
});
|
|
67
79
|
return results;
|
|
68
80
|
}
|
|
69
81
|
const results = [...cachedTestRunResults];
|
|
70
82
|
const testsToRun = (0, run_all_tests_utils_1.getTestsToRun)({ testCases, cachedTestRunResults });
|
|
71
83
|
for (const testCase of testsToRun) {
|
|
72
|
-
const { sessionId, baseReplayId, options } = testCase;
|
|
73
84
|
const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)({
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
85
|
+
replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
|
|
86
|
+
useAssetsSnapshottedInBaseSimulation,
|
|
87
|
+
appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
|
|
88
|
+
testCase,
|
|
89
|
+
}),
|
|
90
|
+
executionOptions,
|
|
91
|
+
screenshottingOptions,
|
|
92
|
+
testCase,
|
|
93
|
+
apiToken: apiToken !== null && apiToken !== void 0 ? apiToken : null,
|
|
77
94
|
commitSha,
|
|
78
|
-
|
|
79
|
-
appUrl,
|
|
80
|
-
headless,
|
|
81
|
-
devTools,
|
|
82
|
-
bypassCSP,
|
|
83
|
-
screenshot: true,
|
|
84
|
-
baseSimulationId: baseReplayId,
|
|
85
|
-
diffThreshold,
|
|
86
|
-
diffPixelThreshold,
|
|
87
|
-
save: false,
|
|
88
|
-
exitOnMismatch: false,
|
|
89
|
-
padTime,
|
|
90
|
-
shiftTime,
|
|
91
|
-
networkStubbing,
|
|
92
|
-
simulationIdForAssets: (0, config_utils_1.getSimulationIdForAssets)(testCase, useAssetsSnapshottedInBaseSimulation),
|
|
93
|
-
accelerate,
|
|
94
|
-
...options,
|
|
95
|
+
deflake: deflake !== null && deflake !== void 0 ? deflake : false,
|
|
95
96
|
});
|
|
96
97
|
results.push(result);
|
|
97
98
|
await (0, test_run_api_1.putTestRunResults)({
|
|
@@ -130,14 +131,11 @@ exports.runAllTests = {
|
|
|
130
131
|
command: "run-all-tests",
|
|
131
132
|
describe: "Run all replay test cases",
|
|
132
133
|
builder: {
|
|
133
|
-
apiToken:
|
|
134
|
-
|
|
135
|
-
},
|
|
136
|
-
commitSha: {
|
|
137
|
-
string: true,
|
|
138
|
-
},
|
|
134
|
+
apiToken: common_options_1.OPTIONS.apiToken,
|
|
135
|
+
commitSha: common_options_1.OPTIONS.commitSha,
|
|
139
136
|
appUrl: {
|
|
140
137
|
string: true,
|
|
138
|
+
description: "The URL to execute the tests against. If left absent here and in the test cases file, then will use the URL the test was originally recorded against.",
|
|
141
139
|
},
|
|
142
140
|
useAssetsSnapshottedInBaseSimulation: {
|
|
143
141
|
boolean: true,
|
|
@@ -145,46 +143,12 @@ exports.runAllTests = {
|
|
|
145
143
|
" from the base simulation/replay the test is comparing against. The sessions will then be replayed against those local urls." +
|
|
146
144
|
" This is an alternative to specifying an appUrl.",
|
|
147
145
|
conflicts: "appUrl",
|
|
148
|
-
|
|
149
|
-
headless: {
|
|
150
|
-
boolean: true,
|
|
151
|
-
description: "Start browser in headless mode",
|
|
152
|
-
},
|
|
153
|
-
devTools: {
|
|
154
|
-
boolean: true,
|
|
155
|
-
description: "Open Chrome Dev Tools",
|
|
156
|
-
},
|
|
157
|
-
bypassCSP: {
|
|
158
|
-
boolean: true,
|
|
159
|
-
description: "Enables bypass CSP in the browser",
|
|
160
|
-
},
|
|
161
|
-
diffThreshold: {
|
|
162
|
-
number: true,
|
|
163
|
-
description: "Acceptable maximum proportion of changed pixels, between 0 and 1. If this proportion is exceeded then the test will fail.",
|
|
164
|
-
},
|
|
165
|
-
diffPixelThreshold: {
|
|
166
|
-
number: true,
|
|
167
|
-
description: "A number between 0 and 1. Color/brightness differences in individual pixels will be ignored if the difference is less than this threshold. A value of 1.0 would accept any difference in color, while a value of 0.0 would accept no difference in color.",
|
|
146
|
+
default: false,
|
|
168
147
|
},
|
|
169
148
|
githubSummary: {
|
|
170
149
|
boolean: true,
|
|
171
150
|
description: "Outputs a summary page for GitHub actions",
|
|
172
151
|
},
|
|
173
|
-
padTime: {
|
|
174
|
-
boolean: true,
|
|
175
|
-
description: "Pad replay time according to recording duration. Please note this option will be ignored if running with the '--accelerate' option.",
|
|
176
|
-
default: true,
|
|
177
|
-
},
|
|
178
|
-
shiftTime: {
|
|
179
|
-
boolean: true,
|
|
180
|
-
description: "Shift time during simulation to be set as the recording time",
|
|
181
|
-
default: true,
|
|
182
|
-
},
|
|
183
|
-
networkStubbing: {
|
|
184
|
-
boolean: true,
|
|
185
|
-
description: "Stub network requests during replay",
|
|
186
|
-
default: true,
|
|
187
|
-
},
|
|
188
152
|
parallelize: {
|
|
189
153
|
boolean: true,
|
|
190
154
|
description: "Run tests in parallel",
|
|
@@ -214,11 +178,8 @@ exports.runAllTests = {
|
|
|
214
178
|
description: "The path to the meticulous.json file containing the list of tests you want to run." +
|
|
215
179
|
" If not set a search will be performed to find a meticulous.json file in the current directory or the nearest parent directory.",
|
|
216
180
|
},
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
description: "Fast forward through any pauses to replay as fast as possible. Warning: this option is experimental and may be deprecated",
|
|
220
|
-
default: false,
|
|
221
|
-
},
|
|
181
|
+
...common_options_1.COMMON_REPLAY_OPTIONS,
|
|
182
|
+
...common_options_1.SCREENSHOT_DIFF_OPTIONS,
|
|
222
183
|
},
|
|
223
184
|
handler: (0, sentry_utils_1.wrapHandler)(handler),
|
|
224
185
|
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AxiosInstance } from "axios";
|
|
2
2
|
import { CommandModule } from "yargs";
|
|
3
|
+
import { ScreenshotDiffOptions } from "../../command-utils/common-types";
|
|
3
4
|
import { CompareImageResult } from "../../image/diff.utils";
|
|
4
5
|
export declare class DiffError extends Error {
|
|
5
6
|
readonly extras?: {
|
|
@@ -28,16 +29,15 @@ export declare const diffScreenshots: (options: {
|
|
|
28
29
|
headReplayId: string;
|
|
29
30
|
baseScreenshotsDir: string;
|
|
30
31
|
headScreenshotsDir: string;
|
|
31
|
-
|
|
32
|
-
pixelThreshold: number | null | undefined;
|
|
32
|
+
diffOptions: ScreenshotDiffOptions;
|
|
33
33
|
exitOnMismatch: boolean;
|
|
34
34
|
}) => Promise<ScreenshotDiffResult[]>;
|
|
35
35
|
interface Options {
|
|
36
36
|
apiToken?: string | null | undefined;
|
|
37
37
|
baseSimulationId: string;
|
|
38
38
|
headSimulationId: string;
|
|
39
|
-
threshold
|
|
40
|
-
pixelThreshold
|
|
39
|
+
threshold: number;
|
|
40
|
+
pixelThreshold: number;
|
|
41
41
|
}
|
|
42
42
|
export declare const screenshotDiff: CommandModule<unknown, Options>;
|
|
43
43
|
export {};
|
|
@@ -9,12 +9,12 @@ const loglevel_1 = __importDefault(require("loglevel"));
|
|
|
9
9
|
const path_1 = require("path");
|
|
10
10
|
const client_1 = require("../../api/client");
|
|
11
11
|
const replay_api_1 = require("../../api/replay.api");
|
|
12
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
12
13
|
const diff_utils_1 = require("../../image/diff.utils");
|
|
13
14
|
const io_utils_1 = require("../../image/io.utils");
|
|
14
15
|
const replays_1 = require("../../local-data/replays");
|
|
15
16
|
const screenshot_diffs_1 = require("../../local-data/screenshot-diffs");
|
|
16
17
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
17
|
-
const DEFAULT_MISMATCH_THRESHOLD = 0.01;
|
|
18
18
|
class DiffError extends Error {
|
|
19
19
|
constructor(message, extras) {
|
|
20
20
|
super(message);
|
|
@@ -22,16 +22,15 @@ class DiffError extends Error {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
exports.DiffError = DiffError;
|
|
25
|
-
const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreenshotsDir, headScreenshotsDir,
|
|
25
|
+
const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreenshotsDir, headScreenshotsDir, diffOptions, exitOnMismatch, }) => {
|
|
26
|
+
const { diffThreshold, diffPixelThreshold } = diffOptions;
|
|
26
27
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
27
|
-
const threshold = threshold_ !== null && threshold_ !== void 0 ? threshold_ : DEFAULT_MISMATCH_THRESHOLD;
|
|
28
28
|
const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
|
|
29
29
|
const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
|
|
30
30
|
// Assume base replay screenshots are always a subset of the head replay screenshots.
|
|
31
31
|
// We report any missing base replay screenshots for visibility but don't count it as a difference.
|
|
32
32
|
const missingHeadImages = new Set([...baseReplayScreenshots].filter((file) => !headReplayScreenshots.includes(file)));
|
|
33
33
|
let totalMismatchPixels = 0;
|
|
34
|
-
let totalComparedPixels = 0;
|
|
35
34
|
const comparisonResults = [];
|
|
36
35
|
try {
|
|
37
36
|
if (missingHeadImages.size > 0) {
|
|
@@ -39,7 +38,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
39
38
|
throw new DiffError(`Head replay is missing screenshots: ${[...missingHeadImages].sort()}`, {
|
|
40
39
|
baseReplayId,
|
|
41
40
|
headReplayId,
|
|
42
|
-
threshold,
|
|
41
|
+
threshold: diffThreshold,
|
|
43
42
|
});
|
|
44
43
|
}
|
|
45
44
|
for (const screenshotFileName of headReplayScreenshots) {
|
|
@@ -54,10 +53,9 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
54
53
|
const comparisonResult = (0, diff_utils_1.compareImages)({
|
|
55
54
|
base: baseScreenshot,
|
|
56
55
|
head: headScreenshot,
|
|
57
|
-
pixelThreshold:
|
|
56
|
+
pixelThreshold: diffPixelThreshold,
|
|
58
57
|
});
|
|
59
58
|
totalMismatchPixels += comparisonResult.mismatchPixels;
|
|
60
|
-
totalComparedPixels += baseScreenshot.width * baseScreenshot.height;
|
|
61
59
|
logger.debug({
|
|
62
60
|
screenshotFileName,
|
|
63
61
|
mismatchPixels: comparisonResult.mismatchPixels,
|
|
@@ -73,7 +71,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
73
71
|
baseScreenshotFile,
|
|
74
72
|
headScreenshotFile,
|
|
75
73
|
comparisonResult,
|
|
76
|
-
outcome: comparisonResult.mismatchFraction >
|
|
74
|
+
outcome: comparisonResult.mismatchFraction > diffThreshold ? "fail" : "pass",
|
|
77
75
|
});
|
|
78
76
|
}
|
|
79
77
|
await (0, replay_api_1.postScreenshotDiffStats)(client, {
|
|
@@ -88,7 +86,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
88
86
|
const diffUrl = await (0, replay_api_1.getDiffUrl)(client, baseReplayId, headReplayId);
|
|
89
87
|
logger.info(`View screenshot diff at ${diffUrl}`);
|
|
90
88
|
comparisonResults.forEach((result) => {
|
|
91
|
-
logComparisonResultMessage(logger, `${Math.round(result.comparisonResult.mismatchFraction * 100)}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${Math.round(
|
|
89
|
+
logComparisonResultMessage(logger, `${Math.round(result.comparisonResult.mismatchFraction * 100)}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${Math.round(diffThreshold * 100)}%)`, result.outcome);
|
|
92
90
|
});
|
|
93
91
|
// Check if individual screenshot mismatch is higher than the threshold.
|
|
94
92
|
const mismatchingScreenshots = comparisonResults.filter((result) => result.outcome == "fail");
|
|
@@ -102,7 +100,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
102
100
|
throw new DiffError(`Screenshots ${mismatchingScreenshots.map((result) => (0, path_1.basename)(result.headScreenshotFile))} do not match!`, {
|
|
103
101
|
baseReplayId,
|
|
104
102
|
headReplayId,
|
|
105
|
-
threshold,
|
|
103
|
+
threshold: diffThreshold,
|
|
106
104
|
});
|
|
107
105
|
}
|
|
108
106
|
}
|
|
@@ -116,7 +114,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
116
114
|
throw new DiffError(`Error while diffing: ${error}`, {
|
|
117
115
|
baseReplayId,
|
|
118
116
|
headReplayId,
|
|
119
|
-
threshold,
|
|
117
|
+
threshold: diffThreshold,
|
|
120
118
|
value: 1,
|
|
121
119
|
});
|
|
122
120
|
}
|
|
@@ -140,8 +138,10 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
|
|
|
140
138
|
headReplayId,
|
|
141
139
|
baseScreenshotsDir,
|
|
142
140
|
headScreenshotsDir,
|
|
143
|
-
|
|
144
|
-
|
|
141
|
+
diffOptions: {
|
|
142
|
+
diffThreshold: threshold,
|
|
143
|
+
diffPixelThreshold: pixelThreshold,
|
|
144
|
+
},
|
|
145
145
|
exitOnMismatch: true,
|
|
146
146
|
});
|
|
147
147
|
};
|
|
@@ -162,12 +162,8 @@ exports.screenshotDiff = {
|
|
|
162
162
|
demandOption: true,
|
|
163
163
|
alias: "headReplayId",
|
|
164
164
|
},
|
|
165
|
-
threshold:
|
|
166
|
-
|
|
167
|
-
},
|
|
168
|
-
pixelThreshold: {
|
|
169
|
-
number: true,
|
|
170
|
-
},
|
|
165
|
+
threshold: common_options_1.SCREENSHOT_DIFF_OPTIONS.diffThreshold,
|
|
166
|
+
pixelThreshold: common_options_1.SCREENSHOT_DIFF_OPTIONS.diffPixelThreshold,
|
|
171
167
|
},
|
|
172
168
|
handler: (0, sentry_utils_1.wrapHandler)(handler),
|
|
173
169
|
};
|
package/dist/config/config.js
CHANGED
|
@@ -22,14 +22,13 @@ const getConfigFilePath = async () => {
|
|
|
22
22
|
return configFilePath;
|
|
23
23
|
};
|
|
24
24
|
const validateReplayOptions = (prevOptions) => {
|
|
25
|
-
const { screenshotSelector, diffThreshold, diffPixelThreshold,
|
|
25
|
+
const { screenshotSelector, diffThreshold, diffPixelThreshold, moveBeforeClick, simulationIdForAssets, } = prevOptions;
|
|
26
26
|
return {
|
|
27
27
|
...(screenshotSelector ? { screenshotSelector } : {}),
|
|
28
28
|
...(diffThreshold ? { diffThreshold } : {}),
|
|
29
29
|
...(diffPixelThreshold ? { diffPixelThreshold } : {}),
|
|
30
|
-
...(cookies ? { cookies } : {}),
|
|
31
30
|
...(moveBeforeClick ? { moveBeforeClick } : {}),
|
|
32
|
-
...(
|
|
31
|
+
...(simulationIdForAssets ? { simulationIdForAssets } : {}),
|
|
33
32
|
};
|
|
34
33
|
};
|
|
35
34
|
const validateConfig = (prevConfig) => {
|
|
@@ -54,7 +53,7 @@ const readConfig = async (configFilePath) => {
|
|
|
54
53
|
const filePath = configFilePath !== null && configFilePath !== void 0 ? configFilePath : (await getConfigFilePath());
|
|
55
54
|
const configStr = await (0, promises_1.readFile)(filePath, "utf-8").catch((error) => {
|
|
56
55
|
// Use an empty config object if there is no config file
|
|
57
|
-
if (configFilePath
|
|
56
|
+
if (configFilePath == null &&
|
|
58
57
|
error instanceof Error &&
|
|
59
58
|
error.code === "ENOENT") {
|
|
60
59
|
return "{}";
|