@alwaysmeticulous/cli 2.3.4 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/command-utils/command-utils.d.ts +9 -0
- package/dist/command-utils/command-utils.js +18 -0
- package/dist/command-utils/common-options.d.ts +110 -0
- package/dist/command-utils/common-options.js +87 -0
- package/dist/command-utils/common-types.d.ts +20 -0
- package/dist/command-utils/common-types.js +2 -0
- package/dist/commands/create-test/create-test.command.d.ts +2 -2
- package/dist/commands/create-test/create-test.command.js +23 -40
- package/dist/commands/record/record.command.d.ts +9 -9
- package/dist/commands/record/record.command.js +2 -1
- package/dist/commands/replay/replay.command.d.ts +24 -26
- package/dist/commands/replay/replay.command.js +84 -76
- package/dist/commands/run-all-tests/run-all-tests.command.d.ts +11 -18
- package/dist/commands/run-all-tests/run-all-tests.command.js +38 -77
- package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts +4 -4
- package/dist/commands/screenshot-diff/screenshot-diff.command.js +15 -19
- package/dist/config/config.js +2 -3
- package/dist/config/config.types.d.ts +7 -8
- package/dist/deflake-tests/deflake-tests.handler.d.ts +12 -3
- package/dist/deflake-tests/deflake-tests.handler.js +41 -6
- package/dist/image/diff.utils.d.ts +8 -1
- package/dist/image/diff.utils.js +1 -5
- package/dist/parallel-tests/messages.types.d.ts +3 -18
- package/dist/parallel-tests/parallel-tests.handler.d.ts +9 -14
- package/dist/parallel-tests/parallel-tests.handler.js +11 -15
- package/dist/parallel-tests/task.handler.js +2 -26
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/utils/config.utils.d.ts +6 -1
- package/dist/utils/config.utils.js +17 -8
- package/package.json +6 -6
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.replay = exports.replayCommandHandler = void 0;
|
|
6
|
+
exports.replay = exports.rawReplayCommandHandler = exports.replayCommandHandler = void 0;
|
|
7
7
|
const common_1 = require("@alwaysmeticulous/common");
|
|
8
8
|
const promises_1 = require("fs/promises");
|
|
9
9
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
@@ -13,17 +13,19 @@ const client_1 = require("../../api/client");
|
|
|
13
13
|
const replay_api_1 = require("../../api/replay.api");
|
|
14
14
|
const upload_1 = require("../../api/upload");
|
|
15
15
|
const archive_1 = require("../../archive/archive");
|
|
16
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
16
17
|
const local_data_utils_1 = require("../../local-data/local-data.utils");
|
|
17
18
|
const replay_assets_1 = require("../../local-data/replay-assets");
|
|
18
19
|
const replays_1 = require("../../local-data/replays");
|
|
20
|
+
const serve_assets_from_simulation_1 = require("../../local-data/serve-assets-from-simulation");
|
|
19
21
|
const sessions_1 = require("../../local-data/sessions");
|
|
20
22
|
const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
|
|
23
|
+
const config_utils_1 = require("../../utils/config.utils");
|
|
21
24
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
22
25
|
const version_utils_1 = require("../../utils/version.utils");
|
|
23
26
|
const screenshot_diff_command_1 = require("../screenshot-diff/screenshot-diff.command");
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId: baseReplayId_, diffThreshold, diffPixelThreshold, save, exitOnMismatch, padTime, shiftTime, networkStubbing, moveBeforeClick, cookies, cookiesFile, accelerate, maxDurationMs, maxEventCount, }) => {
|
|
27
|
+
const command_utils_1 = require("../../command-utils/command-utils");
|
|
28
|
+
const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, }) => {
|
|
27
29
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
28
30
|
const client = (0, client_1.createClient)({ apiToken });
|
|
29
31
|
// 1. Check session files
|
|
@@ -34,9 +36,7 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
34
36
|
logger.debug(`Commit: ${commitSha}`);
|
|
35
37
|
const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
|
|
36
38
|
// 3. If simulationIdForAssets specified then download assets & spin up local server
|
|
37
|
-
const
|
|
38
|
-
? await (0, serve_assets_from_simulation_1.serveAssetsFromSimulation)(client, simulationIdForAssets)
|
|
39
|
-
: undefined;
|
|
39
|
+
const { appUrl, closeServer } = await serveOrGetAppUrl(client, replayTarget);
|
|
40
40
|
// 4. Load replay assets
|
|
41
41
|
const browserUserInteractions = await (0, replay_assets_1.fetchAsset)("replay/v2/snippet-user-interactions.bundle.js");
|
|
42
42
|
const browserPlayback = await (0, replay_assets_1.fetchAsset)("replay/v2/snippet-playback.bundle.js");
|
|
@@ -65,16 +65,14 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
65
65
|
const tempDir = await (0, promises_1.mkdtemp)((0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "replays", tempDirName));
|
|
66
66
|
// 6. Create and save replay parameters
|
|
67
67
|
const replayEventsParams = {
|
|
68
|
-
appUrl
|
|
68
|
+
appUrl,
|
|
69
|
+
replayExecutionOptions: executionOptions,
|
|
69
70
|
browser: null,
|
|
70
71
|
outputDir: tempDir,
|
|
71
72
|
session,
|
|
72
73
|
sessionData,
|
|
73
74
|
recordingId: "manual-replay",
|
|
74
75
|
meticulousSha: "meticulousSha",
|
|
75
|
-
headless: headless || false,
|
|
76
|
-
devTools: devTools || false,
|
|
77
|
-
bypassCSP: bypassCSP || false,
|
|
78
76
|
dependencies: {
|
|
79
77
|
browserUserInteractions: {
|
|
80
78
|
key: "browserUserInteractions",
|
|
@@ -101,23 +99,14 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
101
99
|
location: nodeUserInteractions,
|
|
102
100
|
},
|
|
103
101
|
},
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
screenshot: screenshot,
|
|
107
|
-
screenshotSelector: screenshotSelector || "",
|
|
108
|
-
networkStubbing,
|
|
109
|
-
moveBeforeClick: moveBeforeClick || false,
|
|
110
|
-
cookies: cookies || null,
|
|
111
|
-
cookiesFile: cookiesFile || "",
|
|
112
|
-
accelerate,
|
|
113
|
-
...(maxDurationMs != null ? { maxDurationMs } : {}),
|
|
114
|
-
...(maxEventCount != null ? { maxEventCount } : {}),
|
|
102
|
+
screenshottingOptions,
|
|
103
|
+
cookiesFile: cookiesFile,
|
|
115
104
|
};
|
|
116
105
|
await (0, promises_1.writeFile)((0, path_1.join)(tempDir, "replayEventsParams.json"), JSON.stringify(replayEventsParams));
|
|
117
106
|
// 7. Perform replay
|
|
118
107
|
const startTime = luxon_1.DateTime.utc();
|
|
119
108
|
await replayEvents(replayEventsParams);
|
|
120
|
-
|
|
109
|
+
closeServer === null || closeServer === void 0 ? void 0 : closeServer();
|
|
121
110
|
const endTime = luxon_1.DateTime.utc();
|
|
122
111
|
logger.info(`Simulation time: ${endTime.diff(startTime).as("seconds")} seconds`);
|
|
123
112
|
logger.info("Sending simulation results to Meticulous");
|
|
@@ -157,7 +146,7 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
157
146
|
logger.info("=======");
|
|
158
147
|
// 12. Diff against base replay screenshot if one is provided
|
|
159
148
|
const baseReplayId = baseReplayId_ || "";
|
|
160
|
-
if (
|
|
149
|
+
if (screenshottingOptions.enabled && baseReplayId) {
|
|
161
150
|
logger.info(`Diffing screenshots against replay ${baseReplayId}`);
|
|
162
151
|
await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
|
|
163
152
|
await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
|
|
@@ -169,14 +158,13 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
169
158
|
headReplayId: replay.id,
|
|
170
159
|
baseScreenshotsDir: baseReplayScreenshotsDir,
|
|
171
160
|
headScreenshotsDir: headReplayScreenshotsDir,
|
|
172
|
-
|
|
173
|
-
pixelThreshold: diffPixelThreshold,
|
|
161
|
+
diffOptions: screenshottingOptions.diffOptions,
|
|
174
162
|
exitOnMismatch: !!exitOnMismatch,
|
|
175
163
|
});
|
|
176
164
|
}
|
|
177
165
|
// 13. Add test case to meticulous.json if --save option is passed
|
|
178
166
|
if (save) {
|
|
179
|
-
if (!
|
|
167
|
+
if (!screenshottingOptions.enabled) {
|
|
180
168
|
logger.error("Warning: saving a new test case without screenshot enabled.");
|
|
181
169
|
}
|
|
182
170
|
await (0, config_utils_1.addTestCase)({
|
|
@@ -189,20 +177,75 @@ const replayCommandHandler = async ({ apiToken, commitSha: commitSha_, sessionId
|
|
|
189
177
|
return replay;
|
|
190
178
|
};
|
|
191
179
|
exports.replayCommandHandler = replayCommandHandler;
|
|
192
|
-
const
|
|
193
|
-
|
|
180
|
+
const serveOrGetAppUrl = async (client, replayTarget) => {
|
|
181
|
+
if (replayTarget.type === "snapshotted-assets") {
|
|
182
|
+
const server = await (0, serve_assets_from_simulation_1.serveAssetsFromSimulation)(client, replayTarget.simulationIdForAssets);
|
|
183
|
+
return {
|
|
184
|
+
appUrl: server.url,
|
|
185
|
+
closeServer: server.closeServer,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
if (replayTarget.type === "url") {
|
|
189
|
+
return { appUrl: replayTarget.appUrl };
|
|
190
|
+
}
|
|
191
|
+
if (replayTarget.type === "original-recorded-url") {
|
|
192
|
+
return {};
|
|
193
|
+
}
|
|
194
|
+
return unknownReplayTargetType(replayTarget);
|
|
195
|
+
};
|
|
196
|
+
const unknownReplayTargetType = (replayTarget) => {
|
|
197
|
+
throw new Error(`Unknown type of replay target: ${JSON.stringify(replayTarget)}`);
|
|
198
|
+
};
|
|
199
|
+
const rawReplayCommandHandler = ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId, diffThreshold, diffPixelThreshold, save, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, accelerate, maxDurationMs, maxEventCount, }) => {
|
|
200
|
+
const executionOptions = {
|
|
201
|
+
headless,
|
|
202
|
+
devTools,
|
|
203
|
+
bypassCSP,
|
|
204
|
+
padTime,
|
|
205
|
+
shiftTime,
|
|
206
|
+
networkStubbing,
|
|
207
|
+
accelerate,
|
|
208
|
+
moveBeforeClick,
|
|
209
|
+
maxDurationMs: maxDurationMs !== null && maxDurationMs !== void 0 ? maxDurationMs : undefined,
|
|
210
|
+
maxEventCount: maxEventCount !== null && maxEventCount !== void 0 ? maxEventCount : undefined,
|
|
211
|
+
};
|
|
212
|
+
const screenshottingOptions = screenshot
|
|
213
|
+
? {
|
|
214
|
+
enabled: true,
|
|
215
|
+
screenshotSelector: screenshotSelector !== null && screenshotSelector !== void 0 ? screenshotSelector : undefined,
|
|
216
|
+
diffOptions: { diffPixelThreshold, diffThreshold },
|
|
217
|
+
}
|
|
218
|
+
: { enabled: false };
|
|
219
|
+
return (0, exports.replayCommandHandler)({
|
|
220
|
+
replayTarget: getReplayTarget({ appUrl, simulationIdForAssets }),
|
|
221
|
+
executionOptions,
|
|
222
|
+
screenshottingOptions,
|
|
223
|
+
apiToken,
|
|
224
|
+
commitSha,
|
|
225
|
+
cookiesFile,
|
|
226
|
+
sessionId,
|
|
227
|
+
baseSimulationId,
|
|
228
|
+
save,
|
|
229
|
+
exitOnMismatch: true,
|
|
230
|
+
});
|
|
231
|
+
};
|
|
232
|
+
exports.rawReplayCommandHandler = rawReplayCommandHandler;
|
|
233
|
+
const getReplayTarget = ({ appUrl, simulationIdForAssets, }) => {
|
|
234
|
+
if (simulationIdForAssets) {
|
|
235
|
+
return { type: "snapshotted-assets", simulationIdForAssets };
|
|
236
|
+
}
|
|
237
|
+
if (appUrl) {
|
|
238
|
+
return { type: "url", appUrl };
|
|
239
|
+
}
|
|
240
|
+
return { type: "original-recorded-url" };
|
|
194
241
|
};
|
|
195
242
|
exports.replay = {
|
|
196
243
|
command: "simulate",
|
|
197
244
|
aliases: ["replay"],
|
|
198
245
|
describe: "Simulate (replay) a recorded session",
|
|
199
246
|
builder: {
|
|
200
|
-
apiToken:
|
|
201
|
-
|
|
202
|
-
},
|
|
203
|
-
commitSha: {
|
|
204
|
-
string: true,
|
|
205
|
-
},
|
|
247
|
+
apiToken: common_options_1.OPTIONS.apiToken,
|
|
248
|
+
commitSha: common_options_1.OPTIONS.commitSha,
|
|
206
249
|
sessionId: {
|
|
207
250
|
string: true,
|
|
208
251
|
demandOption: true,
|
|
@@ -216,18 +259,6 @@ exports.replay = {
|
|
|
216
259
|
conflicts: "appUrl",
|
|
217
260
|
description: "If present will run the session against a local server serving up previously snapshotted assets (HTML, JS, CSS etc.) from the specified prior simulation, instead of against a URL. An alternative to specifying an app URL.",
|
|
218
261
|
},
|
|
219
|
-
headless: {
|
|
220
|
-
boolean: true,
|
|
221
|
-
description: "Start browser in headless mode",
|
|
222
|
-
},
|
|
223
|
-
devTools: {
|
|
224
|
-
boolean: true,
|
|
225
|
-
description: "Open Chrome Dev Tools",
|
|
226
|
-
},
|
|
227
|
-
bypassCSP: {
|
|
228
|
-
boolean: true,
|
|
229
|
-
description: "Enables bypass CSP in the browser",
|
|
230
|
-
},
|
|
231
262
|
screenshot: {
|
|
232
263
|
boolean: true,
|
|
233
264
|
description: "Take a screenshot at the end of simulation",
|
|
@@ -242,46 +273,21 @@ exports.replay = {
|
|
|
242
273
|
description: "Base simulation id to diff the final state screenshot against",
|
|
243
274
|
alias: "baseReplayId",
|
|
244
275
|
},
|
|
245
|
-
diffThreshold: {
|
|
246
|
-
number: true,
|
|
247
|
-
description: "Acceptable maximum proportion of changed pixels, between 0 and 1. If this proportion is exceeded then the test will fail.",
|
|
248
|
-
},
|
|
249
|
-
diffPixelThreshold: {
|
|
250
|
-
number: true,
|
|
251
|
-
description: "A number between 0 and 1. Color/brightness differences in individual pixels will be ignored if the difference is less than this threshold. A value of 1.0 would accept any difference in color, while a value of 0.0 would accept no difference in color.",
|
|
252
|
-
},
|
|
253
276
|
save: {
|
|
254
277
|
boolean: true,
|
|
255
278
|
description: "Adds the simulation to the list of test cases in meticulous.json",
|
|
256
279
|
},
|
|
257
|
-
padTime: {
|
|
258
|
-
boolean: true,
|
|
259
|
-
description: "Pad simulation time according to recording duration. Please note this option will be ignored if running with the '--accelerate' option.",
|
|
260
|
-
default: true,
|
|
261
|
-
},
|
|
262
|
-
shiftTime: {
|
|
263
|
-
boolean: true,
|
|
264
|
-
description: "Shift time during simulation to be set as the recording time",
|
|
265
|
-
default: true,
|
|
266
|
-
},
|
|
267
|
-
networkStubbing: {
|
|
268
|
-
boolean: true,
|
|
269
|
-
description: "Stub network requests during simulation",
|
|
270
|
-
default: true,
|
|
271
|
-
},
|
|
272
280
|
moveBeforeClick: {
|
|
273
281
|
boolean: true,
|
|
274
282
|
description: "Simulate mouse movement before clicking",
|
|
283
|
+
default: false,
|
|
275
284
|
},
|
|
276
285
|
cookiesFile: {
|
|
277
286
|
string: true,
|
|
278
287
|
description: "Path to cookies to inject before simulation",
|
|
279
288
|
},
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
description: "Fast forward through any pauses to replay as fast as possible. Warning: this option is experimental and may be deprecated",
|
|
283
|
-
default: false,
|
|
284
|
-
},
|
|
289
|
+
...common_options_1.COMMON_REPLAY_OPTIONS,
|
|
290
|
+
...common_options_1.SCREENSHOT_DIFF_OPTIONS,
|
|
285
291
|
maxDurationMs: {
|
|
286
292
|
number: true,
|
|
287
293
|
description: "Maximum duration (in milliseconds) the simulation will run",
|
|
@@ -291,5 +297,7 @@ exports.replay = {
|
|
|
291
297
|
description: "Maximum number of events the simulation will run",
|
|
292
298
|
},
|
|
293
299
|
},
|
|
294
|
-
handler: (0, sentry_utils_1.wrapHandler)(
|
|
300
|
+
handler: (0, sentry_utils_1.wrapHandler)((0, command_utils_1.handleNulls)(async (options) => {
|
|
301
|
+
await (0, exports.rawReplayCommandHandler)(options);
|
|
302
|
+
})),
|
|
295
303
|
};
|
|
@@ -1,24 +1,17 @@
|
|
|
1
|
+
import { ReplayExecutionOptions } from "@alwaysmeticulous/common";
|
|
1
2
|
import { CommandModule } from "yargs";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
diffPixelThreshold?: number | null | undefined;
|
|
12
|
-
padTime: boolean;
|
|
13
|
-
shiftTime: boolean;
|
|
14
|
-
networkStubbing: boolean;
|
|
15
|
-
githubSummary?: boolean | null | undefined;
|
|
16
|
-
parallelize?: boolean | null | undefined;
|
|
17
|
-
parallelTasks?: number | null | undefined;
|
|
3
|
+
import { ScreenshotDiffOptions } from "../../command-utils/common-types";
|
|
4
|
+
interface Options extends ScreenshotDiffOptions, ReplayExecutionOptions {
|
|
5
|
+
apiToken?: string;
|
|
6
|
+
commitSha?: string;
|
|
7
|
+
appUrl?: string;
|
|
8
|
+
useAssetsSnapshottedInBaseSimulation: boolean;
|
|
9
|
+
githubSummary?: boolean;
|
|
10
|
+
parallelize?: boolean;
|
|
11
|
+
parallelTasks?: number | null;
|
|
18
12
|
deflake: boolean;
|
|
19
13
|
useCache: boolean;
|
|
20
|
-
testsFile?: string
|
|
21
|
-
accelerate: boolean;
|
|
14
|
+
testsFile?: string;
|
|
22
15
|
}
|
|
23
16
|
export declare const runAllTests: CommandModule<unknown, Options>;
|
|
24
17
|
export {};
|
|
@@ -8,6 +8,7 @@ const common_1 = require("@alwaysmeticulous/common");
|
|
|
8
8
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
9
9
|
const client_1 = require("../../api/client");
|
|
10
10
|
const test_run_api_1 = require("../../api/test-run.api");
|
|
11
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
11
12
|
const config_1 = require("../../config/config");
|
|
12
13
|
const deflake_tests_handler_1 = require("../../deflake-tests/deflake-tests.handler");
|
|
13
14
|
const parallel_tests_handler_1 = require("../../parallel-tests/parallel-tests.handler");
|
|
@@ -17,7 +18,25 @@ const github_summary_utils_1 = require("../../utils/github-summary.utils");
|
|
|
17
18
|
const run_all_tests_utils_1 = require("../../utils/run-all-tests.utils");
|
|
18
19
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
19
20
|
const version_utils_1 = require("../../utils/version.utils");
|
|
21
|
+
const command_utils_1 = require("../../command-utils/command-utils");
|
|
20
22
|
const handler = async ({ apiToken, commitSha: commitSha_, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks, deflake, useCache, testsFile, accelerate, }) => {
|
|
23
|
+
const executionOptions = {
|
|
24
|
+
headless,
|
|
25
|
+
devTools,
|
|
26
|
+
bypassCSP,
|
|
27
|
+
padTime,
|
|
28
|
+
shiftTime,
|
|
29
|
+
networkStubbing,
|
|
30
|
+
accelerate,
|
|
31
|
+
moveBeforeClick: false,
|
|
32
|
+
maxDurationMs: undefined,
|
|
33
|
+
maxEventCount: undefined, // we don't expose this option
|
|
34
|
+
};
|
|
35
|
+
const screenshottingOptions = {
|
|
36
|
+
enabled: true,
|
|
37
|
+
screenshotSelector: undefined,
|
|
38
|
+
diffOptions: { diffPixelThreshold, diffThreshold },
|
|
39
|
+
};
|
|
21
40
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
22
41
|
const client = (0, client_1.createClient)({ apiToken });
|
|
23
42
|
const config = await (0, config_1.readConfig)(testsFile || undefined);
|
|
@@ -47,51 +66,33 @@ const handler = async ({ apiToken, commitSha: commitSha_, appUrl, useAssetsSnaps
|
|
|
47
66
|
config,
|
|
48
67
|
client,
|
|
49
68
|
testRun,
|
|
69
|
+
executionOptions,
|
|
70
|
+
screenshottingOptions,
|
|
50
71
|
apiToken,
|
|
51
72
|
commitSha,
|
|
52
73
|
appUrl,
|
|
53
74
|
useAssetsSnapshottedInBaseSimulation,
|
|
54
|
-
|
|
55
|
-
devTools,
|
|
56
|
-
bypassCSP,
|
|
57
|
-
diffThreshold,
|
|
58
|
-
diffPixelThreshold,
|
|
59
|
-
padTime,
|
|
60
|
-
shiftTime,
|
|
61
|
-
networkStubbing,
|
|
62
|
-
parallelTasks,
|
|
75
|
+
parallelTasks: parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : undefined,
|
|
63
76
|
deflake,
|
|
64
77
|
cachedTestRunResults,
|
|
65
|
-
accelerate,
|
|
66
78
|
});
|
|
67
79
|
return results;
|
|
68
80
|
}
|
|
69
81
|
const results = [...cachedTestRunResults];
|
|
70
82
|
const testsToRun = (0, run_all_tests_utils_1.getTestsToRun)({ testCases, cachedTestRunResults });
|
|
71
83
|
for (const testCase of testsToRun) {
|
|
72
|
-
const { sessionId, baseReplayId, options } = testCase;
|
|
73
84
|
const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)({
|
|
74
|
-
|
|
75
|
-
|
|
85
|
+
replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
|
|
86
|
+
useAssetsSnapshottedInBaseSimulation,
|
|
87
|
+
appUrl,
|
|
88
|
+
testCase,
|
|
89
|
+
}),
|
|
90
|
+
executionOptions,
|
|
91
|
+
screenshottingOptions,
|
|
92
|
+
testCase,
|
|
76
93
|
apiToken,
|
|
77
94
|
commitSha,
|
|
78
|
-
|
|
79
|
-
appUrl,
|
|
80
|
-
headless,
|
|
81
|
-
devTools,
|
|
82
|
-
bypassCSP,
|
|
83
|
-
screenshot: true,
|
|
84
|
-
baseSimulationId: baseReplayId,
|
|
85
|
-
diffThreshold,
|
|
86
|
-
diffPixelThreshold,
|
|
87
|
-
save: false,
|
|
88
|
-
exitOnMismatch: false,
|
|
89
|
-
padTime,
|
|
90
|
-
shiftTime,
|
|
91
|
-
networkStubbing,
|
|
92
|
-
simulationIdForAssets: (0, config_utils_1.getSimulationIdForAssets)(testCase, useAssetsSnapshottedInBaseSimulation),
|
|
93
|
-
accelerate,
|
|
94
|
-
...options,
|
|
95
|
+
deflake: deflake !== null && deflake !== void 0 ? deflake : false,
|
|
95
96
|
});
|
|
96
97
|
results.push(result);
|
|
97
98
|
await (0, test_run_api_1.putTestRunResults)({
|
|
@@ -130,14 +131,11 @@ exports.runAllTests = {
|
|
|
130
131
|
command: "run-all-tests",
|
|
131
132
|
describe: "Run all replay test cases",
|
|
132
133
|
builder: {
|
|
133
|
-
apiToken:
|
|
134
|
-
|
|
135
|
-
},
|
|
136
|
-
commitSha: {
|
|
137
|
-
string: true,
|
|
138
|
-
},
|
|
134
|
+
apiToken: common_options_1.OPTIONS.apiToken,
|
|
135
|
+
commitSha: common_options_1.OPTIONS.commitSha,
|
|
139
136
|
appUrl: {
|
|
140
137
|
string: true,
|
|
138
|
+
description: "The URL to execute the tests against. If left absent here and in the test cases file, then will use the URL the test was originally recorded against.",
|
|
141
139
|
},
|
|
142
140
|
useAssetsSnapshottedInBaseSimulation: {
|
|
143
141
|
boolean: true,
|
|
@@ -145,46 +143,12 @@ exports.runAllTests = {
|
|
|
145
143
|
" from the base simulation/replay the test is comparing against. The sessions will then be replayed against those local urls." +
|
|
146
144
|
" This is an alternative to specifying an appUrl.",
|
|
147
145
|
conflicts: "appUrl",
|
|
148
|
-
|
|
149
|
-
headless: {
|
|
150
|
-
boolean: true,
|
|
151
|
-
description: "Start browser in headless mode",
|
|
152
|
-
},
|
|
153
|
-
devTools: {
|
|
154
|
-
boolean: true,
|
|
155
|
-
description: "Open Chrome Dev Tools",
|
|
156
|
-
},
|
|
157
|
-
bypassCSP: {
|
|
158
|
-
boolean: true,
|
|
159
|
-
description: "Enables bypass CSP in the browser",
|
|
160
|
-
},
|
|
161
|
-
diffThreshold: {
|
|
162
|
-
number: true,
|
|
163
|
-
description: "Acceptable maximum proportion of changed pixels, between 0 and 1. If this proportion is exceeded then the test will fail.",
|
|
164
|
-
},
|
|
165
|
-
diffPixelThreshold: {
|
|
166
|
-
number: true,
|
|
167
|
-
description: "A number between 0 and 1. Color/brightness differences in individual pixels will be ignored if the difference is less than this threshold. A value of 1.0 would accept any difference in color, while a value of 0.0 would accept no difference in color.",
|
|
146
|
+
default: false,
|
|
168
147
|
},
|
|
169
148
|
githubSummary: {
|
|
170
149
|
boolean: true,
|
|
171
150
|
description: "Outputs a summary page for GitHub actions",
|
|
172
151
|
},
|
|
173
|
-
padTime: {
|
|
174
|
-
boolean: true,
|
|
175
|
-
description: "Pad replay time according to recording duration. Please note this option will be ignored if running with the '--accelerate' option.",
|
|
176
|
-
default: true,
|
|
177
|
-
},
|
|
178
|
-
shiftTime: {
|
|
179
|
-
boolean: true,
|
|
180
|
-
description: "Shift time during simulation to be set as the recording time",
|
|
181
|
-
default: true,
|
|
182
|
-
},
|
|
183
|
-
networkStubbing: {
|
|
184
|
-
boolean: true,
|
|
185
|
-
description: "Stub network requests during replay",
|
|
186
|
-
default: true,
|
|
187
|
-
},
|
|
188
152
|
parallelize: {
|
|
189
153
|
boolean: true,
|
|
190
154
|
description: "Run tests in parallel",
|
|
@@ -214,11 +178,8 @@ exports.runAllTests = {
|
|
|
214
178
|
description: "The path to the meticulous.json file containing the list of tests you want to run." +
|
|
215
179
|
" If not set a search will be performed to find a meticulous.json file in the current directory or the nearest parent directory.",
|
|
216
180
|
},
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
description: "Fast forward through any pauses to replay as fast as possible. Warning: this option is experimental and may be deprecated",
|
|
220
|
-
default: false,
|
|
221
|
-
},
|
|
181
|
+
...common_options_1.COMMON_REPLAY_OPTIONS,
|
|
182
|
+
...common_options_1.SCREENSHOT_DIFF_OPTIONS,
|
|
222
183
|
},
|
|
223
|
-
handler: (0, sentry_utils_1.wrapHandler)(handler),
|
|
184
|
+
handler: (0, sentry_utils_1.wrapHandler)((0, command_utils_1.handleNulls)(handler)),
|
|
224
185
|
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AxiosInstance } from "axios";
|
|
2
2
|
import { CommandModule } from "yargs";
|
|
3
|
+
import { ScreenshotDiffOptions } from "../../command-utils/common-types";
|
|
3
4
|
import { CompareImageResult } from "../../image/diff.utils";
|
|
4
5
|
export declare class DiffError extends Error {
|
|
5
6
|
readonly extras?: {
|
|
@@ -28,16 +29,15 @@ export declare const diffScreenshots: (options: {
|
|
|
28
29
|
headReplayId: string;
|
|
29
30
|
baseScreenshotsDir: string;
|
|
30
31
|
headScreenshotsDir: string;
|
|
31
|
-
|
|
32
|
-
pixelThreshold: number | null | undefined;
|
|
32
|
+
diffOptions: ScreenshotDiffOptions;
|
|
33
33
|
exitOnMismatch: boolean;
|
|
34
34
|
}) => Promise<ScreenshotDiffResult[]>;
|
|
35
35
|
interface Options {
|
|
36
36
|
apiToken?: string | null | undefined;
|
|
37
37
|
baseSimulationId: string;
|
|
38
38
|
headSimulationId: string;
|
|
39
|
-
threshold
|
|
40
|
-
pixelThreshold
|
|
39
|
+
threshold: number;
|
|
40
|
+
pixelThreshold: number;
|
|
41
41
|
}
|
|
42
42
|
export declare const screenshotDiff: CommandModule<unknown, Options>;
|
|
43
43
|
export {};
|
|
@@ -9,12 +9,12 @@ const loglevel_1 = __importDefault(require("loglevel"));
|
|
|
9
9
|
const path_1 = require("path");
|
|
10
10
|
const client_1 = require("../../api/client");
|
|
11
11
|
const replay_api_1 = require("../../api/replay.api");
|
|
12
|
+
const common_options_1 = require("../../command-utils/common-options");
|
|
12
13
|
const diff_utils_1 = require("../../image/diff.utils");
|
|
13
14
|
const io_utils_1 = require("../../image/io.utils");
|
|
14
15
|
const replays_1 = require("../../local-data/replays");
|
|
15
16
|
const screenshot_diffs_1 = require("../../local-data/screenshot-diffs");
|
|
16
17
|
const sentry_utils_1 = require("../../utils/sentry.utils");
|
|
17
|
-
const DEFAULT_MISMATCH_THRESHOLD = 0.01;
|
|
18
18
|
class DiffError extends Error {
|
|
19
19
|
constructor(message, extras) {
|
|
20
20
|
super(message);
|
|
@@ -22,16 +22,15 @@ class DiffError extends Error {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
exports.DiffError = DiffError;
|
|
25
|
-
const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreenshotsDir, headScreenshotsDir,
|
|
25
|
+
const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreenshotsDir, headScreenshotsDir, diffOptions, exitOnMismatch, }) => {
|
|
26
|
+
const { diffThreshold, diffPixelThreshold } = diffOptions;
|
|
26
27
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
27
|
-
const threshold = threshold_ !== null && threshold_ !== void 0 ? threshold_ : DEFAULT_MISMATCH_THRESHOLD;
|
|
28
28
|
const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
|
|
29
29
|
const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
|
|
30
30
|
// Assume base replay screenshots are always a subset of the head replay screenshots.
|
|
31
31
|
// We report any missing base replay screenshots for visibility but don't count it as a difference.
|
|
32
32
|
const missingHeadImages = new Set([...baseReplayScreenshots].filter((file) => !headReplayScreenshots.includes(file)));
|
|
33
33
|
let totalMismatchPixels = 0;
|
|
34
|
-
let totalComparedPixels = 0;
|
|
35
34
|
const comparisonResults = [];
|
|
36
35
|
try {
|
|
37
36
|
if (missingHeadImages.size > 0) {
|
|
@@ -39,7 +38,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
39
38
|
throw new DiffError(`Head replay is missing screenshots: ${[...missingHeadImages].sort()}`, {
|
|
40
39
|
baseReplayId,
|
|
41
40
|
headReplayId,
|
|
42
|
-
threshold,
|
|
41
|
+
threshold: diffThreshold,
|
|
43
42
|
});
|
|
44
43
|
}
|
|
45
44
|
for (const screenshotFileName of headReplayScreenshots) {
|
|
@@ -54,10 +53,9 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
54
53
|
const comparisonResult = (0, diff_utils_1.compareImages)({
|
|
55
54
|
base: baseScreenshot,
|
|
56
55
|
head: headScreenshot,
|
|
57
|
-
pixelThreshold:
|
|
56
|
+
pixelThreshold: diffPixelThreshold,
|
|
58
57
|
});
|
|
59
58
|
totalMismatchPixels += comparisonResult.mismatchPixels;
|
|
60
|
-
totalComparedPixels += baseScreenshot.width * baseScreenshot.height;
|
|
61
59
|
logger.debug({
|
|
62
60
|
screenshotFileName,
|
|
63
61
|
mismatchPixels: comparisonResult.mismatchPixels,
|
|
@@ -73,7 +71,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
73
71
|
baseScreenshotFile,
|
|
74
72
|
headScreenshotFile,
|
|
75
73
|
comparisonResult,
|
|
76
|
-
outcome: comparisonResult.mismatchFraction >
|
|
74
|
+
outcome: comparisonResult.mismatchFraction > diffThreshold ? "fail" : "pass",
|
|
77
75
|
});
|
|
78
76
|
}
|
|
79
77
|
await (0, replay_api_1.postScreenshotDiffStats)(client, {
|
|
@@ -88,7 +86,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
88
86
|
const diffUrl = await (0, replay_api_1.getDiffUrl)(client, baseReplayId, headReplayId);
|
|
89
87
|
logger.info(`View screenshot diff at ${diffUrl}`);
|
|
90
88
|
comparisonResults.forEach((result) => {
|
|
91
|
-
logComparisonResultMessage(logger, `${Math.round(result.comparisonResult.mismatchFraction * 100)}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${Math.round(
|
|
89
|
+
logComparisonResultMessage(logger, `${Math.round(result.comparisonResult.mismatchFraction * 100)}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${Math.round(diffThreshold * 100)}%)`, result.outcome);
|
|
92
90
|
});
|
|
93
91
|
// Check if individual screenshot mismatch is higher than the threshold.
|
|
94
92
|
const mismatchingScreenshots = comparisonResults.filter((result) => result.outcome == "fail");
|
|
@@ -102,7 +100,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
102
100
|
throw new DiffError(`Screenshots ${mismatchingScreenshots.map((result) => (0, path_1.basename)(result.headScreenshotFile))} do not match!`, {
|
|
103
101
|
baseReplayId,
|
|
104
102
|
headReplayId,
|
|
105
|
-
threshold,
|
|
103
|
+
threshold: diffThreshold,
|
|
106
104
|
});
|
|
107
105
|
}
|
|
108
106
|
}
|
|
@@ -116,7 +114,7 @@ const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreens
|
|
|
116
114
|
throw new DiffError(`Error while diffing: ${error}`, {
|
|
117
115
|
baseReplayId,
|
|
118
116
|
headReplayId,
|
|
119
|
-
threshold,
|
|
117
|
+
threshold: diffThreshold,
|
|
120
118
|
value: 1,
|
|
121
119
|
});
|
|
122
120
|
}
|
|
@@ -140,8 +138,10 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
|
|
|
140
138
|
headReplayId,
|
|
141
139
|
baseScreenshotsDir,
|
|
142
140
|
headScreenshotsDir,
|
|
143
|
-
|
|
144
|
-
|
|
141
|
+
diffOptions: {
|
|
142
|
+
diffThreshold: threshold,
|
|
143
|
+
diffPixelThreshold: pixelThreshold,
|
|
144
|
+
},
|
|
145
145
|
exitOnMismatch: true,
|
|
146
146
|
});
|
|
147
147
|
};
|
|
@@ -162,12 +162,8 @@ exports.screenshotDiff = {
|
|
|
162
162
|
demandOption: true,
|
|
163
163
|
alias: "headReplayId",
|
|
164
164
|
},
|
|
165
|
-
threshold:
|
|
166
|
-
|
|
167
|
-
},
|
|
168
|
-
pixelThreshold: {
|
|
169
|
-
number: true,
|
|
170
|
-
},
|
|
165
|
+
threshold: common_options_1.SCREENSHOT_DIFF_OPTIONS.diffThreshold,
|
|
166
|
+
pixelThreshold: common_options_1.SCREENSHOT_DIFF_OPTIONS.diffPixelThreshold,
|
|
171
167
|
},
|
|
172
168
|
handler: (0, sentry_utils_1.wrapHandler)(handler),
|
|
173
169
|
};
|
package/dist/config/config.js
CHANGED
|
@@ -22,14 +22,13 @@ const getConfigFilePath = async () => {
|
|
|
22
22
|
return configFilePath;
|
|
23
23
|
};
|
|
24
24
|
const validateReplayOptions = (prevOptions) => {
|
|
25
|
-
const { screenshotSelector, diffThreshold, diffPixelThreshold,
|
|
25
|
+
const { screenshotSelector, diffThreshold, diffPixelThreshold, moveBeforeClick, simulationIdForAssets, } = prevOptions;
|
|
26
26
|
return {
|
|
27
27
|
...(screenshotSelector ? { screenshotSelector } : {}),
|
|
28
28
|
...(diffThreshold ? { diffThreshold } : {}),
|
|
29
29
|
...(diffPixelThreshold ? { diffPixelThreshold } : {}),
|
|
30
|
-
...(cookies ? { cookies } : {}),
|
|
31
30
|
...(moveBeforeClick ? { moveBeforeClick } : {}),
|
|
32
|
-
...(
|
|
31
|
+
...(simulationIdForAssets ? { simulationIdForAssets } : {}),
|
|
33
32
|
};
|
|
34
33
|
};
|
|
35
34
|
const validateConfig = (prevConfig) => {
|