@alwaysmeticulous/cli 2.7.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/replay-diff.api.d.ts +13 -0
- package/dist/api/replay-diff.api.js +32 -0
- package/dist/commands/replay/replay.command.d.ts +2 -1
- package/dist/commands/replay/replay.command.js +29 -3
- package/dist/commands/run-all-tests/run-all-tests.command.js +1 -0
- package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts +16 -24
- package/dist/commands/screenshot-diff/screenshot-diff.command.js +153 -99
- package/dist/config/config.js +5 -5
- package/dist/deflake-tests/deflake-tests.handler.d.ts +1 -0
- package/dist/deflake-tests/deflake-tests.handler.js +3 -2
- package/dist/index.d.ts +0 -1
- package/dist/index.js +1 -3
- package/dist/main.js +0 -2
- package/dist/parallel-tests/parallel-tests.handler.js +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +4 -3
- package/dist/api/project-build.api.d.ts +0 -9
- package/dist/api/project-build.api.js +0 -51
- package/dist/commands/upload-build/upload-build.command.d.ts +0 -13
- package/dist/commands/upload-build/upload-build.command.js +0 -88
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ReplayDiff, ReplayDiffData } from "@alwaysmeticulous/api";
|
|
2
|
+
import { AxiosInstance } from "axios";
|
|
3
|
+
export declare const getReplayDiff: ({ client, replayDiffId, }: {
|
|
4
|
+
client: AxiosInstance;
|
|
5
|
+
replayDiffId: string;
|
|
6
|
+
}) => Promise<ReplayDiff | null>;
|
|
7
|
+
export declare const createReplayDiff: ({ client, headReplayId, baseReplayId, testRunId, data: replayDiffData, }: {
|
|
8
|
+
client: AxiosInstance;
|
|
9
|
+
headReplayId: string;
|
|
10
|
+
baseReplayId: string;
|
|
11
|
+
testRunId: string | null;
|
|
12
|
+
data: ReplayDiffData;
|
|
13
|
+
}) => Promise<ReplayDiff>;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createReplayDiff = exports.getReplayDiff = void 0;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const getReplayDiff = async ({ client, replayDiffId, }) => {
|
|
9
|
+
const { data } = await client
|
|
10
|
+
.get(`replay-diffs/${replayDiffId}`)
|
|
11
|
+
.catch((error) => {
|
|
12
|
+
var _a;
|
|
13
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
14
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
15
|
+
return { data: null };
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
throw error;
|
|
19
|
+
});
|
|
20
|
+
return data;
|
|
21
|
+
};
|
|
22
|
+
exports.getReplayDiff = getReplayDiff;
|
|
23
|
+
const createReplayDiff = async ({ client, headReplayId, baseReplayId, testRunId, data: replayDiffData, }) => {
|
|
24
|
+
const { data } = await client.post("replay-diffs", {
|
|
25
|
+
headReplayId,
|
|
26
|
+
baseReplayId,
|
|
27
|
+
...(testRunId ? { testRunId } : {}),
|
|
28
|
+
data: replayDiffData,
|
|
29
|
+
});
|
|
30
|
+
return data;
|
|
31
|
+
};
|
|
32
|
+
exports.createReplayDiff = createReplayDiff;
|
|
@@ -7,8 +7,9 @@ export interface ReplayOptions extends AdditionalReplayOptions {
|
|
|
7
7
|
screenshottingOptions: ScreenshotAssertionsOptions;
|
|
8
8
|
exitOnMismatch: boolean;
|
|
9
9
|
generatedBy: GeneratedBy;
|
|
10
|
+
testRunId: string | null;
|
|
10
11
|
}
|
|
11
|
-
export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, }: ReplayOptions) => Promise<Replay>;
|
|
12
|
+
export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, }: ReplayOptions) => Promise<Replay>;
|
|
12
13
|
export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, Omit<ReplayExecutionOptions, "maxDurationMs" | "maxEventCount">, AdditionalReplayOptions {
|
|
13
14
|
screenshot: boolean;
|
|
14
15
|
appUrl: string | null | undefined;
|
|
@@ -10,6 +10,7 @@ const common_1 = require("@alwaysmeticulous/common");
|
|
|
10
10
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
11
11
|
const luxon_1 = require("luxon");
|
|
12
12
|
const client_1 = require("../../api/client");
|
|
13
|
+
const replay_diff_api_1 = require("../../api/replay-diff.api");
|
|
13
14
|
const replay_api_1 = require("../../api/replay.api");
|
|
14
15
|
const upload_1 = require("../../api/upload");
|
|
15
16
|
const archive_1 = require("../../archive/archive");
|
|
@@ -24,7 +25,7 @@ const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
|
|
|
24
25
|
const config_utils_1 = require("../../utils/config.utils");
|
|
25
26
|
const version_utils_1 = require("../../utils/version.utils");
|
|
26
27
|
const screenshot_diff_command_1 = require("../screenshot-diff/screenshot-diff.command");
|
|
27
|
-
const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, }) => {
|
|
28
|
+
const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, }) => {
|
|
28
29
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
29
30
|
const client = (0, client_1.createClient)({ apiToken });
|
|
30
31
|
// 1. Check session files
|
|
@@ -73,6 +74,7 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
|
|
|
73
74
|
recordingId: "manual-replay",
|
|
74
75
|
meticulousSha: "meticulousSha",
|
|
75
76
|
generatedBy,
|
|
77
|
+
testRunId,
|
|
76
78
|
dependencies: {
|
|
77
79
|
browserUserInteractions: {
|
|
78
80
|
key: "browserUserInteractions",
|
|
@@ -152,15 +154,38 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
|
|
|
152
154
|
await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
|
|
153
155
|
const baseReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(baseReplayId));
|
|
154
156
|
const headReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)(tempDir);
|
|
155
|
-
await (0, screenshot_diff_command_1.diffScreenshots)({
|
|
157
|
+
const screenshotDiffResults = await (0, screenshot_diff_command_1.diffScreenshots)({
|
|
156
158
|
client,
|
|
157
159
|
baseReplayId,
|
|
158
160
|
headReplayId: replay.id,
|
|
159
161
|
baseScreenshotsDir: baseReplayScreenshotsDir,
|
|
160
162
|
headScreenshotsDir: headReplayScreenshotsDir,
|
|
161
163
|
diffOptions: screenshottingOptions.diffOptions,
|
|
162
|
-
exitOnMismatch: !!exitOnMismatch,
|
|
163
164
|
});
|
|
165
|
+
const replayDiff = await (0, replay_diff_api_1.createReplayDiff)({
|
|
166
|
+
client,
|
|
167
|
+
headReplayId: replay.id,
|
|
168
|
+
baseReplayId,
|
|
169
|
+
testRunId,
|
|
170
|
+
data: {
|
|
171
|
+
screenshotAssertionsOptions: screenshottingOptions,
|
|
172
|
+
screenshotDiffResults,
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
logger.debug(replayDiff);
|
|
176
|
+
try {
|
|
177
|
+
(0, screenshot_diff_command_1.checkScreenshotDiffResult)({
|
|
178
|
+
baseReplayId,
|
|
179
|
+
headReplayId: replay.id,
|
|
180
|
+
results: screenshotDiffResults,
|
|
181
|
+
diffOptions: screenshottingOptions.diffOptions,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
if (exitOnMismatch) {
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
164
189
|
}
|
|
165
190
|
// 13. Add test case to meticulous.json if --save option is passed
|
|
166
191
|
if (save) {
|
|
@@ -237,6 +262,7 @@ const rawReplayCommandHandler = ({ apiToken, commitSha, sessionId, appUrl, simul
|
|
|
237
262
|
save,
|
|
238
263
|
exitOnMismatch: true,
|
|
239
264
|
generatedBy: generatedByOption,
|
|
265
|
+
testRunId: null,
|
|
240
266
|
});
|
|
241
267
|
};
|
|
242
268
|
exports.rawReplayCommandHandler = rawReplayCommandHandler;
|
|
@@ -98,6 +98,7 @@ const handler = async ({ apiToken, commitSha: commitSha_, appUrl, useAssetsSnaps
|
|
|
98
98
|
commitSha,
|
|
99
99
|
deflake: deflake !== null && deflake !== void 0 ? deflake : false,
|
|
100
100
|
generatedBy: { type: "testRun", runId: testRun.id },
|
|
101
|
+
testRunId: testRun.id,
|
|
101
102
|
});
|
|
102
103
|
results.push(result);
|
|
103
104
|
await (0, test_run_api_1.putTestRunResults)({
|
|
@@ -1,37 +1,30 @@
|
|
|
1
1
|
/// <reference types="yargs" />
|
|
2
|
+
import { ScreenshotDiffOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
|
|
2
3
|
import { AxiosInstance } from "axios";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
export declare const diffScreenshots: ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, }: {
|
|
5
|
+
client: AxiosInstance;
|
|
6
|
+
baseReplayId: string;
|
|
7
|
+
headReplayId: string;
|
|
8
|
+
baseScreenshotsDir: string;
|
|
9
|
+
headScreenshotsDir: string;
|
|
10
|
+
diffOptions: ScreenshotDiffOptions;
|
|
11
|
+
}) => Promise<ScreenshotDiffResult[]>;
|
|
12
|
+
export declare const checkScreenshotDiffResult: ({ baseReplayId, headReplayId, results, diffOptions, }: {
|
|
13
|
+
baseReplayId: string;
|
|
14
|
+
headReplayId: string;
|
|
15
|
+
results: ScreenshotDiffResult[];
|
|
16
|
+
diffOptions: ScreenshotDiffOptions;
|
|
17
|
+
}) => void;
|
|
18
|
+
export declare class ScreenshotDiffError extends Error {
|
|
6
19
|
readonly extras?: {
|
|
7
20
|
baseReplayId: string;
|
|
8
21
|
headReplayId: string;
|
|
9
|
-
threshold: number;
|
|
10
|
-
value?: number;
|
|
11
22
|
} | undefined;
|
|
12
23
|
constructor(message: string, extras?: {
|
|
13
24
|
baseReplayId: string;
|
|
14
25
|
headReplayId: string;
|
|
15
|
-
threshold: number;
|
|
16
|
-
value?: number;
|
|
17
26
|
} | undefined);
|
|
18
27
|
}
|
|
19
|
-
declare type ComparisonOutcome = "pass" | "fail";
|
|
20
|
-
export interface ScreenshotDiffResult {
|
|
21
|
-
baseScreenshotFile: string;
|
|
22
|
-
headScreenshotFile: string;
|
|
23
|
-
outcome: ComparisonOutcome;
|
|
24
|
-
comparisonResult: CompareImageResult;
|
|
25
|
-
}
|
|
26
|
-
export declare const diffScreenshots: (options: {
|
|
27
|
-
client: AxiosInstance;
|
|
28
|
-
baseReplayId: string;
|
|
29
|
-
headReplayId: string;
|
|
30
|
-
baseScreenshotsDir: string;
|
|
31
|
-
headScreenshotsDir: string;
|
|
32
|
-
diffOptions: ScreenshotDiffOptions;
|
|
33
|
-
exitOnMismatch: boolean;
|
|
34
|
-
}) => Promise<ScreenshotDiffResult[]>;
|
|
35
28
|
export declare const screenshotDiff: import("yargs").CommandModule<unknown, import("yargs").InferredOptionTypes<{
|
|
36
29
|
apiToken: {
|
|
37
30
|
string: true;
|
|
@@ -57,4 +50,3 @@ export declare const screenshotDiff: import("yargs").CommandModule<unknown, impo
|
|
|
57
50
|
readonly default: 0.01;
|
|
58
51
|
};
|
|
59
52
|
}>>;
|
|
60
|
-
export {};
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.screenshotDiff = exports.
|
|
6
|
+
exports.screenshotDiff = exports.ScreenshotDiffError = exports.checkScreenshotDiffResult = exports.diffScreenshots = void 0;
|
|
7
7
|
const path_1 = require("path");
|
|
8
8
|
const common_1 = require("@alwaysmeticulous/common");
|
|
9
9
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
@@ -15,116 +15,163 @@ const diff_utils_1 = require("../../image/diff.utils");
|
|
|
15
15
|
const io_utils_1 = require("../../image/io.utils");
|
|
16
16
|
const replays_1 = require("../../local-data/replays");
|
|
17
17
|
const screenshot_diffs_1 = require("../../local-data/screenshot-diffs");
|
|
18
|
-
|
|
19
|
-
constructor(message, extras) {
|
|
20
|
-
super(message);
|
|
21
|
-
this.extras = extras;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
exports.DiffError = DiffError;
|
|
25
|
-
const diffScreenshots = async ({ client, baseReplayId, headReplayId, baseScreenshotsDir, headScreenshotsDir, diffOptions, exitOnMismatch, }) => {
|
|
26
|
-
const { diffThreshold, diffPixelThreshold } = diffOptions;
|
|
18
|
+
const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, }) => {
|
|
27
19
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
20
|
+
const { diffThreshold, diffPixelThreshold } = diffOptions;
|
|
28
21
|
const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
|
|
29
22
|
const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
try {
|
|
36
|
-
if (missingHeadImages.size > 0) {
|
|
37
|
-
logComparisonResultMessage(logger, `Head replay is missing screenshots: ${[...missingHeadImages].sort()}`, "fail");
|
|
38
|
-
throw new DiffError(`Head replay is missing screenshots: ${[...missingHeadImages].sort()}`, {
|
|
39
|
-
baseReplayId,
|
|
40
|
-
headReplayId,
|
|
41
|
-
threshold: diffThreshold,
|
|
42
|
-
});
|
|
23
|
+
const missingHeadImages = new Set(baseReplayScreenshots.filter((file) => !headReplayScreenshots.includes(file)));
|
|
24
|
+
const missingHeadImagesResults = Array.from(missingHeadImages).flatMap((screenshotFileName) => {
|
|
25
|
+
const identifier = getScreenshotIdentifier(screenshotFileName);
|
|
26
|
+
if (identifier == null) {
|
|
27
|
+
return [];
|
|
43
28
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
pixelThreshold: diffPixelThreshold,
|
|
57
|
-
});
|
|
58
|
-
totalMismatchPixels += comparisonResult.mismatchPixels;
|
|
59
|
-
logger.debug({
|
|
60
|
-
screenshotFileName,
|
|
61
|
-
mismatchPixels: comparisonResult.mismatchPixels,
|
|
62
|
-
mismatchFraction: comparisonResult.mismatchFraction,
|
|
63
|
-
});
|
|
64
|
-
await (0, screenshot_diffs_1.writeScreenshotDiff)({
|
|
65
|
-
baseReplayId,
|
|
66
|
-
headReplayId,
|
|
67
|
-
screenshotFileName,
|
|
68
|
-
diff: comparisonResult.diff,
|
|
69
|
-
});
|
|
70
|
-
comparisonResults.push({
|
|
71
|
-
baseScreenshotFile,
|
|
72
|
-
headScreenshotFile,
|
|
73
|
-
comparisonResult,
|
|
74
|
-
outcome: comparisonResult.mismatchFraction > diffThreshold ? "fail" : "pass",
|
|
75
|
-
});
|
|
29
|
+
return [
|
|
30
|
+
{
|
|
31
|
+
identifier,
|
|
32
|
+
outcome: "missing-head",
|
|
33
|
+
baseScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
34
|
+
},
|
|
35
|
+
];
|
|
36
|
+
});
|
|
37
|
+
const diffAgainstBase = async (screenshotFileName) => {
|
|
38
|
+
const identifier = getScreenshotIdentifier(screenshotFileName);
|
|
39
|
+
if (identifier == null) {
|
|
40
|
+
return [];
|
|
76
41
|
}
|
|
77
|
-
|
|
42
|
+
if (!baseReplayScreenshots.includes(screenshotFileName)) {
|
|
43
|
+
return [
|
|
44
|
+
{
|
|
45
|
+
identifier,
|
|
46
|
+
outcome: "missing-base",
|
|
47
|
+
headScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
const baseScreenshotFile = (0, path_1.join)(baseScreenshotsDir, screenshotFileName);
|
|
52
|
+
const headScreenshotFile = (0, path_1.join)(headScreenshotsDir, screenshotFileName);
|
|
53
|
+
const baseScreenshot = await (0, io_utils_1.readPng)(baseScreenshotFile);
|
|
54
|
+
const headScreenshot = await (0, io_utils_1.readPng)(headScreenshotFile);
|
|
55
|
+
if (baseScreenshot.width !== baseScreenshot.width ||
|
|
56
|
+
headScreenshot.height !== headScreenshot.height) {
|
|
57
|
+
return [
|
|
58
|
+
{
|
|
59
|
+
identifier,
|
|
60
|
+
outcome: "different-size",
|
|
61
|
+
headScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
62
|
+
baseScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
63
|
+
baseWidth: baseScreenshot.width,
|
|
64
|
+
baseHeight: baseScreenshot.height,
|
|
65
|
+
headWidth: headScreenshot.width,
|
|
66
|
+
headHeight: headScreenshot.height,
|
|
67
|
+
},
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
const comparisonResult = (0, diff_utils_1.compareImages)({
|
|
71
|
+
base: baseScreenshot,
|
|
72
|
+
head: headScreenshot,
|
|
73
|
+
pixelThreshold: diffPixelThreshold,
|
|
74
|
+
});
|
|
75
|
+
await (0, screenshot_diffs_1.writeScreenshotDiff)({
|
|
78
76
|
baseReplayId,
|
|
79
77
|
headReplayId,
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
height: 0,
|
|
83
|
-
mismatchPixels: totalMismatchPixels,
|
|
84
|
-
},
|
|
78
|
+
screenshotFileName,
|
|
79
|
+
diff: comparisonResult.diff,
|
|
85
80
|
});
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
81
|
+
return [
|
|
82
|
+
{
|
|
83
|
+
identifier,
|
|
84
|
+
outcome: comparisonResult.mismatchFraction > diffThreshold
|
|
85
|
+
? "diff"
|
|
86
|
+
: "no-diff",
|
|
87
|
+
headScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
88
|
+
baseScreenshotFile: `screenshots/${screenshotFileName}`,
|
|
89
|
+
width: baseScreenshot.width,
|
|
90
|
+
height: baseScreenshot.height,
|
|
91
|
+
mismatchPixels: comparisonResult.mismatchPixels,
|
|
92
|
+
mismatchFraction: comparisonResult.mismatchFraction,
|
|
93
|
+
},
|
|
94
|
+
];
|
|
95
|
+
};
|
|
96
|
+
const headDiffResults = (await Promise.all(headReplayScreenshots.map(diffAgainstBase))).flat();
|
|
97
|
+
const diffUrl = await (0, replay_api_1.getDiffUrl)(client, baseReplayId, headReplayId);
|
|
98
|
+
logger.info(`View screenshot diff at ${diffUrl}`);
|
|
99
|
+
return [...missingHeadImagesResults, ...headDiffResults];
|
|
100
|
+
};
|
|
101
|
+
exports.diffScreenshots = diffScreenshots;
|
|
102
|
+
const checkScreenshotDiffResult = ({ baseReplayId, headReplayId, results, diffOptions, }) => {
|
|
103
|
+
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
104
|
+
const missingHeadImagesResults = results.flatMap((result) => result.outcome === "missing-head" ? [result] : []);
|
|
105
|
+
if (missingHeadImagesResults.length) {
|
|
106
|
+
const message = `Head replay is missing screenshots: ${missingHeadImagesResults
|
|
107
|
+
.map(({ baseScreenshotFile }) => (0, path_1.basename)(baseScreenshotFile))
|
|
108
|
+
.sort()} => FAIL!`;
|
|
109
|
+
logger.info(message);
|
|
110
|
+
throw new ScreenshotDiffError(message, {
|
|
111
|
+
baseReplayId,
|
|
112
|
+
headReplayId,
|
|
90
113
|
});
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
114
|
+
}
|
|
115
|
+
const missingBaseImagesResults = results.flatMap((result) => result.outcome === "missing-base" ? [result] : []);
|
|
116
|
+
if (missingHeadImagesResults.length) {
|
|
117
|
+
const message = `Notice: Base replay is missing screenshots: ${missingBaseImagesResults
|
|
118
|
+
.map(({ headScreenshotFile }) => (0, path_1.basename)(headScreenshotFile))
|
|
119
|
+
.sort()}`;
|
|
120
|
+
logger.info(message);
|
|
121
|
+
}
|
|
122
|
+
results.forEach((result) => {
|
|
123
|
+
const { outcome } = result;
|
|
124
|
+
if (outcome === "different-size") {
|
|
125
|
+
const message = `Screenshots ${(0, path_1.basename)(result.headScreenshotFile)} have different sizes => FAIL!`;
|
|
126
|
+
logger.info(message);
|
|
127
|
+
throw new ScreenshotDiffError(message, {
|
|
101
128
|
baseReplayId,
|
|
102
129
|
headReplayId,
|
|
103
|
-
threshold: diffThreshold,
|
|
104
130
|
});
|
|
105
131
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
132
|
+
if (outcome === "diff" || outcome === "no-diff") {
|
|
133
|
+
const mismatch = (result.mismatchFraction * 100).toFixed(3);
|
|
134
|
+
const threshold = (diffOptions.diffThreshold * 100).toFixed(3);
|
|
135
|
+
const message = `${mismatch}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${threshold}%) => ${outcome === "no-diff" ? "PASS" : "FAIL!"}`;
|
|
136
|
+
logger.info(message);
|
|
137
|
+
if (outcome === "diff") {
|
|
138
|
+
throw new ScreenshotDiffError(message, {
|
|
139
|
+
baseReplayId,
|
|
140
|
+
headReplayId,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
110
143
|
}
|
|
111
|
-
|
|
112
|
-
|
|
144
|
+
});
|
|
145
|
+
};
|
|
146
|
+
exports.checkScreenshotDiffResult = checkScreenshotDiffResult;
|
|
147
|
+
class ScreenshotDiffError extends Error {
|
|
148
|
+
constructor(message, extras) {
|
|
149
|
+
super(message);
|
|
150
|
+
this.extras = extras;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
exports.ScreenshotDiffError = ScreenshotDiffError;
|
|
154
|
+
const getScreenshotIdentifier = (filename) => {
|
|
155
|
+
const name = (0, path_1.basename)(filename);
|
|
156
|
+
if (name === "final-state.png") {
|
|
157
|
+
return {
|
|
158
|
+
type: "end-state",
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
if (name.startsWith("screenshot-after-event")) {
|
|
162
|
+
const match = name.match(/^(?:.*)-(\d+)[.]png$/);
|
|
163
|
+
const eventNumber = match ? parseInt(match[1], 10) : undefined;
|
|
164
|
+
if (match && eventNumber != null && !isNaN(eventNumber)) {
|
|
165
|
+
return {
|
|
166
|
+
type: "after-event",
|
|
167
|
+
eventNumber: eventNumber - 1,
|
|
168
|
+
};
|
|
113
169
|
}
|
|
114
|
-
throw new DiffError(`Error while diffing: ${error}`, {
|
|
115
|
-
baseReplayId,
|
|
116
|
-
headReplayId,
|
|
117
|
-
threshold: diffThreshold,
|
|
118
|
-
value: 1,
|
|
119
|
-
});
|
|
120
170
|
}
|
|
121
|
-
return
|
|
122
|
-
};
|
|
123
|
-
exports.diffScreenshots = diffScreenshots;
|
|
124
|
-
const logComparisonResultMessage = (logger, message, outcome) => {
|
|
125
|
-
logger.info(`${message} => ${outcome === "pass" ? "PASS" : "FAIL!"}`);
|
|
171
|
+
return undefined;
|
|
126
172
|
};
|
|
127
173
|
const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulationId: headReplayId, threshold, pixelThreshold, }) => {
|
|
174
|
+
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
128
175
|
const client = (0, client_1.createClient)({ apiToken });
|
|
129
176
|
await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
|
|
130
177
|
await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
|
|
@@ -132,17 +179,24 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
|
|
|
132
179
|
await (0, replays_1.getOrFetchReplayArchive)(client, headReplayId);
|
|
133
180
|
const baseScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(baseReplayId));
|
|
134
181
|
const headScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(headReplayId));
|
|
135
|
-
|
|
182
|
+
const diffOptions = {
|
|
183
|
+
diffThreshold: threshold,
|
|
184
|
+
diffPixelThreshold: pixelThreshold,
|
|
185
|
+
};
|
|
186
|
+
const results = await (0, exports.diffScreenshots)({
|
|
136
187
|
client,
|
|
137
188
|
baseReplayId,
|
|
138
189
|
headReplayId,
|
|
139
190
|
baseScreenshotsDir,
|
|
140
191
|
headScreenshotsDir,
|
|
141
|
-
diffOptions
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
192
|
+
diffOptions,
|
|
193
|
+
});
|
|
194
|
+
logger.debug(results);
|
|
195
|
+
(0, exports.checkScreenshotDiffResult)({
|
|
196
|
+
baseReplayId,
|
|
197
|
+
headReplayId,
|
|
198
|
+
results,
|
|
199
|
+
diffOptions,
|
|
146
200
|
});
|
|
147
201
|
};
|
|
148
202
|
exports.screenshotDiff = (0, command_builder_1.buildCommand)("screenshot-diff")
|
package/dist/config/config.js
CHANGED
|
@@ -24,11 +24,11 @@ const getConfigFilePath = async () => {
|
|
|
24
24
|
const validateReplayOptions = (prevOptions) => {
|
|
25
25
|
const { screenshotSelector, diffThreshold, diffPixelThreshold, moveBeforeClick, simulationIdForAssets, } = prevOptions;
|
|
26
26
|
return {
|
|
27
|
-
...(screenshotSelector ? { screenshotSelector } : {}),
|
|
28
|
-
...(diffThreshold ? { diffThreshold } : {}),
|
|
29
|
-
...(diffPixelThreshold ? { diffPixelThreshold } : {}),
|
|
30
|
-
...(moveBeforeClick ? { moveBeforeClick } : {}),
|
|
31
|
-
...(simulationIdForAssets ? { simulationIdForAssets } : {}),
|
|
27
|
+
...(screenshotSelector != null ? { screenshotSelector } : {}),
|
|
28
|
+
...(diffThreshold != null ? { diffThreshold } : {}),
|
|
29
|
+
...(diffPixelThreshold != null ? { diffPixelThreshold } : {}),
|
|
30
|
+
...(moveBeforeClick != null ? { moveBeforeClick } : {}),
|
|
31
|
+
...(simulationIdForAssets != null ? { simulationIdForAssets } : {}),
|
|
32
32
|
};
|
|
33
33
|
};
|
|
34
34
|
const validateConfig = (prevConfig) => {
|
|
@@ -12,6 +12,7 @@ interface HandleReplayOptions {
|
|
|
12
12
|
apiToken: string | null;
|
|
13
13
|
commitSha: string;
|
|
14
14
|
generatedBy: GeneratedBy;
|
|
15
|
+
testRunId: string | null;
|
|
15
16
|
}
|
|
16
17
|
export declare const deflakeReplayCommandHandler: (options: DeflakeReplayCommandHandlerOptions) => Promise<TestCaseResult>;
|
|
17
18
|
export {};
|
|
@@ -8,7 +8,7 @@ const common_1 = require("@alwaysmeticulous/common");
|
|
|
8
8
|
const loglevel_1 = __importDefault(require("loglevel"));
|
|
9
9
|
const replay_command_1 = require("../commands/replay/replay.command");
|
|
10
10
|
const screenshot_diff_command_1 = require("../commands/screenshot-diff/screenshot-diff.command");
|
|
11
|
-
const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, }) => {
|
|
11
|
+
const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, }) => {
|
|
12
12
|
var _a, _b;
|
|
13
13
|
const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
|
|
14
14
|
const replayPromise = (0, replay_command_1.replayCommandHandler)({
|
|
@@ -23,6 +23,7 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
|
|
|
23
23
|
exitOnMismatch: false,
|
|
24
24
|
cookiesFile: null,
|
|
25
25
|
generatedBy,
|
|
26
|
+
testRunId,
|
|
26
27
|
});
|
|
27
28
|
const result = await replayPromise
|
|
28
29
|
.then((replay) => ({
|
|
@@ -31,7 +32,7 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
|
|
|
31
32
|
result: "pass",
|
|
32
33
|
}))
|
|
33
34
|
.catch((error) => {
|
|
34
|
-
if (error instanceof screenshot_diff_command_1.
|
|
35
|
+
if (error instanceof screenshot_diff_command_1.ScreenshotDiffError && error.extras) {
|
|
35
36
|
return {
|
|
36
37
|
...testCase,
|
|
37
38
|
headReplayId: error.extras.headReplayId,
|
package/dist/index.d.ts
CHANGED
|
@@ -9,4 +9,3 @@ export { runAllTests } from "./commands/run-all-tests/run-all-tests.command";
|
|
|
9
9
|
export { screenshotDiff } from "./commands/screenshot-diff/screenshot-diff.command";
|
|
10
10
|
export { showProject } from "./commands/show-project/show-project.command";
|
|
11
11
|
export { updateTests } from "./commands/update-tests/update-tests.command";
|
|
12
|
-
export { uploadBuild } from "./commands/upload-build/upload-build.command";
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.updateTests = exports.showProject = exports.screenshotDiff = exports.runAllTests = exports.replay = exports.record = exports.downloadSession = exports.downloadReplay = exports.debugReplay = exports.createTest = exports.bootstrap = void 0;
|
|
4
4
|
var bootstrap_command_1 = require("./commands/bootstrap/bootstrap.command");
|
|
5
5
|
Object.defineProperty(exports, "bootstrap", { enumerable: true, get: function () { return bootstrap_command_1.bootstrap; } });
|
|
6
6
|
var create_test_command_1 = require("./commands/create-test/create-test.command");
|
|
@@ -23,5 +23,3 @@ var show_project_command_1 = require("./commands/show-project/show-project.comma
|
|
|
23
23
|
Object.defineProperty(exports, "showProject", { enumerable: true, get: function () { return show_project_command_1.showProject; } });
|
|
24
24
|
var update_tests_command_1 = require("./commands/update-tests/update-tests.command");
|
|
25
25
|
Object.defineProperty(exports, "updateTests", { enumerable: true, get: function () { return update_tests_command_1.updateTests; } });
|
|
26
|
-
var upload_build_command_1 = require("./commands/upload-build/upload-build.command");
|
|
27
|
-
Object.defineProperty(exports, "uploadBuild", { enumerable: true, get: function () { return upload_build_command_1.uploadBuild; } });
|
package/dist/main.js
CHANGED
|
@@ -18,7 +18,6 @@ const screenshot_diff_command_1 = require("./commands/screenshot-diff/screenshot
|
|
|
18
18
|
const serve_command_1 = require("./commands/serve/serve.command");
|
|
19
19
|
const show_project_command_1 = require("./commands/show-project/show-project.command");
|
|
20
20
|
const update_tests_command_1 = require("./commands/update-tests/update-tests.command");
|
|
21
|
-
const upload_build_command_1 = require("./commands/upload-build/upload-build.command");
|
|
22
21
|
const logger_utils_1 = require("./utils/logger.utils");
|
|
23
22
|
const sentry_utils_1 = require("./utils/sentry.utils");
|
|
24
23
|
const handleDataDir = (dataDir) => {
|
|
@@ -43,7 +42,6 @@ const main = () => {
|
|
|
43
42
|
.command(screenshot_diff_command_1.screenshotDiff)
|
|
44
43
|
.command(show_project_command_1.showProject)
|
|
45
44
|
.command(update_tests_command_1.updateTests)
|
|
46
|
-
.command(upload_build_command_1.uploadBuild)
|
|
47
45
|
.command("serve", false, serve_command_1.serve) // This is a debugging command, so we hide it to not pollute the main menu
|
|
48
46
|
.help()
|
|
49
47
|
.strict()
|