@allurereport/core 3.16.0 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +4 -4
- package/dist/config.d.ts +2 -1
- package/dist/config.js +14 -10
- package/dist/history.d.ts +2 -2
- package/dist/history.js +12 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/qualityGate/qualityGate.d.ts +6 -5
- package/dist/qualityGate/qualityGate.js +5 -3
- package/dist/qualityGate/rules.d.ts +13 -10
- package/dist/qualityGate/rules.js +142 -15
- package/dist/report.d.ts +1 -2
- package/dist/report.js +125 -52
- package/dist/resolutions.d.ts +8 -0
- package/dist/resolutions.js +181 -0
- package/dist/store/convert.js +0 -1
- package/dist/store/store.d.ts +8 -5
- package/dist/store/store.js +110 -38
- package/dist/utils/artifacts.d.ts +14 -0
- package/dist/utils/artifacts.js +35 -0
- package/dist/utils/perf.d.ts +6 -10
- package/dist/utils/perf.js +41 -28
- package/package.json +21 -21
- package/dist/known.d.ts +0 -8
- package/dist/known.js +0 -86
package/dist/api.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CategoriesConfig,
|
|
1
|
+
import type { CategoriesConfig, PerformanceConfig, ResolvedAllureServiceConfig } from "@allurereport/core-api";
|
|
2
2
|
import type { Plugin, ReportFiles, Config } from "@allurereport/plugin-api";
|
|
3
3
|
import type { ResultsReader } from "@allurereport/reader-api";
|
|
4
4
|
export interface PluginInstance {
|
|
@@ -8,19 +8,19 @@ export interface PluginInstance {
|
|
|
8
8
|
options: Record<string, any>;
|
|
9
9
|
}
|
|
10
10
|
type FullConfigRequiredFromConfig = Required<Pick<Config, "name" | "output" | "open">>;
|
|
11
|
-
export interface FullConfig extends Omit<Config, "name" | "output" | "open" | "allureService" | "
|
|
11
|
+
export interface FullConfig extends Omit<Config, "name" | "output" | "open" | "allureService" | "plugins" | "port" | "resultsDir">, FullConfigRequiredFromConfig {
|
|
12
12
|
port: Config["port"] | undefined;
|
|
13
13
|
allowedEnvironments?: Config["allowedEnvironments"];
|
|
14
14
|
reportFiles: ReportFiles;
|
|
15
15
|
readers?: ResultsReader[];
|
|
16
16
|
plugins?: PluginInstance[];
|
|
17
|
-
known?: KnownTestFailure[];
|
|
18
|
-
knownIssuesPath?: Config["knownIssuesPath"];
|
|
19
17
|
realTime?: any;
|
|
20
18
|
qualityGate?: Config["qualityGate"];
|
|
19
|
+
performance?: PerformanceConfig;
|
|
21
20
|
allureService?: ResolvedAllureServiceConfig;
|
|
22
21
|
categories?: CategoriesConfig;
|
|
23
22
|
globalAttachments?: string[];
|
|
23
|
+
cwd?: string;
|
|
24
24
|
resultsDir?: string[];
|
|
25
25
|
}
|
|
26
26
|
export {};
|
package/dist/config.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { Config, Plugin, PluginConstructorContext, PluginDescriptor } from
|
|
|
2
2
|
import type { FullConfig, PluginInstance } from "./api.js";
|
|
3
3
|
type PluginConstructor = new (options?: Record<string, any>, context?: PluginConstructorContext) => Plugin;
|
|
4
4
|
export interface ConfigOverride {
|
|
5
|
+
cwd?: string;
|
|
5
6
|
name?: Config["name"];
|
|
6
7
|
output?: Config["output"];
|
|
7
8
|
open?: Config["open"];
|
|
@@ -9,7 +10,7 @@ export interface ConfigOverride {
|
|
|
9
10
|
hideLabels?: Config["hideLabels"];
|
|
10
11
|
historyPath?: Config["historyPath"];
|
|
11
12
|
historyLimit?: Config["historyLimit"];
|
|
12
|
-
|
|
13
|
+
resolutions?: Pick<NonNullable<Config["resolutions"]>, "knownIssuesPath">;
|
|
13
14
|
plugins?: Config["plugins"];
|
|
14
15
|
}
|
|
15
16
|
export declare const parseIntegerConfigValue: (value: unknown, defaultValue: number, minValue: number) => number;
|
package/dist/config.js
CHANGED
|
@@ -4,8 +4,8 @@ import * as process from "node:process";
|
|
|
4
4
|
import { validateEnvironmentName } from "@allurereport/core-api";
|
|
5
5
|
import { createJiti } from "jiti";
|
|
6
6
|
import { parse } from "yaml";
|
|
7
|
-
import { DEFAULT_KNOWN_ISSUES_PATH, hasKnownIssueRules, readKnownIssues, resolveExactIssuesFilePath } from "./known.js";
|
|
8
7
|
import { FileSystemReportFiles } from "./plugin.js";
|
|
8
|
+
import { DEFAULT_KNOWN_ISSUES_PATH, resolveExactIssuesFilePath, validateResolutionsConfig } from "./resolutions.js";
|
|
9
9
|
import { environmentIdentityById, environmentIdentityByName, validateAllowedEnvironmentIds, normalizeEnvironmentDescriptorMap, validateAllowedEnvironmentId, } from "./utils/environment.js";
|
|
10
10
|
import { importWrapper } from "./utils/module.js";
|
|
11
11
|
import { normalizeImportPath } from "./utils/path.js";
|
|
@@ -98,8 +98,7 @@ export const validateConfig = (config) => {
|
|
|
98
98
|
"hideLabels",
|
|
99
99
|
"historyPath",
|
|
100
100
|
"historyLimit",
|
|
101
|
-
"
|
|
102
|
-
"knownIssues",
|
|
101
|
+
"resolutions",
|
|
103
102
|
"plugins",
|
|
104
103
|
"defaultLabels",
|
|
105
104
|
"variables",
|
|
@@ -108,6 +107,7 @@ export const validateConfig = (config) => {
|
|
|
108
107
|
"environments",
|
|
109
108
|
"appendHistory",
|
|
110
109
|
"qualityGate",
|
|
110
|
+
"performance",
|
|
111
111
|
"allureService",
|
|
112
112
|
"categories",
|
|
113
113
|
"globalAttachments",
|
|
@@ -195,6 +195,7 @@ export const resolveConfig = async (config, override = {}) => {
|
|
|
195
195
|
if (!validationResult.valid) {
|
|
196
196
|
throw new Error(`The provided Allure config contains unsupported fields: ${validationResult.fields.join(", ")}`);
|
|
197
197
|
}
|
|
198
|
+
validateResolutionsConfig(config.resolutions);
|
|
198
199
|
const { environments, environment, allowedEnvironments } = resolveConfigEnvironments(config);
|
|
199
200
|
const name = override.name ?? config.name ?? "Allure Report";
|
|
200
201
|
const open = override.open ?? config.open ?? false;
|
|
@@ -203,11 +204,14 @@ export const resolveConfig = async (config, override = {}) => {
|
|
|
203
204
|
const historyPath = override.historyPath ?? config.historyPath;
|
|
204
205
|
const historyLimit = override.historyLimit ?? config.historyLimit;
|
|
205
206
|
const appendHistory = config.appendHistory ?? true;
|
|
206
|
-
const configuredKnownIssuesPath = override.knownIssuesPath ?? config.knownIssuesPath;
|
|
207
|
-
const knownIssuesPathInput = configuredKnownIssuesPath ?? (
|
|
207
|
+
const configuredKnownIssuesPath = override.resolutions?.knownIssuesPath ?? config.resolutions?.knownIssuesPath;
|
|
208
|
+
const knownIssuesPathInput = configuredKnownIssuesPath ?? (config.resolutions ? DEFAULT_KNOWN_ISSUES_PATH : undefined);
|
|
208
209
|
const knownIssuesPath = await resolveExactIssuesFilePath(knownIssuesPathInput, "known issues");
|
|
210
|
+
const resolutions = config.resolutions || override.resolutions?.knownIssuesPath !== undefined
|
|
211
|
+
? { ...(config.resolutions ?? { rules: [] }), knownIssuesPath }
|
|
212
|
+
: undefined;
|
|
209
213
|
const output = resolve(override.output ?? config.output ?? "./allure-report");
|
|
210
|
-
const
|
|
214
|
+
const cwd = resolve(override.cwd ?? process.cwd());
|
|
211
215
|
const variables = config.variables ?? {};
|
|
212
216
|
const resultsDir = normalizeResultsDir(config.resultsDir);
|
|
213
217
|
let pluginInstances = [];
|
|
@@ -233,13 +237,12 @@ export const resolveConfig = async (config, override = {}) => {
|
|
|
233
237
|
}
|
|
234
238
|
return {
|
|
235
239
|
name,
|
|
240
|
+
cwd,
|
|
236
241
|
output,
|
|
237
242
|
open,
|
|
238
243
|
port,
|
|
239
244
|
hideLabels,
|
|
240
|
-
|
|
241
|
-
known,
|
|
242
|
-
knownIssues: config.knownIssues,
|
|
245
|
+
resolutions,
|
|
243
246
|
environment,
|
|
244
247
|
allowedEnvironments,
|
|
245
248
|
variables,
|
|
@@ -251,6 +254,7 @@ export const resolveConfig = async (config, override = {}) => {
|
|
|
251
254
|
plugins: pluginInstances,
|
|
252
255
|
defaultLabels: config.defaultLabels ?? {},
|
|
253
256
|
qualityGate: config.qualityGate,
|
|
257
|
+
performance: config.performance,
|
|
254
258
|
allureService: config.allureService
|
|
255
259
|
? {
|
|
256
260
|
accessToken: config.allureService.accessToken,
|
|
@@ -289,7 +293,7 @@ export const readConfig = async (cwd = process.cwd(), configPath, override) => {
|
|
|
289
293
|
default:
|
|
290
294
|
config = DEFAULT_CONFIG;
|
|
291
295
|
}
|
|
292
|
-
const fullConfig = await resolveConfig(config, override);
|
|
296
|
+
const fullConfig = await resolveConfig(config, { ...override, cwd });
|
|
293
297
|
return fullConfig;
|
|
294
298
|
};
|
|
295
299
|
export const readRawConfig = async (cwd = process.cwd(), configPath) => {
|
package/dist/history.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type AllureHistory, type HistoryDataPoint, type TestCase, type TestResult } from "@allurereport/core-api";
|
|
2
|
-
export declare const createHistory: (reportUuid: string, reportName: string | undefined, testCases: TestCase[], testResults: TestResult[], remoteUrl?: string) => HistoryDataPoint;
|
|
1
|
+
import { type AllureHistory, type HistoryDataPoint, type MetricSample, type TestCase, type TestResult } from "@allurereport/core-api";
|
|
2
|
+
export declare const createHistory: (reportUuid: string, reportName: string | undefined, testCases: TestCase[], testResults: TestResult[], remoteUrl?: string, metrics?: MetricSample[]) => HistoryDataPoint;
|
|
3
3
|
export declare class AllureLocalHistory implements AllureHistory {
|
|
4
4
|
#private;
|
|
5
5
|
private readonly params;
|
package/dist/history.js
CHANGED
|
@@ -43,7 +43,17 @@ const createHistoryItems = (testResults, remoteUrl) => {
|
|
|
43
43
|
return acc;
|
|
44
44
|
}, {});
|
|
45
45
|
};
|
|
46
|
-
|
|
46
|
+
const metricsToHistoryValues = (metrics) => {
|
|
47
|
+
const grouped = new Map();
|
|
48
|
+
for (const metric of metrics) {
|
|
49
|
+
if (!metric.key || !Number.isFinite(metric.value)) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
grouped.set(metric.key, [...(grouped.get(metric.key) ?? []), metric.value]);
|
|
53
|
+
}
|
|
54
|
+
return Object.fromEntries([...grouped.entries()].map(([key, values]) => [key, values.reduce((acc, value) => acc + value, 0) / values.length]));
|
|
55
|
+
};
|
|
56
|
+
export const createHistory = (reportUuid, reportName = "Allure Report", testCases, testResults, remoteUrl = "", metrics = []) => {
|
|
47
57
|
const knownTestCaseIds = testCases.map((tc) => tc.id);
|
|
48
58
|
return {
|
|
49
59
|
uuid: reportUuid,
|
|
@@ -51,7 +61,7 @@ export const createHistory = (reportUuid, reportName = "Allure Report", testCase
|
|
|
51
61
|
timestamp: new Date().getTime(),
|
|
52
62
|
knownTestCaseIds,
|
|
53
63
|
testResults: createHistoryItems(testResults, remoteUrl),
|
|
54
|
-
metrics:
|
|
64
|
+
metrics: metricsToHistoryValues(metrics),
|
|
55
65
|
url: remoteUrl,
|
|
56
66
|
};
|
|
57
67
|
};
|
package/dist/index.d.ts
CHANGED
|
@@ -7,8 +7,8 @@ export * from "./utils/flaky.js";
|
|
|
7
7
|
export * from "./utils/environment.js";
|
|
8
8
|
export * from "./utils/resultsDir.js";
|
|
9
9
|
export * from "./history.js";
|
|
10
|
-
export * from "./
|
|
10
|
+
export * from "./resolutions.js";
|
|
11
11
|
export { resolveConfig, readConfig, readRawConfig, getPluginInstance } from "./config.js";
|
|
12
12
|
export * from "./report.js";
|
|
13
13
|
export * from "./plugin.js";
|
|
14
|
-
export { QualityGateState, qualityGateDefaultRules, maxFailuresRule, minTestsCountRule, successRateRule, maxDurationRule, allTestsContainEnvRule, environmentsTestedRule, convertQualityGateResultsToTestErrors, stringifyQualityGateResults, } from "./qualityGate/index.js";
|
|
14
|
+
export { QualityGateState, qualityGateDefaultRules, maxFailuresRule, minTestsCountRule, successRateRule, maxDurationRule, allTestsContainEnvRule, environmentsTestedRule, metricMaxRule, metricMinRule, metricMaxDeltaRule, metricMaxDeltaPercentRule, convertQualityGateResultsToTestErrors, stringifyQualityGateResults, } from "./qualityGate/index.js";
|
package/dist/index.js
CHANGED
|
@@ -6,8 +6,8 @@ export * from "./utils/flaky.js";
|
|
|
6
6
|
export * from "./utils/environment.js";
|
|
7
7
|
export * from "./utils/resultsDir.js";
|
|
8
8
|
export * from "./history.js";
|
|
9
|
-
export * from "./
|
|
9
|
+
export * from "./resolutions.js";
|
|
10
10
|
export { resolveConfig, readConfig, readRawConfig, getPluginInstance } from "./config.js";
|
|
11
11
|
export * from "./report.js";
|
|
12
12
|
export * from "./plugin.js";
|
|
13
|
-
export { QualityGateState, qualityGateDefaultRules, maxFailuresRule, minTestsCountRule, successRateRule, maxDurationRule, allTestsContainEnvRule, environmentsTestedRule, convertQualityGateResultsToTestErrors, stringifyQualityGateResults, } from "./qualityGate/index.js";
|
|
13
|
+
export { QualityGateState, qualityGateDefaultRules, maxFailuresRule, minTestsCountRule, successRateRule, maxDurationRule, allTestsContainEnvRule, environmentsTestedRule, metricMaxRule, metricMinRule, metricMaxDeltaRule, metricMaxDeltaPercentRule, convertQualityGateResultsToTestErrors, stringifyQualityGateResults, } from "./qualityGate/index.js";
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
import type { QualityGateConfig, QualityGateValidationResult } from "@allurereport/plugin-api";
|
|
1
|
+
import { type MetricSample, type TestError, type TestResult } from "@allurereport/core-api";
|
|
2
|
+
import type { QualityGateConfig, QualityGateMetricHistoryPoint, QualityGateValidationResult } from "@allurereport/plugin-api";
|
|
3
3
|
export declare const stringifyQualityGateResults: (results: QualityGateValidationResult[]) => string;
|
|
4
4
|
export declare const convertQualityGateResultsToTestErrors: (results: QualityGateValidationResult[]) => TestError[];
|
|
5
5
|
export declare class QualityGateState {
|
|
6
6
|
#private;
|
|
7
|
-
setResult(rule: string, value:
|
|
8
|
-
getResult(rule: string):
|
|
7
|
+
setResult(rule: string, value: unknown, testResults?: string[]): void;
|
|
8
|
+
getResult(rule: string): unknown;
|
|
9
9
|
getTestResults(rule: string): string[];
|
|
10
10
|
markTestResultsProcessed(testResults: TestResult[]): void;
|
|
11
11
|
isTestResultProcessed(trId: string): boolean;
|
|
@@ -16,7 +16,8 @@ export declare class QualityGate {
|
|
|
16
16
|
validate(payload: {
|
|
17
17
|
state?: QualityGateState;
|
|
18
18
|
trs: TestResult[];
|
|
19
|
-
|
|
19
|
+
metrics?: MetricSample[];
|
|
20
|
+
previousHistory?: QualityGateMetricHistoryPoint[];
|
|
20
21
|
environment?: string;
|
|
21
22
|
}): Promise<{
|
|
22
23
|
fastFailed: boolean;
|
|
@@ -5,6 +5,7 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
5
5
|
};
|
|
6
6
|
var _QualityGateState_state, _QualityGateState_processedTestResultIds;
|
|
7
7
|
import { gray, red } from "yoctocolors";
|
|
8
|
+
import { isIgnoredFailure } from "../resolutions.js";
|
|
8
9
|
import { qualityGateDefaultRules } from "./rules.js";
|
|
9
10
|
export const stringifyQualityGateResults = (results) => {
|
|
10
11
|
if (results.length === 0) {
|
|
@@ -58,10 +59,10 @@ export class QualityGate {
|
|
|
58
59
|
this.config = config;
|
|
59
60
|
}
|
|
60
61
|
async validate(payload) {
|
|
61
|
-
const { state, trs,
|
|
62
|
+
const { state, trs, metrics = [], previousHistory = [], environment } = payload;
|
|
62
63
|
const trsToValidateById = new Map();
|
|
63
64
|
for (const tr of trs) {
|
|
64
|
-
if (tr.isRetry || state?.isTestResultProcessed(tr.id)) {
|
|
65
|
+
if (tr.isRetry || isIgnoredFailure(tr) || state?.isTestResultProcessed(tr.id)) {
|
|
65
66
|
continue;
|
|
66
67
|
}
|
|
67
68
|
trsToValidateById.set(tr.id, tr);
|
|
@@ -97,7 +98,8 @@ export class QualityGate {
|
|
|
97
98
|
setResult: (value, testResults) => state?.setResult?.(ruleId, value, testResults),
|
|
98
99
|
},
|
|
99
100
|
expected,
|
|
100
|
-
|
|
101
|
+
metrics,
|
|
102
|
+
previousHistory,
|
|
101
103
|
environment,
|
|
102
104
|
});
|
|
103
105
|
if (result.success) {
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import { type QualityGateRule } from "@allurereport/plugin-api";
|
|
2
|
+
type MetricRuleConfig = {
|
|
3
|
+
key: string;
|
|
4
|
+
value: number;
|
|
5
|
+
title?: string;
|
|
6
|
+
unit?: string;
|
|
7
|
+
};
|
|
2
8
|
export declare const maxFailuresRule: QualityGateRule<number>;
|
|
3
9
|
export declare const minTestsCountRule: QualityGateRule<number>;
|
|
4
|
-
export declare const successRateRule: QualityGateRule<number
|
|
5
|
-
totalCount: number;
|
|
6
|
-
unknownCount: number;
|
|
7
|
-
passedCount: number;
|
|
8
|
-
}>;
|
|
10
|
+
export declare const successRateRule: QualityGateRule<number>;
|
|
9
11
|
export declare const maxDurationRule: QualityGateRule<number>;
|
|
10
12
|
export declare const allTestsContainEnvRule: QualityGateRule<string, number>;
|
|
11
13
|
export declare const environmentsTestedRule: QualityGateRule<string[]>;
|
|
12
|
-
export declare const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
export declare const metricMaxRule: QualityGateRule<MetricRuleConfig>;
|
|
15
|
+
export declare const metricMinRule: QualityGateRule<MetricRuleConfig>;
|
|
16
|
+
export declare const metricMaxDeltaRule: QualityGateRule<MetricRuleConfig>;
|
|
17
|
+
export declare const metricMaxDeltaPercentRule: QualityGateRule<MetricRuleConfig>;
|
|
18
|
+
export declare const qualityGateDefaultRules: (QualityGateRule<number> | QualityGateRule<string, number> | QualityGateRule<string[]> | QualityGateRule<MetricRuleConfig>)[];
|
|
19
|
+
export {};
|
|
@@ -1,12 +1,81 @@
|
|
|
1
1
|
import { filterSuccessful, filterUnsuccessful } from "@allurereport/core-api";
|
|
2
2
|
import { bold } from "yoctocolors";
|
|
3
|
+
const numberStateValue = (value) => (Number.isFinite(value) ? Number(value) : 0);
|
|
4
|
+
const stringArrayStateValue = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
5
|
+
const successRateStateValue = (value) => {
|
|
6
|
+
if (typeof value !== "object" || value === null) {
|
|
7
|
+
return { totalCount: 0, passedCount: 0 };
|
|
8
|
+
}
|
|
9
|
+
const state = value;
|
|
10
|
+
return {
|
|
11
|
+
totalCount: numberStateValue(state.totalCount),
|
|
12
|
+
passedCount: numberStateValue(state.passedCount),
|
|
13
|
+
};
|
|
14
|
+
};
|
|
15
|
+
const metricValues = (metrics, key) => metrics
|
|
16
|
+
.map((metric) => (metric.key === key ? metric.value : undefined))
|
|
17
|
+
.filter((value) => Number.isFinite(value));
|
|
18
|
+
const metricAverage = (metrics, key) => {
|
|
19
|
+
const values = metricValues(metrics, key);
|
|
20
|
+
if (values.length === 0) {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
return values.reduce((acc, value) => acc + value, 0) / values.length;
|
|
24
|
+
};
|
|
25
|
+
const previousMetricValue = (history, key) => {
|
|
26
|
+
const previousHistory = [...history].sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0));
|
|
27
|
+
for (const historyPoint of previousHistory) {
|
|
28
|
+
const value = historyPoint.metrics?.[key];
|
|
29
|
+
if (Number.isFinite(value)) {
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return undefined;
|
|
34
|
+
};
|
|
35
|
+
const metricActual = (metrics, key, value) => {
|
|
36
|
+
const sample = metrics.find((metric) => metric.key === key);
|
|
37
|
+
return {
|
|
38
|
+
key,
|
|
39
|
+
value,
|
|
40
|
+
...(sample?.title ? { title: sample.title } : {}),
|
|
41
|
+
...(sample?.unit ? { unit: sample.unit } : {}),
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
const formatMetricValue = (value, unit) => {
|
|
45
|
+
if (!Number.isFinite(value)) {
|
|
46
|
+
return "n/a";
|
|
47
|
+
}
|
|
48
|
+
const finiteValue = value;
|
|
49
|
+
const formatted = Number.isInteger(finiteValue) ? String(finiteValue) : String(Number(finiteValue.toFixed(3)));
|
|
50
|
+
return unit ? `${formatted} ${unit}` : formatted;
|
|
51
|
+
};
|
|
52
|
+
const metricRuleTitle = (actual) => actual.title ?? actual.key;
|
|
53
|
+
const metricRuleUnit = (actual, expected) => actual.unit ?? expected.unit;
|
|
54
|
+
const missingMetricResult = (actual) => ({
|
|
55
|
+
success: false,
|
|
56
|
+
actual,
|
|
57
|
+
testResults: [],
|
|
58
|
+
});
|
|
59
|
+
const metricRuleResult = (success, actual) => ({
|
|
60
|
+
success,
|
|
61
|
+
actual,
|
|
62
|
+
testResults: [],
|
|
63
|
+
});
|
|
64
|
+
const metricDeltaWithinThreshold = (delta, threshold, better) => {
|
|
65
|
+
if (better === "lower") {
|
|
66
|
+
return delta <= threshold;
|
|
67
|
+
}
|
|
68
|
+
if (better === "higher") {
|
|
69
|
+
return -delta <= threshold;
|
|
70
|
+
}
|
|
71
|
+
return Math.abs(delta) <= threshold;
|
|
72
|
+
};
|
|
3
73
|
export const maxFailuresRule = {
|
|
4
74
|
rule: "maxFailures",
|
|
5
75
|
message: ({ actual, expected }) => `The number of failed tests ${bold(String(actual))} exceeds the allowed threshold value ${bold(String(expected))}`,
|
|
6
76
|
validate: async ({ trs, expected, state }) => {
|
|
7
|
-
const previous = state.getResult()
|
|
8
|
-
const
|
|
9
|
-
const failedTrs = unknown.filter(filterUnsuccessful);
|
|
77
|
+
const previous = numberStateValue(state.getResult());
|
|
78
|
+
const failedTrs = trs.filter(filterUnsuccessful);
|
|
10
79
|
const testResults = failedTrs.map((tr) => tr.id);
|
|
11
80
|
const actual = previous + failedTrs.length;
|
|
12
81
|
state.setResult(actual, testResults);
|
|
@@ -21,7 +90,7 @@ export const minTestsCountRule = {
|
|
|
21
90
|
rule: "minTestsCount",
|
|
22
91
|
message: ({ actual, expected }) => `The total number of tests ${bold(String(actual))} is less than the expected threshold value ${bold(String(expected))}`,
|
|
23
92
|
validate: async ({ trs, expected, state }) => {
|
|
24
|
-
const actual = trs.length + (state.getResult()
|
|
93
|
+
const actual = trs.length + numberStateValue(state.getResult());
|
|
25
94
|
state.setResult(actual, []);
|
|
26
95
|
return {
|
|
27
96
|
success: actual >= expected,
|
|
@@ -34,16 +103,14 @@ export const successRateRule = {
|
|
|
34
103
|
rule: "successRate",
|
|
35
104
|
message: ({ actual, expected }) => `Success rate ${bold(String(actual))} is less, than expected ${bold(String(expected))}`,
|
|
36
105
|
validate: async ({ trs, expected, state }) => {
|
|
37
|
-
const previous = state.getResult()
|
|
38
|
-
const
|
|
39
|
-
const
|
|
40
|
-
const notPassedTrs = unknown.filter((tr) => !filterSuccessful(tr));
|
|
106
|
+
const previous = successRateStateValue(state.getResult());
|
|
107
|
+
const passedTrs = trs.filter(filterSuccessful);
|
|
108
|
+
const notPassedTrs = trs.filter((tr) => !filterSuccessful(tr));
|
|
41
109
|
const totalCount = previous.totalCount + trs.length;
|
|
42
|
-
const unknownCount = previous.unknownCount + unknown.length;
|
|
43
110
|
const passedCount = previous.passedCount + passedTrs.length;
|
|
44
111
|
const testResults = notPassedTrs.map((tr) => tr.id);
|
|
45
|
-
const rate = totalCount === 0 ? 0 :
|
|
46
|
-
state.setResult({ totalCount,
|
|
112
|
+
const rate = totalCount === 0 ? 0 : passedCount / totalCount;
|
|
113
|
+
state.setResult({ totalCount, passedCount }, testResults);
|
|
47
114
|
return {
|
|
48
115
|
success: rate >= expected,
|
|
49
116
|
actual: rate,
|
|
@@ -55,7 +122,7 @@ export const maxDurationRule = {
|
|
|
55
122
|
rule: "maxDuration",
|
|
56
123
|
message: ({ actual, expected }) => `Maximum duration of some tests exceed the defined limit; actual ${bold(String(actual))}, expected ${bold(String(expected))}`,
|
|
57
124
|
validate: async ({ trs, expected, state }) => {
|
|
58
|
-
const previous = state.getResult()
|
|
125
|
+
const previous = numberStateValue(state.getResult());
|
|
59
126
|
const actual = Math.max(previous, ...trs.map((tr) => tr.duration ?? 0));
|
|
60
127
|
const tooLongTrs = trs.filter((tr) => (tr.duration ?? 0) > expected);
|
|
61
128
|
const testResults = tooLongTrs.map((tr) => tr.id);
|
|
@@ -69,9 +136,9 @@ export const maxDurationRule = {
|
|
|
69
136
|
};
|
|
70
137
|
export const allTestsContainEnvRule = {
|
|
71
138
|
rule: "allTestsContainEnv",
|
|
72
|
-
message: ({ actual, expected }) => `Not all tests contain the required "${bold(expected)}" environment, ${bold(actual)} tests have different or missing environment`,
|
|
139
|
+
message: ({ actual, expected }) => `Not all tests contain the required "${bold(expected)}" environment, ${bold(String(actual))} tests have different or missing environment`,
|
|
73
140
|
validate: async ({ trs, expected, state }) => {
|
|
74
|
-
const previous = state.getResult()
|
|
141
|
+
const previous = numberStateValue(state.getResult());
|
|
75
142
|
const testsWithoutEnv = trs.filter((tr) => (tr.environment ?? "") !== expected);
|
|
76
143
|
const testResults = testsWithoutEnv.map((tr) => tr.id);
|
|
77
144
|
const actual = testsWithoutEnv.length + previous;
|
|
@@ -87,7 +154,7 @@ export const environmentsTestedRule = {
|
|
|
87
154
|
rule: "environmentsTested",
|
|
88
155
|
message: ({ actual, expected }) => `The following environments were not tested: "${actual.join('", "')}"; expected all of: "${expected.join('", "')}"`,
|
|
89
156
|
validate: async ({ trs, expected, state }) => {
|
|
90
|
-
const previouslyTested = new Set(state.getResult()
|
|
157
|
+
const previouslyTested = new Set(stringArrayStateValue(state.getResult()));
|
|
91
158
|
const batchTested = trs.map((tr) => tr.environment).filter((env) => env != null && env !== "");
|
|
92
159
|
const testedEnvs = new Set([...previouslyTested, ...batchTested]);
|
|
93
160
|
state.setResult([...testedEnvs], []);
|
|
@@ -99,6 +166,62 @@ export const environmentsTestedRule = {
|
|
|
99
166
|
};
|
|
100
167
|
},
|
|
101
168
|
};
|
|
169
|
+
export const metricMaxRule = {
|
|
170
|
+
rule: "metricMax",
|
|
171
|
+
message: ({ actual, expected }) => `${bold(metricRuleTitle(actual))} ${bold(formatMetricValue(actual.value, metricRuleUnit(actual, expected)))} exceeds the allowed maximum ${bold(formatMetricValue(expected.value, metricRuleUnit(actual, expected)))}`,
|
|
172
|
+
validate: async ({ metrics = [], expected }) => {
|
|
173
|
+
const value = metricAverage(metrics, expected.key);
|
|
174
|
+
const actual = metricActual(metrics, expected.key, value ?? Number.NaN);
|
|
175
|
+
return value === undefined ? missingMetricResult(actual) : metricRuleResult(value <= expected.value, actual);
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
export const metricMinRule = {
|
|
179
|
+
rule: "metricMin",
|
|
180
|
+
message: ({ actual, expected }) => `${bold(metricRuleTitle(actual))} ${bold(formatMetricValue(actual.value, metricRuleUnit(actual, expected)))} is below the required minimum ${bold(formatMetricValue(expected.value, metricRuleUnit(actual, expected)))}`,
|
|
181
|
+
validate: async ({ metrics = [], expected }) => {
|
|
182
|
+
const value = metricAverage(metrics, expected.key);
|
|
183
|
+
const actual = metricActual(metrics, expected.key, value ?? Number.NaN);
|
|
184
|
+
return value === undefined ? missingMetricResult(actual) : metricRuleResult(value >= expected.value, actual);
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
export const metricMaxDeltaRule = {
|
|
188
|
+
rule: "metricMaxDelta",
|
|
189
|
+
message: ({ actual, expected }) => `${bold(metricRuleTitle(actual))} changed by ${bold(formatMetricValue(actual.value, metricRuleUnit(actual, expected)))}, which exceeds ${bold(formatMetricValue(expected.value, metricRuleUnit(actual, expected)))}`,
|
|
190
|
+
validate: async ({ metrics = [], previousHistory = [], expected }) => {
|
|
191
|
+
const current = metricAverage(metrics, expected.key);
|
|
192
|
+
if (current === undefined) {
|
|
193
|
+
return missingMetricResult(metricActual(metrics, expected.key, Number.NaN));
|
|
194
|
+
}
|
|
195
|
+
const previous = previousMetricValue(previousHistory, expected.key);
|
|
196
|
+
if (previous === undefined) {
|
|
197
|
+
return metricRuleResult(true, metricActual(metrics, expected.key, 0));
|
|
198
|
+
}
|
|
199
|
+
const value = current - previous;
|
|
200
|
+
const actual = metricActual(metrics, expected.key, value);
|
|
201
|
+
const better = metrics.find((metric) => metric.key === expected.key)?.better;
|
|
202
|
+
return metricRuleResult(metricDeltaWithinThreshold(value, expected.value, better), actual);
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
export const metricMaxDeltaPercentRule = {
|
|
206
|
+
rule: "metricMaxDeltaPercent",
|
|
207
|
+
message: ({ actual, expected }) => `${bold(metricRuleTitle(actual))} changed by ${bold(formatMetricValue(actual.value, "%"))}, which exceeds ${bold(formatMetricValue(expected.value, "%"))}`,
|
|
208
|
+
validate: async ({ metrics = [], previousHistory = [], expected }) => {
|
|
209
|
+
const current = metricAverage(metrics, expected.key);
|
|
210
|
+
if (current === undefined) {
|
|
211
|
+
return missingMetricResult(metricActual(metrics, expected.key, Number.NaN));
|
|
212
|
+
}
|
|
213
|
+
const previous = previousMetricValue(previousHistory, expected.key);
|
|
214
|
+
if (previous === undefined) {
|
|
215
|
+
return metricRuleResult(true, metricActual(metrics, expected.key, 0));
|
|
216
|
+
}
|
|
217
|
+
const value = previous !== 0 ? ((current - previous) / Math.abs(previous)) * 100 : Number.NaN;
|
|
218
|
+
const actual = metricActual(metrics, expected.key, value);
|
|
219
|
+
const better = metrics.find((metric) => metric.key === expected.key)?.better;
|
|
220
|
+
return Number.isFinite(value)
|
|
221
|
+
? metricRuleResult(metricDeltaWithinThreshold(value, expected.value, better), actual)
|
|
222
|
+
: missingMetricResult(actual);
|
|
223
|
+
},
|
|
224
|
+
};
|
|
102
225
|
export const qualityGateDefaultRules = [
|
|
103
226
|
maxFailuresRule,
|
|
104
227
|
minTestsCountRule,
|
|
@@ -106,4 +229,8 @@ export const qualityGateDefaultRules = [
|
|
|
106
229
|
maxDurationRule,
|
|
107
230
|
allTestsContainEnvRule,
|
|
108
231
|
environmentsTestedRule,
|
|
232
|
+
metricMaxRule,
|
|
233
|
+
metricMinRule,
|
|
234
|
+
metricMaxDeltaRule,
|
|
235
|
+
metricMaxDeltaPercentRule,
|
|
109
236
|
];
|
package/dist/report.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { TestResult } from "@allurereport/core-api";
|
|
2
2
|
import { type ResultFile } from "@allurereport/plugin-api";
|
|
3
3
|
import type { FullConfig } from "./api.js";
|
|
4
4
|
import { type QualityGateState } from "./qualityGate/index.js";
|
|
@@ -19,7 +19,6 @@ export declare class AllureReport {
|
|
|
19
19
|
readResult: (data: ResultFile) => Promise<void>;
|
|
20
20
|
validate: (params: {
|
|
21
21
|
trs: TestResult[];
|
|
22
|
-
knownIssues: KnownTestFailure[];
|
|
23
22
|
state?: QualityGateState;
|
|
24
23
|
environment?: string;
|
|
25
24
|
}) => Promise<{
|