executable-stories-formatters 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -1
- package/dist/adapters.d.cts +1 -1
- package/dist/adapters.d.ts +1 -1
- package/dist/cli.js +810 -6
- package/dist/cli.js.map +1 -1
- package/dist/{index-DyeUWfYK.d.cts → index-C4QO-SVT.d.cts} +33 -8
- package/dist/{index-DyeUWfYK.d.ts → index-C4QO-SVT.d.ts} +33 -8
- package/dist/index.cjs +900 -17
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +403 -5
- package/dist/index.d.ts +403 -5
- package/dist/index.js +881 -17
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schemas/README.md +11 -2
package/dist/index.js
CHANGED
|
@@ -2561,6 +2561,21 @@ body {
|
|
|
2561
2561
|
}
|
|
2562
2562
|
}
|
|
2563
2563
|
|
|
2564
|
+
/* ============================================================================
|
|
2565
|
+
History metric badges
|
|
2566
|
+
============================================================================ */
|
|
2567
|
+
.badge { display: inline-block; padding: 2px 6px; border-radius: 4px; font-size: 0.75em; font-weight: 600; margin-left: 4px; vertical-align: middle; }
|
|
2568
|
+
.badge-grade { color: #fff; }
|
|
2569
|
+
.badge-grade-A { background: var(--success); }
|
|
2570
|
+
.badge-grade-B { background: #2196F3; }
|
|
2571
|
+
.badge-grade-C { background: #FF9800; }
|
|
2572
|
+
.badge-grade-D { background: #f44336; }
|
|
2573
|
+
.badge-grade-F { background: #9E0000; }
|
|
2574
|
+
.badge-flaky { background: #FF9800; color: #fff; }
|
|
2575
|
+
.badge-perf { font-size: 0.7em; }
|
|
2576
|
+
.badge-perf-improving { color: var(--success); }
|
|
2577
|
+
.badge-perf-regressing { color: var(--error); }
|
|
2578
|
+
|
|
2564
2579
|
`;
|
|
2565
2580
|
|
|
2566
2581
|
// src/formatters/html/renderers/status.ts
|
|
@@ -2594,7 +2609,22 @@ function renderMetaInfo(args, deps) {
|
|
|
2594
2609
|
items.push(`<dt>Git:</dt><dd>${deps.escapeHtml(shortSha)}</dd>`);
|
|
2595
2610
|
}
|
|
2596
2611
|
if (args.ciName) {
|
|
2597
|
-
|
|
2612
|
+
if (args.ciUrl && args.ciBuildNumber) {
|
|
2613
|
+
items.push(
|
|
2614
|
+
`<dt>CI:</dt><dd>${deps.escapeHtml(args.ciName)} <a href="${deps.escapeHtml(args.ciUrl)}">#${deps.escapeHtml(args.ciBuildNumber)}</a></dd>`
|
|
2615
|
+
);
|
|
2616
|
+
} else {
|
|
2617
|
+
items.push(`<dt>CI:</dt><dd>${deps.escapeHtml(args.ciName)}</dd>`);
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
if (args.ciBranch) {
|
|
2621
|
+
items.push(`<dt>Branch:</dt><dd>${deps.escapeHtml(args.ciBranch)}</dd>`);
|
|
2622
|
+
}
|
|
2623
|
+
if (args.ciCommitSha) {
|
|
2624
|
+
const shortSha = args.ciCommitSha.length > 7 ? args.ciCommitSha.slice(0, 7) : args.ciCommitSha;
|
|
2625
|
+
items.push(
|
|
2626
|
+
`<dt>Commit:</dt><dd title="${deps.escapeHtml(args.ciCommitSha)}">${deps.escapeHtml(shortSha)}</dd>`
|
|
2627
|
+
);
|
|
2598
2628
|
}
|
|
2599
2629
|
return `<dl class="meta-info">${items.join("")}</dl>`;
|
|
2600
2630
|
}
|
|
@@ -2845,6 +2875,14 @@ function highlightStepParams(text, deps) {
|
|
|
2845
2875
|
return result;
|
|
2846
2876
|
}
|
|
2847
2877
|
|
|
2878
|
+
// src/history/sample-policy.ts
|
|
2879
|
+
var MIN_PERF_SAMPLES = 6;
|
|
2880
|
+
var MIN_METRIC_SAMPLES = 5;
|
|
2881
|
+
var MIN_FLAKINESS_SAMPLES = 3;
|
|
2882
|
+
function hasSufficientHistory(entries, min) {
|
|
2883
|
+
return entries.length >= min;
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2848
2886
|
// src/formatters/html/renderers/scenario.ts
|
|
2849
2887
|
function renderScenario(args, deps) {
|
|
2850
2888
|
const { tc } = args;
|
|
@@ -2865,6 +2903,19 @@ function renderScenario(args, deps) {
|
|
|
2865
2903
|
traceBadge = `<span class="tag trace-tag" title="${deps.escapeHtml(otelMeta.traceId)}">${deps.escapeHtml(shortId)}\u2026</span>`;
|
|
2866
2904
|
}
|
|
2867
2905
|
}
|
|
2906
|
+
let metricBadges = "";
|
|
2907
|
+
const { metrics } = args;
|
|
2908
|
+
if (metrics && metrics.sampleSize >= MIN_METRIC_SAMPLES) {
|
|
2909
|
+
const grade = metrics.stabilityGrade;
|
|
2910
|
+
metricBadges += `<span class="badge badge-grade badge-grade-${grade}" title="Pass rate: ${(metrics.passRate * 100).toFixed(0)}% (${metrics.sampleSize} runs)">${grade}</span>`;
|
|
2911
|
+
if (metrics.flakinessLevel !== "stable") {
|
|
2912
|
+
metricBadges += `<span class="badge badge-flaky">${metrics.flakinessLevel}</span>`;
|
|
2913
|
+
}
|
|
2914
|
+
if (metrics.performanceTrend !== "stable") {
|
|
2915
|
+
const arrow = metrics.performanceTrend === "improving" ? "\u2191" : "\u2193";
|
|
2916
|
+
metricBadges += `<span class="badge badge-perf badge-perf-${metrics.performanceTrend}">${arrow} ${metrics.performanceTrend}</span>`;
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2868
2919
|
const storyDocs = deps.renderDocs(tc.story.docs, "story-docs");
|
|
2869
2920
|
const steps = deps.renderSteps(
|
|
2870
2921
|
{ steps: tc.story.steps, stepResults: tc.stepResults },
|
|
@@ -2899,7 +2950,7 @@ function renderScenario(args, deps) {
|
|
|
2899
2950
|
<span class="status-icon ${statusClass}">${statusIcon}</span>
|
|
2900
2951
|
<span class="scenario-name">${deps.escapeHtml(tc.story.scenario)}</span>
|
|
2901
2952
|
</div>
|
|
2902
|
-
<div class="scenario-meta">${tags}${traceBadge}</div>
|
|
2953
|
+
<div class="scenario-meta">${tags}${traceBadge}${metricBadges}</div>
|
|
2903
2954
|
</div>
|
|
2904
2955
|
<span class="scenario-duration">${duration}</span>
|
|
2905
2956
|
</div>
|
|
@@ -3111,7 +3162,12 @@ function renderFeature(args, deps) {
|
|
|
3111
3162
|
const featureName = suitePaths.length > 0 && suitePaths[0].length > 0 ? suitePaths[0][0] : file.split("/").pop()?.replace(/\.[^.]+$/, "") ?? file;
|
|
3112
3163
|
const collapsedClass = deps.startCollapsed ? " collapsed" : "";
|
|
3113
3164
|
const ariaExpanded = !deps.startCollapsed;
|
|
3114
|
-
const scenarios = testCases.map(
|
|
3165
|
+
const scenarios = testCases.map(
|
|
3166
|
+
(tc) => deps.renderScenario(
|
|
3167
|
+
{ tc, metrics: args.metricsMap?.get(tc.id) },
|
|
3168
|
+
deps.scenarioDeps
|
|
3169
|
+
)
|
|
3170
|
+
).join("\n");
|
|
3115
3171
|
return `
|
|
3116
3172
|
<div class="feature${collapsedClass}">
|
|
3117
3173
|
<div class="feature-header" role="button" tabindex="0" aria-expanded="${ariaExpanded}">
|
|
@@ -3156,7 +3212,11 @@ function buildBody(args, deps) {
|
|
|
3156
3212
|
durationMs: run.durationMs,
|
|
3157
3213
|
packageVersion: run.packageVersion,
|
|
3158
3214
|
gitSha: run.gitSha,
|
|
3159
|
-
ciName: run.ci?.name
|
|
3215
|
+
ciName: run.ci?.name,
|
|
3216
|
+
ciBranch: run.ci?.branch,
|
|
3217
|
+
ciUrl: run.ci?.url,
|
|
3218
|
+
ciCommitSha: run.ci?.commitSha,
|
|
3219
|
+
ciBuildNumber: run.ci?.buildNumber
|
|
3160
3220
|
},
|
|
3161
3221
|
deps.metaDeps
|
|
3162
3222
|
)
|
|
@@ -3185,7 +3245,10 @@ function buildBody(args, deps) {
|
|
|
3185
3245
|
const byFile = groupBy(run.testCases, (tc) => tc.sourceFile);
|
|
3186
3246
|
for (const [file, testCases] of byFile) {
|
|
3187
3247
|
parts.push(
|
|
3188
|
-
deps.renderFeature(
|
|
3248
|
+
deps.renderFeature(
|
|
3249
|
+
{ file, testCases, metricsMap: args.metricsMap },
|
|
3250
|
+
deps.featureDeps
|
|
3251
|
+
)
|
|
3189
3252
|
);
|
|
3190
3253
|
}
|
|
3191
3254
|
return parts.join("\n");
|
|
@@ -5657,45 +5720,102 @@ function clearVersionCache() {
|
|
|
5657
5720
|
|
|
5658
5721
|
// src/utils/ci-detect.ts
|
|
5659
5722
|
function detectCI4(env = process.env) {
|
|
5723
|
+
if (env.TF_BUILD === "True") {
|
|
5724
|
+
const branch = env.BUILD_SOURCEBRANCH?.replace(/^refs\/heads\//, "");
|
|
5725
|
+
const serverUri = env.SYSTEM_TEAMFOUNDATIONSERVERURI;
|
|
5726
|
+
const teamProject = env.SYSTEM_TEAMPROJECT;
|
|
5727
|
+
const buildId = env.BUILD_BUILDID;
|
|
5728
|
+
const url = serverUri && teamProject && buildId ? `${serverUri}${teamProject}/_build/results?buildId=${buildId}` : void 0;
|
|
5729
|
+
return {
|
|
5730
|
+
name: "azure",
|
|
5731
|
+
provider: "azure",
|
|
5732
|
+
buildNumber: buildId,
|
|
5733
|
+
url,
|
|
5734
|
+
branch,
|
|
5735
|
+
commitSha: env.BUILD_SOURCEVERSION,
|
|
5736
|
+
prNumber: env.SYSTEM_PULLREQUEST_PULLREQUESTID
|
|
5737
|
+
};
|
|
5738
|
+
}
|
|
5739
|
+
if (env.BUILDKITE === "true") {
|
|
5740
|
+
const prRaw = env.BUILDKITE_PULL_REQUEST;
|
|
5741
|
+
const prNumber = prRaw && prRaw !== "false" ? prRaw : void 0;
|
|
5742
|
+
return {
|
|
5743
|
+
name: "buildkite",
|
|
5744
|
+
provider: "buildkite",
|
|
5745
|
+
buildNumber: env.BUILDKITE_BUILD_NUMBER,
|
|
5746
|
+
url: env.BUILDKITE_BUILD_URL,
|
|
5747
|
+
branch: env.BUILDKITE_BRANCH,
|
|
5748
|
+
commitSha: env.BUILDKITE_COMMIT,
|
|
5749
|
+
prNumber
|
|
5750
|
+
};
|
|
5751
|
+
}
|
|
5660
5752
|
if (env.GITHUB_ACTIONS === "true") {
|
|
5661
5753
|
const url = env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID ? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` : void 0;
|
|
5754
|
+
const branch = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
5755
|
+
const prMatch = env.GITHUB_REF?.match(/^refs\/pull\/(\d+)\/(merge|head)$/);
|
|
5756
|
+
const prNumber = prMatch ? prMatch[1] : void 0;
|
|
5662
5757
|
return {
|
|
5663
5758
|
name: "github",
|
|
5759
|
+
provider: "github",
|
|
5664
5760
|
buildNumber: env.GITHUB_RUN_NUMBER,
|
|
5665
|
-
url
|
|
5761
|
+
url,
|
|
5762
|
+
branch,
|
|
5763
|
+
commitSha: env.GITHUB_SHA,
|
|
5764
|
+
prNumber
|
|
5765
|
+
};
|
|
5766
|
+
}
|
|
5767
|
+
if (env.GITLAB_CI === "true") {
|
|
5768
|
+
return {
|
|
5769
|
+
name: "gitlab",
|
|
5770
|
+
provider: "gitlab",
|
|
5771
|
+
buildNumber: env.CI_PIPELINE_IID,
|
|
5772
|
+
url: env.CI_PIPELINE_URL,
|
|
5773
|
+
branch: env.CI_COMMIT_REF_NAME,
|
|
5774
|
+
commitSha: env.CI_COMMIT_SHA,
|
|
5775
|
+
prNumber: env.CI_MERGE_REQUEST_IID
|
|
5666
5776
|
};
|
|
5667
5777
|
}
|
|
5668
5778
|
if (env.CIRCLECI === "true") {
|
|
5779
|
+
const prUrl = env.CIRCLE_PULL_REQUEST;
|
|
5780
|
+
const prMatch = prUrl?.match(/\/(\d+)$/);
|
|
5781
|
+
const prNumber = prMatch ? prMatch[1] : void 0;
|
|
5669
5782
|
return {
|
|
5670
5783
|
name: "circleci",
|
|
5784
|
+
provider: "circleci",
|
|
5671
5785
|
buildNumber: env.CIRCLE_BUILD_NUM,
|
|
5672
|
-
url: env.CIRCLE_BUILD_URL
|
|
5786
|
+
url: env.CIRCLE_BUILD_URL,
|
|
5787
|
+
branch: env.CIRCLE_BRANCH,
|
|
5788
|
+
commitSha: env.CIRCLE_SHA1,
|
|
5789
|
+
prNumber
|
|
5673
5790
|
};
|
|
5674
5791
|
}
|
|
5675
5792
|
if (env.JENKINS_URL !== void 0) {
|
|
5676
5793
|
return {
|
|
5677
5794
|
name: "jenkins",
|
|
5795
|
+
provider: "jenkins",
|
|
5678
5796
|
buildNumber: env.BUILD_NUMBER,
|
|
5679
|
-
url: env.BUILD_URL
|
|
5797
|
+
url: env.BUILD_URL,
|
|
5798
|
+
branch: env.GIT_BRANCH,
|
|
5799
|
+
commitSha: env.GIT_COMMIT
|
|
5680
5800
|
};
|
|
5681
5801
|
}
|
|
5682
5802
|
if (env.TRAVIS === "true") {
|
|
5803
|
+
const prRaw = env.TRAVIS_PULL_REQUEST;
|
|
5804
|
+
const prNumber = prRaw && prRaw !== "false" ? prRaw : void 0;
|
|
5683
5805
|
return {
|
|
5684
5806
|
name: "travis",
|
|
5807
|
+
provider: "travis",
|
|
5685
5808
|
buildNumber: env.TRAVIS_BUILD_NUMBER,
|
|
5686
|
-
url: env.TRAVIS_BUILD_WEB_URL
|
|
5687
|
-
|
|
5688
|
-
|
|
5689
|
-
|
|
5690
|
-
return {
|
|
5691
|
-
name: "gitlab",
|
|
5692
|
-
buildNumber: env.CI_PIPELINE_IID,
|
|
5693
|
-
url: env.CI_PIPELINE_URL
|
|
5809
|
+
url: env.TRAVIS_BUILD_WEB_URL,
|
|
5810
|
+
branch: env.TRAVIS_BRANCH,
|
|
5811
|
+
commitSha: env.TRAVIS_COMMIT,
|
|
5812
|
+
prNumber
|
|
5694
5813
|
};
|
|
5695
5814
|
}
|
|
5696
5815
|
if (env.CI === "true") {
|
|
5697
5816
|
return {
|
|
5698
|
-
name: "ci"
|
|
5817
|
+
name: "ci",
|
|
5818
|
+
provider: "unknown"
|
|
5699
5819
|
};
|
|
5700
5820
|
}
|
|
5701
5821
|
return void 0;
|
|
@@ -5726,6 +5846,731 @@ function resolveTraceUrl(template, traceId) {
|
|
|
5726
5846
|
return template.replace(/\{traceId\}/g, traceId);
|
|
5727
5847
|
}
|
|
5728
5848
|
|
|
5849
|
+
// src/notifiers/ansi-strip.ts
|
|
5850
|
+
function stripAnsi(text) {
|
|
5851
|
+
return text.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "");
|
|
5852
|
+
}
|
|
5853
|
+
|
|
5854
|
+
// src/notifiers/slack.ts
|
|
5855
|
+
function truncate(text, maxLen) {
|
|
5856
|
+
if (text.length <= maxLen) return text;
|
|
5857
|
+
return text.slice(0, maxLen - 3) + "...";
|
|
5858
|
+
}
|
|
5859
|
+
function formatDuration3(ms) {
|
|
5860
|
+
const seconds = ms / 1e3;
|
|
5861
|
+
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
|
5862
|
+
const minutes = Math.floor(seconds / 60);
|
|
5863
|
+
const remainingSeconds = seconds % 60;
|
|
5864
|
+
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
|
|
5865
|
+
}
|
|
5866
|
+
function buildSlackPayload(summary, maxFailedTests) {
|
|
5867
|
+
const allPassed = summary.failed === 0;
|
|
5868
|
+
const emoji = allPassed ? ":white_check_mark:" : ":x:";
|
|
5869
|
+
const statusText = allPassed ? "Passed" : "Failed";
|
|
5870
|
+
const blocks = [];
|
|
5871
|
+
blocks.push({
|
|
5872
|
+
type: "header",
|
|
5873
|
+
text: {
|
|
5874
|
+
type: "plain_text",
|
|
5875
|
+
text: `${emoji} Test Results: ${summary.passed} passed, ${summary.failed} failed`,
|
|
5876
|
+
emoji: true
|
|
5877
|
+
}
|
|
5878
|
+
});
|
|
5879
|
+
blocks.push({
|
|
5880
|
+
type: "section",
|
|
5881
|
+
fields: [
|
|
5882
|
+
{ type: "mrkdwn", text: `*Total:* ${summary.total}` },
|
|
5883
|
+
{ type: "mrkdwn", text: `*Passed:* ${summary.passed}` },
|
|
5884
|
+
{ type: "mrkdwn", text: `*Failed:* ${summary.failed}` },
|
|
5885
|
+
{ type: "mrkdwn", text: `*Skipped:* ${summary.skipped}` },
|
|
5886
|
+
{ type: "mrkdwn", text: `*Duration:* ${formatDuration3(summary.durationMs)}` },
|
|
5887
|
+
{ type: "mrkdwn", text: `*Status:* ${statusText}` }
|
|
5888
|
+
]
|
|
5889
|
+
});
|
|
5890
|
+
if (summary.failedTests.length > 0) {
|
|
5891
|
+
const displayedTests = summary.failedTests.slice(0, maxFailedTests);
|
|
5892
|
+
const lines = displayedTests.map((t) => {
|
|
5893
|
+
const name = t.name;
|
|
5894
|
+
if (t.error) {
|
|
5895
|
+
const cleanError = truncate(stripAnsi(t.error), 500);
|
|
5896
|
+
return `*${name}*
|
|
5897
|
+
\`\`\`${cleanError}\`\`\``;
|
|
5898
|
+
}
|
|
5899
|
+
return `*${name}*`;
|
|
5900
|
+
});
|
|
5901
|
+
let text = lines.join("\n\n");
|
|
5902
|
+
if (summary.failedTests.length > maxFailedTests) {
|
|
5903
|
+
text += `
|
|
5904
|
+
|
|
5905
|
+
_...and ${summary.failedTests.length - maxFailedTests} more_`;
|
|
5906
|
+
}
|
|
5907
|
+
blocks.push({
|
|
5908
|
+
type: "section",
|
|
5909
|
+
text: {
|
|
5910
|
+
type: "mrkdwn",
|
|
5911
|
+
text
|
|
5912
|
+
}
|
|
5913
|
+
});
|
|
5914
|
+
}
|
|
5915
|
+
if (summary.ci) {
|
|
5916
|
+
const elements = [];
|
|
5917
|
+
if (summary.ci.displayName) {
|
|
5918
|
+
elements.push({ type: "mrkdwn", text: `*CI:* ${summary.ci.displayName}` });
|
|
5919
|
+
}
|
|
5920
|
+
if (summary.ci.branch) {
|
|
5921
|
+
elements.push({ type: "mrkdwn", text: `*Branch:* ${summary.ci.branch}` });
|
|
5922
|
+
}
|
|
5923
|
+
if (summary.ci.commitSha) {
|
|
5924
|
+
elements.push({ type: "mrkdwn", text: `*Commit:* ${summary.ci.commitSha.slice(0, 7)}` });
|
|
5925
|
+
}
|
|
5926
|
+
if (summary.ci.buildNumber) {
|
|
5927
|
+
elements.push({ type: "mrkdwn", text: `*Build:* #${summary.ci.buildNumber}` });
|
|
5928
|
+
}
|
|
5929
|
+
if (elements.length > 0) {
|
|
5930
|
+
blocks.push({
|
|
5931
|
+
type: "context",
|
|
5932
|
+
elements
|
|
5933
|
+
});
|
|
5934
|
+
}
|
|
5935
|
+
}
|
|
5936
|
+
if (summary.reportUrl) {
|
|
5937
|
+
blocks.push({
|
|
5938
|
+
type: "actions",
|
|
5939
|
+
elements: [
|
|
5940
|
+
{
|
|
5941
|
+
type: "button",
|
|
5942
|
+
text: {
|
|
5943
|
+
type: "plain_text",
|
|
5944
|
+
text: "View Report",
|
|
5945
|
+
emoji: true
|
|
5946
|
+
},
|
|
5947
|
+
url: summary.reportUrl,
|
|
5948
|
+
action_id: "view_report"
|
|
5949
|
+
}
|
|
5950
|
+
]
|
|
5951
|
+
});
|
|
5952
|
+
}
|
|
5953
|
+
return { blocks };
|
|
5954
|
+
}
|
|
5955
|
+
async function sendSlackNotification(args, deps) {
|
|
5956
|
+
const { summary, webhookUrl, maxFailedTests = 5 } = args;
|
|
5957
|
+
const { fetch, logger } = deps;
|
|
5958
|
+
const payload = buildSlackPayload(summary, maxFailedTests);
|
|
5959
|
+
try {
|
|
5960
|
+
const response = await fetch(webhookUrl, {
|
|
5961
|
+
method: "POST",
|
|
5962
|
+
headers: { "Content-Type": "application/json" },
|
|
5963
|
+
body: JSON.stringify(payload)
|
|
5964
|
+
});
|
|
5965
|
+
if (!response.ok) {
|
|
5966
|
+
const requestId = response.headers.get("x-request-id") ?? void 0;
|
|
5967
|
+
let bodyText = "";
|
|
5968
|
+
try {
|
|
5969
|
+
bodyText = await response.text();
|
|
5970
|
+
} catch {
|
|
5971
|
+
}
|
|
5972
|
+
const truncatedBody = truncate(bodyText, 200);
|
|
5973
|
+
const idPart = requestId ? ` x-request-id=${requestId}` : "";
|
|
5974
|
+
const errorMsg = `Slack notifier failed: HTTP ${response.status}${idPart} ${truncatedBody}`;
|
|
5975
|
+
logger.warn(errorMsg);
|
|
5976
|
+
return { ok: false, error: errorMsg };
|
|
5977
|
+
}
|
|
5978
|
+
return { ok: true };
|
|
5979
|
+
} catch (err) {
|
|
5980
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
5981
|
+
const errorMsg = `Slack notifier failed: ${msg}`;
|
|
5982
|
+
logger.warn(errorMsg);
|
|
5983
|
+
return { ok: false, error: errorMsg };
|
|
5984
|
+
}
|
|
5985
|
+
}
|
|
5986
|
+
|
|
5987
|
+
// src/notifiers/teams.ts
|
|
5988
|
+
function truncate2(text, maxLen) {
|
|
5989
|
+
if (text.length <= maxLen) return text;
|
|
5990
|
+
return text.slice(0, maxLen - 3) + "...";
|
|
5991
|
+
}
|
|
5992
|
+
function formatDuration4(ms) {
|
|
5993
|
+
const seconds = ms / 1e3;
|
|
5994
|
+
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
|
5995
|
+
const minutes = Math.floor(seconds / 60);
|
|
5996
|
+
const remainingSeconds = seconds % 60;
|
|
5997
|
+
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
|
|
5998
|
+
}
|
|
5999
|
+
function buildTeamsPayload(summary, maxFailedTests) {
|
|
6000
|
+
const allPassed = summary.failed === 0;
|
|
6001
|
+
const statusEmoji = allPassed ? "\u2705" : "\u274C";
|
|
6002
|
+
const statusColor = allPassed ? "good" : "attention";
|
|
6003
|
+
const bodyItems = [];
|
|
6004
|
+
bodyItems.push({
|
|
6005
|
+
type: "TextBlock",
|
|
6006
|
+
size: "Large",
|
|
6007
|
+
weight: "Bolder",
|
|
6008
|
+
text: `${statusEmoji} Test Results`,
|
|
6009
|
+
color: statusColor
|
|
6010
|
+
});
|
|
6011
|
+
bodyItems.push({
|
|
6012
|
+
type: "FactSet",
|
|
6013
|
+
facts: [
|
|
6014
|
+
{ title: "Total", value: String(summary.total) },
|
|
6015
|
+
{ title: "Passed", value: String(summary.passed) },
|
|
6016
|
+
{ title: "Failed", value: String(summary.failed) },
|
|
6017
|
+
{ title: "Skipped", value: String(summary.skipped) },
|
|
6018
|
+
{ title: "Duration", value: formatDuration4(summary.durationMs) }
|
|
6019
|
+
]
|
|
6020
|
+
});
|
|
6021
|
+
if (summary.failedTests.length > 0) {
|
|
6022
|
+
const displayedTests = summary.failedTests.slice(0, maxFailedTests);
|
|
6023
|
+
const failedItems = [
|
|
6024
|
+
{
|
|
6025
|
+
type: "TextBlock",
|
|
6026
|
+
text: "Failed Tests",
|
|
6027
|
+
weight: "Bolder",
|
|
6028
|
+
spacing: "Medium"
|
|
6029
|
+
}
|
|
6030
|
+
];
|
|
6031
|
+
for (const t of displayedTests) {
|
|
6032
|
+
failedItems.push({
|
|
6033
|
+
type: "TextBlock",
|
|
6034
|
+
text: `**${t.name}**`,
|
|
6035
|
+
wrap: true
|
|
6036
|
+
});
|
|
6037
|
+
if (t.error) {
|
|
6038
|
+
const cleanError = truncate2(stripAnsi(t.error), 500);
|
|
6039
|
+
failedItems.push({
|
|
6040
|
+
type: "TextBlock",
|
|
6041
|
+
text: cleanError,
|
|
6042
|
+
wrap: true,
|
|
6043
|
+
fontType: "Monospace",
|
|
6044
|
+
size: "Small",
|
|
6045
|
+
color: "Attention"
|
|
6046
|
+
});
|
|
6047
|
+
}
|
|
6048
|
+
}
|
|
6049
|
+
if (summary.failedTests.length > maxFailedTests) {
|
|
6050
|
+
failedItems.push({
|
|
6051
|
+
type: "TextBlock",
|
|
6052
|
+
text: `...and ${summary.failedTests.length - maxFailedTests} more`,
|
|
6053
|
+
isSubtle: true,
|
|
6054
|
+
spacing: "Small"
|
|
6055
|
+
});
|
|
6056
|
+
}
|
|
6057
|
+
bodyItems.push({
|
|
6058
|
+
type: "Container",
|
|
6059
|
+
items: failedItems
|
|
6060
|
+
});
|
|
6061
|
+
}
|
|
6062
|
+
if (summary.ci) {
|
|
6063
|
+
const ciFacts = [];
|
|
6064
|
+
if (summary.ci.displayName) {
|
|
6065
|
+
ciFacts.push({ title: "CI", value: summary.ci.displayName });
|
|
6066
|
+
}
|
|
6067
|
+
if (summary.ci.branch) {
|
|
6068
|
+
ciFacts.push({ title: "Branch", value: summary.ci.branch });
|
|
6069
|
+
}
|
|
6070
|
+
if (summary.ci.commitSha) {
|
|
6071
|
+
ciFacts.push({ title: "Commit", value: summary.ci.commitSha.slice(0, 7) });
|
|
6072
|
+
}
|
|
6073
|
+
if (summary.ci.buildNumber) {
|
|
6074
|
+
ciFacts.push({ title: "Build", value: `#${summary.ci.buildNumber}` });
|
|
6075
|
+
}
|
|
6076
|
+
if (ciFacts.length > 0) {
|
|
6077
|
+
bodyItems.push({
|
|
6078
|
+
type: "FactSet",
|
|
6079
|
+
facts: ciFacts,
|
|
6080
|
+
separator: true
|
|
6081
|
+
});
|
|
6082
|
+
}
|
|
6083
|
+
}
|
|
6084
|
+
const card = {
|
|
6085
|
+
type: "AdaptiveCard",
|
|
6086
|
+
$schema: "http://adaptivecards.io/schemas/adaptive-card.json",
|
|
6087
|
+
version: "1.4",
|
|
6088
|
+
body: bodyItems
|
|
6089
|
+
};
|
|
6090
|
+
if (summary.reportUrl) {
|
|
6091
|
+
card.actions = [
|
|
6092
|
+
{
|
|
6093
|
+
type: "Action.OpenUrl",
|
|
6094
|
+
title: "View Report",
|
|
6095
|
+
url: summary.reportUrl
|
|
6096
|
+
}
|
|
6097
|
+
];
|
|
6098
|
+
}
|
|
6099
|
+
return {
|
|
6100
|
+
type: "message",
|
|
6101
|
+
attachments: [
|
|
6102
|
+
{
|
|
6103
|
+
contentType: "application/vnd.microsoft.card.adaptive",
|
|
6104
|
+
content: card
|
|
6105
|
+
}
|
|
6106
|
+
]
|
|
6107
|
+
};
|
|
6108
|
+
}
|
|
6109
|
+
async function sendTeamsNotification(args, deps) {
|
|
6110
|
+
const { summary, webhookUrl, maxFailedTests = 5 } = args;
|
|
6111
|
+
const { fetch, logger } = deps;
|
|
6112
|
+
const payload = buildTeamsPayload(summary, maxFailedTests);
|
|
6113
|
+
try {
|
|
6114
|
+
const response = await fetch(webhookUrl, {
|
|
6115
|
+
method: "POST",
|
|
6116
|
+
headers: { "Content-Type": "application/json" },
|
|
6117
|
+
body: JSON.stringify(payload)
|
|
6118
|
+
});
|
|
6119
|
+
if (!response.ok) {
|
|
6120
|
+
const requestId = response.headers.get("x-request-id") ?? void 0;
|
|
6121
|
+
let bodyText = "";
|
|
6122
|
+
try {
|
|
6123
|
+
bodyText = await response.text();
|
|
6124
|
+
} catch {
|
|
6125
|
+
}
|
|
6126
|
+
const truncatedBody = truncate2(bodyText, 200);
|
|
6127
|
+
const idPart = requestId ? ` x-request-id=${requestId}` : "";
|
|
6128
|
+
const errorMsg = `Teams notifier failed: HTTP ${response.status}${idPart} ${truncatedBody}`;
|
|
6129
|
+
logger.warn(errorMsg);
|
|
6130
|
+
return { ok: false, error: errorMsg };
|
|
6131
|
+
}
|
|
6132
|
+
return { ok: true };
|
|
6133
|
+
} catch (err) {
|
|
6134
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
6135
|
+
const errorMsg = `Teams notifier failed: ${msg}`;
|
|
6136
|
+
logger.warn(errorMsg);
|
|
6137
|
+
return { ok: false, error: errorMsg };
|
|
6138
|
+
}
|
|
6139
|
+
}
|
|
6140
|
+
|
|
6141
|
+
// src/notifiers/hmac.ts
|
|
6142
|
+
import { createHmac } from "crypto";
|
|
6143
|
+
function signBody(args) {
|
|
6144
|
+
let input;
|
|
6145
|
+
let timestamp;
|
|
6146
|
+
if (args.includeTimestamp) {
|
|
6147
|
+
timestamp = args.timestamp ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
6148
|
+
input = `${timestamp}.${args.body}`;
|
|
6149
|
+
} else {
|
|
6150
|
+
input = args.body;
|
|
6151
|
+
}
|
|
6152
|
+
const hex = createHmac("sha256", args.secret).update(input, "utf8").digest("hex");
|
|
6153
|
+
return {
|
|
6154
|
+
signature: `sha256=${hex}`,
|
|
6155
|
+
timestamp
|
|
6156
|
+
};
|
|
6157
|
+
}
|
|
6158
|
+
|
|
6159
|
+
// src/notifiers/webhook.ts
|
|
6160
|
+
async function sendWebhookNotification(args, deps) {
|
|
6161
|
+
const { summary, options } = args;
|
|
6162
|
+
const { fetch, logger } = deps;
|
|
6163
|
+
const payload = {
|
|
6164
|
+
schemaVersion: 1,
|
|
6165
|
+
event: "test_run_finished",
|
|
6166
|
+
summary
|
|
6167
|
+
};
|
|
6168
|
+
const body = JSON.stringify(payload);
|
|
6169
|
+
const headers = { "Content-Type": "application/json" };
|
|
6170
|
+
if (options.headers) {
|
|
6171
|
+
for (const [key, value] of Object.entries(options.headers)) {
|
|
6172
|
+
headers[key] = value;
|
|
6173
|
+
}
|
|
6174
|
+
}
|
|
6175
|
+
if (options.signer) {
|
|
6176
|
+
const { secret, header, includeTimestamp, timestampHeader } = options.signer;
|
|
6177
|
+
const result = signBody({ body, secret, includeTimestamp });
|
|
6178
|
+
headers[header] = result.signature;
|
|
6179
|
+
if (result.timestamp) {
|
|
6180
|
+
headers[timestampHeader ?? "X-Timestamp"] = result.timestamp;
|
|
6181
|
+
}
|
|
6182
|
+
}
|
|
6183
|
+
try {
|
|
6184
|
+
const response = await fetch(options.url, {
|
|
6185
|
+
method: options.method ?? "POST",
|
|
6186
|
+
headers,
|
|
6187
|
+
body
|
|
6188
|
+
});
|
|
6189
|
+
if (!response.ok) {
|
|
6190
|
+
const requestId = response.headers.get("x-request-id") ?? void 0;
|
|
6191
|
+
let snippet = "";
|
|
6192
|
+
try {
|
|
6193
|
+
snippet = (await response.text()).slice(0, 200);
|
|
6194
|
+
} catch {
|
|
6195
|
+
}
|
|
6196
|
+
const idPart = requestId ? ` x-request-id=${requestId}` : "";
|
|
6197
|
+
const errorMsg = `webhook: HTTP ${response.status}${idPart} ${snippet}`;
|
|
6198
|
+
logger.warn(errorMsg);
|
|
6199
|
+
return { ok: false, error: errorMsg };
|
|
6200
|
+
}
|
|
6201
|
+
return { ok: true };
|
|
6202
|
+
} catch (err) {
|
|
6203
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
6204
|
+
const errorMsg = `webhook: ${msg}`;
|
|
6205
|
+
logger.warn(errorMsg);
|
|
6206
|
+
return { ok: false, error: errorMsg };
|
|
6207
|
+
}
|
|
6208
|
+
}
|
|
6209
|
+
|
|
6210
|
+
// src/notifiers/index.ts
|
|
6211
|
+
function buildSummary(run, reportUrl, toCIInfo2) {
|
|
6212
|
+
let passed = 0;
|
|
6213
|
+
let failed = 0;
|
|
6214
|
+
let skipped = 0;
|
|
6215
|
+
const failedTests = [];
|
|
6216
|
+
for (const tc of run.testCases) {
|
|
6217
|
+
switch (tc.status) {
|
|
6218
|
+
case "passed":
|
|
6219
|
+
passed++;
|
|
6220
|
+
break;
|
|
6221
|
+
case "failed":
|
|
6222
|
+
failed++;
|
|
6223
|
+
failedTests.push({
|
|
6224
|
+
testId: tc.id,
|
|
6225
|
+
name: tc.story.scenario,
|
|
6226
|
+
error: tc.errorMessage
|
|
6227
|
+
});
|
|
6228
|
+
break;
|
|
6229
|
+
case "skipped":
|
|
6230
|
+
case "pending":
|
|
6231
|
+
skipped++;
|
|
6232
|
+
break;
|
|
6233
|
+
}
|
|
6234
|
+
}
|
|
6235
|
+
let ci;
|
|
6236
|
+
if (run.ci) {
|
|
6237
|
+
ci = toCIInfo2(run.ci);
|
|
6238
|
+
}
|
|
6239
|
+
return {
|
|
6240
|
+
total: run.testCases.length,
|
|
6241
|
+
passed,
|
|
6242
|
+
failed,
|
|
6243
|
+
skipped,
|
|
6244
|
+
durationMs: run.durationMs,
|
|
6245
|
+
failedTests,
|
|
6246
|
+
ci,
|
|
6247
|
+
reportUrl
|
|
6248
|
+
};
|
|
6249
|
+
}
|
|
6250
|
+
function shouldNotify(condition, failedCount) {
|
|
6251
|
+
if (condition === "never") return false;
|
|
6252
|
+
if (condition === "on-failure" && failedCount === 0) return false;
|
|
6253
|
+
return true;
|
|
6254
|
+
}
|
|
6255
|
+
async function sendNotifications(args, deps) {
|
|
6256
|
+
const { run, notification } = args;
|
|
6257
|
+
const { logger, toCIInfo: toCIInfo2 } = deps;
|
|
6258
|
+
const env = deps.env ?? process.env;
|
|
6259
|
+
if (!deps.fetch) {
|
|
6260
|
+
logger.warn("notifications: skipped (fetch unavailable)");
|
|
6261
|
+
return;
|
|
6262
|
+
}
|
|
6263
|
+
const fetch = deps.fetch;
|
|
6264
|
+
const slackWebhookUrl = notification?.slackWebhookUrl ?? env.SLACK_WEBHOOK_URL;
|
|
6265
|
+
const teamsWebhookUrl = notification?.teamsWebhookUrl ?? env.TEAMS_WEBHOOK_URL;
|
|
6266
|
+
const globalCondition = notification?.condition ?? "on-failure";
|
|
6267
|
+
const reportUrl = notification?.reportUrl;
|
|
6268
|
+
const maxFailedTests = notification?.maxFailedTests ?? 5;
|
|
6269
|
+
const webhooks = notification?.webhooks ?? [];
|
|
6270
|
+
if (!slackWebhookUrl && !teamsWebhookUrl && webhooks.length === 0) {
|
|
6271
|
+
return;
|
|
6272
|
+
}
|
|
6273
|
+
const summary = buildSummary(run, reportUrl, toCIInfo2);
|
|
6274
|
+
const promises = [];
|
|
6275
|
+
if (slackWebhookUrl && shouldNotify(globalCondition, summary.failed)) {
|
|
6276
|
+
promises.push(
|
|
6277
|
+
sendSlackNotification(
|
|
6278
|
+
{ summary, webhookUrl: slackWebhookUrl, maxFailedTests },
|
|
6279
|
+
{ fetch, logger }
|
|
6280
|
+
).then(() => void 0)
|
|
6281
|
+
);
|
|
6282
|
+
}
|
|
6283
|
+
if (teamsWebhookUrl && shouldNotify(globalCondition, summary.failed)) {
|
|
6284
|
+
promises.push(
|
|
6285
|
+
sendTeamsNotification(
|
|
6286
|
+
{ summary, webhookUrl: teamsWebhookUrl, maxFailedTests },
|
|
6287
|
+
{ fetch, logger }
|
|
6288
|
+
).then(() => void 0)
|
|
6289
|
+
);
|
|
6290
|
+
}
|
|
6291
|
+
for (const webhook of webhooks) {
|
|
6292
|
+
const effectiveCondition = webhook.condition ?? globalCondition;
|
|
6293
|
+
if (!shouldNotify(effectiveCondition, summary.failed)) continue;
|
|
6294
|
+
promises.push(
|
|
6295
|
+
sendWebhookNotification(
|
|
6296
|
+
{ summary, options: webhook, maxFailedTests },
|
|
6297
|
+
{ fetch, logger }
|
|
6298
|
+
).then(() => void 0)
|
|
6299
|
+
);
|
|
6300
|
+
}
|
|
6301
|
+
await Promise.allSettled(promises);
|
|
6302
|
+
}
|
|
6303
|
+
|
|
6304
|
+
// src/types/ci.ts
|
|
6305
|
+
var DISPLAY_NAMES = {
|
|
6306
|
+
github: "GitHub Actions",
|
|
6307
|
+
gitlab: "GitLab CI",
|
|
6308
|
+
circleci: "CircleCI",
|
|
6309
|
+
jenkins: "Jenkins",
|
|
6310
|
+
azure: "Azure DevOps",
|
|
6311
|
+
buildkite: "Buildkite",
|
|
6312
|
+
travis: "Travis CI",
|
|
6313
|
+
unknown: "CI"
|
|
6314
|
+
};
|
|
6315
|
+
var NAME_TO_PROVIDER = {
|
|
6316
|
+
github: "github",
|
|
6317
|
+
gitlab: "gitlab",
|
|
6318
|
+
circleci: "circleci",
|
|
6319
|
+
jenkins: "jenkins",
|
|
6320
|
+
azure: "azure",
|
|
6321
|
+
buildkite: "buildkite",
|
|
6322
|
+
travis: "travis",
|
|
6323
|
+
ci: "unknown"
|
|
6324
|
+
};
|
|
6325
|
+
function toCIInfo(raw) {
|
|
6326
|
+
if (!raw) return void 0;
|
|
6327
|
+
const provider = raw.provider ?? NAME_TO_PROVIDER[raw.name] ?? "unknown";
|
|
6328
|
+
return {
|
|
6329
|
+
provider,
|
|
6330
|
+
displayName: DISPLAY_NAMES[provider],
|
|
6331
|
+
url: raw.url,
|
|
6332
|
+
buildNumber: raw.buildNumber,
|
|
6333
|
+
branch: raw.branch,
|
|
6334
|
+
commitSha: raw.commitSha,
|
|
6335
|
+
prNumber: raw.prNumber
|
|
6336
|
+
};
|
|
6337
|
+
}
|
|
6338
|
+
function toRawCIInfo(ci) {
|
|
6339
|
+
if (!ci) return void 0;
|
|
6340
|
+
return {
|
|
6341
|
+
name: ci.provider === "unknown" ? "ci" : ci.provider,
|
|
6342
|
+
provider: ci.provider,
|
|
6343
|
+
url: ci.url,
|
|
6344
|
+
buildNumber: ci.buildNumber,
|
|
6345
|
+
branch: ci.branch,
|
|
6346
|
+
commitSha: ci.commitSha,
|
|
6347
|
+
prNumber: ci.prNumber
|
|
6348
|
+
};
|
|
6349
|
+
}
|
|
6350
|
+
|
|
6351
|
+
// src/history/history-store.ts
|
|
6352
|
+
function emptyStore() {
|
|
6353
|
+
return { version: 1, maxRuns: 10, tests: {}, lastUpdated: 0 };
|
|
6354
|
+
}
|
|
6355
|
+
function loadHistory(args, deps) {
|
|
6356
|
+
const content = deps.readFile(args.filePath);
|
|
6357
|
+
if (content === void 0) {
|
|
6358
|
+
return emptyStore();
|
|
6359
|
+
}
|
|
6360
|
+
let parsed;
|
|
6361
|
+
try {
|
|
6362
|
+
parsed = JSON.parse(content);
|
|
6363
|
+
} catch {
|
|
6364
|
+
deps.logger.warn(`Failed to parse history file: ${args.filePath}`);
|
|
6365
|
+
return emptyStore();
|
|
6366
|
+
}
|
|
6367
|
+
if (typeof parsed !== "object" || parsed === null || parsed.version !== 1) {
|
|
6368
|
+
deps.logger.warn(
|
|
6369
|
+
`Unknown history version in ${args.filePath}, expected version 1`
|
|
6370
|
+
);
|
|
6371
|
+
return emptyStore();
|
|
6372
|
+
}
|
|
6373
|
+
const obj = parsed;
|
|
6374
|
+
if (typeof obj.tests !== "object" || obj.tests === null || Array.isArray(obj.tests)) {
|
|
6375
|
+
deps.logger.warn(
|
|
6376
|
+
`Malformed history store in ${args.filePath}: tests must be a non-null object`
|
|
6377
|
+
);
|
|
6378
|
+
return emptyStore();
|
|
6379
|
+
}
|
|
6380
|
+
return parsed;
|
|
6381
|
+
}
|
|
6382
|
+
function saveHistory(args, deps) {
|
|
6383
|
+
deps.writeFile(args.filePath, JSON.stringify(args.store, null, 2));
|
|
6384
|
+
}
|
|
6385
|
+
function updateHistory(args) {
|
|
6386
|
+
const { store, run, maxRuns } = args;
|
|
6387
|
+
const newTests = { ...store.tests };
|
|
6388
|
+
for (const tc of run.testCases) {
|
|
6389
|
+
const entry = {
|
|
6390
|
+
runId: run.runId,
|
|
6391
|
+
timestamp: run.startedAtMs,
|
|
6392
|
+
status: tc.status,
|
|
6393
|
+
durationMs: tc.durationMs,
|
|
6394
|
+
ci: run.ci ? {
|
|
6395
|
+
provider: void 0,
|
|
6396
|
+
branch: run.ci.branch,
|
|
6397
|
+
commitSha: run.ci.commitSha
|
|
6398
|
+
} : void 0
|
|
6399
|
+
};
|
|
6400
|
+
const existing = newTests[tc.id];
|
|
6401
|
+
if (existing) {
|
|
6402
|
+
const updatedEntries = [...existing.entries, entry];
|
|
6403
|
+
const trimmed = updatedEntries.length > maxRuns ? updatedEntries.slice(updatedEntries.length - maxRuns) : updatedEntries;
|
|
6404
|
+
newTests[tc.id] = {
|
|
6405
|
+
...existing,
|
|
6406
|
+
testName: tc.story.scenario,
|
|
6407
|
+
sourceFile: tc.sourceFile,
|
|
6408
|
+
sourceLine: tc.sourceLine,
|
|
6409
|
+
entries: trimmed
|
|
6410
|
+
};
|
|
6411
|
+
} else {
|
|
6412
|
+
newTests[tc.id] = {
|
|
6413
|
+
testId: tc.id,
|
|
6414
|
+
testName: tc.story.scenario,
|
|
6415
|
+
sourceFile: tc.sourceFile,
|
|
6416
|
+
sourceLine: tc.sourceLine,
|
|
6417
|
+
entries: [entry]
|
|
6418
|
+
};
|
|
6419
|
+
}
|
|
6420
|
+
}
|
|
6421
|
+
return {
|
|
6422
|
+
version: 1,
|
|
6423
|
+
maxRuns,
|
|
6424
|
+
tests: newTests,
|
|
6425
|
+
lastUpdated: Date.now()
|
|
6426
|
+
};
|
|
6427
|
+
}
|
|
6428
|
+
|
|
6429
|
+
// src/history/flakiness.ts
|
|
6430
|
+
function calculateFlakiness(args) {
|
|
6431
|
+
const { entries } = args;
|
|
6432
|
+
const countable = entries.filter(
|
|
6433
|
+
(e) => e.status === "passed" || e.status === "failed"
|
|
6434
|
+
);
|
|
6435
|
+
if (countable.length < MIN_FLAKINESS_SAMPLES) {
|
|
6436
|
+
return {
|
|
6437
|
+
flakinessLevel: "stable",
|
|
6438
|
+
flakinessScore: 0,
|
|
6439
|
+
failureRate: 0,
|
|
6440
|
+
longestPassStreak: countable.length,
|
|
6441
|
+
longestFailStreak: 0
|
|
6442
|
+
};
|
|
6443
|
+
}
|
|
6444
|
+
let transitions = 0;
|
|
6445
|
+
for (let i = 1; i < countable.length; i++) {
|
|
6446
|
+
if (countable[i].status !== countable[i - 1].status) {
|
|
6447
|
+
transitions++;
|
|
6448
|
+
}
|
|
6449
|
+
}
|
|
6450
|
+
const transitionScore = transitions / (countable.length - 1);
|
|
6451
|
+
const failures = countable.filter((e) => e.status === "failed").length;
|
|
6452
|
+
const failureRate = failures / countable.length;
|
|
6453
|
+
let longestPassStreak = 0;
|
|
6454
|
+
let longestFailStreak = 0;
|
|
6455
|
+
let currentPassStreak = 0;
|
|
6456
|
+
let currentFailStreak = 0;
|
|
6457
|
+
for (const e of countable) {
|
|
6458
|
+
if (e.status === "passed") {
|
|
6459
|
+
currentPassStreak++;
|
|
6460
|
+
currentFailStreak = 0;
|
|
6461
|
+
if (currentPassStreak > longestPassStreak) {
|
|
6462
|
+
longestPassStreak = currentPassStreak;
|
|
6463
|
+
}
|
|
6464
|
+
} else {
|
|
6465
|
+
currentFailStreak++;
|
|
6466
|
+
currentPassStreak = 0;
|
|
6467
|
+
if (currentFailStreak > longestFailStreak) {
|
|
6468
|
+
longestFailStreak = currentFailStreak;
|
|
6469
|
+
}
|
|
6470
|
+
}
|
|
6471
|
+
}
|
|
6472
|
+
let flakinessLevel;
|
|
6473
|
+
if (transitionScore > 0.5 || transitionScore > 0.3 && failureRate > 0.2) {
|
|
6474
|
+
flakinessLevel = "flaky";
|
|
6475
|
+
} else if (transitionScore > 0.2 || failureRate > 0.3) {
|
|
6476
|
+
flakinessLevel = "unstable";
|
|
6477
|
+
} else {
|
|
6478
|
+
flakinessLevel = "stable";
|
|
6479
|
+
}
|
|
6480
|
+
return {
|
|
6481
|
+
flakinessLevel,
|
|
6482
|
+
flakinessScore: transitionScore,
|
|
6483
|
+
failureRate,
|
|
6484
|
+
longestPassStreak,
|
|
6485
|
+
longestFailStreak
|
|
6486
|
+
};
|
|
6487
|
+
}
|
|
6488
|
+
|
|
6489
|
+
// src/history/performance.ts
|
|
6490
|
+
function detectPerformanceTrend(args) {
|
|
6491
|
+
const { entries } = args;
|
|
6492
|
+
const countable = entries.filter(
|
|
6493
|
+
(e) => e.status !== "skipped" && e.status !== "pending"
|
|
6494
|
+
);
|
|
6495
|
+
if (countable.length === 0) {
|
|
6496
|
+
return { trend: "stable", avgDurationMs: 0 };
|
|
6497
|
+
}
|
|
6498
|
+
const avgAll = countable.reduce((sum, e) => sum + e.durationMs, 0) / countable.length;
|
|
6499
|
+
if (countable.length < MIN_PERF_SAMPLES) {
|
|
6500
|
+
return { trend: "stable", avgDurationMs: avgAll };
|
|
6501
|
+
}
|
|
6502
|
+
const mid = Math.floor(countable.length / 2);
|
|
6503
|
+
const earlier = countable.slice(0, mid);
|
|
6504
|
+
const recent = countable.slice(mid);
|
|
6505
|
+
const earlierAvg = earlier.reduce((sum, e) => sum + e.durationMs, 0) / earlier.length;
|
|
6506
|
+
const recentAvg = recent.reduce((sum, e) => sum + e.durationMs, 0) / recent.length;
|
|
6507
|
+
let trend;
|
|
6508
|
+
if (earlierAvg === 0) {
|
|
6509
|
+
trend = "stable";
|
|
6510
|
+
} else {
|
|
6511
|
+
const change = (recentAvg - earlierAvg) / earlierAvg;
|
|
6512
|
+
if (change > 0.1) {
|
|
6513
|
+
trend = "regressing";
|
|
6514
|
+
} else if (change < -0.1) {
|
|
6515
|
+
trend = "improving";
|
|
6516
|
+
} else {
|
|
6517
|
+
trend = "stable";
|
|
6518
|
+
}
|
|
6519
|
+
}
|
|
6520
|
+
return { trend, avgDurationMs: avgAll };
|
|
6521
|
+
}
|
|
6522
|
+
|
|
6523
|
+
// src/history/stability.ts
|
|
6524
|
+
function calculateStability(args) {
|
|
6525
|
+
const { passRate, flakinessScore, longestPassStreak, sampleSize } = args;
|
|
6526
|
+
const inverseFlakiness = 1 - flakinessScore;
|
|
6527
|
+
const streakNorm = longestPassStreak / Math.min(sampleSize, 10);
|
|
6528
|
+
const score = passRate * 0.6 + inverseFlakiness * 0.3 + streakNorm * 0.1;
|
|
6529
|
+
if (score >= 0.95) return "A";
|
|
6530
|
+
if (score >= 0.85) return "B";
|
|
6531
|
+
if (score >= 0.7) return "C";
|
|
6532
|
+
if (score >= 0.5) return "D";
|
|
6533
|
+
return "F";
|
|
6534
|
+
}
|
|
6535
|
+
|
|
6536
|
+
// src/history/metrics.ts
|
|
6537
|
+
function computeTestMetrics(args) {
|
|
6538
|
+
const { testId, entries } = args;
|
|
6539
|
+
const flakiness = calculateFlakiness({ entries });
|
|
6540
|
+
const perf = detectPerformanceTrend({ entries });
|
|
6541
|
+
const countable = entries.filter(
|
|
6542
|
+
(e) => e.status === "passed" || e.status === "failed"
|
|
6543
|
+
);
|
|
6544
|
+
const passRate = countable.length > 0 ? countable.filter((e) => e.status === "passed").length / countable.length : 1;
|
|
6545
|
+
const stabilityGrade = calculateStability({
|
|
6546
|
+
passRate,
|
|
6547
|
+
flakinessScore: flakiness.flakinessScore,
|
|
6548
|
+
longestPassStreak: flakiness.longestPassStreak,
|
|
6549
|
+
sampleSize: entries.length
|
|
6550
|
+
});
|
|
6551
|
+
let consecutiveFailures = 0;
|
|
6552
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
6553
|
+
if (entries[i].status === "failed") {
|
|
6554
|
+
consecutiveFailures++;
|
|
6555
|
+
} else {
|
|
6556
|
+
break;
|
|
6557
|
+
}
|
|
6558
|
+
}
|
|
6559
|
+
return {
|
|
6560
|
+
testId,
|
|
6561
|
+
flakinessLevel: flakiness.flakinessLevel,
|
|
6562
|
+
flakinessScore: flakiness.flakinessScore,
|
|
6563
|
+
failureRate: flakiness.failureRate,
|
|
6564
|
+
stabilityGrade,
|
|
6565
|
+
performanceTrend: perf.trend,
|
|
6566
|
+
avgDurationMs: perf.avgDurationMs,
|
|
6567
|
+
passRate,
|
|
6568
|
+
longestPassStreak: flakiness.longestPassStreak,
|
|
6569
|
+
consecutiveFailures,
|
|
6570
|
+
sampleSize: entries.length
|
|
6571
|
+
};
|
|
6572
|
+
}
|
|
6573
|
+
|
|
5729
6574
|
// src/index.ts
|
|
5730
6575
|
var FORMAT_EXTENSIONS = {
|
|
5731
6576
|
markdown: ".md",
|
|
@@ -6074,6 +6919,9 @@ export {
|
|
|
6074
6919
|
CucumberMessagesFormatter,
|
|
6075
6920
|
HtmlFormatter,
|
|
6076
6921
|
JUnitFormatter,
|
|
6922
|
+
MIN_FLAKINESS_SAMPLES,
|
|
6923
|
+
MIN_METRIC_SAMPLES,
|
|
6924
|
+
MIN_PERF_SAMPLES,
|
|
6077
6925
|
MarkdownFormatter,
|
|
6078
6926
|
ReportGenerator,
|
|
6079
6927
|
STORY_META_KEY,
|
|
@@ -6081,15 +6929,21 @@ export {
|
|
|
6081
6929
|
adaptPlaywrightRun,
|
|
6082
6930
|
adaptVitestRun,
|
|
6083
6931
|
assertValidRun,
|
|
6932
|
+
calculateFlakiness,
|
|
6933
|
+
calculateStability,
|
|
6084
6934
|
canonicalizeRun,
|
|
6085
6935
|
clearVersionCache,
|
|
6936
|
+
computeTestMetrics,
|
|
6086
6937
|
createReportGenerator,
|
|
6087
6938
|
deriveStepResults,
|
|
6088
6939
|
detectCI4 as detectCI,
|
|
6940
|
+
detectPerformanceTrend,
|
|
6089
6941
|
findGitDir,
|
|
6090
6942
|
formatDuration2 as formatDuration,
|
|
6091
6943
|
generateRunId,
|
|
6092
6944
|
generateTestCaseId,
|
|
6945
|
+
hasSufficientHistory,
|
|
6946
|
+
loadHistory,
|
|
6093
6947
|
mergeStepResults,
|
|
6094
6948
|
msToNanoseconds,
|
|
6095
6949
|
nanosecondsToMs,
|
|
@@ -6105,8 +6959,18 @@ export {
|
|
|
6105
6959
|
resolveAttachment,
|
|
6106
6960
|
resolveAttachments,
|
|
6107
6961
|
resolveTraceUrl,
|
|
6962
|
+
saveHistory,
|
|
6963
|
+
sendNotifications,
|
|
6964
|
+
sendSlackNotification,
|
|
6965
|
+
sendTeamsNotification,
|
|
6966
|
+
sendWebhookNotification,
|
|
6967
|
+
signBody,
|
|
6108
6968
|
slugify,
|
|
6969
|
+
stripAnsi,
|
|
6970
|
+
toCIInfo,
|
|
6971
|
+
toRawCIInfo,
|
|
6109
6972
|
tryGetActiveOtelContext,
|
|
6973
|
+
updateHistory,
|
|
6110
6974
|
validateCanonicalRun
|
|
6111
6975
|
};
|
|
6112
6976
|
//# sourceMappingURL=index.js.map
|