@regressionbot/sdk 2.10.1 β 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/cli.js +10 -0
- package/dist/index.d.ts +2 -2
- package/dist/types.d.ts +41 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -38,6 +38,24 @@ npm install @regressionbot/sdk
|
|
|
38
38
|
|
|
39
39
|
## Migration from 1.x
|
|
40
40
|
|
|
41
|
+
### 2.11.0
|
|
42
|
+
|
|
43
|
+
Needs an API release past 2.10.1 for these fields to be populated. On an older API they are
|
|
44
|
+
simply absent.
|
|
45
|
+
|
|
46
|
+
- **`intentAssessment.reasons`** can now include `'intended_change_not_found'`. An intent
|
|
47
|
+
that expects a visible change (`expectedChanges`, or text that is not an explicit "no
|
|
48
|
+
visual change") must now actually be seen: zero diffs and all-noise runs no longer pass
|
|
49
|
+
when the intent expected a change. `intentAssessment.missingExpectedChanges` lists each
|
|
50
|
+
`expectedChanges` item not found, or the intent's own words when it listed none.
|
|
51
|
+
- **`intentAssessment.reasons`** can now include `'partial_coverage'`: the checked pages were
|
|
52
|
+
clean but the run covered only part of the site, so a would-be pass became a `review`.
|
|
53
|
+
- **`JobSummary.pathCoverage`** (`PathCoverage`) β which paths the run checked, from where
|
|
54
|
+
(`'paths'` / `'sitemap'` / `'fallback'`), and whether that was partial. A run given no paths
|
|
55
|
+
now defaults to the project's sitemap (or `testOrigin/sitemap.xml`, up to 100 pages) instead
|
|
56
|
+
of silently checking only `/`; `pathCoverage.partial` and the `'partial_coverage'` reason
|
|
57
|
+
above say when it couldn't cover everything asked.
|
|
58
|
+
|
|
41
59
|
### 2.10.1
|
|
42
60
|
|
|
43
61
|
Documentation only; no change to any type's shape or any method.
|
|
@@ -453,6 +471,8 @@ const summary = await job.getSummary();
|
|
|
453
471
|
console.log(summary.intentAssessment?.summary);
|
|
454
472
|
console.log(`Bugs: ${summary.intentAssessment?.bugCount}`);
|
|
455
473
|
// What's behind a `review` or `fail`, worst first (needs API 2.10.0+): summary.intentAssessment?.reasons
|
|
474
|
+
// If an expectedChanges item never showed up, reasons includes 'intended_change_not_found'
|
|
475
|
+
// and summary.intentAssessment?.missingExpectedChanges names which ones.
|
|
456
476
|
|
|
457
477
|
// Per-result verdict, plus the per-region verdicts behind it
|
|
458
478
|
for (const r of summary.regressions) {
|
package/dist/cli.js
CHANGED
|
@@ -81,6 +81,11 @@ function printChangeClusters(clusters) {
|
|
|
81
81
|
console.log(`- ${label}: ${c.pageCount} pages`);
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
|
+
function printPathCoverage(coverage) {
|
|
85
|
+
if (!coverage)
|
|
86
|
+
return;
|
|
87
|
+
console.log(`\nπΊοΈ Coverage: ${coverage.checkedPaths.length} path(s) via ${coverage.source}${coverage.partial ? ' (partial)' : ''}${coverage.reason ? ` β ${coverage.reason}` : ''}`);
|
|
88
|
+
}
|
|
84
89
|
/**
|
|
85
90
|
* A mistake in how the command was invoked, rather than a run that worked and found
|
|
86
91
|
* something. Exits 2 so CI can tell the two apart β a pipeline that treats "regressions
|
|
@@ -98,6 +103,9 @@ function printIntent(summary) {
|
|
|
98
103
|
if (ia.reasons && ia.reasons.length > 0) {
|
|
99
104
|
console.log(` reasons: ${ia.reasons.join(', ')}`);
|
|
100
105
|
}
|
|
106
|
+
if (ia.missingExpectedChanges && ia.missingExpectedChanges.length > 0) {
|
|
107
|
+
console.log(` missing expected changes: ${ia.missingExpectedChanges.join(', ')}`);
|
|
108
|
+
}
|
|
101
109
|
}
|
|
102
110
|
function parseFailOn(raw) {
|
|
103
111
|
if (raw === undefined)
|
|
@@ -405,6 +413,7 @@ Waiting for completion...
|
|
|
405
413
|
}
|
|
406
414
|
}
|
|
407
415
|
printChangeClusters(summary.changeClusters);
|
|
416
|
+
printPathCoverage(summary.pathCoverage);
|
|
408
417
|
printIntent(summary);
|
|
409
418
|
printUsage(summary.usage);
|
|
410
419
|
// Reported even when there are regressions. These used to be hidden behind an
|
|
@@ -453,6 +462,7 @@ Errors: ${summary.errorCount}
|
|
|
453
462
|
});
|
|
454
463
|
}
|
|
455
464
|
printChangeClusters(summary.changeClusters);
|
|
465
|
+
printPathCoverage(summary.pathCoverage);
|
|
456
466
|
printIntent(summary);
|
|
457
467
|
printUsage(summary.usage);
|
|
458
468
|
const doDownload = options.download || options['download-full'];
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { VRConfig, Viewport, Viewports, JobStatus, JobSummary, JobProgress, PageResult, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage } from './types';
|
|
1
|
+
import { VRConfig, Viewport, Viewports, JobStatus, JobSummary, JobProgress, PageResult, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage, PathCoverage } from './types';
|
|
2
2
|
import { sanitizeFilename, sanitizeUrlToPath } from './security';
|
|
3
3
|
export { sanitizeFilename, sanitizeUrlToPath, Viewports };
|
|
4
|
-
export type { PageResult, JobProgress, JobStatus, JobSummary, Viewport, VRConfig, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage };
|
|
4
|
+
export type { PageResult, JobProgress, JobStatus, JobSummary, Viewport, VRConfig, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage, PathCoverage };
|
|
5
5
|
export declare class RegressionBot {
|
|
6
6
|
private apiKey;
|
|
7
7
|
private apiUrl;
|
package/dist/types.d.ts
CHANGED
|
@@ -668,6 +668,30 @@ export interface JobSummary {
|
|
|
668
668
|
* from the API's own JSDoc instead. Needs API 2.10.0 or later.
|
|
669
669
|
*/
|
|
670
670
|
changeClusters: ChangeCluster[];
|
|
671
|
+
/**
|
|
672
|
+
* Which paths the run checked, from where, and whether that was all it was asked to check.
|
|
673
|
+
* Not yet in the API's published openapi.yaml schema; this doc comment is taken from the
|
|
674
|
+
* API's own JSDoc instead.
|
|
675
|
+
*/
|
|
676
|
+
pathCoverage?: PathCoverage;
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Which pages a run covered and why. `partial` is the part a caller must not miss: the run did
|
|
680
|
+
* not look at the whole site, so a clean result is not "nothing else broke".
|
|
681
|
+
*/
|
|
682
|
+
export interface PathCoverage {
|
|
683
|
+
/** `paths`: the caller listed them. `sitemap`: read from the sitemap. `fallback`: nothing
|
|
684
|
+
* usable was given or found, so only "/". */
|
|
685
|
+
source: 'paths' | 'sitemap' | 'fallback';
|
|
686
|
+
checkedPaths: string[];
|
|
687
|
+
/** Sitemap paths the scans matched before the scan limit; absent when no sitemap was read. */
|
|
688
|
+
totalDiscovered?: number;
|
|
689
|
+
truncatedByLimit: boolean;
|
|
690
|
+
/** The run did not check the whole of what it was asked to (fallback, unreadable sitemap,
|
|
691
|
+
* scan limit). */
|
|
692
|
+
partial: boolean;
|
|
693
|
+
reason?: string;
|
|
694
|
+
sitemapUrl?: string;
|
|
671
695
|
}
|
|
672
696
|
export interface ChangeClusterPage {
|
|
673
697
|
url: string;
|
|
@@ -726,14 +750,29 @@ export interface IntentAssessment {
|
|
|
726
750
|
/**
|
|
727
751
|
* Every condition behind `decision`, worst first; empty on a clean pass. Tells a review
|
|
728
752
|
* caused by a page that needs a person from one caused by pages that failed or were never
|
|
729
|
-
* assessed.
|
|
753
|
+
* assessed. `partial_coverage` means the checked pages were clean but the run covered only
|
|
754
|
+
* part of the site (for example a sitemap that could not be read or hit the page cap), so a
|
|
755
|
+
* would-be pass became a review. Needs API 2.10.0 or later.
|
|
730
756
|
*/
|
|
731
757
|
reasons?: GateReason[];
|
|
758
|
+
/**
|
|
759
|
+
* Present when `reasons` has `intended_change_not_found`: each `expectedChanges` item the
|
|
760
|
+
* run did not show, or the intent's own words when it listed none. The decision is
|
|
761
|
+
* `review`, never `pass`.
|
|
762
|
+
*/
|
|
763
|
+
missingExpectedChanges?: string[];
|
|
732
764
|
/** What the run compared and how much of it the verdict covers. Needs API 2.10.0 or later. */
|
|
733
765
|
coverage?: RunCoverage;
|
|
734
766
|
}
|
|
735
767
|
/** Every condition {@link IntentAssessment.decision} can be built from, worst first. */
|
|
736
|
-
export type GateReason = 'no_intent' | 'nothing_compared' | 'bug' | 'needs_review'
|
|
768
|
+
export type GateReason = 'no_intent' | 'nothing_compared' | 'bug' | 'needs_review'
|
|
769
|
+
/** An intent expecting a visible change (`expectedChanges`, or text that is not an explicit
|
|
770
|
+
* "no visual change") did not show it β zero diffs and all-noise runs included. See
|
|
771
|
+
* {@link IntentAssessment.missingExpectedChanges}. */
|
|
772
|
+
| 'intended_change_not_found' | 'assessment_in_progress' | 'assessment_incomplete' | 'device_disagreement' | 'comparison_failures'
|
|
773
|
+
/** Not from the gate itself: added when a would-be pass is turned into a review because the
|
|
774
|
+
* run covered only part of the site. See {@link JobSummary.pathCoverage}. */
|
|
775
|
+
| 'partial_coverage';
|
|
737
776
|
/**
|
|
738
777
|
* Where one changed page stands with the verdict.
|
|
739
778
|
*
|
package/package.json
CHANGED