@regressionbot/sdk 2.10.0 β 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/dist/cli.js +10 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +1 -0
- package/dist/types.d.ts +43 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -38,6 +38,34 @@ npm install @regressionbot/sdk
|
|
|
38
38
|
|
|
39
39
|
## Migration from 1.x
|
|
40
40
|
|
|
41
|
+
### 2.11.0
|
|
42
|
+
|
|
43
|
+
Needs an API release past 2.10.1 for these fields to be populated. On an older API they are
|
|
44
|
+
simply absent.
|
|
45
|
+
|
|
46
|
+
- **`intentAssessment.reasons`** can now include `'intended_change_not_found'`. An intent
|
|
47
|
+
that expects a visible change (`expectedChanges`, or text that is not an explicit "no
|
|
48
|
+
visual change") must now actually be seen: zero diffs and all-noise runs no longer pass
|
|
49
|
+
when the intent expected a change. `intentAssessment.missingExpectedChanges` lists each
|
|
50
|
+
`expectedChanges` item not found, or the intent's own words when it listed none.
|
|
51
|
+
- **`intentAssessment.reasons`** can now include `'partial_coverage'`: the checked pages were
|
|
52
|
+
clean but the run covered only part of the site, so a would-be pass became a `review`.
|
|
53
|
+
- **`JobSummary.pathCoverage`** (`PathCoverage`) β which paths the run checked, from where
|
|
54
|
+
(`'paths'` / `'sitemap'` / `'fallback'`), and whether that was partial. A run given no paths
|
|
55
|
+
now defaults to the project's sitemap (or `testOrigin/sitemap.xml`, up to 100 pages) instead
|
|
56
|
+
of silently checking only `/`; `pathCoverage.partial` and the `'partial_coverage'` reason
|
|
57
|
+
above say when it couldn't cover everything asked.
|
|
58
|
+
|
|
59
|
+
### 2.10.1
|
|
60
|
+
|
|
61
|
+
Documentation only; no change to any type's shape or any method.
|
|
62
|
+
|
|
63
|
+
- **`ResultAssessment.reason` can be `deferred`**: the verdict pass ran short of time for that page
|
|
64
|
+
and will retry it.
|
|
65
|
+
- **`autoApprove` with a `runContext`**: auto-approve fires only if every changed page has an AI
|
|
66
|
+
verdict (a page skipped under `aiSummaryThreshold` counts as having none); otherwise the run is
|
|
67
|
+
held for review.
|
|
68
|
+
|
|
41
69
|
### 2.10.0
|
|
42
70
|
|
|
43
71
|
Needs the current production API for these fields to be populated. On an older API they are simply absent.
|
|
@@ -443,6 +471,8 @@ const summary = await job.getSummary();
|
|
|
443
471
|
console.log(summary.intentAssessment?.summary);
|
|
444
472
|
console.log(`Bugs: ${summary.intentAssessment?.bugCount}`);
|
|
445
473
|
// What's behind a `review` or `fail`, worst first (needs API 2.10.0+): summary.intentAssessment?.reasons
|
|
474
|
+
// If an expectedChanges item never showed up, reasons includes 'intended_change_not_found'
|
|
475
|
+
// and summary.intentAssessment?.missingExpectedChanges names which ones.
|
|
446
476
|
|
|
447
477
|
// Per-result verdict, plus the per-region verdicts behind it
|
|
448
478
|
for (const r of summary.regressions) {
|
|
@@ -496,7 +526,7 @@ const status = await job.waitForCompletion(3000, (s) => {
|
|
|
496
526
|
|
|
497
527
|
### Auto-Approval
|
|
498
528
|
|
|
499
|
-
Set `.autoApprove()` to automatically promote screenshots to baselines when the job completes. Useful for scheduled health checks or first-run baseline seeding.
|
|
529
|
+
Set `.autoApprove()` to automatically promote screenshots to baselines when the job completes. Useful for scheduled health checks or first-run baseline seeding. When the run carries a `runContext`, auto-approve fires only if every changed page has an AI verdict (a page skipped under `aiSummaryThreshold` counts as having none); otherwise the run is held for review.
|
|
500
530
|
|
|
501
531
|
```typescript
|
|
502
532
|
const job = await rb
|
package/dist/cli.js
CHANGED
|
@@ -81,6 +81,11 @@ function printChangeClusters(clusters) {
|
|
|
81
81
|
console.log(`- ${label}: ${c.pageCount} pages`);
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
|
+
function printPathCoverage(coverage) {
|
|
85
|
+
if (!coverage)
|
|
86
|
+
return;
|
|
87
|
+
console.log(`\nπΊοΈ Coverage: ${coverage.checkedPaths.length} path(s) via ${coverage.source}${coverage.partial ? ' (partial)' : ''}${coverage.reason ? ` β ${coverage.reason}` : ''}`);
|
|
88
|
+
}
|
|
84
89
|
/**
|
|
85
90
|
* A mistake in how the command was invoked, rather than a run that worked and found
|
|
86
91
|
* something. Exits 2 so CI can tell the two apart β a pipeline that treats "regressions
|
|
@@ -98,6 +103,9 @@ function printIntent(summary) {
|
|
|
98
103
|
if (ia.reasons && ia.reasons.length > 0) {
|
|
99
104
|
console.log(` reasons: ${ia.reasons.join(', ')}`);
|
|
100
105
|
}
|
|
106
|
+
if (ia.missingExpectedChanges && ia.missingExpectedChanges.length > 0) {
|
|
107
|
+
console.log(` missing expected changes: ${ia.missingExpectedChanges.join(', ')}`);
|
|
108
|
+
}
|
|
101
109
|
}
|
|
102
110
|
function parseFailOn(raw) {
|
|
103
111
|
if (raw === undefined)
|
|
@@ -405,6 +413,7 @@ Waiting for completion...
|
|
|
405
413
|
}
|
|
406
414
|
}
|
|
407
415
|
printChangeClusters(summary.changeClusters);
|
|
416
|
+
printPathCoverage(summary.pathCoverage);
|
|
408
417
|
printIntent(summary);
|
|
409
418
|
printUsage(summary.usage);
|
|
410
419
|
// Reported even when there are regressions. These used to be hidden behind an
|
|
@@ -453,6 +462,7 @@ Errors: ${summary.errorCount}
|
|
|
453
462
|
});
|
|
454
463
|
}
|
|
455
464
|
printChangeClusters(summary.changeClusters);
|
|
465
|
+
printPathCoverage(summary.pathCoverage);
|
|
456
466
|
printIntent(summary);
|
|
457
467
|
printUsage(summary.usage);
|
|
458
468
|
const doDownload = options.download || options['download-full'];
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { VRConfig, Viewport, Viewports, JobStatus, JobSummary, JobProgress, PageResult, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage } from './types';
|
|
1
|
+
import { VRConfig, Viewport, Viewports, JobStatus, JobSummary, JobProgress, PageResult, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage, PathCoverage } from './types';
|
|
2
2
|
import { sanitizeFilename, sanitizeUrlToPath } from './security';
|
|
3
3
|
export { sanitizeFilename, sanitizeUrlToPath, Viewports };
|
|
4
|
-
export type { PageResult, JobProgress, JobStatus, JobSummary, Viewport, VRConfig, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage };
|
|
4
|
+
export type { PageResult, JobProgress, JobStatus, JobSummary, Viewport, VRConfig, ProjectConfig, ProjectPath, ProjectScan, JobAiSummary, RegionVerdict, ResultVerdict, VerdictDecision, RegressionbotSummaryItem, RunContext, IntentAssessment, BaselinePolicy, ProjectSchedule, ProjectConfigUpdate, ProjectUpdateResult, EnvGate, SummaryStatus, ApproveResult, ApproveWhere, Change, ChangeType, StyleDelta, ChangeBox, UsageBlock, QuotaExceededBody, GateReason, AssessmentStatus, ResultAssessment, RunCoverage, ChangeCluster, ChangeClusterPage, PathCoverage };
|
|
5
5
|
export declare class RegressionBot {
|
|
6
6
|
private apiKey;
|
|
7
7
|
private apiUrl;
|
|
@@ -47,6 +47,7 @@ export declare class RegressionBot {
|
|
|
47
47
|
/** CSS injected before screenshotting. Max 4096 characters. */
|
|
48
48
|
customCss?: string;
|
|
49
49
|
concurrency?: number;
|
|
50
|
+
/** Automatically promote all captures to baselines. When the run carries an intent (`runContext`), auto-approve fires only if every changed page has an AI verdict (a page skipped under `aiSummaryThreshold` counts as having none); otherwise the run is held for review. */
|
|
50
51
|
autoApprove?: boolean;
|
|
51
52
|
/** What this run is testing, used to judge whether changes were intended. */
|
|
52
53
|
runContext?: RunContext;
|
|
@@ -94,6 +95,7 @@ export declare class JobBuilder {
|
|
|
94
95
|
* raising it is a load decision about someone else's environment.
|
|
95
96
|
*/
|
|
96
97
|
concurrency(n: number): this;
|
|
98
|
+
/** Automatically promote all captures to baselines. When the run carries an intent (`runContext`), auto-approve fires only if every changed page has an AI verdict (a page skipped under `aiSummaryThreshold` counts as having none); otherwise the run is held for review. */
|
|
97
99
|
autoApprove(val?: boolean): this;
|
|
98
100
|
mask(selectors: string[]): this;
|
|
99
101
|
/**
|
package/dist/index.js
CHANGED
|
@@ -145,6 +145,7 @@ class JobBuilder {
|
|
|
145
145
|
this.manifest.concurrency = n;
|
|
146
146
|
return this;
|
|
147
147
|
}
|
|
148
|
+
/** Automatically promote all captures to baselines. When the run carries an intent (`runContext`), auto-approve fires only if every changed page has an AI verdict (a page skipped under `aiSummaryThreshold` counts as having none); otherwise the run is held for review. */
|
|
148
149
|
autoApprove(val = true) {
|
|
149
150
|
this.manifest.autoApprove = val;
|
|
150
151
|
return this;
|
package/dist/types.d.ts
CHANGED
|
@@ -668,6 +668,30 @@ export interface JobSummary {
|
|
|
668
668
|
* from the API's own JSDoc instead. Needs API 2.10.0 or later.
|
|
669
669
|
*/
|
|
670
670
|
changeClusters: ChangeCluster[];
|
|
671
|
+
/**
|
|
672
|
+
* Which paths the run checked, from where, and whether that was all it was asked to check.
|
|
673
|
+
* Not yet in the API's published openapi.yaml schema; this doc comment is taken from the
|
|
674
|
+
* API's own JSDoc instead.
|
|
675
|
+
*/
|
|
676
|
+
pathCoverage?: PathCoverage;
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Which pages a run covered and why. `partial` is the part a caller must not miss: the run did
|
|
680
|
+
* not look at the whole site, so a clean result is not "nothing else broke".
|
|
681
|
+
*/
|
|
682
|
+
export interface PathCoverage {
|
|
683
|
+
/** `paths`: the caller listed them. `sitemap`: read from the sitemap. `fallback`: nothing
|
|
684
|
+
* usable was given or found, so only "/". */
|
|
685
|
+
source: 'paths' | 'sitemap' | 'fallback';
|
|
686
|
+
checkedPaths: string[];
|
|
687
|
+
/** Sitemap paths the scans matched before the scan limit; absent when no sitemap was read. */
|
|
688
|
+
totalDiscovered?: number;
|
|
689
|
+
truncatedByLimit: boolean;
|
|
690
|
+
/** The run did not check the whole of what it was asked to (fallback, unreadable sitemap,
|
|
691
|
+
* scan limit). */
|
|
692
|
+
partial: boolean;
|
|
693
|
+
reason?: string;
|
|
694
|
+
sitemapUrl?: string;
|
|
671
695
|
}
|
|
672
696
|
export interface ChangeClusterPage {
|
|
673
697
|
url: string;
|
|
@@ -726,14 +750,29 @@ export interface IntentAssessment {
|
|
|
726
750
|
/**
|
|
727
751
|
* Every condition behind `decision`, worst first; empty on a clean pass. Tells a review
|
|
728
752
|
* caused by a page that needs a person from one caused by pages that failed or were never
|
|
729
|
-
* assessed.
|
|
753
|
+
* assessed. `partial_coverage` means the checked pages were clean but the run covered only
|
|
754
|
+
* part of the site (for example a sitemap that could not be read or hit the page cap), so a
|
|
755
|
+
* would-be pass became a review. Needs API 2.10.0 or later.
|
|
730
756
|
*/
|
|
731
757
|
reasons?: GateReason[];
|
|
758
|
+
/**
|
|
759
|
+
* Present when `reasons` has `intended_change_not_found`: each `expectedChanges` item the
|
|
760
|
+
* run did not show, or the intent's own words when it listed none. The decision is
|
|
761
|
+
* `review`, never `pass`.
|
|
762
|
+
*/
|
|
763
|
+
missingExpectedChanges?: string[];
|
|
732
764
|
/** What the run compared and how much of it the verdict covers. Needs API 2.10.0 or later. */
|
|
733
765
|
coverage?: RunCoverage;
|
|
734
766
|
}
|
|
735
767
|
/** Every condition {@link IntentAssessment.decision} can be built from, worst first. */
|
|
736
|
-
export type GateReason = 'no_intent' | 'nothing_compared' | 'bug' | 'needs_review'
|
|
768
|
+
export type GateReason = 'no_intent' | 'nothing_compared' | 'bug' | 'needs_review'
|
|
769
|
+
/** An intent expecting a visible change (`expectedChanges`, or text that is not an explicit
|
|
770
|
+
* "no visual change") did not show it β zero diffs and all-noise runs included. See
|
|
771
|
+
* {@link IntentAssessment.missingExpectedChanges}. */
|
|
772
|
+
| 'intended_change_not_found' | 'assessment_in_progress' | 'assessment_incomplete' | 'device_disagreement' | 'comparison_failures'
|
|
773
|
+
/** Not from the gate itself: added when a would-be pass is turned into a review because the
|
|
774
|
+
* run covered only part of the site. See {@link JobSummary.pathCoverage}. */
|
|
775
|
+
| 'partial_coverage';
|
|
737
776
|
/**
|
|
738
777
|
* Where one changed page stands with the verdict.
|
|
739
778
|
*
|
|
@@ -750,9 +789,9 @@ export interface ResultAssessment {
|
|
|
750
789
|
status: AssessmentStatus;
|
|
751
790
|
/**
|
|
752
791
|
* Why, when status is not `assessed`. One of no_intent, analysis_in_progress, timeout,
|
|
753
|
-
* model_error, malformed_response, incomplete_response, capped, pass_error,
|
|
792
|
+
* model_error, malformed_response, incomplete_response, capped, deferred, pass_error,
|
|
754
793
|
* pass_interrupted, no_assessor, summary_failed, below_ai_threshold, not_analysed,
|
|
755
|
-
* not_reached.
|
|
794
|
+
* not_reached. `deferred` means the analysis ran short of time and will be retried.
|
|
756
795
|
*/
|
|
757
796
|
reason?: string;
|
|
758
797
|
}
|
package/package.json
CHANGED