@testrelic/core 2.14.0-next.114 → 2.16.0-next.124

package/dist/index.d.ts

@@ -4,7 +4,7 @@
  * All interfaces for the JSON output schema, reporter configuration,
  * and internal fixture-to-reporter communication.
  *
- * Schema Version: 1.0.0
+ * Schema Version: 1.6.0
  */
 type NavigationType = 'goto' | 'navigation' | 'back' | 'forward' | 'refresh' | 'spa_route' | 'spa_replace' | 'hash_change' | 'link_click' | 'form_submit' | 'redirect' | 'popstate' | 'page_load' | 'manual_record' | 'fallback' | 'dummy';
 interface TestRunReport {
@@ -56,6 +56,30 @@ interface Summary {
     readonly totalTimelineSteps: number;
     readonly totalActionSteps: number;
     readonly actionStepsByCategory: Record<string, number>;
+    /** Visual baseline comparisons across the run. */
+    readonly visualChecks?: number;
+    /** Of those, how many differed from their baseline. */
+    readonly visualFailures?: number;
+    /** Of those, how many had no baseline and wrote one. */
+    readonly visualNew?: number;
+    /** DOM elements that differed across the run's visual comparisons. */
+    readonly visualDomChanges?: number;
+    /** Action steps across the run, bucketed by the driver's own category. */
+    readonly actionStepsByKind?: Record<string, number>;
+    /** Device-log lines captured across the run (mobile). */
+    readonly deviceLogLines?: number;
+    /** Of those, how many were error or fatal. */
+    readonly deviceLogErrors?: number;
+    /** Of those, how many came from a crash log. */
+    readonly deviceLogCrashes?: number;
+    /** Screenshots captured across the run. */
+    readonly screenshotCount?: number;
+    /** Screen recordings captured across the run. */
+    readonly videoCount?: number;
+    /** AI-detected defects across the run, by severity. */
+    readonly aiDefectsBySeverity?: Record<string, number>;
+    /** Distinct devices the run touched. */
+    readonly devices?: readonly DeviceInfo[];
 }
 interface CIMetadata {
     readonly provider: CIProvider;
@@ -99,6 +123,12 @@ interface TestResult {
     readonly apiAssertions: readonly ApiAssertion[] | null;
     readonly actions: readonly ActionStep[] | null;
     readonly consoleLogs: readonly ConsoleLogEntry[] | null;
+    /** Visual baseline comparisons recorded during this test, if any. */
+    readonly visual?: readonly VisualComparison[] | null;
+    /** The device this test drove (mobile runs only). */
+    readonly device?: DeviceInfo | null;
+    /** AI-detected defects attributed to this test. */
+    readonly aiDefects?: readonly AiDefect[] | null;
 }
 type ResourceType = 'xhr' | 'document' | 'script' | 'stylesheet' | 'image' | 'font' | 'other';
 interface CapturedNetworkRequest {
@@ -121,6 +151,15 @@ interface CapturedNetworkRequest {
 }
 interface TestArtifacts {
     readonly screenshot?: string;
+    /**
+     * Every screenshot captured for this test, report-relative.
+     *
+     * `screenshot` is the first one and stays for compatibility. Mobile
+     * collectors have always gathered an array — one per step, or one per
+     * failure — and only index 0 survived: the artifact copier wrote a single
+     * `screenshot.png` and dropped the rest before the report could ask.
+     */
+    readonly screenshots?: readonly string[];
     readonly video?: string;
     /**
      * Playwright's trace archive — the DOM snapshots, network log and screencast
@@ -138,8 +177,183 @@ interface TestArtifacts {
     readonly videoKey?: string;
     readonly traceKey?: string;
 }
+/**
+ * Outcome of a single visual baseline comparison.
+ *
+ * `new` is a snapshot Playwright had no baseline for and therefore wrote one
+ * for — it is not a pass, because nothing was actually compared. `updated`
+ * means the run was invoked with `--update-snapshots` and the baseline was
+ * rewritten, which likewise compares nothing.
+ */
+type VisualStatus = 'passed' | 'failed' | 'new' | 'updated';
+/** Which assertion produced a comparison. */
+type VisualSource = 'toHaveScreenshot' | 'toMatchSnapshot' | 'toMatchVisualBaseline';
+/**
+ * One visual baseline comparison within a test.
+ *
+ * A test may hold many of these — `TestArtifacts` deliberately is not the home
+ * for them, because it is single-slot per type (one screenshot, one video, one
+ * trace) while a single test can assert against any number of named snapshots.
+ *
+ * Every image field is a report-relative POSIX path, matching `TestArtifacts`.
+ * A passing comparison carries no `actual` or `diff`: Playwright attaches
+ * nothing when the images match, so there is nothing to copy.
+ */
+interface VisualComparison {
+    /** Snapshot identity within the test, e.g. `home` or `checkout/cart`. */
+    readonly name: string;
+    readonly status: VisualStatus;
+    readonly source: VisualSource;
+    /** The committed baseline image. */
+    readonly baseline?: string;
+    /** What this run rendered. Absent when the comparison passed. */
+    readonly actual?: string;
+    /** Playwright's rendered diff. Present only on failure. */
+    readonly diff?: string;
+    /** The previous attempt's render, on a retried failure. */
+    readonly previous?: string;
+    /** Baseline dimensions, read from the PNG header. */
+    readonly width?: number;
+    readonly height?: number;
+    /** Actual dimensions — set only when they differ from the baseline. */
+    readonly actualWidth?: number;
+    readonly actualHeight?: number;
+    /** Differing pixel count, as reported by Playwright's comparator. */
+    readonly diffPixels?: number;
+    /** `diffPixels / (width * height)`, computed at full precision. */
+    readonly diffPixelRatio?: number;
+    /** Cloud storage keys, populated after artifact upload. */
+    readonly baselineKey?: string;
+    readonly actualKey?: string;
+    readonly diffKey?: string;
+    /**
+     * Which elements changed, ranked by the area they affect.
+     *
+     * Present only when a DOM baseline existed to compare against. Capped — see
+     * `domChangeCount` for the number found before the cap.
+     */
+    readonly domChanges?: readonly DomChange[];
+    /** Total changes found, which may exceed `domChanges.length`. */
+    readonly domChangeCount?: number;
+    /**
+     * Of those, how many actually changed rather than merely shifting position.
+     *
+     * The gap between this and `domChangeCount` is the layout knock-on: one card
+     * gaining a margin moves everything below it, and a report that presents all
+     * of those as equals hides the single edit that caused them.
+     */
+    readonly domRootCauses?: number;
+    /** True when the DOM capture itself hit a node or byte cap. */
+    readonly domTruncated?: boolean;
+    /** Report-relative paths to the two captured DOM snapshots. */
+    readonly baselineDom?: string;
+    readonly actualDom?: string;
+    /**
+     * The document-space region the screenshot covered: `[x, y, width, height]`.
+     *
+     * Element rectangles are in document coordinates, so the report needs this to
+     * place a box over the image — for a locator screenshot the image covers only
+     * that element's box, not the viewport.
+     */
+    readonly domCaptureBox?: readonly [number, number, number, number];
+    /** Cloud storage keys for the DOM snapshots. */
+    readonly baselineDomKey?: string;
+    readonly actualDomKey?: string;
+}
+/** How an element differs between the baseline render and this one. */
+type DomChangeKind = 'added' | 'removed' | 'moved' | 'resized' | 'restyled' | 'text'
+/** Its id or class list changed — often the real story behind a restyle. */
+ | 'attrs';
+/** One computed style property that differs. */
+interface DomStyleChange {
+    readonly prop: string;
+    readonly from: string;
+    readonly to: string;
+}
+/**
+ * One element that differs between two renders.
+ *
+ * Deliberately shaped like the platform's existing `RegionEvidence` — rank,
+ * bbox, area share, a kind and a human label — so that a DOM overlay becomes
+ * another caller of the dashboard's existing box-overlay component rather than
+ * a second renderer built alongside it.
+ */
+interface DomChange {
+    /** 1-based, ordered by affected area descending. Drives the overlay colour. */
+    readonly rank: number;
+    /** Structural path used to match the element across the two snapshots. */
+    readonly path: string;
+    /** What a human reads, e.g. `div.card.accent`. */
+    readonly label: string;
+    readonly kinds: readonly DomChangeKind[];
+    /** Document-space `[x, y, width, height]` in this render. */
+    readonly rect?: readonly [number, number, number, number];
+    /** The same, in the baseline. Absent for an added element. */
+    readonly baselineRect?: readonly [number, number, number, number];
+    /** Share of the capture box this element occupies, 0–1. */
+    readonly areaPct: number;
+    readonly styles?: readonly DomStyleChange[];
+    readonly text?: {
+        readonly from: string;
+        readonly to: string;
+    };
+    /** Identity attributes that changed, e.g. `class: "card" → "banner"`. */
+    readonly attrs?: readonly DomStyleChange[];
+    /**
+     * True when this element only moved or resized, in a comparison where
+     * something else actually changed.
+     *
+     * Shifting one card down by 24px shifts everything below it too, so a naive
+     * list reports eleven changes for one edit. This flags the ten that merely
+     * got pushed around, so a report can lead with the one that did something.
+     *
+     * It is a statement about this element — that nothing about it changed except
+     * its position — and deliberately not a claim to have proven what moved it.
+     */
+    readonly consequence?: boolean;
+}
 type TestStatus = 'passed' | 'failed' | 'skipped' | 'flaky' | 'timedout';
 type TestType = 'e2e' | 'api' | 'unit' | 'mobile' | 'unknown';
+/** Platform a mobile test drove. */
+type MobilePlatform = 'android' | 'ios' | 'web' | 'unknown';
+/** How the app under test was driven. */
+type AppTestMode = 'native' | 'hybrid' | 'mobile-web';
+/**
+ * The device a mobile test ran on.
+ *
+ * Appium detects all of this from the W3C capabilities and Maestro reads it
+ * from the CLI flags and driver log. Both used to collapse it to two words in
+ * `tags` before the report saw it; this carries it through intact.
+ */
+interface DeviceInfo {
+    readonly platform: MobilePlatform;
+    /** OS version, e.g. "14" or "17.5". */
+    readonly platformVersion: string | null;
+    /** Human-facing device name, e.g. "Pixel 4a" or "iPhone 15". */
+    readonly deviceName: string | null;
+    /** Driver, e.g. "UiAutomator2", "XCUITest", "maestro". */
+    readonly automationName: string | null;
+    /** App package / bundle id / apk path under test. */
+    readonly appId: string | null;
+    /** Native, hybrid or mobile-web. Null when the runner cannot tell. */
+    readonly appTestMode: AppTestMode | null;
+    /** Device serial or UDID, e.g. "emulator-5554". */
+    readonly udid: string | null;
+}
+/** Severity of an AI-detected defect. */
+type AiDefectSeverity = 'critical' | 'warning' | 'info';
+/** Category of an AI-detected defect. */
+type AiDefectType = 'ui' | 'spelling' | 'i18n' | 'layout' | 'accessibility';
+/** A defect reported by an AI analysis pass over a captured screen. */
+interface AiDefect {
+    readonly type: AiDefectType;
+    readonly severity: AiDefectSeverity;
+    readonly description: string;
+    /** Where in the app it was seen, when the analyser says. */
+    readonly location: string | null;
+    /** Report-relative path to the screenshot it was found on. */
+    readonly screenshot: string | null;
+}
 interface FailureDiagnostic {
     readonly message: string;
     readonly line: number | null;
@@ -218,6 +432,13 @@ interface AssertionLocation {
     /** Column number (1-based), if available */
     readonly column?: number;
 }
+/** A failed assertion reduced to what a summary row needs. */
+interface FailedAssertionSummary {
+    readonly type: AssertionType;
+    readonly expected: string;
+    readonly actual: string;
+    readonly expression?: string;
+}
 /** A single captured assertion on API response data. */
 interface ApiAssertion {
     /** Links to ApiCallRecord.id from the API proxy */
@@ -309,10 +530,24 @@ type TimelineStep = NavigationStep | ApiCallStep;
  * distinct bucket from generic `custom_step`.
  */
 type ActionCategory = 'ui_action' | 'assertion' | 'navigation' | 'custom_step';
+/**
+ * The driver's own step category, preserved alongside `ActionCategory`.
+ *
+ * `ActionCategory` is the four-bucket vocabulary the platform's timeline and
+ * Ask AI tools filter on, and it stays closed. Mobile drivers distinguish far
+ * more than four kinds — Appium reports eleven, Maestro ten — and flattening
+ * them made a swipe, an app launch and a screenshot render identically. This
+ * keeps the finer label for the report to badge.
+ */
+type StepKind = 'find' | 'interaction' | 'gesture' | 'navigation' | 'lifecycle' | 'context' | 'device' | 'keyboard' | 'alert' | 'screenshot' | 'assertion' | 'media' | 'script' | 'flow_control' | 'ai' | 'other';
 /** A single captured test action step. */
 interface ActionStep {
     readonly title: string;
     readonly category: ActionCategory;
+    /** The driver's own finer-grained category, when it reported one. */
+    readonly kind?: StepKind | null;
+    /** Element selector this step acted on, when the driver reported one. */
+    readonly selector?: string | null;
     readonly timestamp: string;
     readonly duration: number;
     readonly videoOffset: number | null;
@@ -323,6 +558,14 @@ interface ActionStep {
 /** Level of a captured console/terminal log message. */
 type ConsoleLogLevel = 'log' | 'warn' | 'error' | 'info' | 'debug' | 'stdout' | 'stderr';
 /** A single captured console or terminal log entry. */
+/**
+ * Where a captured log line came from.
+ *
+ * Browser consoles only ever produce `console`. Mobile runs mix several
+ * streams, and telling a crash log from ordinary logcat chatter is the whole
+ * point of showing them — so the stream survives to the row.
+ */
+type LogSource = 'console' | 'logcat' | 'syslog' | 'crashlog' | 'browser' | 'safariConsole' | 'performance' | 'server' | 'bidi' | 'driver';
 interface ConsoleLogEntry {
     /** Log level or stream type */
     readonly level: ConsoleLogLevel;
@@ -332,6 +575,14 @@ interface ConsoleLogEntry {
     readonly timestamp: string;
     /** Source file location (browser console only, best-effort) */
     readonly location: string | null;
+    /** Which stream produced the line. Defaults to `console` when absent. */
+    readonly source?: LogSource | null;
+    /** Emitting tag, e.g. an Android logcat tag like "ActivityManager". */
+    readonly tag?: string | null;
+    /** Emitting process id, when the stream reports one. */
+    readonly pid?: number | null;
+    /** Stack trace attached to the line (crash logs). */
+    readonly stackTrace?: string | null;
 }
 /** Report generation mode. */
 type ReportMode = 'streaming' | 'embedded' | 'auto';
@@ -466,6 +717,14 @@ interface TestRelicFilePayload {
     readonly apiCallsFile: string | null;
     /** Number of API call items in the file. */
     readonly apiCallsCount: number;
+    /**
+     * Response-status mix of the calls in the file.
+     *
+     * Tallied as each record is appended. The file itself is never read back
+     * into memory in streaming mode, so an aggregate computed at write time is
+     * the only way the grid can show a row's status chips.
+     */
+    readonly apiStatusMix?: ApiCallsByStatusRange | null;
 }
 /** Type guard: validates that a parsed object is a v2 file-based payload. */
 declare function isTestRelicFilePayload(obj: unknown): obj is TestRelicFilePayload;
@@ -562,7 +821,7 @@ interface ArtifactFile {
     /** File name (e.g., `screenshot.png`, `video.webm`, `trace.zip`) */
     readonly name: string;
     /** Artifact type for rendering */
-    readonly type: 'screenshot' | 'video' | 'trace' | 'other';
+    readonly type: 'screenshot' | 'video' | 'trace' | 'baseline' | 'diff' | 'dom' | 'other';
     /** Path relative to the report output directory */
     readonly relativePath: string;
     /** File size in bytes */
@@ -625,6 +884,8 @@ interface TestIndexEntry {
     readonly retryIndex: number;
     /** Test tags */
     readonly tags: readonly string[];
+    /** What kind of test this is — badges the row, and picks the row's evidence. */
+    readonly testType?: TestType;
     /** Whether this test has network request data */
     readonly hasNetworkData: boolean;
     /** Whether this test has console log data */
@@ -633,6 +894,20 @@ interface TestIndexEntry {
     readonly hasArtifacts: boolean;
     /** Whether this test has action step data */
     readonly hasActionSteps: boolean;
+    /** Number of visual baseline comparisons recorded */
+    readonly visualCount?: number;
+    /** Of those, how many differed from their baseline */
+    readonly visualFailedCount?: number;
+    /** DOM elements that differed across this test's comparisons */
+    readonly domChangeCount?: number;
+    /**
+     * Report-relative path to a diff image for this test, when one exists.
+     *
+     * Carried on the index so the run overview can show the failure itself
+     * rather than only describing it — the grid must never fetch per-test
+     * detail files to render.
+     */
+    readonly visualDiffPath?: string;
     /** First line of error message (failed/flaky tests only) */
     readonly errorMessage: string | null;
     /** Number of network requests captured */
@@ -641,6 +916,34 @@ interface TestIndexEntry {
     readonly consoleCount: number;
     /** Number of action steps captured */
     readonly actionCount: number;
+    /** API calls captured for this test. */
+    readonly apiCallCount?: number;
+    /** Response-status mix of those calls, for the row's status chips. */
+    readonly apiStatusMix?: ApiCallsByStatusRange;
+    /** API assertions that passed during this test. */
+    readonly assertionsPassed?: number;
+    /** API assertions that failed during this test. */
+    readonly assertionsFailed?: number;
+    /**
+     * The first failed assertion, flattened to strings.
+     *
+     * The run overview leads with "expected 101, received undefined" and must
+     * do so without opening the test's detail file. The first line of an API
+     * failure message is the matcher wrapper, which says nothing.
+     */
+    readonly firstFailedAssertion?: FailedAssertionSummary;
+    /** The device this test drove (mobile runs only). */
+    readonly device?: DeviceInfo | null;
+    /** Action steps for this test, bucketed by the driver's own category. */
+    readonly stepKinds?: Record<string, number>;
+    /** Device-log errors and fatals recorded during this test. */
+    readonly deviceLogErrors?: number;
+    /** Whether a crash-log line was recorded during this test. */
+    readonly hasCrashLog?: boolean;
+    /** Whether a screen recording exists for this test. */
+    readonly hasVideo?: boolean;
+    /** AI-detected defects attributed to this test. */
+    readonly aiDefectCount?: number;
     /** Whether this is a retry attempt (not the final attempt) */
     readonly isRetry: boolean;
     /** Stable key shared by every attempt of the same test (same across retries, unlike `id`) */
@@ -664,6 +967,8 @@ interface TestDetailData {
     readonly actions: readonly ActionStep[];
     /** References to screenshot/video files */
     readonly artifacts: TestArtifacts | null;
+    /** Visual baseline comparisons (inline — a handful per test at most) */
+    readonly visual?: readonly VisualComparison[] | null;
     /** Error details for failed/flaky tests */
     readonly failureDiagnostic: FailureDiagnostic | null;
     /** Whether network.jsonl exists for this test */
@@ -680,6 +985,10 @@ interface TestDetailData {
     readonly apiCallsCount: number;
     /** Test start time (ISO 8601) — used for video sync */
     readonly startedAt?: string;
+    /** The device this test drove (mobile runs only). */
+    readonly device?: DeviceInfo | null;
+    /** AI-detected defects attributed to this test. */
+    readonly aiDefects?: readonly AiDefect[] | null;
 }
 /** Streaming report summary — running counters written to summary.json. */
 interface StreamingReportSummary {
@@ -833,4 +1142,4 @@ declare function isValidCloudConfig(input: unknown): input is CloudConfig;
 declare function isValidQueueEntry(input: unknown): input is QueueEntry;
 declare function isValidTestRunReport(value: unknown): value is TestRunReport;
 
-export { ATTACHMENT_CONTENT_TYPE, ATTACHMENT_NAME, type ActionCategory, type ActionStep, type ApiAssertion, type ApiCallRecord, type ApiCallStep, type ApiCallStepRequest, type ApiCallStepResponse, type ArtifactFile, type ArtifactRunEntry, type ArtifactRunManifest, type ArtifactTestEntry, type AssertionLocation, type AssertionType, type AuthMode, type AuthState, type CIMetadata, type CIProvider, type CapturedNetworkRequest, type CloudConfig, type CloudReporterOptions, type ConsoleLogEntry, type ConsoleLogLevel, ErrorCode, type FailureDiagnostic, type GitMetadata, type LogLevel, type Logger, type LoggerOptions, type MergeOptions, type NavigationAnnotation, type NavigationStep, type NavigationType, type NetworkStats, PAYLOAD_VERSION, PAYLOAD_VERSION_LEGACY, type QueueEntry, type RepoResolution, type ReportMode, type ReporterConfig, type ResourceBreakdown, type ResourceType, type StepAssertion, type StepTestIdentity, type StreamingFileStats, type StreamingReportManifest, type StreamingReportSummary, type StreamingValidation, type StreamingWriteError, type Summary, type TestArtifacts, type TestDetailData, type TestIndexEntry, type TestRelicDataPayload, TestRelicError, type TestRelicFilePayload, type TestResult, type TestRunReport, type TestStatus, type TestType, type TimelineEntry, type TimelineStep, type UploadStrategy, createError, createLogger, isTestRelicDataPayload, isTestRelicFilePayload, isValidApiKeyFormat, isValidCloudConfig, isValidConfig, isValidEndpointUrl, isValidNavigationType, isValidQueueEntry, isValidTestRunReport, isValidUploadStrategy };
+export { ATTACHMENT_CONTENT_TYPE, ATTACHMENT_NAME, type ActionCategory, type ActionStep, type AiDefect, type AiDefectSeverity, type AiDefectType, type ApiAssertion, type ApiCallRecord, type ApiCallStep, type ApiCallStepRequest, type ApiCallStepResponse, type ApiCallsByStatusRange, type ApiResponseTimeStats, type AppTestMode, type ArtifactFile, type ArtifactRunEntry, type ArtifactRunManifest, type ArtifactTestEntry, type AssertionLocation, type AssertionType, type AuthMode, type AuthState, type CIMetadata, type CIProvider, type CapturedCookie, type CapturedNetworkRequest, type CloudConfig, type CloudReporterOptions, type ConsoleLogEntry, type ConsoleLogLevel, type DeviceInfo, type DomChange, type DomChangeKind, type DomStyleChange, ErrorCode, type FailedAssertionSummary, type FailureDiagnostic, type GitMetadata, type LogLevel, type LogSource, type Logger, type LoggerOptions, type MergeOptions, type MobilePlatform, type NavigationAnnotation, type NavigationStep, type NavigationType, type NetworkStats, PAYLOAD_VERSION, PAYLOAD_VERSION_LEGACY, type QueueEntry, type RepoResolution, type ReportMode, type ReporterConfig, type ResourceBreakdown, type ResourceType, type StepAssertion, type StepKind, type StepTestIdentity, type StreamingFileStats, type StreamingReportManifest, type StreamingReportSummary, type StreamingValidation, type StreamingWriteError, type Summary, type TestArtifacts, type TestDetailData, type TestIndexEntry, type TestRelicDataPayload, TestRelicError, type TestRelicFilePayload, type TestResult, type TestRunReport, type TestStatus, type TestType, type TimelineEntry, type TimelineStep, type UploadStrategy, type VisualComparison, type VisualSource, type VisualStatus, createError, createLogger, isTestRelicDataPayload, isTestRelicFilePayload, isValidApiKeyFormat, isValidCloudConfig, isValidConfig, isValidEndpointUrl, isValidNavigationType, isValidQueueEntry, isValidTestRunReport, isValidUploadStrategy };