@trailmark/playwright 0.1.0

package/src/reporter/index.ts

@@ -0,0 +1,572 @@
+import { existsSync } from "node:fs"
+import type {
+  FullConfig,
+  FullResult,
+  Reporter,
+  TestCase,
+  TestResult,
+} from "@playwright/test/reporter"
+import {
+  BaselineFile,
+  DiffResult,
+  TrailmarkArtifactIndex,
+  TrailmarkAttachment,
+  TrailmarkRun,
+  buildBaselineFromRun,
+  buildReport,
+  computeConfigHash,
+  createEmptyRun,
+  diffRuns,
+  formatConsoleReport,
+  hashObject,
+} from "@trailmark/core"
+import { resolveArtifactOptions } from "../artifacts/options"
+import { TrailmarkMode, TrailmarkReporterOptions } from "../types"
+import { readAttachmentPayload } from "./attachments"
+import { readBaseline, writeJsonAtomic } from "./io"
+import { applyUrlNormalization, detectIdentityCollisions } from "./normalization"
+import { summarizeScanErrors } from "./output"
+import { withRunShape } from "./run"
+import { resolveMode } from "../mode"
+import { ReporterBaselineError } from "./errors"
+
+export const DEFAULT_BASELINE_PATH = ".trailmark/baseline.json"
+export const DEFAULT_REPORT_PATH = ".trailmark/report.json"
+export const CURRENT_RUN_PATH = ".trailmark/current-run.json"
+
+interface NormalizedReporterOptions {
+  baselinePath: string
+  reportPath: string
+  mode: TrailmarkMode
+  failOnImpact: string[]
+  failOnScanError: boolean
+  onConfigMismatch: "warn" | "error" | "ignore"
+  verbose: boolean
+  url: TrailmarkReporterOptions["url"]
+  artifacts: ReturnType<typeof resolveArtifactOptions>
+}
+
+type ReporterStatus = { status: "passed" | "failed" } | void
+
+type ConfigMismatch = {
+  baselineConfigHash: string
+  currentConfigHash: string
+  action: "warn" | "error" | "ignore"
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function toStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) {
+    return []
+  }
+
+  return value.filter((entry): entry is string => typeof entry === "string")
+}
+
+function toAttachments(value: unknown): TrailmarkAttachment[] {
+  if (!Array.isArray(value)) {
+    return []
+  }
+
+  return value
+    .filter((entry): entry is Record<string, unknown> => isRecord(entry))
+    .map((entry) => ({
+      kind: typeof entry.kind === "string" ? entry.kind : "unknown",
+      path: typeof entry.path === "string" ? entry.path : undefined,
+      error: typeof entry.error === "string" ? entry.error : undefined,
+    }))
+}
+
+function toArtifactIndex(value: unknown): TrailmarkArtifactIndex {
+  if (!isRecord(value)) {
+    return {}
+  }
+
+  const byOccurrenceId: Record<string, TrailmarkAttachment[]> = {}
+  const byIssueId: Record<string, TrailmarkAttachment[]> = {}
+
+  if (isRecord(value.byOccurrenceId)) {
+    for (const key of Object.keys(value.byOccurrenceId).sort((a, b) => a.localeCompare(b))) {
+      byOccurrenceId[key] = toAttachments(value.byOccurrenceId[key])
+    }
+  }
+
+  if (isRecord(value.byIssueId)) {
+    for (const key of Object.keys(value.byIssueId).sort((a, b) => a.localeCompare(b))) {
+      byIssueId[key] = toAttachments(value.byIssueId[key])
+    }
+  }
+
+  return {
+    byOccurrenceId,
+    byIssueId,
+  }
+}
+
+function mergeArtifactIndex(
+  left: TrailmarkArtifactIndex,
+  right: TrailmarkArtifactIndex,
+): TrailmarkArtifactIndex {
+  const byOccurrenceId: Record<string, TrailmarkAttachment[]> = {
+    ...left.byOccurrenceId,
+    ...right.byOccurrenceId,
+  }
+  const byIssueId: Record<string, TrailmarkAttachment[]> = {
+    ...left.byIssueId,
+    ...right.byIssueId,
+  }
+
+  return {
+    byOccurrenceId,
+    byIssueId,
+  }
+}
+
+function uniqueStrings(values: string[]): string[] {
+  return Array.from(new Set(values)).sort((left, right) => left.localeCompare(right))
+}
+
+function summarizeImpactCounts(counts: Record<string, number>): string {
+  const orderedImpacts = ["critical", "serious", "moderate", "minor", "unknown"]
+  const orderedEntries: Array<[string, number]> = []
+
+  for (const impact of orderedImpacts) {
+    const count = counts[impact]
+    if (!count) {
+      continue
+    }
+
+    orderedEntries.push([impact, count])
+  }
+
+  for (const impact of Object.keys(counts).sort((left, right) => left.localeCompare(right))) {
+    if (orderedImpacts.includes(impact)) {
+      continue
+    }
+
+    const count = counts[impact]
+    if (!count) {
+      continue
+    }
+
+    orderedEntries.push([impact, count])
+  }
+
+  if (orderedEntries.length === 0) {
+    return "none"
+  }
+
+  return orderedEntries.map(([impact, count]) => `${impact}=${count}`).join(", ")
+}
+
+export default class TrailmarkReporter implements Reporter {
+  private readonly options: NormalizedReporterOptions
+
+  private config: FullConfig | undefined
+
+  private sourceRuns = 0
+
+  private seenRunHashes = new Set<string>()
+
+  private mergedRun = this.createMergedRun()
+
+  constructor(options: TrailmarkReporterOptions = {}) {
+    const mode = resolveMode(options)
+    const envVerbose = process.env.TRAILMARK_VERBOSE
+    const verboseFromEnv = envVerbose === "1" || envVerbose === "true"
+
+    this.options = {
+      baselinePath: options.baselinePath ?? DEFAULT_BASELINE_PATH,
+      reportPath: options.reportPath ?? DEFAULT_REPORT_PATH,
+      mode,
+      failOnImpact: options.failOnImpact ?? ["serious", "critical"],
+      failOnScanError: options.failOnScanError ?? true,
+      onConfigMismatch: options.onConfigMismatch ?? "warn",
+      verbose: options.verbose ?? verboseFromEnv,
+      url: options.url ?? {
+        stripHash: true,
+        stripQuery: true,
+      },
+      artifacts: resolveArtifactOptions(mode, options.artifacts),
+    }
+
+    this.syncFixtureEnv()
+  }
+
+  private syncFixtureEnv(): void {
+    process.env.TRAILMARK_MODE = this.options.mode
+    process.env.TRAILMARK_BASELINE_PATH = this.options.baselinePath
+    process.env.TRAILMARK_ARTIFACTS_ENABLED = String(this.options.artifacts.enabled)
+    process.env.TRAILMARK_ARTIFACTS_OUTPUT_DIR = this.options.artifacts.outputDir
+    process.env.TRAILMARK_ARTIFACTS_LIMIT_PER_RUN = String(this.options.artifacts.limitPerRun)
+    process.env.TRAILMARK_ARTIFACTS_CAPTURE = this.options.artifacts.capture
+    process.env.TRAILMARK_ARTIFACTS_INCLUDE_VIEWPORT = String(
+      this.options.artifacts.includeViewport,
+    )
+    process.env.TRAILMARK_ARTIFACTS_INCLUDE_ELEMENT = String(this.options.artifacts.includeElement)
+  }
+
+  private createMergedRun(): TrailmarkRun {
+    return withRunShape(
+      createEmptyRun({
+        metadata: {
+          sourceRuns: 0,
+          artifactRunIds: [],
+          artifactWarnings: [],
+          artifactIndex: {},
+        },
+      }),
+    )
+  }
+
+  private debug(message: string, context?: Record<string, unknown>): void {
+    if (!this.options.verbose) {
+      return
+    }
+
+    const suffix = context ? ` ${JSON.stringify(context)}` : ""
+    console.log(`trailmark:debug ${message}${suffix}`)
+  }
+
+  private appendRun(run: TrailmarkRun): void {
+    const runHash = hashObject({
+      version: run.version,
+      startedAt: run.startedAt,
+      finishedAt: run.finishedAt,
+      tool: run.tool,
+      findings: run.findings,
+      scanErrors: run.scanErrors,
+      metadata: run.metadata,
+    })
+    if (this.seenRunHashes.has(runHash)) {
+      return
+    }
+    this.seenRunHashes.add(runHash)
+
+    this.sourceRuns += 1
+    this.mergedRun.findings.push(...run.findings)
+    this.mergedRun.scanErrors.push(...run.scanErrors)
+
+    const previousMetadata = isRecord(this.mergedRun.metadata) ? this.mergedRun.metadata : {}
+    const runMetadata = isRecord(run.metadata) ? run.metadata : {}
+    const mergedArtifactIndex = mergeArtifactIndex(
+      toArtifactIndex(previousMetadata.artifactIndex),
+      toArtifactIndex(runMetadata.artifactIndex),
+    )
+
+    const artifactRunIds = uniqueStrings([
+      ...toStringArray(previousMetadata.artifactRunIds),
+      ...toStringArray(runMetadata.artifactRunIds),
+      ...(typeof runMetadata.artifactRunId === "string" ? [runMetadata.artifactRunId] : []),
+    ])
+
+    const artifactWarnings = uniqueStrings([
+      ...toStringArray(previousMetadata.artifactWarnings),
+      ...toStringArray(runMetadata.artifactWarnings),
+    ])
+
+    this.mergedRun.metadata = {
+      ...previousMetadata,
+      sourceRuns: this.sourceRuns,
+      scanCount: Number(previousMetadata.scanCount ?? 0) + Number(runMetadata.scanCount ?? 0),
+      skippedScanCount:
+        Number(previousMetadata.skippedScanCount ?? 0) + Number(runMetadata.skippedScanCount ?? 0),
+      artifactRunIds,
+      artifactWarnings,
+      artifactIndex: mergedArtifactIndex,
+    }
+  }
+
+  private currentRunPayloadBase(configHash: string, run: TrailmarkRun): Record<string, unknown> {
+    return {
+      generatedAt: new Date().toISOString(),
+      mode: this.options.mode,
+      configHash,
+      run,
+    }
+  }
+
+  private computeConfigHash(): string {
+    return computeConfigHash({
+      failOnImpact: this.options.failOnImpact,
+      failOnScanError: this.options.failOnScanError,
+      onConfigMismatch: this.options.onConfigMismatch,
+      url: this.options.url,
+      projects: this.config?.projects.map((project) => project.name) ?? [],
+    })
+  }
+
+  private emitCollisionWarnings(finalizedRun: TrailmarkRun, warnings: string[]): void {
+    const collisions = detectIdentityCollisions(finalizedRun.findings)
+    if (collisions.length === 0) {
+      return
+    }
+
+    const collisionSummary = collisions
+      .slice(0, 5)
+      .map((entry) => `${entry.scopeKey}:${entry.ruleId}:${entry.pageKey} x${entry.count}`)
+      .join(", ")
+    const warning =
+      "trailmark: detected potential target-hash collisions in current run. " +
+      `Repeated elements may merge into one issue identity (${collisionSummary}).`
+
+    warnings.push(warning)
+    console.warn(warning)
+  }
+
+  private emitArtifactWarnings(finalizedRun: TrailmarkRun, warnings: string[]): void {
+    if (!isRecord(finalizedRun.metadata)) {
+      return
+    }
+
+    for (const warning of toStringArray(finalizedRun.metadata.artifactWarnings)) {
+      if (warnings.includes(warning)) {
+        continue
+      }
+
+      warnings.push(warning)
+      console.warn(warning)
+    }
+  }
+
+  private handleScanErrors(finalizedRun: TrailmarkRun): boolean {
+    if (finalizedRun.scanErrors.length === 0) {
+      return false
+    }
+
+    summarizeScanErrors(finalizedRun.scanErrors)
+    return this.options.failOnScanError
+  }
+
+  private runDiff(
+    finalizedRun: TrailmarkRun,
+    configHash: string,
+    warnings: string[],
+    errors: string[],
+  ): { diff?: DiffResult; configMismatch?: ConfigMismatch; shouldFail: boolean } {
+    let shouldFail = false
+    let diff: DiffResult | undefined
+    let configMismatch: ConfigMismatch | undefined
+
+    try {
+      const { data: baseline, error, message } = readBaseline(this.options.baselinePath)
+      if (error) {
+        throw new ReporterBaselineError(message)
+      }
+
+      if (baseline.configHash !== configHash) {
+        configMismatch = {
+          baselineConfigHash: baseline.configHash,
+          currentConfigHash: configHash,
+          action: this.options.onConfigMismatch,
+        }
+
+        const mismatchMessage =
+          "trailmark: baseline config hash mismatch detected. " +
+          `baseline=${baseline.configHash} current=${configHash}. ` +
+          "Diff classification may shift if normalization/gating settings changed."
+
+        if (this.options.onConfigMismatch === "warn") {
+          warnings.push(mismatchMessage)
+          console.warn(mismatchMessage)
+        } else if (this.options.onConfigMismatch === "error") {
+          errors.push(mismatchMessage)
+          console.error(mismatchMessage)
+          shouldFail = true
+        }
+      }
+
+      diff = diffRuns(baseline, finalizedRun, {
+        failOnImpact: this.options.failOnImpact,
+        onDebug: this.options.verbose ? this.debug.bind(this) : undefined,
+      })
+    } catch (error) {
+      const message =
+        error instanceof Error
+          ? error.message
+          : `trailmark: failed to process baseline at ${this.options.baselinePath}.`
+      errors.push(message)
+      console.error(message)
+      shouldFail = true
+    }
+
+    return { diff, configMismatch, shouldFail }
+  }
+
+  private updateBaseline(
+    playwrightStatus: string,
+    finalizedRun: TrailmarkRun,
+    configHash: string,
+    errors: string[],
+    shouldFail: boolean,
+  ): boolean {
+    if (playwrightStatus !== "passed") {
+      const errorMessage =
+        "trailmark: skipping baseline update because Playwright run did not pass."
+      errors.push(errorMessage)
+      console.error(errorMessage)
+      return true
+    }
+
+    if (this.options.failOnScanError && finalizedRun.scanErrors.length > 0) {
+      const errorMessage =
+        "trailmark: skipping baseline update because scan errors were captured in this run."
+      errors.push(errorMessage)
+      console.error(errorMessage)
+      return true
+    }
+
+    const baselineRead = readBaseline(this.options.baselinePath)
+    if (baselineRead.error && baselineRead.error !== "file-missing") {
+      errors.push(baselineRead.message)
+      console.error(baselineRead.message)
+      return true
+    }
+    const previousBaseline = baselineRead.data ?? undefined
+
+    if (shouldFail) {
+      return true
+    }
+
+    let baselineDelta: DiffResult | undefined
+    if (previousBaseline) {
+      baselineDelta = diffRuns(previousBaseline, finalizedRun, {
+        failOnImpact: this.options.failOnImpact,
+        onDebug: this.options.verbose ? this.debug.bind(this) : undefined,
+      })
+    }
+
+    const baseline = buildBaselineFromRun(finalizedRun, {
+      configHash,
+      previousBaseline,
+    })
+    writeJsonAtomic(this.options.baselinePath, baseline)
+    console.log(
+      `trailmark: baseline updated at ${this.options.baselinePath} with ${Object.keys(baseline.findings).length} issue(s).`,
+    )
+    if (baselineDelta) {
+      const exactMatches = baselineDelta.existing.filter(
+        (entry) => entry.matchedBy === "exact",
+      ).length
+      const relaxedMatches = baselineDelta.existing.length - exactMatches
+      console.log(
+        "trailmark: baseline delta vs previous | " +
+          `new=${baselineDelta.stats.newCount} ` +
+          `resolved=${baselineDelta.stats.resolvedCount} ` +
+          `existing=${baselineDelta.stats.existingCount} ` +
+          `(exact=${exactMatches}, relaxed=${relaxedMatches}) ` +
+          `blocking=${baselineDelta.blockingNew.length} ` +
+          `total previous=${baselineDelta.stats.totalBaseline} ` +
+          `total current=${baselineDelta.stats.totalCurrent}`,
+      )
+      console.log(
+        `trailmark: baseline delta impacts (new): ${summarizeImpactCounts(baselineDelta.stats.newByImpact)}`,
+      )
+    }
+
+    return false
+  }
+
+  private reportArtifactIndex(finalizedRun: TrailmarkRun): TrailmarkArtifactIndex | undefined {
+    if (!isRecord(finalizedRun.metadata)) {
+      return undefined
+    }
+
+    return toArtifactIndex(finalizedRun.metadata.artifactIndex)
+  }
+
+  onBegin(config: FullConfig): void {
+    this.config = config
+    this.sourceRuns = 0
+    this.seenRunHashes = new Set<string>()
+    this.mergedRun = this.createMergedRun()
+  }
+
+  onTestEnd(_test: TestCase, result: TestResult): void {
+    for (const run of readAttachmentPayload(result)) {
+      this.appendRun(run)
+    }
+  }
+
+  async onEnd(result: FullResult): Promise<ReporterStatus> {
+    const playwrightStatus = typeof result.status === "string" ? result.status : "passed"
+    const finalizedRun = applyUrlNormalization(
+      withRunShape({
+        ...this.mergedRun,
+        finishedAt: new Date().toISOString(),
+      }),
+      this.options.url,
+      this.options.verbose ? this.debug.bind(this) : undefined,
+    )
+
+    const configHash = this.computeConfigHash()
+    const warnings: string[] = []
+    const errors: string[] = []
+    let shouldFail = this.handleScanErrors(finalizedRun)
+
+    this.emitCollisionWarnings(finalizedRun, warnings)
+    this.emitArtifactWarnings(finalizedRun, warnings)
+
+    let diff: DiffResult | undefined
+    let configMismatch: ConfigMismatch | undefined
+
+    if (this.options.mode === "baseline") {
+      shouldFail = this.updateBaseline(
+        playwrightStatus,
+        finalizedRun,
+        configHash,
+        errors,
+        shouldFail,
+      )
+    } else {
+      const diffResult = this.runDiff(finalizedRun, configHash, warnings, errors)
+      shouldFail ||= diffResult.shouldFail
+      diff = diffResult.diff
+      configMismatch = diffResult.configMismatch
+
+      if (diff?.blockingNew.length) {
+        shouldFail = true
+        console.error(
+          `trailmark: ${diff.blockingNew.length} blocking new issue(s) found for impacts ${this.options.failOnImpact.join(", ")}.`,
+        )
+      }
+    }
+
+    const report = buildReport({
+      run: finalizedRun,
+      mode: this.options.mode,
+      configHash,
+      diff,
+      artifacts: this.reportArtifactIndex(finalizedRun),
+    })
+    writeJsonAtomic(this.options.reportPath, report)
+
+    console.log(
+      formatConsoleReport(report, {
+        failOnImpact: this.options.failOnImpact,
+      }),
+    )
+
+    writeJsonAtomic(CURRENT_RUN_PATH, {
+      ...this.currentRunPayloadBase(configHash, finalizedRun),
+      configMismatch,
+      diff,
+      reportPath: this.options.reportPath,
+      warnings,
+      errors,
+    })
+
+    if (this.options.failOnScanError && finalizedRun.scanErrors.length > 0) {
+      return { status: "failed" }
+    }
+
+    if (shouldFail) {
+      return { status: "failed" }
+    }
+
+    return undefined
+  }
+}

package/src/reporter/io.ts

@@ -0,0 +1,33 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"
+import { dirname } from "node:path"
+import { parseBaselineFile } from "@trailmark/core"
+
+function ensureParentDir(filePath: string): void {
+  mkdirSync(dirname(filePath), { recursive: true })
+}
+
+export function writeJsonAtomic(filePath: string, payload: unknown): void {
+  ensureParentDir(filePath)
+  const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`
+
+  try {
+    writeFileSync(tempPath, JSON.stringify(payload, null, 2), "utf8")
+    renameSync(tempPath, filePath)
+  } finally {
+    if (existsSync(tempPath)) {
+      rmSync(tempPath, { force: true })
+    }
+  }
+}
+
+export function readBaseline(baselinePath: string) {
+  if (!existsSync(baselinePath)) {
+    return {
+      data: null,
+      error: "file-missing" as const,
+      message: `trailmark: baseline file not found at ${baselinePath}. Run in baseline mode first.`,
+    }
+  }
+
+  return parseBaselineFile(readFileSync(baselinePath, "utf8"))
+}