@trailmark/playwright 0.1.0

package/src/reporter/normalization.ts

@@ -0,0 +1,122 @@
+import {
+  TrailmarkFinding,
+  TrailmarkRun,
+  buildIssueId,
+  buildRelaxedIssueId,
+  hashObject,
+  normalizeUrlToPageKey,
+} from "@trailmark/core"
+import { TrailmarkReporterOptions } from "../types"
+import { scopeKeyFromMetaLocal } from "./run"
+
+const GLOBAL_HASH_INPUT_DEBUG_SEEN = new Set<string>()
+
+export interface IdentityCollision {
+  count: number
+  scopeKey: string
+  ruleId: string
+  pageKey: string
+  targetHash: string
+  issueIds: string[]
+}
+
+export function applyUrlNormalization(
+  run: TrailmarkRun,
+  options: TrailmarkReporterOptions["url"],
+  onDebug?: (message: string, context?: Record<string, unknown>) => void,
+): TrailmarkRun {
+  const seenHashInputDebugs = new Set<string>()
+
+  const findings = run.findings.map((finding) => {
+    const pageKey = normalizeUrlToPageKey(finding.pageUrl, options)
+    const scopeKey = scopeKeyFromMetaLocal(finding.meta)
+    const relaxedIssueId = buildRelaxedIssueId({
+      ruleId: finding.ruleId,
+      pageKey,
+      targetHash: finding.targetHash,
+    })
+    const issueId = buildIssueId({
+      ruleId: finding.ruleId,
+      pageKey,
+      targetHash: finding.targetHash,
+      failureHint: finding.failureHint,
+    })
+
+    const debugKey = hashObject({
+      ruleId: finding.ruleId,
+      pageKey,
+      targetHash: finding.targetHash,
+      failureHint: finding.failureHint,
+      issueId,
+      relaxedIssueId,
+      scopeKey,
+    })
+
+    if (!seenHashInputDebugs.has(debugKey) && !GLOBAL_HASH_INPUT_DEBUG_SEEN.has(debugKey)) {
+      seenHashInputDebugs.add(debugKey)
+      GLOBAL_HASH_INPUT_DEBUG_SEEN.add(debugKey)
+      onDebug?.("hash-input", {
+        scopeKey,
+        ruleId: finding.ruleId,
+        pageKey,
+        targetHash: finding.targetHash,
+        failureHint: finding.failureHint,
+        issueId,
+        relaxedIssueId,
+      })
+    }
+
+    return {
+      ...finding,
+      pageKey,
+      issueId,
+      relaxedIssueId,
+    }
+  })
+
+  const scanErrors = run.scanErrors.map((scanError) => {
+    const pageKey =
+      scanError.pageKey ??
+      (scanError.pageUrl ? normalizeUrlToPageKey(scanError.pageUrl, options) : undefined)
+
+    return {
+      ...scanError,
+      pageKey,
+    }
+  })
+
+  return {
+    ...run,
+    findings,
+    scanErrors,
+  }
+}
+
+export function detectIdentityCollisions(findings: TrailmarkFinding[]): IdentityCollision[] {
+  const map = new Map<string, IdentityCollision>()
+
+  for (const finding of findings) {
+    const scopeKey = scopeKeyFromMetaLocal(finding.meta)
+    const collisionKey = `${scopeKey}|${finding.ruleId}|${finding.pageKey}|${finding.targetHash}`
+    const existing = map.get(collisionKey)
+
+    if (existing) {
+      existing.count += 1
+      existing.issueIds.push(finding.issueId)
+      continue
+    }
+
+    map.set(collisionKey, {
+      count: 1,
+      scopeKey,
+      ruleId: finding.ruleId,
+      pageKey: finding.pageKey,
+      targetHash: finding.targetHash,
+      issueIds: [finding.issueId],
+    })
+  }
+
+  return Array.from(map.values())
+    .filter((entry) => new Set(entry.issueIds).size > 1)
+    .sort((left, right) => right.count - left.count)
+}

package/src/reporter/output.ts

@@ -0,0 +1,103 @@
+import { DiffResult, TrailmarkRun } from "@trailmark/core"
+
+function impactWeight(impact: string | undefined): number {
+  switch ((impact ?? "unknown").toLowerCase()) {
+    case "critical":
+      return 0
+    case "serious":
+      return 1
+    case "moderate":
+      return 2
+    case "minor":
+      return 3
+    default:
+      return 4
+  }
+}
+
+export function summarizeRunEnvelope(sourceRuns: number, findingsCount: number): void {
+  if (sourceRuns === 0) {
+    console.log("trailmark: 0 scans / no findings.")
+    return
+  }
+
+  if (findingsCount === 0) {
+    console.log(`trailmark: ${sourceRuns} scan run(s), 0 findings.`)
+  }
+}
+
+export function summarizeScanErrors(scanErrors: TrailmarkRun["scanErrors"]): void {
+  if (scanErrors.length === 0) {
+    return
+  }
+
+  console.error(`trailmark: ${scanErrors.length} scan error(s) captured during run.`)
+  for (const scanError of scanErrors.slice(0, 10)) {
+    const location = scanError.pageKey ?? scanError.pageUrl ?? "<unknown-page>"
+    console.error(`- ${location} :: ${scanError.message}`)
+  }
+}
+
+export function summarizeNewFindings(diff: DiffResult): void {
+  if (diff.newFindings.length === 0) {
+    console.log("trailmark: no new accessibility violations compared to baseline.")
+    return
+  }
+
+  const byImpact = Object.entries(diff.stats.newByImpact)
+    .sort((left, right) => impactWeight(left[0]) - impactWeight(right[0]))
+    .map(([impact, count]) => `${impact}:${count}`)
+    .join(", ")
+
+  const ruleCounts = new Map<string, number>()
+  for (const finding of diff.newFindings) {
+    ruleCounts.set(finding.ruleId, (ruleCounts.get(finding.ruleId) ?? 0) + 1)
+  }
+
+  const topRules = Array.from(ruleCounts.entries())
+    .sort((left, right) => {
+      const countDiff = right[1] - left[1]
+      if (countDiff !== 0) {
+        return countDiff
+      }
+      return left[0].localeCompare(right[0])
+    })
+    .slice(0, 5)
+    .map(([ruleId, count]) => `${ruleId}:${count}`)
+    .join(", ")
+
+  console.log(
+    `trailmark: new=${diff.stats.newCount} existing=${diff.stats.existingCount} resolved=${diff.stats.resolvedCount}.`,
+  )
+  console.log(`trailmark: new by impact -> ${byImpact || "none"}`)
+  console.log(`trailmark: top new rules -> ${topRules || "none"}`)
+
+  const sortedFindings = [...diff.newFindings].sort((left, right) => {
+    const impactDiff =
+      impactWeight(left.impact as string | undefined) -
+      impactWeight(right.impact as string | undefined)
+    if (impactDiff !== 0) {
+      return impactDiff
+    }
+
+    const ruleDiff = left.ruleId.localeCompare(right.ruleId)
+    if (ruleDiff !== 0) {
+      return ruleDiff
+    }
+
+    const pageDiff = left.pageKey.localeCompare(right.pageKey)
+    if (pageDiff !== 0) {
+      return pageDiff
+    }
+
+    const leftSelector = left.target[0] ?? left.selectorHint ?? ""
+    const rightSelector = right.target[0] ?? right.selectorHint ?? ""
+    return leftSelector.localeCompare(rightSelector)
+  })
+
+  for (const finding of sortedFindings) {
+    const impact = (finding.impact ?? "unknown").toLowerCase()
+    const selector = finding.target[0] ?? finding.selectorHint ?? "<unknown-target>"
+    console.log(`- ${impact} | ${finding.ruleId} | ${finding.pageKey} | ${selector}`)
+  }
+}

package/src/reporter/run.ts

@@ -0,0 +1,17 @@
+import { TrailmarkFindingMeta, TrailmarkRun } from "@trailmark/core"
+
+export function withRunShape(run: TrailmarkRun): TrailmarkRun {
+  if (Array.isArray(run.scanErrors)) {
+    return run
+  }
+
+  return {
+    ...run,
+    scanErrors: [],
+  }
+}
+
+export function scopeKeyFromMetaLocal(meta: TrailmarkFindingMeta | undefined): string {
+  const normalized = meta?.projectName?.trim().toLowerCase()
+  return normalized || "default"
+}

package/src/types.ts

@@ -0,0 +1,53 @@
+import { Page } from "@playwright/test"
+import { TrailmarkFindingMeta, TrailmarkRun, UrlNormalizationOptions } from "@trailmark/core"
+
+export interface TrailmarkAxeOptions {
+  include?: string[]
+  exclude?: string[]
+  tags?: string[]
+  rules?: Record<string, { enabled: boolean }>
+  disableRules?: string[]
+}
+
+export type TrailmarkArtifactCaptureMode = "new-only" | "all"
+
+export interface TrailmarkArtifactOptions {
+  enabled?: boolean
+  outputDir?: string
+  limitPerRun?: number
+  capture?: TrailmarkArtifactCaptureMode
+  includeViewport?: boolean
+  includeElement?: boolean
+}
+
+export interface ScanOptions {
+  enabled?: boolean
+  continueOnError?: boolean
+  axe?: TrailmarkAxeOptions
+  url?: UrlNormalizationOptions
+  artifacts?: TrailmarkArtifactOptions
+  meta?: Partial<TrailmarkFindingMeta>
+  scanLabel?: string
+}
+
+export interface TrailmarkFixture {
+  scan(page: Page, opts?: ScanOptions): Promise<void>
+  getRun(): TrailmarkRun
+}
+
+export type TrailmarkMode = "baseline" | "check"
+
+export type TrailmarkModeOptions = {
+  mode?: TrailmarkMode
+}
+
+export type TrailmarkReporterOptions = {
+  baselinePath?: string
+  reportPath?: string
+  failOnImpact?: string[]
+  failOnScanError?: boolean
+  onConfigMismatch?: "warn" | "error" | "ignore"
+  verbose?: boolean
+  url?: UrlNormalizationOptions
+  artifacts?: TrailmarkArtifactOptions
+} & TrailmarkModeOptions

package/test/artifactManager.unit.test.ts

@@ -0,0 +1,141 @@
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import path from "node:path"
+import { afterEach, beforeEach, describe, expect, it } from "vitest"
+import { TrailmarkFinding } from "@trailmark/core"
+import {
+  ArtifactManager,
+  createArtifactRunId,
+  sanitizeArtifactSegment,
+} from "../src/artifacts/artifactManager"
+
+function makeFinding(overrides: Partial<TrailmarkFinding>): TrailmarkFinding {
+  return {
+    issueId: "issue-abc1234567890",
+    relaxedIssueId: "relaxed-abc1234567890",
+    ruleId: "color-contrast",
+    impact: "critical",
+    tags: ["wcag2aa"],
+    pageUrl: "https://example.com/checkout",
+    pageKey: "example.com/checkout",
+    target: ["button.checkout"],
+    targetHash: "target-a",
+    selectorHint: "button.checkout",
+    ...overrides,
+  }
+}
+
+describe("artifact manager", () => {
+  let originalCwd: string
+  let tempDir: string
+
+  beforeEach(() => {
+    originalCwd = process.cwd()
+    tempDir = mkdtempSync(path.join(tmpdir(), "trailmark-artifacts-"))
+    process.chdir(tempDir)
+  })
+
+  afterEach(() => {
+    process.chdir(originalCwd)
+    rmSync(tempDir, { recursive: true, force: true })
+  })
+
+  it("sanitizes path segments and builds deterministic run ids", () => {
+    expect(sanitizeArtifactSegment("  Checkout Page / Hero  ")).toBe("checkout_page_hero")
+    expect(sanitizeArtifactSegment("***", 20)).toBe("unknown")
+    expect(createArtifactRunId(new Date("2026-01-02T03:04:05.678Z"))).toBe("20260102T030405678Z")
+  })
+
+  it("captures viewport and element artifacts with expected file names", async () => {
+    const manager = new ArtifactManager({
+      enabled: true,
+      outputDir: ".trailmark/artifacts",
+      limitPerRun: 10,
+      capture: "all",
+      includeViewport: true,
+      includeElement: true,
+      runId: "run-123",
+    })
+
+    const page = {
+      screenshot: async ({ path: outputPath }: { path: string }) => {
+        writeFileSync(outputPath, "viewport", "utf8")
+      },
+      locator: (selector: string) => ({
+        first: () => ({
+          screenshot: async ({ path: outputPath }: { path: string }) => {
+            writeFileSync(outputPath, `element:${selector}`, "utf8")
+          },
+        }),
+      }),
+    }
+
+    const result = await manager.captureForIssue({
+      page: page as never,
+      finding: makeFinding({
+        issueId: "issue-abc1234567890",
+      }),
+    })
+
+    expect(result.skipped).toBe(false)
+    expect(result.attachments).toHaveLength(2)
+    expect(result.attachments).toEqual([
+      {
+        kind: "viewport",
+        path: ".trailmark/artifacts/run-123/critical__color-contrast__example.com_checkout__issue-abc123__viewport.png",
+      },
+      {
+        kind: "element",
+        path: ".trailmark/artifacts/run-123/critical__color-contrast__example.com_checkout__issue-abc123__element.png",
+      },
+    ])
+
+    for (const attachment of result.attachments) {
+      expect(attachment.path && existsSync(path.resolve(tempDir, attachment.path))).toBe(true)
+    }
+
+    const elementAttachment = result.attachments.find((attachment) => attachment.kind === "element")
+    expect(readFileSync(path.resolve(tempDir, elementAttachment?.path ?? ""), "utf8")).toContain(
+      "button.checkout",
+    )
+  })
+
+  it("enforces artifact limit per run", async () => {
+    const manager = new ArtifactManager({
+      enabled: true,
+      outputDir: ".trailmark/artifacts",
+      limitPerRun: 1,
+      capture: "all",
+      includeViewport: true,
+      includeElement: false,
+      runId: "run-limited",
+    })
+
+    const page = {
+      screenshot: async ({ path: outputPath }: { path: string }) => {
+        writeFileSync(outputPath, "viewport", "utf8")
+      },
+      locator: () => ({
+        first: () => ({
+          screenshot: async () => undefined,
+        }),
+      }),
+    }
+
+    const first = await manager.captureForIssue({ page: page as never, finding: makeFinding({}) })
+    const second = await manager.captureForIssue({
+      page: page as never,
+      finding: makeFinding({
+        issueId: "issue-b",
+        relaxedIssueId: "relaxed-b",
+        targetHash: "target-b",
+      }),
+    })
+
+    expect(first.skipped).toBe(false)
+    expect(second.skipped).toBe(true)
+    expect(manager.getWarnings()).toEqual([
+      "trailmark: artifact capture limit reached (1); skipping remaining captures.",
+    ])
+  })
+})

package/test/reporter.integration.test.ts

@@ -0,0 +1,153 @@
+import { existsSync, readFileSync, rmSync } from "node:fs"
+import path from "node:path"
+import { spawnSync } from "node:child_process"
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"
+import { chromium } from "@playwright/test"
+
+type Mode = "baseline" | "check"
+type Target = "base" | "extra"
+
+interface CommandResult {
+  status: number | null
+  stdout: string
+  stderr: string
+}
+
+const packageDir = path.resolve(__dirname, "..")
+const repoRoot = path.resolve(packageDir, "..", "..")
+const pnpmBin = process.platform === "win32" ? "pnpm.cmd" : "pnpm"
+const baselinePath = path.resolve(packageDir, "integration", "tmp", "baseline.json")
+const currentRunPath = path.resolve(packageDir, ".trailmark", "current-run.json")
+const reportPath = path.resolve(packageDir, ".trailmark", "report.json")
+let browserReady = false
+let _browserError: string | undefined
+
+function readJson(filePath: string): unknown {
+  return JSON.parse(readFileSync(filePath, "utf8"))
+}
+
+function run(command: string[], cwd: string, env: NodeJS.ProcessEnv = {}): CommandResult {
+  const result = spawnSync(pnpmBin, command, {
+    cwd,
+    env: {
+      ...process.env,
+      ...env,
+    },
+    encoding: "utf8",
+  })
+
+  return {
+    status: result.status,
+    stdout: result.stdout ?? "",
+    stderr: result.stderr ?? "",
+  }
+}
+
+function runPlaywright(mode: Mode, target: Target): CommandResult {
+  return run(["exec", "playwright", "test", "-c", "integration/playwright.config.ts"], packageDir, {
+    TRAILMARK_MODE: mode,
+    TRAILMARK_TARGET: target,
+  })
+}
+
+beforeAll(() => {
+  const coreBuild = run(["--filter", "@trailmark/core", "build"], repoRoot)
+  expect(coreBuild.status, `${coreBuild.stdout}\n${coreBuild.stderr}`).toBe(0)
+
+  const pwBuild = run(["--filter", "@trailmark/playwright", "build"], repoRoot)
+  expect(pwBuild.status, `${pwBuild.stdout}\n${pwBuild.stderr}`).toBe(0)
+})
+
+beforeAll(async () => {
+  try {
+    const browser = await chromium.launch({ headless: true })
+    await browser.close()
+    browserReady = true
+  } catch (error) {
+    browserReady = false
+    _browserError = error instanceof Error ? error.message : String(error)
+  }
+})
+
+beforeEach(() => {
+  rmSync(path.resolve(packageDir, "integration", "tmp"), { recursive: true, force: true })
+  rmSync(path.resolve(packageDir, ".trailmark"), { recursive: true, force: true })
+})
+
+afterAll(() => {
+  rmSync(path.resolve(packageDir, "integration", "tmp"), { recursive: true, force: true })
+  rmSync(path.resolve(packageDir, ".trailmark"), { recursive: true, force: true })
+})
+
+describe("trailmark reporter integration", () => {
+  it("creates baseline and then passes check when findings are unchanged", (context) => {
+    if (!browserReady) {
+      context.skip()
+      return
+    }
+
+    const baselineResult = runPlaywright("baseline", "base")
+    expect(baselineResult.status, `${baselineResult.stdout}\n${baselineResult.stderr}`).toBe(0)
+    expect(existsSync(baselinePath)).toBe(true)
+    expect(existsSync(currentRunPath)).toBe(true)
+
+    const baseline = readJson(baselinePath) as {
+      findings: Record<string, unknown>
+    }
+
+    console.log("Baseline findings:", baseline)
+    expect(Object.keys(baseline.findings).length).toBeGreaterThan(0)
+
+    const checkResult = runPlaywright("check", "base")
+    expect(checkResult.status, `${checkResult.stdout}\n${checkResult.stderr}`).toBe(0)
+    expect(existsSync(reportPath)).toBe(true)
+
+    const currentRun = readJson(currentRunPath) as {
+      diff?: { newFindings: unknown[]; blockingNew: unknown[] }
+    }
+    const report = readJson(reportPath) as {
+      diff?: { newIssueIds: string[] }
+      summary: { totalIssues: number }
+    }
+
+    expect(currentRun.diff?.newFindings.length ?? -1).toBe(0)
+    expect(currentRun.diff?.blockingNew.length ?? -1).toBe(0)
+    expect(report.diff?.newIssueIds).toEqual([])
+    expect(report.summary.totalIssues).toBeGreaterThan(0)
+  })
+
+  it("fails check when new issues are introduced", (context) => {
+    if (!browserReady) {
+      context.skip()
+      return
+    }
+
+    const baselineResult = runPlaywright("baseline", "base")
+    expect(baselineResult.status, `${baselineResult.stdout}\n${baselineResult.stderr}`).toBe(0)
+
+    const failResult = runPlaywright("check", "extra")
+    expect(failResult.status).not.toBe(0)
+    expect(existsSync(currentRunPath)).toBe(true)
+    expect(existsSync(reportPath)).toBe(true)
+
+    const currentRun = readJson(currentRunPath) as {
+      diff?: { newFindings: unknown[]; blockingNew: unknown[] }
+    }
+    const report = readJson(reportPath) as {
+      diff?: { newIssueIds: string[] }
+      issues: Array<{ occurrences: Array<{ attachments: Array<{ path?: string }> }> }>
+    }
+
+    expect((currentRun.diff?.newFindings.length ?? 0) > 0).toBe(true)
+    expect((currentRun.diff?.blockingNew.length ?? 0) > 0).toBe(true)
+    expect((report.diff?.newIssueIds.length ?? 0) > 0).toBe(true)
+
+    const attachmentPaths = report.issues.flatMap((issue) =>
+      issue.occurrences.flatMap((occurrence) =>
+        occurrence.attachments.map((attachment) => attachment.path).filter(Boolean),
+      ),
+    )
+    expect(attachmentPaths.length > 0).toBe(true)
+    expect(existsSync(path.resolve(packageDir, attachmentPaths[0] ?? ""))).toBe(true)
+  })
+})