@trailmark/playwright 0.1.0

package/test/reporter.unit.test.ts

@@ -0,0 +1,337 @@
+import { Buffer } from "node:buffer"
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import path from "node:path"
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
+import {
+  BaselineFile,
+  buildBaselineFromRun,
+  createEmptyRun,
+  normalizeAxeResults,
+} from "@trailmark/core"
+import { TRAILMARK_ATTACHMENT_NAME } from "../src/constants"
+import TrailmarkReporter from "../src/reporter"
+
+describe("trailmark reporter attachment handling", () => {
+  let originalCwd: string
+  let tempDir: string
+
+  beforeEach(() => {
+    originalCwd = process.cwd()
+    tempDir = mkdtempSync(path.join(tmpdir(), "trailmark-reporter-"))
+    process.chdir(tempDir)
+  })
+
+  afterEach(() => {
+    rmSync(path.join(tempDir, "baseline.json"), { force: true })
+    rmSync(path.join(tempDir, ".trailmark"), { recursive: true, force: true })
+    process.chdir(originalCwd)
+    rmSync(tempDir, { recursive: true, force: true })
+  })
+
+  it("builds a baseline from attachments nested in test steps", async () => {
+    const reporter = new TrailmarkReporter({
+      mode: "baseline",
+      baselinePath: path.join(tempDir, "baseline.json"),
+      reportPath: path.join(tempDir, ".trailmark", "report.json"),
+      failOnImpact: ["minor", "moderate", "serious", "critical", "unknown"],
+    })
+
+    reporter.onBegin({ projects: [{ name: "chromium" }] } as never)
+
+    const run = createEmptyRun()
+    run.findings.push(
+      ...normalizeAxeResults({
+        pageUrl: "file:///tmp/base.html",
+        results: {
+          violations: [
+            {
+              id: "image-alt",
+              impact: "critical",
+              tags: ["wcag2a"],
+              nodes: [
+                {
+                  target: ["img.hero"],
+                  html: '<img class="hero" src="hero.jpg">',
+                  failureSummary: "Fix any of the following: Element does not have alt text",
+                },
+              ],
+            },
+          ],
+        },
+      }),
+    )
+
+    reporter.onTestEnd(
+      {} as never,
+      {
+        attachments: [],
+        steps: [
+          {
+            attachments: [
+              {
+                name: TRAILMARK_ATTACHMENT_NAME,
+                contentType: "application/json",
+                body: Buffer.from(JSON.stringify(run), "utf8"),
+              },
+            ],
+            steps: [],
+          },
+        ],
+      } as never,
+    )
+
+    const endResult = await reporter.onEnd({} as never)
+    expect(endResult).toBeUndefined()
+
+    const baseline = JSON.parse(
+      readFileSync(path.join(tempDir, "baseline.json"), "utf8"),
+    ) as BaselineFile
+    expect(Object.keys(baseline.findings).length).toBeGreaterThan(0)
+
+    const report = JSON.parse(
+      readFileSync(path.join(tempDir, ".trailmark", "report.json"), "utf8"),
+    ) as {
+      run: { mode: string }
+      summary: { totalIssues: number }
+      issues: unknown[]
+      diff?: unknown
+    }
+
+    expect(report.run.mode).toBe("baseline")
+    expect(report.summary.totalIssues).toBeGreaterThan(0)
+    expect(report.issues.length).toBeGreaterThan(0)
+    expect(report.diff).toBeUndefined()
+  })
+
+  it("aggregates findings across attachments and computes diff in check mode", async () => {
+    const baselinePath = path.join(tempDir, "baseline.json")
+    const reporter = new TrailmarkReporter({
+      mode: "check",
+      baselinePath,
+      reportPath: path.join(tempDir, ".trailmark", "report.json"),
+      failOnImpact: ["serious", "critical"],
+    })
+
+    reporter.onBegin({ projects: [{ name: "chromium" }, { name: "firefox" }] } as never)
+
+    const baseRun = createEmptyRun()
+    baseRun.findings.push(
+      ...normalizeAxeResults({
+        pageUrl: "https://example.com/base",
+        meta: { projectName: "chromium", testId: "t-1" },
+        results: {
+          violations: [
+            {
+              id: "image-alt",
+              impact: "serious",
+              tags: ["wcag2a"],
+              nodes: [
+                {
+                  target: ["img.hero"],
+                  html: '<img data-testid="hero-image" src="hero.jpg">',
+                  failureSummary: "Element does not have alt text",
+                },
+              ],
+            },
+          ],
+        },
+      }),
+    )
+
+    const baseline = buildBaselineFromRun(baseRun, {
+      configHash: "cfg",
+      createdAt: "2026-01-01T00:00:00.000Z",
+    })
+    writeFileSync(baselinePath, JSON.stringify(baseline, null, 2), "utf8")
+
+    const followupRun = createEmptyRun()
+    followupRun.findings.push(...baseRun.findings)
+    const newFindings = normalizeAxeResults({
+      pageUrl: "https://example.com/extra",
+      meta: { projectName: "firefox", testId: "t-2" },
+      results: {
+        violations: [
+          {
+            id: "color-contrast",
+            impact: "critical",
+            tags: ["wcag2aa"],
+            nodes: [
+              {
+                target: ["button.checkout"],
+                html: '<button data-testid="checkout">Checkout</button>',
+                failureSummary: "Element has insufficient color contrast",
+              },
+            ],
+          },
+        ],
+      },
+    })
+    newFindings[0].occurrenceId = "occurrence-firefox-extra"
+    newFindings[0].occurredAt = "2026-01-02T00:00:00.000Z"
+    newFindings[0].attachments = [
+      {
+        kind: "viewport",
+        path: ".trailmark/artifacts/run-123/critical__color-contrast__example.com_extra__new__viewport.png",
+      },
+    ]
+    followupRun.findings.push(...newFindings)
+
+    reporter.onTestEnd(
+      {} as never,
+      {
+        attachments: [
+          {
+            name: TRAILMARK_ATTACHMENT_NAME,
+            contentType: "application/json",
+            body: Buffer.from(JSON.stringify(baseRun), "utf8"),
+          },
+        ],
+        steps: [],
+      } as never,
+    )
+
+    reporter.onTestEnd(
+      {} as never,
+      {
+        attachments: [
+          {
+            name: TRAILMARK_ATTACHMENT_NAME,
+            contentType: "application/json",
+            body: Buffer.from(JSON.stringify(followupRun), "utf8"),
+          },
+        ],
+        steps: [],
+      } as never,
+    )
+
+    const endResult = await reporter.onEnd({} as never)
+    expect(endResult?.status).toBe("failed")
+
+    const currentRunPath = path.join(tempDir, ".trailmark", "current-run.json")
+    const payload = JSON.parse(readFileSync(currentRunPath, "utf8")) as {
+      run: { findings: { meta?: { projectName?: string } }[] }
+      diff: { existing: unknown[]; newFindings: unknown[]; blockingNew: unknown[] }
+    }
+
+    expect(payload.run.findings).toHaveLength(3)
+    const projectNames = new Set(
+      payload.run.findings.map((finding) => finding.meta?.projectName).filter(Boolean),
+    )
+    expect(projectNames).toEqual(new Set(["chromium", "firefox"]))
+    expect(payload.diff.existing).toHaveLength(1)
+    expect(payload.diff.newFindings).toHaveLength(1)
+    expect(payload.diff.blockingNew).toHaveLength(1)
+
+    const report = JSON.parse(
+      readFileSync(path.join(tempDir, ".trailmark", "report.json"), "utf8"),
+    ) as {
+      summary: { newIssues: number; totalIssues: number }
+      diff?: { newIssueIds: string[] }
+      issues: Array<{
+        issueId: string
+        occurrences: Array<{ attachments: Array<{ path?: string }> }>
+      }>
+    }
+
+    expect(report.summary.totalIssues).toBe(2)
+    expect(report.summary.newIssues).toBe(1)
+    expect(report.diff?.newIssueIds).toHaveLength(1)
+
+    const newIssue = report.issues.find((issue) => issue.issueId === newFindings[0].issueId)
+    expect(newIssue).toBeDefined()
+    expect(newIssue?.occurrences[0].attachments[0].path).toContain(".trailmark/artifacts/run-123")
+  })
+
+  it("logs baseline delta details when updating an existing baseline", async () => {
+    const baselinePath = path.join(tempDir, "baseline.json")
+    const reporter = new TrailmarkReporter({
+      mode: "baseline",
+      baselinePath,
+      reportPath: path.join(tempDir, ".trailmark", "report.json"),
+      failOnImpact: ["serious", "critical"],
+    })
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined)
+
+    reporter.onBegin({ projects: [{ name: "chromium" }] } as never)
+
+    const previousRun = createEmptyRun()
+    previousRun.findings.push(
+      ...normalizeAxeResults({
+        pageUrl: "https://example.com/base",
+        meta: { projectName: "chromium", testId: "t-1" },
+        results: {
+          violations: [
+            {
+              id: "image-alt",
+              impact: "serious",
+              tags: ["wcag2a"],
+              nodes: [
+                {
+                  target: ["img.hero"],
+                  html: '<img data-testid="hero-image" src="hero.jpg">',
+                  failureSummary: "Element does not have alt text",
+                },
+              ],
+            },
+          ],
+        },
+      }),
+    )
+    const previousBaseline = buildBaselineFromRun(previousRun, {
+      configHash: "cfg",
+      createdAt: "2026-01-01T00:00:00.000Z",
+    })
+    writeFileSync(baselinePath, JSON.stringify(previousBaseline, null, 2), "utf8")
+
+    const currentRun = createEmptyRun()
+    currentRun.findings.push(
+      ...normalizeAxeResults({
+        pageUrl: "https://example.com/extra",
+        meta: { projectName: "chromium", testId: "t-2" },
+        results: {
+          violations: [
+            {
+              id: "color-contrast",
+              impact: "critical",
+              tags: ["wcag2aa"],
+              nodes: [
+                {
+                  target: ["button.checkout"],
+                  html: '<button data-testid="checkout">Checkout</button>',
+                  failureSummary: "Element has insufficient color contrast",
+                },
+              ],
+            },
+          ],
+        },
+      }),
+    )
+
+    reporter.onTestEnd(
+      {} as never,
+      {
+        attachments: [
+          {
+            name: TRAILMARK_ATTACHMENT_NAME,
+            contentType: "application/json",
+            body: Buffer.from(JSON.stringify(currentRun), "utf8"),
+          },
+        ],
+        steps: [],
+      } as never,
+    )
+
+    const endResult = await reporter.onEnd({} as never)
+    expect(endResult).toBeUndefined()
+
+    const logged = logSpy.mock.calls.map((call) => call.join(" ")).join("\n")
+    expect(logged).toContain("trailmark: baseline delta vs previous")
+    expect(logged).toContain("new=1")
+    expect(logged).toContain("resolved=1")
+    expect(logged).toContain("existing=0")
+    expect(logged).toContain("trailmark: baseline delta impacts (new): critical=1")
+
+    logSpy.mockRestore()
+  })
+})

package/test-results/.last-run.json

@@ -0,0 +1,4 @@
+{
+  "status": "passed",
+  "failedTests": []
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*.ts"]
+}