@trailmark/playwright 0.1.0

package/src/artifacts/options.ts

@@ -0,0 +1,79 @@
+import { TrailmarkArtifactCaptureMode, TrailmarkArtifactOptions, TrailmarkMode } from "../types"
+
+export interface ResolvedTrailmarkArtifactOptions {
+  enabled: boolean
+  outputDir: string
+  limitPerRun: number
+  capture: TrailmarkArtifactCaptureMode
+  includeViewport: boolean
+  includeElement: boolean
+}
+
+const DEFAULT_OUTPUT_DIR = ".trailmark/artifacts"
+const DEFAULT_LIMIT_PER_RUN = 50
+
+function parseBoolean(value: string | undefined): boolean | undefined {
+  if (!value) {
+    return undefined
+  }
+
+  const normalized = value.trim().toLowerCase()
+  if (normalized === "1" || normalized === "true") {
+    return true
+  }
+
+  if (normalized === "0" || normalized === "false") {
+    return false
+  }
+
+  return undefined
+}
+
+function parsePositiveInteger(value: string | undefined): number | undefined {
+  if (!value) {
+    return undefined
+  }
+
+  const parsed = Number.parseInt(value, 10)
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return undefined
+  }
+
+  return parsed
+}
+
+function parseCaptureMode(value: string | undefined): TrailmarkArtifactCaptureMode | undefined {
+  if (!value) {
+    return undefined
+  }
+
+  if (value === "new-only" || value === "all") {
+    return value
+  }
+
+  return undefined
+}
+
+export function resolveArtifactOptions(
+  mode: TrailmarkMode,
+  options: TrailmarkArtifactOptions | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): ResolvedTrailmarkArtifactOptions {
+  // const defaultEnabled = mode === "check"
+  const defaultEnabled = false
+
+  const envEnabled = parseBoolean(env.TRAILMARK_ARTIFACTS_ENABLED)
+  const envIncludeViewport = parseBoolean(env.TRAILMARK_ARTIFACTS_INCLUDE_VIEWPORT)
+  const envIncludeElement = parseBoolean(env.TRAILMARK_ARTIFACTS_INCLUDE_ELEMENT)
+  const envLimitPerRun = parsePositiveInteger(env.TRAILMARK_ARTIFACTS_LIMIT_PER_RUN)
+  const envCaptureMode = parseCaptureMode(env.TRAILMARK_ARTIFACTS_CAPTURE)
+
+  return {
+    enabled: options?.enabled ?? envEnabled ?? defaultEnabled,
+    outputDir: options?.outputDir ?? env.TRAILMARK_ARTIFACTS_OUTPUT_DIR ?? DEFAULT_OUTPUT_DIR,
+    limitPerRun: options?.limitPerRun ?? envLimitPerRun ?? DEFAULT_LIMIT_PER_RUN,
+    capture: options?.capture ?? envCaptureMode ?? "new-only",
+    includeViewport: options?.includeViewport ?? envIncludeViewport ?? true,
+    includeElement: options?.includeElement ?? envIncludeElement ?? true,
+  }
+}

package/src/artifacts/registry.ts

@@ -0,0 +1,143 @@
+import {
+  BaselineEntry,
+  BaselineFile,
+  TrailmarkAttachment,
+  TrailmarkFinding,
+  hashObject,
+  scopeKeyFromBaselineEntry,
+  scopeKeyFromMeta,
+  scopedIssueKey,
+} from "@trailmark/core"
+import { TrailmarkArtifactCaptureMode } from "../types"
+
+function makeBaselineExactKey(entry: BaselineEntry): string {
+  return scopedIssueKey(entry.issueId, scopeKeyFromBaselineEntry(entry))
+}
+
+function makeFindingExactKey(finding: TrailmarkFinding): string {
+  return scopedIssueKey(finding.issueId, scopeKeyFromMeta(finding.meta))
+}
+
+function makeFindingRelaxedKey(finding: TrailmarkFinding): string {
+  return scopedIssueKey(finding.relaxedIssueId, scopeKeyFromMeta(finding.meta))
+}
+
+function makeRelaxedMap(baselineEntries: BaselineEntry[]): Map<string, BaselineEntry[]> {
+  const map = new Map<string, BaselineEntry[]>()
+  for (const entry of baselineEntries) {
+    const key = scopedIssueKey(entry.relaxedIssueId, scopeKeyFromBaselineEntry(entry))
+    const list = map.get(key) ?? []
+    list.push(entry)
+    map.set(key, list)
+  }
+
+  for (const entries of map.values()) {
+    entries.sort((left, right) =>
+      makeBaselineExactKey(left).localeCompare(makeBaselineExactKey(right)),
+    )
+  }
+
+  return map
+}
+
+function pickUnmatched(
+  entries: BaselineEntry[] | undefined,
+  matchedExactKeys: Set<string>,
+): BaselineEntry | undefined {
+  if (!entries || entries.length === 0) {
+    return undefined
+  }
+
+  return entries.find((entry) => !matchedExactKeys.has(makeBaselineExactKey(entry)))
+}
+
+export class CaptureDecisionRegistry {
+  private readonly seenIssueKeys = new Set<string>()
+
+  private readonly baselineByExact: Map<string, BaselineEntry>
+
+  private readonly baselineByRelaxed: Map<string, BaselineEntry[]>
+
+  private readonly matchedBaselineExactKeys = new Set<string>()
+
+  constructor(
+    private readonly captureMode: TrailmarkArtifactCaptureMode,
+    baseline: BaselineFile | undefined,
+  ) {
+    const entries = Object.values(baseline?.findings ?? {})
+    this.baselineByExact = new Map(entries.map((entry) => [makeBaselineExactKey(entry), entry]))
+    this.baselineByRelaxed = makeRelaxedMap(entries)
+  }
+
+  shouldCaptureIssue(finding: TrailmarkFinding): boolean {
+    const findingExactKey = makeFindingExactKey(finding)
+    if (this.seenIssueKeys.has(findingExactKey)) {
+      return false
+    }
+
+    this.seenIssueKeys.add(findingExactKey)
+
+    if (this.captureMode === "all") {
+      return true
+    }
+
+    const exact = this.baselineByExact.get(findingExactKey)
+    if (exact) {
+      this.matchedBaselineExactKeys.add(makeBaselineExactKey(exact))
+      return false
+    }
+
+    const relaxed = pickUnmatched(
+      this.baselineByRelaxed.get(makeFindingRelaxedKey(finding)),
+      this.matchedBaselineExactKeys,
+    )
+
+    if (relaxed) {
+      this.matchedBaselineExactKeys.add(makeBaselineExactKey(relaxed))
+      return false
+    }
+
+    return true
+  }
+}
+
+export class ArtifactRegistry {
+  private readonly occurrenceOrdinalByIssueKey = new Map<string, number>()
+
+  private readonly attachmentsByOccurrenceId = new Map<string, TrailmarkAttachment[]>()
+
+  registerOccurrence(finding: TrailmarkFinding): { occurrenceId: string; issueKey: string } {
+    const issueKey = makeFindingExactKey(finding)
+    const ordinal = (this.occurrenceOrdinalByIssueKey.get(issueKey) ?? 0) + 1
+    this.occurrenceOrdinalByIssueKey.set(issueKey, ordinal)
+
+    const occurrenceId = hashObject({
+      issueKey,
+      issueId: finding.issueId,
+      pageKey: finding.pageKey,
+      testId: finding.meta?.testId,
+      projectName: finding.meta?.projectName,
+      ordinal,
+    })
+
+    return {
+      occurrenceId,
+      issueKey,
+    }
+  }
+
+  registerAttachments(occurrenceId: string, attachments: TrailmarkAttachment[]): void {
+    this.attachmentsByOccurrenceId.set(occurrenceId, attachments)
+  }
+
+  attachmentIndex(): { byOccurrenceId: Record<string, TrailmarkAttachment[]> } {
+    const byOccurrenceId: Record<string, TrailmarkAttachment[]> = {}
+    for (const key of Array.from(this.attachmentsByOccurrenceId.keys()).sort((a, b) =>
+      a.localeCompare(b),
+    )) {
+      byOccurrenceId[key] = this.attachmentsByOccurrenceId.get(key) ?? []
+    }
+
+    return { byOccurrenceId }
+  }
+}

package/src/constants.ts

@@ -0,0 +1 @@
+export const TRAILMARK_ATTACHMENT_NAME = "trailmark-run"

package/src/fixture.ts

@@ -0,0 +1,239 @@
+import { existsSync } from "node:fs"
+import AxeBuilder from "@axe-core/playwright"
+import { expect, test as base } from "@playwright/test"
+import {
+  AxeLikeResults,
+  BaselineFile,
+  TrailmarkFindingMeta,
+  TrailmarkRun,
+  createEmptyRun,
+  normalizeAxeResults,
+  normalizeUrlToPageKey,
+} from "@trailmark/core"
+import { TRAILMARK_ATTACHMENT_NAME } from "./constants"
+import { ArtifactManager, createArtifactRunId } from "./artifacts/artifactManager"
+import { resolveArtifactOptions } from "./artifacts/options"
+import { ArtifactRegistry, CaptureDecisionRegistry } from "./artifacts/registry"
+import { ScanOptions, TrailmarkArtifactOptions, TrailmarkFixture, TrailmarkMode } from "./types"
+import { resolveMode } from "./mode"
+import { DEFAULT_BASELINE_PATH } from "./reporter"
+import { readBaseline } from "./reporter/io"
+
+function cloneRun(run: TrailmarkRun): TrailmarkRun {
+  return JSON.parse(JSON.stringify(run)) as TrailmarkRun
+}
+
+function applyAxeOptions(builder: AxeBuilder, opts: ScanOptions | undefined): AxeBuilder {
+  let configured = builder
+
+  for (const selector of opts?.axe?.include ?? []) {
+    configured = configured.include(selector)
+  }
+
+  for (const selector of opts?.axe?.exclude ?? []) {
+    configured = configured.exclude(selector)
+  }
+
+  if (opts?.axe?.tags && opts.axe.tags.length > 0) {
+    configured = configured.withTags(opts.axe.tags)
+  }
+
+  const rules: Record<string, { enabled: boolean }> = {
+    ...opts?.axe?.rules,
+  }
+
+  for (const ruleId of opts?.axe?.disableRules ?? []) {
+    rules[ruleId] = { enabled: false }
+  }
+
+  if (Object.keys(rules).length > 0) {
+    configured = configured.options({ rules })
+  }
+
+  return configured
+}
+
+function shouldSkipScan(opts: ScanOptions | undefined): boolean {
+  if (opts?.enabled === false) {
+    return true
+  }
+
+  const envToggle = process.env.TRAILMARK_DISABLE_SCAN
+  return envToggle === "1" || envToggle === "true"
+}
+
+function toError(input: unknown): Error {
+  if (input instanceof Error) {
+    return input
+  }
+
+  return new Error(typeof input === "string" ? input : "Unknown Trailmark scan failure.")
+}
+
+function loadBaselineForArtifacts(
+  mode: TrailmarkMode,
+  artifactOptions: TrailmarkArtifactOptions | undefined,
+  warnings: string[],
+): BaselineFile | undefined {
+  if (mode !== "check") {
+    return undefined
+  }
+
+  const captureMode = artifactOptions?.capture ?? "new-only"
+  if (captureMode !== "new-only") {
+    return undefined
+  }
+
+  const baselinePath = process.env.TRAILMARK_BASELINE_PATH ?? DEFAULT_BASELINE_PATH
+  if (!existsSync(baselinePath)) {
+    return undefined
+  }
+
+  const { data: baseline, error, message } = readBaseline(baselinePath)
+  if (error) {
+    warnings.push(
+      `trailmark: failed to read baseline for artifact capture decisions: ${message}. Falling back to capturing all issues.`,
+    )
+    return undefined
+  }
+  return baseline
+}
+
+export const test = base.extend<{ trailmark: TrailmarkFixture }>({
+  // oxlint-disable-next-line no-empty-pattern: First argument must use the object destructuring pattern: _fixtures https://github.com/microsoft/playwright/issues/14590
+  trailmark: async ({}, use, testInfo) => {
+    const mode = resolveMode()
+    const artifactWarnings: string[] = []
+    const resolvedArtifactOptions = resolveArtifactOptions(mode, undefined)
+    const artifactRunId = createArtifactRunId()
+    const artifactManager = new ArtifactManager({
+      ...resolvedArtifactOptions,
+      runId: artifactRunId,
+    })
+    const artifactRegistry = new ArtifactRegistry()
+
+    const baselineForCapture = loadBaselineForArtifacts(
+      mode,
+      resolvedArtifactOptions,
+      artifactWarnings,
+    )
+    const captureMode = baselineForCapture ? resolvedArtifactOptions.capture : "all"
+    const captureDecision = new CaptureDecisionRegistry(captureMode, baselineForCapture)
+
+    const run = createEmptyRun({
+      metadata: {
+        projectName: testInfo.project.name,
+        testFile: testInfo.file,
+        scanCount: 0,
+        skippedScanCount: 0,
+        artifactRunId,
+        artifactOptions: resolvedArtifactOptions,
+      },
+    })
+
+    if (!Array.isArray(run.scanErrors)) {
+      run.scanErrors = []
+    }
+
+    const fixture: TrailmarkFixture = {
+      async scan(page, opts) {
+        if (shouldSkipScan(opts)) {
+          run.metadata = {
+            ...run.metadata,
+            skippedScanCount: Number(run.metadata?.skippedScanCount ?? 0) + 1,
+          }
+          return
+        }
+
+        const meta: TrailmarkFindingMeta = {
+          testId: testInfo.testId,
+          testTitle: testInfo.title,
+          testFile: testInfo.file,
+          projectName: testInfo.project.name,
+          scanLabel: opts?.scanLabel,
+          ...opts?.meta,
+        }
+
+        const pageUrl = page.url()
+        const builder = applyAxeOptions(new AxeBuilder({ page }), opts)
+        const scanArtifactEnabled = opts?.artifacts?.enabled ?? resolvedArtifactOptions.enabled
+
+        try {
+          const results = (await builder.analyze()) as AxeLikeResults
+          const findings = normalizeAxeResults({
+            results,
+            pageUrl,
+            urlOptions: opts?.url,
+            meta,
+          })
+
+          const occurredAt = new Date().toISOString()
+          const finalizedFindings: TrailmarkRun["findings"] = []
+
+          for (const finding of findings) {
+            const { occurrenceId } = artifactRegistry.registerOccurrence(finding)
+            const withOccurrence = {
+              ...finding,
+              occurrenceId,
+              occurredAt,
+            }
+
+            if (scanArtifactEnabled && captureDecision.shouldCaptureIssue(withOccurrence)) {
+              const captureResult = await artifactManager.captureForIssue({
+                page,
+                finding: withOccurrence,
+              })
+              artifactRegistry.registerAttachments(occurrenceId, captureResult.attachments)
+              finalizedFindings.push({
+                ...withOccurrence,
+                attachments: captureResult.attachments,
+              })
+              continue
+            }
+
+            finalizedFindings.push(withOccurrence)
+          }
+
+          run.findings.push(...finalizedFindings)
+          run.metadata = {
+            ...run.metadata,
+            scanCount: Number(run.metadata?.scanCount ?? 0) + 1,
+          }
+        } catch (error) {
+          const normalized = toError(error)
+          run.scanErrors.push({
+            pageUrl,
+            pageKey: normalizeUrlToPageKey(pageUrl, opts?.url),
+            message: normalized.message,
+            stack: normalized.stack,
+            occurredAt: new Date().toISOString(),
+            meta,
+          })
+
+          if (!opts?.continueOnError) {
+            throw normalized
+          }
+        }
+      },
+      getRun() {
+        return cloneRun(run)
+      },
+    }
+
+    await use(fixture)
+
+    run.finishedAt = new Date().toISOString()
+    run.metadata = {
+      ...run.metadata,
+      artifactWarnings: [...artifactWarnings, ...artifactManager.getWarnings()],
+      artifactIndex: artifactRegistry.attachmentIndex(),
+    }
+
+    await testInfo.attach(TRAILMARK_ATTACHMENT_NAME, {
+      contentType: "application/json",
+      body: Buffer.from(JSON.stringify(run), "utf8"),
+    })
+  },
+})
+
+export { expect }

package/src/index.ts

@@ -0,0 +1,3 @@
+export { expect, test } from "./fixture"
+export type { ScanOptions, TrailmarkFixture, TrailmarkReporterOptions } from "./types"
+export type { TrailmarkRun } from "@trailmark/core"

package/src/mode.ts

@@ -0,0 +1,7 @@
+import { TrailmarkMode, TrailmarkModeOptions } from "./types"
+
+export function resolveMode(options?: TrailmarkModeOptions): TrailmarkMode {
+  const envMode = process.env.TRAILMARK_MODE as TrailmarkMode | undefined
+  const mode = options?.mode ?? envMode ?? "check"
+  return mode
+}

package/src/reporter/attachments.ts

@@ -0,0 +1,139 @@
+import { Buffer } from "node:buffer"
+import { existsSync, readFileSync } from "node:fs"
+import type { TestResult } from "@playwright/test/reporter"
+import { TrailmarkRun, createEmptyRun, hashObject } from "@trailmark/core"
+import { TRAILMARK_ATTACHMENT_NAME } from "../constants"
+import { withRunShape } from "./run"
+
+type AttachmentLike = {
+  name: string
+  path?: string
+  body?: unknown
+}
+
+type StepLike = {
+  attachments?: AttachmentLike[]
+  steps?: StepLike[]
+}
+
+function readAttachmentBody(body: unknown): string | undefined {
+  if (!body) {
+    return undefined
+  }
+
+  if (typeof body === "string") {
+    return body
+  }
+
+  if (Buffer.isBuffer(body) || body instanceof Uint8Array) {
+    return Buffer.from(body).toString("utf8")
+  }
+
+  return undefined
+}
+
+function attachmentSignature(attachment: AttachmentLike): string {
+  if (attachment.path) {
+    return `${attachment.name}|path:${attachment.path}`
+  }
+
+  const body = readAttachmentBody(attachment.body)
+  if (body !== undefined) {
+    return `${attachment.name}|body:${body}`
+  }
+
+  return `${attachment.name}|fallback:${typeof attachment.body}`
+}
+
+function collectAttachments(result: TestResult): AttachmentLike[] {
+  const attachments: AttachmentLike[] = []
+  const seen = new Set<string>()
+
+  for (const attachment of result.attachments) {
+    const signature = attachmentSignature(attachment)
+    if (seen.has(signature)) {
+      continue
+    }
+    seen.add(signature)
+    attachments.push(attachment)
+  }
+
+  const queue: StepLike[] = [...result.steps]
+  while (queue.length > 0) {
+    const step = queue.pop()
+    if (!step) {
+      continue
+    }
+
+    for (const attachment of step.attachments ?? []) {
+      const signature = attachmentSignature(attachment)
+      if (seen.has(signature)) {
+        continue
+      }
+      seen.add(signature)
+      attachments.push(attachment)
+    }
+
+    queue.push(...(step.steps ?? []))
+  }
+
+  return attachments
+}
+
+export function readAttachmentPayload(result: TestResult): TrailmarkRun[] {
+  const runs: TrailmarkRun[] = []
+  const seenPayloadHashes = new Set<string>()
+
+  for (const attachment of collectAttachments(result)) {
+    if (attachment.name !== TRAILMARK_ATTACHMENT_NAME) {
+      continue
+    }
+
+    try {
+      let raw: string | undefined
+      if (attachment.body) {
+        raw = readAttachmentBody(attachment.body)
+      } else if (attachment.path && existsSync(attachment.path)) {
+        raw = readFileSync(attachment.path, "utf8")
+      }
+
+      if (!raw) {
+        continue
+      }
+
+      const parsed = JSON.parse(raw) as Partial<TrailmarkRun>
+      if (!Array.isArray(parsed.findings)) {
+        continue
+      }
+
+      const payloadHash = hashObject({
+        attachmentName: attachment.name,
+        version: parsed.version,
+        startedAt: parsed.startedAt,
+        finishedAt: parsed.finishedAt,
+        tool: parsed.tool,
+        findings: parsed.findings,
+        scanErrors: parsed.scanErrors ?? [],
+        metadata: parsed.metadata,
+      })
+      if (seenPayloadHashes.has(payloadHash)) {
+        continue
+      }
+      seenPayloadHashes.add(payloadHash)
+
+      const run = createEmptyRun({
+        metadata: parsed.metadata,
+        tool: parsed.tool,
+        startedAt: parsed.startedAt,
+      })
+      run.finishedAt = parsed.finishedAt
+      run.findings.push(...parsed.findings)
+      run.scanErrors.push(...(Array.isArray(parsed.scanErrors) ? parsed.scanErrors : []))
+      runs.push(withRunShape(run))
+    } catch {
+      continue
+    }
+  }
+
+  return runs
+}

package/src/reporter/errors.ts

@@ -0,0 +1,6 @@
+export class ReporterBaselineError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ReporterBaselineError"
+  }
+}