@aethervtt/shard-verify 0.0.0-stage → 0.2.0

package/dist/node/files.js

@@ -0,0 +1,159 @@
+import { copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, join, relative, resolve } from "node:path";
+import { ShardError } from "@aethervtt/shard-core";
+import { compareImages, diffHeatmap } from "../compare.js";
+import { parsePlan } from "../plan.js";
+import { parsePerfRecord } from "../record.js";
+import { renderReport } from "../report.js";
+import { addApproval } from "../run.js";
+import { checkThresholds } from "../thresholds.js";
+import { decodePng, encodePng, sha256 } from "./png.js";
+async function readJson(file, what) {
+  let text;
+  try {
+    text = await readFile(file, "utf8");
+  } catch {
+    throw new ShardError("verify/not-found", `No ${what} at ${file}`, { path: file });
+  }
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    throw new ShardError(
+      "verify/invalid-json",
+      `${file} isn't valid JSON: ${err.message}`,
+      {
+        path: file
+      }
+    );
+  }
+}
+async function readPlan(file) {
+  return parsePlan(await readJson(file, "capture plan"), file);
+}
+async function readManifest(captures) {
+  const json = await readJson(join(captures, "manifest.json"), "capture run");
+  return json;
+}
+async function readApprovals(approved) {
+  try {
+    return JSON.parse(await readFile(join(approved, "approvals.json"), "utf8"));
+  } catch {
+    return void 0;
+  }
+}
+async function exists(file) {
+  return stat(file).then(
+    () => true,
+    () => false
+  );
+}
+async function compareRun(captures, approved, options = {}) {
+  const manifest = await readManifest(captures);
+  const approvals = await readApprovals(approved);
+  const reportFile = resolve(options.report ?? join(captures, "report.html"));
+  const base = dirname(reportFile);
+  const link = (file) => relative(base, file).split("\\").join("/");
+  const shots = [];
+  for (const shot of manifest.shots) {
+    const captureFile = join(captures, `${shot.id}.png`);
+    const approvedFile = join(approved, `${shot.id}.png`);
+    const approval = approvals?.approvals.filter((a) => a.shot === shot.id).at(-1);
+    const entry = {
+      id: shot.id,
+      hash: shot.hash,
+      status: "new",
+      capture: link(captureFile),
+      ...approval && { approval }
+    };
+    if (!await exists(approvedFile)) {
+      shots.push({ ...entry, reason: "no approved image" });
+      continue;
+    }
+    const approvedBytes = new Uint8Array(await readFile(approvedFile));
+    entry.approved = link(approvedFile);
+    if (sha256(approvedBytes) === shot.hash) {
+      shots.push({ ...entry, status: "pass", share: 0, maxDeltaE: 0, ssim: 1 });
+      continue;
+    }
+    const before = await decodePng(approvedBytes);
+    const after = await decodePng(new Uint8Array(await readFile(captureFile)));
+    let diff;
+    try {
+      diff = compareImages(before, after, shot.tolerance);
+    } catch (err) {
+      shots.push({ ...entry, status: "fail", reason: err.message });
+      continue;
+    }
+    const heatmapFile = join(captures, "diff", `${shot.id}.png`);
+    await mkdir(dirname(heatmapFile), { recursive: true });
+    await writeFile(heatmapFile, encodePng(diffHeatmap(before, diff), before.width, before.height));
+    shots.push({
+      ...entry,
+      status: diff.pass ? "pass" : "fail",
+      reason: diff.reason,
+      heatmap: link(heatmapFile),
+      share: diff.share,
+      maxDeltaE: diff.maxDeltaE,
+      ssim: diff.ssim
+    });
+  }
+  await mkdir(base, { recursive: true });
+  await writeFile(reportFile, renderReport(`Captures of ${manifest.fixture}`, shots));
+  return { pass: shots.every((s) => s.status === "pass"), shots, report: reportFile };
+}
+async function approveShot(options) {
+  const { captures, approved, shot } = options;
+  const approval = {
+    shot,
+    hash: "",
+    reason: options.reason ?? "",
+    by: options.by,
+    date: (options.date ?? /* @__PURE__ */ new Date()).toISOString()
+  };
+  const approvals = addApproval(await readApprovals(approved), approval);
+  const manifest = await readManifest(captures);
+  const captured = manifest.shots.find((s) => s.id === shot);
+  if (!captured) {
+    throw new ShardError("verify/unknown-shot", `The run in ${captures} has no shot "${shot}"`, {
+      hint: `Its shots: ${manifest.shots.map((s) => s.id).join(", ") || "none"}.`
+    });
+  }
+  const target = join(approved, `${shot}.png`);
+  await mkdir(dirname(target), { recursive: true });
+  await copyFile(join(captures, `${shot}.png`), target);
+  const recorded = approvals.approvals.at(-1);
+  recorded.hash = captured.hash;
+  await writeFile(join(approved, "approvals.json"), `${JSON.stringify(approvals, null, 2)}
+`);
+  return recorded;
+}
+async function readRecords(paths) {
+  const records = [];
+  const visit = async (path) => {
+    const info = await stat(path).catch(() => void 0);
+    if (!info) throw new ShardError("verify/not-found", `No records at ${path}`, { path });
+    if (info.isDirectory()) {
+      for (const entry of (await readdir(path)).sort()) {
+        const child = join(path, entry);
+        if (entry.endsWith(".json") || (await stat(child)).isDirectory()) await visit(child);
+      }
+    } else if (!path.endsWith("manifest.json")) {
+      records.push(parsePerfRecord(await readJson(path, "record"), path));
+    }
+  };
+  for (const path of paths) await visit(path);
+  return records;
+}
+async function perfCheck(planFile, paths) {
+  const plan = await readPlan(planFile);
+  const records = await readRecords(paths);
+  return { ...checkThresholds(records, plan.thresholds), records: records.length };
+}
+export {
+  approveShot,
+  compareRun,
+  perfCheck,
+  readManifest,
+  readPlan,
+  readRecords
+};

package/dist/node/index.d.ts

@@ -0,0 +1,3 @@
+export { browserLaunch, type CaptureOptions, type CaptureRun, clientUrl, ISOLATION_HEADERS, runCapture, } from './capture.js';
+export { approveShot, type CompareResult, type CompareRun, compareRun, perfCheck, readManifest, readPlan, readRecords, } from './files.js';
+export { decodePng, encodePng, sha256 } from './png.js';

package/dist/node/index.js

@@ -0,0 +1,30 @@
+import {
+  browserLaunch,
+  clientUrl,
+  ISOLATION_HEADERS,
+  runCapture
+} from "./capture.js";
+import {
+  approveShot,
+  compareRun,
+  perfCheck,
+  readManifest,
+  readPlan,
+  readRecords
+} from "./files.js";
+import { decodePng, encodePng, sha256 } from "./png.js";
+export {
+  ISOLATION_HEADERS,
+  approveShot,
+  browserLaunch,
+  clientUrl,
+  compareRun,
+  decodePng,
+  encodePng,
+  perfCheck,
+  readManifest,
+  readPlan,
+  readRecords,
+  runCapture,
+  sha256
+};

package/dist/node/png.d.ts

@@ -0,0 +1,7 @@
+import type { RgbaImage } from '../compare.js';
+/** RGBA8 PNG, filter none: diff heatmaps. */
+export declare function encodePng(rgba: Uint8Array, width: number, height: number): Uint8Array;
+/** A PNG as straight-alpha RGBA8. */
+export declare function decodePng(png: Uint8Array): Promise<RgbaImage>;
+/** SHA-256 of a file's bytes, hex. */
+export declare function sha256(bytes: Uint8Array): string;

package/dist/node/png.js

@@ -0,0 +1,50 @@
+import { createHash } from "node:crypto";
+import { crc32, deflateSync } from "node:zlib";
+import { decodePngImage } from "@aethervtt/shard-texture";
+function encodePng(rgba, width, height) {
+  const stride = width * 4;
+  const raw = new Uint8Array((stride + 1) * height);
+  for (let y = 0; y < height; y++) {
+    raw.set(rgba.subarray(y * stride, (y + 1) * stride), y * (stride + 1) + 1);
+  }
+  const chunk = (type, data) => {
+    const out2 = new Uint8Array(12 + data.length);
+    const view2 = new DataView(out2.buffer);
+    view2.setUint32(0, data.length);
+    for (let i = 0; i < 4; i++) out2[4 + i] = type.charCodeAt(i);
+    out2.set(data, 8);
+    view2.setUint32(8 + data.length, crc32(out2.subarray(4, 8 + data.length)));
+    return out2;
+  };
+  const header = new Uint8Array(13);
+  const view = new DataView(header.buffer);
+  view.setUint32(0, width);
+  view.setUint32(4, height);
+  header[8] = 8;
+  header[9] = 6;
+  const parts = [
+    new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]),
+    chunk("IHDR", header),
+    chunk("IDAT", deflateSync(raw)),
+    chunk("IEND", new Uint8Array(0))
+  ];
+  const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
+  let at = 0;
+  for (const p of parts) {
+    out.set(p, at);
+    at += p.length;
+  }
+  return out;
+}
+async function decodePng(png) {
+  const image = await decodePngImage(png);
+  return { width: image.width, height: image.height, data: image.data };
+}
+function sha256(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+export {
+  decodePng,
+  encodePng,
+  sha256
+};

package/dist/page-api.d.ts

@@ -0,0 +1,32 @@
+import type { RecordMeta } from './metrics/metrics.js';
+import type { PlanConditions } from './plan.js';
+import type { PerfRecord } from './record.js';
+/** `window.__shardCapture`: what `shard capture` drives (0062). Types only: Node imports it too. */
+export interface CapturePage {
+    apply(state: Record<string, unknown>): Promise<void>;
+    /** Resolves once every app has nothing loading and its last frame is presented. */
+    idle(timeoutMs?: number): Promise<void>;
+    /** Runs a host step; with `trace`, stamps it and reports the latency to the frame showing it. */
+    step(name: string, args?: unknown, trace?: boolean): Promise<{
+        result: unknown;
+        latencyMs?: number;
+    }>;
+    probe(): Promise<unknown>;
+    conditions(conditions: PlanConditions): Promise<void>;
+    /** A PNG of a canvas's own pixels, alpha kept, as base64. */
+    snapshot(selector: string): Promise<{
+        png: string;
+        width: number;
+        height: number;
+    }>;
+    /** Starts a new measurement window on every app. */
+    reset(): void;
+    /** The first app's record (the one with metrics). */
+    record(meta: RecordMeta): PerfRecord;
+}
+declare global {
+    interface Window {
+        __shardCapture?: CapturePage;
+        __shardReady?: boolean;
+    }
+}

package/dist/page.d.ts

@@ -0,0 +1,17 @@
+import { type App } from '@aethervtt/shard-runtime';
+import type { CapturePage } from './page-api.js';
+export type { CapturePage } from './page-api.js';
+/** What a host page gives the capture tool (0062). Every part is optional. */
+export interface CaptureHost {
+    /** Applies a shot's state: the camera, the active view, scene toggles. */
+    apply?(state: Record<string, unknown>): void | Promise<void>;
+    /** Host actions a plan's steps run by name: move a token, switch scenes, reconnect. */
+    steps?: Record<string, (args: unknown) => unknown>;
+    /** What checks read after a step: entity sets, owner and mirror counts. JSON only. */
+    probe?(): unknown;
+}
+/**
+ * Installs `window.__shardCapture` for `shard capture`, and sets `window.__shardReady` once every
+ * app has presented its first usable frame (the host calls `app.markUsable()`).
+ */
+export declare function installCapturePage(apps: App | readonly App[], host?: CaptureHost): CapturePage;

package/dist/page.js

@@ -0,0 +1,104 @@
+import { ShardError } from "@aethervtt/shard-core";
+import { Gpu, RenderScale } from "@aethervtt/shard-render";
+import { FrameDemand, LOADING_DEMAND } from "@aethervtt/shard-runtime";
+import { MetricsResource } from "./metrics/metrics.js";
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function loading(app) {
+  const gpu = app.world.tryResource(Gpu);
+  return app.world.resource(FrameDemand).isHeld(LOADING_DEMAND) || (gpu?.pipelines.pending ?? 0) > 0;
+}
+function settled(app) {
+  const demand = app.world.resource(FrameDemand);
+  return demand.mode !== "on-demand" || app.world.asleep && !demand.active;
+}
+async function idleApp(app, deadline) {
+  for (; ; ) {
+    if (performance.now() > deadline) {
+      const held = app.world.resource(FrameDemand).held().join(", ") || "nothing";
+      const compiling = app.world.tryResource(Gpu)?.pipelines.pending ?? 0;
+      throw new ShardError(
+        "verify/idle-timeout",
+        `The app never went idle for the capture (holding frames: ${held}; pipelines compiling: ${compiling})`,
+        { hint: "Holding nothing: it was slow, not stuck; raise the plan's timeoutMs." }
+      );
+    }
+    if (loading(app) || !settled(app)) {
+      await sleep(16);
+      continue;
+    }
+    await app.whenPresented();
+    if (!loading(app)) return;
+  }
+}
+function base64(bytes) {
+  let text = "";
+  for (let i = 0; i < bytes.length; i += 32768) {
+    text += String.fromCharCode(...bytes.subarray(i, i + 32768));
+  }
+  return btoa(text);
+}
+function installCapturePage(apps, host = {}) {
+  const list = Array.isArray(apps) ? apps : [apps];
+  const page = {
+    async apply(state) {
+      await host.apply?.(state);
+    },
+    async idle(timeoutMs = 3e4) {
+      const deadline = performance.now() + timeoutMs;
+      for (const app of list) await idleApp(app, deadline);
+    },
+    async step(name, args, trace) {
+      const run = host.steps?.[name];
+      if (!run) {
+        throw new ShardError("verify/unknown-step", `The page has no step "${name}"`, {
+          hint: `Its steps: ${Object.keys(host.steps ?? {}).join(", ") || "none"}.`
+        });
+      }
+      const result = await run(args);
+      if (!trace) return { result: result ?? null };
+      const latencies = await Promise.all(list.map((app) => app.trace(`step:${name}`)));
+      return { result: result ?? null, latencyMs: Math.max(...latencies) };
+    },
+    async probe() {
+      return await host.probe?.() ?? null;
+    },
+    async conditions(conditions) {
+      const scale = conditions.renderScale;
+      if (!scale) return;
+      for (const app of list) {
+        if (!app.world.tryResource(RenderScale)) continue;
+        app.world.patchResource(RenderScale, { mode: scale.mode, scale: scale.scale });
+      }
+    },
+    async snapshot(selector) {
+      const canvas = document.querySelector(selector);
+      if (!(canvas instanceof HTMLCanvasElement)) {
+        throw new ShardError("verify/no-canvas", `No canvas matches "${selector}"`);
+      }
+      const blob = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+      if (!blob) throw new ShardError("verify/no-canvas", `The canvas "${selector}" has no pixels`);
+      const png = base64(new Uint8Array(await blob.arrayBuffer()));
+      return { png, width: canvas.width, height: canvas.height };
+    },
+    reset() {
+      for (const app of list) app.world.tryResource(MetricsResource)?.reset();
+    },
+    record(meta) {
+      for (const app of list) {
+        const metrics = app.world.tryResource(MetricsResource);
+        if (metrics) return metrics.record(meta);
+      }
+      throw new ShardError("verify/no-metrics", "No app on this page has metrics", {
+        hint: "Add metricsPlugin() from @aethervtt/shard-verify/metrics to the app."
+      });
+    }
+  };
+  window.__shardCapture = page;
+  void Promise.all(list.map((app) => app.whenUsable())).then(() => {
+    window.__shardReady = true;
+  });
+  return page;
+}
+export {
+  installCapturePage
+};

package/dist/plan.d.ts

@@ -0,0 +1,132 @@
+import type { JsonSchema } from '@aethervtt/shard-core';
+import type { Tolerance } from './compare.js';
+export type BrowserName = 'chromium' | 'webkit' | 'firefox';
+export type CaptureScope = 'canvas' | 'page';
+/**
+ * A capture and measurement run (0062): which page, in which browsers, at which sizes, with what
+ * shots, host steps and checks, measured scenarios, and pass thresholds. Every renderer in a
+ * comparison runs the same plan.
+ */
+export interface CapturePlan {
+    url: string;
+    browsers: BrowserName[];
+    dpr: number[];
+    /** CSS pixels. */
+    viewport: [number, number];
+    /** canvas: the canvas's own pixels, alpha included. page: the viewport, DOM and all. */
+    scope: CaptureScope;
+    /** The canvas a canvas-scope shot reads. Default the page's first `canvas`. */
+    canvas: string;
+    /** What records name the fixture. Default the URL's path and hash. */
+    fixture: string;
+    /** What records name the renderer, when the page doesn't say. */
+    renderer?: string;
+    /**
+     * The graphics API the page is asked for (0064), as `?backend=`. Unset, the page picks. A browser
+     * without it skips the plan: no WebGPU skips a 'webgpu' plan, not a 'webgl2' one.
+     */
+    backend?: 'auto' | 'webgpu' | 'webgl2';
+    conditions: PlanConditions;
+    /** Default tolerance for every shot. */
+    tolerance: Tolerance;
+    /** Browser contexts against one host session. Default one, `main`. */
+    clients: PlanClient[];
+    /** Shots every client takes before the steps run. */
+    shots: PlanShot[];
+    steps: PlanStep[];
+    scenarios: PlanScenario[];
+    thresholds: Thresholds;
+    /** Longest wait for readiness, an idle frame, or a step, in ms. Default 60 s. */
+    timeoutMs: number;
+}
+export interface PlanConditions {
+    /** Pins Shard's RenderScale (0051) for the run, so resolution can't buy frame time. */
+    renderScale?: {
+        mode: 'fixed';
+        scale: number;
+    };
+}
+export interface PlanClient {
+    name: string;
+    /** Passed to the page as `?role=`. */
+    role?: string;
+    /** More query parameters for this client's page. */
+    query?: Record<string, string>;
+    /** Its own shots, after the plan's. */
+    shots?: PlanShot[];
+}
+export interface PlanShot {
+    name: string;
+    /** Given to the page's `__shardCapture.apply(state)`: camera, view, toggles. */
+    state?: Record<string, unknown>;
+    scope?: CaptureScope;
+    canvas?: string;
+    tolerance?: Tolerance;
+}
+/** One host action, run in each of its clients in order, then checked in every client. */
+export interface PlanStep {
+    name: string;
+    /** The page's step to run (`__shardCapture.step`). Default `name`. */
+    run?: string;
+    args?: unknown;
+    /** Clients that run it. Default all; every client is still checked. */
+    clients?: string[];
+    /** Stamp the step with `app.trace` and report `step.latencyMs` to the checks. */
+    trace?: boolean;
+    /** Probe path → matcher, per client name. */
+    expect?: Record<string, Expectations>;
+    /** Take a shot of every client after the step (named after it), or the named shots. */
+    capture?: boolean | PlanShot[];
+}
+/** Dotted path into what the page's `probe()` returned (plus `step.latencyMs`) → matcher. */
+export type Expectations = Record<string, Matcher>;
+export interface Matcher {
+    equals?: unknown;
+    includes?: unknown[];
+    excludes?: unknown[];
+    min?: number;
+    max?: number;
+    /** Deep-equal to the value at another path (a fresh load's entity set, say). */
+    sameAs?: string;
+}
+/** A measured run: steps start it, the window runs `seconds`, then each client records. */
+export interface PlanScenario {
+    name: string;
+    steps?: PlanStep[];
+    seconds: number;
+    /** Clients that record. Default all. */
+    clients?: string[];
+}
+export interface ThresholdRule {
+    max?: number;
+    min?: number;
+    /** A renderer in the same run whose value, times `ratio`, is the budget. */
+    maxRatioTo?: string;
+    ratio?: number;
+}
+/**
+ * A budget on one span of a record's `breakdown` (0074): `{ "span": "partition/spawn", "p95": 2 }`.
+ * Its p50, p95 or max (ms) must stay at or under the number given.
+ */
+export interface SpanThreshold {
+    span: string;
+    p50?: number;
+    p95?: number;
+    max?: number;
+}
+/** A scenario's rules: metric path (`frameTime.p95`) → rule, and span budgets under `spans`. */
+export interface ScenarioThresholds {
+    [metric: string]: ThresholdRule | readonly SpanThreshold[] | undefined;
+    spans?: readonly SpanThreshold[];
+}
+/** Scenario → its rules. */
+export type Thresholds = Record<string, ScenarioThresholds>;
+/** JSON Schema for capture plans (`.shard/schemas/capture-plan.schema.json`). */
+export declare function capturePlanJsonSchema(): JsonSchema;
+/**
+ * Reads a plan, filling in defaults, or throws `verify/invalid-plan` listing every problem: schema
+ * errors, then references to clients that don't exist and names used twice.
+ */
+export declare function parsePlan(json: unknown, file?: string): CapturePlan;
+/** A shot's id in a capture run: `chromium/main/map-close@2x` (the capture's path, less `.png`). */
+export declare function shotId(browser: string, client: string, shot: string, dpr: number): string;