@aethervtt/shard-verify 0.1.0 → 0.2.1

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 export { compareImages, DEFAULT_TOLERANCE, deltaE2000, diffHeatmap, type ImageDiff, type RgbaImage, type Tolerance, } from './compare.js';
 export { type CheckFailure, checkExpectations, deepEqual, valueAt } from './expect.js';
-export { type BrowserName, type CapturePlan, type CaptureScope, capturePlanJsonSchema, type Expectations, type Matcher, type PlanClient, type PlanConditions, type PlanScenario, type PlanShot, type PlanStep, parsePlan, shotId, type ThresholdRule, type Thresholds, } from './plan.js';
+export { type BrowserName, type CapturePlan, type CaptureScope, capturePlanJsonSchema, type Expectations, type Matcher, type PlanClient, type PlanConditions, type PlanScenario, type PlanShot, type PlanStep, parsePlan, type ScenarioThresholds, type SpanThreshold, shotId, type ThresholdRule, type Thresholds, } from './plan.js';
 export { type PerfDevice, type PerfRecord, parsePerfRecord, perfRecordJsonSchema, rendererName, validateJson, } from './record.js';
 export { type ReportShot, renderReport } from './report.js';
 export { type Approval, type ApprovalsFile, addApproval, type CapturedShot, type CaptureManifest, } from './run.js';
-export { type Breach, checkThresholds, type PerfCheck } from './thresholds.js';
+export { type Breach, checkThresholds, type PerfCheck, spansThatGrew } from './thresholds.js';

package/dist/index.js

@@ -20,7 +20,7 @@ import { renderReport } from "./report.js";
 import {
   addApproval
 } from "./run.js";
-import { checkThresholds } from "./thresholds.js";
+import { checkThresholds, spansThatGrew } from "./thresholds.js";
 export {
   DEFAULT_TOLERANCE,
   addApproval,
@@ -37,6 +37,7 @@ export {
   renderReport,
   rendererName,
   shotId,
+  spansThatGrew,
   validateJson,
   valueAt
 };

package/dist/metrics/metrics.js

@@ -11,7 +11,8 @@ import { Gpu, Graph, RenderScale, Window } from "@aethervtt/shard-render";
 import {
   AppControlResource,
   definePlugin,
-  FrameDemand
+  FrameDemand,
+  perfBreakdown
 } from "@aethervtt/shard-runtime";
 import { distribution, percentile, round, SampleRing } from "../stats.js";
 const FRAME_SAMPLES = 1 << 15;
@@ -130,7 +131,7 @@ class Metrics {
     const device = this.app.world.tryResource(Gpu);
     const memory = device?.memory() ?? { bytes: 0, byCategory: {} };
     return {
-      version: 1,
+      version: 2,
       renderer: meta.renderer ?? this.renderer,
       fixture: meta.fixture,
       scenario: meta.scenario,
@@ -152,7 +153,8 @@ class Metrics {
       frameTime,
       longTasks: { count: tasks.length, totalMs: round(totalMs), maxMs: round(maxMs) },
       gpuMemory: { bytes: memory.bytes, byCategory: memory.byCategory },
-      download: this.performance?.downloads?.() ?? { transferred: 0, decoded: 0 }
+      download: this.performance?.downloads?.() ?? { transferred: 0, decoded: 0 },
+      breakdown: perfBreakdown(this.app.world)
     };
   }
   device() {
@@ -181,7 +183,7 @@ const frameStart = defineSystem({
 const metricsMethods = [
   {
     name: "metrics.record",
-    description: "A performance record (spec 0062) of the running app: cold start, first usable frame, patch-to-frame latency, frame times, long tasks, GPU memory, and downloads, over the window since metrics.reset (at most 30 s).",
+    description: "A performance record (spec 0062) of the running app: cold start, first usable frame, patch-to-frame latency, frame times, long tasks, GPU memory, downloads, and the breakdown (0074: the 10 CPU and GPU spans with the highest p95), over the window since metrics.reset (at most 30 s).",
     params: defineSchema("verify/MetricsRecordParams", {
       fixture: t.string({ required: true, description: 'What was loaded, e.g. "tabletop-small".' }),
       scenario: t.string({ required: true, description: 'What ran, e.g. "tabletop-pan".' }),

package/dist/node/capture.d.ts

@@ -10,6 +10,11 @@ export interface CaptureOptions {
     headed?: boolean;
     /** Extra browser flags (also `SHARD_BROWSER_ARGS`, space-separated). */
     browserArgs?: string[];
+    /**
+     * Chromium's release channel (also `SHARD_BROWSER_CHANNEL`): `chrome` runs the installed Chrome,
+     * headed at the display's refresh rate for frame-interval records. Default Playwright's Chromium.
+     */
+    channel?: string;
     /** Progress lines. */
     log?: (message: string) => void;
 }
@@ -29,8 +34,15 @@ export interface CaptureRun {
  * three.js runs its tests), drawing without a Vulkan surface. `SHARD_BROWSER_HEADED=1` shows the
  * window (CI runs it under `xvfb-run`), and `SHARD_BROWSER_ARGS` adds flags.
  */
-export declare function browserLaunch(browser: BrowserName, dpr: number, options?: Pick<CaptureOptions, 'headed' | 'browserArgs'>): LaunchOptions;
+export declare function browserLaunch(browser: BrowserName, dpr: number, options?: Pick<CaptureOptions, 'headed' | 'browserArgs' | 'channel'>): LaunchOptions;
 /** Runs a capture plan in real browsers (0062). Checks that fail don't stop the run. */
 export declare function runCapture(plan: CapturePlan, options: CaptureOptions): Promise<CaptureRun>;
 /** The plan's URL with the backend it asks for, and the client's role and query parameters. */
 export declare function clientUrl(url: string, client: PlanClient, backend?: string): string;
+/**
+ * Headers for pages Shard serves to measure (0074): cross-origin isolation, so `performance.now()`
+ * steps 5 µs instead of 100 µs (`credentialless` COEP keeps cross-origin loads working without
+ * CORP headers), and `Document-Policy: js-profiling` for the JS Self-Profiling API. `shard dev`,
+ * the playground and the capture server send them.
+ */
+export declare const ISOLATION_HEADERS: Readonly<Record<string, string>>;

package/dist/node/capture.js

@@ -21,7 +21,8 @@ function browserLaunch(browser, dpr, options = {}) {
   ];
   if (process.platform === "linux")
     args.push("--enable-features=Vulkan", "--disable-vulkan-surface");
-  return { headless, channel: "chromium", args: [...args, ...extra] };
+  const channel = options.channel ?? process.env.SHARD_BROWSER_CHANNEL ?? "chromium";
+  return { headless, channel, args: [...args, ...extra] };
 }
 async function runCapture(plan, options) {
   const playwright = await import("playwright");
@@ -170,6 +171,16 @@ async function runSteps(plan, steps, browser, dpr, clients, out, result, prefix)
           { run: step.run ?? step.name, args: step.args ?? null, trace: step.trace ?? false }
         );
         latency.set(client.plan.name, ran.latencyMs);
+        if (ran.result !== null && ran.result !== void 0) {
+          const file = join(
+            out,
+            "results",
+            `${browser}@${dpr}x-${client.plan.name}`,
+            `${name}.json`
+          );
+          await writeOut(file, `${JSON.stringify(ran.result, null, 2)}
+`);
+        }
       } catch (err) {
         failures.push({ step: name, client: client.plan.name, path: "", message: pageMessage(err) });
       }
@@ -246,7 +257,13 @@ function pageMessage(err) {
   const message = err.message ?? String(err);
   return message.replace(/^page\.evaluate: /, "").split("\n")[0];
 }
+const ISOLATION_HEADERS = {
+  "Cross-Origin-Opener-Policy": "same-origin",
+  "Cross-Origin-Embedder-Policy": "credentialless",
+  "Document-Policy": "js-profiling"
+};
 export {
+  ISOLATION_HEADERS,
   browserLaunch,
   clientUrl,
   runCapture

package/dist/node/index.d.ts

@@ -1,3 +1,3 @@
-export { browserLaunch, type CaptureOptions, type CaptureRun, clientUrl, runCapture, } from './capture.js';
+export { browserLaunch, type CaptureOptions, type CaptureRun, clientUrl, ISOLATION_HEADERS, runCapture, } from './capture.js';
 export { approveShot, type CompareResult, type CompareRun, compareRun, perfCheck, readManifest, readPlan, readRecords, } from './files.js';
 export { decodePng, encodePng, sha256 } from './png.js';

package/dist/node/index.js

@@ -1,6 +1,7 @@
 import {
   browserLaunch,
   clientUrl,
+  ISOLATION_HEADERS,
   runCapture
 } from "./capture.js";
 import {
@@ -13,6 +14,7 @@ import {
 } from "./files.js";
 import { decodePng, encodePng, sha256 } from "./png.js";
 export {
+  ISOLATION_HEADERS,
   approveShot,
   browserLaunch,
   clientUrl,

package/dist/plan.d.ts

@@ -104,8 +104,23 @@ export interface ThresholdRule {
     maxRatioTo?: string;
     ratio?: number;
 }
-/** Scenario → metric path (`frameTime.p95`) → rule. */
-export type Thresholds = Record<string, Record<string, ThresholdRule>>;
+/**
+ * A budget on one span of a record's `breakdown` (0074): `{ "span": "partition/spawn", "p95": 2 }`.
+ * Its p50, p95 or max (ms) must stay at or under the number given.
+ */
+export interface SpanThreshold {
+    span: string;
+    p50?: number;
+    p95?: number;
+    max?: number;
+}
+/** A scenario's rules: metric path (`frameTime.p95`) → rule, and span budgets under `spans`. */
+export interface ScenarioThresholds {
+    [metric: string]: ThresholdRule | readonly SpanThreshold[] | undefined;
+    spans?: readonly SpanThreshold[];
+}
+/** Scenario → its rules. */
+export type Thresholds = Record<string, ScenarioThresholds>;
 /** JSON Schema for capture plans (`.shard/schemas/capture-plan.schema.json`). */
 export declare function capturePlanJsonSchema(): JsonSchema;
 /**

package/dist/plan.js

@@ -122,6 +122,23 @@ function capturePlanJsonSchema() {
         type: "object",
         additionalProperties: {
           type: "object",
+          properties: {
+            spans: {
+              type: "array",
+              description: `Span budgets on records' breakdown (spec 0074): { "span": "partition/spawn", "p95": 2 }.`,
+              items: {
+                type: "object",
+                additionalProperties: false,
+                required: ["span"],
+                properties: {
+                  span: { type: "string", minLength: 1 },
+                  p50: { type: "number", minimum: 0 },
+                  p95: { type: "number", minimum: 0 },
+                  max: { type: "number", minimum: 0 }
+                }
+              }
+            }
+          },
           additionalProperties: {
             type: "object",
             additionalProperties: false,

package/dist/record.d.ts

@@ -1,4 +1,4 @@
-import type { JsonSchema } from '@aethervtt/shard-core';
+import type { JsonSchema, SpanStats } from '@aethervtt/shard-core';
 import { ShardError } from '@aethervtt/shard-core';
 /**
  * One measured run of one scenario on one renderer (0062). Shard's `metrics.record` writes it, and
@@ -6,7 +6,8 @@ import { ShardError } from '@aethervtt/shard-core';
  * sizes bytes.
  */
 export interface PerfRecord {
-    version: 1;
+    /** 2 adds `breakdown` (0074); version 1 records still read. */
+    version: 1 | 2;
     /** `shard@<sha>`, `three@0.160.1`: the part before `@` names the renderer in thresholds. */
     renderer: string;
     fixture: string;
@@ -61,6 +62,14 @@ export interface PerfRecord {
         transferred: number;
         decoded: number;
     };
+    /**
+     * Where the time went (0074, version 2): the 10 spans with the highest p95, CPU (systems,
+     * schedules, render encodes, workers) and GPU passes, over the profiler's window.
+     */
+    breakdown?: {
+        cpu: SpanStats[];
+        gpu: SpanStats[];
+    };
 }
 export interface PerfDevice {
     ua: string;

package/dist/record.js

@@ -1,6 +1,23 @@
 import { ShardError } from "@aethervtt/shard-core";
 const ms = { type: "number", minimum: 0 };
 const count = { type: "integer", minimum: 0 };
+const spanStats = {
+  type: "array",
+  items: {
+    type: "object",
+    additionalProperties: false,
+    required: ["span", "track", "calls", "total", "p50", "p95", "max"],
+    properties: {
+      span: { type: "string", minLength: 1 },
+      track: { enum: ["main", "gpu", "async", "gc", "worker"] },
+      calls: count,
+      total: ms,
+      p50: ms,
+      p95: ms,
+      max: ms
+    }
+  }
+};
 function perfRecordJsonSchema() {
   return {
     $schema: "https://json-schema.org/draft/2020-12/schema",
@@ -26,7 +43,7 @@ function perfRecordJsonSchema() {
     ],
     properties: {
       $schema: { type: "string" },
-      version: { const: 1 },
+      version: { enum: [1, 2] },
       renderer: {
         type: "string",
         minLength: 1,
@@ -98,6 +115,13 @@ function perfRecordJsonSchema() {
         additionalProperties: false,
         required: ["transferred", "decoded"],
         properties: { transferred: count, decoded: count }
+      },
+      breakdown: {
+        type: "object",
+        additionalProperties: false,
+        required: ["cpu", "gpu"],
+        description: "Version 2 (spec 0074): the 10 CPU and 10 GPU spans with the highest p95 over the window.",
+        properties: { cpu: spanStats, gpu: spanStats }
       }
     }
   };

package/dist/thresholds.d.ts

@@ -15,6 +15,17 @@ export interface Breach {
         value: number | undefined;
     };
     message: string;
+    /** A span budget's span (0074). */
+    span?: string;
+    /**
+     * Against a baseline: the spans whose p95 grew the most from its record to this one (0074), when
+     * both records have a breakdown.
+     */
+    grew?: {
+        span: string;
+        p95: number;
+        baselineP95: number;
+    }[];
 }
 export interface PerfCheck {
     pass: boolean;
@@ -29,3 +40,12 @@ export interface PerfCheck {
  * run has one; with no baseline record, the rule fails.
  */
 export declare function checkThresholds(records: readonly PerfRecord[], thresholds: Thresholds): PerfCheck;
+/**
+ * The spans whose p95 grew the most from `baseline` to `record` (0074), largest growth first: a
+ * span missing from the baseline's breakdown counts from 0. Empty unless both have a breakdown.
+ */
+export declare function spansThatGrew(record: PerfRecord, baseline: PerfRecord, limit?: number): {
+    span: string;
+    p95: number;
+    baselineP95: number;
+}[];

package/dist/thresholds.js

@@ -4,18 +4,28 @@ function checkThresholds(records, thresholds) {
   const breaches = [];
   let checked = 0;
   for (const [scenario, rules] of Object.entries(thresholds)) {
+    const metrics = [];
+    for (const [metric, rule] of Object.entries(rules)) {
+      if (metric !== "spans" && rule && !Array.isArray(rule))
+        metrics.push([metric, rule]);
+    }
     const baselines = new Set(
-      Object.values(rules).map((r) => r.maxRatioTo).filter((r) => r !== void 0)
+      metrics.map(([, r]) => r.maxRatioTo).filter((r) => r !== void 0)
     );
     const subjects = records.filter(
       (r) => r.scenario === scenario && !baselines.has(rendererName(r))
     );
     for (const record of subjects) {
-      for (const [metric, rule] of Object.entries(rules)) {
+      for (const [metric, rule] of metrics) {
         checked++;
         const breach = apply(record, metric, rule, records);
         if (breach) breaches.push(breach);
       }
+      for (const rule of rules.spans ?? []) {
+        checked++;
+        const breach = applySpan(record, rule);
+        if (breach) breaches.push(breach);
+      }
     }
   }
   return { pass: breaches.length === 0, checked, breaches };
@@ -64,12 +74,62 @@ function apply(record, metric, rule, records) {
   const baseValue = metricOf(baseline, metric);
   const limit = baseValue === void 0 ? void 0 : baseValue * ratio;
   if (limit !== void 0 && value <= limit) return void 0;
-  return {
+  const grew = spansThatGrew(record, baseline);
+  const breach = {
     ...base,
     budget: `max ${ratio} \xD7 ${baseline.renderer} (${baseValue ?? "none"})${limit === void 0 ? "" : ` = ${round(limit)}`}`,
     baseline: { renderer: baseline.renderer, value: baseValue },
-    message: limit === void 0 ? `${where}: ${baseline.renderer}'s record has no ${metric}` : `${where}: ${value} is over ${ratio} \xD7 ${baseline.renderer}'s ${baseValue} (${round(limit)})`
+    message: limit === void 0 ? `${where}: ${baseline.renderer}'s record has no ${metric}` : `${where}: ${value} is over ${ratio} \xD7 ${baseline.renderer}'s ${baseValue} (${round(limit)})${grew.length > 0 ? `; grew most: ${grew.map((g) => `${g.span} ${g.baselineP95} \u2192 ${g.p95} ms`).join(", ")}` : ""}`
+  };
+  if (grew.length > 0) breach.grew = grew;
+  return breach;
+}
+function spansOf(record) {
+  return record.breakdown ? [...record.breakdown.cpu, ...record.breakdown.gpu] : [];
+}
+function spansThatGrew(record, baseline, limit = 3) {
+  if (!record.breakdown || !baseline.breakdown) return [];
+  const before = new Map(spansOf(baseline).map((s) => [s.span, s.p95]));
+  return spansOf(record).map((s) => ({ span: s.span, p95: s.p95, baselineP95: before.get(s.span) ?? 0 })).filter((g) => g.p95 > g.baselineP95).sort((a, b) => b.p95 - b.baselineP95 - (a.p95 - a.baselineP95) || (a.span < b.span ? -1 : 1)).slice(0, limit);
+}
+function applySpan(record, rule) {
+  const stats = spansOf(record).find((s) => s.span === rule.span);
+  const where = `${record.scenario} span ${rule.span} (${record.renderer})`;
+  const base = {
+    scenario: record.scenario,
+    renderer: record.renderer,
+    fixture: record.fixture,
+    span: rule.span
   };
+  if (!stats) {
+    const metric = `span:${rule.span}`;
+    return {
+      ...base,
+      metric,
+      value: void 0,
+      budget: describeSpan(rule),
+      message: record.breakdown ? `${where}: the record's breakdown has no ${rule.span}` : `${where}: the record has no breakdown (version 1)`
+    };
+  }
+  for (const key of ["p50", "p95", "max"]) {
+    const limit = rule[key];
+    if (limit === void 0 || stats[key] <= limit) continue;
+    return {
+      ...base,
+      metric: `span:${rule.span}.${key}`,
+      value: stats[key],
+      budget: `${key} ${limit}`,
+      message: `${where}: ${key} ${stats[key]} ms is over ${limit} ms`
+    };
+  }
+  return void 0;
+}
+function describeSpan(rule) {
+  const parts = [];
+  for (const key of ["p50", "p95", "max"]) {
+    if (rule[key] !== void 0) parts.push(`${key} ${rule[key]}`);
+  }
+  return parts.join(", ");
 }
 function findBaseline(record, renderer, records) {
   const candidates = records.filter(
@@ -91,5 +151,6 @@ function round(value) {
   return Math.round(value * 100) / 100;
 }
 export {
-  checkThresholds
+  checkThresholds,
+  spansThatGrew
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aethervtt/shard-verify",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "license": "MIT",
   "type": "module",
   "sideEffects": false,
@@ -28,13 +28,13 @@
     "README.md"
   ],
   "dependencies": {
-    "@aethervtt/shard-assets": "0.1.0",
-    "@aethervtt/shard-core": "0.1.0",
-    "@aethervtt/shard-gpu": "0.1.0",
-    "@aethervtt/shard-platform": "0.1.0",
-    "@aethervtt/shard-render": "0.1.0",
-    "@aethervtt/shard-runtime": "0.1.0",
-    "@aethervtt/shard-texture": "0.1.0"
+    "@aethervtt/shard-assets": "0.2.1",
+    "@aethervtt/shard-core": "0.2.1",
+    "@aethervtt/shard-gpu": "0.2.1",
+    "@aethervtt/shard-platform": "0.2.1",
+    "@aethervtt/shard-render": "0.2.1",
+    "@aethervtt/shard-runtime": "0.2.1",
+    "@aethervtt/shard-texture": "0.2.1"
   },
   "repository": {
     "type": "git",
@@ -46,6 +46,6 @@
     "provenance": true
   },
   "shard": {
-    "buildSha": "8e14c9896f82a3554b156f1ffdd68f2cc3e7d78a"
+    "buildSha": "772eb0d2b8a5fc499105708f4ae92c58788cc9ea"
   }
 }