@pixel-point/toolcraft 0.0.15 → 0.0.16

package/templates/starter/e2e/browser-performance-reporter.spec.ts

@@ -0,0 +1,456 @@
+import { expect, test } from "@playwright/test";
+import type { TestCase } from "@playwright/test/reporter";
+import { existsSync, mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  defineToolcraft,
+  defineToolcraftPerformance,
+  deriveToolcraftPerformancePaths,
+  registerToolcraftRendererPipeline,
+  type ToolcraftRendererPipelinePassContract,
+} from "@repo/toolcraft-runtime";
+
+import {
+  TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+  type ToolcraftBrowserRuntimeRequirement,
+} from "../src/app/test-evidence/browser-runtime-contract";
+import {
+  TOOLCRAFT_BROWSER_PERFORMANCE_EVIDENCE_ATTACHMENT_NAME,
+  TOOLCRAFT_BROWSER_PERFORMANCE_EVIDENCE_CONTENT_TYPE,
+  serializeToolcraftBrowserPerformanceEvidence,
+  type ToolcraftPerformancePipelinePhase,
+} from "../src/app/test-evidence/browser-performance-contract";
+import {
+  TOOLCRAFT_BROWSER_PERFORMANCE_MEASUREMENT_ATTACHMENT_NAME,
+  TOOLCRAFT_BROWSER_PERFORMANCE_MEASUREMENT_CONTENT_TYPE,
+  serializeToolcraftPerformanceMeasurement,
+} from "../src/app/test-evidence/browser-performance-measurement-contract";
+import {
+  fakeSuite,
+  fakeTestCase,
+  passedResultWithEvidence,
+} from "./browser-runtime-evidence-reporter-test-fixtures";
+import ToolcraftBrowserRuntimeEvidenceReporter from "./browser-runtime-evidence-reporter";
+import { getToolcraftPerformancePathTestName } from "./performance-path-helpers";
+import {
+  TOOLCRAFT_PERFORMANCE_ENVIRONMENT_ATTACHMENT_NAME,
+  TOOLCRAFT_PERFORMANCE_ENVIRONMENT_CONTENT_TYPE,
+} from "./performance-environment-evidence";
+import { TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR_ENV } from "./performance-fixture-selection";
+
+const originalFixtureSelector =
+  process.env[TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR_ENV];
+
+test.beforeEach(() => {
+  process.env[TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR_ENV] = "maximum";
+});
+
+test.afterEach(() => {
+  if (originalFixtureSelector === undefined) {
+    delete process.env[TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR_ENV];
+  } else {
+    process.env[TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR_ENV] =
+      originalFixtureSelector;
+  }
+});
+
+const performanceSchema = defineToolcraft({
+  canvas: { enabled: true, sizing: { mode: "editable-output" } },
+  panels: {
+    controls: {
+      sections: [
+        {
+          controls: {
+            opacity: {
+              defaultValue: 0.5,
+              label: "Opacity",
+              max: 1,
+              min: 0,
+              performanceRole: "responsiveness",
+              target: "appearance.opacity",
+              type: "slider",
+            },
+          },
+          title: "Appearance",
+        },
+      ],
+      title: "Controls",
+    },
+  },
+});
+
+type ReporterPipelinePasses = {
+  composite: ToolcraftRendererPipelinePassContract<void>;
+};
+
+const performanceRegistration =
+  registerToolcraftRendererPipeline<ReporterPipelinePasses>()({
+    interactionInvalidation: [
+      {
+        interaction: "control-drag",
+        invalidates: ["composite"],
+        targets: ["appearance.opacity"],
+      },
+    ],
+    passes: [
+      {
+        cost: {
+          dimensions: [],
+          frequency: "interaction",
+          relationship: "constant",
+        },
+        id: "composite",
+        inputs: ["appearance.opacity"],
+        invalidatedBy: ["appearance.opacity"],
+        kind: "composite",
+        lifecycle: { cache: "none", resourceScope: "call" },
+        output: "preview",
+        quality: "full",
+        runsOn: "main",
+      },
+    ],
+    runtimeId: "reporter-pipeline-v1",
+  });
+
+const performanceModel = defineToolcraftPerformance({
+  rendererPipeline: performanceRegistration,
+  rendererStrategy: "dom",
+  scenarios: [],
+  usesCustomRenderer: true,
+  workloadEnvelope: { dimensions: [] },
+});
+const [reporterPerformancePath] = deriveToolcraftPerformancePaths(
+  performanceSchema,
+  performanceModel,
+);
+const richTestName = "browser perf: reporter pipeline evidence";
+const performanceConfig = defineToolcraftPerformance({
+  ...performanceModel,
+  scenarios: [
+    {
+      automated: true,
+      automatedTestName: "perf: reporter pipeline evidence",
+      browser: true,
+      browserTestName: richTestName,
+      coversTargets: reporterPerformancePath!.targets,
+      expectedObservable: "Opacity changes the preview.",
+      fixture: "reporter fixture",
+      id: "reporter-pipeline",
+      interaction: "control-drag",
+      pathId: reporterPerformancePath!.id,
+      target: "appearance.opacity",
+    },
+  ],
+});
+
+function richPipelineAttachment(phase: ToolcraftPerformancePipelinePhase) {
+  const phaseIndex = (["cold", "warm", "sustained"] as const).indexOf(phase);
+  const counters = (executions: number) => ({
+    activeResources: 0,
+    cacheHits: 0,
+    cacheMisses: executions,
+    durationMax: executions === 0 ? 0 : 1,
+    durationTotal: executions,
+    executions,
+    resourceCreations: 0,
+    resourceDisposals: 0,
+    transfers: 0,
+  });
+  return {
+    body: Buffer.from(
+      serializeToolcraftBrowserPerformanceEvidence({
+        after: {
+          disposed: false,
+          passes: [
+            {
+              ...counters(phaseIndex + 1),
+              passId: "composite",
+            },
+          ],
+          runtimeId: performanceRegistration.runtimeId,
+        },
+        before: {
+          disposed: false,
+          passes: [{ ...counters(phaseIndex), passId: "composite" }],
+          runtimeId: performanceRegistration.runtimeId,
+        },
+        evidenceType: "performance-pipeline",
+        pathId: reporterPerformancePath!.id,
+        phase,
+        profile: reporterPerformancePath!.profile,
+      }),
+    ),
+    contentType: TOOLCRAFT_BROWSER_PERFORMANCE_EVIDENCE_CONTENT_TYPE,
+    name: TOOLCRAFT_BROWSER_PERFORMANCE_EVIDENCE_ATTACHMENT_NAME,
+  };
+}
+
+function richMeasurementAttachment(phase: ToolcraftPerformancePipelinePhase) {
+  return {
+    body: Buffer.from(
+      serializeToolcraftPerformanceMeasurement({
+        evidenceType: "performance-measurement-metrics",
+        kind: "interaction",
+        metrics: {
+          droppedFrameCount: 0,
+          droppedFrameRatio: 0,
+          durationMs: 10,
+          frameGapP50Ms: 10,
+          frameGapP95Ms: 16,
+          frameGapP99Ms: 16,
+          longTaskCount: 0,
+          longTaskMaxMs: 0,
+          maxFrameGapMs: 16,
+          sampleCount: 3,
+        },
+        pathId: reporterPerformancePath!.id,
+        phase,
+        profile: reporterPerformancePath!.profile,
+      }),
+    ),
+    contentType: TOOLCRAFT_BROWSER_PERFORMANCE_MEASUREMENT_CONTENT_TYPE,
+    name: TOOLCRAFT_BROWSER_PERFORMANCE_MEASUREMENT_ATTACHMENT_NAME,
+  };
+}
+
+function environmentAttachment() {
+  return {
+    body: Buffer.from(
+      JSON.stringify({
+        browser: { name: "chromium", version: "test" },
+        calibration: { durationMs: 1, iterations: 10 },
+        cpuThrottling: { mode: "none", rate: 1 },
+        evidenceType: "performance-environment",
+        hardwareConcurrency: 4,
+        version: 1,
+        viewport: { height: 720, width: 1280 },
+      }),
+    ),
+    contentType: TOOLCRAFT_PERFORMANCE_ENVIRONMENT_CONTENT_TYPE,
+    name: TOOLCRAFT_PERFORMANCE_ENVIRONMENT_ATTACHMENT_NAME,
+  };
+}
+
+test("custom renderer reports require rich phases in addition to runtime registration evidence", () => {
+  const booleanRequirement: ToolcraftBrowserRuntimeRequirement = {
+    evidenceType: "performance-budget",
+    requirementId: "reporter-pipeline",
+    testName: richTestName,
+  };
+  const booleanResult = passedResultWithEvidence(booleanRequirement);
+  const marker = fakeTestCase({
+    file: "/product/e2e/app-browser-runtime-evidence.spec.ts",
+    title: TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+  });
+  const reports: unknown[] = [];
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    performanceConfig,
+    performanceRequirements: [booleanRequirement],
+    performanceSchema,
+    reportPerformance: (report) => reports.push(report),
+  });
+  const scenarioTest = fakeTestCase({
+    results: [
+      {
+        ...booleanResult,
+        attachments: [
+          ...booleanResult.attachments,
+          richPipelineAttachment("cold"),
+          richPipelineAttachment("warm"),
+          richPipelineAttachment("sustained"),
+          richMeasurementAttachment("cold"),
+          richMeasurementAttachment("warm"),
+          richMeasurementAttachment("sustained"),
+        ],
+      } as TestCase["results"][number],
+    ],
+    title: richTestName,
+  });
+
+  reporter.onBegin?.({} as never, fakeSuite([marker, scenarioTest]));
+  expect(reporter.onEnd?.({ status: "passed" } as never)).toBeUndefined();
+  expect(reports).toHaveLength(1);
+
+  const missingPhaseReporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    performanceConfig,
+    performanceRequirements: [booleanRequirement],
+    performanceSchema,
+    reportError: () => undefined,
+  });
+  const missingPhaseTest = fakeTestCase({
+    results: [
+      {
+        ...booleanResult,
+        attachments: [
+          ...booleanResult.attachments,
+          richPipelineAttachment("cold"),
+          richPipelineAttachment("warm"),
+          richMeasurementAttachment("cold"),
+          richMeasurementAttachment("warm"),
+          richMeasurementAttachment("sustained"),
+        ],
+      } as TestCase["results"][number],
+    ],
+    title: richTestName,
+  });
+  missingPhaseReporter.onBegin?.(
+    {} as never,
+    fakeSuite([marker, missingPhaseTest]),
+  );
+  expect(missingPhaseReporter.onEnd?.({ status: "passed" } as never)).toEqual({
+    status: "failed",
+  });
+});
+
+test("protected checkpoint reporter writes only a complete independently validated report", () => {
+  const booleanRequirement: ToolcraftBrowserRuntimeRequirement = {
+    evidenceType: "performance-budget",
+    requirementId: "reporter-pipeline",
+    testName: richTestName,
+  };
+  const booleanResult = passedResultWithEvidence(booleanRequirement);
+  const reportPath = path.join(
+    mkdtempSync(path.join(tmpdir(), "toolcraft-reporter-")),
+    "report.json",
+  );
+  const marker = fakeTestCase({
+    file: "/product/e2e/app-browser-runtime-evidence.spec.ts",
+    title: TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+  });
+  const environmentTest = fakeTestCase({
+    results: [
+      {
+        ...booleanResult,
+        attachments: [environmentAttachment()],
+      } as TestCase["results"][number],
+    ],
+    title: "browser perf: toolcraft environment",
+  });
+  const scenarioTest = fakeTestCase({
+    results: [
+      {
+        ...booleanResult,
+        attachments: [
+          ...booleanResult.attachments,
+          ...(["cold", "warm", "sustained"] as const).flatMap((phase) => [
+            richPipelineAttachment(phase),
+            richMeasurementAttachment(phase),
+          ]),
+        ],
+      } as TestCase["results"][number],
+    ],
+    title: richTestName,
+  });
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    checkpointReport: {
+      nonce: "checkpoint-nonce",
+      path: reportPath,
+      requestAuthorityHash: "b".repeat(64),
+      sourceHash: "a".repeat(64),
+    },
+    performanceConfig,
+    performanceRequirements: [booleanRequirement],
+    performanceSchema,
+  });
+
+  reporter.onBegin?.(
+    {} as never,
+    fakeSuite([marker, environmentTest, scenarioTest]),
+  );
+  expect(reporter.onEnd?.({ status: "passed" } as never)).toBeUndefined();
+  const report = JSON.parse(readFileSync(reportPath, "utf8"));
+  expect(report.nonce).toBe("checkpoint-nonce");
+  expect(report.measurements).toHaveLength(3);
+  expect(report.pipelineSummaries).toHaveLength(3);
+});
+
+test("targeted reporter writes only exact impacted canonical path evidence", () => {
+  const targetedTestName = getToolcraftPerformancePathTestName(
+    reporterPerformancePath!,
+  );
+  const booleanRequirement: ToolcraftBrowserRuntimeRequirement = {
+    evidenceType: "performance-budget",
+    requirementId: "reporter-pipeline",
+    testName: targetedTestName,
+  };
+  const booleanResult = passedResultWithEvidence(booleanRequirement);
+  const reportPath = path.join(
+    mkdtempSync(path.join(tmpdir(), "toolcraft-targeted-reporter-")),
+    "report.json",
+  );
+  const scenarioTest = fakeTestCase({
+    results: [
+      {
+        ...booleanResult,
+        attachments: [
+          ...booleanResult.attachments,
+          ...(["cold", "warm", "sustained"] as const).flatMap((phase) => [
+            richPipelineAttachment(phase),
+            richMeasurementAttachment(phase),
+          ]),
+        ],
+      } as TestCase["results"][number],
+    ],
+    title: targetedTestName,
+  });
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    performanceConfig,
+    performanceRequirements: [booleanRequirement],
+    performanceSchema,
+    targetedPerformanceReport: {
+      fixtureResolutionMode: "strict-development",
+      fixtureSelector: "development",
+      nonce: "targeted-nonce",
+      passIds: ["composite"],
+      path: reportPath,
+      pathIds: [reporterPerformancePath!.id],
+      requestAuthorityHash: "b".repeat(64),
+      sourceHash: "a".repeat(64),
+    },
+  });
+
+  reporter.onBegin?.({} as never, fakeSuite([scenarioTest]));
+  expect(reporter.onEnd?.({ status: "passed" } as never)).toBeUndefined();
+  expect(JSON.parse(readFileSync(reportPath, "utf8"))).toMatchObject({
+    fixtureResolutionMode: "strict-development",
+    fixtureSelector: "development",
+    nonce: "targeted-nonce",
+    performancePassIds: ["composite"],
+    performancePathIds: [reporterPerformancePath!.id],
+    requestAuthorityHash: "b".repeat(64),
+    testNames: [targetedTestName],
+    version: 3,
+  });
+
+  const unrelatedReportPath = `${reportPath}.unrelated`;
+  const unrelatedReporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    performanceConfig,
+    performanceRequirements: [booleanRequirement],
+    performanceSchema,
+    reportError: () => undefined,
+    targetedPerformanceReport: {
+      fixtureResolutionMode: "strict-development",
+      fixtureSelector: "development",
+      nonce: "targeted-nonce",
+      passIds: ["composite"],
+      path: unrelatedReportPath,
+      pathIds: [reporterPerformancePath!.id],
+      requestAuthorityHash: "b".repeat(64),
+      sourceHash: "a".repeat(64),
+    },
+  });
+  unrelatedReporter.onBegin?.(
+    {} as never,
+    fakeSuite([fakeTestCase({ title: "browser perf: unrelated workload" })]),
+  );
+  expect(unrelatedReporter.onEnd?.({ status: "passed" } as never)).toEqual({
+    status: "failed",
+  });
+  expect(existsSync(unrelatedReportPath)).toBe(false);
+});

package/templates/starter/e2e/browser-performance-targeted-selection.spec.ts

@@ -0,0 +1,145 @@
+import { expect, test } from "@playwright/test";
+import type { ToolcraftPerformancePath } from "@repo/toolcraft-runtime";
+
+import { fakeTestCase } from "./browser-runtime-evidence-reporter-test-fixtures";
+import {
+  evaluateToolcraftTargetedPerformanceSelection,
+  type ToolcraftTargetedPerformanceReportTarget,
+} from "./browser-performance-reporter-evaluation";
+import { getToolcraftPerformancePathTestName } from "./performance-path-helpers";
+
+function pathFixture(
+  id: string,
+  invalidates: readonly string[],
+): ToolcraftPerformancePath {
+  return {
+    id,
+    interaction: "control-change",
+    invalidates,
+    profile: "interactive-discrete",
+    runsOn: [],
+    targets: [`target.${id}`],
+    workloadDimensions: [],
+  };
+}
+
+const compositePath = pathFixture("path-composite", ["composite"]);
+const passlessPath = pathFixture("path-passless", []);
+const canonicalPaths = [compositePath, passlessPath];
+
+function target(
+  pathIds: readonly string[],
+  passIds: readonly string[],
+): ToolcraftTargetedPerformanceReportTarget {
+  return {
+    fixtureResolutionMode: "strict-development",
+    fixtureSelector: "development",
+    nonce: "targeted-nonce",
+    passIds,
+    path: "/unused/report.json",
+    pathIds,
+    requestAuthorityHash: "b".repeat(64),
+    sourceHash: "a".repeat(64),
+  };
+}
+
+function evaluate(
+  paths: readonly ToolcraftPerformancePath[],
+  reportTarget: ToolcraftTargetedPerformanceReportTarget,
+) {
+  return evaluateToolcraftTargetedPerformanceSelection({
+    canonicalPaths,
+    expectedPaths: paths,
+    selectedTests: paths.map((performancePath) =>
+      fakeTestCase({ title: getToolcraftPerformancePathTestName(performancePath) }),
+    ),
+    target: reportTarget,
+  });
+}
+
+test("targeted selection accepts an exact pass-bearing path tuple", () => {
+  expect(
+    evaluate([compositePath], target([compositePath.id], ["composite"])).errors,
+  ).toEqual([]);
+});
+
+test("targeted selection keeps the signed path when one pass has several paths", () => {
+  const sibling = pathFixture("path-composite-sibling", ["composite"]);
+  const evaluation = evaluateToolcraftTargetedPerformanceSelection({
+    canonicalPaths: [compositePath, sibling],
+    expectedPaths: [compositePath],
+    selectedTests: [
+      fakeTestCase({ title: getToolcraftPerformancePathTestName(compositePath) }),
+    ],
+    target: target([compositePath.id], ["composite"]),
+  });
+
+  expect(evaluation.errors).toEqual([]);
+  expect(evaluation.paths.map(({ id }) => id)).toEqual([compositePath.id]);
+});
+
+test("targeted selection accepts an exact passless path tuple", () => {
+  expect(evaluate([passlessPath], target([passlessPath.id], [])).errors).toEqual(
+    [],
+  );
+});
+
+test("targeted selection rejects duplicate and unknown canonical path ids", () => {
+  expect(
+    evaluate(
+      [compositePath],
+      target([compositePath.id, compositePath.id], ["composite"]),
+    ).errors.join("\n"),
+  ).toMatch(/unique canonical path ids/iu);
+  expect(
+    evaluate([compositePath], target(["path-unknown"], ["composite"])).errors.join(
+      "\n",
+    ),
+  ).toMatch(/unknown canonical paths/iu);
+});
+
+test("targeted selection rejects duplicate and unknown renderer pass ids", () => {
+  expect(
+    evaluate(
+      [compositePath],
+      target([compositePath.id], ["composite", "composite"]),
+    ).errors.join("\n"),
+  ).toMatch(/unique renderer pass ids/iu);
+  expect(
+    evaluate([compositePath], target([compositePath.id], ["unknown"])).errors.join(
+      "\n",
+    ),
+  ).toMatch(/passes without canonical paths/iu);
+});
+
+test("targeted selection requires the exact union of selected path invalidations", () => {
+  expect(
+    evaluate([compositePath], target([compositePath.id], [])).errors.join("\n"),
+  ).toMatch(/exactly match selected path invalidations/iu);
+  expect(
+    evaluate(
+      [passlessPath],
+      target([passlessPath.id], ["composite"]),
+    ).errors.join("\n"),
+  ).toMatch(/exactly match selected path invalidations/iu);
+});
+
+test("targeted selection rejects requestless or full-certification targets", () => {
+  const valid = target([compositePath.id], ["composite"]);
+  for (const invalid of [
+    { ...valid, requestAuthorityHash: null },
+    { ...valid, requestAuthorityHash: "not-a-hash" },
+    {
+      ...valid,
+      fixtureResolutionMode: "full-certification",
+      fixtureSelector: "maximum",
+    },
+  ]) {
+    expect(
+      evaluate(
+        [compositePath],
+        invalid as unknown as ToolcraftTargetedPerformanceReportTarget,
+      ).errors.join("\n"),
+    ).toMatch(/strict development fixtures.*canonical request authority/iu);
+  }
+});

package/templates/starter/e2e/browser-proof-session.ts

@@ -36,6 +36,10 @@ export type ToolcraftBrowserProofSession = {
     target: string,
     run: (control: Locator, page: Page) => Promise<Result>,
   ): ToolcraftBrowserAction<"interaction", Result>;
+  targetAction<Result = void>(
+    target: string,
+    run: (page: Page) => Promise<Result>,
+  ): ToolcraftBrowserAction<"interaction", Result>;
   observe<T>(
     read: (root: HTMLElement) => T | Promise<T>,
   ): ToolcraftBrowserObservation<T>;
@@ -212,6 +216,16 @@ function requireSession(value: unknown): SessionRecord {
   return record;
 }
 
+function normalizeActionTarget(target: string): string {
+  const normalizedTarget = target.trim();
+  if (!normalizedTarget) {
+    throw new Error(
+      "A target-scoped Toolcraft browser action requires a non-empty semantic target.",
+    );
+  }
+  return normalizedTarget;
+}
+
 export async function createToolcraftBrowserProofSession(
   page: Page,
 ): Promise<ToolcraftBrowserProofSession> {
@@ -260,7 +274,7 @@ export async function createToolcraftBrowserProofSession(
       target: string,
       run: (control: Locator, currentPage: Page) => Promise<Result>,
     ): ToolcraftBrowserAction<"interaction", Result> {
-      const normalizedTarget = target.trim();
+      const normalizedTarget = normalizeActionTarget(target);
       const action = Object.freeze({}) as ToolcraftBrowserAction<
         "interaction",
         Result
@@ -279,6 +293,22 @@ export async function createToolcraftBrowserProofSession(
       });
       return action;
     },
+    targetAction<Result = void>(
+      target: string,
+      run: (currentPage: Page) => Promise<Result>,
+    ): ToolcraftBrowserAction<"interaction", Result> {
+      const action = Object.freeze({}) as ToolcraftBrowserAction<
+        "interaction",
+        Result
+      >;
+      actionRecords.set(action, {
+        kind: "interaction",
+        run,
+        session: record,
+        target: normalizeActionTarget(target),
+      });
+      return action;
+    },
     observe<T>(
       read: (root: HTMLElement) => T | Promise<T>,
     ): ToolcraftBrowserObservation<T> {