@pixel-point/toolcraft 0.0.15 → 0.0.16

package/templates/runtime/testing/performance-path-codec.test.ts

@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  createToolcraftPerformancePathId,
+  decodeToolcraftPerformancePathId,
+} from "./performance-path-codec.mjs";
+
+const canonicalPath = {
+  interaction: "control-change",
+  invalidates: ["composite", "source"],
+  profile: "interactive-discrete",
+  runsOn: ["main", "worker"],
+  workloadDimensions: ["pixel-count"],
+} as const;
+
+function encodeSignature(signature: readonly unknown[]): string {
+  return `performance-path:${encodeURIComponent(JSON.stringify(signature))}`;
+}
+
+describe("Toolcraft performance path codec", () => {
+  it("round trips one canonical runtime signature", () => {
+    const pathId = createToolcraftPerformancePathId(canonicalPath);
+
+    expect(decodeToolcraftPerformancePathId(pathId)).toEqual(canonicalPath);
+  });
+
+  it.each([
+    [
+      "unknown profile",
+      ["unknown", "control-change", ["composite"], ["main"], []],
+    ],
+    [
+      "unknown interaction",
+      ["interactive-discrete", "unknown", ["composite"], ["main"], []],
+    ],
+    [
+      "mismatched profile",
+      ["interactive-continuous", "control-change", ["composite"], ["main"], []],
+    ],
+    [
+      "duplicate pass",
+      [
+        "interactive-discrete",
+        "control-change",
+        ["composite", "composite"],
+        ["main"],
+        [],
+      ],
+    ],
+    [
+      "unsorted pass",
+      [
+        "interactive-discrete",
+        "control-change",
+        ["source", "composite"],
+        ["main"],
+        [],
+      ],
+    ],
+    [
+      "unknown run location",
+      [
+        "interactive-discrete",
+        "control-change",
+        ["composite"],
+        ["satellite"],
+        [],
+      ],
+    ],
+    [
+      "duplicate dimension",
+      [
+        "interactive-discrete",
+        "control-change",
+        ["composite"],
+        ["main"],
+        ["pixels", "pixels"],
+      ],
+    ],
+  ])("rejects %s", (_case, signature) => {
+    expect(decodeToolcraftPerformancePathId(encodeSignature(signature))).toBeUndefined();
+  });
+
+  it("rejects non-array and non-canonically encoded JSON", () => {
+    expect(
+      decodeToolcraftPerformancePathId(
+        `performance-path:${encodeURIComponent(JSON.stringify({ ...canonicalPath }))}`,
+      ),
+    ).toBeUndefined();
+    expect(
+      decodeToolcraftPerformancePathId(
+        `${createToolcraftPerformancePathId(canonicalPath)}%20`,
+      ),
+    ).toBeUndefined();
+  });
+});

package/templates/runtime/testing/performance-path-coverage-policy.ts

@@ -0,0 +1,111 @@
+import type { ToolcraftEnvelopeValidationContext } from "./performance-envelope-validation-context";
+import type { ToolcraftPerformancePath } from "./performance-path-model";
+import {
+  getToolcraftPerformanceScenarioCommonErrors,
+  hasToolcraftBrowserTestName,
+} from "./performance-scenario-common-validation";
+import type { ToolcraftPerformanceScenario } from "./performance-types";
+
+function compareCodeUnits(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function uniqueSorted(values: readonly string[]): string[] {
+  return [...new Set(values)].sort(compareCodeUnits);
+}
+
+function arraysEqual(left: readonly string[], right: readonly string[]): boolean {
+  return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
+function getScenarioPathErrors(
+  scenario: ToolcraftPerformanceScenario,
+  pathsById: ReadonlyMap<string, ToolcraftPerformancePath>,
+): string[] {
+  const path = pathsById.get(scenario.pathId);
+  if (!path) {
+    return [
+      `Performance scenario ${scenario.id} references unknown pathId "${scenario.pathId}".`,
+    ];
+  }
+
+  const errors: string[] = [];
+  if (scenario.interaction !== path.interaction) {
+    errors.push(
+      `Performance scenario ${scenario.id} interaction ${scenario.interaction} does not exercise path ${path.id} interaction ${path.interaction}.`,
+    );
+  }
+  const declaredCoveredTargets = scenario.coversTargets;
+  const coveredTargets = uniqueSorted(declaredCoveredTargets);
+  if (coveredTargets.length !== declaredCoveredTargets.length) {
+    errors.push(
+      `Performance scenario ${scenario.id} coversTargets must not contain duplicate targets.`,
+    );
+  }
+  if (!arraysEqual(coveredTargets, path.targets)) {
+    errors.push(
+      `Performance scenario ${scenario.id} coversTargets must equal path ${path.id} targets ${JSON.stringify(path.targets)}.`,
+    );
+  }
+  if (scenario.target && !path.targets.includes(scenario.target)) {
+    errors.push(
+      `Performance scenario ${scenario.id} target "${scenario.target}" does not belong to path ${path.id}.`,
+    );
+  }
+  return errors;
+}
+
+export function getToolcraftPerformancePathCoverageErrors(
+  context: ToolcraftEnvelopeValidationContext,
+): string[] {
+  const { config, paths, schema } = context;
+  const pathsById = new Map(paths.map((path) => [path.id, path] as const));
+  const scenariosByPathId = new Map<string, ToolcraftPerformanceScenario[]>();
+  const browserNameOwners = new Map<string, { pathId: string; scenarioId: string }>();
+  const errors: string[] = [];
+
+  for (const scenario of config.scenarios) {
+    errors.push(...getToolcraftPerformanceScenarioCommonErrors(schema, scenario));
+    const scenarios = scenariosByPathId.get(scenario.pathId) ?? [];
+    scenarios.push(scenario);
+    scenariosByPathId.set(scenario.pathId, scenarios);
+    errors.push(...getScenarioPathErrors(scenario, pathsById));
+
+    if (hasToolcraftBrowserTestName(scenario)) {
+      const previous = browserNameOwners.get(scenario.browserTestName);
+      if (previous && previous.pathId !== scenario.pathId) {
+        errors.push(
+          `${scenario.id} browserTestName "${scenario.browserTestName}" is already used by ${previous.scenarioId} for a different performance path.`,
+        );
+      } else if (!previous) {
+        browserNameOwners.set(scenario.browserTestName, {
+          pathId: scenario.pathId,
+          scenarioId: scenario.id,
+        });
+      }
+    }
+  }
+
+  for (const path of paths) {
+    const scenarios = scenariosByPathId.get(path.id) ?? [];
+    if (scenarios.length > 1) {
+      errors.push(
+        `Performance path ${path.id} must have exactly one path scenario; found ${scenarios.length}. Equivalent controls share the derived path instead of declaring per-control scenarios.`,
+      );
+    }
+    if (
+      !scenarios.some(
+        (scenario) =>
+          scenario.automated &&
+          scenario.automatedTestName.trim() &&
+          hasToolcraftBrowserTestName(scenario),
+      )
+    ) {
+      errors.push(
+        `Performance path ${path.id} must have an automated browser performance scenario.`,
+      );
+    }
+  }
+
+  return errors;
+}

package/templates/runtime/testing/performance-path-coverage.test.ts

@@ -0,0 +1,219 @@
+import { describe, expect, it } from "vitest";
+
+import { validateToolcraftPerformanceCoverage } from "./performance-coverage-validator";
+import { deriveToolcraftPerformancePaths } from "./performance-paths";
+import {
+  createConfig,
+  createInvalidation,
+  createScenario,
+  groupedSchema,
+} from "./performance-paths.test-fixtures";
+
+describe("Toolcraft performance path coverage", () => {
+  it("requires one real browser scenario to cover each derived path", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const validScenario = createScenario(path);
+
+    expect(
+      validateToolcraftPerformanceCoverage(groupedSchema, createConfig()),
+    ).toContain(
+      `Performance path ${path.id} must have an automated browser performance scenario.`,
+    );
+    expect(
+      validateToolcraftPerformanceCoverage(
+        groupedSchema,
+        createConfig({ scenarios: [validScenario] }),
+      ),
+    ).toEqual([]);
+  });
+
+  it("validates every path scenario instead of accepting one valid sibling", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        scenarios: [
+          createScenario(path),
+          createScenario(path, {
+            automatedTestName: "",
+            browserTestName: "browser perf: malformed sibling",
+            fixture: "",
+            id: "malformed-sibling",
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain("malformed-sibling must name a representative fixture.");
+    expect(errors).toContain(
+      "malformed-sibling must point to an automated performance test.",
+    );
+  });
+
+  it("rejects duplicate coversTargets even when the unique set matches", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        scenarios: [
+          createScenario(path, {
+            coversTargets: [...path.targets, path.targets[0]!],
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain(
+      `Performance scenario shared-uniform-drag coversTargets must not contain duplicate targets.`,
+    );
+  });
+
+  it("rejects per-control scenarios that duplicate one derived path", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const sharedName = "browser perf: shared uniform drag stays responsive";
+
+    expect(
+      validateToolcraftPerformanceCoverage(
+        groupedSchema,
+        createConfig({
+          scenarios: [
+            createScenario(path, { browserTestName: sharedName }),
+            createScenario(path, {
+              browserTestName: sharedName,
+              id: "shared-uniform-drag-secondary-assertion",
+            }),
+          ],
+        }),
+      ),
+    ).toContain(
+      `Performance path ${path.id} must have exactly one path scenario; found 2. Equivalent controls share the derived path instead of declaring per-control scenarios.`,
+    );
+
+    const distinctConfig = createConfig({
+      interactions: [
+        createInvalidation({ targets: ["effect.opacity"] }),
+        createInvalidation({
+          interaction: "control-change",
+          targets: ["effect.strength"],
+        }),
+      ],
+    });
+    const [changePath, dragPath] = deriveToolcraftPerformancePaths(
+      groupedSchema,
+      distinctConfig,
+    );
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        interactions: distinctConfig.rendererPipeline!.interactionInvalidation,
+        scenarios: [
+          createScenario(changePath!, {
+            browserTestName: sharedName,
+            id: "change-path",
+            interaction: "control-change",
+          }),
+          createScenario(dragPath!, {
+            browserTestName: sharedName,
+            id: "drag-path",
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain(
+      `drag-path browserTestName "${sharedName}" is already used by change-path for a different performance path.`,
+    );
+  });
+
+  it("does not let disabled or blank browser checks own a shared name", () => {
+    const distinctConfig = createConfig({
+      interactions: [
+        createInvalidation({ targets: ["effect.opacity"] }),
+        createInvalidation({
+          interaction: "control-change",
+          targets: ["effect.strength"],
+        }),
+      ],
+    });
+    const [changePath, dragPath] = deriveToolcraftPerformancePaths(
+      groupedSchema,
+      distinctConfig,
+    );
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        interactions: distinctConfig.rendererPipeline!.interactionInvalidation,
+        scenarios: [
+          createScenario(changePath!, {
+            browserTestName: "browser perf: valid change path",
+            id: "valid-change-path",
+            interaction: "control-change",
+          }),
+          createScenario(dragPath!, {
+            browserTestName: "browser perf: valid drag path",
+            id: "valid-drag-path",
+          }),
+          createScenario(changePath!, {
+            browser: false,
+            browserTestName: "",
+            id: "disabled-change-path",
+            interaction: "control-change",
+          }),
+          createScenario(dragPath!, {
+            browser: false,
+            browserTestName: "",
+            id: "disabled-drag-path",
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain(
+      "disabled-change-path must point to a browser performance test.",
+    );
+    expect(errors).toContain(
+      "disabled-drag-path must point to a browser performance test.",
+    );
+    expect(errors.some((error) => error.includes("already used"))).toBe(false);
+  });
+
+  it("rejects path claims that omit equivalent targets", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        scenarios: [
+          createScenario(path, {
+            coversTargets: ["effect.opacity"],
+            id: "opacity-drag",
+            target: "effect.opacity",
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain(
+      `Performance scenario opacity-drag coversTargets must equal path ${path.id} targets ["effect.opacity","effect.strength"].`,
+    );
+  });
+
+  it("rejects scenarios that do not own a canonical path", () => {
+    const path = deriveToolcraftPerformancePaths(groupedSchema, createConfig())[0]!;
+    const errors = validateToolcraftPerformanceCoverage(
+      groupedSchema,
+      createConfig({
+        scenarios: [
+          createScenario(path),
+          createScenario(path, {
+            id: "unknown-path",
+            pathId: "performance-path:unknown",
+          }),
+        ],
+      }),
+    );
+
+    expect(errors).toContain(
+      'Performance scenario unknown-path references unknown pathId "performance-path:unknown".',
+    );
+  });
+});

package/templates/runtime/testing/performance-path-model.ts

@@ -0,0 +1,15 @@
+import type {
+  ToolcraftPipelineInteraction,
+  ToolcraftRenderPassRunLocation,
+} from "./performance-types";
+import type { ToolcraftPerformanceProfileName } from "./performance-workload-types";
+
+export type ToolcraftPerformancePath = {
+  id: string;
+  interaction: ToolcraftPipelineInteraction;
+  invalidates: readonly string[];
+  profile: ToolcraftPerformanceProfileName;
+  runsOn: readonly ToolcraftRenderPassRunLocation[];
+  targets: readonly string[];
+  workloadDimensions: readonly string[];
+};

package/templates/runtime/testing/performance-paths.test-fixtures.ts

@@ -0,0 +1,165 @@
+import { defineToolcraft } from "../schema/define-toolcraft";
+import {
+  registerToolcraftRendererPipeline,
+  type ToolcraftRendererPipelinePassContract,
+} from "../rendering/renderer-pipeline-registration";
+import type { ToolcraftPerformancePath } from "./performance-path-model";
+import type {
+  ToolcraftEnvelopePerformanceConfig,
+  ToolcraftInteractionInvalidation,
+  ToolcraftPerformanceScenario,
+  ToolcraftRenderPass,
+} from "./performance-types";
+
+export const groupedSchema = defineToolcraft({
+  canvas: { enabled: true, sizing: { mode: "editable-output" } },
+  panels: {
+    controls: {
+      sections: [
+        {
+          controls: {
+            opacity: {
+              defaultValue: 0.5,
+              label: "Opacity",
+              max: 1,
+              min: 0,
+              performanceRole: "responsiveness",
+              target: "effect.opacity",
+              type: "slider",
+            },
+            strength: {
+              defaultValue: 0.5,
+              label: "Strength",
+              max: 1,
+              min: 0,
+              performanceRole: "responsiveness",
+              target: "effect.strength",
+              type: "slider",
+            },
+          },
+          title: "Effect",
+        },
+      ],
+      title: "Controls",
+    },
+  },
+});
+
+export const workloadSchema = defineToolcraft({
+  canvas: { enabled: true, sizing: { mode: "editable-output" } },
+  panels: {
+    controls: {
+      sections: [
+        {
+          controls: {
+            density: {
+              defaultValue: 4,
+              label: "Density",
+              max: 12,
+              min: 1,
+              performanceRole: "workload",
+              target: "render.density",
+              type: "slider",
+            },
+          },
+          title: "Render",
+        },
+      ],
+      title: "Controls",
+    },
+  },
+});
+
+export function createPass(
+  overrides: Partial<ToolcraftRenderPass>,
+): ToolcraftRenderPass {
+  return {
+    cacheKey: ["effect.source"],
+    cost: {
+      dimensions: [],
+      frequency: "interaction",
+      relationship: "constant",
+    },
+    id: "uniform-composite",
+    inputs: ["effect.source"],
+    invalidatedBy: ["effect.opacity", "effect.strength"],
+    kind: "composite",
+    lifecycle: { cache: "memoized", resourceScope: "renderer" },
+    output: "preview",
+    quality: "full",
+    runsOn: "gpu",
+    ...overrides,
+  };
+}
+
+export function createInvalidation(
+  overrides: Partial<ToolcraftInteractionInvalidation>,
+): ToolcraftInteractionInvalidation {
+  return {
+    interaction: "control-drag",
+    invalidates: ["uniform-composite"],
+    targets: ["effect.opacity"],
+    ...overrides,
+  };
+}
+
+export function createConfig(options: {
+  dimensions?: ToolcraftEnvelopePerformanceConfig["workloadEnvelope"]["dimensions"];
+  interactions?: readonly ToolcraftInteractionInvalidation[];
+  passes?: readonly ToolcraftRenderPass[];
+  scenarios?: ToolcraftEnvelopePerformanceConfig["scenarios"];
+} = {}): ToolcraftEnvelopePerformanceConfig {
+  const rendererPipeline = registerToolcraftRendererPipeline<
+    Record<string, ToolcraftRendererPipelinePassContract<unknown>>
+  >()({
+    interactionInvalidation: options.interactions ?? [
+      createInvalidation({ targets: ["effect.opacity"] }),
+      createInvalidation({ targets: ["effect.strength"] }),
+    ],
+    passes: options.passes ?? [createPass({})],
+    runtimeId: "grouped-effect-renderer-v1",
+  });
+
+  return {
+    rendererPipeline,
+    rendererStrategy: "webgl",
+    rendererTechnique: {
+      exportRenderer: "webgl",
+      fidelityRisks: ["The fixture must keep preview and export output equivalent."],
+      performanceRisks: ["Interactive composite updates must remain GPU-backed."],
+      previewRenderer: "webgl",
+      productRepresentation: "pixel",
+      rendererStrategy: "webgl",
+      sourceRepresentation: "procedural-data",
+      whyNotAlternativeStrategies: [
+        "The fixture exercises an explicitly GPU-owned composite path.",
+      ],
+    },
+    scenarios: options.scenarios ?? [],
+    usesCustomRenderer: true,
+    workloadEnvelope: { dimensions: options.dimensions ?? [] },
+  };
+}
+
+export function createScenario(
+  path: ToolcraftPerformancePath,
+  overrides: Partial<
+    Exclude<ToolcraftPerformanceScenario, { interaction: "export" }>
+  > = {},
+): Exclude<ToolcraftPerformanceScenario, { interaction: "export" }> {
+  return {
+    automated: true,
+    automatedTestName: "perf: shared uniform drag stays responsive",
+    browser: true,
+    browserTestName: "browser perf: shared uniform drag stays responsive",
+    controlLabel: "Opacity",
+    coversTargets: path.targets,
+    expectedObservable: "Dragging a uniform control keeps the preview responsive.",
+    fixture: "combined uniform path fixture",
+    id: "shared-uniform-drag",
+    interaction: "control-drag",
+    pathId: path.id,
+    target: path.targets[0],
+    ...overrides,
+  };
+}