@pixel-point/toolcraft 0.0.14 → 0.0.15

package/templates/runtime/testing/performance-budget-validation.ts

@@ -0,0 +1,84 @@
+import type {
+  ToolcraftPerformanceBudget,
+  ToolcraftPerformanceScenario,
+} from "./performance-types";
+
+const maxPerformanceBudgetCaps: Required<ToolcraftPerformanceBudget> = {
+  maxExportMs: 8000,
+  maxFrameGapMs: 120,
+  maxInteractionMs: 2000,
+  maxLongTaskMs: 250,
+  maxPreviewMs: 2000,
+  maxRenderMs: 2000,
+};
+
+export function hasAnyBudget(budget: ToolcraftPerformanceBudget): boolean {
+  return Object.values(budget).some((value) => typeof value === "number" && value > 0);
+}
+
+export function hasPositiveBudgetField(
+  budget: ToolcraftPerformanceBudget,
+  field: keyof ToolcraftPerformanceBudget,
+): boolean {
+  const value = budget[field];
+  return typeof value === "number" && value > 0;
+}
+
+export function getMissingInteractionBudgetFields(
+  scenario: ToolcraftPerformanceScenario,
+): string[] {
+  switch (scenario.interaction) {
+    case "animation-frame":
+      return hasPositiveBudgetField(scenario.budget, "maxFrameGapMs")
+        ? hasPositiveBudgetField(scenario.budget, "maxLongTaskMs")
+          ? []
+          : ["maxLongTaskMs"]
+        : hasPositiveBudgetField(scenario.budget, "maxLongTaskMs")
+          ? ["maxFrameGapMs"]
+          : ["maxFrameGapMs", "maxLongTaskMs"];
+    case "animation-viewport-drag":
+    case "mask-drag":
+    case "viewport-zoom-stress":
+      return (["maxInteractionMs", "maxFrameGapMs", "maxLongTaskMs"] as const).filter(
+        (field) => !hasPositiveBudgetField(scenario.budget, field),
+      );
+    case "viewport-stability":
+      return hasPositiveBudgetField(scenario.budget, "maxFrameGapMs")
+        ? []
+        : ["maxFrameGapMs"];
+    case "control-change":
+    case "control-drag":
+    case "media-import":
+      return (["maxInteractionMs", "maxFrameGapMs"] as const).filter(
+        (field) => !hasPositiveBudgetField(scenario.budget, field),
+      );
+    case "export-copy":
+      return hasPositiveBudgetField(scenario.budget, "maxExportMs") ? [] : ["maxExportMs"];
+    case "preview-render":
+      return hasPositiveBudgetField(scenario.budget, "maxPreviewMs") ||
+        hasPositiveBudgetField(scenario.budget, "maxRenderMs")
+        ? []
+        : ["maxPreviewMs or maxRenderMs"];
+    case "timeline-playback":
+      return (["maxFrameGapMs", "maxLongTaskMs"] as const).filter(
+        (field) => !hasPositiveBudgetField(scenario.budget, field),
+      );
+    case "timeline-scrub":
+      return (["maxInteractionMs", "maxFrameGapMs", "maxLongTaskMs"] as const).filter(
+        (field) => !hasPositiveBudgetField(scenario.budget, field),
+      );
+  }
+}
+
+export function getBudgetCapErrors(scenario: ToolcraftPerformanceScenario): string[] {
+  return Object.entries(scenario.budget).flatMap(([field, value]) => {
+    const budgetField = field as keyof ToolcraftPerformanceBudget;
+    const cap = maxPerformanceBudgetCaps[budgetField];
+
+    if (typeof value !== "number" || value <= cap) {
+      return [];
+    }
+
+    return [`${scenario.id} ${budgetField} budget must be <= ${cap}ms, received ${value}ms.`];
+  });
+}

package/templates/runtime/testing/performance-control-classification.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from "vitest";
+
+import { defineToolcraft } from "../schema/define-toolcraft";
+import {
+  collectToolcraftPerformanceSensitiveControls,
+  collectToolcraftUnclassifiedPerformanceControls,
+  defineToolcraftPerformance,
+  defaultToolcraftBrowserCheckPolicy,
+  type ToolcraftBrowserCheckPolicy,
+  validateToolcraftPerformanceCoverage,
+} from "./performance";
+import {
+  ordinarySliderSchema,
+  testSchema,
+} from "./performance-test-fixtures";
+
+describe("Toolcraft performance control classification contract", () => {
+  it("collects performance-sensitive controls from schema", () => {
+    expect(collectToolcraftPerformanceSensitiveControls(testSchema)).toEqual([
+      expect.objectContaining({
+        controlId: "density",
+        target: "render.density",
+      }),
+    ]);
+  });
+
+  it("does not classify every slider as workload-sensitive", () => {
+    expect(collectToolcraftPerformanceSensitiveControls(ordinarySliderSchema)).toEqual([]);
+  });
+
+  it("collects explicit workload controls even when labels do not match fallback keywords", () => {
+    const explicitWorkloadSchema = defineToolcraft({
+      canvas: {
+        enabled: true,
+        sizing: { mode: "intrinsic-media" },
+      },
+      panels: {
+        controls: {
+          sections: [
+            {
+              controls: {
+                complexity: {
+                  defaultValue: 4,
+                  label: "Complexity",
+                  max: 12,
+                  min: 1,
+                  performanceReason: "Complexity changes the number of render passes.",
+                  performanceRole: "workload",
+                  target: "render.complexity",
+                  type: "slider",
+                },
+              },
+            },
+          ],
+          title: "Runtime Controls",
+        },
+      },
+    });
+
+    expect(collectToolcraftPerformanceSensitiveControls(explicitWorkloadSchema)).toEqual([
+      expect.objectContaining({
+        controlId: "complexity",
+        target: "render.complexity",
+      }),
+    ]);
+  });
+
+  it("collects visible controls without explicit performance classification", () => {
+    const unclassifiedSchema = defineToolcraft({
+      canvas: {
+        enabled: true,
+        renderScale: true,
+        sizing: { mode: "intrinsic-media" },
+      },
+      panels: {
+        controls: {
+          sections: [
+            {
+              controls: {
+                mode: {
+                  defaultValue: "soft",
+                  label: "Mode",
+                  options: [
+                    { label: "Soft", value: "soft" },
+                    { label: "Sharp", value: "sharp" },
+                  ],
+                  target: "render.mode",
+                  type: "select",
+                },
+              },
+            },
+          ],
+          title: "Runtime Controls",
+        },
+        timeline: { mode: "playback" },
+      },
+    });
+
+    expect(collectToolcraftUnclassifiedPerformanceControls(unclassifiedSchema)).toEqual([
+      expect.objectContaining({
+        controlId: "mode",
+        target: "render.mode",
+      }),
+    ]);
+  });
+});

package/templates/runtime/testing/performance-control-classification.ts

@@ -0,0 +1,101 @@
+import type {
+  ResolvedToolcraftAppSchema,
+  ToolcraftControlSchema,
+} from "../schema/types";
+import { isToolcraftRuntimeOwnedTarget } from "../schema/runtime-targets";
+import type {
+  ToolcraftPerformanceSensitiveControl,
+  ToolcraftUnclassifiedPerformanceControl,
+} from "./performance-types";
+
+const workloadControlPattern =
+  /char\s*size|cell|density|glyph|grid|iteration|matrix|particle|quality|radius|resolution|sample|scale|size/i;
+
+const heavyTextInputPattern = /code|css|instruction|json|prompt|script|shader|template/i;
+
+function getControlSemanticText(control: ToolcraftControlSchema): string {
+  return [
+    control.target,
+    typeof control.label === "string" ? control.label : "",
+    control.unit ?? "",
+    control.valueLabel ?? "",
+    control.xLabel ?? "",
+    control.yLabel ?? "",
+    ...(control.options ?? []).flatMap((option) => [option.label, option.value]),
+  ].join(" ");
+}
+
+export function isSemanticallyHeavyTextControl(control: ToolcraftControlSchema): boolean {
+  return heavyTextInputPattern.test(getControlSemanticText(control));
+}
+
+function isSemanticallyWorkloadControl(control: ToolcraftControlSchema): boolean {
+  const semanticText = getControlSemanticText(control);
+
+  if (control.type === "code" || control.type === "text") {
+    return isSemanticallyHeavyTextControl(control);
+  }
+
+  return workloadControlPattern.test(semanticText);
+}
+
+function isPotentialWorkloadControl(control: ToolcraftControlSchema): boolean {
+  return control.performanceRole === "workload" || isSemanticallyWorkloadControl(control);
+}
+
+export function collectToolcraftPerformanceSensitiveControls(
+  schema: ResolvedToolcraftAppSchema,
+): ToolcraftPerformanceSensitiveControl[] {
+  return (schema.panels.controls?.sections ?? []).flatMap((section) =>
+    Object.entries(section.controls)
+      .filter(
+        ([, control]) =>
+          !isToolcraftRuntimeOwnedTarget(control.target) && isPotentialWorkloadControl(control),
+      )
+      .map(([controlId, control]) => ({
+        control,
+        controlId,
+        target: control.target,
+      })),
+  );
+}
+
+export function collectToolcraftPerformanceRoleConflicts(
+  schema: ResolvedToolcraftAppSchema,
+): ToolcraftPerformanceSensitiveControl[] {
+  return (schema.panels.controls?.sections ?? []).flatMap((section) =>
+    Object.entries(section.controls)
+      .filter(
+        ([, control]) =>
+          !isToolcraftRuntimeOwnedTarget(control.target) &&
+          control.performanceRole === "responsiveness" &&
+          isSemanticallyWorkloadControl(control),
+      )
+      .map(([controlId, control]) => ({
+        control,
+        controlId,
+        target: control.target,
+      })),
+  );
+}
+
+export function collectToolcraftUnclassifiedPerformanceControls(
+  schema: ResolvedToolcraftAppSchema,
+): ToolcraftUnclassifiedPerformanceControl[] {
+  return (schema.panels.controls?.sections ?? []).flatMap((section) =>
+    Object.entries(section.controls)
+      .filter(
+        ([, control]) =>
+          control.type !== "panelActions" &&
+          control.type !== "settingsTransfer" &&
+          !isToolcraftRuntimeOwnedTarget(control.target) &&
+          control.performanceRole !== "workload" &&
+          control.performanceRole !== "responsiveness",
+      )
+      .map(([controlId, control]) => ({
+        control,
+        controlId,
+        target: control.target,
+      })),
+  );
+}

package/templates/runtime/testing/performance-coverage-context.ts

@@ -0,0 +1,44 @@
+import type { ResolvedToolcraftAppSchema, ToolcraftControlSchema } from "../schema/types";
+import type { ToolcraftPerformanceConfig, ToolcraftPerformanceScenario } from "./performance-types";
+import { getAllSchemaControls } from "./performance-schema-queries";
+
+export type ToolcraftPerformanceCoverageContext = {
+  config: ToolcraftPerformanceConfig;
+  controlsByTarget: ReadonlyMap<string, ToolcraftControlSchema>;
+  scenariosByTarget: ReadonlyMap<string, readonly ToolcraftPerformanceScenario[]>;
+  schema: ResolvedToolcraftAppSchema;
+};
+
+export type ToolcraftPerformanceCoverageValidator = (
+  context: ToolcraftPerformanceCoverageContext,
+) => string[];
+
+export function createToolcraftPerformanceCoverageContext(
+  schema: ResolvedToolcraftAppSchema,
+  config: ToolcraftPerformanceConfig,
+): ToolcraftPerformanceCoverageContext {
+  const scenariosByTarget = new Map<string, ToolcraftPerformanceScenario[]>();
+
+  for (const scenario of config.scenarios) {
+    if (!scenario.target) continue;
+    const scenarios = scenariosByTarget.get(scenario.target) ?? [];
+    scenarios.push(scenario);
+    scenariosByTarget.set(scenario.target, scenarios);
+  }
+
+  return {
+    config,
+    controlsByTarget: new Map(
+      getAllSchemaControls(schema).map((control) => [control.target, control] as const),
+    ),
+    scenariosByTarget,
+    schema,
+  };
+}
+
+export function runToolcraftPerformanceCoverageValidators(
+  context: ToolcraftPerformanceCoverageContext,
+  validators: readonly ToolcraftPerformanceCoverageValidator[],
+): string[] {
+  return validators.flatMap((validate) => validate(context));
+}

package/templates/runtime/testing/performance-coverage-renderer-policy.ts

@@ -0,0 +1,221 @@
+import type { ToolcraftRendererStrategy } from "./performance-types";
+import { getBrowserCheckPolicyErrors } from "./performance-browser-policy";
+import {
+  getRendererPipelineErrors,
+  getRendererTechniqueErrors,
+  hasDetailHeavyRendererLayer,
+  hasHighCountCanvas2DRendererLayer,
+  hasLongTaskBudgetScenario,
+  hasMainThreadCanvasRasterCompositePreviewPressure,
+  hasMeasuredGpuAlternativeEvidence,
+  hasStressPreviewOrAnimationScenario,
+  hasZoomSensitiveRenderer,
+} from "./performance-renderer-validation";
+import { hasKeyframeTimeline, hasLayersPanel } from "./performance-schema-queries";
+import type { ToolcraftPerformanceCoverageContext } from "./performance-coverage-context";
+
+export function getToolcraftRendererConfigurationCoverageErrors({
+  config,
+  schema,
+}: ToolcraftPerformanceCoverageContext): string[] {
+  const errors: string[] = [];
+  const customRendererStrategies = new Set<ToolcraftRendererStrategy>([
+    "dom",
+    "svg",
+    "canvas-2d",
+    "webgl",
+    "webgpu",
+  ]);
+  const gpuRendererStrategies = new Set<ToolcraftRendererStrategy>(["webgl", "webgpu"]);
+
+  errors.push(...getBrowserCheckPolicyErrors(config));
+  errors.push(...getRendererTechniqueErrors(config));
+  errors.push(...getRendererPipelineErrors(schema, config));
+
+  if (config.usesCustomRenderer && !customRendererStrategies.has(config.rendererStrategy)) {
+    errors.push(
+      'Custom renderers must declare rendererStrategy "dom", "svg", "canvas-2d", "webgl", or "webgpu".',
+    );
+  }
+
+  if (!config.usesCustomRenderer && config.rendererStrategy !== "none") {
+    errors.push(
+      `Non-custom renderer configs must use rendererStrategy "none", received "${config.rendererStrategy}".`,
+    );
+  }
+
+  if (config.usesCustomRenderer && config.rendererWorkload === "none") {
+    errors.push(
+      'Custom renderers must declare rendererWorkload "simple-composition", "text-output", "vector-output", or "pixel-output".',
+    );
+  }
+
+  if (!config.usesCustomRenderer && config.rendererWorkload !== "none") {
+    errors.push(
+      `Non-custom renderer configs must use rendererWorkload "none", received "${config.rendererWorkload}".`,
+    );
+  }
+
+  if (
+    hasMainThreadCanvasRasterCompositePreviewPressure(config) &&
+    config.rendererWorkload !== "pixel-output"
+  ) {
+    errors.push(
+      `Canvas 2D pipelines with main-thread rasterize/composite preview pressure must use rendererWorkload "pixel-output" or move expensive passes off the main thread; received "${config.rendererWorkload}".`,
+    );
+  }
+
+  if (
+    config.rendererWorkload === "pixel-output" &&
+    !gpuRendererStrategies.has(config.rendererStrategy) &&
+    (!config.rendererTechnique || !hasMeasuredGpuAlternativeEvidence(config.rendererTechnique))
+  ) {
+    errors.push(
+      `rendererWorkload "pixel-output" should use rendererStrategy "webgl" or "webgpu", received "${config.rendererStrategy}". Keeping a CPU renderer requires rendererTechnique.measuredAlternativeEvidence for WebGL/WebGPU stress comparison.`,
+    );
+  }
+
+  if (config.rendererWorkload === "pixel-output") {
+    if (!hasStressPreviewOrAnimationScenario(config)) {
+      errors.push(
+        'rendererWorkload "pixel-output" must include a stress preview-render or animation-frame scenario with stress: true for the largest product canvas and heaviest workload values.',
+      );
+    }
+
+    if (!hasLongTaskBudgetScenario(config)) {
+      errors.push(
+        'rendererWorkload "pixel-output" must include at least one maxLongTaskMs budget so GPU-backed previews cannot pass while freezing the main thread.',
+      );
+    }
+  }
+
+  if (config.usesCustomRenderer && hasDetailHeavyRendererLayer(config.rendererTechnique)) {
+    if (!hasStressPreviewOrAnimationScenario(config)) {
+      errors.push(
+        "Detail-heavy custom renderers must include a stress preview-render or animation-frame scenario for the largest product canvas and heaviest workload values.",
+      );
+    }
+
+    if (!hasLongTaskBudgetScenario(config)) {
+      errors.push(
+        "Detail-heavy custom renderers must include at least one maxLongTaskMs budget so renderer technology can be revised when main-thread work stalls.",
+      );
+    }
+  }
+
+  if (
+    config.usesCustomRenderer &&
+    hasHighCountCanvas2DRendererLayer(config.rendererTechnique) &&
+    !hasStressPreviewOrAnimationScenario(config)
+  ) {
+    errors.push(
+      "High-count Canvas 2D renderer layers must include stress preview-render or animation-frame evidence before delivery. If that stress evidence fails, revise renderer strategy instead of only lowering product workload.",
+    );
+  }
+
+  return errors;
+}
+
+export function getToolcraftCustomRendererInteractionCoverageErrors({
+  config,
+  schema,
+}: ToolcraftPerformanceCoverageContext): string[] {
+  const errors: string[] = [];
+
+  if (config.usesCustomRenderer) {
+    const interactions = new Set(config.scenarios.map((scenario) => scenario.interaction));
+    const hasAnimatedRendererScenario = interactions.has("animation-frame");
+    const needsViewportZoomStress =
+      hasAnimatedRendererScenario || hasZoomSensitiveRenderer(config);
+
+    for (const requiredInteraction of [
+      "preview-render",
+      "control-drag",
+      "viewport-stability",
+    ] as const) {
+      if (!interactions.has(requiredInteraction)) {
+        errors.push(
+          `Custom renderers must include a ${requiredInteraction} performance scenario.`,
+        );
+      }
+    }
+
+    if (hasAnimatedRendererScenario) {
+      const hasAnimatedViewportDrag = config.scenarios.some(
+        (scenario) =>
+          scenario.interaction === "animation-viewport-drag" &&
+          scenario.stress === true,
+      );
+
+      if (!hasAnimatedViewportDrag) {
+        errors.push(
+          "Animated custom renderers must include an animation-viewport-drag performance scenario that samples frames while physically moving the canvas viewport.",
+        );
+      }
+    }
+
+    if (needsViewportZoomStress) {
+      const hasViewportZoomStress = config.scenarios.some(
+        (scenario) =>
+          scenario.interaction === "viewport-zoom-stress" &&
+          scenario.stress === true,
+      );
+
+      if (!hasViewportZoomStress) {
+        errors.push(
+          "Detail-heavy or animated custom renderers must include a viewport-zoom-stress performance scenario that uses real zoom controls while sampling frame gaps and long tasks.",
+        );
+      }
+    }
+
+    if (schema.canvas.upload) {
+      const mediaImportScenarios = config.scenarios.filter(
+        (scenario) => scenario.interaction === "media-import",
+      );
+
+      if (mediaImportScenarios.length === 0) {
+        errors.push(
+          "Custom renderers with canvas upload must include a media-import performance scenario.",
+        );
+      } else if (
+        !mediaImportScenarios.some(
+          (scenario) => scenario.workload && scenario.stressFixture?.kind === "media",
+        )
+      ) {
+        errors.push(
+          'Custom renderers with canvas upload must include a workload media-import performance scenario with stressFixture.kind "media".',
+        );
+      }
+    }
+
+    if (hasKeyframeTimeline(schema)) {
+      const hasKeyframeViewportStability = config.scenarios.some(
+        (scenario) =>
+          scenario.interaction === "viewport-stability" &&
+          scenario.target === "timeline.keyframes",
+      );
+
+      if (!hasKeyframeViewportStability) {
+        errors.push(
+          'Keyframe custom renderers must include a viewport-stability performance scenario with target "timeline.keyframes" that exercises zoom/radar, expanded keyframes, keyframe creation, and playback or scrubbing.',
+        );
+      }
+    }
+
+    if (hasLayersPanel(schema)) {
+      const hasLayerViewportStability = config.scenarios.some(
+        (scenario) =>
+          scenario.interaction === "viewport-stability" &&
+          scenario.target === "layers.interactions",
+      );
+
+      if (!hasLayerViewportStability) {
+        errors.push(
+          'Layer-enabled custom renderers must include a viewport-stability performance scenario with target "layers.interactions" that exercises zoom/radar, layer selection, visibility, reorder or grouping, and selected-layer output stability.',
+        );
+      }
+    }
+  }
+
+  return errors;
+}