@pixel-point/toolcraft 0.0.14 → 0.0.15

package/templates/runtime/testing/performance-coverage-scenario-policy.ts

@@ -0,0 +1,147 @@
+import {
+  getBudgetCapErrors,
+  getMissingInteractionBudgetFields,
+  hasAnyBudget,
+} from "./performance-budget-validation";
+import {
+  getStressFixtureErrors,
+  getWorkloadFixtureErrors,
+} from "./performance-fixture-validation";
+import { arePerformanceFixtureValuesEqual } from "./performance-fixture-values";
+import {
+  hasMinDefaultMax,
+  requiresConcreteUiTarget,
+} from "./performance-scenario-validation";
+import {
+  getToolcraftControlPerformanceValues,
+  hasOutputDeliveryActionReference,
+} from "./performance-schema-queries";
+import type { ToolcraftPerformanceCoverageContext } from "./performance-coverage-context";
+
+export function getToolcraftPerformanceScenarioCoverageErrors({
+  config,
+  controlsByTarget,
+  schema,
+}: ToolcraftPerformanceCoverageContext): string[] {
+  const errors: string[] = [];
+  const browserTestNamesByScenario = new Map<string, string>();
+
+  for (const scenario of config.scenarios) {
+    if (!scenario.id.trim()) {
+      errors.push("Performance scenario is missing an id.");
+    }
+
+    if (!scenario.fixture.trim()) {
+      errors.push(`${scenario.id} must name a representative fixture.`);
+    }
+
+    if (!scenario.expectedObservable.trim()) {
+      errors.push(`${scenario.id} must describe a product-level performance observable.`);
+    }
+
+    if (!hasAnyBudget(scenario.budget)) {
+      errors.push(`${scenario.id} must declare at least one numeric performance budget.`);
+    }
+
+    const missingBudgetFields = getMissingInteractionBudgetFields(scenario);
+    if (missingBudgetFields.length > 0) {
+      errors.push(
+        `${scenario.id} ${scenario.interaction} scenario must declare ${missingBudgetFields.join(
+          " and ",
+        )}.`,
+      );
+    }
+
+    errors.push(...getBudgetCapErrors(scenario));
+
+    if (!scenario.automated || !scenario.automatedTestName.trim()) {
+      errors.push(`${scenario.id} must point to an automated performance test.`);
+    }
+
+    if (!scenario.browser || !scenario.browserTestName.trim()) {
+      errors.push(`${scenario.id} must point to a browser performance test.`);
+    }
+
+    if (scenario.browser && scenario.browserTestName.trim()) {
+      const previousScenarioId = browserTestNamesByScenario.get(scenario.browserTestName);
+
+      if (previousScenarioId) {
+        errors.push(
+          `${scenario.id} browserTestName "${scenario.browserTestName}" is already used by ${previousScenarioId}. Give each performance scenario its own browser test so every control is actually exercised.`,
+        );
+      } else {
+        browserTestNamesByScenario.set(scenario.browserTestName, scenario.id);
+      }
+    }
+
+    const scenarioControl = controlsByTarget.get(scenario.target ?? "");
+    const schemaPerformanceValues = getToolcraftControlPerformanceValues(scenarioControl);
+
+    if (scenario.workload && scenario.interaction !== "media-import") {
+      if (schemaPerformanceValues) {
+        if (
+          scenario.values &&
+          (!arePerformanceFixtureValuesEqual(
+            scenario.values.default,
+            schemaPerformanceValues.default,
+          ) ||
+            !arePerformanceFixtureValuesEqual(
+              scenario.values.max,
+              schemaPerformanceValues.max,
+            ) ||
+            !arePerformanceFixtureValuesEqual(
+              scenario.values.min,
+              schemaPerformanceValues.min,
+            ))
+        ) {
+          errors.push(
+            `${scenario.id} values redefine schema range for "${scenario.target}"; expected ${JSON.stringify(
+              schemaPerformanceValues,
+            )}. Omit values to use the schema-derived range.`,
+          );
+        }
+      } else if (!hasMinDefaultMax(scenario.values)) {
+        errors.push(`${scenario.id} workload scenario must include min/default/max values.`);
+      }
+    }
+
+    errors.push(...getStressFixtureErrors(scenario, scenarioControl));
+    errors.push(...getWorkloadFixtureErrors(config, scenario, scenarioControl));
+
+    if (
+      requiresConcreteUiTarget(scenario.interaction) &&
+      !scenario.controlLabel?.trim() &&
+      !scenario.uiSelector?.trim() &&
+      !(scenario.interaction === "export-copy" && scenario.actionValue?.trim())
+    ) {
+      errors.push(
+        `${scenario.id} ${scenario.interaction} scenario must declare controlLabel or uiSelector for its real browser interaction.`,
+      );
+    }
+
+    if (scenario.interaction === "export-copy") {
+      if (!scenario.actionValue?.trim()) {
+        errors.push(
+          `${scenario.id} export-copy scenario must declare actionValue for the exact output delivery action it exercises.`,
+        );
+      } else if (!scenario.controlLabel?.trim()) {
+        errors.push(
+          `${scenario.id} export-copy scenario must declare controlLabel for the visible output delivery action it clicks.`,
+        );
+      } else if (
+        scenario.completionEvidence !== "download" &&
+        scenario.completionEvidence !== "clipboard"
+      ) {
+        errors.push(
+          `${scenario.id} export-copy scenario must declare completionEvidence as download or clipboard.`,
+        );
+      } else if (!hasOutputDeliveryActionReference(schema, scenario)) {
+        errors.push(
+          `${scenario.id} export-copy scenario references actionValue "${scenario.actionValue.trim()}" with label "${scenario.controlLabel.trim()}", but schema panelActions has no matching output delivery action.`,
+        );
+      }
+    }
+  }
+
+  return errors;
+}

package/templates/runtime/testing/performance-coverage-target-policy.ts

@@ -0,0 +1,121 @@
+import { isToolcraftRuntimeOwnedTarget } from "../schema/runtime-targets";
+import {
+  collectToolcraftPerformanceRoleConflicts,
+  collectToolcraftPerformanceSensitiveControls,
+} from "./performance-control-classification";
+import {
+  getSliderDragControlType,
+  hasControlDragScenario,
+  hasMinDefaultMax,
+  isWorkloadCoverageInteraction,
+} from "./performance-scenario-validation";
+import {
+  getToolcraftControlPerformanceValues,
+  getVisiblePerformanceControlTargets,
+  hasOutputDeliveryAction,
+} from "./performance-schema-queries";
+import type { ToolcraftPerformanceCoverageContext } from "./performance-coverage-context";
+
+export function getToolcraftPerformanceTargetCoverageErrors({
+  config,
+  controlsByTarget,
+  scenariosByTarget,
+  schema,
+}: ToolcraftPerformanceCoverageContext): string[] {
+  const errors: string[] = [];
+
+  const performanceTargets = new Set(config.workloadTargets);
+  const schemaTargets = new Set(
+    (schema.panels.controls?.sections ?? []).flatMap((section) =>
+      Object.values(section.controls).map((control) => control.target),
+    ),
+  );
+  const sensitiveTargets = new Set(
+    collectToolcraftPerformanceSensitiveControls(schema)
+      .map((entry) => entry.target)
+      .filter((target) => !isToolcraftRuntimeOwnedTarget(target)),
+  );
+  const performanceRoleConflicts = collectToolcraftPerformanceRoleConflicts(schema);
+
+  for (const { controlId, target } of performanceRoleConflicts) {
+    errors.push(
+      `${controlId} (${target}) looks performance-sensitive but declares performanceRole "responsiveness". Use performanceRole "workload" with workloadTargets and min/default/max coverage, or rename/restructure the control if it is truly lightweight.`,
+    );
+  }
+
+  for (const target of sensitiveTargets) {
+    if (!performanceTargets.has(target)) {
+      errors.push(
+        `${target} is performance-sensitive and must be listed in workloadTargets with min/default/max workload coverage.`,
+      );
+    }
+  }
+
+  for (const target of performanceTargets) {
+    if (!schemaTargets.has(target)) {
+      errors.push(`Performance workload target ${target} does not exist in schema controls.`);
+    }
+
+    const targetScenarios = scenariosByTarget.get(target) ?? [];
+    const targetControl = controlsByTarget.get(target);
+    const targetRequiresDrag = getSliderDragControlType(targetControl) !== null;
+    const hasWorkloadCoverage = targetScenarios.some(
+      (scenario) => {
+        const values = getToolcraftControlPerformanceValues(targetControl) ?? scenario.values;
+
+        return (
+          scenario.workload &&
+          isWorkloadCoverageInteraction(scenario.interaction, targetRequiresDrag) &&
+          hasMinDefaultMax(values)
+        );
+      },
+    );
+
+    if (!hasWorkloadCoverage) {
+      errors.push(
+        targetRequiresDrag
+          ? `${target} must have min/default/max workload performance coverage through a real control-drag scenario.`
+          : `${target} must have min/default/max workload performance coverage.`,
+      );
+    }
+  }
+
+  for (const target of getVisiblePerformanceControlTargets(schema)) {
+    if (isToolcraftRuntimeOwnedTarget(target)) {
+      continue;
+    }
+
+    const targetScenarios = scenariosByTarget.get(target) ?? [];
+    const targetControl = controlsByTarget.get(target);
+    const sliderControlType = getSliderDragControlType(targetControl);
+
+    if (targetScenarios.length === 0) {
+      errors.push(
+        `${target} must have a performance scenario because every visible control can affect app responsiveness.`,
+      );
+    } else if (sliderControlType && !hasControlDragScenario(targetScenarios)) {
+      errors.push(
+        `${target} is a ${sliderControlType} and must have a control-drag performance scenario proving live canvas/product feedback while dragging.`,
+      );
+    }
+  }
+
+  return errors;
+}
+
+export function getToolcraftPerformanceOutputCoverageErrors({
+  config,
+  schema,
+}: ToolcraftPerformanceCoverageContext): string[] {
+  const errors: string[] = [];
+
+  if (hasOutputDeliveryAction(schema)) {
+    const interactions = new Set(config.scenarios.map((scenario) => scenario.interaction));
+
+    if (!interactions.has("export-copy")) {
+      errors.push("Output actions must include an export-copy performance scenario.");
+    }
+  }
+
+  return errors;
+}

package/templates/runtime/testing/performance-coverage-validator.ts

@@ -0,0 +1,34 @@
+import type { ResolvedToolcraftAppSchema } from "../schema/types";
+import type { ToolcraftPerformanceConfig } from "./performance-types";
+import {
+  createToolcraftPerformanceCoverageContext,
+  runToolcraftPerformanceCoverageValidators,
+  type ToolcraftPerformanceCoverageValidator,
+} from "./performance-coverage-context";
+import {
+  getToolcraftCustomRendererInteractionCoverageErrors,
+  getToolcraftRendererConfigurationCoverageErrors,
+} from "./performance-coverage-renderer-policy";
+import { getToolcraftPerformanceScenarioCoverageErrors } from "./performance-coverage-scenario-policy";
+import {
+  getToolcraftPerformanceOutputCoverageErrors,
+  getToolcraftPerformanceTargetCoverageErrors,
+} from "./performance-coverage-target-policy";
+
+const toolcraftPerformanceCoverageValidators: readonly ToolcraftPerformanceCoverageValidator[] = [
+  getToolcraftRendererConfigurationCoverageErrors,
+  getToolcraftPerformanceScenarioCoverageErrors,
+  getToolcraftPerformanceTargetCoverageErrors,
+  getToolcraftPerformanceOutputCoverageErrors,
+  getToolcraftCustomRendererInteractionCoverageErrors,
+];
+
+export function validateToolcraftPerformanceCoverage(
+  schema: ResolvedToolcraftAppSchema,
+  config: ToolcraftPerformanceConfig,
+): string[] {
+  return runToolcraftPerformanceCoverageValidators(
+    createToolcraftPerformanceCoverageContext(schema, config),
+    toolcraftPerformanceCoverageValidators,
+  );
+}

package/templates/runtime/testing/performance-export-coverage.test.ts

@@ -0,0 +1,234 @@
+import { describe, expect, expectTypeOf, it } from "vitest";
+
+import { defineToolcraft } from "../schema/define-toolcraft";
+import type { ToolcraftActionSchema } from "../schema/types";
+import {
+  defineToolcraftPerformance,
+  type ToolcraftPerformanceScenario,
+  validateToolcraftPerformanceCoverage,
+} from "./performance";
+import { hasOutputDeliveryAction } from "./performance-schema-queries";
+
+function createImageExportSchema(
+  includeOutputAction: boolean,
+  outputAction: ToolcraftActionSchema = {
+    label: "Export PNG",
+    role: "export-image",
+    value: "export.png",
+  },
+) {
+  return defineToolcraft({
+    canvas: {
+      enabled: true,
+      sizing: { mode: "editable-output" },
+    },
+    panels: {
+      controls: {
+        sections: [
+          {
+            controls: {
+              resolution: {
+                defaultValue: "4k",
+                label: "Resolution",
+                options: [
+                  { label: "2K", value: "2k" },
+                  { label: "4K", value: "4k" },
+                  { label: "8K", value: "8k" },
+                ],
+                target: "export.image.resolution",
+                type: "select",
+              },
+            },
+            title: "Image Export",
+          },
+          ...(includeOutputAction
+            ? [
+                {
+                  actionGroup: "secondary" as const,
+                  controls: {
+                    outputActions: {
+                      actions: [outputAction],
+                      target: "actions.output",
+                      type: "panelActions" as const,
+                    },
+                  },
+                  title: "Export",
+                },
+              ]
+            : []),
+        ],
+        title: "Controls",
+      },
+    },
+  });
+}
+
+function createImageExportPerformanceConfig(
+  controlLabel: string | undefined = "Export PNG",
+  uiSelector?: string,
+  actionValue: string | undefined = "export.png",
+  completionEvidence: "clipboard" | "download" | null = "download",
+) {
+  const validScenario = {
+    actionValue: "export.png",
+    automated: true,
+    automatedTestName: "perf: 8K PNG export stays responsive",
+    browser: true,
+    browserTestName: "browser perf: 8K PNG export stays responsive",
+    budget: { maxExportMs: 2000 },
+    controlLabel: "Export PNG",
+    completionEvidence: "download",
+    expectedObservable: "Selecting 8K and exporting PNG measures the real output workload.",
+    fixture: "8K PNG export fixture",
+    id: "export-png",
+    interaction: "export-copy",
+    stressFixture: {
+      kind: "max-value",
+      loadProfile: {
+        hardLimit: "8k",
+        metric: "custom",
+        smoothTarget: "8k",
+        smoothTargetRatio: 1,
+        target: "export.image.resolution",
+        userFacingRange: "fully-guaranteed",
+      },
+      reason: "8K is the highest user-visible export resolution.",
+      value: "8k",
+    },
+    target: "export.image.resolution",
+    values: { default: "4k", max: "8k", min: "2k" },
+    workload: true,
+  } satisfies ToolcraftPerformanceScenario;
+
+  return defineToolcraftPerformance({
+    rendererStrategy: "none",
+    rendererWorkload: "none",
+    scenarios: [
+      {
+        ...validScenario,
+        actionValue,
+        completionEvidence: completionEvidence ?? undefined,
+        controlLabel,
+        uiSelector,
+      } as unknown as ToolcraftPerformanceScenario,
+    ],
+    usesCustomRenderer: false,
+    workloadTargets: ["export.image.resolution"],
+  });
+}
+
+describe("Toolcraft export performance coverage", () => {
+  it("requires exact action metadata in the export-copy scenario type", () => {
+    type ExportScenario = Extract<
+      ToolcraftPerformanceScenario,
+      { interaction: "export-copy" }
+    >;
+
+    expectTypeOf<ExportScenario>().toMatchTypeOf<{
+      actionValue: string;
+      completionEvidence: "clipboard" | "download";
+      controlLabel: string;
+    }>();
+  });
+
+  it("recognizes output actions by typed role instead of visible copy", () => {
+    expect(hasOutputDeliveryAction(createImageExportSchema(true))).toBe(true);
+    expect(
+      hasOutputDeliveryAction(
+        createImageExportSchema(true, {
+          label: "Guardar archivo",
+          role: "download-output",
+          value: "download-file",
+        }),
+      ),
+    ).toBe(true);
+    expect(
+      hasOutputDeliveryAction(
+        createImageExportSchema(true, {
+          label: "Copyright",
+          value: "copyright.info",
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it("rejects export workload coverage when the schema has no output action", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(false),
+        createImageExportPerformanceConfig(),
+      ),
+    ).toContain(
+      'export-png export-copy scenario references actionValue "export.png" with label "Export PNG", but schema panelActions has no matching output delivery action.',
+    );
+  });
+
+  it("allows non-slider workload controls to be covered by their real export path", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig(),
+      ),
+    ).toEqual([]);
+  });
+
+  it("rejects export workload coverage that names a different action", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig("Save Image"),
+      ),
+    ).toContain(
+      'export-png export-copy scenario references actionValue "export.png" with label "Save Image", but schema panelActions has no matching output delivery action.',
+    );
+  });
+
+  it("rejects export workload coverage that names a different action value", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig("Export PNG", undefined, "export.video"),
+      ),
+    ).toContain(
+      'export-png export-copy scenario references actionValue "export.video" with label "Export PNG", but schema panelActions has no matching output delivery action.',
+    );
+  });
+
+  it("rejects selector-only export workload evidence", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig(undefined, "#unrelated-control", ""),
+      ),
+    ).toContain(
+      'export-png export-copy scenario must declare actionValue for the exact output delivery action it exercises.',
+    );
+  });
+
+  it("rejects export workload evidence without the visible action label", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig("", "#unrelated-control", "export.png"),
+      ),
+    ).toContain(
+      'export-png export-copy scenario must declare controlLabel for the visible output delivery action it clicks.',
+    );
+  });
+
+  it("rejects export workload evidence without a protected completion kind", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(
+        createImageExportSchema(true),
+        createImageExportPerformanceConfig(
+          "Export PNG",
+          undefined,
+          "export.png",
+          null,
+        ),
+      ),
+    ).toContain(
+      "export-png export-copy scenario must declare completionEvidence as download or clipboard.",
+    );
+  });
+});