@pixel-point/toolcraft 0.0.13 → 0.0.15

package/templates/starter/src/app/starter-performance.budgets.test.ts

@@ -0,0 +1,374 @@
+import { describe, expect, it } from "vitest";
+import { defineToolcraft, validateToolcraftPerformanceCoverage } from "@repo/toolcraft-runtime";
+
+import { starterSchema } from "./starter-schema";
+
+describe("Toolcraft starter performance budgets and control coverage", () => {
+  it("requires every visible non-action control to have performance coverage", () => {
+    const rendererSchema = defineToolcraft({
+      canvas: {
+        enabled: true,
+        size: { height: 1080, unit: "px", width: 1440 },
+        sizing: { mode: "editable-output" },
+      },
+      panels: {
+        controls: {
+          sections: [
+            {
+              controls: {
+                depth: {
+                  defaultValue: 50,
+                  label: "Depth",
+                  max: 100,
+                  min: 0,
+                  target: "shader.depth",
+                  type: "slider",
+                },
+                gradient: {
+                  defaultValue: "aurora",
+                  label: "Gradient",
+                  options: [
+                    { label: "Aurora", value: "aurora" },
+                    { label: "Prism", value: "prism" },
+                  ],
+                  target: "shader.gradient",
+                  type: "select",
+                },
+                mode: {
+                  defaultValue: "soft",
+                  label: "Mode",
+                  options: [
+                    { label: "Soft", value: "soft" },
+                    { label: "Sharp", value: "sharp" },
+                  ],
+                  target: "shader.mode",
+                  type: "segmented",
+                },
+              },
+              title: "Shader",
+            },
+          ],
+          title: "Shader Controls",
+        },
+      },
+    });
+
+    expect(
+      validateToolcraftPerformanceCoverage(rendererSchema, {
+        rendererStrategy: "canvas-2d",
+        rendererWorkload: "simple-composition",
+        scenarios: [
+          {
+            automated: true,
+            automatedTestName: "perf: preview render stays under budget",
+            browser: true,
+            browserTestName: "browser perf: preview render stays under budget",
+            budget: { maxPreviewMs: 1000 },
+            expectedObservable: "Preview renders without freezing.",
+            fixture: "1440x1080 shader fixture",
+            id: "preview-render",
+            interaction: "preview-render",
+            workload: false,
+          },
+          {
+            automated: true,
+            automatedTestName: "perf: depth drag stays responsive",
+            browser: true,
+            browserTestName: "browser perf: depth drag stays responsive",
+            budget: { maxFrameGapMs: 50, maxInteractionMs: 250 },
+            controlLabel: "Depth",
+            expectedObservable: "Dragging Depth remains responsive.",
+            fixture: "1440x1080 shader fixture",
+            id: "depth-drag",
+            interaction: "control-drag",
+            target: "shader.depth",
+            values: { default: 50, max: 100, min: 0 },
+            workload: true,
+          },
+          {
+            automated: true,
+            automatedTestName: "perf: viewport stays stable",
+            browser: true,
+            browserTestName: "browser perf: viewport stays stable",
+            budget: { maxFrameGapMs: 50 },
+            expectedObservable: "Canvas zoom and offset do not jump.",
+            fixture: "1440x1080 shader fixture",
+            id: "viewport-stability",
+            interaction: "viewport-stability",
+            workload: false,
+          },
+        ],
+        usesCustomRenderer: true,
+        workloadTargets: ["shader.depth"],
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "shader.gradient must have a performance scenario because every visible control can affect app responsiveness.",
+        "shader.mode must have a performance scenario because every visible control can affect app responsiveness.",
+      ]),
+    );
+  });
+
+  it("requires visible control performance coverage even without a custom renderer", () => {
+    const controlsSchema = defineToolcraft({
+      canvas: {
+        enabled: true,
+        sizing: { mode: "intrinsic-media" },
+      },
+      panels: {
+        controls: {
+          sections: [
+            {
+              controls: {
+                mode: {
+                  defaultValue: "soft",
+                  label: "Mode",
+                  options: [
+                    { label: "Soft", value: "soft" },
+                    { label: "Sharp", value: "sharp" },
+                  ],
+                  target: "app.mode",
+                  type: "select",
+                },
+              },
+              title: "Display",
+            },
+          ],
+          title: "Controls",
+        },
+      },
+    });
+
+    expect(
+      validateToolcraftPerformanceCoverage(controlsSchema, {
+        rendererStrategy: "none",
+        rendererWorkload: "none",
+        scenarios: [],
+        usesCustomRenderer: false,
+        workloadTargets: [],
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "app.mode must have a performance scenario because every visible control can affect app responsiveness.",
+      ]),
+    );
+  });
+
+  it("requires budgets that match each performance interaction type", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(starterSchema, {
+        rendererStrategy: "none",
+        rendererWorkload: "none",
+        scenarios: [
+          {
+            automated: true,
+            automatedTestName: "perf: char size drag stays responsive",
+            browser: true,
+            browserTestName: "browser perf: char size drag stays responsive",
+            budget: { maxExportMs: 100 },
+            expectedObservable: "Dragging Char Size remains responsive.",
+            fixture: "1600x1000 transparent glyph fixture",
+            id: "char-size-drag",
+            interaction: "control-drag",
+            target: "generation.prompt",
+            values: { default: 12, max: 32, min: 4 },
+            workload: true,
+          },
+          {
+            actionValue: "export.png",
+            automated: true,
+            automatedTestName: "perf: export png stays under budget",
+            browser: true,
+            browserTestName: "browser perf: export png stays under budget",
+            budget: { maxFrameGapMs: 50 },
+            completionEvidence: "download",
+            controlLabel: "Export PNG",
+            expectedObservable: "Export completes without blocking the UI.",
+            fixture: "1920x1080 output fixture",
+            id: "export-png",
+            interaction: "export-copy",
+            workload: false,
+          },
+          {
+            automated: true,
+            automatedTestName: "perf: preview render stays under budget",
+            browser: true,
+            browserTestName: "browser perf: preview render stays under budget",
+            budget: { maxFrameGapMs: 50 },
+            expectedObservable: "Preview renders without freezing.",
+            fixture: "1920x1080 output fixture",
+            id: "preview-render",
+            interaction: "preview-render",
+            workload: false,
+          },
+          {
+            automated: true,
+            automatedTestName: "perf: animation frame loop stays smooth",
+            browser: true,
+            browserTestName: "browser perf: animation frame loop stays smooth",
+            budget: { maxFrameGapMs: 50 },
+            expectedObservable: "Animation advances without frame stalls.",
+            fixture: "animated output fixture",
+            id: "animation-frame-loop",
+            interaction: "animation-frame",
+            workload: false,
+          },
+        ],
+        usesCustomRenderer: false,
+        workloadTargets: ["generation.prompt"],
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "char-size-drag control-drag scenario must declare maxInteractionMs and maxFrameGapMs.",
+        "export-png export-copy scenario must declare maxExportMs.",
+        "preview-render preview-render scenario must declare maxPreviewMs or maxRenderMs.",
+        "animation-frame-loop animation-frame scenario must declare maxLongTaskMs.",
+      ]),
+    );
+  });
+
+  it("rejects performance budgets that are too loose to catch lag", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(starterSchema, {
+        rendererStrategy: "none",
+        rendererWorkload: "none",
+        scenarios: [
+          {
+            automated: true,
+            automatedTestName: "perf: char size drag stays responsive",
+            browser: true,
+            browserTestName: "browser perf: char size drag stays responsive",
+            budget: { maxFrameGapMs: 1000, maxInteractionMs: 10000 },
+            controlLabel: "Char Size",
+            expectedObservable: "Dragging Char Size remains responsive.",
+            fixture: "1600x1000 transparent glyph fixture",
+            id: "char-size-drag",
+            interaction: "control-drag",
+            target: "generation.prompt",
+            values: { default: 12, max: 32, min: 4 },
+            workload: true,
+          },
+        ],
+        usesCustomRenderer: false,
+        workloadTargets: ["generation.prompt"],
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "char-size-drag maxFrameGapMs budget must be <= 120ms, received 1000ms.",
+        "char-size-drag maxInteractionMs budget must be <= 2000ms, received 10000ms.",
+      ]),
+    );
+  });
+
+  it("does not let load-profile smooth targets bypass required interaction budgets", () => {
+    const schema = defineToolcraft({
+      canvas: { enabled: true },
+      panels: {
+        controls: {
+          sections: [
+            {
+              controls: {
+                density: {
+                  defaultValue: 4,
+                  label: "Density",
+                  max: 12,
+                  min: 1,
+                  performanceReason: "Density changes rendered output size.",
+                  performanceRole: "workload",
+                  target: "render.density",
+                  type: "slider",
+                },
+              },
+              title: "Render",
+            },
+          ],
+          title: "Controls",
+        },
+      },
+    });
+
+    expect(
+      validateToolcraftPerformanceCoverage(schema, {
+        rendererStrategy: "none",
+        rendererWorkload: "none",
+        scenarios: [
+          {
+            automated: true,
+            automatedTestName: "perf: density drag stays responsive",
+            browser: true,
+            browserTestName: "browser perf: density drag stays responsive",
+            budget: { maxFrameGapMs: 80 },
+            controlLabel: "Density",
+            expectedObservable: "Dragging Density updates product output without blocking the UI.",
+            fixture: "density smooth target fixture",
+            id: "density-drag",
+            interaction: "control-drag",
+            stressFixture: {
+              kind: "max-value",
+              loadProfile: {
+                degradationStepPercent: 10,
+                evidence: [
+                  {
+                    attemptedRatio: 1,
+                    attemptedTarget: 12,
+                    decision: "Keep 12 as experimental and guarantee smoothness through 11.",
+                    measuredResult: "At density 12, maxFrameGapMs 148 exceeded the 80ms budget.",
+                    optimizationAttempted:
+                      "Cached expensive render inputs and coalesced preview work to requestAnimationFrame.",
+                    result: "failed",
+                    scenarioId: "density-drag",
+                    target: "render.density",
+                  },
+                ],
+                hardLimit: 12,
+                metric: "numeric-max",
+                smoothTarget: 11,
+                smoothTargetRatio: 0.9,
+                target: "render.density",
+                userFacingRange: "experimental-above-smooth",
+              },
+              reason: "Density 11 is the measured smooth target after hard-limit testing.",
+              value: 11,
+            },
+            target: "render.density",
+            values: { default: 4, max: 12, min: 1 },
+            workload: true,
+          },
+        ],
+        usesCustomRenderer: false,
+        workloadTargets: ["render.density"],
+      }),
+    ).toContain("density-drag control-drag scenario must declare maxInteractionMs.");
+  });
+
+  it("requires real browser interaction metadata for control performance scenarios", () => {
+    expect(
+      validateToolcraftPerformanceCoverage(starterSchema, {
+        rendererStrategy: "none",
+        rendererWorkload: "none",
+        scenarios: [
+          {
+            automated: true,
+            automatedTestName: "perf: char size drag stays responsive",
+            browser: true,
+            browserTestName: "browser perf: char size drag stays responsive",
+            budget: { maxFrameGapMs: 50, maxInteractionMs: 250 },
+            expectedObservable: "Dragging Char Size remains responsive.",
+            fixture: "1600x1000 transparent glyph fixture",
+            id: "char-size-drag",
+            interaction: "control-drag",
+            target: "generation.prompt",
+            values: { default: 12, max: 32, min: 4 },
+            workload: true,
+          },
+        ],
+        usesCustomRenderer: false,
+        workloadTargets: ["generation.prompt"],
+      }),
+    ).toEqual(
+      expect.arrayContaining([
+        "char-size-drag control-drag scenario must declare controlLabel or uiSelector for its real browser interaction.",
+      ]),
+    );
+  });
+});

package/templates/starter/src/app/starter-performance.gates.test.ts

@@ -0,0 +1,128 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+import {
+  collectToolcraftUnclassifiedPerformanceControls,
+  validateToolcraftPerformanceCoverage,
+} from "@repo/toolcraft-runtime";
+
+import { starterPerformance } from "./starter-performance";
+import { starterSchema } from "./starter-schema";
+import {
+  e2eDir,
+  playwrightConfigForbidsFocusedTests,
+  projectDir,
+} from "./starter-performance-test-utils";
+
+describe("Toolcraft starter performance gates", () => {
+  it("publishes separate browser acceptance and performance fallback gates", () => {
+    const packageJson = JSON.parse(readFileSync(join(projectDir, "package.json"), "utf8")) as {
+      scripts?: Record<string, string>;
+    };
+    const generatedAppTestScript =
+      packageJson.scripts?.["test:generated"] ?? packageJson.scripts?.test;
+    const runScriptPattern = (scriptName: string) =>
+      `(?:pnpm ${scriptName}|npm run ${scriptName})`;
+
+    expect(
+      generatedAppTestScript,
+      "Generated app tests must invoke the Toolcraft integrity checker.",
+    ).toContain("node scripts/check-toolcraft-integrity.mjs");
+
+    expect(
+      packageJson.scripts?.["test:browser"],
+      "Generated apps must keep full performance scenarios out of the default browser acceptance gate.",
+    ).toBe('playwright install chromium && playwright test --grep-invert "browser perf:"');
+    expect(
+      packageJson.scripts?.["test:browser:perf"],
+      "Generated apps must expose a protected sequential Playwright checkpoint for final performance proof.",
+    ).toBe("node scripts/run-browser-performance.mjs");
+    const performanceRunner = readFileSync(
+      join(projectDir, "scripts/run-browser-performance.mjs"),
+      "utf8",
+    );
+    const receiptModule = readFileSync(
+      join(projectDir, "scripts/toolcraft-verification-receipt.mjs"),
+      "utf8",
+    );
+    expect(performanceRunner).toContain('runBinary(viteBin, ["build"])');
+    expect(performanceRunner).toContain('TOOLCRAFT_BROWSER_SERVER_MODE: "preview"');
+    expect(performanceRunner).toContain('"--workers=1"');
+    expect(performanceRunner).not.toContain('"--pass-with-no-tests"');
+    expect(performanceRunner).not.toContain("process.argv.slice(2)");
+    expect(performanceRunner).toContain(
+      "assertToolcraftVerificationInputsUnchanged",
+    );
+    expect(performanceRunner).toContain("clearToolcraftPerformanceReceipt");
+    expect(performanceRunner).toContain("writeToolcraftPerformanceCheckpointReceipt");
+    expect(receiptModule).not.toContain(
+      "export async function writeToolcraftPerformanceReceipt",
+    );
+    expect(packageJson.scripts?.["verify:quick"]).toMatch(
+      new RegExp(`^${runScriptPattern("ai:check")} && ${runScriptPattern("test")}$`),
+    );
+    expect(packageJson.scripts?.["verify:ui"]).toMatch(
+      new RegExp(`^${runScriptPattern("test:browser")}$`),
+    );
+    expect(packageJson.scripts?.["verify:perf"]).toMatch(
+      new RegExp(`^${runScriptPattern("test:browser:perf")}$`),
+    );
+    expect(packageJson.scripts?.["verify:perf:playwright"]).toMatch(
+      new RegExp(`^${runScriptPattern("test:browser:perf")}$`),
+    );
+    expect(
+      packageJson.scripts?.["verify:perf:record-agent-browser"],
+      "A terminal-only command must not mint an agent-browser performance receipt without browser evidence.",
+    ).toBeUndefined();
+    expect(packageJson.scripts?.["verify:perf:record-exemption"]).toContain(
+      "record-exemption",
+    );
+    expect(packageJson.scripts?.["verify:receipt"]).toContain("validate");
+    expect(packageJson.scripts?.["verify:final"]).toMatch(
+      new RegExp(
+        `^${runScriptPattern("ai:check")} && ${runScriptPattern("test")} && ${runScriptPattern("build")} && ${runScriptPattern("test:browser")} && ${runScriptPattern("verify:receipt")}$`,
+      ),
+    );
+  });
+
+  it("keeps the Playwright performance audit inside the browser perf tag", () => {
+    const source = readFileSync(join(e2eDir, "app-performance.spec.ts"), "utf8");
+    const testNames = [...source.matchAll(/test\(\s*(["'`])([^"'`]+)\1/g)].map(
+      (match) => match[2],
+    );
+
+    expect(source).not.toContain("TOOLCRAFT_PERF_CHECK");
+    expect(testNames.length).toBeGreaterThan(0);
+    expect(
+      testNames.every((name) => name.includes("browser perf:")),
+      `app-performance.spec.ts tests must all be tagged for the dedicated perf checkpoint: ${testNames.join(", ")}`,
+    ).toBe(true);
+  });
+
+  it("rejects focused Playwright tests that would bypass protected browser gates", () => {
+    expect(playwrightConfigForbidsFocusedTests()).toBe(true);
+  });
+
+  it("declares agent browser as the preferred performance runner", () => {
+    expect(starterPerformance.browserCheckPolicy).toEqual({
+      fallbackRunner: "playwright",
+      fallbackWhen: ["agent-browser-unavailable", "ci"],
+      preferredRunner: "agent-browser",
+    });
+  });
+
+  it("requires valid performance coverage for declared workload scenarios", () => {
+    expect(validateToolcraftPerformanceCoverage(starterSchema, starterPerformance)).toEqual([]);
+  });
+
+  it("requires every visible control to classify its performance role", () => {
+    const unclassifiedControls =
+      collectToolcraftUnclassifiedPerformanceControls(starterSchema);
+
+    expect(
+      unclassifiedControls,
+      "Every visible non-action control must declare performanceRole as workload or responsiveness so AI cannot skip the performance decision.",
+    ).toEqual([]);
+  });
+});

package/templates/starter/src/app/starter-performance.renderer-source.test.ts

@@ -0,0 +1,136 @@
+import { describe, expect, it } from "vitest";
+
+import { starterPerformance } from "./starter-performance";
+import { starterSchema } from "./starter-schema";
+import {
+  appPerformanceHasInteractionInvalidation,
+  appPerformanceHasRenderPipelinePass,
+  sourceCreatesWebGlContextInComponentRender,
+  sourceMayUploadTextureFromTimelineDrivenEffect,
+  sourceResyncsTimelineDurationFromRuntimeDuration,
+  sourceUsesAnimationFrameWithoutCleanup,
+  sourceUsesCpuPixelLoop,
+  sourceUsesDirectStorageApi,
+  sourceUsesGpuRenderer,
+  sourceUsesLowResolutionPreviewUpscale,
+  sourceUsesWebGlLifecycleGuard,
+} from "./starter-performance-test-utils";
+
+describe("Toolcraft starter renderer source performance contract", () => {
+  it("detects low-resolution preview upscale code paths", () => {
+    expect(
+      sourceUsesLowResolutionPreviewUpscale(`
+        const maxPreviewPixels = 1_250_000;
+        const previewScale = Math.sqrt(maxPreviewPixels / (outputWidth * outputHeight));
+        previewContext.drawImage(offscreenCanvas, 0, 0, outputWidth, outputHeight);
+      `),
+    ).toBe(true);
+
+    expect(
+      sourceUsesLowResolutionPreviewUpscale(`
+        canvas.width = outputWidth;
+        canvas.height = outputHeight;
+        drawAsciiTextToCanvas({ canvas, text });
+      `),
+    ).toBe(false);
+  });
+
+  it("rejects low-resolution preview upscale for text and vector output renderers", () => {
+    if (
+      starterPerformance.rendererWorkload !== "text-output" &&
+      starterPerformance.rendererWorkload !== "vector-output"
+    ) {
+      return;
+    }
+
+    expect(
+      sourceUsesLowResolutionPreviewUpscale(),
+      "Text/vector product previews must preserve native output fidelity. Do not render a low-resolution preview canvas/texture and upscale it to state.canvas.size; optimize layout/drawing instead.",
+    ).toBe(false);
+  });
+
+  it("requires procedural pixel-loop renderers to use a GPU strategy", () => {
+    if (!sourceUsesCpuPixelLoop()) {
+      return;
+    }
+
+    expect(
+      starterPerformance.rendererWorkload,
+      "Procedural ImageData/getImageData/putImageData renderers must be classified as pixel-output.",
+    ).toBe("pixel-output");
+    expect(
+      starterPerformance.rendererStrategy,
+      "Procedural ImageData/getImageData/putImageData renderers must be converted to WebGL/WebGPU or removed from the critical render path.",
+    ).toMatch(/^(webgl|webgpu)$/);
+    expect(
+      sourceUsesGpuRenderer(),
+      "Procedural pixel renderers must contain an actual WebGL/WebGPU code path, not only declare a GPU strategy.",
+    ).toBe(true);
+    expect(
+      appPerformanceHasRenderPipelinePass("pixel-transform"),
+      "Procedural pixel renderers must declare a rendererPipeline pixel-transform pass so caching and invalidation are machine-checkable.",
+    ).toBe(true);
+  });
+
+  it("requires custom renderers to declare high-frequency viewport invalidation", () => {
+    if (!starterPerformance.usesCustomRenderer) {
+      return;
+    }
+
+    expect(
+      appPerformanceHasInteractionInvalidation("viewport-zoom"),
+      "Custom renderer apps must declare rendererPipeline interactionInvalidation for viewport-zoom so zoom can stay responsive without recomputing expensive passes.",
+    ).toBe(true);
+  });
+
+  it("requires WebGL/WebGPU renderers to keep their pipeline lifecycle outside React render", () => {
+    if (!sourceUsesGpuRenderer()) {
+      return;
+    }
+
+    expect(
+      sourceUsesWebGlLifecycleGuard(),
+      "WebGL/WebGPU renderer setup must be guarded by refs, memoized setup, an effect, or a renderer class so control changes update uniforms/buffers instead of rebuilding the pipeline.",
+    ).toBe(true);
+    expect(
+      sourceCreatesWebGlContextInComponentRender(),
+      "Do not create a WebGL context directly in the component render path; initialize it once and update uniforms/buffers on runtime value changes.",
+    ).toBe(false);
+  });
+
+  it("requires animation loops to clean up scheduled frames", () => {
+    expect(
+      sourceUsesAnimationFrameWithoutCleanup(),
+      "Renderers that schedule requestAnimationFrame must cancelAnimationFrame on cleanup to avoid runaway loops during control changes or route unmount.",
+    ).toBe(false);
+  });
+
+  it("rejects direct app storage writes outside the runtime persistence policy", () => {
+    expect(
+      sourceUsesDirectStorageApi(),
+      "Generated apps must not read or write app state through localStorage/sessionStorage directly. Use runtime persistence policy when product persistence is required.",
+    ).toBe(false);
+  });
+
+  it("rejects renderer effects that overwrite user-edited timeline duration", () => {
+    expect(
+      sourceResyncsTimelineDurationFromRuntimeDuration(),
+      "Renderers must not watch state.timeline.durationSeconds and dispatch timeline.setDuration back to a computed local duration. Compute a default only during initialization/reset, then map renderer progress to state.timeline.durationSeconds.",
+    ).toBe(false);
+  });
+
+  it("rejects timeline-driven texture uploads in GPU keyframe renderers", () => {
+    if (
+      !sourceUsesGpuRenderer() ||
+      starterSchema.panels.timeline?.enabled !== true ||
+      starterSchema.panels.timeline.mode !== "keyframes"
+    ) {
+      return;
+    }
+
+    expect(
+      sourceMayUploadTextureFromTimelineDrivenEffect(),
+      "GPU keyframe renderers must upload source textures only when media/resource keys change. Timeline-driven effects may update uniforms and draw, but must not call texImage2D or renderer.setImage from an effect that depends on settings/currentTime/keyframeGroups.",
+    ).toBe(false);
+  });
+});