@pixel-point/toolcraft 0.0.14 → 0.0.15

package/templates/starter/e2e/app-browser-proof-session.spec.ts

@@ -0,0 +1,196 @@
+import { expect, test } from "@playwright/test";
+
+import {
+  createToolcraftBrowserProofSession,
+  getToolcraftBrowserActionTarget,
+  runToolcraftBrowserAction,
+  runToolcraftBrowserValueAction,
+  type ToolcraftBrowserProofSession,
+} from "./browser-proof-session";
+import { expectToolcraftTimelineDuration } from "./browser-timeline-evidence-helpers";
+
+const TOOLCRAFT_APP_ROOT_SELECTOR = '[data-slot="toolcraft-runtime-app"]';
+
+async function createDurationFixture(
+  page: Parameters<typeof createToolcraftBrowserProofSession>[0],
+): Promise<ToolcraftBrowserProofSession> {
+  await page.goto("/");
+  const session = await createToolcraftBrowserProofSession(page);
+  await page.locator(TOOLCRAFT_APP_ROOT_SELECTOR).evaluate((root) => {
+    root.setAttribute(
+      "data-duration",
+      JSON.stringify({
+        renderedCycleDurationSeconds: 8,
+        timelineDurationSeconds: 8,
+      }),
+    );
+  });
+  return session;
+}
+
+test("semantic recipes reject raw in-memory callbacks", async ({ page }) => {
+  let duration = {
+    renderedCycleDurationSeconds: 8,
+    timelineDurationSeconds: 8,
+  };
+
+  await expect(
+    expectToolcraftTimelineDuration(
+      (async () => duration) as never,
+      (async () => {
+        duration = {
+          renderedCycleDurationSeconds: 6,
+          timelineDurationSeconds: 6,
+        };
+      }) as never,
+      6,
+      { requirementId: "timeline.raw-callback", stabilityIntervalMs: 0 },
+    ),
+  ).rejects.toThrow(/browser proof session/u);
+});
+
+test("semantic recipes reject action and observation from different sessions", async ({
+  page,
+}) => {
+  await page.goto("/");
+  const first = await createToolcraftBrowserProofSession(page);
+  const second = await createToolcraftBrowserProofSession(page);
+  const observation = first.observe((root) =>
+    JSON.parse(root.getAttribute("data-duration") ?? "null"),
+  );
+  const action = second.action(async (currentPage) => {
+    await currentPage.locator('[data-slot="toolcraft-runtime-app"]').evaluate((root) => {
+      root.setAttribute(
+        "data-duration",
+        JSON.stringify({
+          renderedCycleDurationSeconds: 6,
+          timelineDurationSeconds: 6,
+        }),
+      );
+    });
+  });
+
+  await expect(
+    expectToolcraftTimelineDuration(observation, action, 6, {
+      requirementId: "timeline.mixed-session",
+      stabilityIntervalMs: 0,
+    }),
+  ).rejects.toThrow(/same browser proof session/u);
+});
+
+test("semantic recipes accept observations rooted in the current Toolcraft page", async ({
+  page,
+}) => {
+  const session = await createDurationFixture(page);
+  const observation = session.observe((root) =>
+    JSON.parse(root.getAttribute("data-duration") ?? "null"),
+  );
+  const action = session.action(async (currentPage) => {
+    await currentPage.locator('[data-slot="toolcraft-runtime-app"]').evaluate((root) => {
+      root.setAttribute(
+        "data-duration",
+        JSON.stringify({
+          renderedCycleDurationSeconds: 6,
+          timelineDurationSeconds: 6,
+        }),
+      );
+    });
+  });
+
+  await expectToolcraftTimelineDuration(observation, action, 6, {
+    requirementId: "timeline.browser-session",
+    stabilityIntervalMs: 0,
+  });
+});
+
+test("target-scoped actions resolve the exact field inside a grouped control", async ({
+  page,
+}) => {
+  await page.goto("/");
+  const session = await createToolcraftBrowserProofSession(page);
+  await page.locator(TOOLCRAFT_APP_ROOT_SELECTOR).evaluate((root) => {
+    const boundary = document.createElement("div");
+    boundary.className = "contents";
+    boundary.dataset.toolcraftControlTargets = JSON.stringify([
+      "appearance.fill",
+      "appearance.stroke",
+    ]);
+    for (const target of ["appearance.fill", "appearance.stroke"]) {
+      const field = document.createElement("div");
+      field.dataset.slot = "field";
+      const button = document.createElement("button");
+      button.dataset.target = target;
+      button.textContent = target;
+      field.append(button);
+      boundary.append(field);
+    }
+    root.append(boundary);
+  });
+  const action = session.controlAction("appearance.stroke", async (control) => {
+    await control.getByRole("button").evaluate((button) => {
+      button.dataset.clicked = "true";
+    });
+  });
+
+  expect(getToolcraftBrowserActionTarget(action)).toBe("appearance.stroke");
+  await runToolcraftBrowserAction(action);
+  await expect(
+    page.locator('button[data-target="appearance.stroke"]'),
+  ).toHaveAttribute("data-clicked", "true");
+  await expect(page.locator('button[data-target="appearance.fill"]')).not.toHaveAttribute(
+    "data-clicked",
+    "true",
+  );
+
+  const valueAction = session.controlAction(
+    "appearance.stroke",
+    async () => "stroke-result",
+  );
+  await expect(runToolcraftBrowserValueAction(valueAction)).resolves.toBe(
+    "stroke-result",
+  );
+});
+
+test("proof sessions reject pages without the Toolcraft app identity", async ({ page }) => {
+  await page.setContent('<main data-slot="toolcraft-runtime-app"></main>');
+
+  await expect(createToolcraftBrowserProofSession(page)).rejects.toThrow(
+    /server-backed http\(s\) page/u,
+  );
+});
+
+test("proof sessions start from a fresh server document instead of synthetic DOM", async ({
+  page,
+}) => {
+  await page.goto("/");
+  await page.setContent(`
+    <meta name="toolcraft-app-title" content="Toolcraft Starter" />
+    <main data-slot="toolcraft-runtime-app" data-synthetic="true"></main>
+  `);
+
+  await createToolcraftBrowserProofSession(page);
+
+  await expect(page.locator('[data-synthetic="true"]')).toHaveCount(0);
+  await expect(page.locator(TOOLCRAFT_APP_ROOT_SELECTOR)).toBeVisible();
+});
+
+test("proof sessions reject a synthetic root replacement after verification", async ({
+  page,
+}) => {
+  const session = await createDurationFixture(page);
+  const observation = session.observe((root) => root.getAttribute("data-duration"));
+  const action = session.action(async (currentPage) => {
+    await currentPage.locator(TOOLCRAFT_APP_ROOT_SELECTOR).evaluate((root) => {
+      const replacement = document.createElement("main");
+      replacement.dataset.slot = "toolcraft-runtime-app";
+      root.replaceWith(replacement);
+    });
+  });
+
+  await expect(
+    expectToolcraftTimelineDuration(observation, action, 6, {
+      requirementId: "timeline.synthetic-root",
+      stabilityIntervalMs: 0,
+    }),
+  ).rejects.toThrow(/original live runtime root/u);
+});

package/templates/starter/e2e/app-browser-runtime-evidence.spec.ts

@@ -0,0 +1,9 @@
+import { test } from "@playwright/test";
+
+import {
+  TOOLCRAFT_BROWSER_ACCEPTANCE_MARKER_TEST_NAME,
+  TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+} from "../src/app/test-evidence/browser-runtime-contract";
+
+test(TOOLCRAFT_BROWSER_ACCEPTANCE_MARKER_TEST_NAME, () => undefined);
+test(TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME, () => undefined);

package/templates/starter/e2e/app-browser-runtime-provenance.spec.ts

@@ -0,0 +1,240 @@
+import { expect, test, type TestInfo } from "@playwright/test";
+
+import {
+  TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_ATTACHMENT_NAME,
+  TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_CONTENT_TYPE,
+  parseToolcraftBrowserRuntimeEvidence,
+} from "../src/app/test-evidence/browser-runtime-contract";
+import {
+  expectToolcraftAcceptanceOutcome,
+  expectToolcraftReferenceParity,
+} from "./browser-acceptance-outcome-helpers";
+import { expectToolcraftLayerSelection } from "./browser-layer-evidence-helpers";
+import { createToolcraftBrowserProofSession } from "./browser-proof-session";
+import { expectToolcraftScenarioPerformanceBudget } from "./performance-budget-helpers";
+import { measureToolcraftInteraction } from "./performance-probe-helpers";
+import { expectToolcraftProductObservableToChange } from "./product-observable-helpers";
+import { expectToolcraftTimelineDuration } from "./browser-timeline-evidence-helpers";
+
+test("performance budgets accept only branded measurements for the same scenario", async ({
+  page,
+}, testInfo: TestInfo) => {
+  await page.setContent('<div id="performance-outcome">before</div>');
+  const config = {
+    rendererStrategy: "none",
+    rendererWorkload: "none",
+    scenarios: [
+      {
+        automated: true,
+        automatedTestName: "perf: runtime brand",
+        browser: true,
+        browserTestName: "browser perf: runtime brand",
+        budget: { maxInteractionMs: 1_000 },
+        expectedObservable: "The measured action completes.",
+        fixture: "runtime fixture",
+        id: "runtime-brand",
+        interaction: "control-change",
+        workload: false,
+      },
+    ],
+    usesCustomRenderer: false,
+    workloadTargets: [],
+  } as const;
+  const attachmentCount = testInfo.attachments.length;
+  const result = await measureToolcraftInteraction(
+    page,
+    async () => {
+      await page.locator("#performance-outcome").evaluate((node) => {
+        node.textContent = "after";
+      });
+    },
+    {
+      observeOutcome: () => page.locator("#performance-outcome").textContent(),
+      scenarioId: "runtime-brand",
+    },
+  );
+  await expectToolcraftScenarioPerformanceBudget(result, config, "runtime-brand");
+
+  expect(
+    testInfo.attachments.slice(attachmentCount).map((attachment) =>
+      parseToolcraftBrowserRuntimeEvidence(attachment)?.evidenceType,
+    ),
+  ).toEqual([
+    "performance-product-outcome",
+    "performance-measurement",
+    "performance-budget",
+  ]);
+  expect(Object.isFrozen(result)).toBe(true);
+  await expect(
+    expectToolcraftScenarioPerformanceBudget(
+      { ...result },
+      config,
+      "runtime-brand",
+    ),
+  ).rejects.toThrow(/protected measurement helper/u);
+});
+
+test("protected observable evidence is attached only after its assertion succeeds", async ({
+  page,
+}, testInfo: TestInfo) => {
+  await page.goto("/");
+  const session = await createToolcraftBrowserProofSession(page);
+  await page.locator('[data-slot="toolcraft-runtime-app"]').evaluate((root) => {
+    const output = document.createElement("div");
+    output.dataset.toolcraftProductOutput = "";
+    output.textContent = "Before";
+    const button = document.createElement("button");
+    button.id = "change-output";
+    button.type = "button";
+    button.textContent = "Change output";
+    const controlBoundary = document.createElement("div");
+    controlBoundary.className = "contents";
+    controlBoundary.dataset.toolcraftControlTarget = "test.output";
+    const field = document.createElement("div");
+    field.dataset.slot = "field";
+    field.append(button);
+    controlBoundary.append(field);
+    root.append(output, controlBoundary);
+  });
+
+  const attachmentCount = testInfo.attachments.length;
+  await expectToolcraftProductObservableToChange(
+    session,
+    session.controlAction("test.output", async (control, currentPage) => {
+      await control.locator("#change-output").evaluate((button) => {
+        button.dataset.activated = "true";
+      });
+      await currentPage.locator('[data-toolcraft-product-output]').evaluate((output) => {
+        output.textContent = "After";
+      });
+    }),
+    { requirementId: "test.successful-observable" },
+  );
+
+  const attachment = testInfo.attachments.at(-1);
+  expect(testInfo.attachments).toHaveLength(attachmentCount + 1);
+  expect(attachment).toMatchObject({
+    contentType: TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_CONTENT_TYPE,
+    name: TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_ATTACHMENT_NAME,
+  });
+  expect(parseToolcraftBrowserRuntimeEvidence(attachment)).toEqual({
+    evidenceType: "product-observable-change",
+    requirementId: "test.successful-observable",
+    target: "test.output",
+    version: 2,
+  });
+
+  const countAfterSuccess = testInfo.attachments.length;
+  await expect(
+    expectToolcraftProductObservableToChange(
+      session,
+      session.controlAction("test.output", async () => undefined),
+      {
+        requirementId: "test.failed-observable",
+        timeoutMs: 100,
+      },
+    ),
+  ).rejects.toThrow(/Product output should change/);
+  expect(testInfo.attachments).toHaveLength(countAfterSuccess);
+});
+
+test("acceptance evidence is attached only after semantic verification succeeds", async ({
+  page,
+}, testInfo: TestInfo) => {
+  await page.goto("/");
+  const session = await createToolcraftBrowserProofSession(page);
+  await page.locator('[data-slot="toolcraft-runtime-app"]').evaluate((root) => {
+    root.setAttribute(
+      "data-duration",
+      JSON.stringify({
+        renderedCycleDurationSeconds: 8,
+        timelineDurationSeconds: 8,
+      }),
+    );
+    root.setAttribute(
+      "data-selection",
+      JSON.stringify({ selectedLayerId: "layer-a" }),
+    );
+    const outcome = document.createElement("div");
+    outcome.id = "outcome";
+    outcome.textContent = "before";
+    root.append(outcome);
+  });
+  const attachmentCount = testInfo.attachments.length;
+  const duration = session.observe((root) =>
+    JSON.parse(root.getAttribute("data-duration") ?? "null"),
+  );
+
+  await expectToolcraftTimelineDuration(
+    duration,
+    session.action(async (currentPage) => {
+      await currentPage.locator('[data-slot="toolcraft-runtime-app"]').evaluate((root) => {
+        root.setAttribute(
+          "data-duration",
+          JSON.stringify({
+            renderedCycleDurationSeconds: 6,
+            timelineDurationSeconds: 6,
+          }),
+        );
+      });
+    }),
+    6,
+    { requirementId: "timeline.playback", stabilityIntervalMs: 0 },
+  );
+
+  expect(parseToolcraftBrowserRuntimeEvidence(testInfo.attachments.at(-1))).toEqual({
+    evidenceType: "timeline-duration",
+    requirementId: "timeline.playback",
+    version: 2,
+  });
+  expect(testInfo.attachments).toHaveLength(attachmentCount + 1);
+
+  const selection = session.observe((root) =>
+    JSON.parse(root.getAttribute("data-selection") ?? "null"),
+  );
+  await expect(
+    expectToolcraftLayerSelection(
+      selection,
+      session.action(async () => undefined),
+      { selectedLayerId: "layer-b" },
+      {
+        requirementId: "layers.selection",
+        stabilityIntervalMs: 0,
+        timeoutMs: 100,
+      },
+    ),
+  ).rejects.toThrow(/select the expected layer/);
+  expect(testInfo.attachments).toHaveLength(attachmentCount + 1);
+
+  let transientActionStarted = false;
+  let transientObservationCount = 0;
+  await expect(
+    expectToolcraftAcceptanceOutcome(
+      async () => {
+        if (!transientActionStarted) {
+          return "before";
+        }
+        transientObservationCount += 1;
+        return transientObservationCount === 1 ? "transient" : "before";
+      },
+      async () => {
+        transientActionStarted = true;
+      },
+      {
+        evidenceType: "command-side-effect",
+        requirementId: "command.transient",
+      },
+    ),
+  ).rejects.toThrow(/stability window/);
+  expect(testInfo.attachments).toHaveLength(attachmentCount + 1);
+
+  await page.locator("#outcome").evaluate((node) => {
+    node.textContent = "after";
+  });
+  await expectToolcraftReferenceParity(
+    () => page.locator("#outcome").textContent(),
+    "after",
+    { requirementId: "reference.renderer-state" },
+  );
+  expect(testInfo.attachments).toHaveLength(attachmentCount + 2);
+});

package/templates/starter/e2e/app-browser-runtime-reporter.spec.ts

@@ -0,0 +1,180 @@
+import { expect, test } from "@playwright/test";
+import type { Suite, TestCase } from "@playwright/test/reporter";
+
+import {
+  TOOLCRAFT_BROWSER_ACCEPTANCE_MARKER_TEST_NAME,
+  TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+  TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_ATTACHMENT_NAME,
+  TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_CONTENT_TYPE,
+  type ToolcraftBrowserRuntimeRequirement,
+} from "../src/app/test-evidence/browser-runtime-contract";
+import ToolcraftBrowserRuntimeEvidenceReporter from "./browser-runtime-evidence-reporter";
+
+const fakeRequirement: ToolcraftBrowserRuntimeRequirement = {
+  evidenceType: "product-observable-change",
+  requirementId: "appearance.opacity",
+  testName: "browser: opacity changes product output",
+};
+
+function fakeTestCase({
+  expectedStatus = "passed",
+  file = "/product/e2e/app-controls.spec.ts",
+  results = [],
+  title,
+}: {
+  expectedStatus?: TestCase["expectedStatus"];
+  file?: string;
+  results?: TestCase["results"];
+  title: string;
+}): TestCase {
+  return {
+    expectedStatus,
+    location: { column: 1, file, line: 1 },
+    results,
+    title,
+  } as TestCase;
+}
+
+function fakeSuite(tests: TestCase[]): Suite {
+  return { allTests: () => tests } as Suite;
+}
+
+function passedResultWithEvidence(
+  requirement: ToolcraftBrowserRuntimeRequirement,
+): TestCase["results"][number] {
+  return {
+    attachments: [
+      {
+        body: Buffer.from(
+          JSON.stringify({
+            evidenceType: requirement.evidenceType,
+            requirementId: requirement.requirementId,
+            ...(requirement.target === undefined
+              ? {}
+              : { target: requirement.target }),
+            version: 2,
+          }),
+        ),
+        contentType: TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_CONTENT_TYPE,
+        name: TOOLCRAFT_BROWSER_RUNTIME_EVIDENCE_ATTACHMENT_NAME,
+      },
+    ],
+    retry: 0,
+    status: "passed",
+  } as TestCase["results"][number];
+}
+
+test("runtime evidence reporter fails full runs with a missing required test", async () => {
+  const errors: string[] = [];
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [fakeRequirement],
+    reportError: (error) => errors.push(error),
+  });
+  const marker = fakeTestCase({
+    file: "/product/e2e/app-browser-runtime-evidence.spec.ts",
+    title: TOOLCRAFT_BROWSER_ACCEPTANCE_MARKER_TEST_NAME,
+  });
+
+  reporter.onBegin?.({} as never, fakeSuite([marker]));
+  const status = await reporter.onEnd?.({ status: "passed" } as never);
+
+  expect(status).toEqual({ status: "failed" });
+  expect(errors).toContainEqual(
+    expect.stringContaining(`Missing required browser test "${fakeRequirement.testName}"`),
+  );
+});
+
+test("runtime evidence reporter limits marker-free runs to selected declared tests", async () => {
+  const secondRequirement: ToolcraftBrowserRuntimeRequirement = {
+    ...fakeRequirement,
+    requirementId: "appearance.color",
+    testName: "browser: color changes product output",
+  };
+  const selectedTest = fakeTestCase({
+    results: [passedResultWithEvidence(fakeRequirement)],
+    title: fakeRequirement.testName,
+  });
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [fakeRequirement, secondRequirement],
+    reportError: () => undefined,
+  });
+  const unprotectedMarkerTitle = fakeTestCase({
+    title: TOOLCRAFT_BROWSER_ACCEPTANCE_MARKER_TEST_NAME,
+  });
+
+  reporter.onBegin?.(
+    {} as never,
+    fakeSuite([unprotectedMarkerTitle, selectedTest]),
+  );
+  const status = await reporter.onEnd?.({ status: "passed" } as never);
+
+  expect(status).toBeUndefined();
+});
+
+test("runtime evidence reporter rejects unreachable evidence and runtime skips", () => {
+  const candidates = [
+    fakeTestCase({
+      results: [
+        { attachments: [], retry: 0, status: "passed" } as TestCase["results"][number],
+      ],
+      title: fakeRequirement.testName,
+    }),
+    fakeTestCase({
+      expectedStatus: "skipped",
+      results: [
+        {
+          ...passedResultWithEvidence(fakeRequirement),
+          status: "skipped",
+        } as TestCase["results"][number],
+      ],
+      title: fakeRequirement.testName,
+    }),
+  ];
+
+  for (const candidate of candidates) {
+    const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+      acceptanceRequirements: [fakeRequirement],
+      reportError: () => undefined,
+    });
+
+    reporter.onBegin?.({} as never, fakeSuite([candidate]));
+    expect(reporter.onEnd?.({ status: "passed" } as never)).toEqual({
+      status: "failed",
+    });
+  }
+});
+
+test("performance marker requires every derived scenario evidence type", () => {
+  const performanceRequirements: ToolcraftBrowserRuntimeRequirement[] = [
+    {
+      evidenceType: "performance-measurement",
+      requirementId: "preview-heavy",
+      testName: "browser perf: preview heavy",
+    },
+    {
+      evidenceType: "performance-budget",
+      requirementId: "preview-heavy",
+      testName: "browser perf: preview heavy",
+    },
+  ];
+  const marker = fakeTestCase({
+    file: "/product/e2e/app-browser-runtime-evidence.spec.ts",
+    title: TOOLCRAFT_BROWSER_PERFORMANCE_MARKER_TEST_NAME,
+  });
+  const scenarioTest = fakeTestCase({
+    results: [passedResultWithEvidence(performanceRequirements[0]!)],
+    title: "browser perf: preview heavy",
+  });
+  const errors: string[] = [];
+  const reporter = new ToolcraftBrowserRuntimeEvidenceReporter({
+    acceptanceRequirements: [],
+    performanceRequirements,
+    reportError: (error) => errors.push(error),
+  });
+
+  reporter.onBegin?.({} as never, fakeSuite([marker, scenarioTest]));
+  expect(reporter.onEnd?.({ status: "passed" } as never)).toEqual({
+    status: "failed",
+  });
+  expect(errors).toContainEqual(expect.stringContaining("performance-budget"));
+});