@pixel-point/toolcraft 0.0.15 → 0.0.16

package/templates/starter/scripts/toolcraft-targeted-performance-execution.mjs

@@ -0,0 +1,274 @@
+import { randomUUID } from "node:crypto";
+import { rm } from "node:fs/promises";
+import path from "node:path";
+
+import {
+  decodeToolcraftPerformancePathId,
+} from "@repo/toolcraft-runtime/performance-path-codec";
+import {
+  getToolcraftPlaywrightExactGrepPattern,
+} from "./playwright-test-title-selection.mjs";
+import {
+  createToolcraftPerformanceIterationComparison,
+} from "./toolcraft-performance-iteration-comparison.mjs";
+import {
+  createToolcraftTargetedPerformanceEvidenceHash,
+  readToolcraftTargetedPerformanceReport,
+} from "./toolcraft-targeted-performance-report.mjs";
+import {
+  combineToolcraftTargetedPerformanceReports,
+} from "./toolcraft-targeted-performance-report-combination.mjs";
+import {
+  runToolcraftProofProcess,
+} from "./toolcraft-proof-process.mjs";
+
+function getPathPassIds(pathId) {
+  const signature = decodeToolcraftPerformancePathId(pathId);
+  return signature ? [...signature.invalidates] : undefined;
+}
+
+function isCanonicalTargets(value, { allowEmpty = false } = {}) {
+  return (
+    Array.isArray(value) &&
+    (allowEmpty || value.length > 0) &&
+    value.every(
+      (item) =>
+        typeof item === "string" &&
+        item.length > 0 &&
+        item.trim() === item,
+    ) &&
+    value.every(
+      (item, index) => index === 0 || value[index - 1] < item,
+    )
+  );
+}
+
+function assertInputs(options) {
+  if (
+    options.fixtureResolutionMode !== "strict-development" ||
+    !/^[a-f0-9]{64}$/u.test(options.requestAuthorityHash ?? "") ||
+    !isCanonicalTargets(options.performancePassIds, { allowEmpty: true }) ||
+    !isCanonicalTargets(options.performancePathIds) ||
+    !isCanonicalTargets(options.performanceTestNames) ||
+    options.performancePathIds.length !==
+      options.performanceTestNames.length ||
+    !Array.isArray(options.performanceSelections) ||
+    options.performanceSelections.length !==
+      options.performanceTestNames.length ||
+    !options.performanceTestNames.every(
+      (testName) =>
+        options.performanceSelections.filter(
+          ({ leafTitle }) => leafTitle === testName,
+        ).length === 1,
+    ) ||
+    options.performancePathIds.some(
+      (pathId) => getPathPassIds(pathId) === undefined,
+    ) ||
+    JSON.stringify(
+      [...new Set(
+        options.performancePathIds.flatMap(
+          (pathId) => getPathPassIds(pathId) ?? [],
+        ),
+      )].sort(),
+    ) !== JSON.stringify(options.performancePassIds) ||
+    !(
+      options.performanceComparison?.kind === "none" ||
+      (options.performanceComparison?.kind ===
+        "compatible-targeted-report" &&
+        typeof options.performanceComparison.report === "object" &&
+        /^[a-f0-9]{64}$/u.test(
+          options.performanceComparison.comparisonHash ?? "",
+        ))
+    )
+  ) {
+    throw new Error(
+      "Toolcraft planned targeted performance execution input is malformed.",
+    );
+  }
+}
+
+function assertOperations(operations) {
+  if (
+    typeof operations !== "object" ||
+    operations === null ||
+    !Object.isFrozen(operations) ||
+    ![
+      "createNonce",
+      "readReport",
+      "removeReport",
+      "runPlaywright",
+    ].every((name) => typeof operations[name] === "function")
+  ) {
+    throw new Error(
+      "Toolcraft targeted performance execution requires explicit frozen operations.",
+    );
+  }
+}
+
+function createTestEvidence(selections) {
+  return selections
+    .map(({ fullTitle, leafTitle }) => ({ fullTitle, leafTitle }))
+    .sort((left, right) =>
+      left.fullTitle < right.fullTitle
+        ? -1
+        : left.fullTitle > right.fullTitle
+          ? 1
+          : 0,
+    );
+}
+
+export async function executeToolcraftTargetedPerformanceVerificationCore(
+  options,
+) {
+  assertInputs(options);
+  assertOperations(options.operations);
+  const {
+    fixtureResolutionMode,
+    operations,
+    performanceComparison,
+    performancePassIds,
+    performancePathIds,
+    performanceSelections,
+    performanceTestNames,
+    playwrightBin,
+    projectDir,
+    requestAuthorityHash,
+    sourceHash,
+  } = options;
+  const reports = [];
+  const combinedNonce = operations.createNonce();
+  for (const [index, pathId] of performancePathIds.entries()) {
+    const testName = performanceTestNames[index];
+    const pathPassIds = getPathPassIds(pathId);
+    const selection = performanceSelections.find(
+      ({ leafTitle }) => leafTitle === testName,
+    );
+    const nonce = operations.createNonce();
+    const reportPath = path.join(
+      projectDir,
+      ".toolcraft",
+      "verification",
+      `targeted-performance-${nonce}.json`,
+    );
+    await operations.removeReport(reportPath);
+    try {
+      await operations.runPlaywright({
+        fixtureResolutionMode,
+        nonce,
+        pathId,
+        performancePassIds: pathPassIds,
+        playwrightBin,
+        projectDir,
+        reportPath,
+        requestAuthorityHash,
+        selection,
+        sourceHash,
+        testName,
+      });
+      reports.push(
+        await operations.readReport(reportPath, {
+          fixtureResolutionMode,
+          fixtureSelector: "development",
+          nonce,
+          performancePassIds: pathPassIds,
+          performancePathIds: [pathId],
+          requestAuthorityHash,
+          sourceHash,
+          testNames: [testName],
+          version: 3,
+        }),
+      );
+    } finally {
+      await operations.removeReport(reportPath);
+    }
+  }
+  const report = combineToolcraftTargetedPerformanceReports({
+    nonce: combinedNonce,
+    performancePassIds,
+    reports,
+    requestAuthorityHash,
+    sourceHash,
+  });
+  const testEvidence = createTestEvidence(performanceSelections);
+  const evidenceHash = createToolcraftTargetedPerformanceEvidenceHash({
+    report,
+    testEvidence,
+  });
+  const comparison =
+    performanceComparison.kind === "compatible-targeted-report"
+      ? createToolcraftPerformanceIterationComparison({
+          currentReport: report,
+          previousReport: performanceComparison.report,
+          comparisonHash: performanceComparison.comparisonHash,
+        })
+      : null;
+  if (
+    performanceComparison.kind === "compatible-targeted-report" &&
+    comparison.status !== "compared"
+  ) {
+    throw new Error(
+      "Toolcraft targeted performance comparison is no longer compatible.",
+    );
+  }
+  return Object.freeze({
+    comparison,
+    evidenceHash,
+    report,
+    testEvidence: Object.freeze(testEvidence.map(Object.freeze)),
+  });
+}
+
+function createProductionOperations() {
+  return Object.freeze({
+    createNonce: randomUUID,
+    readReport: readToolcraftTargetedPerformanceReport,
+    removeReport: (reportPath) => rm(reportPath, { force: true }),
+    runPlaywright: ({
+      fixtureResolutionMode,
+      nonce,
+      pathId,
+      performancePassIds,
+      playwrightBin,
+      projectDir,
+      reportPath,
+      requestAuthorityHash,
+      selection,
+      sourceHash,
+    }) =>
+      runToolcraftProofProcess(
+        playwrightBin,
+        [
+          "test",
+          "--grep",
+          getToolcraftPlaywrightExactGrepPattern([selection]),
+          "--workers=1",
+        ],
+        {
+          cwd: projectDir,
+          env: {
+            ...process.env,
+            TOOLCRAFT_BROWSER_SERVER_MODE: "preview",
+            TOOLCRAFT_PERFORMANCE_FIXTURE_RESOLUTION_MODE:
+              fixtureResolutionMode,
+            TOOLCRAFT_PERFORMANCE_FIXTURE_SELECTOR: "development",
+            TOOLCRAFT_TARGETED_PERFORMANCE_PASS_IDS:
+              JSON.stringify(performancePassIds),
+            TOOLCRAFT_TARGETED_PERFORMANCE_PATH_IDS:
+              JSON.stringify([pathId]),
+            TOOLCRAFT_TARGETED_PERFORMANCE_REPORT_NONCE: nonce,
+            TOOLCRAFT_TARGETED_PERFORMANCE_REPORT_PATH: reportPath,
+            TOOLCRAFT_TARGETED_PERFORMANCE_REPORT_SOURCE_HASH: sourceHash,
+            TOOLCRAFT_PERFORMANCE_REQUEST_AUTHORITY_HASH:
+              requestAuthorityHash,
+          },
+        },
+      ),
+  });
+}
+
+export function executeToolcraftTargetedPerformanceVerification(options) {
+  return executeToolcraftTargetedPerformanceVerificationCore({
+    ...options,
+    operations: createProductionOperations(),
+  });
+}

package/templates/starter/scripts/toolcraft-targeted-performance-execution.test.mjs

@@ -0,0 +1,225 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  executeToolcraftTargetedPerformanceVerificationCore,
+} from "./toolcraft-targeted-performance-execution.mjs";
+import {
+  createToolcraftTargetedPerformanceReport,
+} from "./toolcraft-targeted-performance-report.mjs";
+
+const hash = (character) => character.repeat(64);
+const performancePathId = (invalidates) =>
+  `performance-path:${encodeURIComponent(
+    JSON.stringify([
+      "interactive-discrete",
+      "control-change",
+      invalidates,
+      ["main"],
+      ["control-count"],
+    ]),
+  )}`;
+
+function freeze(value) {
+  if (value && typeof value === "object" && !Object.isFrozen(value)) {
+    Object.values(value).forEach(freeze);
+    Object.freeze(value);
+  }
+  return value;
+}
+
+test("runs planned performance paths serially with exact per-path pass ids", async () => {
+  const sourceHash = hash("a");
+  const requestAuthorityHash = hash("b");
+  const pathIds = [
+    performancePathId([]),
+    performancePathId(["composite"]),
+  ].sort();
+  const testNames = pathIds.map(
+    (pathId) => `browser perf: toolcraft path ${pathId}`,
+  );
+  const selections = testNames.map((leafTitle) => ({
+    fullTitle: `app-performance.spec.ts › ${leafTitle}`,
+    grepTitle: `app-performance.spec.ts ${leafTitle}`,
+    leafTitle,
+  }));
+  const metrics = {
+    droppedFrameCount: 0,
+    droppedFrameRatio: 0,
+    durationMs: 10,
+    frameGapP50Ms: 10,
+    frameGapP95Ms: 16,
+    frameGapP99Ms: 16,
+    longTaskCount: 0,
+    longTaskMaxMs: 0,
+    maxFrameGapMs: 16,
+    sampleCount: 3,
+  };
+  const calls = [];
+  let active = 0;
+  let maximumActive = 0;
+  const operations = freeze({
+    createNonce: (() => {
+      let index = 0;
+      return () => `nonce-${index += 1}`;
+    })(),
+    readReport: async (_reportPath, expected) =>
+      createToolcraftTargetedPerformanceReport({
+        fixtureResolutionMode: expected.fixtureResolutionMode,
+        fixtureSelector: "development",
+        measurements: ["cold", "warm", "sustained"].map((phase) => ({
+          evidenceType: "performance-measurement-metrics",
+          kind: "interaction",
+          metrics,
+          pathId: expected.performancePathIds[0],
+          phase,
+          profile: "interactive-discrete",
+          profileCatalogVersion: 1,
+          version: 1,
+        })),
+        nonce: expected.nonce,
+        performancePassIds: expected.performancePassIds,
+        performancePathIds: expected.performancePathIds,
+        requestAuthorityHash,
+        sourceHash,
+        testNames: expected.testNames,
+      }),
+    removeReport: async () => {},
+    runPlaywright: async ({ pathId, performancePassIds }) => {
+      calls.push({ pathId, performancePassIds });
+      active += 1;
+      maximumActive = Math.max(maximumActive, active);
+      await new Promise((resolve) => setImmediate(resolve));
+      active -= 1;
+    },
+  });
+  const result = await executeToolcraftTargetedPerformanceVerificationCore({
+    fixtureResolutionMode: "strict-development",
+    operations,
+    performanceComparison: { kind: "none" },
+    performancePassIds: ["composite"],
+    performancePathIds: pathIds,
+    performanceSelections: selections,
+    performanceTestNames: testNames,
+    playwrightBin: "/workspace/node_modules/.bin/playwright",
+    projectDir: "/workspace",
+    requestAuthorityHash,
+    sourceHash,
+  });
+
+  assert.deepEqual(
+    calls,
+    pathIds.map((pathId) => ({
+      pathId,
+      performancePassIds: pathId.includes(
+        encodeURIComponent(JSON.stringify(["composite"])),
+      )
+        ? ["composite"]
+        : [],
+    })),
+  );
+  assert.equal(maximumActive, 1);
+  assert.deepEqual(result.report.performancePathIds, pathIds);
+  assert.deepEqual(
+    result.report.measurements.map(({ pathId }) => pathId),
+    pathIds.flatMap((pathId) => [pathId, pathId, pathId]),
+  );
+  assert.deepEqual(
+    result.testEvidence,
+    selections.map(({ fullTitle, leafTitle }) => ({ fullTitle, leafTitle })),
+  );
+  assert.match(result.evidenceHash, /^[a-f0-9]{64}$/u);
+  assert.equal(Object.hasOwn(result, "reportHash"), false);
+});
+
+test("rejects a performance path outside the canonical runtime codec", async () => {
+  const pathId =
+    "performance-path:%5B%22unknown-profile%22%2C%22control-change%22%2C%5B%5D%2C%5B%22main%22%5D%2C%5B%5D%5D";
+  const testName = `browser perf: toolcraft path ${pathId}`;
+  const operations = freeze({
+    createNonce: () => "unused",
+    readReport: async () => {},
+    removeReport: async () => {},
+    runPlaywright: async () => {},
+  });
+
+  await assert.rejects(
+    executeToolcraftTargetedPerformanceVerificationCore({
+      fixtureResolutionMode: "strict-development",
+      operations,
+      performanceComparison: { kind: "none" },
+      performancePassIds: [],
+      performancePathIds: [pathId],
+      performanceSelections: [{
+        fullTitle: `app-performance.spec.ts › ${testName}`,
+        grepTitle: `app-performance.spec.ts ${testName}`,
+        leafTitle: testName,
+      }],
+      performanceTestNames: [testName],
+      playwrightBin: "/workspace/node_modules/.bin/playwright",
+      projectDir: "/workspace",
+      requestAuthorityHash: hash("b"),
+      sourceHash: hash("a"),
+    }),
+    /planned targeted performance execution input is malformed/iu,
+  );
+});
+
+test("rejects requestless targeted execution before invoking operations", async () => {
+  const pathId = performancePathId([]);
+  const testName = `browser perf: toolcraft path ${pathId}`;
+  const calls = [];
+  const operations = freeze({
+    createNonce: () => {
+      calls.push("createNonce");
+      return "unused";
+    },
+    readReport: async () => {
+      calls.push("readReport");
+    },
+    removeReport: async () => {
+      calls.push("removeReport");
+    },
+    runPlaywright: async () => {
+      calls.push("runPlaywright");
+    },
+  });
+  const base = {
+    operations,
+    performanceComparison: { kind: "none" },
+    performancePassIds: [],
+    performancePathIds: [pathId],
+    performanceSelections: [
+      {
+        fullTitle: `app-performance.spec.ts › ${testName}`,
+        grepTitle: `app-performance.spec.ts ${testName}`,
+        leafTitle: testName,
+      },
+    ],
+    performanceTestNames: [testName],
+    playwrightBin: "/workspace/node_modules/.bin/playwright",
+    projectDir: "/workspace",
+    sourceHash: hash("a"),
+  };
+
+  for (const invalid of [
+    { fixtureResolutionMode: "strict-development" },
+    {
+      fixtureResolutionMode: "strict-development",
+      requestAuthorityHash: null,
+    },
+    {
+      fixtureResolutionMode: "default",
+      requestAuthorityHash: hash("b"),
+    },
+  ]) {
+    await assert.rejects(
+      executeToolcraftTargetedPerformanceVerificationCore({
+        ...base,
+        ...invalid,
+      }),
+      /planned targeted performance execution input is malformed/iu,
+    );
+  }
+  assert.deepEqual(calls, []);
+});

package/templates/starter/scripts/toolcraft-targeted-performance-report-combination.mjs

@@ -0,0 +1,83 @@
+import {
+  TOOLCRAFT_TARGETED_PERFORMANCE_REPORT_VERSION,
+  createToolcraftTargetedPerformanceReport,
+  getToolcraftTargetedPerformanceReportError,
+} from "./toolcraft-targeted-performance-report.mjs";
+
+const compare = (left, right) =>
+  left < right ? -1 : left > right ? 1 : 0;
+
+function isCanonicalTargets(value, { allowEmpty = false } = {}) {
+  return (
+    Array.isArray(value) &&
+    (allowEmpty || value.length > 0) &&
+    value.every(
+      (item) =>
+        typeof item === "string" &&
+        item.length > 0 &&
+        item.trim() === item,
+    ) &&
+    value.every(
+      (item, index) => index === 0 || compare(value[index - 1], item) < 0,
+    )
+  );
+}
+
+export function combineToolcraftTargetedPerformanceReports({
+  nonce,
+  performancePassIds,
+  reports,
+  requestAuthorityHash,
+  sourceHash,
+}) {
+  const canonicalPassIds = Array.isArray(performancePassIds)
+    ? [...performancePassIds].sort(compare)
+    : [];
+  if (
+    !isCanonicalTargets(canonicalPassIds, { allowEmpty: true }) ||
+    !Array.isArray(reports) ||
+    reports.length === 0 ||
+    reports.some(
+      (report) =>
+        getToolcraftTargetedPerformanceReportError(report) ||
+        report.version !== TOOLCRAFT_TARGETED_PERFORMANCE_REPORT_VERSION ||
+        report.performancePathIds.length !== 1 ||
+        report.testNames.length !== 1 ||
+        report.fixtureResolutionMode !== reports[0].fixtureResolutionMode ||
+        report.fixtureSelector !== reports[0].fixtureSelector ||
+        report.requestAuthorityHash !== requestAuthorityHash ||
+        report.sourceHash !== sourceHash ||
+        !isCanonicalTargets(report.performancePassIds, { allowEmpty: true }),
+    )
+  ) {
+    throw new Error(
+      "Toolcraft serial targeted performance reports are malformed or inconsistent.",
+    );
+  }
+  const pathIds = reports.map((report) => report.performancePathIds[0]);
+  const testNames = reports.map((report) => report.testNames[0]);
+  const reportPassIds = [
+    ...new Set(reports.flatMap((report) => report.performancePassIds)),
+  ].sort(compare);
+  if (!isCanonicalTargets(pathIds) || !isCanonicalTargets(testNames)) {
+    throw new Error(
+      "Toolcraft serial targeted performance reports are not in canonical planned order.",
+    );
+  }
+  if (JSON.stringify(reportPassIds) !== JSON.stringify(canonicalPassIds)) {
+    throw new Error(
+      "Toolcraft serial targeted performance reports are malformed or inconsistent with the canonical planned pass union.",
+    );
+  }
+  return createToolcraftTargetedPerformanceReport({
+    fixtureResolutionMode: reports[0].fixtureResolutionMode,
+    fixtureSelector: reports[0].fixtureSelector,
+    measurements: reports.flatMap((report) => report.measurements),
+    nonce,
+    performancePassIds: canonicalPassIds,
+    performancePathIds: pathIds,
+    requestAuthorityHash,
+    sourceHash,
+    testNames,
+  });
+}

package/templates/starter/scripts/toolcraft-targeted-performance-report-combination.test.mjs

@@ -0,0 +1,131 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  combineToolcraftTargetedPerformanceReports,
+} from "./toolcraft-targeted-performance-report-combination.mjs";
+import {
+  createToolcraftTargetedPerformanceReport,
+} from "./toolcraft-targeted-performance-report.mjs";
+
+const sourceHash = "a".repeat(64);
+const requestAuthorityHash = "b".repeat(64);
+const metrics = {
+  droppedFrameCount: 0,
+  droppedFrameRatio: 0,
+  durationMs: 10,
+  frameGapP50Ms: 10,
+  frameGapP95Ms: 16,
+  frameGapP99Ms: 16,
+  longTaskCount: 0,
+  longTaskMaxMs: 0,
+  maxFrameGapMs: 16,
+  sampleCount: 3,
+};
+
+function report(pathId, testName, performancePassIds = ["composite"]) {
+  return createToolcraftTargetedPerformanceReport({
+    fixtureResolutionMode: "strict-development",
+    fixtureSelector: "development",
+    measurements: ["cold", "warm", "sustained"].map((phase) => ({
+      evidenceType: "performance-measurement-metrics",
+      kind: "interaction",
+      metrics,
+      pathId,
+      phase,
+      profile: "interactive-discrete",
+      profileCatalogVersion: 1,
+      version: 1,
+    })),
+    nonce: `nonce-${pathId}`,
+    performancePassIds,
+    performancePathIds: [pathId],
+    requestAuthorityHash,
+    sourceHash,
+    testNames: [testName],
+  });
+}
+
+test("combines validated single-path reports in canonical planned order", () => {
+  const first = report(
+    "control-drag:composite",
+    "browser perf: control-drag composite",
+  );
+  const second = report(
+    "control-drag:overlay",
+    "browser perf: control-drag overlay",
+    ["overlay"],
+  );
+  const combined = combineToolcraftTargetedPerformanceReports({
+    nonce: "combined-nonce",
+    performancePassIds: ["composite", "overlay"],
+    reports: [first, second],
+    requestAuthorityHash,
+    sourceHash,
+  });
+
+  assert.deepEqual(combined.performancePathIds, [
+    "control-drag:composite",
+    "control-drag:overlay",
+  ]);
+  assert.deepEqual(combined.testNames, [
+    "browser perf: control-drag composite",
+    "browser perf: control-drag overlay",
+  ]);
+  assert.equal(combined.measurements.length, 6);
+  assert.deepEqual(combined.performancePassIds, ["composite", "overlay"]);
+  assert.throws(
+    () =>
+      combineToolcraftTargetedPerformanceReports({
+        nonce: "combined-nonce",
+        performancePassIds: ["composite", "overlay"],
+        reports: [second, first],
+        requestAuthorityHash,
+        sourceHash,
+      }),
+    /canonical planned order/iu,
+  );
+});
+
+test("combines passless and pass-bearing single-path reports into the exact union", () => {
+  const first = report(
+    "control-change:passless",
+    "browser perf: control-change passless",
+    [],
+  );
+  const second = report(
+    "control-drag:composite",
+    "browser perf: control-drag composite",
+  );
+  const combined = combineToolcraftTargetedPerformanceReports({
+    nonce: "combined-nonce",
+    performancePassIds: ["composite"],
+    reports: [first, second],
+    requestAuthorityHash,
+    sourceHash,
+  });
+
+  assert.deepEqual(combined.performancePassIds, ["composite"]);
+  assert.deepEqual(combined.performancePathIds, [
+    "control-change:passless",
+    "control-drag:composite",
+  ]);
+});
+
+test("rejects inconsistent single-path report inputs", () => {
+  const first = report(
+    "control-drag:composite",
+    "browser perf: control-drag composite",
+  );
+  assert.throws(
+    () =>
+      combineToolcraftTargetedPerformanceReports({
+        nonce: "combined-nonce",
+        performancePassIds: ["overlay"],
+        reports: [first],
+        requestAuthorityHash,
+        sourceHash,
+      }),
+    /malformed or inconsistent/iu,
+  );
+});