@zaip-ai/cli 6.2.0 → 7.5.0

package/templates/.agents/skills/zaip-skill-performance-engineering/scripts/performance-report.test.mjs

@@ -0,0 +1,122 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { captureSite } from "./capture-site.mjs";
+import { compareRuns, evaluatePerformance } from "./performance-report.mjs";
+
+const budgets = JSON.parse(
+  await readFile(new URL("../references/site-budgets.json", import.meta.url), "utf8"),
+);
+const conditions = {
+  tool: "Chromium CDP",
+  tool_version: "test",
+  commit: "a".repeat(40),
+  viewport: { width: 390, height: 844 },
+  dpr: 1,
+  cpu: "unthrottled",
+  network: "local",
+  cache: "cold",
+  url: "http://127.0.0.1/",
+  date: "2026-10-21",
+};
+test("network capture cannot establish Lighthouse or field CWV approval", () => {
+  const r = evaluatePerformance(
+    { conditions, bytes: { initial: 10, full: 20, images: 5, javascript: 2, css: 1 } },
+    budgets,
+  );
+  assert.equal(r.measured_checks_passed, true);
+  assert.equal(r.full_approval, false);
+  assert.equal(r.unavailable.length, 2);
+});
+test("invalid conditions, negative metrics and scores over 100 never approve", () => {
+  const r = evaluatePerformance(
+    {
+      conditions: Object.fromEntries(Object.keys(conditions).map((k) => [k, ""])),
+      bytes: { initial: -1, full: -1, images: -1, javascript: -1, css: -1 },
+      lighthouse: {
+        tool: "Lighthouse",
+        version: "test",
+        raw_report: "fixture",
+        scores: { performance: 1000, seo: 1000, accessibility: 1000, "best-practices": 1000 },
+      },
+      field: {
+        source: "fixture",
+        period: "test",
+        percentile: 75,
+        device: "mobile",
+        lcp_ms: -1,
+        inp_ms: -1,
+        cls: -1,
+      },
+    },
+    budgets,
+  );
+  assert.equal(r.full_approval, false);
+  assert.ok(r.findings.includes("invalid:bytes:initial"));
+  assert.ok(r.findings.includes("invalid:lighthouse:performance"));
+  assert.ok(r.findings.includes("conditions-invalid:commit"));
+});
+test("budgets and heavy/or opposite viewport images fail independently", () => {
+  for (const mutation of [
+    { bytes: { initial: 2e6 } },
+    { originalHeavyImagesRequested: ["original.png"] },
+    { oppositeViewportImagesRequested: ["desktop.webp"] },
+  ]) {
+    const r = evaluatePerformance({ conditions, ...mutation }, budgets);
+    assert.equal(r.measured_checks_passed, false);
+  }
+});
+test("TBT cannot substitute for INP and unproven Lighthouse provenance fails", () => {
+  const r = evaluatePerformance(
+    {
+      conditions,
+      field: {
+        percentile: 75,
+        source: "fixture",
+        period: "test",
+        device: "mobile",
+        lcp_ms: 2000,
+        tbt_ms: 20,
+        cls: 0,
+      },
+      lighthouse: { tool: "CDP", scores: { performance: 100 } },
+    },
+    budgets,
+  );
+  assert.ok(r.unavailable.includes("field:inp_ms"));
+  assert.ok(r.findings.includes("lighthouse:provenance"));
+});
+test("equivalent runs use medians and separate initial/full transfer", () => {
+  const a = {
+    conditions,
+    samples: [
+      { initial: 3, full: 6 },
+      { initial: 1, full: 2 },
+      { initial: 2, full: 4 },
+    ],
+  };
+  const b = {
+    conditions: { ...conditions, commit: "b".repeat(40) },
+    samples: [
+      { initial: 1, full: 2 },
+      { initial: 1, full: 2 },
+      { initial: 5, full: 8 },
+    ],
+  };
+  assert.deepEqual(compareRuns(a, b).initial, { before: 2, after: 1 });
+  assert.throws(
+    () => compareRuns(a, { ...b, conditions: { ...conditions, dpr: 2 } }),
+    /Non-equivalent/,
+  );
+  assert.throws(() => compareRuns(a, { ...b, samples: [{ initial: -1, full: 2 }] }), /nonnegative/);
+  assert.throws(
+    () => compareRuns(a, { ...b, conditions: { ...conditions, commit: "unknown" } }),
+    /exact commit/,
+  );
+});
+test("browser capture rejects unbound candidate before launching browser", async () => {
+  await assert.rejects(
+    captureSite({ url: "http://localhost/", outputDir: "unused", commit: "working-tree-fixture" }),
+    /exact candidate Git SHA/,
+  );
+});

package/templates/.agents/skills/zaip-skill-performance-engineering/scripts/report-fixtures.test-support.mjs

@@ -0,0 +1,51 @@
+import { importReport } from "./report-import.mjs";
+export const context = {
+  candidate_sha: "a".repeat(40),
+  scenario: "representative",
+  consent: "denied",
+  tags: ["consent-manager"],
+  cache: "cold",
+  external_policy: "authorized-import",
+};
+export const budgets = {
+  lab: { lcp_ms: 2500, tbt_ms: 200, cls: 0.1, js_cpu_ms: 1000, bytes: 1000000 },
+  scores: { performance: 80 },
+  field_p75: { lcp_ms: 2500, inp_ms: 200, cls: 0.1 },
+  max_variation_ratio: 0.2,
+  regression_ratio: 0.1,
+};
+// Synthetic LHR-shaped fixture. It proves parser behavior, never a real browser measurement.
+export function raw(index = 0, overrides = {}) {
+  return {
+    lighthouseVersion: "13.6.0",
+    fetchTime: `2026-10-29T12:00:0${index}.000Z`,
+    requestedUrl: "https://example.test/",
+    finalDisplayedUrl: "https://example.test/",
+    userAgent: "Synthetic Chrome",
+    configSettings: {
+      formFactor: "mobile",
+      screenEmulation: { mobile: true, width: 390, height: 844, deviceScaleFactor: 1 },
+      throttlingMethod: "simulate",
+      throttling: { cpuSlowdownMultiplier: 4 },
+      disableStorageReset: false,
+    },
+    categories: {
+      performance: { score: 0.95 },
+      accessibility: { score: 1 },
+      seo: { score: 1 },
+      "best-practices": { score: 1 },
+    },
+    audits: {
+      "largest-contentful-paint": { numericValue: 1500 },
+      "total-blocking-time": { numericValue: 50 },
+      "cumulative-layout-shift": { numericValue: 0.01 },
+      "bootup-time": { numericValue: 300 },
+      "total-byte-weight": { numericValue: 100000 },
+    },
+    ...overrides,
+  };
+}
+export const series = (candidate = "a", count = 3) =>
+  Array.from({ length: count }, (_, i) =>
+    importReport(raw(i), { ...context, candidate_sha: candidate.repeat(40) }),
+  );

package/templates/.agents/skills/zaip-skill-performance-engineering/scripts/report-import.mjs

@@ -0,0 +1,209 @@
+import { createHash } from "node:crypto";
+import { lstat, readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+export const digest = (value) => createHash("sha256").update(value).digest("hex");
+export function safeUrl(value) {
+  const url = new URL(value);
+  if (
+    !["http:", "https:"].includes(url.protocol) ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash
+  )
+    throw new Error("report_url_invalid");
+  return url.href;
+}
+export async function readBoundedJson(root, path, maxBytes = 8 * 1024 * 1024) {
+  if (
+    typeof path !== "string" ||
+    !path ||
+    isAbsolute(path) ||
+    /(^|[\\/])\.\.([\\/]|$)/.test(path) ||
+    /^[a-z]:/i.test(path)
+  )
+    throw new Error("report_path_invalid");
+  const absolute = resolve(root, path),
+    rel = relative(resolve(root), absolute);
+  if (!rel || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new Error("report_path_invalid");
+  let current = resolve(root);
+  if ((await lstat(current)).isSymbolicLink()) throw new Error("report_link_forbidden");
+  for (const part of rel.split(sep)) {
+    current = resolve(current, part);
+    if ((await lstat(current)).isSymbolicLink()) throw new Error("report_link_forbidden");
+  }
+  const info = await lstat(absolute);
+  if (!info.isFile() || info.size > maxBytes) throw new Error("report_size_invalid");
+  const bytes = await readFile(absolute);
+  if (bytes.length > maxBytes) throw new Error("report_size_invalid");
+  return { value: JSON.parse(bytes.toString("utf8")), sha256: digest(bytes) };
+}
+const number = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
+const nonempty = (value) => typeof value === "string" && value.length > 0 && value.length <= 256;
+const label = (value) => typeof value === "string" && /^[a-z0-9_.-]{1,64}$/i.test(value);
+const canonical = (value) =>
+  JSON.stringify(value, (_, item) =>
+    item && typeof item === "object" && !Array.isArray(item)
+      ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b)))
+      : item,
+  );
+
+export function importReport(raw, context) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw))
+    throw new Error("report_object_required");
+  if (
+    !context ||
+    !/^[a-f0-9]{40}$/i.test(context.candidate_sha ?? "") ||
+    !["isolated", "representative"].includes(context.scenario) ||
+    !label(context.consent) ||
+    !Array.isArray(context.tags) ||
+    context.tags.length > 64 ||
+    context.tags.some((tag) => !label(tag)) ||
+    !label(context.cache) ||
+    !["blocked", "authorized-import"].includes(context.external_policy)
+  )
+    throw new Error("report_context_required");
+  const lhr = raw.lighthouseResult ?? raw;
+  if (!nonempty(lhr.lighthouseVersion))
+    return {
+      status: "unsupported",
+      provider: ["GTmetrix", "Pingdom"].includes(context.provider) ? context.provider : "unknown",
+      reason: "manual_review_required",
+      authenticity: "not_authenticated",
+    };
+  if (!/^\d+\.\d+\.\d+(?:-[a-z0-9.-]+)?$/i.test(lhr.lighthouseVersion))
+    throw new Error("lighthouse_version_invalid");
+  if (lhr.runtimeError) throw new Error("lighthouse_runtime_error");
+  if (
+    !Number.isFinite(Date.parse(lhr.fetchTime)) ||
+    !nonempty(lhr.userAgent) ||
+    !lhr.configSettings ||
+    !lhr.audits ||
+    !lhr.categories
+  )
+    throw new Error("lighthouse_structure_invalid");
+  const settings = lhr.configSettings;
+  if (settings.extraHeaders && Object.keys(settings.extraHeaders).length)
+    throw new Error("sensitive_report_configuration");
+  if (
+    !["mobile", "desktop"].includes(settings.formFactor) ||
+    !settings.screenEmulation ||
+    !nonempty(settings.throttlingMethod) ||
+    typeof settings.throttling !== "object"
+  )
+    throw new Error("lighthouse_config_missing");
+  const url = safeUrl(lhr.finalDisplayedUrl ?? lhr.finalUrl),
+    requested = safeUrl(lhr.requestedUrl);
+  const metrics = {};
+  for (const [name, audit] of Object.entries({
+    fcp_ms: "first-contentful-paint",
+    speed_index_ms: "speed-index",
+    server_response_ms: "server-response-time",
+    lcp_ms: "largest-contentful-paint",
+    tbt_ms: "total-blocking-time",
+    cls: "cumulative-layout-shift",
+    js_cpu_ms: "bootup-time",
+    bytes: "total-byte-weight",
+  })) {
+    const value = lhr.audits[audit]?.numericValue;
+    metrics[name] = number(value) ? value : null;
+  }
+  const scores = {};
+  for (const name of ["performance", "accessibility", "best-practices", "seo"]) {
+    const value = lhr.categories[name]?.score;
+    if (value !== null && value !== undefined && (!number(value) || value > 1))
+      throw new Error("lighthouse_score_invalid");
+    scores[name] = number(value) ? value * 100 : null;
+  }
+  const comparison = {
+    settings_sha256: digest(canonical(settings)),
+    user_agent_sha256: digest(String(lhr.userAgent ?? "")),
+    environment_sha256: digest(
+      canonical({
+        host: lhr.environment?.hostUserAgent ?? null,
+        network: lhr.environment?.networkUserAgent ?? null,
+      }),
+    ),
+    scenario: context.scenario,
+    consent: context.consent,
+    tags: [...context.tags].sort(),
+    cache: context.cache,
+    external_policy: context.external_policy,
+    version: lhr.lighthouseVersion,
+    url,
+    requested,
+  };
+  const field = [];
+  for (const [scope, experience] of [
+    ["url", raw.loadingExperience],
+    ["origin", raw.originLoadingExperience],
+  ]) {
+    if (!experience?.metrics || !experience.id) continue;
+    let fieldUrl = url;
+    if (requested !== url) {
+      const redirect = context.redirect;
+      if (
+        !redirect ||
+        !["requested", "final"].includes(redirect.field_url) ||
+        safeUrl(redirect.requested_url) !== requested ||
+        safeUrl(redirect.final_url) !== url
+      )
+        throw new Error("field_redirect_mapping_required");
+      fieldUrl = redirect.field_url === "requested" ? requested : url;
+    }
+    const identity = safeUrl(experience.id);
+    const expected = scope === "url" ? fieldUrl : `${new URL(fieldUrl).origin}/`;
+    if (identity !== expected) throw new Error("field_identity_mismatch");
+    const values = experience.metrics;
+    const get = (key, divisor = 1) =>
+      number(values[key]?.percentile) ? values[key].percentile / divisor : null;
+    const period = context.field_period
+      ? Object.fromEntries(
+          ["start", "end"].map((key) => [
+            key,
+            typeof context.field_period[key] === "string" &&
+            /^\d{4}-\d{2}-\d{2}$/.test(context.field_period[key])
+              ? context.field_period[key]
+              : null,
+          ]),
+        )
+      : null;
+    field.push({
+      source: "PSI-CrUX",
+      scope,
+      url: identity,
+      percentile: 75,
+      device: settings.formFactor,
+      period,
+      period_provenance: "caller_declared_not_authenticated",
+      fcp_ms: get("FIRST_CONTENTFUL_PAINT_MS"),
+      ttfb_ms: get("EXPERIMENTAL_TIME_TO_FIRST_BYTE"),
+      lcp_ms: get("LARGEST_CONTENTFUL_PAINT_MS"),
+      inp_ms: get("INTERACTION_TO_NEXT_PAINT"),
+      cls: get("CUMULATIVE_LAYOUT_SHIFT_SCORE", 100),
+    });
+  }
+  return {
+    status: "imported",
+    provider: raw.lighthouseResult ? "PSI" : "Lighthouse",
+    raw_sha256: digest(canonical(raw)),
+    semantic_sample_sha256: digest(canonical(lhr)),
+    authenticity: "not_authenticated",
+    candidate_sha: context.candidate_sha,
+    fetched_at: lhr.fetchTime,
+    url,
+    requested_url: requested,
+    tool_version: lhr.lighthouseVersion,
+    device: settings.formFactor,
+    host_browser:
+      /(?:HeadlessChrome|Chrome|Chromium)\/[\d.]+/.exec(
+        lhr.environment?.hostUserAgent ?? lhr.userAgent ?? "",
+      )?.[0] ?? null,
+    config_sha256: digest(canonical(comparison)),
+    comparison,
+    metrics,
+    scores,
+    field,
+  };
+}

package/templates/.agents/skills/zaip-skill-performance-engineering/scripts/report-import.test.mjs

@@ -0,0 +1,164 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { context, raw } from "./report-fixtures.test-support.mjs";
+import { importReport, safeUrl } from "./report-import.mjs";
+
+test("diagnostics preserve numeric units and never treat server response as full TTFB", () => {
+  const report = raw();
+  report.audits["first-contentful-paint"] = { numericValue: 1100 };
+  report.audits["speed-index"] = { numericValue: 1700 };
+  report.audits["server-response-time"] = { numericValue: 100 };
+  const result = importReport(report, context);
+  assert.equal(result.metrics.fcp_ms, 1100);
+  assert.equal(result.metrics.speed_index_ms, 1700);
+  assert.equal(result.metrics.server_response_ms, 100);
+  assert.equal(result.metrics.ttfb_ms, undefined);
+  for (const value of [undefined, null, "1100", -1, Infinity, NaN]) {
+    report.audits["first-contentful-paint"] = { numericValue: value, displayValue: "1.1 s" };
+    assert.equal(importReport(report, context).metrics.fcp_ms, null);
+  }
+  assert.equal(importReport(raw(), context).metrics.speed_index_ms, null);
+  assert.equal(importReport(raw(), context).metrics.server_response_ms, null);
+});
+
+test("PSI field FCP and TTFB remain p75 diagnostics distinct from lab server response", () => {
+  const lhr = raw();
+  lhr.audits["server-response-time"] = { numericValue: 100 };
+  const report = {
+    lighthouseResult: lhr,
+    loadingExperience: {
+      id: "https://example.test/",
+      metrics: {
+        FIRST_CONTENTFUL_PAINT_MS: { percentile: 1600 },
+        EXPERIMENTAL_TIME_TO_FIRST_BYTE: { percentile: 700 },
+      },
+    },
+  };
+  const result = importReport(report, context);
+  assert.equal(result.metrics.server_response_ms, 100);
+  assert.equal(result.field[0].ttfb_ms, 700);
+  assert.equal(result.field[0].fcp_ms, 1600);
+  assert.equal(result.field[0].percentile, 75);
+  assert.equal(result.field[0].source, "PSI-CrUX");
+  report.loadingExperience.metrics = {};
+  assert.equal(importReport(report, context).field[0].ttfb_ms, null);
+  assert.equal(importReport(report, context).field[0].fcp_ms, null);
+});
+
+test("Lighthouse numeric audits and category fractions are parsed from the real report structure", () => {
+  const result = importReport(raw(), context);
+  assert.equal(result.metrics.lcp_ms, 1500);
+  assert.equal(result.scores.performance, 95);
+  assert.equal(result.metrics.inp_ms, undefined);
+  assert.equal(result.authenticity, "not_authenticated");
+  assert.equal(result.raw_sha256.length, 64);
+});
+test("PSI wrapper retains distinct URL and origin CrUX p75, not TBT as INP", () => {
+  const metric = {
+    LARGEST_CONTENTFUL_PAINT_MS: { percentile: 2200 },
+    CUMULATIVE_LAYOUT_SHIFT_SCORE: { percentile: 6 },
+  };
+  const result = importReport(
+    {
+      lighthouseResult: raw(),
+      loadingExperience: { id: "https://example.test/", metrics: metric },
+      originLoadingExperience: {
+        id: "https://example.test",
+        metrics: { ...metric, INTERACTION_TO_NEXT_PAINT: { percentile: 180 } },
+      },
+    },
+    { ...context, field_period: { start: "2026-09-01", end: "2026-09-28" } },
+  );
+  assert.equal(result.provider, "PSI");
+  assert.equal(result.field[0].inp_ms, null);
+  assert.equal(result.field[0].cls, 0.06);
+  assert.equal(result.field[1].inp_ms, 180);
+  assert.deepEqual(
+    result.field.map((f) => f.scope),
+    ["url", "origin"],
+  );
+});
+test("declared scores and raw_report string cannot stand in for an LHR", () => {
+  for (const report of [
+    { scores: { performance: 100 }, raw_report: "report.json" },
+    { gtmetrix: { grade: "A" } },
+    { pingdom: { grade: 99 } },
+  ])
+    assert.equal(importReport(report, context).status, "unsupported");
+  assert.throws(() => importReport(raw(0, { lighthouseVersion: "13", audits: null }), context));
+  assert.throws(() =>
+    importReport(raw(0, { runtimeError: { code: "FAILED_DOCUMENT_REQUEST" } }), context),
+  );
+});
+test("invalid scores, sensitive URLs and absent context fail closed", () => {
+  const invalid = raw();
+  invalid.categories.performance.score = 100;
+  assert.throws(() => importReport(invalid, context));
+  for (const url of [
+    "https://user:secret@example.test/",
+    "https://example.test/?email=private",
+    "file:///secret",
+    "https://example.test/#token",
+  ])
+    assert.throws(() => safeUrl(url));
+  assert.throws(() => importReport(raw(), {}));
+});
+test("review regression: CrUX URL and origin must belong to the audited route", () => {
+  const metrics = { LARGEST_CONTENTFUL_PAINT_MS: { percentile: 2000 } };
+  for (const field of [
+    { loadingExperience: { id: "https://other.test/", metrics } },
+    { loadingExperience: { id: "https://example.test/another-route", metrics } },
+    { originLoadingExperience: { id: "https://other.test", metrics } },
+    { originLoadingExperience: { id: "https://example.test/path", metrics } },
+  ])
+    assert.throws(
+      () => importReport({ lighthouseResult: raw(), ...field }, context),
+      /field_identity_mismatch/,
+    );
+});
+test("review regression: field redirects need exact explicit mapping, without blocking lab-only redirects", () => {
+  const lhr = raw(0, {
+    requestedUrl: "http://example.test/old",
+    finalDisplayedUrl: "https://example.test/new",
+  });
+  assert.equal(importReport(lhr, context).status, "imported");
+  const report = {
+    lighthouseResult: lhr,
+    loadingExperience: {
+      id: "http://example.test/old",
+      metrics: { LARGEST_CONTENTFUL_PAINT_MS: { percentile: 2000 } },
+    },
+  };
+  assert.throws(() => importReport(report, context), /field_redirect_mapping_required/);
+  const redirect = {
+    requested_url: lhr.requestedUrl,
+    final_url: lhr.finalDisplayedUrl,
+    field_url: "requested",
+  };
+  assert.equal(importReport(report, { ...context, redirect }).field[0].url, lhr.requestedUrl);
+  const finalReport = {
+    ...report,
+    loadingExperience: { ...report.loadingExperience, id: lhr.finalDisplayedUrl },
+    originLoadingExperience: {
+      id: "https://example.test",
+      metrics: report.loadingExperience.metrics,
+    },
+  };
+  assert.deepEqual(
+    importReport(finalReport, {
+      ...context,
+      redirect: { ...redirect, field_url: "final" },
+    }).field.map((field) => field.url),
+    [lhr.finalDisplayedUrl, "https://example.test/"],
+  );
+  assert.throws(
+    () => importReport(finalReport, { ...context, redirect }),
+    /field_identity_mismatch/,
+  );
+  assert.throws(() =>
+    importReport(report, {
+      ...context,
+      redirect: { ...redirect, final_url: "https://other.test/" },
+    }),
+  );
+});