@analitikisler/ai-tag-debugger 0.15.0 → 0.17.0

package/src/ci-output.js

@@ -0,0 +1,104 @@
+// Machine-readable outputs for CI: junit.xml, which GitHub Actions, GitLab and Jenkins show as test
+// results, and summary.json, a small file a script can read without parsing capture.json.
+import path from "node:path";
+import { groupFindings, summarize } from "./checks.js";
+import { messages } from "./i18n.js";
+import { platformLabel } from "./report.js";
+
+// The same order the CLI uses for --fail-on: a finding fails when its severity is at or above the level.
+const LEVELS = ["broken", "risk", "warning", "info"];
+
+/** Whether a finding counts as a failure under --fail-on. Accepted (baselined) findings never do. */
+export function failsAt(finding, failOn = "broken") {
+  if (failOn === "never" || finding.accepted === true) return false;
+  const level = LEVELS.indexOf(finding.severity);
+  return level !== -1 && level <= LEVELS.indexOf(failOn);
+}
+
+/** How long a run took in seconds, from its start to its last step or hit; 0 when the run has no timing. */
+export function runSeconds(run) {
+  if (typeof run?.startedAt !== "number") return 0;
+  const last = Math.max(...[...(run.steps ?? []), ...(run.hits ?? [])].map((x) => x?.t ?? 0));
+  return last > run.startedAt ? Math.round((last - run.startedAt) / 10) / 100 : 0;
+}
+
+/** The one-line name of a finding: platform and event when it has them, then the message. */
+export function findingTitle(f) {
+  const what = [f.platform ? platformLabel(f.platform) : "", f.event ?? ""].filter(Boolean).join(" ");
+  const line = what ? `${what}: ${f.detail}` : f.detail ?? f.check;
+  return line.length > 120 ? `${line.slice(0, 119).trimEnd()}…` : line;
+}
+
+const escapeXml = (s) => String(s ?? "").replace(/[<>&"']/g, (c) => ({ "<": "&lt;", ">": "&gt;", "&": "&amp;", '"': "&quot;", "'": "&apos;" })[c]);
+// Control characters are not allowed in XML 1.0 at all, so they are dropped rather than escaped.
+// eslint-disable-next-line no-control-regex
+const xmlText = (s) => escapeXml(String(s ?? "").replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""));
+const attrs = (o) => Object.entries(o).map(([k, v]) => ` ${k}="${xmlText(v)}"`).join("");
+
+const belongsTo = (f, run) => f.journey === run.journeyId && f.viewport === run.viewport;
+
+// A case is { xml, failed }: a failure carries the detail in its body, a pass keeps it in system-out.
+const testcase = ({ classname, name, time = 0, failed, type, text, message = name }) => ({
+  failed,
+  xml: `<testcase${attrs({ classname, name, time })}>${failed ? `<failure${attrs({ message, type })}>${xmlText(text)}</failure>` : text ? `<system-out>${xmlText(text)}</system-out>` : ""}</testcase>`,
+});
+
+const findingCase = (f, failOn) => testcase({ classname: f.check, name: findingTitle(f), failed: failsAt(f, failOn), type: f.severity, text: f.detail });
+
+function suite(name, cases, seconds) {
+  const failures = cases.filter((c) => c.failed).length;
+  return { failures, tests: cases.length, xml: `  <testsuite${attrs({ name, tests: cases.length, failures, errors: 0, skipped: 0, time: seconds })}>\n${cases.map((c) => `    ${c.xml}`).join("\n")}\n  </testsuite>` };
+}
+
+/**
+ * JUnit XML: one suite per journey and viewport, one case per finding, plus a "journey completed" case
+ * that fails when the journey did not reach its goal. Findings not tied to a run go in a suite of their own.
+ */
+export function renderJunit({ runs = [], findings = [], lang = "en", failOn = "broken", version = "", site = "" }) {
+  const m = messages(lang).ui;
+  const suites = runs.map((run) => {
+    // With --fail-on never the CLI exits 0 whatever happened, so the JUnit file must not fail the job on its own.
+    const journey = testcase({ classname: "journey", name: m.junitJourney, time: runSeconds(run), failed: failOn !== "never" && run.completed === false, type: "incomplete", text: run.note, message: run.note ?? "" });
+    return suite(`${run.journeyId} (${run.viewport})`, [journey, ...findings.filter((f) => belongsTo(f, run)).map((f) => findingCase(f, failOn))], runSeconds(run));
+  });
+  const loose = findings.filter((f) => !runs.some((run) => belongsTo(f, run)));
+  if (loose.length) suites.push(suite(m.junitSite, loose.map((f) => findingCase(f, failOn)), 0));
+  const totals = { tests: suites.reduce((n, s) => n + s.tests, 0), failures: suites.reduce((n, s) => n + s.failures, 0), time: runs.reduce((n, r) => n + runSeconds(r), 0) };
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites${attrs({ name: `ai-tag-debugger ${site}`.trim(), ...(version ? { version } : {}), ...totals })}>\n${suites.map((s) => s.xml).join("\n")}\n</testsuites>\n`;
+}
+
+/** The run in one small object: counts, journeys, findings and where the other files are. */
+export function buildSummary({ runs = [], findings = [], counts, plan = null, against = null, ai = null, lang = "en", reportPath, outDir, changes = null, version = "", site = "", metrics = null, trend = null }) {
+  // Only the run folder's name and file names relative to it: a summary uploaded as a CI artifact must not carry the runner's paths.
+  const dir = outDir ?? (reportPath ? path.dirname(reportPath) : ".");
+  const file = (name) => name;
+  return {
+    version,
+    site,
+    generatedAt: new Date().toISOString(),
+    lang,
+    ai: ai ? { name: ai.name, model: ai.model } : null,
+    plan: { id: against?.id ?? null, events: (plan?.events ?? []).map((e) => `${String(e.platform ?? "").toLowerCase()}:${e.name}`) },
+    journeys: runs.map((r) => ({ id: r.journeyId, viewport: r.viewport, completed: r.completed !== false, note: r.note ?? "", steps: r.steps?.length ?? 0, hits: r.hits?.length ?? 0 })),
+    counts: counts ?? summarize(groupFindings(findings)),
+    // The deterministic verdict (see src/metrics.js): its status, the headline key, and the data quality score.
+    verdict: metrics ? { status: metrics.verdict.status, headline: metrics.verdict.headline } : null,
+    score: metrics?.score?.score ?? null,
+    trend: trend ? { direction: trend.direction, runs: trend.runs } : null,
+    findings: findings.map((f) => ({
+      check: f.check,
+      severity: f.severity,
+      platform: f.platform ?? null,
+      event: f.event ?? null,
+      journey: f.journey ?? null,
+      viewport: f.viewport ?? null,
+      title: findingTitle(f),
+      message: f.detail ?? "",
+      ...(f.platform && f.event ? { key: `${f.platform}:${f.event}` } : {}),
+      ...(f.accepted === true ? { accepted: true } : {}),
+    })),
+    changes: changes ? { fixed: changes.fixed ?? [], new: changes.new ?? [] } : null,
+    run: path.basename(dir),
+    files: { html: file("report.html"), md: file("report.md"), csv: file("timeline.csv"), capture: file("capture.json"), ...(against ? { planCheck: file("plan-check.json") } : {}), junit: file("junit.xml"), summary: file("summary.json") },
+  };
+}

package/src/cli.js

@@ -5,7 +5,9 @@ import { parseArgs } from "node:util";
 import { loadConfig, runAudit } from "./audit.js";
 import { messages } from "./i18n.js";
 import { helpText } from "./help.js";
+import { validatePlan } from "./schema.js";
 import { replaySavedSite, writeSavedSteps } from "./replay.js";
+import { readBaseline, writeBaseline } from "./baseline.js";
 import { savePlan } from "./plan.js";
 import { DEFAULT_DATA_DIR } from "./workspace.js";
 import { commandText, intentToArgs, isSentence, needsWhatToTest, translate } from "./ask.js";
@@ -25,6 +27,7 @@ async function main(args = process.argv.slice(2)) {
       out: { type: "string" },
       journey: { type: "string", multiple: true },
       viewport: { type: "string", multiple: true },
+      concurrency: { type: "string" },
       platforms: { type: "string", multiple: true },
       providers: { type: "string", multiple: true },
       "allow-host": { type: "string", multiple: true },
@@ -36,6 +39,8 @@ async function main(args = process.argv.slice(2)) {
       "base-url": { type: "string" },
       lang: { type: "string" },
       "fail-on": { type: "string", default: "broken" },
+      baseline: { type: "string" },
+      "update-baseline": { type: "boolean", default: false },
       headed: { type: "boolean", default: false },
       "no-branding": { type: "boolean", default: false },
       "keep-bodies": { type: "boolean", default: false },
@@ -53,6 +58,8 @@ async function main(args = process.argv.slice(2)) {
       from: { type: "string" },
       topic: { type: "string" },
       format: { type: "string" },
+      limit: { type: "string" },
+      json: { type: "boolean", default: false },
       force: { type: "boolean", default: false },
       yes: { type: "boolean", short: "y", default: false },
       help: { type: "boolean", short: "h", default: false },
@@ -131,6 +138,18 @@ async function main(args = process.argv.slice(2)) {
     console.log(`✓ Saved plan ${id}: ${plan.events.length} events (${custom} custom) and ${plan.user_properties.length} user properties in ${rel.startsWith("..") ? dir : rel}`);
     return;
   }
+  if (command === "history" && !values.help) {
+    await showHistory(values);
+    return;
+  }
+  if (command === "doctor" && !values.help) {
+    const { doctorChecks } = await import("./doctor.js");
+    const checks = await doctorChecks({ site: values.site, dataDir: values["data-dir"] });
+    for (const c of checks) console.log(`${c.ok ? "✓" : "✗"} ${c.text}`);
+    // Only a check an audit cannot run without fails the command; a missing AI key is advice.
+    if (checks.some((c) => c.hard && !c.ok)) process.exitCode = 1;
+    return;
+  }
   if (values.help || command !== "run" || (!values.config && !values.site && !values.against)) {
     console.log(helpText(command));
     process.exit(values.help ? 0 : 2);
@@ -161,6 +180,8 @@ async function main(args = process.argv.slice(2)) {
     against = await resolvePlan(values.against, { dataDir: values["data-dir"] ?? DEFAULT_DATA_DIR, site: loaded.config.site });
     const plan = await readJson(against.file);
     if (!Array.isArray(plan?.events)) throw new Error(`${against.file} is not a plan: it needs an "events" list.`);
+    // loadConfig checked the config's own plan; the one --against names replaces it and gets the same check.
+    for (const w of validatePlan(plan, { file: against.file }).warnings) console.log(w);
     if (!loaded.config.site) {
       if (!plan.site) throw new Error(`${against.file} has no "site". Add --site <url>.`);
       loaded.config = { ...loaded.config, site: plan.site };
@@ -178,6 +199,17 @@ async function main(args = process.argv.slice(2)) {
   if (maxSteps !== undefined && !(Number.isInteger(maxSteps) && maxSteps > 0)) throw new Error("--max-steps must be a positive whole number.");
   // run --out is a hidden alias kept for one minor version; --data-dir replaces it.
   if (values.out) console.log("Note: run --out will be removed in a later version. Runs are saved in aitd/<domain>/runs/<id>/; use --data-dir <dir> to keep them elsewhere.");
+  const baselineFile = values.baseline ?? loaded.baselineFile ?? null;
+  if (values["update-baseline"] && !baselineFile) throw new Error('--update-baseline needs --baseline <file> or "baseline" in the config.');
+  // A baseline that doesn't exist yet is fine when this run writes it.
+  const baseline = baselineFile
+    ? await readBaseline(baselineFile).catch((err) => {
+        if (values["update-baseline"] && err.code === "ENOENT") return null;
+        throw err;
+      })
+    : null;
+  // A baseline written for another site matches nothing of this one, so say so, but let the run go on.
+  if (baseline?.site && baseline.site !== loaded.config.site) console.log(`Warning: the baseline ${baselineFile} was written for ${baseline.site}, not ${loaded.config.site}. Its entries may not match this site.`);
   const result = await runAudit({
     ...loaded,
     journeys: values.journey,
@@ -185,6 +217,7 @@ async function main(args = process.argv.slice(2)) {
     events: (values.events ?? []).flatMap((e) => e.split(",")).map((e) => e.trim()).filter(Boolean),
     maxSteps,
     viewports: values.viewport,
+    concurrency: values.concurrency,
     providers: platforms.length ? platforms.flatMap((p) => p.split(",")) : undefined,
     scripted: values.scripted,
     ai: !values["no-ai"],
@@ -198,12 +231,27 @@ async function main(args = process.argv.slice(2)) {
     dataDir: values["data-dir"],
     saveSite: values["save-site"],
     against,
+    failOn,
+    baseline,
     log: (msg) => console.log(msg),
   });
 
-  console.log(`\n${messages(result.lang).ui.counts(result.counts)}`);
-  const rel = path.relative(process.cwd(), result.reportPath);
-  console.log(`Report: ${rel.startsWith("..") ? result.reportPath : rel}`);
+  const m = messages(result.lang);
+  console.log(`\n${m.ui.counts(result.counts)}`);
+  if (result.baseline) console.log(`${result.baseline.accepted} accepted (baseline ${baselineFile}), ${result.baseline.partial ? `${result.baseline.partial} ${result.baseline.partial === 1 ? "problem" : "problems"} accepted in some places only (still counted), ` : ""}${result.baseline.stale.length} stale baseline ${result.baseline.stale.length === 1 ? "entry" : "entries"}`);
+  if (result.verdict) console.log(m.ui.cli.verdict(result.verdict.statusText, result.verdict.headline, result.verdict.score));
+  // The trend needs two saved runs; with one there is nothing to say yet.
+  if (result.trend?.runs >= 2) {
+    const { historySummaryLines } = await import("./history.js");
+    for (const line of historySummaryLines(result.trend, result.lang, result.stability ?? []).slice(0, 3)) console.log(line);
+  }
+  const shown = (p) => (path.relative(process.cwd(), p).startsWith("..") ? p : path.relative(process.cwd(), p));
+  console.log(`Report: ${shown(result.reportPath)}`);
+  console.log(`Summary: ${shown(result.summaryPath)} (junit.xml next to it)`);
+  if (values["update-baseline"]) {
+    const n = await writeBaseline(baselineFile, { site: loaded.config.site, findings: result.findings });
+    console.log(`Baseline ${baselineFile} now holds ${n} ${n === 1 ? "finding" : "findings"}. Later runs with --baseline fail only on new ones.`);
+  }
 
   if (!against && !values.out) await offerLearnedPlan(result, loaded.config.site, values);
 
@@ -224,6 +272,39 @@ async function main(args = process.argv.slice(2)) {
   }
 }
 
+// aitd history --site <url>: the trend of the site's saved runs as a small table, the text sparkline and the summary lines.
+async function showHistory(values) {
+  if (!values.site) throw new Error("history needs --site <url>: aitd history --site https://www.example.com");
+  const limit = values.limit ? Number(values.limit) : 30;
+  if (!(Number.isInteger(limit) && limit > 0)) throw new Error("--limit must be a positive whole number.");
+  const lang = values.lang ?? "en";
+  const m = messages(lang);
+  const { domainOf } = await import("./workspace.js");
+  domainOf(values.site);
+  const { findingTimeline, historySummaryLines, journeyStability, loadHistory, sparkline, trend } = await import("./history.js");
+  const history = await loadHistory(values["data-dir"] ?? DEFAULT_DATA_DIR, values.site, { limit });
+  const t = trend(history);
+  const stability = journeyStability(history);
+  if (values.json) {
+    console.log(JSON.stringify({ site: values.site, runs: history.length, trend: t, timeline: findingTimeline(history), stability }, null, 2));
+    return;
+  }
+  if (!history.length) {
+    console.log(m.ui.history.noRuns);
+    return;
+  }
+  const cols = [m.ui.cli.runCol, m.severity.broken, m.severity.risk, m.severity.warning, m.ui.cli.scoreCol, m.ui.cli.completeCol];
+  const rows = t.series.map((p) => [p.id, p.broken, p.risk, p.warning, p.score, p.complete ? m.ui.md.yes : m.ui.md.no].map(String));
+  const widths = cols.map((c, i) => Math.max(c.length, ...rows.map((r) => r[i].length)));
+  const line = (cells) => cells.map((c, i) => (i ? c.padStart(widths[i]) : c.padEnd(widths[i]))).join("  ");
+  console.log(line(cols));
+  for (const r of rows) console.log(line(r));
+  console.log("");
+  const s = m.ui.history;
+  if (t.runs >= 2) console.log(s.sparkline(sparkline(t), s.direction[t.direction], t.runs, t.first.slice(0, 10), t.last.slice(0, 10)));
+  for (const l of historySummaryLines(t, lang, stability)) console.log(l);
+}
+
 // The first run of a site with no plan: offer what it already tracks as its plan.
 async function offerLearnedPlan(result, site, values) {
   const dataDir = values["data-dir"] ?? DEFAULT_DATA_DIR;

package/src/consent.js

@@ -2,6 +2,7 @@
 // (accept all, reject all, analytics only), to show which tags each choice lets through.
 // Runs on its own with every audit when the page shows a consent banner.
 import { AuditSession, CMP_BANNER_SELECTORS } from "./browser.js";
+import { runPool } from "./pool.js";
 
 export const CONSENT_STATES = ["accept", "reject", "analytics"];
 // Tags that are advertising, so an "analytics only" choice should keep them quiet.
@@ -110,6 +111,7 @@ async function runState(config, state, opts) {
 /**
  * Runs every consent scenario on the site's start page. Returns null when the page has no consent
  * banner the tool can answer (the first choice found nothing to click), so the report leaves the section out.
+ * opts.concurrency runs the other choices side by side, each in its own browser; default 1.
  * @returns {Promise<null | { cmp: string, before: object, states: Record<string, { via: string|null, after?: object, error?: string }> }>}
  */
 export async function runConsentScenarios(config, opts = {}) {
@@ -117,8 +119,8 @@ export async function runConsentScenarios(config, opts = {}) {
   const first = await runState(config, "accept", opts);
   if (!first.via) return null;
   log(`\n▶ consent scenarios (${first.via})`);
-  const results = [first];
-  for (const state of CONSENT_STATES.slice(1)) results.push(await runState(config, state, opts));
+  // The first choice shows whether there is a banner to answer at all; the others don't depend on each other.
+  const results = [first, ...(await runPool(CONSENT_STATES.slice(1), opts.concurrency ?? 1, (state) => runState(config, state, opts)))];
   const states = {};
   for (const r of results) {
     states[r.state] = r.error ? { via: null, error: r.error } : { via: r.via, ...(r.after ? { after: r.after } : {}) };

package/src/doctor.js

@@ -0,0 +1,68 @@
+// `aitd doctor`: one line per thing an audit needs, so a broken setup is found before a run starts.
+import { existsSync, readFileSync } from "node:fs";
+import { access, constants, stat } from "node:fs/promises";
+import { lookup } from "node:dns/promises";
+import path from "node:path";
+import { aiStatus } from "./providers.js";
+import { DEFAULT_DATA_DIR } from "./workspace.js";
+
+const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+
+// The same as minNodeMajor() in scripts/setup.js, which the published package doesn't ship.
+const minNodeMajor = (range) => Number(/(\d+)/.exec(range ?? "")?.[1] ?? 0);
+
+/** The nearest existing directory at or above dir, since the data dir is only created by the first run. */
+function existingAncestor(dir, exists) {
+  let d = dir;
+  while (!exists(d) && path.dirname(d) !== d) d = path.dirname(d);
+  return d;
+}
+
+/**
+ * Runs the checks and returns [{ ok, hard, text }]. A failed hard check means audits can't run;
+ * a soft one is advice. Everything that touches the machine can be injected, so tests need no
+ * browser or network.
+ */
+export async function doctorChecks({ site, dataDir = DEFAULT_DATA_DIR, nodeVersion = process.version, chromiumPath, env = process.env, exists = existsSync, writable = writableDir, resolve = lookup } = {}) {
+  const checks = [];
+  const range = pkg.engines?.node;
+  const nodeOk = Number(/^v?(\d+)/.exec(nodeVersion)?.[1] ?? 0) >= minNodeMajor(range);
+  checks.push({ ok: nodeOk, hard: true, text: nodeOk ? `Node ${nodeVersion} (needs ${range})` : `Node ${nodeVersion} is too old: this needs Node ${range}. Install a newer Node, for example with nvm: nvm install 22` });
+
+  const pw = pkg.dependencies.playwright;
+  chromiumPath ??= (await import("playwright")).chromium.executablePath();
+  const chromiumOk = Boolean(chromiumPath && exists(chromiumPath));
+  checks.push({ ok: chromiumOk, hard: true, text: chromiumOk ? `Playwright ${pw} with Chromium at ${chromiumPath}` : `Playwright ${pw} has no Chromium installed. Run: npx -y playwright@${pw} install chromium` });
+
+  // The same line `aitd` with no arguments prints. No key is not a failure: --no-ai and replay work without one.
+  const ai = aiStatus({}, env);
+  const aiOk = !ai.startsWith("AI: no key set");
+  checks.push({ ok: aiOk, hard: false, text: aiOk ? ai : `${ai}. Optional: --no-ai and replay work without a key` });
+
+  const dir = path.resolve(dataDir);
+  const probe = existingAncestor(dir, exists);
+  const dirOk = await writable(probe).then(() => true, () => false);
+  checks.push({ ok: dirOk, hard: true, text: dirOk ? `Data folder ${dir} is writable${probe === dir ? "" : " (it will be created)"}` : `Data folder ${dir} is not writable (checked ${probe}). Pick another with --data-dir <dir>` });
+
+  if (site !== undefined) checks.push(await siteCheck(site, resolve));
+  return checks;
+}
+
+/** Rejects unless d is a directory the process may write to: a data dir under a file can never be created, whatever the permissions say. */
+async function writableDir(d) {
+  if (!(await stat(d)).isDirectory()) throw new Error(`${d} is not a directory`);
+  await access(d, constants.W_OK);
+}
+
+/** The site is an http(s) address whose host resolves. Nothing is fetched. */
+async function siteCheck(site, resolve) {
+  let u;
+  try {
+    u = new URL(site);
+  } catch {
+    return { ok: false, hard: true, text: `"${site}" is not a full URL. Use the form https://www.example.com` };
+  }
+  if (!["http:", "https:"].includes(u.protocol)) return { ok: false, hard: true, text: `${site} is not an http or https address, so the browser can't open it` };
+  const found = await resolve(u.hostname).then(() => true, () => false);
+  return { ok: found, hard: true, text: found ? `${u.hostname} resolves` : `${u.hostname} does not resolve. Check the address, or your network and DNS` };
+}