htmlineage 0.0.0 → 0.1.0-alpha.0

package/src/check/rules.js

@@ -0,0 +1,379 @@
+/**
+ * The check rules that need more than one node or the page.
+ * `validateGraph()` (`../graph/schema.js`) owns the per-node ones (S1–S9, C4,
+ * C9). Pure: no file I/O.
+ */
+
+import { bareNumbers } from "../graph/numbers.js";
+import { exprProblem, exprRefs } from "../graph/expr.js";
+import { STEPS } from "../graph/schema.js";
+import { checkSqlStyle, dateFunctions, maskSql } from "../sql/style.js";
+
+/** Every rule this module can raise, with its fixed severity. */
+export const RULES = {
+  C1: { severity: "error", title: "binding and node do not match up" },
+  C2: {
+    severity: "warning",
+    title: "number in prose is outside a figure's span",
+  },
+  C3: {
+    severity: "warning",
+    title: "figure text does not match its display or value",
+  },
+  C5: {
+    severity: "error",
+    title: "`from` breaks the source → view → figure → insight chain",
+  },
+  C6: { severity: "error", title: "nodes depend on each other in a cycle" },
+  C7: {
+    severity: "warning",
+    title: "sql source is incomplete or not pinned to its period",
+  },
+  C8: { severity: "warning", title: "SQL breaks the style rules" },
+};
+
+const isObject = (value) =>
+  value !== null && typeof value === "object" && !Array.isArray(value);
+const isStringArray = (value) =>
+  Array.isArray(value) && value.every((v) => typeof v === "string");
+
+const NUMBER_PATTERN =
+  /^([+\-−])?[$€£¥]?#?(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d+))?( ?%)?$/;
+
+/**
+ * Read a figure's page text as a number, for C3's fallback.
+ * @param {string} text
+ * @returns {{ number: number, decimals: number, percent: boolean } | null}
+ */
+export function parseDisplayNumber(text) {
+  const match = NUMBER_PATTERN.exec(text);
+  if (!match) return null;
+  const [, sign, digits, decimals, percent] = match;
+  const magnitude = Number(`${digits.replace(/,/g, "")}.${decimals ?? "0"}`);
+  const number = sign === "-" || sign === "−" ? -magnitude : magnitude;
+  return {
+    number,
+    decimals: decimals ? decimals.length : 0,
+    percent: Boolean(percent),
+  };
+}
+
+/**
+ * Run the check rules over a graph and its page.
+ * @param {unknown} graph  as returned by readGraph; may be malformed
+ * @param {{ bindings: {id,text,offset}[], prose: {text,node,offset}[] }} page
+ * @returns {{ code, severity, node: string|null, field: string|null, message: string }[]}
+ */
+export function checkRules(graph, page) {
+  const nodes = graph?.nodes;
+  if (!isObject(nodes)) return [];
+
+  const has = (id) => Object.hasOwn(nodes, id);
+  const node = (id) => (has(id) && isObject(nodes[id]) ? nodes[id] : null);
+
+  const issues = [];
+  const report = (code, n, field, message) =>
+    issues.push({
+      code,
+      severity: RULES[code].severity,
+      node: n,
+      field: field ?? null,
+      message,
+    });
+
+  checkC1(nodes, has, node, page, report);
+  checkC2(page, report);
+  checkC3(node, page, report);
+  checkC5(nodes, has, node, report);
+  checkC6(nodes, has, report);
+  checkC7(nodes, report);
+  checkC8(nodes, report);
+
+  return issues;
+}
+
+function checkC1(nodes, has, node, page, report) {
+  const seen = new Set();
+  for (const binding of page.bindings) {
+    const id = binding.id;
+    if (seen.has(id)) continue;
+    if (!has(id)) {
+      seen.add(id);
+      report("C1", id, null, `data-lineage="${id}" resolves to no node`);
+      continue;
+    }
+    const n = node(id);
+    if (n?.step === "source") {
+      seen.add(id);
+      report(
+        "C1",
+        id,
+        null,
+        `\`${id}\` is a source; bind a figure over it instead`,
+      );
+    }
+  }
+
+  for (const [id, n] of Object.entries(nodes)) {
+    if (!isObject(n)) continue;
+    if (n.step !== "view" && n.step !== "figure" && n.step !== "insight")
+      continue;
+    const bound = page.bindings.some((b) => b.id === id);
+    if (!bound) {
+      report(
+        "C1",
+        id,
+        null,
+        `${n.step} is not bound to any element on the page`,
+      );
+    }
+  }
+}
+
+function checkC2(page, report) {
+  for (const entry of page.prose) {
+    for (const token of bareNumbers(entry.text)) {
+      report(
+        "C2",
+        entry.node,
+        null,
+        `bare number "${token.text}" is not inside a figure's span`,
+      );
+    }
+  }
+}
+
+function checkC3(node, page, report) {
+  const byId = new Map();
+  for (const binding of page.bindings) {
+    if (!byId.has(binding.id)) byId.set(binding.id, []);
+    byId.get(binding.id).push(binding.text);
+  }
+
+  for (const [id, texts] of byId) {
+    const n = node(id);
+    if (n?.step !== "figure") continue;
+    const display = typeof n.display === "string" ? n.display : null;
+    const value =
+      typeof n.value === "number" && Number.isFinite(n.value) ? n.value : null;
+    if (display === null && value === null) continue;
+
+    const bad = [];
+    const seen = new Set();
+    for (const text of texts) {
+      if (text === display) continue;
+      let ok = false;
+      if (value !== null) {
+        const parsed = parseDisplayNumber(text);
+        if (parsed) {
+          const s = parsed.percent ? 100 : 1;
+          const slack = ((0.5 * 10 ** -parsed.decimals) / s) * (1 + 1e-9);
+          if (Math.abs(parsed.number / s - value) <= slack) ok = true;
+        }
+      }
+      if (!ok && !seen.has(text)) {
+        seen.add(text);
+        bad.push(text);
+      }
+    }
+    if (bad.length > 0) {
+      const quoted = bad.map((t) => `"${t}"`).join(", ");
+      const against =
+        display !== null ? `display is "${display}"` : `value is ${value}`;
+      report("C3", id, "display", `page shows ${quoted} but ${against}`);
+    }
+  }
+}
+
+/** Read a node's step, or null when it is not a plain object or has an unknown step. */
+function stepOf(n) {
+  return isObject(n) && STEPS.includes(n.step) ? n.step : null;
+}
+
+function checkC5(nodes, has, node, report) {
+  for (const [id, n] of Object.entries(nodes)) {
+    if (!isObject(n)) continue;
+    const step = n.step;
+    if (step !== "view" && step !== "figure" && step !== "insight") continue;
+    if (!isStringArray(n.from) || n.from.length === 0) continue;
+
+    let exprIds = null;
+    if (step === "figure") {
+      const hasSql = n.sql !== undefined;
+      const hasExpr = n.expr !== undefined;
+      if (hasSql === hasExpr) continue;
+      if (hasExpr) {
+        if (typeof n.expr !== "string") continue;
+        if (exprProblem(n.expr)) continue;
+        exprIds = exprRefs(n.expr, Object.keys(nodes));
+      }
+    }
+
+    const parts = [];
+
+    const unknown = n.from.filter((p) => !has(p));
+    if (unknown.length > 0) {
+      parts.push(
+        `\`from\` names ${unknown.join(", ")}, which ${unknown.length > 1 ? "are not nodes" : "is not a node"}`,
+      );
+    }
+
+    const knownParents = n.from
+      .filter((p) => has(p))
+      .map((p) => ({ id: p, step: stepOf(node(p)) }))
+      .filter((p) => p.step !== null);
+
+    if (knownParents.length > 0) {
+      let ok;
+      let message;
+      if (step === "view") {
+        ok = knownParents.every(
+          (p) => p.step === "source" || p.step === "view",
+        );
+        message = "a view rests on sources and views only";
+      } else if (step === "figure" && exprIds === null) {
+        ok = knownParents.every(
+          (p) => p.step === "source" || p.step === "view",
+        );
+        message = "a figure with `sql` rests on sources and views only";
+      } else if (step === "figure") {
+        ok = knownParents.every((p) => p.step === "figure");
+        message = "a figure with `expr` rests on figures only";
+      } else {
+        ok = knownParents.every((p) => p.step !== "source");
+        message = "an insight rests on views, figures and insights only";
+      }
+      if (!ok) parts.push(message);
+    }
+
+    if (step === "figure" && exprIds !== null) {
+      const missing = exprIds.filter((r) => !n.from.includes(r));
+      if (missing.length > 0) {
+        parts.push(
+          `\`expr\` uses ${missing.join(", ")}, which ${missing.length > 1 ? "are not in `from`" : "is not in `from`"}`,
+        );
+      }
+    }
+
+    if (parts.length > 0) {
+      report("C5", id, "from", parts.join("; "));
+    }
+  }
+}
+
+function checkC6(nodes, has, report) {
+  const edges = (id) => {
+    const n = nodes[id];
+    if (!isObject(n) || !isStringArray(n.from)) return [];
+    return n.from.filter((p) => has(p));
+  };
+
+  const keyOrder = Object.keys(nodes);
+  const indexOf = new Map(keyOrder.map((id, i) => [id, i]));
+  const state = new Map(); // 0 = visiting, 1 = done
+  const stack = [];
+  const stackSet = new Set();
+  const reportedSets = new Set();
+
+  const visit = (id) => {
+    state.set(id, 0);
+    stack.push(id);
+    stackSet.add(id);
+    for (const next of edges(id)) {
+      if (stackSet.has(next)) {
+        const cycleStart = stack.indexOf(next);
+        const cycle = stack.slice(cycleStart);
+        const memberSet = [...cycle].sort().join(",");
+        if (!reportedSets.has(memberSet)) {
+          reportedSets.add(memberSet);
+          let firstIndex = 0;
+          for (let i = 1; i < cycle.length; i++) {
+            if (indexOf.get(cycle[i]) < indexOf.get(cycle[firstIndex]))
+              firstIndex = i;
+          }
+          const rotated = [
+            ...cycle.slice(firstIndex),
+            ...cycle.slice(0, firstIndex),
+          ];
+          report(
+            "C6",
+            rotated[0],
+            "from",
+            `cycle ${[...rotated, rotated[0]].join(" -> ")}`,
+          );
+        }
+      } else if (!state.has(next)) {
+        visit(next);
+      }
+    }
+    stack.pop();
+    stackSet.delete(id);
+    state.set(id, 1);
+  };
+
+  for (const id of keyOrder) {
+    if (!state.has(id)) visit(id);
+  }
+}
+
+function checkC7(nodes, report) {
+  for (const [id, n] of Object.entries(nodes)) {
+    if (!isObject(n) || n.step !== "source" || n.kind !== "sql") continue;
+
+    const parts = [];
+    const missing = ["mart", "period", "sql", "columns"].filter(
+      (field) => n[field] === undefined,
+    );
+    if (missing.length > 0) {
+      parts.push(`sql source is missing \`${missing.join("`, `")}\``);
+    }
+
+    if (typeof n.sql === "string") {
+      if (
+        isObject(n.period) &&
+        typeof n.period.from === "string" &&
+        typeof n.period.to === "string"
+      ) {
+        const masked = maskSql(n.sql, { keepStrings: true });
+        const dates = [...new Set([n.period.from, n.period.to])];
+        const missingDates = dates.filter(
+          (date) => !masked.includes(`'${date}'`),
+        );
+        if (missingDates.length > 0) {
+          parts.push(
+            `SQL does not contain the period as literal dates: ${missingDates.map((d) => `'${d}'`).join(", ")}`,
+          );
+        }
+      }
+      const dates = dateFunctions(n.sql);
+      if (dates.length > 0) {
+        parts.push(`SQL uses date functions: ${dates.join(", ")}`);
+      }
+    }
+
+    if (parts.length > 0) report("C7", id, null, parts.join("; "));
+  }
+}
+
+function checkC8(nodes, report) {
+  for (const [id, n] of Object.entries(nodes)) {
+    if (!isObject(n)) continue;
+    const isSource = n.step === "source" && n.kind === "sql";
+    const isViewOrFigure = n.step === "view" || n.step === "figure";
+    if (!isSource && !isViewOrFigure) continue;
+    if (typeof n.sql !== "string" || n.sql.trim() === "") continue;
+
+    const parents = isViewOrFigure && isStringArray(n.from) ? n.from : [];
+    let findings = checkSqlStyle(n.sql, { parents });
+    if (isSource) findings = findings.filter((f) => f.rule !== "date-function");
+
+    if (findings.length > 0) {
+      report(
+        "C8",
+        id,
+        "sql",
+        `SQL style: ${findings.map((f) => f.message).join("; ")}`,
+      );
+    }
+  }
+}

package/src/check/summary.js

@@ -0,0 +1,154 @@
+/**
+ * The check summary: counts, what needs attention, and its printed form.
+ * Pure: no file I/O, no printing.
+ */
+
+import { traceGraph } from "../graph/trace.js";
+
+/** `[id, node]` pairs in graph order; none when the graph has no readable `nodes`. */
+const nodeEntries = (graph) =>
+  graph !== null &&
+  typeof graph === "object" &&
+  graph.nodes !== null &&
+  typeof graph.nodes === "object" &&
+  !Array.isArray(graph.nodes)
+    ? Object.entries(graph.nodes)
+    : [];
+
+/** The node's `text`, else its `label`; only strings count. */
+const describe = (node) =>
+  [node.text, node.label].find((value) => typeof value === "string") ?? null;
+
+/**
+ * @param {unknown} graph
+ * @param {{ bindings: { id: string }[] }} page
+ * @param {object[]} issues
+ * @param {{ report: string }} options
+ */
+export function summarize(graph, page, issues, { report }) {
+  const counts = {
+    sources: { total: 0, external: 0 },
+    views: { total: 0, shown: 0 },
+    figures: { total: 0, full: 0, partial: 0, ungrounded: 0 },
+    insights: { total: 0, full: 0, partial: 0, ungrounded: 0 },
+  };
+  const ungrounded = [];
+  const external = [];
+  const states = traceGraph(graph);
+  const bound = new Set(page.bindings.map((binding) => binding.id));
+  for (const [id, node] of nodeEntries(graph)) {
+    if (!states.has(id)) continue;
+    if (node.ungrounded === true) {
+      ungrounded.push({ node: id, text: describe(node) });
+    }
+    if (node.step === "view") {
+      counts.views.total += 1;
+      if (bound.has(id)) counts.views.shown += 1;
+    }
+    if (node.step === "figure" || node.step === "insight") {
+      const tally = counts[`${node.step}s`];
+      tally.total += 1;
+      tally[states.get(id)] += 1;
+    }
+    if (node.step === "source") {
+      counts.sources.total += 1;
+      if (node.kind === "external") {
+        counts.sources.external += 1;
+        external.push({
+          node: id,
+          ref: typeof node.ref === "string" ? node.ref : null,
+        });
+      }
+    }
+  }
+  return { report, counts, ungrounded, external, issues };
+}
+
+/** One count line: label padded to 11, count padded to 4, then the detail. */
+const countLine = (label, total, detail) =>
+  `  ${label.padEnd(11)}${String(total).padEnd(4)}${detail}`.trimEnd();
+
+const tracedDetail = ({ total, full, partial, ungrounded }) => {
+  if (total > 0 && full === total) return "all fully traced";
+  return [
+    [full, "fully traced"],
+    [partial, "partially traced"],
+    [ungrounded, "ungrounded"],
+  ]
+    .filter(([n]) => n > 0)
+    .map(([n, what]) => `${n} ${what}`)
+    .join(", ");
+};
+
+const shownDetail = ({ total, shown }) => {
+  if (total === 0) return "";
+  if (shown === total) return "all shown on the page";
+  return `${shown} shown, ${total - shown} not on the page`;
+};
+
+const RED = "\x1b[31m";
+const YELLOW = "\x1b[33m";
+const RESET = "\x1b[0m";
+
+const shorten = (text) => {
+  const flat = text.replace(/\s+/g, " ").trim();
+  return flat.length <= 40 ? flat : `${flat.slice(0, 40).trimEnd()}…`;
+};
+
+/**
+ * @param {object} summary
+ * @param {{ color?: boolean }} [options]
+ * @returns {string}
+ */
+export function formatSummary(summary, { color = false } = {}) {
+  const { sources, views, figures, insights } = summary.counts;
+  const colour = (text, paint) => (color ? `${paint}${text}${RESET}` : text);
+  const lines = [
+    summary.report,
+    "",
+    countLine(
+      "sources",
+      sources.total,
+      sources.external > 0 ? `(${sources.external} external)` : "",
+    ),
+    countLine("views", views.total, shownDetail(views)),
+    countLine("figures", figures.total, tracedDetail(figures)),
+    countLine("insights", insights.total, tracedDetail(insights)),
+  ];
+  const rows = summary.ungrounded.map(({ node, text }) => ({
+    node,
+    tag: "ungrounded",
+    paint: YELLOW,
+    detail: text === null ? "" : `"${shorten(text)}"`,
+  }));
+  for (const { node, ref } of summary.external) {
+    const ask = "a reader will be asked to confirm this with you";
+    rows.push({
+      node,
+      tag: "external",
+      paint: YELLOW,
+      detail: ref === null ? ask : `${ref} — ${ask}`,
+    });
+  }
+  for (const severity of ["warning", "error"]) {
+    for (const issue of summary.issues) {
+      if (issue.severity !== severity) continue;
+      rows.push({
+        node: issue.node ?? "graph",
+        tag: `${issue.code} ${severity}`,
+        paint: severity === "error" ? RED : YELLOW,
+        detail: issue.message,
+      });
+    }
+  }
+  if (rows.length > 0) {
+    const width = Math.max(5, ...rows.map((row) => row.node.length + 3));
+    lines.push("", "  needs attention");
+    for (const { node, tag, paint, detail } of rows) {
+      lines.push(
+        `    ${node.padEnd(width)}${colour(tag, paint)}${" ".repeat(13 - tag.length)}${detail}`.trimEnd(),
+      );
+    }
+  }
+  return lines.join("\n");
+}

package/src/cli.js

@@ -0,0 +1,99 @@
+import { parseArgs } from "node:util";
+
+import { byName, commands } from "./commands/index.js";
+import { EXIT_OK, EXIT_USAGE, UsageError } from "./errors.js";
+import { topLevelHelp, versionLine } from "./help.js";
+
+const defaultIo = {
+  out: (text) => console.log(text),
+  err: (text) => console.error(text),
+  color: process.stdout.isTTY === true,
+};
+
+const HELP_FLAGS = new Set(["-h", "--help"]);
+const VERSION_FLAGS = new Set(["-v", "--version"]);
+
+/**
+ * Parse one command's arguments. parseArgs throws a handful of ERR_PARSE_ARGS_*
+ * codes with readable messages; they are all invocation mistakes, so they all
+ * become a UsageError and exit 2.
+ */
+function parseCommand(command, argv) {
+  try {
+    return parseArgs({
+      args: argv,
+      options: { ...command.options, help: { type: "boolean", short: "h" } },
+      allowPositionals: true,
+      strict: true,
+    });
+  } catch (error) {
+    throw new UsageError(error.message, { command: command.name });
+  }
+}
+
+async function dispatch(argv, io) {
+  if (argv.length === 0) {
+    io.out(topLevelHelp());
+    return EXIT_OK;
+  }
+
+  const [first, ...rest] = argv;
+
+  if (HELP_FLAGS.has(first)) {
+    // `htmlineage --help check` is the same as `htmlineage check --help`.
+    const named = byName.get(rest[0]);
+    io.out(named ? named.usage : topLevelHelp());
+    return EXIT_OK;
+  }
+
+  if (VERSION_FLAGS.has(first)) {
+    io.out(versionLine());
+    return EXIT_OK;
+  }
+
+  if (first.startsWith("-")) {
+    throw new UsageError(`unknown option \`${first}\``);
+  }
+
+  const command = byName.get(first);
+  if (!command) {
+    const known = commands.map((c) => c.name).join(", ");
+    throw new UsageError(
+      `unknown command \`${first}\`. Expected one of: ${known}`,
+    );
+  }
+
+  const parsed = parseCommand(command, rest);
+  if (parsed.values.help) {
+    io.out(command.usage);
+    return EXIT_OK;
+  }
+
+  const code = await command.run(parsed, io);
+  return code ?? EXIT_OK;
+}
+
+/**
+ * The whole CLI as one function, so tests can drive it in-process and assert
+ * on both the exit code and the output without spawning Node.
+ */
+export async function main(argv = process.argv.slice(2), io = defaultIo) {
+  try {
+    return await dispatch(argv, io);
+  } catch (error) {
+    if (error instanceof UsageError) {
+      io.err(`htmlineage: ${error.message}`);
+      io.err(
+        error.command
+          ? `Try \`htmlineage ${error.command} --help\`.`
+          : "Try `htmlineage --help`.",
+      );
+      return EXIT_USAGE;
+    }
+    if (typeof error?.exitCode === "number") {
+      io.err(`htmlineage: ${error.message}`);
+      return error.exitCode;
+    }
+    throw error;
+  }
+}

package/src/commands/check.js

@@ -0,0 +1,71 @@
+import { basename } from "node:path";
+
+import { checkRules } from "../check/rules.js";
+import { formatSummary, summarize } from "../check/summary.js";
+import { UsageError } from "../errors.js";
+import { readGraph, readText } from "../graph/parse.js";
+import { validateGraph } from "../graph/schema.js";
+import { embedFile } from "../html/embed.js";
+import { proseSkip, readPage } from "../html/page.js";
+
+export const name = "check";
+export const summary = "Validate a report's lineage and embed the graph";
+
+export const usage = `htmlineage check <report.html> [options]
+
+Reads the lineage graph beside the report, checks it against the report's
+data-lineage bindings, embeds it into the HTML, and prints a summary.
+
+Exit code is 0 when the report passes (warnings still pass) and 1 when it
+does not.
+
+Options
+  --graph <path>   Graph JSON to read (default: .lineage/<report>.json
+                   beside the report, or else the block already embedded
+                   in it)
+  --no-embed       Check only; leave the report file untouched
+  --json           Print the summary as JSON instead of prose
+  --quiet          Print nothing; rely on the exit code
+  -h, --help       Show this help`;
+
+export const options = {
+  graph: { type: "string" },
+  "no-embed": { type: "boolean", default: false },
+  json: { type: "boolean", default: false },
+  quiet: { type: "boolean", default: false },
+};
+
+export async function run(parsed, io) {
+  const { positionals } = parsed;
+  if (positionals.length === 0) {
+    throw new UsageError("missing the report to check", { command: name });
+  }
+  if (positionals.length > 1) {
+    throw new UsageError(`expected one report, got ${positionals.length}`, {
+      command: name,
+    });
+  }
+  const reportPath = positionals[0];
+  const { graph } = readGraph({
+    reportPath,
+    graphPath: parsed.values.graph,
+  });
+  const html = readText(reportPath, "report");
+  const page = readPage(html, { skip: proseSkip(graph) });
+  const issues = [...validateGraph(graph), ...checkRules(graph, page)];
+  const summary = summarize(graph, page, issues, {
+    report: basename(reportPath),
+  });
+  const failed = issues.some((issue) => issue.severity === "error");
+  // Embed before printing: if the report cannot take the graph, check fails
+  // with that error alone rather than after a summary that reads as passing.
+  if (!failed && !parsed.values["no-embed"]) embedFile(reportPath, html, graph);
+  if (!parsed.values.quiet) {
+    io.out(
+      parsed.values.json
+        ? JSON.stringify(summary, null, 2)
+        : formatSummary(summary, { color: io.color === true }),
+    );
+  }
+  return failed ? 1 : 0;
+}