htmlineage 0.0.0 → 0.1.0-alpha.0

package/src/html/page.js

@@ -0,0 +1,101 @@
+/**
+ * Read the parts of a report that `check` looks at.
+ *
+ * A report is read into two flat lists, in document order:
+ *
+ *   bindings  every element carrying `data-lineage`, however deep or
+ *             hidden, for C1 (unknown/invalid ids) and C3 (missing bindings).
+ *   prose     every visible text run, tagged with the id of its nearest
+ *             bound ancestor (or `null` when unbound), for C2 (bare
+ *             numbers).
+ *
+ * The HTML is only ever read here, never written back: `check` edits a
+ * report with string surgery over the raw bytes, using the offsets this
+ * module returns, so parsing it into a tree and reserializing would risk
+ * changing bytes the report's author never touched.
+ */
+
+import { parse } from "node-html-parser";
+
+const HIDDEN_TAGS = new Set([
+  "head",
+  "script",
+  "style",
+  "noscript",
+  "template",
+]);
+
+const clean = (s) => s.replace(/\s+/g, " ").trim();
+
+/**
+ * Read a report's bindings and prose.
+ *
+ * @param {string} html  the report, as read from disk
+ * @param {{ skip?: (id: string) => boolean }} [options]
+ *   skip(id) true: leave that bound element's text, and everything inside
+ *   it, out of `prose`. Default: skip nothing.
+ * @returns {{
+ *   bindings: { id: string, text: string, offset: number }[],
+ *   prose: { text: string, node: string | null, offset: number }[],
+ * }}
+ */
+export function readPage(html, { skip = () => false } = {}) {
+  const root = parse(html, {
+    comment: false,
+    blockTextElements: { script: true, style: true, noscript: true },
+  });
+
+  const bindings = [];
+  const prose = [];
+
+  const walk = (el, node, hidden) => {
+    for (const child of el.childNodes) {
+      if (child.nodeType === 1) {
+        let childNode = node;
+        let childHidden = hidden;
+        if (child.hasAttribute("data-lineage")) {
+          const id = child.getAttribute("data-lineage") ?? "";
+          bindings.push({
+            id,
+            text: clean(child.textContent),
+            offset: child.range[0],
+          });
+          childNode = id;
+          if (skip(id)) childHidden = true;
+        }
+        if (HIDDEN_TAGS.has(child.rawTagName.toLowerCase())) {
+          childHidden = true;
+        }
+        walk(child, childNode, childHidden);
+      } else if (child.nodeType === 3) {
+        if (hidden) continue;
+        if (child.rawText.startsWith("<!")) continue;
+        const text = clean(child.text);
+        if (text === "") continue;
+        prose.push({ text, node, offset: child.range[0] });
+      }
+    }
+  };
+
+  walk(root, null, false);
+
+  return { bindings, prose };
+}
+
+/**
+ * The skip predicate C2 uses: text counts only when it is unbound or its
+ * nearest binding is a grounded insight.
+ *
+ * @param {unknown} graph  as returned by readGraph; may be malformed
+ * @returns {(id: string) => boolean}
+ */
+export function proseSkip(graph) {
+  const nodes = graph?.nodes;
+  if (nodes === null || typeof nodes !== "object" || Array.isArray(nodes)) {
+    return () => true;
+  }
+  return (id) => {
+    const n = Object.hasOwn(nodes, id) ? nodes[id] : null;
+    return !(n?.step === "insight" && n.ungrounded !== true);
+  };
+}

package/src/meta.js

@@ -0,0 +1,13 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Read name and version straight from package.json rather than importing it,
+ * so this works identically on every supported Node version without relying
+ * on import attributes.
+ */
+export function packageMeta() {
+  const path = fileURLToPath(new URL("../package.json", import.meta.url));
+  const { name, version } = JSON.parse(readFileSync(path, "utf8"));
+  return { name, version };
+}

package/src/sql/compose.js

@@ -0,0 +1,215 @@
+/**
+ * Compose a lineage graph into one SQL script. Pure: no file I/O.
+ */
+
+import { exprProblem, exprRefs } from "../graph/expr.js";
+import { traceGraph } from "../graph/trace.js";
+
+/** Text collapsed to one line, so it cannot end a `--` comment early. */
+const oneLine = (text) => text.replace(/\s+/g, " ").trim();
+
+const quote = (text) => `'${text.replaceAll("'", "''")}'`;
+
+/** A recorded value as a SQL literal: number as written, string quoted, else null. */
+const literal = (value) => {
+  if (typeof value === "number") return String(value);
+  if (typeof value === "string") return quote(value);
+  return "null";
+};
+
+/** The node's SQL as written, minus outer whitespace and one final `;`, indented. */
+const indent = (sql) =>
+  sql
+    .trim()
+    .replace(/\s*;$/, "")
+    .replace(/^(?=.)/gm, "    ");
+
+/**
+ * An expression compiler that reads figures from `nodes`. The text is copied
+ * as written; only each word that names a node changes (not quoted text, and
+ * not a function name before `(`).
+ */
+function compiler(nodes) {
+  const ids = new Set(Object.keys(nodes));
+  const compile = (expr) =>
+    expr.replace(
+      /('[^']*'|"[^"]*")|([A-Za-z_][A-Za-z0-9_]*\b(?!\s*\())/g,
+      (match, quoted, word) => {
+        if (quoted || !ids.has(word)) return match;
+        const { expr: inner } = nodes[word];
+        return inner ? `(${compile(inner)})` : `(select value from ${word})`;
+      },
+    );
+  return compile;
+}
+
+/** Why a report that fails `check` gets no script. */
+export const refusal = (report) =>
+  `${report} does not pass \`htmlineage check\`, so no script was composed`;
+
+const NOTHING = ", so there is nothing to compose";
+
+/** Whether `script`, from `composeSql`, says there was nothing to compose. */
+export const isEmptyScript = (script) =>
+  script.split("\n")[1].endsWith(NOTHING);
+
+/**
+ * @param {object} graph a graph with no error-severity issues
+ * @param {{ report: string, id?: string }} options `report` is the file name
+ *   for the header; `id` limits the script to that node and what is below it
+ * @returns {string} the script, with no trailing newline
+ */
+export function composeSql(graph, options) {
+  const report = oneLine(options.report);
+  const header =
+    options.id === undefined ? report : `${report} (${options.id})`;
+  // The node and everything below it through `from`; the whole graph without an id.
+  const scope = new Set();
+  const collect = (id) => {
+    if (scope.has(id)) return;
+    scope.add(id);
+    (graph.nodes[id].from ?? []).forEach(collect);
+  };
+  if (options.id === undefined) Object.keys(graph.nodes).forEach(collect);
+  else collect(options.id);
+  const entries = Object.entries(graph.nodes).filter(([id]) => scope.has(id));
+  const figures = entries.filter(([, node]) => node.step === "figure");
+  // A sql source whose query was never recorded cannot run.
+  const noSql = (node) =>
+    node.step === "source" && node.kind === "sql" && node.sql === undefined;
+  const unreadable = (node) =>
+    node.expr !== undefined && exprProblem(node.expr) !== null;
+  const notCovered = entries.flatMap(([id, node]) => {
+    if (noSql(node)) return [`-- ${id} has no sql`];
+    if (unreadable(node)) return [`-- ${id} expr cannot be read`];
+    if (node.kind === "external") {
+      const ref = oneLine(node.ref ?? "");
+      return [ref ? `-- ${id} external: ${ref}` : `-- ${id} external`];
+    }
+    if (node.ungrounded) {
+      const text = oneLine(node.text ?? node.label ?? "");
+      return [text ? `-- ${id} ungrounded: ${text}` : `-- ${id} ungrounded`];
+    }
+    return [];
+  });
+  const tail =
+    notCovered.length > 0
+      ? `\n\n-- not covered by this script:\n${notCovered.join("\n")}`
+      : "";
+  // Sources anywhere below a node that pass `test`, in graph order.
+  const sourcesBelow = (id, test) => {
+    const below = new Set();
+    const walk = (parent) => {
+      if (test(graph.nodes[parent])) below.add(parent);
+      (graph.nodes[parent].from ?? []).forEach(walk);
+    };
+    walk(id);
+    return entries.map(([other]) => other).filter((other) => below.has(other));
+  };
+  const externalsBelow = (id) =>
+    sourcesBelow(id, (node) => node.kind === "external");
+  const states = traceGraph(graph);
+  const runs = (id) =>
+    states.get(id) === "full" && sourcesBelow(id, noSql).length === 0;
+  const target = graph.nodes[options.id];
+  // The first view below an insight, looking through insights only.
+  const viewBelow = (id) => {
+    const node = graph.nodes[id];
+    if (node.step === "view") return id;
+    if (node.step !== "insight") return null;
+    for (const parent of node.from) {
+      const found = viewBelow(parent);
+      if (found !== null) return found;
+    }
+    return null;
+  };
+  // A source or view with SQL to run ends in a select of itself; so does an
+  // insight that rests on no figures, through the first view below it.
+  const selected =
+    target?.step === "source" || target?.step === "view"
+      ? options.id
+      : target?.step === "insight" && figures.length === 0
+        ? viewBelow(options.id)
+        : null;
+  const selectsTarget = selected !== null && runs(selected);
+  if (figures.length === 0 && !selectsTarget) {
+    const { id } = options;
+    const externals = id === undefined ? [] : externalsBelow(id);
+    const ungrounded = entries
+      .filter(([, node]) => node.ungrounded)
+      .map(([other]) => other);
+    const missing = id === undefined ? [] : sourcesBelow(id, noSql);
+    const ref = oneLine(target?.ref ?? "");
+    let why = `${id} rests on no figures`;
+    if (id === undefined) why = "no figures";
+    else if (target.kind === "external") {
+      why = ref ? `${id} is external (${ref})` : `${id} is external`;
+    } else if (target.ungrounded) why = `${id} is ungrounded`;
+    else if (noSql(target)) why = `${id} has no sql`;
+    else if (externals.length > 0) {
+      why = `${id} rests on external ${externals.join(", ")}`;
+    } else if (missing.length > 0) {
+      why = `${id} rests on ${missing.join(", ")}, which ${missing.length > 1 ? "have" : "has"} no sql`;
+    } else if (ungrounded.length > 0) {
+      why = `${id} rests on ungrounded ${ungrounded.join(", ")}`;
+    }
+    return `-- ${header}\n-- ${why}${NOTHING}${tail}`;
+  }
+  const compile = compiler(graph.nodes);
+  const ofStep = (step) => entries.filter(([, node]) => node.step === step);
+  const views = [];
+  const visit = ([id, node]) => {
+    if (views.some(([seen]) => seen === id)) return;
+    for (const parent of node.from) {
+      if (graph.nodes[parent].step === "view") {
+        visit([parent, graph.nodes[parent]]);
+      }
+    }
+    views.push([id, node]);
+  };
+  ofStep("view")
+    .filter(([id]) => runs(id))
+    .forEach(visit);
+  const sqlSources = ofStep("source").filter(([id]) => runs(id));
+  const ctes = [...sqlSources, ...views].map(
+    ([id, node]) => `${id} as (\n${indent(node.sql)}\n)`,
+  );
+  // A sql figure with no CTE cannot be read: its value is null.
+  const unread = new Set();
+  for (const [id, node] of figures.filter(([, node]) => node.sql)) {
+    const externals = externalsBelow(id);
+    if (runs(id)) {
+      ctes.push(`${id} (value) as (\n${indent(node.sql)}\n)`);
+    } else if (externals.length > 0 && !node.ungrounded) {
+      ctes.push(
+        `${id} (value) as (\n    -- from the report: rests on external ${externals.join(", ")}\n    select ${literal(node.value)}\n)`,
+      );
+    } else {
+      unread.add(id);
+    }
+  }
+  const isNull = (id) =>
+    graph.nodes[id].ungrounded === true ||
+    unread.has(id) ||
+    unreadable(graph.nodes[id]) ||
+    (graph.nodes[id].expr !== undefined &&
+      exprRefs(graph.nodes[id].expr, Object.keys(graph.nodes)).some(isNull));
+  const rows = figures.map(([id, node], i) => {
+    const reading = node.expr
+      ? compile(node.expr)
+      : `(select value from ${id})`;
+    const [figure, label, value] = [
+      `'${id}'`,
+      node.label === undefined ? "null" : quote(node.label),
+      isNull(id) ? "null" : `cast(${reading} as varchar)`,
+    ];
+    return i === 0
+      ? `select ${figure} as figure, ${label} as label, ${value} as value`
+      : `select ${figure}, ${label}, ${value}`;
+  });
+  const final = selectsTarget
+    ? `select * from ${selected}`
+    : rows.join("\nunion all\n");
+  const withClause = ctes.length > 0 ? `with\n${ctes.join(",\n")}\n` : "";
+  return `-- ${header}\n${withClause}${final}${tail}`;
+}

package/src/sql/style.js

@@ -0,0 +1,193 @@
+/**
+ * The mechanical SQL checks: no `select *`, no date functions, and parents
+ * named by node id so the composed script runs. They scan a query's words and
+ * parentheses, never its meaning. `check` runs them over sources, views and
+ * figure queries (rule C8) and over a source's own SQL for date functions
+ * (rule C7). A SQL parser is deliberately out of scope; that is rule C10,
+ * deferred.
+ */
+
+const CLAUSE_PATTERN =
+  /\b(select|from|where|group\s+by|having|order\s+by|limit|qualify|union|join)\b/gi;
+
+const DATE_FUNCTION_PATTERN =
+  /\b(current_date|current_timestamp|current_time|localtimestamp|localtime|sysdate|systimestamp|getdate|now)\b/gi;
+
+/** A select item that is a star, bare or qualified, with Snowflake's modifiers. */
+const STAR_PATTERN = /(^|\.)\*(\s+(exclude|replace|rename|ilike)\b[\s\S]*)?$/i;
+
+/**
+ * Blank out comments, and the contents of quoted strings and identifiers, so
+ * rules can scan the code with regexes. Same length as `sql`; line breaks
+ * kept; quote characters kept.
+ * @param {string} sql
+ * @param {{ keepStrings?: boolean }} [options]  keep single-quoted string
+ *   contents (for finding date literals). Default false.
+ * @returns {string}
+ */
+export function maskSql(sql, { keepStrings = false } = {}) {
+  const out = [];
+  let i = 0;
+  const n = sql.length;
+
+  while (i < n) {
+    const c = sql[i];
+
+    if (c === "-" && sql[i + 1] === "-") {
+      while (i < n && sql[i] !== "\n") {
+        out.push(" ");
+        i++;
+      }
+      continue;
+    }
+
+    if (c === "/" && sql[i + 1] === "*") {
+      out.push(" ", " ");
+      i += 2;
+      while (i < n && !(sql[i] === "*" && sql[i + 1] === "/")) {
+        out.push(sql[i] === "\n" ? "\n" : " ");
+        i++;
+      }
+      if (i < n) {
+        out.push(" ", " ");
+        i += 2;
+      }
+      continue;
+    }
+
+    if (c === "'" || c === '"') {
+      const quote = c;
+      const keep = keepStrings && quote === "'";
+      out.push(quote);
+      i++;
+      while (i < n) {
+        if (sql[i] === quote) {
+          if (sql[i + 1] === quote) {
+            out.push(keep ? quote : " ", keep ? quote : " ");
+            i += 2;
+            continue;
+          }
+          out.push(quote);
+          i++;
+          break;
+        }
+        out.push(keep ? sql[i] : sql[i] === "\n" ? "\n" : " ");
+        i++;
+      }
+      continue;
+    }
+
+    out.push(c);
+    i++;
+  }
+
+  return out.join("");
+}
+
+/**
+ * Functions that read the clock instead of a literal date: current_date,
+ * current_timestamp, current_time, localtimestamp, localtime, sysdate,
+ * systimestamp, getdate, now. Date arithmetic over literals (`dateadd`) is
+ * pinned and allowed. Whole words, any case, outside strings and comments.
+ * @param {string} sql
+ * @returns {string[]}  each function once, as first written, in order
+ */
+export function dateFunctions(sql) {
+  const seen = new Set();
+  const found = [];
+  for (const match of maskSql(sql).matchAll(DATE_FUNCTION_PATTERN)) {
+    const name = match[0].toLowerCase();
+    if (seen.has(name)) continue;
+    seen.add(name);
+    found.push(match[0]);
+  }
+  return found;
+}
+
+/** Index of each char in `code`: paren depth before that index. */
+function parenDepths(code) {
+  const depths = new Array(code.length);
+  let depth = 0;
+  for (let i = 0; i < code.length; i++) {
+    if (code[i] === ")") depth = Math.max(0, depth - 1);
+    depths[i] = depth;
+    if (code[i] === "(") depth++;
+  }
+  return depths;
+}
+
+/** Escape a string for use inside a regex, so an arbitrary id cannot break it. */
+const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * The mechanical SQL rules for one query.
+ * @param {string} sql
+ * @param {{ parents?: string[] }} [options]  node ids the query must name
+ * @returns {{ rule: string, message: string }[]}  at most one finding per
+ *   rule, in this order: select-star, date-function, parent-ref
+ */
+export function checkSqlStyle(sql, { parents = [] } = {}) {
+  const code = maskSql(sql);
+  const depths = parenDepths(code);
+  const findings = [];
+
+  const clauses = [...code.matchAll(CLAUSE_PATTERN)];
+
+  // select-star: split each select list, at any nesting depth, into items.
+  // A list runs from its "select" (after any "distinct") to the next clause
+  // at the same depth, the parenthesis that closes it, or the end.
+  let star = false;
+  for (const match of clauses) {
+    if (match[0].toLowerCase() !== "select") continue;
+    const depth = depths[match.index];
+    let start = match.index + match[0].length;
+    const distinct = /^\s*\bdistinct\b/i.exec(code.slice(start));
+    if (distinct) start += distinct[0].length;
+    const next = clauses.find(
+      (m) => m.index > match.index && depths[m.index] === depth,
+    );
+    let end = next ? next.index : code.length;
+    for (let i = start; i < end; i++) {
+      if (depths[i] < depth) {
+        end = i;
+        break;
+      }
+    }
+
+    let itemStart = start;
+    for (let i = start; i <= end; i++) {
+      if (i === end || (code[i] === "," && depths[i] === depth)) {
+        if (STAR_PATTERN.test(code.slice(itemStart, i).trim())) star = true;
+        itemStart = i + 1;
+      }
+    }
+  }
+  if (star) {
+    findings.push({
+      rule: "select-star",
+      message: "uses select *; list the columns",
+    });
+  }
+
+  // date-function
+  const dates = dateFunctions(sql);
+  if (dates.length > 0) {
+    findings.push({
+      rule: "date-function",
+      message: `use literal ISO dates, not date functions: ${dates.join(", ")}`,
+    });
+  }
+
+  // parent-ref
+  const missingParents = parents.filter(
+    (id) => !new RegExp(`\\b${escapeRegExp(id)}\\b`, "i").test(code),
+  );
+  if (missingParents.length > 0) {
+    findings.push({
+      rule: "parent-ref",
+      message: `refer to parents by node id: ${missingParents.join(", ")}`,
+    });
+  }
+
+  return findings;
+}

package/src/viewer/client/bridge.js

@@ -0,0 +1,141 @@
+// Runs inside the report, added by the viewer's server to the copy it serves.
+// The report sits in a sandboxed frame, so the panel cannot touch it; the two
+// talk only through postMessage. A classic script: it is inlined in a tag, and
+// the frame's origin is "null", so it cannot import or fetch anything.
+(function () {
+  var BOUND = "[data-lineage]";
+  var CSS_TEXT =
+    BOUND +
+    " { cursor: pointer; transition: outline-color .15s, background-color .15s; outline: 1.5px solid transparent; outline-offset: 2px; border-radius: 2px; }" +
+    " .tl-hover { outline-color: rgba(35, 70, 176, .45) !important; }" +
+    " .tl-peek { outline: 1.5px dashed #111 !important; }" +
+    " .tl-sel { outline: 2px solid #2346b0 !important; background-color: rgba(35, 70, 176, .07) !important; }" +
+    " @keyframes tl-flash { 0% { background-color: rgba(35,70,176,.28); } 100% { background-color: rgba(35,70,176,.07); } }" +
+    " .tl-flash { animation: tl-flash .9s ease-out; }" +
+    " " +
+    BOUND +
+    ":focus-visible { outline: 2px solid #2346b0; }";
+
+  var marked = null; // the id the panel has open
+  var current = null; // the element of it that is outlined
+
+  function post(message) {
+    parent.postMessage(message, "*");
+  }
+
+  function bound() {
+    return Array.prototype.slice.call(document.querySelectorAll(BOUND));
+  }
+
+  function places(id) {
+    return bound().filter(function (el) {
+      return el.getAttribute("data-lineage") === id;
+    });
+  }
+
+  function clear(name) {
+    Array.prototype.forEach.call(
+      document.querySelectorAll("." + name),
+      function (el) {
+        el.classList.remove(name);
+      },
+    );
+  }
+
+  function paint() {
+    clear("tl-sel");
+    if (marked === null) return;
+    var id = marked;
+    if (
+      !current ||
+      !current.isConnected ||
+      current.getAttribute("data-lineage") !== id
+    ) {
+      current = places(id)[0] || null;
+    }
+    if (current) current.classList.add("tl-sel");
+  }
+
+  function open(el) {
+    current = el;
+    post({ lineage: "select", id: el.getAttribute("data-lineage") });
+  }
+
+  function start() {
+    var style = document.createElement("style");
+    style.textContent = CSS_TEXT;
+    document.head.appendChild(style);
+  }
+
+  function ready() {
+    var counts = {};
+    bound().forEach(function (el) {
+      if (!el.hasAttribute("tabindex")) el.setAttribute("tabindex", "0");
+      var id = el.getAttribute("data-lineage");
+      counts[id] = (counts[id] || 0) + 1;
+    });
+    post({ lineage: "ready", counts: counts });
+  }
+
+  document.addEventListener("click", function (e) {
+    var el = e.target.closest && e.target.closest(BOUND);
+    if (!el) return;
+    e.preventDefault();
+    open(el);
+  });
+
+  document.addEventListener("keydown", function (e) {
+    if (e.key === "Escape") return post({ lineage: "back" });
+    if (e.key !== "Enter" && e.key !== " ") return;
+    var el = e.target.closest && e.target.closest(BOUND);
+    if (!el || el !== e.target) return;
+    e.preventDefault();
+    open(el);
+  });
+
+  document.addEventListener("mouseover", function (e) {
+    clear("tl-hover");
+    var el = e.target.closest && e.target.closest(BOUND);
+    if (el) el.classList.add("tl-hover");
+  });
+
+  document.addEventListener("mouseleave", function () {
+    clear("tl-hover");
+  });
+
+  // What the panel asks for: outline the open node, outline one on hover,
+  // scroll to one of a node's places on the page.
+  window.addEventListener("message", function (e) {
+    if (e.source !== parent) return;
+    var m = e.data;
+    if (!m || typeof m !== "object") return;
+    if (m.lineage === "mark") {
+      marked = typeof m.id === "string" ? m.id : null;
+      paint();
+    } else if (m.lineage === "peek") {
+      clear("tl-peek");
+      if (typeof m.id === "string") {
+        places(m.id).forEach(function (el) {
+          el.classList.add("tl-peek");
+        });
+      }
+    } else if (m.lineage === "scroll") {
+      var el = typeof m.id === "string" && places(m.id)[m.index];
+      if (!el) return;
+      current = el;
+      paint();
+      var calm = matchMedia("(prefers-reduced-motion: reduce)").matches;
+      el.scrollIntoView({
+        behavior: calm ? "auto" : "smooth",
+        block: "center",
+      });
+      el.classList.remove("tl-flash");
+      void el.offsetWidth;
+      el.classList.add("tl-flash");
+    }
+  });
+
+  start();
+  if (document.readyState === "complete") ready();
+  else window.addEventListener("load", ready);
+})();