htmlineage 0.0.0 → 0.1.0-alpha.0

package/src/commands/index.js

@@ -0,0 +1,7 @@
+import * as check from "./check.js";
+import * as sql from "./sql.js";
+import * as view from "./view.js";
+
+export const commands = [check, view, sql];
+
+export const byName = new Map(commands.map((c) => [c.name, c]));

package/src/commands/sql.js

@@ -0,0 +1,65 @@
+import { basename } from "node:path";
+
+import { checkRules } from "../check/rules.js";
+import { HtmlineageError, UsageError } from "../errors.js";
+import { readGraph, readText } from "../graph/parse.js";
+import { validateGraph } from "../graph/schema.js";
+import { proseSkip, readPage } from "../html/page.js";
+import { composeSql, refusal } from "../sql/compose.js";
+
+export const name = "sql";
+export const summary = "Compose the report into one runnable SQL script";
+
+export const usage = `htmlineage sql <report.html> [--id <node>] [options]
+
+Joins every source, view and figure in the report's lineage graph into a
+single script a reviewer can paste into the warehouse. External and
+ungrounded nodes are listed as trailing comments. A report that fails
+\`htmlineage check\` is refused.
+
+Options
+  --id <node>      Compose only this node and what it rests on
+  --graph <path>   Graph JSON to read (default: .lineage/<report>.json
+                   beside the report, or else the block already embedded
+                   in it)
+  -h, --help       Show this help`;
+
+export const options = {
+  graph: { type: "string" },
+  id: { type: "string", multiple: true },
+};
+
+export async function run(parsed, io) {
+  const { positionals } = parsed;
+  if (positionals.length === 0) {
+    throw new UsageError("missing the report to compose", { command: name });
+  }
+  if (positionals.length > 1) {
+    throw new UsageError(`expected one report, got ${positionals.length}`, {
+      command: name,
+    });
+  }
+  const ids = parsed.values.id ?? [];
+  if (ids.length > 1) {
+    throw new UsageError(`expected one --id, got ${ids.length}`, {
+      command: name,
+    });
+  }
+  const reportPath = positionals[0];
+  const { graph } = readGraph({ reportPath, graphPath: parsed.values.graph });
+  const [id] = ids;
+  if (id !== undefined && !Object.hasOwn(graph?.nodes ?? {}, id)) {
+    throw new UsageError(`no node \`${id}\` in ${basename(reportPath)}`, {
+      command: name,
+    });
+  }
+  const html = readText(reportPath, "report");
+  const page = readPage(html, { skip: proseSkip(graph) });
+  const issues = [...validateGraph(graph), ...checkRules(graph, page)];
+  const report = basename(reportPath);
+  if (issues.some((issue) => issue.severity === "error")) {
+    throw new HtmlineageError(refusal(report));
+  }
+  io.out(composeSql(graph, { report, id }));
+  return 0;
+}

package/src/commands/view.js

@@ -0,0 +1,80 @@
+import { basename } from "node:path";
+
+import { UsageError } from "../errors.js";
+import { buildPayload } from "../viewer/payload.js";
+import { browserCommand, openBrowser, startServer } from "../viewer/server.js";
+
+const DEFAULT_PORT = 7171;
+
+export const name = "view";
+export const summary = "Serve the report beside its lineage";
+
+export const usage = `htmlineage view <report.html> [options]
+
+Serves a two-pane page on localhost: the report on the left, the lineage of
+whatever you click on the right. Reads the report and its graph, nothing
+else. No database, no credentials. The viewer itself makes no request off
+this machine; the report loads what it would load in any browser.
+
+Options
+  --port <number>  Port to serve on (default: the first free port from 7171)
+  --no-open        Print the URL instead of opening a browser
+  --graph <path>   Graph JSON to read (default: .lineage/<report>.json
+                   beside the report, or else the block already embedded
+                   in it)
+  -h, --help       Show this help`;
+
+export const options = {
+  port: { type: "string" },
+  "no-open": { type: "boolean", default: false },
+  graph: { type: "string" },
+};
+
+export async function run(parsed, io) {
+  const { positionals } = parsed;
+  if (positionals.length === 0) {
+    throw new UsageError("missing the report to view", { command: name });
+  }
+  if (positionals.length > 1) {
+    throw new UsageError(`expected one report, got ${positionals.length}`, {
+      command: name,
+    });
+  }
+  const { port } = parsed.values;
+  if (port !== undefined && !(/^\d+$/.test(port) && Number(port) <= 65535)) {
+    throw new UsageError(
+      `--port must be a whole number from 0 to 65535, got \`${port}\``,
+      { command: name },
+    );
+  }
+  const reportPath = positionals[0];
+  const graphPath = parsed.values.graph;
+  // Read everything once before listening: an unreadable graph exits 1
+  // without ever printing a URL.
+  buildPayload({ reportPath, graphPath });
+  const server = await startServer({
+    reportPath,
+    graphPath,
+    port: port === undefined ? DEFAULT_PORT : Number(port),
+    exact: port !== undefined,
+  });
+  io.out(`Serving ${basename(reportPath)} at ${server.url}`);
+  io.out("Press Ctrl-C to stop.");
+  if (!parsed.values["no-open"]) {
+    const open =
+      io.open ?? ((url) => openBrowser(browserCommand(process.platform, url)));
+    // A browser that will not open is not an error: the URL is printed.
+    open(server.url);
+  }
+  await stopped(io.signal);
+  await server.close();
+  return 0;
+}
+
+/** Resolves when `signal` aborts; without one, on the first Ctrl-C. */
+function stopped(signal) {
+  return new Promise((resolve) => {
+    if (signal === undefined) return process.once("SIGINT", resolve);
+    signal.addEventListener("abort", resolve, { once: true });
+  });
+}

package/src/errors.js

@@ -0,0 +1,39 @@
+/**
+ * Exit codes. These are part of the CLI contract: the skill and the Claude
+ * Code hook both branch on them, so they do not change without a major bump.
+ *
+ *   0  the command did what it was asked to do (warnings are still 0)
+ *   1  the command ran but the report or graph did not pass
+ *   2  the command was invoked wrongly and never ran
+ */
+export const EXIT_OK = 0;
+export const EXIT_FAILED = 1;
+export const EXIT_USAGE = 2;
+
+/** Bad invocation: unknown command, missing argument, unreadable flag. */
+export class UsageError extends Error {
+  constructor(message, { command } = {}) {
+    super(message);
+    this.name = "UsageError";
+    this.command = command ?? null;
+    this.exitCode = EXIT_USAGE;
+  }
+}
+
+/** The command ran and reported a real failure. */
+export class HtmlineageError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "HtmlineageError";
+    this.exitCode = EXIT_FAILED;
+  }
+}
+
+/** A command whose surface is fixed but whose body is not built yet. */
+export class NotImplementedError extends HtmlineageError {
+  constructor(command) {
+    super(`\`htmlineage ${command}\` is not implemented yet.`);
+    this.name = "NotImplementedError";
+    this.command = command;
+  }
+}

package/src/graph/duplicates.js

@@ -0,0 +1,78 @@
+/**
+ * Find keys that appear twice in the same JSON object.
+ *
+ * JSON.parse keeps the last of two equal keys and drops the first without a
+ * word. In a graph the model writes by hand, a repeated node id or field is
+ * an easy slip, and the silent loss of a node is the worst way to find out.
+ * This scans text JSON.parse has already accepted, so it can assume the
+ * input is well formed and only needs to track where it is.
+ *
+ * Returns `[{ path, first, second }]`: `path` like "nodes.f3" or
+ * "nodes.f3.sql", and the character offsets of both occurrences.
+ */
+export function findDuplicateKeys(text) {
+  const duplicates = [];
+  const stack = [];
+  // The key or index a new container will be stored under in its parent.
+  const childPath = () => {
+    const top = stack[stack.length - 1];
+    if (!top) return [];
+    return [...top.path, top.object ? top.key : top.index];
+  };
+
+  let i = 0;
+  while (i < text.length) {
+    const char = text[i];
+    const top = stack[stack.length - 1];
+
+    if (char === "{" || char === "[") {
+      stack.push({
+        object: char === "{",
+        path: childPath(),
+        keys: new Map(),
+        expectKey: true,
+        key: null,
+        index: 0,
+      });
+      i++;
+    } else if (char === "}" || char === "]") {
+      stack.pop();
+      i++;
+    } else if (char === ":") {
+      top.expectKey = false;
+      i++;
+    } else if (char === ",") {
+      if (top.object) top.expectKey = true;
+      else top.index++;
+      i++;
+    } else if (char === '"') {
+      const end = stringEnd(text, i);
+      if (top?.object && top.expectKey) {
+        const key = JSON.parse(text.slice(i, end));
+        if (top.keys.has(key)) {
+          duplicates.push({
+            path: [...top.path, key].join("."),
+            first: top.keys.get(key),
+            second: i,
+          });
+        } else {
+          top.keys.set(key, i);
+        }
+        top.key = key;
+      }
+      i = end;
+    } else {
+      i++;
+    }
+  }
+  return duplicates;
+}
+
+/** Offset just past the closing quote of the string starting at `start`. */
+function stringEnd(text, start) {
+  let i = start + 1;
+  while (text[i] !== '"') {
+    i += text[i] === "\\" ? 2 : 1;
+  }
+  return i + 1;
+}

package/src/graph/expr.js

@@ -0,0 +1,54 @@
+/**
+ * A figure's `expr`: a SQL scalar expression over figure ids, passed through
+ * as written. htmlineage has no grammar for it. It only finds the ids the text
+ * names as bare words, and says whether the text can be read at all (brackets
+ * and quotes close, and no comment).
+ */
+
+/**
+ * The ids from `ids` that the text names as bare words (not inside quotes,
+ * and not a function name before `(`), distinct, in first-seen order.
+ */
+export function exprRefs(source, ids) {
+  const known = new Set(ids);
+  const seen = new Set();
+  const bare = source.replace(/'[^']*'|"[^"]*"/g, " ");
+  for (const [word] of bare.matchAll(/[A-Za-z_][A-Za-z0-9_]*\b(?!\s*\()/g)) {
+    if (known.has(word)) seen.add(word);
+  }
+  return [...seen];
+}
+
+/**
+ * Whether the text can be read at all: brackets and quotes close, and there
+ * is no comment. Returns null, or { message, position } (0-based).
+ */
+export function exprProblem(source) {
+  const open = [];
+  for (let i = 0; i < source.length; i++) {
+    const char = source[i];
+    if (char === "'" || char === '"') {
+      const end = source.indexOf(char, i + 1);
+      if (end === -1) {
+        return { message: `\`${char}\` is never closed`, position: i };
+      }
+      i = end;
+    } else if (source.startsWith("--", i) || source.startsWith("/*", i)) {
+      const mark = source.slice(i, i + 2);
+      return {
+        message: `\`${mark}\` starts a comment; put notes in the node's \`label\``,
+        position: i,
+      };
+    } else if (source[i] === "(") open.push(i);
+    else if (source[i] === ")") {
+      if (open.length === 0) {
+        return { message: "`)` has no matching `(`", position: i };
+      }
+      open.pop();
+    }
+  }
+  if (open.length > 0) {
+    return { message: "`(` is never closed", position: open.pop() };
+  }
+  return null;
+}

package/src/graph/ids.js

@@ -0,0 +1,24 @@
+/**
+ * Node ids double as table names in the composed script (`from v1`), so they
+ * must be plain SQL identifiers: lowercase, starting with a letter.
+ */
+export const ID_PATTERN = /^[a-z][a-z0-9_]*$/;
+
+/**
+ * The conventional first letter for each kind of node. A mismatch is only a
+ * warning: the id still works, it just reads worse in the viewer and script.
+ */
+export function expectedPrefix(node) {
+  switch (node.step) {
+    case "source":
+      return node.kind === "external" ? "x" : "q";
+    case "view":
+      return "v";
+    case "figure":
+      return "f";
+    case "insight":
+      return "i";
+    default:
+      return null;
+  }
+}

package/src/graph/numbers.js

@@ -0,0 +1,61 @@
+/**
+ * What counts as a number in prose, for rule C2.
+ *
+ * C2 says every number a report states must sit inside a figure's span. That
+ * only works if "number" means "a claim someone could check", not "any
+ * digit". This module is that definition, on plain text; `check` decides
+ * which text to feed it (prose outside figure spans, outside bound charts
+ * and tables, outside ungrounded insights).
+ *
+ * Counts:  46.6%   $61.20   4,120   +11.8%   #1   3rd   2.5x   $2bn   12 pp
+ * Exempt:  2026 (a bare year)   2026-08-01 (an ISO date)
+ *          30-day, 12-month (a number hyphenated to a time unit: a metric's
+ *          definition, not a claim about it)
+ * Ignored: Q3, H1, FY26, B2B (digits inside a word are part of a name)
+ *
+ * Deliberately not exempt: "12-point gain" is a claim. "Over 30 days" is
+ * ambiguous and counts; the skill tells the model to write "30-day".
+ */
+
+const ISO_DATE = /(?<![\w-])\d{4}-\d{2}-\d{2}(?![\w-])/g;
+
+const NUMBER =
+  /(?<![\w.,])[+\-−]?[$€£¥]?#?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?:\s?(?:%|pp|bps)|bn|st|nd|rd|th|[xkmb])?(?![\w%])/gi;
+
+const YEAR = /^(?:19|20)\d{2}$/;
+const TIME_QUALIFIER =
+  /^[-‑](?:second|minute|hour|day|week|month|quarter|year)s?\b/i;
+
+/**
+ * Every number-like token in `text`, in order, as
+ * `{ text, index, exempt }` where `exempt` is null for a token C2 counts,
+ * or the reason it does not: "date", "year" or "qualifier".
+ */
+export function findNumbers(text) {
+  const tokens = [];
+  const dates = [];
+  for (const match of text.matchAll(ISO_DATE)) {
+    dates.push([match.index, match.index + match[0].length]);
+    tokens.push({ text: match[0], index: match.index, exempt: "date" });
+  }
+  const inDate = (index) =>
+    dates.some(([start, end]) => index >= start && index < end);
+
+  for (const match of text.matchAll(NUMBER)) {
+    if (inDate(match.index)) continue;
+    const token = match[0];
+    const after = text.slice(match.index + token.length);
+    let exempt = null;
+    if (YEAR.test(token)) exempt = "year";
+    else if (/^\d+$/.test(token) && TIME_QUALIFIER.test(after)) {
+      exempt = "qualifier";
+    }
+    tokens.push({ text: token, index: match.index, exempt });
+  }
+  return tokens.sort((a, b) => a.index - b.index);
+}
+
+/** Only the tokens C2 counts. */
+export function bareNumbers(text) {
+  return findNumbers(text).filter((token) => token.exempt === null);
+}

package/src/graph/parse.js

@@ -0,0 +1,196 @@
+/**
+ * Finding and reading a report's lineage graph.
+ *
+ * A graph lives in one of two places:
+ *
+ *   sidecar   .lineage/<report-stem>.json beside the report. The model
+ *             writes this during the build; it is the working copy.
+ *   embedded  <script type="application/lineage+json"> inside the report.
+ *             `check` writes this; it is what travels when the file is sent.
+ *
+ * Both hold the same object. When both exist the sidecar wins, because a
+ * build in progress has not been re-embedded yet.
+ *
+ * The HTML is scanned as text, not parsed: the graph block is the one element
+ * htmlineage owns, and `check` will replace it in place using the offsets
+ * returned here, without reserializing anything else in the page.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { basename, dirname, extname, join, relative } from "node:path";
+
+import { HtmlineageError } from "../errors.js";
+import { findDuplicateKeys } from "./duplicates.js";
+
+export const GRAPH_MIME = "application/lineage+json";
+export const GRAPH_ELEMENT_ID = "lineage-graph";
+export const SIDECAR_DIR = ".lineage";
+
+/** `reports/aug.html` → `reports/.lineage/aug.json`. The report need not exist. */
+export function sidecarPath(reportPath) {
+  const stem = basename(reportPath, extname(reportPath));
+  return join(dirname(reportPath), SIDECAR_DIR, `${stem}.json`);
+}
+
+const OPEN_SCRIPT = /<script\b[^>]*>/gi;
+// HTML ends a script element only at `</script` followed by whitespace, `/`,
+// `>` or the end of input, so `</scripture>` inside the JSON does not close it.
+const CLOSE_SCRIPT = /<\/script(?=[\s/>]|$)/gi;
+const TYPE_ATTR =
+  /\stype\s*=\s*(["']?)application\/lineage\+json\1(?=[\s/>]|$)/i;
+
+/**
+ * Locate the embedded graph block in an HTML string. Returns null when there
+ * is none, or `{ start, end, contentStart, contentEnd, content }` where
+ * start..end spans the whole element and contentStart..contentEnd its JSON.
+ * Throws when the page holds more than one block, since then it is unclear
+ * which one is the report's lineage.
+ */
+export function findEmbeddedGraph(html) {
+  const blocks = [];
+  for (const match of html.matchAll(OPEN_SCRIPT)) {
+    if (!TYPE_ATTR.test(match[0])) continue;
+    const contentStart = match.index + match[0].length;
+    CLOSE_SCRIPT.lastIndex = contentStart;
+    const contentEnd = CLOSE_SCRIPT.exec(html)?.index ?? -1;
+    if (contentEnd === -1) {
+      throw new HtmlineageError(
+        `the embedded lineage graph at ${lineCol(html, match.index)} is never closed with </script>`,
+      );
+    }
+    const closeEnd = html.indexOf(">", contentEnd);
+    blocks.push({
+      start: match.index,
+      end: closeEnd === -1 ? html.length : closeEnd + 1,
+      contentStart,
+      contentEnd,
+      content: html.slice(contentStart, contentEnd),
+    });
+  }
+  if (blocks.length > 1) {
+    const where = blocks.map((b) => lineCol(html, b.start)).join(", ");
+    throw new HtmlineageError(
+      `the report holds ${blocks.length} embedded lineage graphs (at ${where}); expected one`,
+    );
+  }
+  return blocks[0] ?? null;
+}
+
+/**
+ * Turn a graph into the text that goes inside the embedded <script>. Every
+ * `<` is escaped so that SQL containing `</script>` or `<!--` cannot end the
+ * block early; JSON.parse reads the escape back as the same character.
+ */
+export function serializeForEmbed(graph) {
+  return JSON.stringify(graph, null, 2).replace(/</g, "\\u003c");
+}
+
+/**
+ * Parse graph JSON, turning a syntax error into one that names the file and
+ * the line and column. `offset` is where `text` starts inside `source`, so
+ * positions inside an embedded block are reported as positions in the page.
+ * A key repeated within one object is also an error: JSON.parse would keep
+ * the second and silently drop the first, losing a node or a field.
+ */
+export function parseGraphText(text, { label, source = text, offset = 0 }) {
+  let graph;
+  try {
+    graph = JSON.parse(text);
+  } catch (error) {
+    if (!(error instanceof SyntaxError)) throw error;
+    const position = /\bposition (\d+)/.exec(error.message);
+    const where = position
+      ? ` at ${lineCol(source, offset + Number(position[1]))}`
+      : "";
+    const reason = error.message.replace(
+      /\s*(in JSON )?at position \d+.*$/,
+      "",
+    );
+    throw new HtmlineageError(`${label} is not valid JSON${where}: ${reason}`);
+  }
+
+  const duplicates = findDuplicateKeys(text);
+  if (duplicates.length > 0) {
+    const lines = duplicates.map(
+      ({ path, first, second }) =>
+        `  \`${path}\` at ${lineOf(source, offset + first)} and ${lineOf(source, offset + second)}`,
+    );
+    throw new HtmlineageError(
+      `${label} defines the same key twice, so the first would be silently dropped:\n${lines.join("\n")}`,
+    );
+  }
+  return graph;
+}
+
+/**
+ * Read a report's graph. With `graphPath`, read exactly that file. Otherwise
+ * try the sidecar, then the block embedded in the report. Returns
+ * `{ graph, origin: "file" | "sidecar" | "embedded", path }`.
+ */
+export function readGraph({ reportPath, graphPath } = {}) {
+  if (graphPath) {
+    return { graph: readJsonFile(graphPath), origin: "file", path: graphPath };
+  }
+  if (!reportPath) {
+    throw new HtmlineageError("no report or graph file given");
+  }
+
+  const sidecar = sidecarPath(reportPath);
+  if (existsSync(sidecar)) {
+    return { graph: readJsonFile(sidecar), origin: "sidecar", path: sidecar };
+  }
+
+  const html = readText(reportPath, "report");
+  const block = findEmbeddedGraph(html);
+  if (!block) {
+    throw new HtmlineageError(
+      `no lineage graph found for ${display(reportPath)}. Looked for ` +
+        `${display(sidecar)} and a <script type="${GRAPH_MIME}"> block in the report.`,
+    );
+  }
+  const graph = parseGraphText(block.content, {
+    label: `the lineage graph embedded in ${display(reportPath)}`,
+    source: html,
+    offset: block.contentStart,
+  });
+  return { graph, origin: "embedded", path: reportPath };
+}
+
+function readJsonFile(path) {
+  return parseGraphText(readText(path, "graph file"), { label: display(path) });
+}
+
+/** Read a file as text; failures become `HtmlineageError`s worded for the builder. */
+export function readText(path, what) {
+  try {
+    return readFileSync(path, "utf8");
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      throw new HtmlineageError(`${what} not found: ${display(path)}`);
+    }
+    throw new HtmlineageError(
+      `could not read ${display(path)}: ${error.message}`,
+    );
+  }
+}
+
+function display(path) {
+  const rel = relative(process.cwd(), path);
+  return rel && !rel.startsWith("..") ? rel : path;
+}
+
+function lineOf(text, index) {
+  return lineCol(text, index).replace(/, column \d+$/, "");
+}
+
+function lineCol(text, index) {
+  let line = 1;
+  let lineStart = 0;
+  for (let i = 0; i < index && i < text.length; i++) {
+    if (text[i] === "\n") {
+      line++;
+      lineStart = i + 1;
+    }
+  }
+  return `line ${line}, column ${index - lineStart + 1}`;
+}