@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/app-dev/data-catalog/scripts/data_catalog.py

@@ -0,0 +1,1392 @@
+#!/usr/bin/env python3
+"""Data catalog engine: read column metadata from table scripts, generate the catalog, audit scripts.
+
+Commands:
+  init      set a repo up: config, this engine, the Field/Table module, CI workflow
+  generate  write docs/catalog/*.md from the scripts; --check fails when they are stale
+  audit     report scripts that write tables without complete column definitions (never edits)
+  drift     local check: code changed since a git base while its definitions did not
+  version   print the engine identity
+
+Metadata carriers:
+  Python  module-level CATALOG_TABLE = Table(...) and CATALOG_FIELDS = (Field(...), ...),
+          read with ast (the script is never imported, so Spark is not needed)
+  SQL     a comment block of `-- @table <name>` and `-- @field <name>` entries
+
+Standard library only. Copied into a repo by `init`; refresh it the same way.
+"""
+from __future__ import annotations
+
+import argparse
+import ast
+import hashlib
+import io
+import json
+import os
+import re
+import subprocess
+import sys
+import tokenize
+from dataclasses import dataclass, field
+from pathlib import Path
+
+ENGINE_VERSION = "1.0.0"
+CONFIG_FILE = "data-catalog.json"
+ENGINE_PATH = "scripts/data-catalog/data_catalog.py"
+WORKFLOW_PATH = ".github/workflows/data-catalog.yml"
+GENERATED_MARK = "<!-- generated by data_catalog.py: edit the table script, then run generate -->"
+WORKFLOW_MARK = "# generated by data_catalog.py init: edit data-catalog.json and rerun init"
+TABLE_VAR = "CATALOG_TABLE"
+FIELDS_VAR = "CATALOG_FIELDS"
+DEFAULT_CONFIG = {
+    "version": 1,
+    "scripts": [],
+    "exclude": [],
+    "output": "docs/catalog",
+    "languages": ["en"],
+    "contracts": ["PUBLISHED_COLUMNS", "OUTPUT_COLUMNS"],
+    "fieldsModule": None,
+    "tableNames": [],
+    "schemaNames": [],
+}
+FIELD_EXTRAS = ("source", "window", "unit")
+LANGUAGE_LABELS = {"en": "English", "zh": "中文", "zh_hant": "繁體中文", "ja": "日本語", "ko": "한국어"}
+LANGUAGE_RE = re.compile(r"^[a-z]{2}(_[a-z]+)?$")
+SKIP_DIRS = {"node_modules", "__pycache__", "venv"}
+# Assignments (module or class body) whose string value names the table a script writes,
+# and the schema/database that may sit beside it (`dst_db = "dwd"`, `dst_tbl = "orders"`).
+TABLE_NAME_RE = re.compile(r"^(?:\w*_)?(?:target|output|sink|dest|destination|dst)_(?:table|tbl)(?:_name)?$|^table_name$", re.I)
+SCHEMA_NAME_RE = re.compile(r"^(?:target|output|sink|dest|destination|dst)_(?:db|database|schema)(?:_name)?$", re.I)
+SQL_KEYWORDS = {
+    "end", "from", "where", "and", "or", "not", "null", "then", "else", "when", "case", "as",
+    "on", "join", "by", "asc", "desc", "over", "distinct", "is", "in", "like", "between", "with", "select",
+    "union", "all", "partition", "rows", "range", "interval", "true", "false",
+}
+
+
+class CatalogError(Exception):
+    """A metadata or config problem, reported as `path:line: message`."""
+
+
+# ---------------------------------------------------------------------------
+# Config and file discovery
+# ---------------------------------------------------------------------------
+
+def read_config_file(root: Path) -> dict:
+    try:
+        data = json.loads((root / CONFIG_FILE).read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise CatalogError(f"{CONFIG_FILE}: invalid JSON: {exc}") from exc
+    if not isinstance(data, dict):
+        raise CatalogError(f"{CONFIG_FILE}: must be a JSON object")
+    return data
+
+
+def load_config(root: Path) -> dict:
+    if not (root / CONFIG_FILE).exists():
+        raise CatalogError(f"{CONFIG_FILE} not found in {root}; run `init` first")
+    config = {**DEFAULT_CONFIG, **read_config_file(root)}
+    validate_config(config)
+    return config
+
+
+def validate_config(config: dict) -> None:
+    config["output"] = str(config.get("output") or DEFAULT_CONFIG["output"]).strip().strip("/") or DEFAULT_CONFIG["output"]
+    if not config["scripts"]:
+        raise CatalogError(f"{CONFIG_FILE}: `scripts` must list at least one glob")
+    langs = config["languages"]
+    if not langs or not all(isinstance(lang, str) and LANGUAGE_RE.match(lang) for lang in langs):
+        raise CatalogError(f"{CONFIG_FILE}: `languages` must be codes like \"en\" or \"zh\", got {langs!r}")
+    clash = [lang for lang in langs if lang in FIELD_EXTRAS or lang in ("name", "grain", "notes")]
+    if clash:
+        raise CatalogError(f"{CONFIG_FILE}: language codes clash with field arguments: {clash}")
+
+
+def glob_regex(pattern: str) -> re.Pattern:
+    out, i = [], 0
+    while i < len(pattern):
+        if pattern.startswith("**/", i):
+            out.append("(?:.*/)?"); i += 3
+        elif pattern.startswith("**", i):
+            out.append(".*"); i += 2
+        elif pattern[i] == "*":
+            out.append("[^/]*"); i += 1
+        elif pattern[i] == "?":
+            out.append("[^/]"); i += 1
+        else:
+            out.append(re.escape(pattern[i])); i += 1
+    return re.compile("".join(out) + r"\Z")
+
+
+def find_scripts(root: Path, patterns: list[str], exclude: list[str]) -> list[Path]:
+    """One pruned walk: hidden directories and node_modules are never entered."""
+    wanted = [glob_regex(p) for p in patterns]
+    skipped = [glob_regex(p) for p in exclude]
+    found = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = sorted(d for d in dirnames if not d.startswith(".") and d not in SKIP_DIRS)
+        for name in filenames:
+            if not name.endswith((".py", ".sql")):
+                continue
+            path = Path(dirpath) / name
+            relpath = rel(root, path)
+            if any(r.match(relpath) for r in wanted) and not any(r.match(relpath) for r in skipped):
+                found.append(path)
+    return sorted(found)
+
+
+def rel(root: Path, path: Path) -> str:
+    return path.relative_to(root).as_posix()
+
+
+# ---------------------------------------------------------------------------
+# Metadata model
+# ---------------------------------------------------------------------------
+
+@dataclass
+class FieldDoc:
+    name: str
+    text: dict
+    source: str
+    window: str | None
+    unit: str | None
+    line: int
+
+
+@dataclass
+class TableDoc:
+    name: str
+    script: str
+    grain: str
+    text: dict
+    notes: list
+    fields: list
+    line: int
+    contracts: dict = field(default_factory=dict)  # name -> list[str] | "derived" | "unchecked"
+
+
+# ---------------------------------------------------------------------------
+# Python carrier (ast, never imported)
+# ---------------------------------------------------------------------------
+
+def module_assignments(tree: ast.Module) -> dict:
+    out = {}
+    for node in tree.body:
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name):
+                    out[target.id] = node.value
+        elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None:
+            out[node.target.id] = node.value
+    return out
+
+
+def call_name(node: ast.AST) -> str | None:
+    if not isinstance(node, ast.Call):
+        return None
+    if isinstance(node.func, ast.Name):
+        return node.func.id
+    if isinstance(node.func, ast.Attribute):
+        return node.func.attr
+    return None
+
+
+def eval_text(node: ast.AST, names: dict, where: str, seen: frozenset = frozenset()):
+    """Evaluate a string, tuple of strings, or None built from literals and module constants."""
+    if isinstance(node, ast.Constant) and (isinstance(node.value, str) or node.value is None):
+        return node.value
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        left = eval_text(node.left, names, where, seen)
+        right = eval_text(node.right, names, where, seen)
+        if isinstance(left, str) and isinstance(right, str):
+            return left + right
+        if isinstance(left, tuple) and isinstance(right, tuple):
+            return left + right
+    if isinstance(node, (ast.Tuple, ast.List)):
+        return tuple(eval_text(elt, names, where, seen) for elt in node.elts)
+    if isinstance(node, ast.Name) and node.id in names and node.id not in seen:
+        return eval_text(names[node.id], names, where, seen | {node.id})
+    raise CatalogError(f"{where}:{getattr(node, 'lineno', '?')}: metadata must be a string literal "
+                       "(or module-level string constant), not an expression")
+
+
+def resolve_sequence(node: ast.AST, names: dict, where: str, seen: frozenset = frozenset()) -> list:
+    """Flatten a tuple/list (with *NAME, A + B, NAME) into its element nodes."""
+    if isinstance(node, (ast.Tuple, ast.List)):
+        out = []
+        for elt in node.elts:
+            if isinstance(elt, ast.Starred):
+                out.extend(resolve_sequence(elt.value, names, where, seen))
+            else:
+                out.append(elt)
+        return out
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        return resolve_sequence(node.left, names, where, seen) + resolve_sequence(node.right, names, where, seen)
+    if isinstance(node, ast.Name) and node.id in names and node.id not in seen:
+        return resolve_sequence(names[node.id], names, where, seen | {node.id})
+    raise CatalogError(f"{where}:{getattr(node, 'lineno', '?')}: {FIELDS_VAR} must be a tuple of Field(...) "
+                       "built in this module (literals, *NAME or A + B); imported lists cannot be read")
+
+
+def call_kwargs(call: ast.Call, names: dict, where: str, kind: str) -> dict:
+    if len(call.args) > 1 or any(isinstance(a, ast.Starred) for a in call.args):
+        raise CatalogError(f"{where}:{call.lineno}: pass only the name positionally to {kind}(...); "
+                           "everything else by keyword")
+    values = {}
+    if call.args:
+        values["name"] = eval_text(call.args[0], names, where)
+    for kw in call.keywords:
+        if kw.arg is None:
+            raise CatalogError(f"{where}:{call.lineno}: **kwargs is not readable in {kind}(...)")
+        values[kw.arg] = eval_text(kw.value, names, where)
+    return values
+
+
+def require_text(values: dict, key: str, where: str, line: int, label: str) -> str:
+    value = values.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise CatalogError(f"{where}:{line}: {label} is missing `{key}`")
+    return value.strip()
+
+
+def build_field(values: dict, languages: list, where: str, line: int) -> FieldDoc:
+    name = require_text(values, "name", where, line, "Field")
+    known = {"name", *FIELD_EXTRAS, *languages}
+    unknown = sorted(set(values) - known)
+    if unknown:
+        raise CatalogError(f"{where}:{line}: Field {name!r} has unknown argument(s) {unknown}; "
+                           f"languages configured: {languages}")
+    text = {lang: require_text(values, lang, where, line, f"Field {name!r}") for lang in languages}
+    source = require_text(values, "source", where, line, f"Field {name!r}")
+    return FieldDoc(name, text, source, (values.get("window") or None), (values.get("unit") or None), line)
+
+
+def build_table(values: dict, fields: list, languages: list, where: str, line: int, script: str) -> TableDoc:
+    name = require_text(values, "name", where, line, "Table")
+    known = {"name", "grain", "notes", *languages}
+    unknown = sorted(set(values) - known)
+    if unknown:
+        raise CatalogError(f"{where}:{line}: Table {name!r} has unknown argument(s) {unknown}")
+    grain = require_text(values, "grain", where, line, f"Table {name!r}")
+    text = {lang: require_text(values, lang, where, line, f"Table {name!r}") for lang in languages}
+    notes = values.get("notes") or ()
+    if isinstance(notes, str):
+        notes = (notes,)
+    return TableDoc(name, script, grain, text, [n.strip() for n in notes if n and n.strip()], fields, line)
+
+
+def contract_state(node: ast.AST, names: dict):
+    if any(isinstance(n, ast.Name) and n.id == FIELDS_VAR for n in ast.walk(node)):
+        return "derived"
+    try:
+        items = resolve_sequence(node, names, "", frozenset())
+        values = [eval_text(item, names, "") for item in items]
+    except CatalogError:
+        return "unchecked"
+    if all(isinstance(v, str) for v in values):
+        return values
+    return "unchecked"
+
+
+def parse_python(text: str, where: str, languages: list, contracts: list) -> TableDoc | None:
+    try:
+        tree = ast.parse(text)
+    except SyntaxError as exc:
+        raise CatalogError(f"{where}:{exc.lineno}: cannot parse: {exc.msg}") from exc
+    names = module_assignments(tree)
+    table_node, fields_node = names.get(TABLE_VAR), names.get(FIELDS_VAR)
+    if table_node is None and fields_node is None:
+        return None
+    if table_node is None or fields_node is None:
+        missing = TABLE_VAR if table_node is None else FIELDS_VAR
+        raise CatalogError(f"{where}:1: has one of {TABLE_VAR}/{FIELDS_VAR} but not {missing}")
+    if call_name(table_node) != "Table":
+        raise CatalogError(f"{where}:{table_node.lineno}: {TABLE_VAR} must be Table(...)")
+    fields = []
+    for item in resolve_sequence(fields_node, names, where):
+        if call_name(item) != "Field":
+            raise CatalogError(f"{where}:{getattr(item, 'lineno', '?')}: every {FIELDS_VAR} entry must be Field(...)")
+        fields.append(build_field(call_kwargs(item, names, where, "Field"), languages, where, item.lineno))
+    table = build_table(call_kwargs(table_node, names, where, "Table"), fields, languages, where, table_node.lineno, where)
+    for contract in contracts:
+        if contract in names:
+            table.contracts[contract] = contract_state(names[contract], names)
+    return table
+
+
+# ---------------------------------------------------------------------------
+# SQL carrier: `-- @table` / `-- @field` comment blocks
+# ---------------------------------------------------------------------------
+
+SQL_DIRECTIVE_RE = re.compile(r"^@(table|field)\s+(\S+)\s*$")
+SQL_DIRECTIVE_START_RE = re.compile(r"^@(table|field)\b")
+SQL_KEY_RE = re.compile(r"^  ([a-z][a-z_]*):\s?(.*)$")
+SQL_CONTINUE_RE = re.compile(r"^ {4,}(\S.*)$")
+
+
+def parse_sql(text: str, where: str, languages: list) -> TableDoc | None:
+    entries = []  # [kind, name, values, line, last_key]
+    current = None
+    for lineno, raw in enumerate(text.splitlines(), 1):
+        stripped = raw.lstrip()
+        if not stripped.startswith("--"):
+            if stripped:
+                break  # the block sits above the code; a directive after the first statement is ignored
+            current = None
+            continue
+        body = stripped[2:]
+        body = body[1:] if body.startswith(" ") else body
+        directive = SQL_DIRECTIVE_RE.match(body)
+        if not directive and SQL_DIRECTIVE_START_RE.match(body):
+            raise CatalogError(f"{where}:{lineno}: write `-- @table <name>` or `-- @field <name>` with nothing after the name")
+        if directive:
+            current = [directive.group(1), directive.group(2), {}, lineno, None]
+            entries.append(current)
+            continue
+        if current is None:
+            continue
+        key = SQL_KEY_RE.match(body)
+        cont = SQL_CONTINUE_RE.match(body)
+        if key:
+            current[4] = key.group(1)
+            current[2][key.group(1)] = key.group(2).strip()
+        elif cont and current[4]:
+            prev = current[2][current[4]]
+            current[2][current[4]] = f"{prev} {cont.group(1).strip()}".strip()
+        else:
+            current = None
+    if not entries:
+        return None
+    tables = [e for e in entries if e[0] == "table"]
+    if len(tables) != 1:
+        raise CatalogError(f"{where}:{entries[0][3]}: a SQL file needs exactly one `-- @table`, found {len(tables)}")
+    fields = []
+    for kind, name, values, line, _ in entries:
+        if kind == "field":
+            if "notes" in values:
+                raise CatalogError(f"{where}:{line}: `notes` belongs on @table, not @field")
+            fields.append(build_field({"name": name, **values}, languages, where, line))
+    _, name, values, line, _ = tables[0]
+    notes = values.pop("notes", None)
+    values = {"name": name, **values}
+    if notes:
+        values["notes"] = tuple(n.strip() for n in notes.split(";"))
+    return build_table(values, fields, languages, where, line, where)
+
+
+def parse_script(path: Path, root: Path, config: dict) -> TableDoc | None:
+    where = rel(root, path)
+    text = path.read_text(encoding="utf-8")
+    if path.suffix == ".py":
+        return parse_python(text, where, config["languages"], config["contracts"])
+    return parse_sql(text, where, config["languages"])
+
+
+# ---------------------------------------------------------------------------
+# Validation and rendering
+# ---------------------------------------------------------------------------
+
+def collect(root: Path, config: dict):
+    """Parse every configured script; returns (tables, errors, warnings)."""
+    tables, errors, warnings = [], [], []
+    for path in find_scripts(root, config["scripts"], config["exclude"]):
+        try:
+            table = parse_script(path, root, config)
+        except CatalogError as exc:
+            errors.append(str(exc))
+            continue
+        except (UnicodeDecodeError, OSError) as exc:
+            errors.append(f"{rel(root, path)}:1: cannot read: {exc}")
+            continue
+        if table is None:
+            continue
+        seen = {}
+        for f in table.fields:
+            if f.name in seen:
+                errors.append(f"{table.script}:{f.line}: field {f.name!r} is defined twice (first at line {seen[f.name]})")
+            seen.setdefault(f.name, f.line)
+        for contract, state in table.contracts.items():
+            if isinstance(state, list):
+                missing = [c for c in state if c not in seen]
+                extra = [n for n in seen if n not in state]
+                if missing:
+                    errors.append(f"{table.script}:{table.line}: {contract} has column(s) with no Field: {', '.join(missing)}")
+                if extra:
+                    warnings.append(f"{table.script}: Field(s) not in {contract}: {', '.join(extra)} "
+                                    "(fine for partition or provenance columns)")
+                warnings.append(f"{table.script}: {contract} is a literal list; derive it from {FIELDS_VAR} "
+                                "so code and docs cannot drift")
+        if not table.contracts:
+            check_select_contract(path, table, seen, errors, warnings)
+        tables.append(table)
+    by_name = {}
+    for table in tables:
+        if table.name in by_name:
+            errors.append(f"{table.script}:{table.line}: table {table.name!r} is also documented in {by_name[table.name]}")
+        by_name.setdefault(table.name, table.script)
+    errors += page_collisions([t for t in tables if by_name[t.name] == t.script])
+    return sorted(tables, key=lambda t: t.name), errors, warnings
+
+
+def select_of(path: Path, table_name: str):
+    """(columns, complete) of the SELECT that writes table_name, or None when no such SELECT is found."""
+    text = path.read_text(encoding="utf-8")
+    chunks = [text] if path.suffix == ".sql" else python_strings(ast.parse(text))
+    writes = [w for chunk in chunks for w in sql_writes(chunk)]
+    exact = [w for w in writes if same_table(w[0], table_name)]
+    chosen = exact or (writes if len(writes) == 1 and is_dynamic(writes[0][0]) else [])
+    if not chosen or not chosen[0][1]:
+        return None
+    return chosen[0][1], chosen[0][2]
+
+
+def check_select_contract(path: Path, table: TableDoc, documented: dict, errors: list, warnings: list) -> None:
+    """With no column contract, the SELECT that writes the table is the contract for its comments."""
+    found = select_of(path, table.name)
+    if found is None:
+        warnings.append(f"{table.script}: no column contract and no readable SELECT writing {table.name}; "
+                        "columns are not checked")
+        return
+    columns, complete = found
+    missing = [c for c in columns if c not in documented]
+    if missing:
+        errors.append(f"{table.script}:{table.line}: the SELECT writing {table.name} outputs column(s) with no "
+                      f"definition: {', '.join(missing)}")
+    extra = [n for n in documented if n not in columns]
+    if extra and complete:
+        errors.append(f"{table.script}:{table.line}: definition(s) for column(s) the SELECT does not output: "
+                      f"{', '.join(extra)}")
+    elif not complete:
+        warnings.append(f"{table.script}: the SELECT writing {table.name} has `*` or unnamed items; "
+                        "only its named columns are checked")
+
+
+def cell(text: str | None) -> str:
+    if not text:
+        return ""
+    return text.replace("\\", "\\\\").replace("|", "\\|").replace("\r\n", "\n").replace("\n", "<br>")
+
+
+def label(lang: str) -> str:
+    return LANGUAGE_LABELS.get(lang, lang)
+
+
+def page_name(table: TableDoc) -> str:
+    return re.sub(r"[^A-Za-z0-9_.-]", "_", table.name) + ".md"
+
+
+def page_collisions(tables: list) -> list:
+    errors, by_page = [], {}
+    for t in tables:
+        page = page_name(t)
+        if page.lower() == "readme.md":
+            errors.append(f"{t.script}:{t.line}: table name {t.name!r} would overwrite the catalog index")
+        elif page.lower() in by_page:
+            other = by_page[page.lower()]
+            errors.append(f"{t.script}:{t.line}: tables {t.name!r} and {other.name!r} ({other.script}) map to the same page {page}")
+        by_page.setdefault(page.lower(), t)
+    return errors
+
+
+def link_to(root: Path, output: str, target: str) -> str:
+    return Path(os.path.relpath(root / target, root / output)).as_posix()
+
+
+def render_table(table: TableDoc, root: Path, config: dict) -> str:
+    langs = config["languages"]
+    out = [GENERATED_MARK, "", f"# `{table.name}`", ""]
+    out.append(f"- **Script:** [{table.script}]({link_to(root, config['output'], table.script)})")
+    out.append(f"- **Grain:** {cell(table.grain)}")
+    out.append(f"- **Fields:** {len(table.fields)}")
+    out.append("")
+    for lang in langs:
+        out.append(f"**{label(lang)}:** {cell(table.text[lang])}" if len(langs) > 1 else cell(table.text[lang]))
+        out.append("")
+    if table.notes:
+        out.append("**Notes:**")
+        out.append("")
+        out.extend(f"- {cell(note)}" for note in table.notes)
+        out.append("")
+    has_window = any(f.window for f in table.fields)
+    has_unit = any(f.unit for f in table.fields)
+    head = ["Field", *(label(lang) for lang in langs), "Source"]
+    head += ["Window"] if has_window else []
+    head += ["Unit"] if has_unit else []
+    out.append("| " + " | ".join(head) + " |")
+    out.append("|" + "---|" * len(head))
+    for f in table.fields:
+        row = [f"`{f.name}`", *(cell(f.text[lang]) for lang in langs), cell(f.source)]
+        row += [cell(f.window)] if has_window else []
+        row += [cell(f.unit)] if has_unit else []
+        out.append("| " + " | ".join(row) + " |")
+    return "\n".join(out) + "\n"
+
+
+def render_index(tables: list, root: Path, config: dict) -> str:
+    lang = config["languages"][0]
+    engine = ENGINE_PATH
+    out = [
+        GENERATED_MARK, "", "# Data catalog", "",
+        "Table and column definitions, generated from the metadata in each table script. "
+        f"To change a definition, edit `{TABLE_VAR}` / `{FIELDS_VAR}` (or the `-- @field` block in a .sql file), "
+        f"run `python3 {engine} generate`, and commit the script and these pages together.",
+        "",
+    ]
+    if not tables:
+        out.append("No table script carries catalog metadata yet. Run "
+                   f"`python3 {engine} audit` to see which scripts need it.")
+        return "\n".join(out) + "\n"
+    out += ["| Table | Summary | Grain | Fields | Script |", "|---|---|---|---|---|"]
+    for t in tables:
+        script = f"[{t.script}]({link_to(root, config['output'], t.script)})"
+        out.append(f"| [`{t.name}`]({page_name(t)}) | {cell(t.text[lang])} | {cell(t.grain)} | {len(t.fields)} | {script} |")
+    return "\n".join(out) + "\n"
+
+
+def generation_plan(root: Path, config: dict):
+    tables, errors, warnings = collect(root, config)
+    plan = {}
+    if not errors:
+        out_dir = config["output"]
+        plan[f"{out_dir}/README.md"] = render_index(tables, root, config)
+        for table in tables:
+            plan[f"{out_dir}/{page_name(table)}"] = render_table(table, root, config)
+    return plan, errors, warnings
+
+
+def orphan_pages(root: Path, config: dict, plan: dict) -> list:
+    out_dir = root / config["output"]
+    if not out_dir.is_dir():
+        return []
+    orphans = []
+    for path in sorted(out_dir.glob("*.md")):
+        key = rel(root, path)
+        if key in plan:
+            continue
+        try:
+            first = path.read_text(encoding="utf-8").split("\n", 1)[0]
+        except (UnicodeDecodeError, OSError):
+            continue
+        if first == GENERATED_MARK:
+            orphans.append(key)
+    return orphans
+
+
+def cmd_generate(root: Path, check: bool) -> int:
+    config = load_config(root)
+    plan, errors, warnings = generation_plan(root, config)
+    for warning in warnings:
+        print(f"warning: {warning}", file=sys.stderr)
+    if errors:
+        for error in errors:
+            print(f"error: {error}", file=sys.stderr)
+        print(f"data-catalog: {len(errors)} metadata error(s); nothing written", file=sys.stderr)
+        return 1
+    stale = [p for p, text in plan.items() if read_or_none(root / p) != text]
+    orphans = orphan_pages(root, config, plan)
+    if check:
+        if stale or orphans:
+            for p in stale:
+                print(f"stale: {p}", file=sys.stderr)
+            for p in orphans:
+                print(f"orphan: {p} (its table no longer has metadata)", file=sys.stderr)
+            print(f"data-catalog: the catalog does not match the table scripts. Run `python3 {ENGINE_PATH} generate` "
+                  "and commit the result.", file=sys.stderr)
+            return 1
+        print(f"data-catalog: catalog is current ({len(plan) - 1} table(s))")
+        return 0
+    for p in stale:
+        target = root / p
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(plan[p], encoding="utf-8")
+        print(f"wrote {p}")
+    for p in orphans:
+        (root / p).unlink()
+        print(f"removed {p}")
+    print(f"data-catalog: {len(plan) - 1} table(s); {len(stale)} written, {len(orphans)} removed")
+    return 0
+
+
+def read_or_none(path: Path) -> str | None:
+    try:
+        return path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Audit: what scripts write, which columns, what is documented
+# ---------------------------------------------------------------------------
+
+WRITE_RES = [
+    re.compile(r"\binsert\s+(?:overwrite|into)\s+(?:table\s+)?([`\"\w.{}$-]+)", re.I),
+    re.compile(r"\bcreate\s+(?:or\s+replace\s+)?(?:temporary\s+|temp\s+)?table\s+(?:if\s+not\s+exists\s+)?([`\"\w.{}$-]+)[^;]*?\bas\b\s*(?=with\b|select\b|\()", re.I | re.S),
+    re.compile(r"\bmerge\s+into\s+([`\"\w.{}$-]+)", re.I),
+]
+API_WRITE_RES = [
+    re.compile(r"""\.(?:saveAsTable|insertInto)\(\s*[fr]?["']([^"']+)["']"""),
+    re.compile(r"""["']hoodie\.table\.name["']\s*[:,]\s*[fr]?["']([^"']+)["']"""),
+]
+
+
+def mask_sql(sql: str, strings: bool = True) -> str:
+    """Blank out comments (and string literals unless strings=False), keeping offsets."""
+    out, i, n = [], 0, len(sql)
+    while i < n:
+        two = sql[i:i + 2]
+        if two == "--":
+            j = sql.find("\n", i)
+            j = n if j == -1 else j
+            out.append(" " * (j - i)); i = j
+        elif two == "/*":
+            j = sql.find("*/", i + 2)
+            j = n if j == -1 else j + 2
+            out.append(re.sub(r"[^\n]", " ", sql[i:j])); i = j
+        elif sql[i] == "'":
+            j = i + 1
+            while j < n and not (sql[j] == "'" and sql[j + 1:j + 2] != "'"):
+                j += 2 if sql[j:j + 2] in ("''", "\\'", "\\\\") else 1
+            j = min(j + 1, n)
+            out.append(sql[i:j] if not strings else "'" + " " * (j - i - 2) + "'" if j - i >= 2 else sql[i:j]); i = j
+        else:
+            out.append(sql[i]); i += 1
+    return "".join(out)
+
+
+def split_top(text: str, sep: str = ",") -> list:
+    parts, depth, start = [], 0, 0
+    for i, ch in enumerate(text):
+        if ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+        elif ch == sep and depth == 0:
+            parts.append(text[start:i]); start = i + 1
+    parts.append(text[start:])
+    return parts
+
+
+def top_level_word(text: str, word: str, start: int = 0) -> int:
+    depth, quote = 0, None
+    pattern = re.compile(rf"{word}\b", re.I)
+    i = start
+    while i < len(text):
+        ch = text[i]
+        if quote:
+            quote = None if ch == quote else quote
+        elif ch in "`\"":
+            quote = ch
+        elif ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+        elif depth == 0 and pattern.match(text, i) and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
+            if not (word.lower() == "from" and re.search(r"\bdistinct\s*$", text[:i], re.I)):
+                return i
+        i += 1
+    return -1
+
+
+IDENT = r"(?:`[^`]+`|\"[^\"]+\"|[A-Za-z_]\w*)"
+
+
+def unquote(ident: str) -> str:
+    return ident[1:-1] if ident[:1] in "`\"" else ident
+
+
+def select_items(statement: str, original: str | None = None):
+    """Outermost SELECT items as [(name or None, expression)], plus (select_end, body_end) offsets.
+
+    `statement` is masked text; `original` (same offsets) supplies expressions with literals intact.
+    """
+    original = statement if original is None else original
+    at = top_level_word(statement, "select")
+    if at == -1:
+        return None
+    body_start = at + len("select")
+    end = top_level_word(statement, "from", body_start)
+    end = len(statement) if end == -1 else end
+    body = statement[body_start:end]
+    lead = re.match(r"\s*(?:distinct|all)\b", body, re.I)
+    offset = body_start + (lead.end() if lead else 0)
+    items, pos = [], offset
+    for part in split_top(statement[offset:end]):
+        expr_masked = " ".join(part.split())
+        expr = " ".join(original[pos:pos + len(part)].split())
+        pos += len(part) + 1
+        if not expr_masked:
+            continue
+        items.append((item_name(expr_masked), expr))
+    return items, at, end
+
+
+def item_name(item: str) -> str | None:
+    """The output name of one SELECT item, or None when it has no name the engine can read for sure."""
+    if item == "*" or item.endswith(".*") or "{" in item:
+        return None
+    alias = re.search(rf"\bas\s+({IDENT})\s*$", item, re.I)
+    if alias:
+        return unquote(alias.group(1))
+    ref = rf"(?:{IDENT}\.)*({IDENT})"
+    bare = re.fullmatch(ref, item)
+    if bare:
+        return None if bare.group(1).lower() in SQL_KEYWORDS else unquote(bare.group(1))
+    implicit = re.fullmatch(rf"(.*\S)\s+({IDENT})", item)  # `sum(x) total`, `t.col c`, `CASE ... END flag`
+    if implicit:
+        head, name = implicit.group(1), unquote(implicit.group(2))
+        plain = re.fullmatch(ref, head)
+        head_ok = head.endswith(")") or re.match(r"(?is)case\b.*\bend$", head) \
+            or (plain and plain.group(1).lower() not in SQL_KEYWORDS)
+        if head_ok and name.lower() not in SQL_KEYWORDS:
+            return name
+    return None
+
+
+def select_columns(statement: str):
+    """Output column names of a statement's outermost SELECT; (names, complete)."""
+    found = select_items(statement)
+    if found is None:
+        return [], False
+    items = found[0]
+    names = [n for n, _ in items if n]
+    return names, len(names) == len(items)
+
+
+def statement_at(masked: str, start: int) -> str:
+    end = masked.find(";", start)
+    return masked[start:end if end != -1 else len(masked)]
+
+
+def sql_writes(sql: str):
+    """Tables a SQL text writes, each with the columns of its SELECT."""
+    masked = mask_sql(sql)
+    found = []
+    for pattern in WRITE_RES:
+        for match in pattern.finditer(masked):
+            table = match.group(1).strip("`\"")
+            if table.lower() in ("table", "select", "with", "partition", "if", "directory", "local"):
+                continue
+            names, complete = select_columns(statement_at(masked, match.end()))
+            found.append((table, names, complete))
+    return found
+
+
+def python_strings(tree: ast.Module) -> list:
+    out = []
+    fragments = {id(v) for node in ast.walk(tree) if isinstance(node, ast.JoinedStr) for v in node.values}
+    docstrings = {id(n.body[0].value) for n in ast.walk(tree)
+                  if isinstance(n, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))
+                  and n.body and isinstance(n.body[0], ast.Expr) and isinstance(n.body[0].value, ast.Constant)}
+    for node in ast.walk(tree):
+        if id(node) in fragments or id(node) in docstrings:
+            continue
+        if isinstance(node, ast.Constant) and isinstance(node.value, str):
+            out.append(node.value)
+        elif isinstance(node, ast.JoinedStr):
+            parts = [v.value if isinstance(v, ast.Constant) else "{expr}" for v in node.values]
+            out.append("".join(str(p) for p in parts))
+    return out
+
+
+@dataclass
+class ScriptAudit:
+    script: str
+    writes: list = field(default_factory=list)       # [(table, evidence)]
+    columns: list = field(default_factory=list)
+    columns_from: str | None = None                  # "PUBLISHED_COLUMNS" | "sql select" | None
+    columns_complete: bool = False
+    table: TableDoc | None = None
+    findings: list = field(default_factory=list)     # [(kind, message)]
+
+
+def declared_tables(tree: ast.Module, config: dict) -> list:
+    """Table names assigned in module or class bodies, joined to a sibling schema name."""
+    table_names, schema_names = set(config.get("tableNames") or []), set(config.get("schemaNames") or [])
+    found = []
+    scopes = [tree.body] + [n.body for n in ast.walk(tree) if isinstance(n, ast.ClassDef)]
+    for body in scopes:
+        strings = {}
+        for node in body:
+            targets = node.targets if isinstance(node, ast.Assign) else [node.target] if isinstance(node, ast.AnnAssign) else []
+            value = getattr(node, "value", None)
+            if isinstance(value, ast.Constant) and isinstance(value.value, str):
+                for target in targets:
+                    if isinstance(target, ast.Name):
+                        strings[target.id] = value.value
+        schema = next((v for k, v in strings.items() if k in schema_names or SCHEMA_NAME_RE.match(k)), None)
+        for key, value in strings.items():
+            if (key in table_names or TABLE_NAME_RE.match(key)) and value.strip():
+                full = value if "." in value or not schema else f"{schema}.{value}"
+                found.append((full, key))
+    return found
+
+
+def strip_py_comments(text: str) -> str:
+    out = []
+    try:
+        for tok in tokenize.generate_tokens(io.StringIO(text).readline):
+            if tok.type != tokenize.COMMENT:
+                out.append(tok)
+        return tokenize.untokenize(out)
+    except (tokenize.TokenError, IndentationError, SyntaxError):
+        return text
+
+
+def audit_python(text: str, where: str, config: dict, audit: ScriptAudit) -> None:
+    tree = ast.parse(text)
+    names = module_assignments(tree)
+    audit.writes.extend(declared_tables(tree, config))
+    code = strip_py_comments(text)
+    for pattern in API_WRITE_RES:
+        for match in pattern.finditer(code):
+            audit.writes.append((match.group(1), "DataFrame write"))
+    for contract in config["contracts"]:
+        if contract in names:
+            state = contract_state(names[contract], names)
+            if isinstance(state, list):
+                audit.columns, audit.columns_from, audit.columns_complete = state, contract, True
+                break
+            if state == "derived" and audit.table is not None:
+                audit.columns = [f.name for f in audit.table.fields]
+                audit.columns_from, audit.columns_complete = f"{contract} (derived)", True
+                break
+    for chunk in python_strings(tree):
+        for table, cols, complete in sql_writes(chunk):
+            audit.writes.append((table, "SQL string"))
+            if audit.columns_from is None and cols:
+                audit.columns, audit.columns_from, audit.columns_complete = cols, "SQL select", complete
+
+
+def audit_sql(text: str, audit: ScriptAudit) -> None:
+    for table, cols, complete in sql_writes(text):
+        audit.writes.append((table, "SQL statement"))
+        if audit.columns_from is None and cols:
+            audit.columns, audit.columns_from, audit.columns_complete = cols, "SQL select", complete
+
+
+RELATIVE_RE = re.compile(r"^\s*(同上|同前|如上|same as (the )?(above|previous)|as above|ditto|see above)", re.I)
+
+
+CODE_IN_TEXT_RE = re.compile(
+    r"\b[a-z_][a-z0-9_]+\.[a-z_][a-z0-9_]+\b"      # schema.table or table.column (not "a.m.", "e.g.")
+    r"|<>|!=|<=|>=|\s=\s*'"                           # comparisons
+    r"|\b(?:inner|left|right|full|cross)\s+join\b"
+    r"|\b(?:date_sub|date_add|coalesce|count|sum|avg|min|max|cast|nvl|if|case)\s*\(", re.I)
+CJK_RE = re.compile(r"[\u3400-\u9fff]")
+LONG_WORDS, LONG_CJK = 30, 60
+
+
+def text_size(text: str) -> tuple[int, int]:
+    """(latin words, CJK characters) in a definition."""
+    cjk = len(CJK_RE.findall(text))
+    return len(CJK_RE.sub(" ", text).split()), cjk
+
+
+def shingles(text: str) -> set:
+    """Word 5-grams, plus 8-character runs of CJK text: the unit a repeated phrase is measured in."""
+    words = re.findall(r"[a-z0-9_']+", CJK_RE.sub(" ", text.lower()))
+    grams = {" ".join(words[i:i + 5]) for i in range(len(words) - 4)}
+    for run in re.findall(r"[\u3400-\u9fff]{8,}", text):
+        grams |= {run[i:i + 8] for i in range(len(run) - 7)}
+    return grams
+
+
+def repeated_phrase(fields: list):
+    """The phrase shared by the most rows, when at least 3 rows and half of them carry it."""
+    if len(fields) < 3:
+        return None
+    counts = {}
+    for f in fields:
+        for gram in set().union(*(shingles(t) for t in (*f.text.values(), f.source))):
+            counts[gram] = counts.get(gram, 0) + 1
+    best = max(counts.items(), key=lambda kv: (kv[1], len(kv[0])), default=None)
+    if best and best[1] >= 3 and best[1] * 2 >= len(fields):
+        return best
+    return None
+
+
+def restates_name(name: str, text: str) -> bool:
+    words = re.sub(r"[\W_]+", " ", text).strip().lower()
+    return words == re.sub(r"[\W_]+", " ", name).strip().lower() or len(text.strip()) < 4
+
+
+def audit_script(path: Path, root: Path, config: dict) -> ScriptAudit:
+    where = rel(root, path)
+    audit = ScriptAudit(where)
+    try:
+        text = path.read_text(encoding="utf-8")
+    except (UnicodeDecodeError, OSError) as exc:
+        audit.findings.append(("unreadable", str(exc)))
+        return audit
+    try:
+        audit.table = parse_script(path, root, config)
+    except CatalogError as exc:
+        audit.findings.append(("invalid-metadata", str(exc)))
+        if ": cannot parse:" in str(exc):
+            return audit
+    try:
+        if path.suffix == ".py":
+            audit_python(text, where, config, audit)
+        else:
+            audit_sql(text, audit)
+    except SyntaxError as exc:
+        audit.findings.append(("unreadable", f"cannot parse Python: {exc.msg} (line {exc.lineno})"))
+        return audit
+    seen, unique = set(), []
+    for table, evidence in audit.writes:
+        if table not in seen:
+            seen.add(table); unique.append((table, evidence))
+    audit.writes = unique
+    return audit
+
+
+def short_name(table: str) -> str:
+    return table.split(".")[-1].strip("`\"").lower()
+
+
+def is_dynamic(table: str) -> bool:
+    return "{" in table or "$" in table
+
+
+def same_table(a: str, b: str) -> bool:
+    """Equal names; when one side has no schema, equal short names. Templated names match nothing."""
+    if is_dynamic(a) or is_dynamic(b):
+        return False
+    a, b = a.strip("`\"").lower(), b.strip("`\"").lower()
+    return a == b or (("." not in a or "." not in b) and short_name(a) == short_name(b))
+
+
+def lookup(table: str, by_table: dict):
+    return [v for k, v in by_table.items() if same_table(k, table)]
+
+
+def judge(audit: ScriptAudit, documented_in: dict, writers_of: dict) -> None:
+    """Findings for one script; documented_in/writers_of are keyed by short table name."""
+    t = audit.table
+    targets = [w[0] for w in audit.writes]
+    if t is None:
+        open_targets = [x for x in targets if not is_dynamic(x) and not lookup(x, documented_in)]
+        if open_targets and not any(k == "invalid-metadata" for k, _ in audit.findings):
+            others = sorted({s for x in open_targets for group in lookup(x, writers_of) for s in group if s != audit.script})
+            also = f"; also written by {', '.join(others)} (put the metadata in the script that builds the table)" if others else ""
+            audit.findings.append(("no-catalog", f"writes {', '.join(open_targets)} but carries no {TABLE_VAR}/{FIELDS_VAR} "
+                                                 f"(or -- @table block){also}"))
+        if audit.columns and open_targets:
+            audit.findings.append(("undocumented", f"{len(audit.columns)} column(s) with no definition"))
+        return
+    documented = [f.name for f in t.fields]
+    static = [x for x in targets if not is_dynamic(x)]
+    if static and not any(same_table(t.name, x) for x in static):
+        audit.findings.append(("table-mismatch", f"metadata names {t.name} but the script writes {', '.join(static)}"))
+    if audit.columns:
+        missing = [c for c in audit.columns if c not in documented]
+        if missing:
+            audit.findings.append(("undocumented", f"{len(missing)} column(s) with no Field: {preview(missing)}"))
+        if audit.columns_complete and audit.columns_from == "SQL select":
+            extra = [n for n in documented if n not in audit.columns]
+            if extra:
+                audit.findings.append(("not-in-output", f"Field(s) not in the SELECT output: {preview(extra)}"))
+    else:
+        audit.findings.append(("columns-unknown", "output columns could not be read statically; declare a "
+                                                  f"contract tuple derived from {FIELDS_VAR}, or compare against the warehouse schema"))
+    for contract, state in t.contracts.items():
+        if isinstance(state, list):
+            audit.findings.append(("contract-not-derived", f"{contract} is a literal list; derive it from {FIELDS_VAR}"))
+    relative = [f.name for f in t.fields if any(RELATIVE_RE.match(x) for x in (*f.text.values(), f.source))]
+    if relative:
+        audit.findings.append(("relative-definition", f"definition or source points at another row (\"same as above\"); "
+                                                      f"each row must stand alone: {preview(relative)}"))
+    thin = [f.name for f in t.fields if f.name not in relative and (
+        any(restates_name(f.name, f.text[lang]) for lang in f.text) or restates_name(f.name, f.source))]
+    repeated = repeated_phrase(t.fields)
+    if repeated:
+        audit.findings.append(("repeated-text", f"\"{repeated[0]}…\" appears in {repeated[1]} of {len(t.fields)} rows; "
+                                                "a fact true for every column belongs once in the table notes"))
+    coded = [f.name for f in t.fields if any(CODE_IN_TEXT_RE.search(x) for x in f.text.values())]
+    if coded:
+        audit.findings.append(("code-in-description", "table names, operators or SQL functions in the meaning text; "
+                                                      f"move lineage to source: {preview(coded)}"))
+    long = [f.name for f in t.fields if any(w > LONG_WORDS or c > LONG_CJK for w, c in map(text_size, f.text.values()))]
+    if long:
+        audit.findings.append(("long-definition", f"over {LONG_WORDS} words or {LONG_CJK} Chinese characters; "
+                                                  f"usually a table-level fact or lineage: {preview(long)}"))
+    if thin:
+        audit.findings.append(("thin-definition", f"definition or source only restates the name: {preview(thin)}"))
+
+
+def preview(items: list, limit: int = 12) -> str:
+    shown = ", ".join(items[:limit])
+    return shown + (f", … (+{len(items) - limit})" if len(items) > limit else "")
+
+
+def cmd_audit(root: Path, globs: list, as_json: bool, verbose: bool, table_names: list, schema_names: list) -> int:
+    config_path = root / CONFIG_FILE
+    config = load_config(root) if config_path.exists() else dict(DEFAULT_CONFIG)
+    config = {**config, "tableNames": [*config["tableNames"], *table_names],
+              "schemaNames": [*config["schemaNames"], *schema_names]}
+    if globs:
+        patterns, exclude = globs, config.get("exclude", [])
+    elif config_path.exists():
+        patterns, exclude = config["scripts"], config["exclude"]
+    else:
+        raise CatalogError(f"no {CONFIG_FILE}: pass globs to audit, e.g. `audit 'dags/**/*.py' '**/*.sql'`")
+    audits = [audit_script(p, root, config) for p in find_scripts(root, patterns, exclude)]
+    documented_in, writers_of = {}, {}
+    for a in audits:
+        if a.table:
+            documented_in.setdefault(a.table.name.lower(), a.script)
+        for table, _ in a.writes:
+            if not is_dynamic(table):
+                writers_of.setdefault(table.strip("`\"").lower(), []).append(a.script)
+    for a in audits:
+        judge(a, documented_in, writers_of)
+    relevant = [a for a in audits if a.writes or a.table or a.findings]
+    writers = [a for a in audits if a.writes or a.table]
+    tables = sorted(set(documented_in) | {k for k in writers_of if not lookup(k, documented_in)})
+    complete = sorted({a.table.name.lower() for a in writers if a.table
+                       and not any(k in ("undocumented", "invalid-metadata") for k, _ in a.findings)})
+    if as_json:
+        print(json.dumps({
+            "engine": ENGINE_VERSION,
+            "scanned": len(audits),
+            "writers": len(writers),
+            "tables": len(tables),
+            "documentedTables": len(complete),
+            "scripts": [{
+                "script": a.script,
+                "writes": [{"table": t, "evidence": e} for t, e in a.writes],
+                "documentedTable": a.table.name if a.table else None,
+                "documentedFields": len(a.table.fields) if a.table else 0,
+                "columns": a.columns,
+                "columnsFrom": a.columns_from,
+                "columnsComplete": a.columns_complete,
+                "findings": [{"kind": k, "message": m} for k, m in a.findings],
+            } for a in relevant],
+        }, ensure_ascii=False, indent=2))
+        return 0
+    print(f"data-catalog audit: {len(audits)} script(s) scanned; {len(writers)} write {len(tables)} table(s); "
+          f"{len(complete)} table(s) fully documented")
+    for a in sorted(relevant, key=lambda x: (x.table is not None, x.script)):
+        if not a.findings and not verbose:
+            continue
+        print(f"\n{a.script}")
+        if a.writes:
+            print("  writes:  " + ", ".join(f"{t} ({e})" for t, e in a.writes))
+        cols = f"{len(a.columns)} from {a.columns_from}" if a.columns_from else "unknown"
+        if a.columns_from and not a.columns_complete:
+            cols += " (partial: wildcard or dynamic SELECT items)"
+        print(f"  columns: {cols}")
+        print(f"  catalog: {a.table.name + f' ({len(a.table.fields)} fields)' if a.table else 'none'}")
+        for kind, message in a.findings:
+            print(f"  - {kind}: {message}")
+    quiet = len(audits) - len(relevant)
+    if quiet:
+        print(f"\n{quiet} script(s) write no table the audit can see (helpers, DAGs, tests).")
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# drift: code changed since a git base while its definitions did not
+# ---------------------------------------------------------------------------
+
+def git(root: Path, *args: str) -> tuple[int, str]:
+    try:
+        r = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True)
+    except FileNotFoundError as exc:
+        raise CatalogError("drift needs git on PATH") from exc
+    return r.returncode, r.stdout
+
+
+def statement_shape(text: str, suffix: str, table_name: str):
+    """({column: expression}, rest) for the statement writing table_name; rest is its FROM/WHERE/JOIN/CTE text."""
+    try:
+        chunks = [text] if suffix == ".sql" else python_strings(ast.parse(text))
+    except SyntaxError:
+        return None
+    for chunk in chunks:
+        masked, readable = mask_sql(chunk), mask_sql(chunk, strings=False)
+        for pattern in WRITE_RES:
+            for match in pattern.finditer(masked):
+                if not same_table(match.group(1).strip("`\""), table_name):
+                    continue
+                stop = masked.find(";", match.end())
+                stop = len(masked) if stop == -1 else stop
+                found = select_items(masked[match.end():stop], readable[match.end():stop])
+                if found is None:
+                    continue
+                items, at, end = found
+                base = match.end()
+                rest = " ".join((readable[match.start():base + at] + " " + readable[base + end:stop]).split())
+                return {n: e for n, e in items if n}, rest
+    return None
+
+
+def code_without_metadata(text: str, suffix: str) -> str:
+    if suffix == ".sql":
+        return " ".join(mask_sql(text, strings=False).split())
+    try:
+        tree = ast.parse(text)
+    except SyntaxError:
+        return text
+    tree.body = [n for n in tree.body if not (isinstance(n, (ast.Assign, ast.AnnAssign)) and any(
+        isinstance(t, ast.Name) and t.id in (TABLE_VAR, FIELDS_VAR)
+        for t in (n.targets if isinstance(n, ast.Assign) else [n.target])))]
+    return ast.dump(tree)
+
+
+def definitions(table: TableDoc | None) -> tuple[dict, tuple]:
+    if table is None:
+        return {}, ()
+    fields = {f.name: (tuple(sorted(f.text.items())), f.source, f.window, f.unit) for f in table.fields}
+    return fields, (table.grain, tuple(sorted(table.text.items())), tuple(table.notes))
+
+
+def drift_items(root: Path, config: dict, base: str) -> list:
+    code, _ = git(root, "rev-parse", "--verify", f"{base}^{{commit}}")
+    if code != 0:
+        raise CatalogError(f"drift: {base!r} is not a commit in this git repository")
+    _, out = git(root, "diff", "--name-only", base, "--")
+    changed = set(out.split())
+    items = []
+    for path in find_scripts(root, config["scripts"], config["exclude"]):
+        where = rel(root, path)
+        if where not in changed:
+            continue
+        code, old_text = git(root, "show", f"{base}:{where}")
+        if code != 0:
+            continue  # a new script: generate checks that every column is defined
+        new_text = path.read_text(encoding="utf-8")
+        parse = parse_python if path.suffix == ".py" else None
+        try:
+            new_t = (parse(new_text, where, config["languages"], config["contracts"]) if parse
+                     else parse_sql(new_text, where, config["languages"]))
+        except CatalogError:
+            continue  # generate reports invalid metadata
+        if new_t is None:
+            continue  # undocumented: audit reports it
+        try:
+            old_t = (parse(old_text, where, config["languages"], config["contracts"]) if parse
+                     else parse_sql(old_text, where, config["languages"]))
+        except CatalogError:
+            old_t = None
+        old_fields, old_table = definitions(old_t)
+        new_fields, new_table = definitions(new_t)
+        edited = {n for n, d in new_fields.items() if old_fields.get(n) != d}
+        any_edit = bool(edited) or old_table != new_table
+        old_shape = statement_shape(old_text, path.suffix, new_t.name)
+        new_shape = statement_shape(new_text, path.suffix, new_t.name)
+        found = []
+        if old_shape and new_shape:
+            for name, expr in new_shape[0].items():
+                if name in old_shape[0] and old_shape[0][name] != expr and name in new_fields and name not in edited:
+                    found.append(f"{name}: its SELECT expression changed; its definition did not")
+            if old_shape[1] != new_shape[1] and not any_edit:
+                found.append("filters, joins or CTEs changed; no definition changed (they shape every column)")
+        elif code_without_metadata(old_text, path.suffix) != code_without_metadata(new_text, path.suffix) and not any_edit:
+            found.append("code changed and no definition changed; the columns could not be read, so check each one")
+        if found:
+            items.append((where, new_t.name, found))
+    return items
+
+
+def cmd_drift(root: Path, base: str) -> int:
+    config = load_config(root)
+    items = drift_items(root, config, base)
+    if not items:
+        print(f"data-catalog drift since {base}: none")
+        return 0
+    count = sum(len(f) for _, _, f in items)
+    print(f"data-catalog drift since {base}: {count} item(s) to confirm")
+    for where, table, found in items:
+        print(f"\n{where} ({table})")
+        for line in found:
+            print(f"  - {line}")
+    print("\nUpdate each definition to match the code, or confirm it still holds and say so in the PR.")
+    return 1
+
+
+# ---------------------------------------------------------------------------
+# init: config, engine copy, Field/Table module, workflow
+# (the CONTRIBUTING.md section and AGENTS.md pointer come from persist-conventions.mjs)
+# ---------------------------------------------------------------------------
+
+def render_fields_module(languages: list) -> str:
+    lang_fields = "\n".join(f"    {lang}: str" for lang in languages)
+    return f'''"""Column metadata types for the data catalog.
+
+Generated by data_catalog.py init from data-catalog.json; rerun init after changing
+`languages` there, never edit this file. Table scripts declare:
+
+    CATALOG_TABLE = Table("schema.table", grain=..., {", ".join(f"{lang}=..." for lang in languages)})
+    {FIELDS_VAR} = (Field("column", {", ".join(f"{lang}=..." for lang in languages)}, source=...), ...)
+    PUBLISHED_COLUMNS = tuple(f.name for f in {FIELDS_VAR})
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Field:
+    name: str
+{lang_fields}
+    source: str
+    window: str | None = None
+    unit: str | None = None
+
+
+@dataclass(frozen=True)
+class Table:
+    name: str
+    grain: str
+{lang_fields}
+    notes: tuple[str, ...] = ()
+'''
+
+
+def render_workflow(config: dict) -> str:
+    paths = [CONFIG_FILE, ENGINE_PATH, f"{config['output']}/**", *config["scripts"]]
+    if config.get("fieldsModule"):
+        paths.append(config["fieldsModule"])
+    path_lines = "\n".join("      - '" + p.replace("'", "''") + "'" for p in dict.fromkeys(paths))
+    return f"""{WORKFLOW_MARK}
+name: Data catalog
+
+on:
+  pull_request:
+    paths:
+{path_lines}
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: data-catalog-${{{{ github.ref }}}}
+  cancel-in-progress: true
+
+jobs:
+  catalog:
+    name: Data catalog
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Check the catalog matches the table scripts
+        run: python3 {ENGINE_PATH} generate --check
+"""
+
+
+def default_fields_module(root: Path, scripts: list) -> str:
+    py = [p for p in find_scripts(root, scripts, []) if p.suffix == ".py"]
+    if not py:
+        return "catalog_fields.py"
+    common = os.path.commonpath([str(p.parent) for p in py])
+    return rel(root, Path(common) / "catalog_fields.py")
+
+
+def init_plan(root: Path, args) -> tuple[dict, list]:
+    path = root / CONFIG_FILE
+    config = {**DEFAULT_CONFIG, **(read_config_file(root) if path.exists() else {})}
+    if args.scripts:
+        config["scripts"] = args.scripts
+    if args.exclude:
+        config["exclude"] = args.exclude
+    if args.languages:
+        config["languages"] = [x.strip() for x in args.languages.split(",") if x.strip()]
+    if args.output:
+        config["output"] = args.output.rstrip("/")
+    if args.fields_module:
+        config["fieldsModule"] = args.fields_module
+    validate_config(config)
+    if not config.get("fieldsModule"):
+        config["fieldsModule"] = default_fields_module(root, config["scripts"])
+    plan = [
+        (CONFIG_FILE, json.dumps(config, ensure_ascii=False, indent=2) + "\n", "config"),
+        (ENGINE_PATH, Path(__file__).read_text(encoding="utf-8"), f"engine {ENGINE_VERSION}"),
+        (config["fieldsModule"], render_fields_module(config["languages"]), "Field/Table types for table scripts"),
+        (WORKFLOW_PATH, render_workflow(config), "CI: fail a PR whose catalog is stale"),
+    ]
+    return config, plan
+
+
+def cmd_init(root: Path, args) -> int:
+    config, plan = init_plan(root, args)
+    changed = [(p, text, why) for p, text, why in plan if read_or_none(root / p) != text]
+    for p, text, why in plan:
+        state = "unchanged" if (p, text, why) not in changed else ("write" if args.write else "would write")
+        print(f"{state:12} {p}  ({why})")
+    if not args.write:
+        print("\ndry run: rerun with --write to apply")
+        return 0
+    for p, text, _ in changed:
+        target = root / p
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(text, encoding="utf-8")
+    print()
+    return cmd_generate(root, check=False)
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def engine_identity() -> str:
+    digest = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:12]
+    return f"{ENGINE_VERSION}+{digest}"
+
+
+def find_root(start: Path) -> Path:
+    for candidate in [start, *start.parents]:
+        if (candidate / CONFIG_FILE).exists() or (candidate / ".git").exists():
+            return candidate
+    return start
+
+
+def main(argv: list | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="data_catalog.py", description=__doc__.split("\n\n")[0])
+    parser.add_argument("--root", type=Path, help="repo root (default: nearest dir with data-catalog.json or .git)")
+    sub = parser.add_subparsers(dest="command", required=True)
+    p_init = sub.add_parser("init", help="set the repo up (dry run unless --write)")
+    p_init.add_argument("--scripts", action="append", help="glob of table scripts (.py/.sql); repeatable")
+    p_init.add_argument("--exclude", action="append", help="glob to skip; repeatable")
+    p_init.add_argument("--languages", help="comma-separated codes, e.g. en,zh")
+    p_init.add_argument("--output", help="catalog directory (default docs/catalog)")
+    p_init.add_argument("--fields-module", help="path for the generated Field/Table module")
+    p_init.add_argument("--write", action="store_true")
+    p_gen = sub.add_parser("generate", help="write the catalog pages")
+    p_gen.add_argument("--check", action="store_true", help="write nothing; exit 1 when stale or invalid")
+    p_audit = sub.add_parser("audit", help="report scripts with missing or weak column definitions")
+    p_audit.add_argument("globs", nargs="*", help="globs to scan (default: config scripts)")
+    p_audit.add_argument("--json", action="store_true")
+    p_audit.add_argument("--verbose", action="store_true", help="also list scripts with no findings")
+    p_audit.add_argument("--table-name", action="append", default=[], metavar="NAME",
+                         help="extra variable/attribute whose string value is the table written; repeatable")
+    p_audit.add_argument("--schema-name", action="append", default=[], metavar="NAME",
+                         help="extra variable/attribute holding that table's schema; repeatable")
+    p_drift = sub.add_parser("drift", help="local check: code changed since a git base while its definitions did not")
+    p_drift.add_argument("--base", default="HEAD", help="git ref to compare with (default HEAD: uncommitted work)")
+    sub.add_parser("version", help="print the engine identity")
+    args = parser.parse_args(argv)
+    root = (args.root or find_root(Path.cwd())).resolve()
+    try:
+        if args.command == "version":
+            print(engine_identity())
+            return 0
+        if args.command == "init":
+            return cmd_init(root, args)
+        if args.command == "generate":
+            return cmd_generate(root, args.check)
+        if args.command == "drift":
+            return cmd_drift(root, args.base)
+        return cmd_audit(root, args.globs, args.json, args.verbose, args.table_name, args.schema_name)
+    except CatalogError as exc:
+        print(f"data-catalog: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())