@1aboveio/app-dev-skills 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +2 -0
  2. package/package.json +7 -6
  3. package/skills/airflow/SKILL.md +1 -1
  4. package/skills/android-foundation/SKILL.md +1 -1
  5. package/skills/app-debug/SKILL.md +1 -1
  6. package/skills/better-auth/SKILL.md +1 -1
  7. package/skills/data-catalog/LICENSE +3 -0
  8. package/skills/data-catalog/SKILL.md +208 -0
  9. package/skills/data-catalog/agents/openai.yaml +3 -0
  10. package/skills/data-catalog/references/data-catalog-conventions.md +77 -0
  11. package/skills/data-catalog/references/field-definitions.md +124 -0
  12. package/skills/data-catalog/references/format.md +134 -0
  13. package/skills/data-catalog/release-check.mjs +93 -0
  14. package/skills/data-catalog/scripts/data-catalog.test.mjs +36 -0
  15. package/skills/data-catalog/scripts/data_catalog.py +1447 -0
  16. package/skills/data-catalog/scripts/persist-conventions-core.mjs +117 -0
  17. package/skills/data-catalog/scripts/persist-conventions.mjs +17 -0
  18. package/skills/data-catalog/scripts/test_data_catalog.py +618 -0
  19. package/skills/data-quality/LICENSE +3 -0
  20. package/skills/data-quality/SKILL.md +96 -0
  21. package/skills/data-quality/agents/openai.yaml +3 -0
  22. package/skills/data-quality/references/checks.md +56 -0
  23. package/skills/data-quality/release-check.mjs +93 -0
  24. package/skills/data-quality/scripts/data-quality.test.mjs +12 -0
  25. package/skills/data-quality/scripts/data_quality.py +405 -0
  26. package/skills/data-quality/scripts/test_data_quality.py +277 -0
  27. package/skills/ios-foundation/SKILL.md +1 -1
  28. package/skills/monorepo/SKILL.md +1 -1
  29. package/skills/nextjs-fullstack/SKILL.md +1 -1
  30. package/skills/prisma-setup/SKILL.md +1 -1
  31. package/skills/pyspark/SKILL.md +1 -1
  32. package/skills/python-backend/SKILL.md +1 -1
  33. package/skills/shadcn/SKILL.md +1 -1
  34. package/skills/zod-v4/SKILL.md +1 -1
@@ -0,0 +1,1447 @@
1
+ #!/usr/bin/env python3
2
+ """Data catalog engine: read column metadata from table scripts, generate the catalog, audit scripts.
3
+
4
+ Commands:
5
+ init set a repo up: config, this engine, the Field/Table module, CI workflow
6
+ generate write docs/catalog/*.md from the scripts; --check fails when they are stale
7
+ audit report scripts that write tables without complete column definitions (never edits)
8
+ drift local check: code changed since a git base while its definitions did not
9
+ version print the engine identity
10
+
11
+ Metadata carriers:
12
+ Python module-level CATALOG_TABLE = Table(...) and CATALOG_FIELDS = (Field(...), ...),
13
+ read with ast (the script is never imported, so Spark is not needed)
14
+ SQL a comment block of `-- @table <name>` and `-- @field <name>` entries
15
+
16
+ Standard library only. Copied into a repo by `init`; refresh it the same way.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import argparse
21
+ import ast
22
+ import hashlib
23
+ import io
24
+ import json
25
+ import os
26
+ import re
27
+ import subprocess
28
+ import sys
29
+ import tokenize
30
+ from dataclasses import dataclass, field
31
+ from pathlib import Path
32
+
33
+ ENGINE_VERSION = "1.0.0"
34
+ CONFIG_FILE = "data-catalog.json"
35
+ ENGINE_PATH = "scripts/data-catalog/data_catalog.py"
36
+ WORKFLOW_PATH = ".github/workflows/data-catalog.yml"
37
+ GENERATED_MARK = "<!-- generated by data_catalog.py: edit the table script, then run generate -->"
38
+ WORKFLOW_MARK = "# generated by data_catalog.py init: edit data-catalog.json and rerun init"
39
+ TABLE_VAR = "CATALOG_TABLE"
40
+ FIELDS_VAR = "CATALOG_FIELDS"
41
+ DEFAULT_CONFIG = {
42
+ "version": 1,
43
+ "scripts": [],
44
+ "exclude": [],
45
+ "output": "docs/catalog",
46
+ "languages": ["en"],
47
+ "contracts": ["PUBLISHED_COLUMNS", "OUTPUT_COLUMNS"],
48
+ "fieldsModule": None,
49
+ "tableNames": [],
50
+ "schemaNames": [],
51
+ }
52
+ FIELD_EXTRAS = ("source", "window", "unit", "values")
53
+ LANGUAGE_LABELS = {"en": "English", "zh": "中文", "zh_hant": "繁體中文", "ja": "日本語", "ko": "한국어"}
54
+ LANGUAGE_RE = re.compile(r"^[a-z]{2}(_[a-z]+)?$")
55
+ SKIP_DIRS = {"node_modules", "__pycache__", "venv"}
56
+ # Assignments (module or class body) whose string value names the table a script writes,
57
+ # and the schema/database that may sit beside it (`dst_db = "dwd"`, `dst_tbl = "orders"`).
58
+ TABLE_NAME_RE = re.compile(r"^(?:\w*_)?(?:target|output|sink|dest|destination|dst)_(?:table|tbl)(?:_name)?$|^table_name$", re.I)
59
+ SCHEMA_NAME_RE = re.compile(r"^(?:target|output|sink|dest|destination|dst)_(?:db|database|schema)(?:_name)?$", re.I)
60
+ SQL_KEYWORDS = {
61
+ "end", "from", "where", "and", "or", "not", "null", "then", "else", "when", "case", "as",
62
+ "on", "join", "by", "asc", "desc", "over", "distinct", "is", "in", "like", "between", "with", "select",
63
+ "union", "all", "partition", "rows", "range", "interval", "true", "false",
64
+ }
65
+
66
+
67
+ class CatalogError(Exception):
68
+ """A metadata or config problem, reported as `path:line: message`."""
69
+
70
+
71
+ # ---------------------------------------------------------------------------
72
+ # Config and file discovery
73
+ # ---------------------------------------------------------------------------
74
+
75
+ def read_config_file(root: Path) -> dict:
76
+ try:
77
+ data = json.loads((root / CONFIG_FILE).read_text(encoding="utf-8"))
78
+ except json.JSONDecodeError as exc:
79
+ raise CatalogError(f"{CONFIG_FILE}: invalid JSON: {exc}") from exc
80
+ if not isinstance(data, dict):
81
+ raise CatalogError(f"{CONFIG_FILE}: must be a JSON object")
82
+ return data
83
+
84
+
85
+ def load_config(root: Path) -> dict:
86
+ if not (root / CONFIG_FILE).exists():
87
+ raise CatalogError(f"{CONFIG_FILE} not found in {root}; run `init` first")
88
+ config = {**DEFAULT_CONFIG, **read_config_file(root)}
89
+ validate_config(config)
90
+ return config
91
+
92
+
93
+ def validate_config(config: dict) -> None:
94
+ config["output"] = str(config.get("output") or DEFAULT_CONFIG["output"]).strip().strip("/") or DEFAULT_CONFIG["output"]
95
+ if not config["scripts"]:
96
+ raise CatalogError(f"{CONFIG_FILE}: `scripts` must list at least one glob")
97
+ langs = config["languages"]
98
+ if not langs or not all(isinstance(lang, str) and LANGUAGE_RE.match(lang) for lang in langs):
99
+ raise CatalogError(f"{CONFIG_FILE}: `languages` must be codes like \"en\" or \"zh\", got {langs!r}")
100
+ clash = [lang for lang in langs if lang in FIELD_EXTRAS or lang in ("name", "grain", "notes")]
101
+ if clash:
102
+ raise CatalogError(f"{CONFIG_FILE}: language codes clash with field arguments: {clash}")
103
+
104
+
105
+ def glob_regex(pattern: str) -> re.Pattern:
106
+ out, i = [], 0
107
+ while i < len(pattern):
108
+ if pattern.startswith("**/", i):
109
+ out.append("(?:.*/)?"); i += 3
110
+ elif pattern.startswith("**", i):
111
+ out.append(".*"); i += 2
112
+ elif pattern[i] == "*":
113
+ out.append("[^/]*"); i += 1
114
+ elif pattern[i] == "?":
115
+ out.append("[^/]"); i += 1
116
+ else:
117
+ out.append(re.escape(pattern[i])); i += 1
118
+ return re.compile("".join(out) + r"\Z")
119
+
120
+
121
+ def find_scripts(root: Path, patterns: list[str], exclude: list[str]) -> list[Path]:
122
+ """One pruned walk: hidden directories and node_modules are never entered."""
123
+ wanted = [glob_regex(p) for p in patterns]
124
+ skipped = [glob_regex(p) for p in exclude]
125
+ found = []
126
+ for dirpath, dirnames, filenames in os.walk(root):
127
+ dirnames[:] = sorted(d for d in dirnames if not d.startswith(".") and d not in SKIP_DIRS)
128
+ for name in filenames:
129
+ if not name.endswith((".py", ".sql")):
130
+ continue
131
+ path = Path(dirpath) / name
132
+ relpath = rel(root, path)
133
+ if any(r.match(relpath) for r in wanted) and not any(r.match(relpath) for r in skipped):
134
+ found.append(path)
135
+ return sorted(found)
136
+
137
+
138
+ def rel(root: Path, path: Path) -> str:
139
+ return path.relative_to(root).as_posix()
140
+
141
+
142
+ # ---------------------------------------------------------------------------
143
+ # Metadata model
144
+ # ---------------------------------------------------------------------------
145
+
146
+ @dataclass
147
+ class FieldDoc:
148
+ name: str
149
+ text: dict
150
+ source: str
151
+ window: str | None
152
+ unit: str | None
153
+ line: int
154
+ values: tuple = () # the values an enumerated column can take; checked by data-quality
155
+
156
+
157
+ @dataclass
158
+ class TableDoc:
159
+ name: str
160
+ script: str
161
+ grain: dict # language -> what one row is
162
+ text: dict
163
+ notes: dict # language -> [shared rules]
164
+ fields: list
165
+ line: int
166
+ contracts: dict = field(default_factory=dict) # name -> list[str] | "derived" | "unchecked"
167
+
168
+
169
+ # ---------------------------------------------------------------------------
170
+ # Python carrier (ast, never imported)
171
+ # ---------------------------------------------------------------------------
172
+
173
+ def module_assignments(tree: ast.Module) -> dict:
174
+ out = {}
175
+ for node in tree.body:
176
+ if isinstance(node, ast.Assign):
177
+ for target in node.targets:
178
+ if isinstance(target, ast.Name):
179
+ out[target.id] = node.value
180
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None:
181
+ out[node.target.id] = node.value
182
+ return out
183
+
184
+
185
+ def call_name(node: ast.AST) -> str | None:
186
+ if not isinstance(node, ast.Call):
187
+ return None
188
+ if isinstance(node.func, ast.Name):
189
+ return node.func.id
190
+ if isinstance(node.func, ast.Attribute):
191
+ return node.func.attr
192
+ return None
193
+
194
+
195
+ def eval_text(node: ast.AST, names: dict, where: str, seen: frozenset = frozenset()):
196
+ """Evaluate a string, tuple of strings, or None built from literals and module constants."""
197
+ if isinstance(node, ast.Constant) and (isinstance(node.value, str) or node.value is None):
198
+ return node.value
199
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
200
+ left = eval_text(node.left, names, where, seen)
201
+ right = eval_text(node.right, names, where, seen)
202
+ if isinstance(left, str) and isinstance(right, str):
203
+ return left + right
204
+ if isinstance(left, tuple) and isinstance(right, tuple):
205
+ return left + right
206
+ if isinstance(node, (ast.Tuple, ast.List)):
207
+ return tuple(eval_text(elt, names, where, seen) for elt in node.elts)
208
+ if isinstance(node, ast.Name) and node.id in names and node.id not in seen:
209
+ return eval_text(names[node.id], names, where, seen | {node.id})
210
+ raise CatalogError(f"{where}:{getattr(node, 'lineno', '?')}: metadata must be a string literal "
211
+ "(or module-level string constant), not an expression")
212
+
213
+
214
+ def resolve_sequence(node: ast.AST, names: dict, where: str, seen: frozenset = frozenset()) -> list:
215
+ """Flatten a tuple/list (with *NAME, A + B, NAME) into its element nodes."""
216
+ if isinstance(node, (ast.Tuple, ast.List)):
217
+ out = []
218
+ for elt in node.elts:
219
+ if isinstance(elt, ast.Starred):
220
+ out.extend(resolve_sequence(elt.value, names, where, seen))
221
+ else:
222
+ out.append(elt)
223
+ return out
224
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
225
+ return resolve_sequence(node.left, names, where, seen) + resolve_sequence(node.right, names, where, seen)
226
+ if isinstance(node, ast.Name) and node.id in names and node.id not in seen:
227
+ return resolve_sequence(names[node.id], names, where, seen | {node.id})
228
+ raise CatalogError(f"{where}:{getattr(node, 'lineno', '?')}: {FIELDS_VAR} must be a tuple of Field(...) "
229
+ "built in this module (literals, *NAME or A + B); imported lists cannot be read")
230
+
231
+
232
+ def call_kwargs(call: ast.Call, names: dict, where: str, kind: str) -> dict:
233
+ if len(call.args) > 1 or any(isinstance(a, ast.Starred) for a in call.args):
234
+ raise CatalogError(f"{where}:{call.lineno}: pass only the name positionally to {kind}(...); "
235
+ "everything else by keyword")
236
+ values = {}
237
+ if call.args:
238
+ values["name"] = eval_text(call.args[0], names, where)
239
+ for kw in call.keywords:
240
+ if kw.arg is None:
241
+ raise CatalogError(f"{where}:{call.lineno}: **kwargs is not readable in {kind}(...)")
242
+ values[kw.arg] = eval_text(kw.value, names, where)
243
+ return values
244
+
245
+
246
+ def require_text(values: dict, key: str, where: str, line: int, label: str) -> str:
247
+ value = values.get(key)
248
+ if not isinstance(value, str) or not value.strip():
249
+ raise CatalogError(f"{where}:{line}: {label} is missing `{key}`")
250
+ return value.strip()
251
+
252
+
253
+ def build_field(values: dict, languages: list, where: str, line: int) -> FieldDoc:
254
+ name = require_text(values, "name", where, line, "Field")
255
+ known = {"name", *FIELD_EXTRAS, *languages}
256
+ unknown = sorted(set(values) - known)
257
+ if unknown:
258
+ raise CatalogError(f"{where}:{line}: Field {name!r} has unknown argument(s) {unknown}; "
259
+ f"languages configured: {languages}")
260
+ text = {lang: require_text(values, lang, where, line, f"Field {name!r}") for lang in languages}
261
+ source = require_text(values, "source", where, line, f"Field {name!r}")
262
+ allowed = values.get("values") or ()
263
+ if isinstance(allowed, str):
264
+ allowed = tuple(v.strip() for v in re.split(r"[;;]", allowed))
265
+ if not isinstance(allowed, tuple) or not all(isinstance(v, str) and v != "" for v in allowed):
266
+ raise CatalogError(f"{where}:{line}: Field {name!r}: values must be a tuple of non-empty strings")
267
+ if len(set(allowed)) != len(allowed):
268
+ raise CatalogError(f"{where}:{line}: Field {name!r}: values lists a value twice")
269
+ return FieldDoc(name, text, source, (values.get("window") or None), (values.get("unit") or None), line, allowed)
270
+
271
+
272
+ def build_table(values: dict, fields: list, languages: list, where: str, line: int, script: str) -> TableDoc:
273
+ name = require_text(values, "name", where, line, "Table")
274
+ if "grain" in values or "notes" in values:
275
+ keys = ", ".join(f"grain_{lang}=..., notes_{lang}=(...)" for lang in languages)
276
+ raise CatalogError(f"{where}:{line}: Table {name!r}: grain and notes are written per language: {keys}")
277
+ known = {"name", *languages, *(f"grain_{lang}" for lang in languages), *(f"notes_{lang}" for lang in languages)}
278
+ unknown = sorted(set(values) - known)
279
+ if unknown:
280
+ raise CatalogError(f"{where}:{line}: Table {name!r} has unknown argument(s) {unknown}")
281
+ text = {lang: require_text(values, lang, where, line, f"Table {name!r}") for lang in languages}
282
+ grain = {lang: require_text(values, f"grain_{lang}", where, line, f"Table {name!r}") for lang in languages}
283
+ notes = {}
284
+ for lang in languages:
285
+ raw = values.get(f"notes_{lang}") or ()
286
+ raw = (raw,) if isinstance(raw, str) else raw
287
+ notes[lang] = [n.strip() for n in raw if n and n.strip()]
288
+ counts = {lang: len(n) for lang, n in notes.items()}
289
+ if len(set(counts.values())) > 1:
290
+ raise CatalogError(f"{where}:{line}: Table {name!r}: every language needs the same notes, got {counts}")
291
+ return TableDoc(name, script, grain, text, notes, fields, line)
292
+
293
+
294
+ def struct_fields(node: ast.AST, names: dict):
295
+ """Column names of a literal StructType([StructField("name", ...), ...]), or None."""
296
+ if call_name(node) != "StructType" or not node.args:
297
+ return None
298
+ try:
299
+ items = resolve_sequence(node.args[0], names, "", frozenset())
300
+ except CatalogError:
301
+ return None
302
+ out = []
303
+ for item in items:
304
+ if call_name(item) != "StructField" or not item.args:
305
+ return None
306
+ try:
307
+ value = eval_text(item.args[0], names, "")
308
+ except CatalogError:
309
+ return None
310
+ if not isinstance(value, str):
311
+ return None
312
+ out.append(value)
313
+ return out
314
+
315
+
316
+ def contract_state(node: ast.AST, names: dict):
317
+ if any(isinstance(n, ast.Name) and n.id == FIELDS_VAR for n in ast.walk(node)):
318
+ return "derived"
319
+ columns = struct_fields(node, names)
320
+ if columns is not None:
321
+ return columns
322
+ try:
323
+ items = resolve_sequence(node, names, "", frozenset())
324
+ values = [eval_text(item, names, "") for item in items]
325
+ except CatalogError:
326
+ return "unchecked"
327
+ if all(isinstance(v, str) for v in values):
328
+ return values
329
+ return "unchecked"
330
+
331
+
332
+ def parse_python(text: str, where: str, languages: list, contracts: list) -> TableDoc | None:
333
+ try:
334
+ tree = ast.parse(text)
335
+ except SyntaxError as exc:
336
+ raise CatalogError(f"{where}:{exc.lineno}: cannot parse: {exc.msg}") from exc
337
+ names = module_assignments(tree)
338
+ table_node, fields_node = names.get(TABLE_VAR), names.get(FIELDS_VAR)
339
+ if table_node is None and fields_node is None:
340
+ return None
341
+ if table_node is None or fields_node is None:
342
+ missing = TABLE_VAR if table_node is None else FIELDS_VAR
343
+ raise CatalogError(f"{where}:1: has one of {TABLE_VAR}/{FIELDS_VAR} but not {missing}")
344
+ if call_name(table_node) != "Table":
345
+ raise CatalogError(f"{where}:{table_node.lineno}: {TABLE_VAR} must be Table(...)")
346
+ fields = []
347
+ for item in resolve_sequence(fields_node, names, where):
348
+ if call_name(item) != "Field":
349
+ raise CatalogError(f"{where}:{getattr(item, 'lineno', '?')}: every {FIELDS_VAR} entry must be Field(...)")
350
+ fields.append(build_field(call_kwargs(item, names, where, "Field"), languages, where, item.lineno))
351
+ table = build_table(call_kwargs(table_node, names, where, "Table"), fields, languages, where, table_node.lineno, where)
352
+ for contract in contracts:
353
+ if contract in names:
354
+ table.contracts[contract] = contract_state(names[contract], names)
355
+ return table
356
+
357
+
358
+ # ---------------------------------------------------------------------------
359
+ # SQL carrier: `-- @table` / `-- @field` comment blocks
360
+ # ---------------------------------------------------------------------------
361
+
362
+ SQL_DIRECTIVE_RE = re.compile(r"^@(table|field)\s+(\S+)\s*$")
363
+ SQL_DIRECTIVE_START_RE = re.compile(r"^@(table|field)\b")
364
+ SQL_KEY_RE = re.compile(r"^ ([a-z][a-z_]*):\s?(.*)$")
365
+ SQL_CONTINUE_RE = re.compile(r"^ {4,}(\S.*)$")
366
+
367
+
368
+ def parse_sql(text: str, where: str, languages: list) -> TableDoc | None:
369
+ entries = [] # [kind, name, values, line, last_key]
370
+ current = None
371
+ for lineno, raw in enumerate(text.splitlines(), 1):
372
+ stripped = raw.lstrip()
373
+ if not stripped.startswith("--"):
374
+ if stripped:
375
+ break # the block sits above the code; a directive after the first statement is ignored
376
+ current = None
377
+ continue
378
+ body = stripped[2:]
379
+ body = body[1:] if body.startswith(" ") else body
380
+ directive = SQL_DIRECTIVE_RE.match(body)
381
+ if not directive and SQL_DIRECTIVE_START_RE.match(body):
382
+ raise CatalogError(f"{where}:{lineno}: write `-- @table <name>` or `-- @field <name>` with nothing after the name")
383
+ if directive:
384
+ current = [directive.group(1), directive.group(2), {}, lineno, None]
385
+ entries.append(current)
386
+ continue
387
+ if current is None:
388
+ continue
389
+ key = SQL_KEY_RE.match(body)
390
+ cont = SQL_CONTINUE_RE.match(body)
391
+ if key:
392
+ current[4] = key.group(1)
393
+ current[2][key.group(1)] = key.group(2).strip()
394
+ elif cont and current[4]:
395
+ prev = current[2][current[4]]
396
+ current[2][current[4]] = f"{prev} {cont.group(1).strip()}".strip()
397
+ else:
398
+ current = None
399
+ if not entries:
400
+ return None
401
+ tables = [e for e in entries if e[0] == "table"]
402
+ if len(tables) != 1:
403
+ raise CatalogError(f"{where}:{entries[0][3]}: a SQL file needs exactly one `-- @table`, found {len(tables)}")
404
+ fields = []
405
+ for kind, name, values, line, _ in entries:
406
+ if kind == "field":
407
+ if any(k.startswith(("notes", "grain")) for k in values):
408
+ raise CatalogError(f"{where}:{line}: `grain_*` and `notes_*` belong on @table, not @field")
409
+ fields.append(build_field({"name": name, **values}, languages, where, line))
410
+ _, name, values, line, _ = tables[0]
411
+ values = {"name": name, **{k: tuple(n.strip() for n in re.split(r"[;;]", v)) if k.startswith("notes_") else v
412
+ for k, v in values.items()}}
413
+ return build_table(values, fields, languages, where, line, where)
414
+
415
+
416
+ def parse_script(path: Path, root: Path, config: dict) -> TableDoc | None:
417
+ where = rel(root, path)
418
+ text = path.read_text(encoding="utf-8")
419
+ if path.suffix == ".py":
420
+ return parse_python(text, where, config["languages"], config["contracts"])
421
+ return parse_sql(text, where, config["languages"])
422
+
423
+
424
+ # ---------------------------------------------------------------------------
425
+ # Validation and rendering
426
+ # ---------------------------------------------------------------------------
427
+
428
+ def collect(root: Path, config: dict):
429
+ """Parse every configured script; returns (tables, errors, warnings)."""
430
+ tables, errors, warnings = [], [], []
431
+ for path in find_scripts(root, config["scripts"], config["exclude"]):
432
+ try:
433
+ table = parse_script(path, root, config)
434
+ except CatalogError as exc:
435
+ errors.append(str(exc))
436
+ continue
437
+ except (UnicodeDecodeError, OSError) as exc:
438
+ errors.append(f"{rel(root, path)}:1: cannot read: {exc}")
439
+ continue
440
+ if table is None:
441
+ continue
442
+ seen = {}
443
+ for f in table.fields:
444
+ if f.name in seen:
445
+ errors.append(f"{table.script}:{f.line}: field {f.name!r} is defined twice (first at line {seen[f.name]})")
446
+ seen.setdefault(f.name, f.line)
447
+ for contract, state in table.contracts.items():
448
+ if isinstance(state, list):
449
+ missing = [c for c in state if c not in seen]
450
+ extra = [n for n in seen if n not in state]
451
+ if missing:
452
+ errors.append(f"{table.script}:{table.line}: {contract} has column(s) with no Field: {', '.join(missing)}")
453
+ if extra:
454
+ warnings.append(f"{table.script}: Field(s) not in {contract}: {', '.join(extra)} "
455
+ "(fine for partition or provenance columns)")
456
+ warnings.append(f"{table.script}: {contract} is a literal list; derive it from {FIELDS_VAR} "
457
+ "so code and docs cannot drift")
458
+ if not table.contracts:
459
+ check_select_contract(path, table, seen, errors, warnings)
460
+ tables.append(table)
461
+ by_name = {}
462
+ for table in tables:
463
+ if table.name in by_name:
464
+ errors.append(f"{table.script}:{table.line}: table {table.name!r} is also documented in {by_name[table.name]}")
465
+ by_name.setdefault(table.name, table.script)
466
+ errors += page_collisions([t for t in tables if by_name[t.name] == t.script])
467
+ return sorted(tables, key=lambda t: t.name), errors, warnings
468
+
469
+
470
+ def select_of(path: Path, table_name: str):
471
+ """(columns, complete) of the SELECT that writes table_name, or None when no such SELECT is found."""
472
+ text = path.read_text(encoding="utf-8")
473
+ chunks = [text] if path.suffix == ".sql" else python_strings(ast.parse(text))
474
+ writes = [w for chunk in chunks for w in sql_writes(chunk)]
475
+ exact = [w for w in writes if same_table(w[0], table_name)]
476
+ chosen = exact or (writes if len(writes) == 1 and is_dynamic(writes[0][0]) else [])
477
+ if not chosen or not chosen[0][1]:
478
+ return None
479
+ return chosen[0][1], chosen[0][2]
480
+
481
+
482
+ def check_select_contract(path: Path, table: TableDoc, documented: dict, errors: list, warnings: list) -> None:
483
+ """With no column contract, the SELECT that writes the table is the contract for its comments."""
484
+ found = select_of(path, table.name)
485
+ if found is None:
486
+ warnings.append(f"{table.script}: no column contract and no readable SELECT writing {table.name}; "
487
+ "columns are not checked")
488
+ return
489
+ columns, complete = found
490
+ missing = [c for c in columns if c not in documented]
491
+ if missing:
492
+ errors.append(f"{table.script}:{table.line}: the SELECT writing {table.name} outputs column(s) with no "
493
+ f"definition: {', '.join(missing)}")
494
+ extra = [n for n in documented if n not in columns]
495
+ if extra and complete:
496
+ errors.append(f"{table.script}:{table.line}: definition(s) for column(s) the SELECT does not output: "
497
+ f"{', '.join(extra)}")
498
+ elif not complete:
499
+ warnings.append(f"{table.script}: the SELECT writing {table.name} has `*` or unnamed items; "
500
+ "only its named columns are checked")
501
+
502
+
503
+ def cell(text: str | None) -> str:
504
+ if not text:
505
+ return ""
506
+ return text.replace("\\", "\\\\").replace("|", "\\|").replace("\r\n", "\n").replace("\n", "<br>")
507
+
508
+
509
+ SECTION_LABELS = {"en": ("Grain:", "Notes:"), "zh": ("粒度:", "说明:"), "zh_hant": ("粒度:", "說明:")}
510
+
511
+
512
+ def label(lang: str) -> str:
513
+ return LANGUAGE_LABELS.get(lang, lang)
514
+
515
+
516
+ def page_name(table: TableDoc) -> str:
517
+ return re.sub(r"[^A-Za-z0-9_.-]", "_", table.name) + ".md"
518
+
519
+
520
+ def page_collisions(tables: list) -> list:
521
+ errors, by_page = [], {}
522
+ for t in tables:
523
+ page = page_name(t)
524
+ if page.lower() == "readme.md":
525
+ errors.append(f"{t.script}:{t.line}: table name {t.name!r} would overwrite the catalog index")
526
+ elif page.lower() in by_page:
527
+ other = by_page[page.lower()]
528
+ errors.append(f"{t.script}:{t.line}: tables {t.name!r} and {other.name!r} ({other.script}) map to the same page {page}")
529
+ by_page.setdefault(page.lower(), t)
530
+ return errors
531
+
532
+
533
+ def link_to(root: Path, output: str, target: str) -> str:
534
+ return Path(os.path.relpath(root / target, root / output)).as_posix()
535
+
536
+
537
+ def render_table(table: TableDoc, root: Path, config: dict) -> str:
538
+ langs = config["languages"]
539
+ out = [GENERATED_MARK, "", f"# `{table.name}`", ""]
540
+ out.append(f"- **Script:** [{table.script}]({link_to(root, config['output'], table.script)})")
541
+ out.append(f"- **Fields:** {len(table.fields)}")
542
+ out.append("")
543
+ for lang in langs:
544
+ grain_label, notes_label = SECTION_LABELS.get(lang, SECTION_LABELS["en"])
545
+ if len(langs) > 1:
546
+ out += [f"## {label(lang)}", ""]
547
+ out += [cell(table.text[lang]), "", f"- **{grain_label}** {cell(table.grain[lang])}"]
548
+ if table.notes[lang]:
549
+ out.append(f"- **{notes_label}**")
550
+ out.extend(f" - {cell(note)}" for note in table.notes[lang])
551
+ out.append("")
552
+ if len(langs) > 1:
553
+ out += ["## Fields", ""]
554
+ has_window = any(f.window for f in table.fields)
555
+ has_unit = any(f.unit for f in table.fields)
556
+ has_values = any(f.values for f in table.fields)
557
+ head = ["Field", *(label(lang) for lang in langs), "Source"]
558
+ head += ["Window"] if has_window else []
559
+ head += ["Unit"] if has_unit else []
560
+ head += ["Values"] if has_values else []
561
+ out.append("| " + " | ".join(head) + " |")
562
+ out.append("|" + "---|" * len(head))
563
+ for f in table.fields:
564
+ row = [f"`{f.name}`", *(cell(f.text[lang]) for lang in langs), cell(f.source)]
565
+ row += [cell(f.window)] if has_window else []
566
+ row += [cell(f.unit)] if has_unit else []
567
+ row += [cell(", ".join(f"`{v}`" for v in f.values))] if has_values else []
568
+ out.append("| " + " | ".join(row) + " |")
569
+ return "\n".join(out) + "\n"
570
+
571
+
572
+ def render_index(tables: list, root: Path, config: dict) -> str:
573
+ lang = config["languages"][0]
574
+ engine = ENGINE_PATH
575
+ out = [
576
+ GENERATED_MARK, "", "# Data catalog", "",
577
+ "Table and column definitions, generated from the metadata in each table script. "
578
+ f"To change a definition, edit `{TABLE_VAR}` / `{FIELDS_VAR}` (or the `-- @field` block in a .sql file), "
579
+ f"run `python3 {engine} generate`, and commit the script and these pages together.",
580
+ "",
581
+ ]
582
+ if not tables:
583
+ out.append("No table script carries catalog metadata yet. Run "
584
+ f"`python3 {engine} audit` to see which scripts need it.")
585
+ return "\n".join(out) + "\n"
586
+ out += ["| Table | Summary | Grain | Fields | Script |", "|---|---|---|---|---|"]
587
+ for t in tables:
588
+ script = f"[{t.script}]({link_to(root, config['output'], t.script)})"
589
+ out.append(f"| [`{t.name}`]({page_name(t)}) | {cell(t.text[lang])} | {cell(t.grain[lang])} | {len(t.fields)} | {script} |")
590
+ return "\n".join(out) + "\n"
591
+
592
+
593
+ def generation_plan(root: Path, config: dict):
594
+ tables, errors, warnings = collect(root, config)
595
+ plan = {}
596
+ if not errors:
597
+ out_dir = config["output"]
598
+ plan[f"{out_dir}/README.md"] = render_index(tables, root, config)
599
+ for table in tables:
600
+ plan[f"{out_dir}/{page_name(table)}"] = render_table(table, root, config)
601
+ return plan, errors, warnings
602
+
603
+
604
+ def orphan_pages(root: Path, config: dict, plan: dict) -> list:
605
+ out_dir = root / config["output"]
606
+ if not out_dir.is_dir():
607
+ return []
608
+ orphans = []
609
+ for path in sorted(out_dir.glob("*.md")):
610
+ key = rel(root, path)
611
+ if key in plan:
612
+ continue
613
+ try:
614
+ first = path.read_text(encoding="utf-8").split("\n", 1)[0]
615
+ except (UnicodeDecodeError, OSError):
616
+ continue
617
+ if first == GENERATED_MARK:
618
+ orphans.append(key)
619
+ return orphans
620
+
621
+
622
+ def cmd_generate(root: Path, check: bool) -> int:
623
+ config = load_config(root)
624
+ plan, errors, warnings = generation_plan(root, config)
625
+ for warning in warnings:
626
+ print(f"warning: {warning}", file=sys.stderr)
627
+ if errors:
628
+ for error in errors:
629
+ print(f"error: {error}", file=sys.stderr)
630
+ print(f"data-catalog: {len(errors)} metadata error(s); nothing written", file=sys.stderr)
631
+ return 1
632
+ stale = [p for p, text in plan.items() if read_or_none(root / p) != text]
633
+ orphans = orphan_pages(root, config, plan)
634
+ if check:
635
+ if stale or orphans:
636
+ for p in stale:
637
+ print(f"stale: {p}", file=sys.stderr)
638
+ for p in orphans:
639
+ print(f"orphan: {p} (its table no longer has metadata)", file=sys.stderr)
640
+ print(f"data-catalog: the catalog does not match the table scripts. Run `python3 {ENGINE_PATH} generate` "
641
+ "and commit the result.", file=sys.stderr)
642
+ return 1
643
+ print(f"data-catalog: catalog is current ({len(plan) - 1} table(s))")
644
+ return 0
645
+ for p in stale:
646
+ target = root / p
647
+ target.parent.mkdir(parents=True, exist_ok=True)
648
+ target.write_text(plan[p], encoding="utf-8")
649
+ print(f"wrote {p}")
650
+ for p in orphans:
651
+ (root / p).unlink()
652
+ print(f"removed {p}")
653
+ print(f"data-catalog: {len(plan) - 1} table(s); {len(stale)} written, {len(orphans)} removed")
654
+ return 0
655
+
656
+
657
+ def read_or_none(path: Path) -> str | None:
658
+ try:
659
+ return path.read_text(encoding="utf-8")
660
+ except FileNotFoundError:
661
+ return None
662
+
663
+
664
+ # ---------------------------------------------------------------------------
665
+ # Audit: what scripts write, which columns, what is documented
666
+ # ---------------------------------------------------------------------------
667
+
668
+ WRITE_RES = [
669
+ re.compile(r"\binsert\s+(?:overwrite|into)\s+(?:table\s+)?([`\"\w.{}$-]+)", re.I),
670
+ re.compile(r"\bcreate\s+(?:or\s+replace\s+)?(?:temporary\s+|temp\s+)?table\s+(?:if\s+not\s+exists\s+)?([`\"\w.{}$-]+)[^;]*?\bas\b\s*(?=with\b|select\b|\()", re.I | re.S),
671
+ re.compile(r"\bmerge\s+into\s+([`\"\w.{}$-]+)", re.I),
672
+ ]
673
+ API_WRITE_RES = [
674
+ re.compile(r"""\.(?:saveAsTable|insertInto)\(\s*[fr]?["']([^"']+)["']"""),
675
+ re.compile(r"""["']hoodie\.table\.name["']\s*[:,]\s*[fr]?["']([^"']+)["']"""),
676
+ ]
677
+
678
+
679
+ def mask_sql(sql: str, strings: bool = True) -> str:
680
+ """Blank out comments (and string literals unless strings=False), keeping offsets."""
681
+ out, i, n = [], 0, len(sql)
682
+ while i < n:
683
+ two = sql[i:i + 2]
684
+ if two == "--":
685
+ j = sql.find("\n", i)
686
+ j = n if j == -1 else j
687
+ out.append(" " * (j - i)); i = j
688
+ elif two == "/*":
689
+ j = sql.find("*/", i + 2)
690
+ j = n if j == -1 else j + 2
691
+ out.append(re.sub(r"[^\n]", " ", sql[i:j])); i = j
692
+ elif sql[i] == "'":
693
+ j = i + 1
694
+ while j < n and not (sql[j] == "'" and sql[j + 1:j + 2] != "'"):
695
+ j += 2 if sql[j:j + 2] in ("''", "\\'", "\\\\") else 1
696
+ j = min(j + 1, n)
697
+ out.append(sql[i:j] if not strings else "'" + " " * (j - i - 2) + "'" if j - i >= 2 else sql[i:j]); i = j
698
+ else:
699
+ out.append(sql[i]); i += 1
700
+ return "".join(out)
701
+
702
+
703
+ def split_top(text: str, sep: str = ",") -> list:
704
+ parts, depth, start = [], 0, 0
705
+ for i, ch in enumerate(text):
706
+ if ch in "([{":
707
+ depth += 1
708
+ elif ch in ")]}":
709
+ depth -= 1
710
+ elif ch == sep and depth == 0:
711
+ parts.append(text[start:i]); start = i + 1
712
+ parts.append(text[start:])
713
+ return parts
714
+
715
+
716
+ def top_level_word(text: str, word: str, start: int = 0) -> int:
717
+ depth, quote = 0, None
718
+ pattern = re.compile(rf"{word}\b", re.I)
719
+ i = start
720
+ while i < len(text):
721
+ ch = text[i]
722
+ if quote:
723
+ quote = None if ch == quote else quote
724
+ elif ch in "`\"":
725
+ quote = ch
726
+ elif ch in "([{":
727
+ depth += 1
728
+ elif ch in ")]}":
729
+ depth -= 1
730
+ elif depth == 0 and pattern.match(text, i) and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
731
+ if not (word.lower() == "from" and re.search(r"\bdistinct\s*$", text[:i], re.I)):
732
+ return i
733
+ i += 1
734
+ return -1
735
+
736
+
737
+ IDENT = r"(?:`[^`]+`|\"[^\"]+\"|[A-Za-z_]\w*)"
738
+
739
+
740
+ def unquote(ident: str) -> str:
741
+ return ident[1:-1] if ident[:1] in "`\"" else ident
742
+
743
+
744
+ def select_items(statement: str, original: str | None = None):
745
+ """Outermost SELECT items as [(name or None, expression)], plus (select_end, body_end) offsets.
746
+
747
+ `statement` is masked text; `original` (same offsets) supplies expressions with literals intact.
748
+ """
749
+ original = statement if original is None else original
750
+ at = top_level_word(statement, "select")
751
+ if at == -1:
752
+ return None
753
+ body_start = at + len("select")
754
+ end = top_level_word(statement, "from", body_start)
755
+ end = len(statement) if end == -1 else end
756
+ body = statement[body_start:end]
757
+ lead = re.match(r"\s*(?:distinct|all)\b", body, re.I)
758
+ offset = body_start + (lead.end() if lead else 0)
759
+ items, pos = [], offset
760
+ for part in split_top(statement[offset:end]):
761
+ expr_masked = " ".join(part.split())
762
+ expr = " ".join(original[pos:pos + len(part)].split())
763
+ pos += len(part) + 1
764
+ if not expr_masked:
765
+ continue
766
+ items.append((item_name(expr_masked), expr))
767
+ return items, at, end
768
+
769
+
770
+ def item_name(item: str) -> str | None:
771
+ """The output name of one SELECT item, or None when it has no name the engine can read for sure."""
772
+ if item == "*" or item.endswith(".*") or "{" in item:
773
+ return None
774
+ alias = re.search(rf"\bas\s+({IDENT})\s*$", item, re.I)
775
+ if alias:
776
+ return unquote(alias.group(1))
777
+ ref = rf"(?:{IDENT}\.)*({IDENT})"
778
+ bare = re.fullmatch(ref, item)
779
+ if bare:
780
+ return None if bare.group(1).lower() in SQL_KEYWORDS else unquote(bare.group(1))
781
+ implicit = re.fullmatch(rf"(.*\S)\s+({IDENT})", item) # `sum(x) total`, `t.col c`, `CASE ... END flag`
782
+ if implicit:
783
+ head, name = implicit.group(1), unquote(implicit.group(2))
784
+ plain = re.fullmatch(ref, head)
785
+ head_ok = head.endswith(")") or re.match(r"(?is)case\b.*\bend$", head) \
786
+ or (plain and plain.group(1).lower() not in SQL_KEYWORDS)
787
+ if head_ok and name.lower() not in SQL_KEYWORDS:
788
+ return name
789
+ return None
790
+
791
+
792
+ def select_columns(statement: str):
793
+ """Output column names of a statement's outermost SELECT; (names, complete)."""
794
+ found = select_items(statement)
795
+ if found is None:
796
+ return [], False
797
+ items = found[0]
798
+ names = [n for n, _ in items if n]
799
+ return names, len(names) == len(items)
800
+
801
+
802
+ def statement_at(masked: str, start: int) -> str:
803
+ end = masked.find(";", start)
804
+ return masked[start:end if end != -1 else len(masked)]
805
+
806
+
807
+ def sql_writes(sql: str):
808
+ """Tables a SQL text writes, each with the columns of its SELECT."""
809
+ masked = mask_sql(sql)
810
+ found = []
811
+ for pattern in WRITE_RES:
812
+ for match in pattern.finditer(masked):
813
+ table = match.group(1).strip("`\"")
814
+ if table.lower() in ("table", "select", "with", "partition", "if", "directory", "local"):
815
+ continue
816
+ names, complete = select_columns(statement_at(masked, match.end()))
817
+ found.append((table, names, complete))
818
+ return found
819
+
820
+
821
+ def python_strings(tree: ast.Module) -> list:
822
+ out = []
823
+ fragments = {id(v) for node in ast.walk(tree) if isinstance(node, ast.JoinedStr) for v in node.values}
824
+ docstrings = {id(n.body[0].value) for n in ast.walk(tree)
825
+ if isinstance(n, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))
826
+ and n.body and isinstance(n.body[0], ast.Expr) and isinstance(n.body[0].value, ast.Constant)}
827
+ for node in ast.walk(tree):
828
+ if id(node) in fragments or id(node) in docstrings:
829
+ continue
830
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
831
+ out.append(node.value)
832
+ elif isinstance(node, ast.JoinedStr):
833
+ parts = [v.value if isinstance(v, ast.Constant) else "{expr}" for v in node.values]
834
+ out.append("".join(str(p) for p in parts))
835
+ return out
836
+
837
+
838
+ @dataclass
839
+ class ScriptAudit:
840
+ script: str
841
+ writes: list = field(default_factory=list) # [(table, evidence)]
842
+ columns: list = field(default_factory=list)
843
+ columns_from: str | None = None # "PUBLISHED_COLUMNS" | "sql select" | None
844
+ columns_complete: bool = False
845
+ table: TableDoc | None = None
846
+ findings: list = field(default_factory=list) # [(kind, message)]
847
+
848
+
849
+ def declared_tables(tree: ast.Module, config: dict) -> list:
850
+ """Table names assigned in module or class bodies, joined to a sibling schema name."""
851
+ table_names, schema_names = set(config.get("tableNames") or []), set(config.get("schemaNames") or [])
852
+ found = []
853
+ scopes = [tree.body] + [n.body for n in ast.walk(tree) if isinstance(n, ast.ClassDef)]
854
+ for body in scopes:
855
+ strings = {}
856
+ for node in body:
857
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target] if isinstance(node, ast.AnnAssign) else []
858
+ value = getattr(node, "value", None)
859
+ if isinstance(value, ast.Constant) and isinstance(value.value, str):
860
+ for target in targets:
861
+ if isinstance(target, ast.Name):
862
+ strings[target.id] = value.value
863
+ schema = next((v for k, v in strings.items() if k in schema_names or SCHEMA_NAME_RE.match(k)), None)
864
+ for key, value in strings.items():
865
+ if (key in table_names or TABLE_NAME_RE.match(key)) and value.strip():
866
+ full = value if "." in value or not schema else f"{schema}.{value}"
867
+ found.append((full, key))
868
+ return found
869
+
870
+
871
+ def strip_py_comments(text: str) -> str:
872
+ out = []
873
+ try:
874
+ for tok in tokenize.generate_tokens(io.StringIO(text).readline):
875
+ if tok.type != tokenize.COMMENT:
876
+ out.append(tok)
877
+ return tokenize.untokenize(out)
878
+ except (tokenize.TokenError, IndentationError, SyntaxError):
879
+ return text
880
+
881
+
882
+ def audit_python(text: str, where: str, config: dict, audit: ScriptAudit) -> None:
883
+ tree = ast.parse(text)
884
+ names = module_assignments(tree)
885
+ audit.writes.extend(declared_tables(tree, config))
886
+ code = strip_py_comments(text)
887
+ for pattern in API_WRITE_RES:
888
+ for match in pattern.finditer(code):
889
+ audit.writes.append((match.group(1), "DataFrame write"))
890
+ for contract in config["contracts"]:
891
+ if contract in names:
892
+ state = contract_state(names[contract], names)
893
+ if isinstance(state, list):
894
+ audit.columns, audit.columns_from, audit.columns_complete = state, contract, True
895
+ break
896
+ if state == "derived" and audit.table is not None:
897
+ audit.columns = [f.name for f in audit.table.fields]
898
+ audit.columns_from, audit.columns_complete = f"{contract} (derived)", True
899
+ break
900
+ for chunk in python_strings(tree):
901
+ for table, cols, complete in sql_writes(chunk):
902
+ audit.writes.append((table, "SQL string"))
903
+ if audit.columns_from is None and cols:
904
+ audit.columns, audit.columns_from, audit.columns_complete = cols, "SQL select", complete
905
+
906
+
907
+ def audit_sql(text: str, audit: ScriptAudit) -> None:
908
+ for table, cols, complete in sql_writes(text):
909
+ audit.writes.append((table, "SQL statement"))
910
+ if audit.columns_from is None and cols:
911
+ audit.columns, audit.columns_from, audit.columns_complete = cols, "SQL select", complete
912
+
913
+
914
+ RELATIVE_RE = re.compile(r"^\s*(同上|同前|如上|same as (the )?(above|previous)|as above|ditto|see above)", re.I)
915
+
916
+
917
+ CODE_IN_TEXT_RE = re.compile(
918
+ r"\b[a-z_][a-z0-9_]+\.[a-z_][a-z0-9_]+\b" # schema.table or table.column (not "a.m.", "e.g.")
919
+ r"|<>|!=|<=|>=|\s=\s*'" # comparisons
920
+ r"|\b(?:inner|left|right|full|cross)\s+join\b"
921
+ r"|\b(?:date_sub|date_add|coalesce|count|sum|avg|min|max|cast|nvl|if|case)\s*\(", re.I)
922
+ CJK_RE = re.compile(r"[\u3400-\u9fff]")
923
+ LONG_WORDS, LONG_CJK = 30, 60
924
+
925
+
926
+ def text_size(text: str) -> tuple[int, int]:
927
+ """(latin words, CJK characters) in a definition."""
928
+ cjk = len(CJK_RE.findall(text))
929
+ return len(CJK_RE.sub(" ", text).split()), cjk
930
+
931
+
932
+ def shingles(text: str) -> set:
933
+ """Word 5-grams, plus 8-character runs of CJK text: the unit a repeated phrase is measured in."""
934
+ words = re.findall(r"[a-z0-9_']+", CJK_RE.sub(" ", text.lower()))
935
+ grams = {" ".join(words[i:i + 5]) for i in range(len(words) - 4)}
936
+ for run in re.findall(r"[\u3400-\u9fff]{8,}", text):
937
+ grams |= {run[i:i + 8] for i in range(len(run) - 7)}
938
+ return grams
939
+
940
+
941
+ def repeated_phrase(fields: list):
942
+ """The phrase shared by the most rows, when at least 3 rows and half of them carry it."""
943
+ if len(fields) < 3:
944
+ return None
945
+ counts = {}
946
+ for f in fields:
947
+ for gram in set().union(*(shingles(t) for t in (*f.text.values(), f.source))):
948
+ counts[gram] = counts.get(gram, 0) + 1
949
+ best = max(counts.items(), key=lambda kv: (kv[1], len(kv[0])), default=None)
950
+ if best and best[1] >= 3 and best[1] * 2 >= len(fields):
951
+ return best
952
+ return None
953
+
954
+
955
+ def restates_name(name: str, text: str) -> bool:
956
+ words = re.sub(r"[\W_]+", " ", text).strip().lower()
957
+ return words == re.sub(r"[\W_]+", " ", name).strip().lower() or len(text.strip()) < 4
958
+
959
+
960
+ def audit_script(path: Path, root: Path, config: dict) -> ScriptAudit:
961
+ where = rel(root, path)
962
+ audit = ScriptAudit(where)
963
+ try:
964
+ text = path.read_text(encoding="utf-8")
965
+ except (UnicodeDecodeError, OSError) as exc:
966
+ audit.findings.append(("unreadable", str(exc)))
967
+ return audit
968
+ try:
969
+ audit.table = parse_script(path, root, config)
970
+ except CatalogError as exc:
971
+ audit.findings.append(("invalid-metadata", str(exc)))
972
+ if ": cannot parse:" in str(exc):
973
+ return audit
974
+ try:
975
+ if path.suffix == ".py":
976
+ audit_python(text, where, config, audit)
977
+ else:
978
+ audit_sql(text, audit)
979
+ except SyntaxError as exc:
980
+ audit.findings.append(("unreadable", f"cannot parse Python: {exc.msg} (line {exc.lineno})"))
981
+ return audit
982
+ seen, unique = set(), []
983
+ for table, evidence in audit.writes:
984
+ if table not in seen:
985
+ seen.add(table); unique.append((table, evidence))
986
+ audit.writes = unique
987
+ return audit
988
+
989
+
990
+ def short_name(table: str) -> str:
991
+ return table.split(".")[-1].strip("`\"").lower()
992
+
993
+
994
+ def is_dynamic(table: str) -> bool:
995
+ return "{" in table or "$" in table
996
+
997
+
998
+ def same_table(a: str, b: str) -> bool:
999
+ """Equal names; when one side has no schema, equal short names. Templated names match nothing."""
1000
+ if is_dynamic(a) or is_dynamic(b):
1001
+ return False
1002
+ a, b = a.strip("`\"").lower(), b.strip("`\"").lower()
1003
+ return a == b or (("." not in a or "." not in b) and short_name(a) == short_name(b))
1004
+
1005
+
1006
+ def lookup(table: str, by_table: dict):
1007
+ return [v for k, v in by_table.items() if same_table(k, table)]
1008
+
1009
+
1010
+ def judge(audit: ScriptAudit, documented_in: dict, writers_of: dict) -> None:
1011
+ """Findings for one script; documented_in/writers_of are keyed by short table name."""
1012
+ t = audit.table
1013
+ targets = [w[0] for w in audit.writes]
1014
+ if t is None:
1015
+ open_targets = [x for x in targets if not is_dynamic(x) and not lookup(x, documented_in)]
1016
+ if open_targets and not any(k == "invalid-metadata" for k, _ in audit.findings):
1017
+ others = sorted({s for x in open_targets for group in lookup(x, writers_of) for s in group if s != audit.script})
1018
+ also = f"; also written by {', '.join(others)} (put the metadata in the script that builds the table)" if others else ""
1019
+ audit.findings.append(("no-catalog", f"writes {', '.join(open_targets)} but carries no {TABLE_VAR}/{FIELDS_VAR} "
1020
+ f"(or -- @table block){also}"))
1021
+ if audit.columns and open_targets:
1022
+ audit.findings.append(("undocumented", f"{len(audit.columns)} column(s) with no definition"))
1023
+ return
1024
+ documented = [f.name for f in t.fields]
1025
+ static = [x for x in targets if not is_dynamic(x)]
1026
+ if static and not any(same_table(t.name, x) for x in static):
1027
+ audit.findings.append(("table-mismatch", f"metadata names {t.name} but the script writes {', '.join(static)}"))
1028
+ if audit.columns:
1029
+ missing = [c for c in audit.columns if c not in documented]
1030
+ if missing:
1031
+ audit.findings.append(("undocumented", f"{len(missing)} column(s) with no Field: {preview(missing)}"))
1032
+ if audit.columns_complete and audit.columns_from == "SQL select":
1033
+ extra = [n for n in documented if n not in audit.columns]
1034
+ if extra:
1035
+ audit.findings.append(("not-in-output", f"Field(s) not in the SELECT output: {preview(extra)}"))
1036
+ else:
1037
+ audit.findings.append(("columns-unknown", "output columns could not be read statically; declare a "
1038
+ f"contract tuple derived from {FIELDS_VAR}, or compare against the warehouse schema"))
1039
+ for contract, state in t.contracts.items():
1040
+ if isinstance(state, list):
1041
+ audit.findings.append(("contract-not-derived", f"{contract} is a literal list; derive it from {FIELDS_VAR}"))
1042
+ relative = [f.name for f in t.fields if any(RELATIVE_RE.match(x) for x in (*f.text.values(), f.source))]
1043
+ if relative:
1044
+ audit.findings.append(("relative-definition", f"definition or source points at another row (\"same as above\"); "
1045
+ f"each row must stand alone: {preview(relative)}"))
1046
+ thin = [f.name for f in t.fields if f.name not in relative and (
1047
+ any(restates_name(f.name, f.text[lang]) for lang in f.text) or restates_name(f.name, f.source))]
1048
+ repeated = repeated_phrase(t.fields)
1049
+ if repeated:
1050
+ audit.findings.append(("repeated-text", f"\"{repeated[0]}…\" appears in {repeated[1]} of {len(t.fields)} rows; "
1051
+ "a fact true for every column belongs once in the table notes"))
1052
+ coded = [f.name for f in t.fields if any(CODE_IN_TEXT_RE.search(x) for x in f.text.values())]
1053
+ if coded:
1054
+ audit.findings.append(("code-in-description", "table names, operators or SQL functions in the meaning text; "
1055
+ f"move lineage to source: {preview(coded)}"))
1056
+ long = [f.name for f in t.fields if any(w > LONG_WORDS or c > LONG_CJK for w, c in map(text_size, f.text.values()))]
1057
+ if long:
1058
+ audit.findings.append(("long-definition", f"over {LONG_WORDS} words or {LONG_CJK} Chinese characters; "
1059
+ f"usually a table-level fact or lineage: {preview(long)}"))
1060
+ if thin:
1061
+ audit.findings.append(("thin-definition", f"definition or source only restates the name: {preview(thin)}"))
1062
+
1063
+
1064
+ def preview(items: list, limit: int = 12) -> str:
1065
+ shown = ", ".join(items[:limit])
1066
+ return shown + (f", … (+{len(items) - limit})" if len(items) > limit else "")
1067
+
1068
+
1069
+ def cmd_audit(root: Path, globs: list, as_json: bool, verbose: bool, table_names: list, schema_names: list,
1070
+ contracts: list) -> int:
1071
+ config_path = root / CONFIG_FILE
1072
+ config = load_config(root) if config_path.exists() else dict(DEFAULT_CONFIG)
1073
+ config = {**config, "tableNames": [*config["tableNames"], *table_names],
1074
+ "schemaNames": [*config["schemaNames"], *schema_names],
1075
+ "contracts": [*config["contracts"], *contracts]}
1076
+ if globs:
1077
+ patterns, exclude = globs, config.get("exclude", [])
1078
+ elif config_path.exists():
1079
+ patterns, exclude = config["scripts"], config["exclude"]
1080
+ else:
1081
+ raise CatalogError(f"no {CONFIG_FILE}: pass globs to audit, e.g. `audit 'dags/**/*.py' '**/*.sql'`")
1082
+ audits = [audit_script(p, root, config) for p in find_scripts(root, patterns, exclude)]
1083
+ documented_in, writers_of = {}, {}
1084
+ for a in audits:
1085
+ if a.table:
1086
+ documented_in.setdefault(a.table.name.lower(), a.script)
1087
+ for table, _ in a.writes:
1088
+ if not is_dynamic(table):
1089
+ writers_of.setdefault(table.strip("`\"").lower(), []).append(a.script)
1090
+ for a in audits:
1091
+ judge(a, documented_in, writers_of)
1092
+ relevant = [a for a in audits if a.writes or a.table or a.findings]
1093
+ writers = [a for a in audits if a.writes or a.table]
1094
+ tables = sorted(set(documented_in) | {k for k in writers_of if not lookup(k, documented_in)})
1095
+ complete = sorted({a.table.name.lower() for a in writers if a.table
1096
+ and not any(k in ("undocumented", "invalid-metadata") for k, _ in a.findings)})
1097
+ if as_json:
1098
+ print(json.dumps({
1099
+ "engine": ENGINE_VERSION,
1100
+ "scanned": len(audits),
1101
+ "writers": len(writers),
1102
+ "tables": len(tables),
1103
+ "documentedTables": len(complete),
1104
+ "scripts": [{
1105
+ "script": a.script,
1106
+ "writes": [{"table": t, "evidence": e} for t, e in a.writes],
1107
+ "documentedTable": a.table.name if a.table else None,
1108
+ "documentedFields": len(a.table.fields) if a.table else 0,
1109
+ "columns": a.columns,
1110
+ "columnsFrom": a.columns_from,
1111
+ "columnsComplete": a.columns_complete,
1112
+ "findings": [{"kind": k, "message": m} for k, m in a.findings],
1113
+ } for a in relevant],
1114
+ }, ensure_ascii=False, indent=2))
1115
+ return 0
1116
+ print(f"data-catalog audit: {len(audits)} script(s) scanned; {len(writers)} write {len(tables)} table(s); "
1117
+ f"{len(complete)} table(s) fully documented")
1118
+ for a in sorted(relevant, key=lambda x: (x.table is not None, x.script)):
1119
+ if not a.findings and not verbose:
1120
+ continue
1121
+ print(f"\n{a.script}")
1122
+ if a.writes:
1123
+ print(" writes: " + ", ".join(f"{t} ({e})" for t, e in a.writes))
1124
+ cols = f"{len(a.columns)} from {a.columns_from}" if a.columns_from else "unknown"
1125
+ if a.columns_from and not a.columns_complete:
1126
+ cols += " (partial: wildcard or dynamic SELECT items)"
1127
+ print(f" columns: {cols}")
1128
+ print(f" catalog: {a.table.name + f' ({len(a.table.fields)} fields)' if a.table else 'none'}")
1129
+ for kind, message in a.findings:
1130
+ print(f" - {kind}: {message}")
1131
+ quiet = len(audits) - len(relevant)
1132
+ if quiet:
1133
+ print(f"\n{quiet} script(s) write no table the audit can see (helpers, DAGs, tests).")
1134
+ return 0
1135
+
1136
+
1137
+ # ---------------------------------------------------------------------------
1138
+ # drift: code changed since a git base while its definitions did not
1139
+ # ---------------------------------------------------------------------------
1140
+
1141
+ def git(root: Path, *args: str) -> tuple[int, str]:
1142
+ try:
1143
+ r = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True)
1144
+ except FileNotFoundError as exc:
1145
+ raise CatalogError("drift needs git on PATH") from exc
1146
+ return r.returncode, r.stdout
1147
+
1148
+
1149
+ def statement_shape(text: str, suffix: str, table_name: str):
1150
+ """({column: expression}, rest) for the statement writing table_name; rest is its FROM/WHERE/JOIN/CTE text."""
1151
+ try:
1152
+ chunks = [text] if suffix == ".sql" else python_strings(ast.parse(text))
1153
+ except SyntaxError:
1154
+ return None
1155
+ for chunk in chunks:
1156
+ masked, readable = mask_sql(chunk), mask_sql(chunk, strings=False)
1157
+ for pattern in WRITE_RES:
1158
+ for match in pattern.finditer(masked):
1159
+ if not same_table(match.group(1).strip("`\""), table_name):
1160
+ continue
1161
+ stop = masked.find(";", match.end())
1162
+ stop = len(masked) if stop == -1 else stop
1163
+ found = select_items(masked[match.end():stop], readable[match.end():stop])
1164
+ if found is None:
1165
+ continue
1166
+ items, at, end = found
1167
+ base = match.end()
1168
+ rest = " ".join((readable[match.start():base + at] + " " + readable[base + end:stop]).split())
1169
+ return {n: e for n, e in items if n}, rest
1170
+ return None
1171
+
1172
+
1173
+ def code_without_metadata(text: str, suffix: str) -> str:
1174
+ if suffix == ".sql":
1175
+ return " ".join(mask_sql(text, strings=False).split())
1176
+ try:
1177
+ tree = ast.parse(text)
1178
+ except SyntaxError:
1179
+ return text
1180
+ tree.body = [n for n in tree.body if not (isinstance(n, (ast.Assign, ast.AnnAssign)) and any(
1181
+ isinstance(t, ast.Name) and t.id in (TABLE_VAR, FIELDS_VAR)
1182
+ for t in (n.targets if isinstance(n, ast.Assign) else [n.target])))]
1183
+ return ast.dump(tree)
1184
+
1185
+
1186
+ def definitions(table: TableDoc | None) -> tuple[dict, tuple]:
1187
+ if table is None:
1188
+ return {}, ()
1189
+ fields = {f.name: (tuple(sorted(f.text.items())), f.source, f.window, f.unit, f.values) for f in table.fields}
1190
+ notes = tuple((lang, tuple(n)) for lang, n in sorted(table.notes.items()))
1191
+ return fields, (tuple(sorted(table.grain.items())), tuple(sorted(table.text.items())), notes)
1192
+
1193
+
1194
+ def drift_items(root: Path, config: dict, base: str) -> list:
1195
+ code, _ = git(root, "rev-parse", "--verify", f"{base}^{{commit}}")
1196
+ if code != 0:
1197
+ raise CatalogError(f"drift: {base!r} is not a commit in this git repository")
1198
+ _, out = git(root, "diff", "--name-only", base, "--")
1199
+ changed = set(out.split())
1200
+ items = []
1201
+ for path in find_scripts(root, config["scripts"], config["exclude"]):
1202
+ where = rel(root, path)
1203
+ if where not in changed:
1204
+ continue
1205
+ code, old_text = git(root, "show", f"{base}:{where}")
1206
+ if code != 0:
1207
+ continue # a new script: generate checks that every column is defined
1208
+ new_text = path.read_text(encoding="utf-8")
1209
+ parse = parse_python if path.suffix == ".py" else None
1210
+ try:
1211
+ new_t = (parse(new_text, where, config["languages"], config["contracts"]) if parse
1212
+ else parse_sql(new_text, where, config["languages"]))
1213
+ except CatalogError:
1214
+ continue # generate reports invalid metadata
1215
+ if new_t is None:
1216
+ continue # undocumented: audit reports it
1217
+ try:
1218
+ old_t = (parse(old_text, where, config["languages"], config["contracts"]) if parse
1219
+ else parse_sql(old_text, where, config["languages"]))
1220
+ except CatalogError:
1221
+ old_t = None
1222
+ old_fields, old_table = definitions(old_t)
1223
+ new_fields, new_table = definitions(new_t)
1224
+ edited = {n for n, d in new_fields.items() if old_fields.get(n) != d}
1225
+ any_edit = bool(edited) or old_table != new_table
1226
+ old_shape = statement_shape(old_text, path.suffix, new_t.name)
1227
+ new_shape = statement_shape(new_text, path.suffix, new_t.name)
1228
+ found = []
1229
+ if old_shape and new_shape:
1230
+ for name, expr in new_shape[0].items():
1231
+ if name in old_shape[0] and old_shape[0][name] != expr and name in new_fields and name not in edited:
1232
+ found.append(f"{name}: its SELECT expression changed; its definition did not")
1233
+ if old_shape[1] != new_shape[1] and not any_edit:
1234
+ found.append("filters, joins or CTEs changed; no definition changed (they shape every column)")
1235
+ elif code_without_metadata(old_text, path.suffix) != code_without_metadata(new_text, path.suffix) and not any_edit:
1236
+ found.append("code changed and no definition changed; the columns could not be read, so check each one")
1237
+ if found:
1238
+ items.append((where, new_t.name, found))
1239
+ return items
1240
+
1241
+
1242
+ def cmd_drift(root: Path, base: str) -> int:
1243
+ config = load_config(root)
1244
+ items = drift_items(root, config, base)
1245
+ if not items:
1246
+ print(f"data-catalog drift since {base}: none")
1247
+ return 0
1248
+ count = sum(len(f) for _, _, f in items)
1249
+ print(f"data-catalog drift since {base}: {count} item(s) to confirm")
1250
+ for where, table, found in items:
1251
+ print(f"\n{where} ({table})")
1252
+ for line in found:
1253
+ print(f" - {line}")
1254
+ print("\nUpdate each definition to match the code, or confirm it still holds and say so in the PR.")
1255
+ return 1
1256
+
1257
+
1258
+ # ---------------------------------------------------------------------------
1259
+ # init: config, engine copy, Field/Table module, workflow
1260
+ # (the CONTRIBUTING.md section and AGENTS.md pointer come from persist-conventions.mjs)
1261
+ # ---------------------------------------------------------------------------
1262
+
1263
+ def render_fields_module(languages: list) -> str:
1264
+ lang_fields = "\n".join(f" {lang}: str" for lang in languages)
1265
+ grain_fields = "\n".join(f" grain_{lang}: str" for lang in languages)
1266
+ notes_fields = "\n".join(f" notes_{lang}: tuple[str, ...] = ()" for lang in languages)
1267
+ return f'''"""Column metadata types for the data catalog.
1268
+
1269
+ Generated by data_catalog.py init from data-catalog.json; rerun init after changing
1270
+ `languages` there, never edit this file. Table scripts declare:
1271
+
1272
+ CATALOG_TABLE = Table("schema.table", {", ".join(f"{lang}=..., grain_{lang}=..." for lang in languages)})
1273
+ {FIELDS_VAR} = (Field("column", {", ".join(f"{lang}=..." for lang in languages)}, source=...), ...)
1274
+ PUBLISHED_COLUMNS = tuple(f.name for f in {FIELDS_VAR})
1275
+ """
1276
+ from __future__ import annotations
1277
+
1278
+ from dataclasses import dataclass
1279
+
1280
+
1281
+ @dataclass(frozen=True)
1282
+ class Field:
1283
+ name: str
1284
+ {lang_fields}
1285
+ source: str
1286
+ window: str | None = None
1287
+ unit: str | None = None
1288
+ values: tuple[str, ...] = ()
1289
+
1290
+
1291
+ @dataclass(frozen=True)
1292
+ class Table:
1293
+ name: str
1294
+ {lang_fields}
1295
+ {grain_fields}
1296
+ {notes_fields}
1297
+ '''
1298
+
1299
+
1300
+ def render_workflow(config: dict) -> str:
1301
+ paths = [CONFIG_FILE, ENGINE_PATH, f"{config['output']}/**", *config["scripts"]]
1302
+ if config.get("fieldsModule"):
1303
+ paths.append(config["fieldsModule"])
1304
+ path_lines = "\n".join(" - '" + p.replace("'", "''") + "'" for p in dict.fromkeys(paths))
1305
+ return f"""{WORKFLOW_MARK}
1306
+ name: Data catalog
1307
+
1308
+ on:
1309
+ pull_request:
1310
+ paths:
1311
+ {path_lines}
1312
+ workflow_dispatch:
1313
+
1314
+ permissions:
1315
+ contents: read
1316
+
1317
+ concurrency:
1318
+ group: data-catalog-${{{{ github.ref }}}}
1319
+ cancel-in-progress: true
1320
+
1321
+ jobs:
1322
+ catalog:
1323
+ name: Data catalog
1324
+ runs-on: ubuntu-latest
1325
+ timeout-minutes: 5
1326
+ steps:
1327
+ - uses: actions/checkout@v4
1328
+ - uses: actions/setup-python@v5
1329
+ with:
1330
+ python-version: '3.12'
1331
+ - name: Check the catalog matches the table scripts
1332
+ run: python3 {ENGINE_PATH} generate --check
1333
+ """
1334
+
1335
+
1336
+ def default_fields_module(root: Path, scripts: list) -> str:
1337
+ py = [p for p in find_scripts(root, scripts, []) if p.suffix == ".py"]
1338
+ if not py:
1339
+ return "catalog_fields.py"
1340
+ common = os.path.commonpath([str(p.parent) for p in py])
1341
+ return rel(root, Path(common) / "catalog_fields.py")
1342
+
1343
+
1344
+ def init_plan(root: Path, args) -> tuple[dict, list]:
1345
+ path = root / CONFIG_FILE
1346
+ config = {**DEFAULT_CONFIG, **(read_config_file(root) if path.exists() else {})}
1347
+ if args.scripts:
1348
+ config["scripts"] = args.scripts
1349
+ if args.exclude:
1350
+ config["exclude"] = args.exclude
1351
+ if args.languages:
1352
+ config["languages"] = [x.strip() for x in args.languages.split(",") if x.strip()]
1353
+ if args.output:
1354
+ config["output"] = args.output.rstrip("/")
1355
+ if args.fields_module:
1356
+ config["fieldsModule"] = args.fields_module
1357
+ validate_config(config)
1358
+ if not config.get("fieldsModule"):
1359
+ config["fieldsModule"] = default_fields_module(root, config["scripts"])
1360
+ plan = [
1361
+ (CONFIG_FILE, json.dumps(config, ensure_ascii=False, indent=2) + "\n", "config"),
1362
+ (ENGINE_PATH, Path(__file__).read_text(encoding="utf-8"), f"engine {ENGINE_VERSION}"),
1363
+ (config["fieldsModule"], render_fields_module(config["languages"]), "Field/Table types for table scripts"),
1364
+ (WORKFLOW_PATH, render_workflow(config), "CI: fail a PR whose catalog is stale"),
1365
+ ]
1366
+ return config, plan
1367
+
1368
+
1369
+ def cmd_init(root: Path, args) -> int:
1370
+ config, plan = init_plan(root, args)
1371
+ changed = [(p, text, why) for p, text, why in plan if read_or_none(root / p) != text]
1372
+ for p, text, why in plan:
1373
+ state = "unchanged" if (p, text, why) not in changed else ("write" if args.write else "would write")
1374
+ print(f"{state:12} {p} ({why})")
1375
+ if not args.write:
1376
+ print("\ndry run: rerun with --write to apply")
1377
+ return 0
1378
+ for p, text, _ in changed:
1379
+ target = root / p
1380
+ target.parent.mkdir(parents=True, exist_ok=True)
1381
+ target.write_text(text, encoding="utf-8")
1382
+ print()
1383
+ return cmd_generate(root, check=False)
1384
+
1385
+
1386
+ # ---------------------------------------------------------------------------
1387
+ # CLI
1388
+ # ---------------------------------------------------------------------------
1389
+
1390
+ def engine_identity() -> str:
1391
+ digest = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()[:12]
1392
+ return f"{ENGINE_VERSION}+{digest}"
1393
+
1394
+
1395
+ def find_root(start: Path) -> Path:
1396
+ for candidate in [start, *start.parents]:
1397
+ if (candidate / CONFIG_FILE).exists() or (candidate / ".git").exists():
1398
+ return candidate
1399
+ return start
1400
+
1401
+
1402
+ def main(argv: list | None = None) -> int:
1403
+ parser = argparse.ArgumentParser(prog="data_catalog.py", description=__doc__.split("\n\n")[0])
1404
+ parser.add_argument("--root", type=Path, help="repo root (default: nearest dir with data-catalog.json or .git)")
1405
+ sub = parser.add_subparsers(dest="command", required=True)
1406
+ p_init = sub.add_parser("init", help="set the repo up (dry run unless --write)")
1407
+ p_init.add_argument("--scripts", action="append", help="glob of table scripts (.py/.sql); repeatable")
1408
+ p_init.add_argument("--exclude", action="append", help="glob to skip; repeatable")
1409
+ p_init.add_argument("--languages", help="comma-separated codes, e.g. en,zh")
1410
+ p_init.add_argument("--output", help="catalog directory (default docs/catalog)")
1411
+ p_init.add_argument("--fields-module", help="path for the generated Field/Table module")
1412
+ p_init.add_argument("--write", action="store_true")
1413
+ p_gen = sub.add_parser("generate", help="write the catalog pages")
1414
+ p_gen.add_argument("--check", action="store_true", help="write nothing; exit 1 when stale or invalid")
1415
+ p_audit = sub.add_parser("audit", help="report scripts with missing or weak column definitions")
1416
+ p_audit.add_argument("globs", nargs="*", help="globs to scan (default: config scripts)")
1417
+ p_audit.add_argument("--json", action="store_true")
1418
+ p_audit.add_argument("--verbose", action="store_true", help="also list scripts with no findings")
1419
+ p_audit.add_argument("--table-name", action="append", default=[], metavar="NAME",
1420
+ help="extra variable/attribute whose string value is the table written; repeatable")
1421
+ p_audit.add_argument("--schema-name", action="append", default=[], metavar="NAME",
1422
+ help="extra variable/attribute holding that table's schema; repeatable")
1423
+ p_audit.add_argument("--contract", action="append", default=[], metavar="NAME",
1424
+ help="extra column-contract name (tuple of names or StructType); repeatable")
1425
+ p_drift = sub.add_parser("drift", help="local check: code changed since a git base while its definitions did not")
1426
+ p_drift.add_argument("--base", default="HEAD", help="git ref to compare with (default HEAD: uncommitted work)")
1427
+ sub.add_parser("version", help="print the engine identity")
1428
+ args = parser.parse_args(argv)
1429
+ root = (args.root or find_root(Path.cwd())).resolve()
1430
+ try:
1431
+ if args.command == "version":
1432
+ print(engine_identity())
1433
+ return 0
1434
+ if args.command == "init":
1435
+ return cmd_init(root, args)
1436
+ if args.command == "generate":
1437
+ return cmd_generate(root, args.check)
1438
+ if args.command == "drift":
1439
+ return cmd_drift(root, args.base)
1440
+ return cmd_audit(root, args.globs, args.json, args.verbose, args.table_name, args.schema_name, args.contract)
1441
+ except CatalogError as exc:
1442
+ print(f"data-catalog: {exc}", file=sys.stderr)
1443
+ return 2
1444
+
1445
+
1446
+ if __name__ == "__main__":
1447
+ sys.exit(main())