@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/app-dev/data-catalog/scripts/test_data_catalog.py

@@ -0,0 +1,571 @@
+"""Tests for data_catalog.py. Run: python3 test_data_catalog.py"""
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import subprocess
+import io
+import json
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import data_catalog as dc  # noqa: E402
+
+PY_SCRIPT = '''
+"""Build dwd.customer_daily."""
+from catalog_fields import Field, Table
+
+TARGET_TABLE = "dwd.customer_daily"
+WINDOW_NOTE = "[dt-7d, dt)"
+
+CATALOG_TABLE = Table(
+    "dwd.customer_daily",
+    grain="one row per (dt, cust_id)",
+    en="Daily customer collection rollup.",
+    zh="客户每日收款汇总。",
+    notes=("Amounts in USD via dim_forex.",),
+)
+KEYS = (
+    Field("dt", en="Snapshot date.", zh="快照日期。", source="Run parameter `dt`."),
+)
+CATALOG_FIELDS = (
+    *KEYS,
+    Field(
+        "coll_amt_7d_usd",
+        en="Collection amount | last 7 days.",
+        zh="近 7 天收款金额。",
+        source="dwd_transaction POSTED collections, "
+               "sum of txn_amount_usd.",
+        window=WINDOW_NOTE,
+        unit="USD",
+    ),
+)
+PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
+'''
+
+SQL_SCRIPT = """\
+-- @table ads.merchant_monthly
+--   grain: one row per (month, merchant_id)
+--   en: Monthly merchant volume.
+--   zh: 商户月度交易量。
+--   notes: Built monthly; Excludes test merchants
+-- @field merchant_id
+--   en: Merchant id.
+--   zh: 商户 ID。
+--   source: dim_merchant.merchant_id
+-- @field volume_usd
+--   en: Settled volume in the month.
+--   zh: 当月结算金额。
+--   source: dwd_settlement settled rows,
+--     summed amount_usd
+--   unit: USD
+
+INSERT OVERWRITE TABLE ads.merchant_monthly
+WITH s AS (SELECT merchant_id, amount_usd FROM dwd_settlement WHERE status = 'SETTLED, final')
+SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd
+FROM s GROUP BY s.merchant_id;
+"""
+
+
+def make_repo(files: dict, config: dict | None = None) -> Path:
+    root = Path(tempfile.mkdtemp(prefix="data-catalog-test-"))
+    (root / ".git").mkdir()
+    for name, text in files.items():
+        path = root / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(textwrap.dedent(text) if name.endswith(".py") else text, encoding="utf-8")
+    if config is not None:
+        (root / dc.CONFIG_FILE).write_text(json.dumps({**{"version": 1, "output": "docs/catalog"}, **config}), encoding="utf-8")
+    return root
+
+
+def run(root: Path, *argv: str) -> tuple[int, str, str]:
+    out, err = io.StringIO(), io.StringIO()
+    with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+        code = dc.main(["--root", str(root), *argv])
+    return code, out.getvalue(), err.getvalue()
+
+
+BILINGUAL = {"scripts": ["jobs/**/*.py", "sql/**/*.sql"], "languages": ["en", "zh"]}
+
+
+class GenerateTest(unittest.TestCase):
+    def test_python_and_sql_metadata_render_pages_and_index(self):
+        root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "sql/merchant_monthly.sql": SQL_SCRIPT}, BILINGUAL)
+        code, out, err = run(root, "generate")
+        self.assertEqual(code, 0, err)
+        page = (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8")
+        self.assertTrue(page.startswith(dc.GENERATED_MARK))
+        self.assertIn("- **Script:** [jobs/dwd/customer_daily.py](../../jobs/dwd/customer_daily.py)", page)
+        self.assertIn("| Field | English | 中文 | Source | Window | Unit |", page)
+        # starred KEYS come first, `|` is escaped, implicit concatenation and module constants resolve
+        self.assertIn("| `dt` | Snapshot date. | 快照日期。 | Run parameter `dt`. |  |  |", page)
+        self.assertIn("| `coll_amt_7d_usd` | Collection amount \\| last 7 days. | 近 7 天收款金额。 | "
+                      "dwd_transaction POSTED collections, sum of txn_amount_usd. | [dt-7d, dt) | USD |", page)
+        sql_page = (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8")
+        self.assertIn("| `volume_usd` | Settled volume in the month. | 当月结算金额。 | "
+                      "dwd_settlement settled rows, summed amount_usd | USD |", sql_page)
+        self.assertNotIn("Window", sql_page)
+        self.assertIn("- Excludes test merchants", sql_page)
+        index = (root / "docs/catalog/README.md").read_text(encoding="utf-8")
+        self.assertLess(index.index("ads.merchant_monthly"), index.index("dwd.customer_daily"))
+        self.assertIn("| [`dwd.customer_daily`](dwd.customer_daily.md) | Daily customer collection rollup. | "
+                      "one row per (dt, cust_id) | 2 |", index)
+
+    def test_check_passes_when_current_and_fails_after_a_definition_changes(self):
+        root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
+        self.assertEqual(run(root, "generate")[0], 0)
+        self.assertEqual(run(root, "generate", "--check")[0], 0)
+        script = root / "jobs/dwd/customer_daily.py"
+        script.write_text(script.read_text(encoding="utf-8").replace("Snapshot date.", "Partition date."), encoding="utf-8")
+        code, _, err = run(root, "generate", "--check")
+        self.assertEqual(code, 1)
+        self.assertIn("stale: docs/catalog/dwd.customer_daily.md", err)
+        self.assertNotIn("Partition date.", (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8"))
+
+    def test_removing_metadata_makes_the_page_an_orphan_that_generate_removes(self):
+        root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
+        run(root, "generate")
+        (root / "docs/catalog/hand-written.md").write_text("# kept\n", encoding="utf-8")
+        (root / "jobs/dwd/customer_daily.py").write_text("x = 1\n", encoding="utf-8")
+        code, _, err = run(root, "generate", "--check")
+        self.assertEqual(code, 1)
+        self.assertIn("orphan: docs/catalog/dwd.customer_daily.md", err)
+        self.assertEqual(run(root, "generate")[0], 0)
+        self.assertFalse((root / "docs/catalog/dwd.customer_daily.md").exists())
+        self.assertTrue((root / "docs/catalog/hand-written.md").exists())
+
+    def test_missing_language_text_fails_with_location_and_writes_nothing(self):
+        script = PY_SCRIPT.replace('zh="近 7 天收款金额。",', "")
+        root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertRegex(err, r"jobs/dwd/customer_daily\.py:\d+: Field 'coll_amt_7d_usd' is missing `zh`")
+        self.assertFalse((root / "docs/catalog").exists())
+
+    def test_literal_contract_missing_a_field_is_an_error_and_extra_fields_only_warn(self):
+        script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
+                                   'PUBLISHED_COLUMNS = ("coll_amt_7d_usd", "payout_amt_7d_usd")')
+        root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("PUBLISHED_COLUMNS has column(s) with no Field: payout_amt_7d_usd", err)
+        self.assertIn("Field(s) not in PUBLISHED_COLUMNS: dt", err)
+
+    def test_sql_select_is_the_contract_for_its_comment_block(self):
+        added = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, COUNT(*) AS txn_cnt")
+        root = make_repo({"sql/merchant_monthly.sql": added}, BILINGUAL)
+        code, _, err = run(root, "generate", "--check")
+        self.assertEqual(code, 1)
+        self.assertIn("sql/merchant_monthly.sql:1: the SELECT writing ads.merchant_monthly outputs column(s) "
+                      "with no definition: txn_cnt", err)
+        dropped = SQL_SCRIPT.replace(", SUM(s.amount_usd) AS volume_usd", "")
+        root = make_repo({"sql/merchant_monthly.sql": dropped}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("definition(s) for column(s) the SELECT does not output: volume_usd", err)
+
+    def test_sql_inside_a_documented_python_script_without_a_contract_is_checked(self):
+        script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)", '''
+SQL = """
+INSERT OVERWRITE TABLE dwd.customer_daily
+SELECT dt, SUM(amt) AS coll_amt_7d_usd, COUNT(*) AS coll_cnt_7d FROM dwd.txn GROUP BY dt
+"""
+''')
+        root = make_repo({"jobs/a.py": script}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("outputs column(s) with no definition: coll_cnt_7d", err)
+
+    def test_expression_metadata_and_imported_field_lists_are_rejected(self):
+        expr = PY_SCRIPT.replace('en="Snapshot date."', 'en=f"Snapshot {1}"')
+        root = make_repo({"jobs/a.py": expr}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("metadata must be a string literal", err)
+        imported = PY_SCRIPT.replace("    *KEYS,\n", "    *other.KEYS,\n")
+        root = make_repo({"jobs/a.py": imported}, BILINGUAL)
+        self.assertIn("imported lists cannot be read", run(root, "generate")[2])
+
+    def test_same_table_in_two_scripts_is_an_error(self):
+        root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("table 'dwd.customer_daily' is also documented in jobs/a.py", err)
+
+    def test_hidden_directories_are_not_scanned(self):
+        root = make_repo({"jobs/a.py": PY_SCRIPT, ".worktree/jobs/a.py": PY_SCRIPT},
+                         {**BILINGUAL, "scripts": ["**/jobs/*.py"]})
+        self.assertEqual(run(root, "generate")[0], 0)
+
+
+class AuditTest(unittest.TestCase):
+    def test_reports_undocumented_writers_with_their_columns(self):
+        legacy = '''
+        TARGET_TABLE = "dwd.legacy"
+        PUBLISHED_COLUMNS = ("dt", "cust_id", "amt")
+        '''
+        root = make_repo({"jobs/legacy.py": legacy, "jobs/helper.py": "def f():\n    return 1\n"})
+        code, out, _ = run(root, "audit", "--json", "jobs/*.py")
+        self.assertEqual(code, 0)
+        report = json.loads(out)
+        self.assertEqual((report["scanned"], report["writers"], report["tables"], report["documentedTables"]), (2, 1, 1, 0))
+        [entry] = report["scripts"]
+        self.assertEqual(entry["writes"], [{"table": "dwd.legacy", "evidence": "TARGET_TABLE"}])
+        self.assertEqual(entry["columns"], ["dt", "cust_id", "amt"])
+        kinds = [f["kind"] for f in entry["findings"]]
+        self.assertEqual(kinds, ["no-catalog", "undocumented"])
+
+    def test_a_table_documented_in_its_builder_does_not_flag_backfill_scripts(self):
+        backfill = '''
+        """Backfill: INSERT INTO docs.only_in_docstring SELECT 1"""
+        TARGET_TABLE = "dwd.customer_daily"
+        '''
+        twin = '''
+        TARGET_TABLE = "dwd.undocumented"
+        '''
+        root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/backfill_a.py": backfill,
+                          "jobs/b.py": twin, "jobs/backfill_b.py": twin}, BILINGUAL)
+        report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
+        self.assertEqual(report["jobs/backfill_a.py"]["findings"], [])
+        self.assertEqual([w["table"] for w in report["jobs/backfill_a.py"]["writes"]], ["dwd.customer_daily"])
+        self.assertEqual(report["jobs/b.py"]["findings"][0]["message"],
+                         "writes dwd.undocumented but carries no CATALOG_TABLE/CATALOG_FIELDS (or -- @table block); "
+                         "also written by jobs/backfill_b.py (put the metadata in the script that builds the table)")
+
+    def test_class_attributes_name_the_table_with_their_schema(self):
+        script = '''
+        class OrdersEtl(Etl):
+            dst_db = "dwd"
+            dst_tbl = "orders"
+
+        class Other:
+            out_tbl = "scratch"
+            out_db = "tmp"
+        '''
+        root = make_repo({"jobs/orders.py": script})
+        [entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
+        self.assertEqual(entry["writes"], [{"table": "dwd.orders", "evidence": "dst_tbl"}])
+        [entry] = json.loads(run(root, "audit", "--json", "--table-name", "out_tbl", "--schema-name", "out_db",
+                                 "jobs/*.py")[1])["scripts"]
+        self.assertEqual([w["table"] for w in entry["writes"]], ["dwd.orders", "tmp.scratch"])
+
+    def test_sql_select_list_columns_ignore_cte_columns_comments_and_strings(self):
+        sql = """\
+        -- INSERT INTO not.this
+        INSERT OVERWRITE TABLE ads.daily PARTITION (dt)
+        WITH base AS (SELECT a, b AS inner_only FROM src)
+        SELECT DISTINCT base.a,
+               CASE WHEN b > 0 THEN 'x, y' ELSE 'z' END AS flag,
+               sum(b) total,
+               coalesce(c, 0),
+               t.*
+        FROM base JOIN t ON t.a = base.a;
+        """
+        root = make_repo({"sql/daily.sql": textwrap.dedent(sql)})
+        report = json.loads(run(root, "audit", "--json", "sql/*.sql")[1])
+        [entry] = report["scripts"]
+        self.assertEqual([w["table"] for w in entry["writes"]], ["ads.daily"])
+        self.assertEqual(entry["columns"], ["a", "flag", "total"])
+        self.assertFalse(entry["columnsComplete"])
+
+    def test_sql_inside_python_strings_is_read(self):
+        script = '''
+        def run(spark, target):
+            spark.sql(f"""
+                INSERT INTO TABLE {target}
+                SELECT cust_id, count(*) AS txn_cnt FROM dwd.txn GROUP BY cust_id
+            """)
+            df.write.format("hudi").option("hoodie.table.name", "dwd_txn_cnt").save(path)
+        '''
+        root = make_repo({"jobs/cnt.py": script})
+        [entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
+        self.assertEqual([w["table"] for w in entry["writes"]], ["dwd_txn_cnt", "{expr}"])
+        self.assertEqual(entry["columns"], ["cust_id", "txn_cnt"])
+
+    def test_partial_catalog_names_missing_columns_and_thin_definitions(self):
+        script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
+                                   'PUBLISHED_COLUMNS = ("dt", "coll_amt_7d_usd", "payout_amt_7d_usd")')
+        script = script.replace('en="Snapshot date."', 'en="dt"').replace('zh="近 7 天收款金额。"', 'zh="同上，7 天窗口。"')
+        root = make_repo({"jobs/a.py": script}, BILINGUAL)
+        [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
+        findings = {f["kind"]: f["message"] for f in entry["findings"]}
+        self.assertEqual(findings["undocumented"], "1 column(s) with no Field: payout_amt_7d_usd")
+        self.assertEqual(findings["contract-not-derived"], "PUBLISHED_COLUMNS is a literal list; derive it from CATALOG_FIELDS")
+        self.assertEqual(findings["thin-definition"], "definition or source only restates the name: dt")
+        self.assertEqual(findings["relative-definition"], 'definition or source points at another row ("same as above"); '
+                                                          "each row must stand alone: coll_amt_7d_usd")
+
+    def test_writing_rule_leads_flag_bloated_rows_and_spare_plain_ones(self):
+        shared = "dwd.transaction where status = 'POSTED' and cust_type <> 'TEST' and txn_date < dt"
+        rows = "\n".join(
+            f'''    Field("{name}", en="{en}", zh="{zh}", source="{shared}, {agg}."),'''
+            for name, en, zh, agg in [
+                ("a_cnt", "Number of posted rows whose currency has a dim.forex rate on txn_date.", "近 7 天收款笔数。", "count rows"),
+                ("b_cnt", "Number of distinct payer banks.", "不同付款银行数。", "count distinct payer_bank"),
+                ("c_amt", " ".join(["word"] * 31), "近 7 天收款总额。", "sum of amount"),
+            ])
+        script = f'''
+CATALOG_TABLE = Table("dwd.t", grain="One row per id.", en="Table.", zh="表。")
+CATALOG_FIELDS = (
+{rows}
+)
+'''
+        root = make_repo({"jobs/t.py": script}, BILINGUAL)
+        [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
+        findings = {f["kind"]: f["message"] for f in entry["findings"]}
+        self.assertIn("appears in 3 of 3 rows", findings["repeated-text"])
+        self.assertTrue(findings["code-in-description"].endswith(": a_cnt"))
+        self.assertTrue(findings["long-definition"].endswith(": c_amt"))
+
+    def test_abbreviations_are_not_mistaken_for_table_names(self):
+        self.assertIsNone(dc.CODE_IN_TEXT_RE.search("Created between 0 a.m. and 6 a.m., e.g. at night."))
+        self.assertIsNotNone(dc.CODE_IN_TEXT_RE.search("Rate from dim.forex."))
+
+    def test_shared_vocabulary_and_plain_units_are_not_flagged(self):
+        script = '''
+CATALOG_TABLE = Table("dwd.t", grain="One row per id.", en="Table.", zh="表。",
+                      notes=("All metrics count posted collections.", "Every metric is 0 if there were no collections."))
+CATALOG_FIELDS = (
+    Field("a", en="Number of collections in the last 7 days.", zh="近 7 天收款笔数。",
+          source="count of dwd.transaction rows", window="[dt-7d, dt)"),
+    Field("b", en="Number of collections in the last 30 days.", zh="近 30 天收款笔数。",
+          source="count of dwd.transaction rows", window="[dt-30d, dt)"),
+    Field("c", en="Total collected in the last 7 days, in USD.", zh="近 7 天收款总额（USD）。",
+          source="sum of dwd.transaction.amount_usd", window="[dt-7d, dt)"),
+)
+PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
+'''
+        root = make_repo({"jobs/t.py": script}, BILINGUAL)
+        [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
+        self.assertEqual(entry["findings"], [])
+
+    def test_documented_script_with_derived_contract_has_no_findings(self):
+        root = make_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
+        report = json.loads(run(root, "audit", "--json")[1])
+        self.assertEqual((report["tables"], report["documentedTables"]), (1, 1))
+        self.assertEqual(report["scripts"][0]["findings"], [])
+
+    def test_audit_without_config_or_globs_explains_what_to_pass(self):
+        root = make_repo({})
+        code, _, err = run(root, "audit")
+        self.assertEqual(code, 2)
+        self.assertIn("pass globs to audit", err)
+
+
+class ReviewFixTest(unittest.TestCase):
+    def test_trailing_slash_in_output_neither_fails_check_nor_deletes_pages(self):
+        root = make_repo({"sql/merchant_monthly.sql": SQL_SCRIPT}, {**BILINGUAL, "output": "docs/catalog/"})
+        self.assertEqual(run(root, "generate")[0], 0)
+        code, _, err = run(root, "generate", "--check")
+        self.assertEqual(code, 0, err)
+        self.assertIn("0 written, 0 removed", run(root, "generate")[1])
+        self.assertTrue((root / "docs/catalog/ads.merchant_monthly.md").exists())
+
+    def test_plain_ctas_select_is_the_contract(self):
+        sql = SQL_SCRIPT.split("INSERT OVERWRITE")[0] + "CREATE TABLE ads.merchant_monthly AS SELECT merchant_id, volume_usd, extra FROM s;\n"
+        root = make_repo({"sql/m.sql": sql}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("outputs column(s) with no definition: extra", err)
+
+    def test_unaliased_expressions_never_fail_ci_on_a_guessed_name(self):
+        sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, s.price * s.qty, CAST(b AS INT) + 1")
+        root = make_repo({"sql/m.sql": sql}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 0, err)
+        self.assertIn("has `*` or unnamed items; only its named columns are checked", err)
+
+    def test_an_unrelated_statement_is_never_used_as_the_contract(self):
+        sql = SQL_SCRIPT.replace("SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd", "SELECT *") + \
+            "INSERT INTO tmp.stage SELECT zz FROM q;\n"
+        root = make_repo({"sql/m.sql": sql}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 0, err)
+        self.assertIn("no column contract and no readable SELECT writing ads.merchant_monthly", err)
+
+    def test_tables_mapping_to_one_page_or_the_index_are_errors(self):
+        a = PY_SCRIPT.replace('"dwd.customer_daily",\n    grain', '"ads.a_b",\n    grain')
+        b = PY_SCRIPT.replace('"dwd.customer_daily",\n    grain', '"ads.a b",\n    grain')
+        readme = PY_SCRIPT.replace('"dwd.customer_daily",\n    grain', '"README",\n    grain')
+        root = make_repo({"jobs/a.py": a, "jobs/b.py": b, "jobs/c.py": readme}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("map to the same page ads.a_b.md", err)
+        self.assertIn("would overwrite the catalog index", err)
+
+    def test_exclude_globs_match_root_level_files_like_script_globs(self):
+        root = make_repo({"a_legacy.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT.replace("dwd.customer_daily", "dwd.other")},
+                         {**BILINGUAL, "scripts": ["**/*.py"], "exclude": ["**/*_legacy.py"]})
+        self.assertEqual(run(root, "generate")[0], 0)
+        self.assertEqual(sorted(p.name for p in (root / "docs/catalog").iterdir()), ["README.md", "dwd.other.md"])
+
+    def test_same_short_name_in_another_schema_is_a_different_table(self):
+        ods = '''
+        TARGET_TABLE = "ods.customer_daily"
+        '''
+        root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/ods.py": ods}, BILINGUAL)
+        report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
+        self.assertEqual([f["kind"] for f in report["jobs/ods.py"]["findings"]], ["no-catalog"])
+
+    def test_audit_ignores_commented_writes_dynamic_names_and_directories(self):
+        script = PY_SCRIPT + '''
+# df.write.insertInto("old.dropped_table")
+def run(spark, db, tbl):
+    spark.sql(f"INSERT OVERWRITE TABLE {db}.{tbl} SELECT 1")
+    spark.sql("INSERT OVERWRITE LOCAL DIRECTORY '/tmp/x' SELECT 1")
+'''
+        root = make_repo({"jobs/a.py": script}, BILINGUAL)
+        [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
+        self.assertEqual(sorted(w["table"] for w in entry["writes"]), ["dwd.customer_daily", "{expr}.{expr}"])
+        self.assertEqual(entry["findings"], [])
+
+    def test_python_syntax_error_is_one_finding(self):
+        root = make_repo({"jobs/a.py": "CATALOG_TABLE = (\n"}, BILINGUAL)
+        [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
+        self.assertEqual([f["kind"] for f in entry["findings"]], ["invalid-metadata"])
+
+    def test_sql_block_ends_at_code_and_malformed_directives_fail(self):
+        late = SQL_SCRIPT + "-- @field ghost\n--   en: x\n"
+        root = make_repo({"sql/m.sql": late}, BILINGUAL)
+        self.assertEqual(run(root, "generate")[0], 0)
+        self.assertNotIn("ghost", (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8"))
+        bad = SQL_SCRIPT.replace("-- @table ads.merchant_monthly", "-- @table ads.merchant_monthly   (monthly)")
+        root = make_repo({"sql/m.sql": bad}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("sql/m.sql:1: write `-- @table <name>`", err)
+
+    def test_backslash_escaped_quote_keeps_the_select_readable(self):
+        sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, 'it\\'s' AS note")
+        root = make_repo({"sql/m.sql": sql}, BILINGUAL)
+        code, _, err = run(root, "generate")
+        self.assertEqual(code, 1)
+        self.assertIn("outputs column(s) with no definition: note", err)
+
+    def test_init_with_invalid_json_config_exits_2(self):
+        root = make_repo({"jobs/a.py": "x = 1\n"})
+        (root / dc.CONFIG_FILE).write_text("{oops", encoding="utf-8")
+        code, _, err = run(root, "init", "--write")
+        self.assertEqual(code, 2)
+        self.assertIn("invalid JSON", err)
+
+
+def git_repo(files: dict, config: dict) -> Path:
+    root = make_repo(files, config)
+    (root / ".git").rmdir()
+    for cmd in (["init", "-q"], ["add", "-A"], ["-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "base"]):
+        subprocess.run(["git", *cmd], cwd=root, check=True)
+    return root
+
+
+def edit(root: Path, name: str, old: str, new: str) -> None:
+    path = root / name
+    text = path.read_text(encoding="utf-8")
+    assert old in text, old
+    path.write_text(text.replace(old, new), encoding="utf-8")
+
+
+class DriftTest(unittest.TestCase):
+    def test_changed_expression_without_changed_definition_is_reported(self):
+        root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
+        self.assertEqual(run(root, "drift")[0], 0)
+        edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd * 0.9) AS volume_usd")
+        code, out, _ = run(root, "drift")
+        self.assertEqual(code, 1)
+        self.assertIn("sql/m.sql (ads.merchant_monthly)\n  - volume_usd: its SELECT expression changed; its definition did not", out)
+        self.assertNotIn("filters, joins", out)
+        edit(root, "sql/m.sql", "Settled volume in the month.", "Settled volume net of the 10% fee.")
+        self.assertEqual(run(root, "drift")[0], 0)
+
+    def test_changed_filter_flags_the_table_until_a_definition_changes(self):
+        root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
+        edit(root, "sql/m.sql", "'SETTLED, final'", "'SETTLED'")
+        code, out, _ = run(root, "drift")
+        self.assertEqual(code, 1)
+        self.assertIn("filters, joins or CTEs changed; no definition changed", out)
+        edit(root, "sql/m.sql", "dwd_settlement settled rows,", "dwd_settlement rows with status SETTLED,")
+        self.assertEqual(run(root, "drift")[0], 0)
+
+    def test_dataframe_script_change_without_definition_change_is_reported(self):
+        script = PY_SCRIPT + "\ndef build(df):\n    return df.filter('status = 1')\n"
+        root = git_repo({"jobs/a.py": script}, BILINGUAL)
+        edit(root, "jobs/a.py", "status = 1", "status = 2")
+        code, out, _ = run(root, "drift")
+        self.assertEqual(code, 1)
+        self.assertIn("code changed and no definition changed; the columns could not be read", out)
+
+    def test_definition_only_edit_and_untracked_new_script_are_quiet(self):
+        root = git_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
+        edit(root, "jobs/a.py", "Snapshot date.", "Snapshot date (UTC).")
+        (root / "jobs/new.py").write_text(textwrap.dedent(PY_SCRIPT).replace("dwd.customer_daily", "dwd.new"), encoding="utf-8")
+        self.assertEqual(run(root, "drift")[0], 0)
+
+    def test_base_ref_compares_a_whole_branch(self):
+        root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
+        edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "MAX(s.amount_usd) AS volume_usd")
+        subprocess.run(["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "change"], cwd=root, check=True)
+        self.assertEqual(run(root, "drift")[0], 0)
+        self.assertEqual(run(root, "drift", "--base", "HEAD~1")[0], 1)
+
+    def test_outside_git_drift_exits_2(self):
+        root = make_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
+        (root / ".git").rmdir()
+        code, _, err = run(root, "drift")
+        self.assertEqual(code, 2)
+        self.assertIn("is not a commit", err)
+
+
+class InitTest(unittest.TestCase):
+    def test_dry_run_writes_nothing(self):
+        root = make_repo({"jobs/dwd/a.py": "x = 1\n"})
+        code, out, _ = run(root, "init", "--scripts", "jobs/**/*.py")
+        self.assertEqual(code, 0)
+        self.assertIn("would write  data-catalog.json", out)
+        self.assertEqual(sorted(p.name for p in root.iterdir()), [".git", "jobs"])
+
+    def test_write_sets_up_repo_and_generated_module_accepts_documented_scripts(self):
+        root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "jobs/dwd/other.py": "x = 1\n",
+                          "AGENTS.md": "# Agents\n\nExisting rules.\n"})
+        code, out, err = run(root, "init", "--scripts", "jobs/**/*.py", "--languages", "en,zh", "--write")
+        self.assertEqual(code, 0, err)
+        config = json.loads((root / dc.CONFIG_FILE).read_text(encoding="utf-8"))
+        self.assertEqual(config["fieldsModule"], "jobs/dwd/catalog_fields.py")
+        self.assertEqual((root / dc.ENGINE_PATH).read_text(encoding="utf-8"), (HERE / "data_catalog.py").read_text(encoding="utf-8"))
+        workflow = (root / dc.WORKFLOW_PATH).read_text(encoding="utf-8")
+        self.assertIn("      - 'jobs/**/*.py'\n", workflow)
+        self.assertIn("run: python3 scripts/data-catalog/data_catalog.py generate --check", workflow)
+        # agent pointers are persist-conventions.mjs's job; init leaves them alone
+        self.assertEqual((root / "AGENTS.md").read_text(encoding="utf-8"), "# Agents\n\nExisting rules.\n")
+        self.assertFalse((root / "CLAUDE.md").exists())
+        self.assertTrue((root / "docs/catalog/dwd.customer_daily.md").exists())
+        # the generated runtime module takes exactly the arguments the engine reads
+        spec = importlib.util.spec_from_file_location("catalog_fields", root / config["fieldsModule"])
+        module = importlib.util.module_from_spec(spec)
+        sys.modules["catalog_fields"] = module  # dataclasses resolve string annotations through sys.modules
+        self.addCleanup(sys.modules.pop, "catalog_fields", None)
+        spec.loader.exec_module(module)
+        f = module.Field("x", en="e", zh="z", source="s", window="7d", unit="USD")
+        self.assertEqual((f.name, f.zh, f.unit), ("x", "z", "USD"))
+        with self.assertRaises(TypeError):
+            module.Field("x", en="e", source="s")
+        # rerunning is a no-op, and the catalog check passes
+        code, out, _ = run(root, "init", "--write")
+        self.assertEqual(code, 0)
+        self.assertNotIn("write ", out.replace("would write", ""))
+        self.assertEqual(run(root, "generate", "--check")[0], 0)
+
+    def test_bad_language_code_is_refused(self):
+        root = make_repo({"jobs/a.py": "x = 1\n"})
+        code, _, err = run(root, "init", "--scripts", "jobs/*.py", "--languages", "English", "--write")
+        self.assertEqual(code, 2)
+        self.assertIn("`languages` must be codes", err)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/skills/app-dev/ios-foundation/LICENSE

@@ -0,0 +1,3 @@
+MIT License
+
+Copyright (c) 2026 1AboveIO