@1aboveio/app-dev-skills 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,603 @@
1
+ """Tests for data_catalog.py. Run: python3 test_data_catalog.py"""
2
+ from __future__ import annotations
3
+
4
+ import contextlib
5
+ import importlib.util
6
+ import subprocess
7
+ import io
8
+ import json
9
+ import sys
10
+ import tempfile
11
+ import textwrap
12
+ import unittest
13
+ from pathlib import Path
14
+
15
+ HERE = Path(__file__).resolve().parent
16
+ sys.path.insert(0, str(HERE))
17
+ import data_catalog as dc # noqa: E402
18
+
19
+ PY_SCRIPT = '''
20
+ """Build dwd.customer_daily."""
21
+ from catalog_fields import Field, Table
22
+
23
+ TARGET_TABLE = "dwd.customer_daily"
24
+ WINDOW_NOTE = "[dt-7d, dt)"
25
+
26
+ CATALOG_TABLE = Table(
27
+ "dwd.customer_daily",
28
+ en="Daily customer collection rollup.",
29
+ zh="客户每日收款汇总。",
30
+ grain_en="one row per (dt, cust_id)",
31
+ grain_zh="每个 (dt, cust_id) 一行",
32
+ notes_en=("Amounts in USD via dim_forex.",),
33
+ notes_zh=("金额经 dim_forex 换算为 USD。",),
34
+ )
35
+ KEYS = (
36
+ Field("dt", en="Snapshot date.", zh="快照日期。", source="Run parameter `dt`."),
37
+ )
38
+ CATALOG_FIELDS = (
39
+ *KEYS,
40
+ Field(
41
+ "coll_amt_7d_usd",
42
+ en="Collection amount | last 7 days.",
43
+ zh="近 7 天收款金额。",
44
+ source="dwd_transaction POSTED collections, "
45
+ "sum of txn_amount_usd.",
46
+ window=WINDOW_NOTE,
47
+ unit="USD",
48
+ ),
49
+ )
50
+ PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
51
+ '''
52
+
53
+ SQL_SCRIPT = """\
54
+ -- @table ads.merchant_monthly
55
+ -- en: Monthly merchant volume.
56
+ -- zh: 商户月度交易量。
57
+ -- grain_en: one row per (month, merchant_id)
58
+ -- grain_zh: 每个 (month, merchant_id) 一行
59
+ -- notes_en: Built monthly; Excludes test merchants
60
+ -- notes_zh: 每月构建;不含测试商户
61
+ -- @field merchant_id
62
+ -- en: Merchant id.
63
+ -- zh: 商户 ID。
64
+ -- source: dim_merchant.merchant_id
65
+ -- @field volume_usd
66
+ -- en: Settled volume in the month.
67
+ -- zh: 当月结算金额。
68
+ -- source: dwd_settlement settled rows,
69
+ -- summed amount_usd
70
+ -- unit: USD
71
+
72
+ INSERT OVERWRITE TABLE ads.merchant_monthly
73
+ WITH s AS (SELECT merchant_id, amount_usd FROM dwd_settlement WHERE status = 'SETTLED, final')
74
+ SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd
75
+ FROM s GROUP BY s.merchant_id;
76
+ """
77
+
78
+
79
+ def make_repo(files: dict, config: dict | None = None) -> Path:
80
+ root = Path(tempfile.mkdtemp(prefix="data-catalog-test-"))
81
+ (root / ".git").mkdir()
82
+ for name, text in files.items():
83
+ path = root / name
84
+ path.parent.mkdir(parents=True, exist_ok=True)
85
+ path.write_text(textwrap.dedent(text) if name.endswith(".py") else text, encoding="utf-8")
86
+ if config is not None:
87
+ (root / dc.CONFIG_FILE).write_text(json.dumps({**{"version": 1, "output": "docs/catalog"}, **config}), encoding="utf-8")
88
+ return root
89
+
90
+
91
+ def run(root: Path, *argv: str) -> tuple[int, str, str]:
92
+ out, err = io.StringIO(), io.StringIO()
93
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
94
+ code = dc.main(["--root", str(root), *argv])
95
+ return code, out.getvalue(), err.getvalue()
96
+
97
+
98
+ BILINGUAL = {"scripts": ["jobs/**/*.py", "sql/**/*.sql"], "languages": ["en", "zh"]}
99
+
100
+
101
+ class GenerateTest(unittest.TestCase):
102
+ def test_python_and_sql_metadata_render_pages_and_index(self):
103
+ root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "sql/merchant_monthly.sql": SQL_SCRIPT}, BILINGUAL)
104
+ code, out, err = run(root, "generate")
105
+ self.assertEqual(code, 0, err)
106
+ page = (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8")
107
+ self.assertTrue(page.startswith(dc.GENERATED_MARK))
108
+ self.assertIn("- **Script:** [jobs/dwd/customer_daily.py](../../jobs/dwd/customer_daily.py)", page)
109
+ self.assertIn("| Field | English | 中文 | Source | Window | Unit |", page)
110
+ # starred KEYS come first, `|` is escaped, implicit concatenation and module constants resolve
111
+ self.assertIn("| `dt` | Snapshot date. | 快照日期。 | Run parameter `dt`. | | |", page)
112
+ self.assertIn("| `coll_amt_7d_usd` | Collection amount \\| last 7 days. | 近 7 天收款金额。 | "
113
+ "dwd_transaction POSTED collections, sum of txn_amount_usd. | [dt-7d, dt) | USD |", page)
114
+ sql_page = (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8")
115
+ self.assertIn("| `volume_usd` | Settled volume in the month. | 当月结算金额。 | "
116
+ "dwd_settlement settled rows, summed amount_usd | USD |", sql_page)
117
+ self.assertNotIn("Window", sql_page)
118
+ self.assertIn(" - Excludes test merchants", sql_page)
119
+ self.assertIn("## 中文", sql_page)
120
+ self.assertIn("- **粒度:** 每个 (month, merchant_id) 一行", sql_page)
121
+ self.assertIn(" - 不含测试商户", sql_page)
122
+ index = (root / "docs/catalog/README.md").read_text(encoding="utf-8")
123
+ self.assertLess(index.index("ads.merchant_monthly"), index.index("dwd.customer_daily"))
124
+ self.assertIn("| [`dwd.customer_daily`](dwd.customer_daily.md) | Daily customer collection rollup. | "
125
+ "one row per (dt, cust_id) | 2 |", index)
126
+
127
+ def test_check_passes_when_current_and_fails_after_a_definition_changes(self):
128
+ root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
129
+ self.assertEqual(run(root, "generate")[0], 0)
130
+ self.assertEqual(run(root, "generate", "--check")[0], 0)
131
+ script = root / "jobs/dwd/customer_daily.py"
132
+ script.write_text(script.read_text(encoding="utf-8").replace("Snapshot date.", "Partition date."), encoding="utf-8")
133
+ code, _, err = run(root, "generate", "--check")
134
+ self.assertEqual(code, 1)
135
+ self.assertIn("stale: docs/catalog/dwd.customer_daily.md", err)
136
+ self.assertNotIn("Partition date.", (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8"))
137
+
138
+ def test_removing_metadata_makes_the_page_an_orphan_that_generate_removes(self):
139
+ root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
140
+ run(root, "generate")
141
+ (root / "docs/catalog/hand-written.md").write_text("# kept\n", encoding="utf-8")
142
+ (root / "jobs/dwd/customer_daily.py").write_text("x = 1\n", encoding="utf-8")
143
+ code, _, err = run(root, "generate", "--check")
144
+ self.assertEqual(code, 1)
145
+ self.assertIn("orphan: docs/catalog/dwd.customer_daily.md", err)
146
+ self.assertEqual(run(root, "generate")[0], 0)
147
+ self.assertFalse((root / "docs/catalog/dwd.customer_daily.md").exists())
148
+ self.assertTrue((root / "docs/catalog/hand-written.md").exists())
149
+
150
+ def test_missing_language_text_fails_with_location_and_writes_nothing(self):
151
+ script = PY_SCRIPT.replace('zh="近 7 天收款金额。",', "")
152
+ root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
153
+ code, _, err = run(root, "generate")
154
+ self.assertEqual(code, 1)
155
+ self.assertRegex(err, r"jobs/dwd/customer_daily\.py:\d+: Field 'coll_amt_7d_usd' is missing `zh`")
156
+ self.assertFalse((root / "docs/catalog").exists())
157
+
158
+ def test_literal_contract_missing_a_field_is_an_error_and_extra_fields_only_warn(self):
159
+ script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
160
+ 'PUBLISHED_COLUMNS = ("coll_amt_7d_usd", "payout_amt_7d_usd")')
161
+ root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
162
+ code, _, err = run(root, "generate")
163
+ self.assertEqual(code, 1)
164
+ self.assertIn("PUBLISHED_COLUMNS has column(s) with no Field: payout_amt_7d_usd", err)
165
+ self.assertIn("Field(s) not in PUBLISHED_COLUMNS: dt", err)
166
+
167
+ def test_sql_select_is_the_contract_for_its_comment_block(self):
168
+ added = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, COUNT(*) AS txn_cnt")
169
+ root = make_repo({"sql/merchant_monthly.sql": added}, BILINGUAL)
170
+ code, _, err = run(root, "generate", "--check")
171
+ self.assertEqual(code, 1)
172
+ self.assertIn("sql/merchant_monthly.sql:1: the SELECT writing ads.merchant_monthly outputs column(s) "
173
+ "with no definition: txn_cnt", err)
174
+ dropped = SQL_SCRIPT.replace(", SUM(s.amount_usd) AS volume_usd", "")
175
+ root = make_repo({"sql/merchant_monthly.sql": dropped}, BILINGUAL)
176
+ code, _, err = run(root, "generate")
177
+ self.assertEqual(code, 1)
178
+ self.assertIn("definition(s) for column(s) the SELECT does not output: volume_usd", err)
179
+
180
+ def test_sql_inside_a_documented_python_script_without_a_contract_is_checked(self):
181
+ script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)", '''
182
+ SQL = """
183
+ INSERT OVERWRITE TABLE dwd.customer_daily
184
+ SELECT dt, SUM(amt) AS coll_amt_7d_usd, COUNT(*) AS coll_cnt_7d FROM dwd.txn GROUP BY dt
185
+ """
186
+ ''')
187
+ root = make_repo({"jobs/a.py": script}, BILINGUAL)
188
+ code, _, err = run(root, "generate")
189
+ self.assertEqual(code, 1)
190
+ self.assertIn("outputs column(s) with no definition: coll_cnt_7d", err)
191
+
192
+ def test_expression_metadata_and_imported_field_lists_are_rejected(self):
193
+ expr = PY_SCRIPT.replace('en="Snapshot date."', 'en=f"Snapshot {1}"')
194
+ root = make_repo({"jobs/a.py": expr}, BILINGUAL)
195
+ code, _, err = run(root, "generate")
196
+ self.assertEqual(code, 1)
197
+ self.assertIn("metadata must be a string literal", err)
198
+ imported = PY_SCRIPT.replace(" *KEYS,\n", " *other.KEYS,\n")
199
+ root = make_repo({"jobs/a.py": imported}, BILINGUAL)
200
+ self.assertIn("imported lists cannot be read", run(root, "generate")[2])
201
+
202
+ def test_table_grain_and_notes_are_per_language_with_matching_notes(self):
203
+ old = PY_SCRIPT.replace('grain_en="one row per (dt, cust_id)",', 'grain="one row per (dt, cust_id)",')
204
+ root = make_repo({"jobs/a.py": old}, BILINGUAL)
205
+ code, _, err = run(root, "generate")
206
+ self.assertEqual(code, 1)
207
+ self.assertIn("grain and notes are written per language: grain_en=..., notes_en=(...), grain_zh=...", err)
208
+ uneven = PY_SCRIPT.replace('notes_zh=("金额经 dim_forex 换算为 USD。",),', 'notes_zh=(),')
209
+ root = make_repo({"jobs/a.py": uneven}, BILINGUAL)
210
+ code, _, err = run(root, "generate")
211
+ self.assertEqual(code, 1)
212
+ self.assertIn("every language needs the same notes, got {'en': 1, 'zh': 0}", err)
213
+
214
+ def test_same_table_in_two_scripts_is_an_error(self):
215
+ root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT}, BILINGUAL)
216
+ code, _, err = run(root, "generate")
217
+ self.assertEqual(code, 1)
218
+ self.assertIn("table 'dwd.customer_daily' is also documented in jobs/a.py", err)
219
+
220
+ def test_hidden_directories_are_not_scanned(self):
221
+ root = make_repo({"jobs/a.py": PY_SCRIPT, ".worktree/jobs/a.py": PY_SCRIPT},
222
+ {**BILINGUAL, "scripts": ["**/jobs/*.py"]})
223
+ self.assertEqual(run(root, "generate")[0], 0)
224
+
225
+
226
+ class AuditTest(unittest.TestCase):
227
+ def test_reports_undocumented_writers_with_their_columns(self):
228
+ legacy = '''
229
+ TARGET_TABLE = "dwd.legacy"
230
+ PUBLISHED_COLUMNS = ("dt", "cust_id", "amt")
231
+ '''
232
+ root = make_repo({"jobs/legacy.py": legacy, "jobs/helper.py": "def f():\n return 1\n"})
233
+ code, out, _ = run(root, "audit", "--json", "jobs/*.py")
234
+ self.assertEqual(code, 0)
235
+ report = json.loads(out)
236
+ self.assertEqual((report["scanned"], report["writers"], report["tables"], report["documentedTables"]), (2, 1, 1, 0))
237
+ [entry] = report["scripts"]
238
+ self.assertEqual(entry["writes"], [{"table": "dwd.legacy", "evidence": "TARGET_TABLE"}])
239
+ self.assertEqual(entry["columns"], ["dt", "cust_id", "amt"])
240
+ kinds = [f["kind"] for f in entry["findings"]]
241
+ self.assertEqual(kinds, ["no-catalog", "undocumented"])
242
+
243
+ def test_a_table_documented_in_its_builder_does_not_flag_backfill_scripts(self):
244
+ backfill = '''
245
+ """Backfill: INSERT INTO docs.only_in_docstring SELECT 1"""
246
+ TARGET_TABLE = "dwd.customer_daily"
247
+ '''
248
+ twin = '''
249
+ TARGET_TABLE = "dwd.undocumented"
250
+ '''
251
+ root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/backfill_a.py": backfill,
252
+ "jobs/b.py": twin, "jobs/backfill_b.py": twin}, BILINGUAL)
253
+ report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
254
+ self.assertEqual(report["jobs/backfill_a.py"]["findings"], [])
255
+ self.assertEqual([w["table"] for w in report["jobs/backfill_a.py"]["writes"]], ["dwd.customer_daily"])
256
+ self.assertEqual(report["jobs/b.py"]["findings"][0]["message"],
257
+ "writes dwd.undocumented but carries no CATALOG_TABLE/CATALOG_FIELDS (or -- @table block); "
258
+ "also written by jobs/backfill_b.py (put the metadata in the script that builds the table)")
259
+
260
+ def test_a_structtype_contract_is_read_for_its_column_names(self):
261
+ script = '''
262
+ L1A_PUBLISHED_SCHEMA = StructType([
263
+ StructField("dt", DateType(), False),
264
+ StructField("cust_id", StringType(), False),
265
+ ])
266
+ TARGET_TABLE = "dwd.flow"
267
+ '''
268
+ root = make_repo({"jobs/flow.py": script})
269
+ [entry] = json.loads(run(root, "audit", "--json", "--contract", "L1A_PUBLISHED_SCHEMA", "jobs/*.py")[1])["scripts"]
270
+ self.assertEqual((entry["columns"], entry["columnsFrom"]), (["dt", "cust_id"], "L1A_PUBLISHED_SCHEMA"))
271
+
272
+ def test_class_attributes_name_the_table_with_their_schema(self):
273
+ script = '''
274
+ class OrdersEtl(Etl):
275
+ dst_db = "dwd"
276
+ dst_tbl = "orders"
277
+
278
+ class Other:
279
+ out_tbl = "scratch"
280
+ out_db = "tmp"
281
+ '''
282
+ root = make_repo({"jobs/orders.py": script})
283
+ [entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
284
+ self.assertEqual(entry["writes"], [{"table": "dwd.orders", "evidence": "dst_tbl"}])
285
+ [entry] = json.loads(run(root, "audit", "--json", "--table-name", "out_tbl", "--schema-name", "out_db",
286
+ "jobs/*.py")[1])["scripts"]
287
+ self.assertEqual([w["table"] for w in entry["writes"]], ["dwd.orders", "tmp.scratch"])
288
+
289
+ def test_sql_select_list_columns_ignore_cte_columns_comments_and_strings(self):
290
+ sql = """\
291
+ -- INSERT INTO not.this
292
+ INSERT OVERWRITE TABLE ads.daily PARTITION (dt)
293
+ WITH base AS (SELECT a, b AS inner_only FROM src)
294
+ SELECT DISTINCT base.a,
295
+ CASE WHEN b > 0 THEN 'x, y' ELSE 'z' END AS flag,
296
+ sum(b) total,
297
+ coalesce(c, 0),
298
+ t.*
299
+ FROM base JOIN t ON t.a = base.a;
300
+ """
301
+ root = make_repo({"sql/daily.sql": textwrap.dedent(sql)})
302
+ report = json.loads(run(root, "audit", "--json", "sql/*.sql")[1])
303
+ [entry] = report["scripts"]
304
+ self.assertEqual([w["table"] for w in entry["writes"]], ["ads.daily"])
305
+ self.assertEqual(entry["columns"], ["a", "flag", "total"])
306
+ self.assertFalse(entry["columnsComplete"])
307
+
308
+ def test_sql_inside_python_strings_is_read(self):
309
+ script = '''
310
+ def run(spark, target):
311
+ spark.sql(f"""
312
+ INSERT INTO TABLE {target}
313
+ SELECT cust_id, count(*) AS txn_cnt FROM dwd.txn GROUP BY cust_id
314
+ """)
315
+ df.write.format("hudi").option("hoodie.table.name", "dwd_txn_cnt").save(path)
316
+ '''
317
+ root = make_repo({"jobs/cnt.py": script})
318
+ [entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
319
+ self.assertEqual([w["table"] for w in entry["writes"]], ["dwd_txn_cnt", "{expr}"])
320
+ self.assertEqual(entry["columns"], ["cust_id", "txn_cnt"])
321
+
322
+ def test_partial_catalog_names_missing_columns_and_thin_definitions(self):
323
+ script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
324
+ 'PUBLISHED_COLUMNS = ("dt", "coll_amt_7d_usd", "payout_amt_7d_usd")')
325
+ script = script.replace('en="Snapshot date."', 'en="dt"').replace('zh="近 7 天收款金额。"', 'zh="同上,7 天窗口。"')
326
+ root = make_repo({"jobs/a.py": script}, BILINGUAL)
327
+ [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
328
+ findings = {f["kind"]: f["message"] for f in entry["findings"]}
329
+ self.assertEqual(findings["undocumented"], "1 column(s) with no Field: payout_amt_7d_usd")
330
+ self.assertEqual(findings["contract-not-derived"], "PUBLISHED_COLUMNS is a literal list; derive it from CATALOG_FIELDS")
331
+ self.assertEqual(findings["thin-definition"], "definition or source only restates the name: dt")
332
+ self.assertEqual(findings["relative-definition"], 'definition or source points at another row ("same as above"); '
333
+ "each row must stand alone: coll_amt_7d_usd")
334
+
335
+ def test_writing_rule_leads_flag_bloated_rows_and_spare_plain_ones(self):
336
+ shared = "dwd.transaction where status = 'POSTED' and cust_type <> 'TEST' and txn_date < dt"
337
+ rows = "\n".join(
338
+ f''' Field("{name}", en="{en}", zh="{zh}", source="{shared}, {agg}."),'''
339
+ for name, en, zh, agg in [
340
+ ("a_cnt", "Number of posted rows whose currency has a dim.forex rate on txn_date.", "近 7 天收款笔数。", "count rows"),
341
+ ("b_cnt", "Number of distinct payer banks.", "不同付款银行数。", "count distinct payer_bank"),
342
+ ("c_amt", " ".join(["word"] * 31), "近 7 天收款总额。", "sum of amount"),
343
+ ])
344
+ script = f'''
345
+ CATALOG_TABLE = Table("dwd.t", en="Table.", zh="表。", grain_en="One row per id.", grain_zh="每个 id 一行。")
346
+ CATALOG_FIELDS = (
347
+ {rows}
348
+ )
349
+ '''
350
+ root = make_repo({"jobs/t.py": script}, BILINGUAL)
351
+ [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
352
+ findings = {f["kind"]: f["message"] for f in entry["findings"]}
353
+ self.assertIn("appears in 3 of 3 rows", findings["repeated-text"])
354
+ self.assertTrue(findings["code-in-description"].endswith(": a_cnt"))
355
+ self.assertTrue(findings["long-definition"].endswith(": c_amt"))
356
+
357
+ def test_abbreviations_are_not_mistaken_for_table_names(self):
358
+ self.assertIsNone(dc.CODE_IN_TEXT_RE.search("Created between 0 a.m. and 6 a.m., e.g. at night."))
359
+ self.assertIsNotNone(dc.CODE_IN_TEXT_RE.search("Rate from dim.forex."))
360
+
361
+ def test_shared_vocabulary_and_plain_units_are_not_flagged(self):
362
+ script = '''
363
+ CATALOG_TABLE = Table("dwd.t", en="Table.", zh="表。", grain_en="One row per id.", grain_zh="每个 id 一行。",
364
+ notes_en=("All metrics count posted collections.", "Every metric is 0 if there were no collections."),
365
+ notes_zh=("所有指标只计已入账收款。", "若窗口内无收款,则指标为 0。"))
366
+ CATALOG_FIELDS = (
367
+ Field("a", en="Number of collections in the last 7 days.", zh="近 7 天收款笔数。",
368
+ source="count of dwd.transaction rows", window="[dt-7d, dt)"),
369
+ Field("b", en="Number of collections in the last 30 days.", zh="近 30 天收款笔数。",
370
+ source="count of dwd.transaction rows", window="[dt-30d, dt)"),
371
+ Field("c", en="Total collected in the last 7 days, in USD.", zh="近 7 天收款总额(USD)。",
372
+ source="sum of dwd.transaction.amount_usd", window="[dt-7d, dt)"),
373
+ )
374
+ PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
375
+ '''
376
+ root = make_repo({"jobs/t.py": script}, BILINGUAL)
377
+ [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
378
+ self.assertEqual(entry["findings"], [])
379
+
380
+ def test_documented_script_with_derived_contract_has_no_findings(self):
381
+ root = make_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
382
+ report = json.loads(run(root, "audit", "--json")[1])
383
+ self.assertEqual((report["tables"], report["documentedTables"]), (1, 1))
384
+ self.assertEqual(report["scripts"][0]["findings"], [])
385
+
386
+ def test_audit_without_config_or_globs_explains_what_to_pass(self):
387
+ root = make_repo({})
388
+ code, _, err = run(root, "audit")
389
+ self.assertEqual(code, 2)
390
+ self.assertIn("pass globs to audit", err)
391
+
392
+
393
+ class ReviewFixTest(unittest.TestCase):
394
+ def test_trailing_slash_in_output_neither_fails_check_nor_deletes_pages(self):
395
+ root = make_repo({"sql/merchant_monthly.sql": SQL_SCRIPT}, {**BILINGUAL, "output": "docs/catalog/"})
396
+ self.assertEqual(run(root, "generate")[0], 0)
397
+ code, _, err = run(root, "generate", "--check")
398
+ self.assertEqual(code, 0, err)
399
+ self.assertIn("0 written, 0 removed", run(root, "generate")[1])
400
+ self.assertTrue((root / "docs/catalog/ads.merchant_monthly.md").exists())
401
+
402
+ def test_plain_ctas_select_is_the_contract(self):
403
+ sql = SQL_SCRIPT.split("INSERT OVERWRITE")[0] + "CREATE TABLE ads.merchant_monthly AS SELECT merchant_id, volume_usd, extra FROM s;\n"
404
+ root = make_repo({"sql/m.sql": sql}, BILINGUAL)
405
+ code, _, err = run(root, "generate")
406
+ self.assertEqual(code, 1)
407
+ self.assertIn("outputs column(s) with no definition: extra", err)
408
+
409
+ def test_unaliased_expressions_never_fail_ci_on_a_guessed_name(self):
410
+ sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, s.price * s.qty, CAST(b AS INT) + 1")
411
+ root = make_repo({"sql/m.sql": sql}, BILINGUAL)
412
+ code, _, err = run(root, "generate")
413
+ self.assertEqual(code, 0, err)
414
+ self.assertIn("has `*` or unnamed items; only its named columns are checked", err)
415
+
416
+ def test_an_unrelated_statement_is_never_used_as_the_contract(self):
417
+ sql = SQL_SCRIPT.replace("SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd", "SELECT *") + \
418
+ "INSERT INTO tmp.stage SELECT zz FROM q;\n"
419
+ root = make_repo({"sql/m.sql": sql}, BILINGUAL)
420
+ code, _, err = run(root, "generate")
421
+ self.assertEqual(code, 0, err)
422
+ self.assertIn("no column contract and no readable SELECT writing ads.merchant_monthly", err)
423
+
424
+ def test_tables_mapping_to_one_page_or_the_index_are_errors(self):
425
+ a = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"ads.a_b",\n en=')
426
+ b = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"ads.a b",\n en=')
427
+ readme = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"README",\n en=')
428
+ root = make_repo({"jobs/a.py": a, "jobs/b.py": b, "jobs/c.py": readme}, BILINGUAL)
429
+ code, _, err = run(root, "generate")
430
+ self.assertEqual(code, 1)
431
+ self.assertIn("map to the same page ads.a_b.md", err)
432
+ self.assertIn("would overwrite the catalog index", err)
433
+
434
+ def test_exclude_globs_match_root_level_files_like_script_globs(self):
435
+ root = make_repo({"a_legacy.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT.replace("dwd.customer_daily", "dwd.other")},
436
+ {**BILINGUAL, "scripts": ["**/*.py"], "exclude": ["**/*_legacy.py"]})
437
+ self.assertEqual(run(root, "generate")[0], 0)
438
+ self.assertEqual(sorted(p.name for p in (root / "docs/catalog").iterdir()), ["README.md", "dwd.other.md"])
439
+
440
+ def test_same_short_name_in_another_schema_is_a_different_table(self):
441
+ ods = '''
442
+ TARGET_TABLE = "ods.customer_daily"
443
+ '''
444
+ root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/ods.py": ods}, BILINGUAL)
445
+ report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
446
+ self.assertEqual([f["kind"] for f in report["jobs/ods.py"]["findings"]], ["no-catalog"])
447
+
448
+ def test_audit_ignores_commented_writes_dynamic_names_and_directories(self):
449
+ script = PY_SCRIPT + '''
450
+ # df.write.insertInto("old.dropped_table")
451
+ def run(spark, db, tbl):
452
+ spark.sql(f"INSERT OVERWRITE TABLE {db}.{tbl} SELECT 1")
453
+ spark.sql("INSERT OVERWRITE LOCAL DIRECTORY '/tmp/x' SELECT 1")
454
+ '''
455
+ root = make_repo({"jobs/a.py": script}, BILINGUAL)
456
+ [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
457
+ self.assertEqual(sorted(w["table"] for w in entry["writes"]), ["dwd.customer_daily", "{expr}.{expr}"])
458
+ self.assertEqual(entry["findings"], [])
459
+
460
+ def test_python_syntax_error_is_one_finding(self):
461
+ root = make_repo({"jobs/a.py": "CATALOG_TABLE = (\n"}, BILINGUAL)
462
+ [entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
463
+ self.assertEqual([f["kind"] for f in entry["findings"]], ["invalid-metadata"])
464
+
465
+ def test_sql_block_ends_at_code_and_malformed_directives_fail(self):
466
+ late = SQL_SCRIPT + "-- @field ghost\n-- en: x\n"
467
+ root = make_repo({"sql/m.sql": late}, BILINGUAL)
468
+ self.assertEqual(run(root, "generate")[0], 0)
469
+ self.assertNotIn("ghost", (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8"))
470
+ bad = SQL_SCRIPT.replace("-- @table ads.merchant_monthly", "-- @table ads.merchant_monthly (monthly)")
471
+ root = make_repo({"sql/m.sql": bad}, BILINGUAL)
472
+ code, _, err = run(root, "generate")
473
+ self.assertEqual(code, 1)
474
+ self.assertIn("sql/m.sql:1: write `-- @table <name>`", err)
475
+
476
+ def test_backslash_escaped_quote_keeps_the_select_readable(self):
477
+ sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, 'it\\'s' AS note")
478
+ root = make_repo({"sql/m.sql": sql}, BILINGUAL)
479
+ code, _, err = run(root, "generate")
480
+ self.assertEqual(code, 1)
481
+ self.assertIn("outputs column(s) with no definition: note", err)
482
+
483
+ def test_init_with_invalid_json_config_exits_2(self):
484
+ root = make_repo({"jobs/a.py": "x = 1\n"})
485
+ (root / dc.CONFIG_FILE).write_text("{oops", encoding="utf-8")
486
+ code, _, err = run(root, "init", "--write")
487
+ self.assertEqual(code, 2)
488
+ self.assertIn("invalid JSON", err)
489
+
490
+
491
+ def git_repo(files: dict, config: dict) -> Path:
492
+ root = make_repo(files, config)
493
+ (root / ".git").rmdir()
494
+ for cmd in (["init", "-q"], ["add", "-A"], ["-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "base"]):
495
+ subprocess.run(["git", *cmd], cwd=root, check=True)
496
+ return root
497
+
498
+
499
+ def edit(root: Path, name: str, old: str, new: str) -> None:
500
+ path = root / name
501
+ text = path.read_text(encoding="utf-8")
502
+ assert old in text, old
503
+ path.write_text(text.replace(old, new), encoding="utf-8")
504
+
505
+
506
+ class DriftTest(unittest.TestCase):
507
+ def test_changed_expression_without_changed_definition_is_reported(self):
508
+ root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
509
+ self.assertEqual(run(root, "drift")[0], 0)
510
+ edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd * 0.9) AS volume_usd")
511
+ code, out, _ = run(root, "drift")
512
+ self.assertEqual(code, 1)
513
+ self.assertIn("sql/m.sql (ads.merchant_monthly)\n - volume_usd: its SELECT expression changed; its definition did not", out)
514
+ self.assertNotIn("filters, joins", out)
515
+ edit(root, "sql/m.sql", "Settled volume in the month.", "Settled volume net of the 10% fee.")
516
+ self.assertEqual(run(root, "drift")[0], 0)
517
+
518
+ def test_changed_filter_flags_the_table_until_a_definition_changes(self):
519
+ root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
520
+ edit(root, "sql/m.sql", "'SETTLED, final'", "'SETTLED'")
521
+ code, out, _ = run(root, "drift")
522
+ self.assertEqual(code, 1)
523
+ self.assertIn("filters, joins or CTEs changed; no definition changed", out)
524
+ edit(root, "sql/m.sql", "dwd_settlement settled rows,", "dwd_settlement rows with status SETTLED,")
525
+ self.assertEqual(run(root, "drift")[0], 0)
526
+
527
+ def test_dataframe_script_change_without_definition_change_is_reported(self):
528
+ script = PY_SCRIPT + "\ndef build(df):\n return df.filter('status = 1')\n"
529
+ root = git_repo({"jobs/a.py": script}, BILINGUAL)
530
+ edit(root, "jobs/a.py", "status = 1", "status = 2")
531
+ code, out, _ = run(root, "drift")
532
+ self.assertEqual(code, 1)
533
+ self.assertIn("code changed and no definition changed; the columns could not be read", out)
534
+
535
+ def test_definition_only_edit_and_untracked_new_script_are_quiet(self):
536
+ root = git_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
537
+ edit(root, "jobs/a.py", "Snapshot date.", "Snapshot date (UTC).")
538
+ (root / "jobs/new.py").write_text(textwrap.dedent(PY_SCRIPT).replace("dwd.customer_daily", "dwd.new"), encoding="utf-8")
539
+ self.assertEqual(run(root, "drift")[0], 0)
540
+
541
+ def test_base_ref_compares_a_whole_branch(self):
542
+ root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
543
+ edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "MAX(s.amount_usd) AS volume_usd")
544
+ subprocess.run(["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "change"], cwd=root, check=True)
545
+ self.assertEqual(run(root, "drift")[0], 0)
546
+ self.assertEqual(run(root, "drift", "--base", "HEAD~1")[0], 1)
547
+
548
+ def test_outside_git_drift_exits_2(self):
549
+ root = make_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
550
+ (root / ".git").rmdir()
551
+ code, _, err = run(root, "drift")
552
+ self.assertEqual(code, 2)
553
+ self.assertIn("is not a commit", err)
554
+
555
+
556
+ class InitTest(unittest.TestCase):
557
+ def test_dry_run_writes_nothing(self):
558
+ root = make_repo({"jobs/dwd/a.py": "x = 1\n"})
559
+ code, out, _ = run(root, "init", "--scripts", "jobs/**/*.py")
560
+ self.assertEqual(code, 0)
561
+ self.assertIn("would write data-catalog.json", out)
562
+ self.assertEqual(sorted(p.name for p in root.iterdir()), [".git", "jobs"])
563
+
564
+ def test_write_sets_up_repo_and_generated_module_accepts_documented_scripts(self):
565
+ root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "jobs/dwd/other.py": "x = 1\n",
566
+ "AGENTS.md": "# Agents\n\nExisting rules.\n"})
567
+ code, out, err = run(root, "init", "--scripts", "jobs/**/*.py", "--languages", "en,zh", "--write")
568
+ self.assertEqual(code, 0, err)
569
+ config = json.loads((root / dc.CONFIG_FILE).read_text(encoding="utf-8"))
570
+ self.assertEqual(config["fieldsModule"], "jobs/dwd/catalog_fields.py")
571
+ self.assertEqual((root / dc.ENGINE_PATH).read_text(encoding="utf-8"), (HERE / "data_catalog.py").read_text(encoding="utf-8"))
572
+ workflow = (root / dc.WORKFLOW_PATH).read_text(encoding="utf-8")
573
+ self.assertIn(" - 'jobs/**/*.py'\n", workflow)
574
+ self.assertIn("run: python3 scripts/data-catalog/data_catalog.py generate --check", workflow)
575
+ # agent pointers are persist-conventions.mjs's job; init leaves them alone
576
+ self.assertEqual((root / "AGENTS.md").read_text(encoding="utf-8"), "# Agents\n\nExisting rules.\n")
577
+ self.assertFalse((root / "CLAUDE.md").exists())
578
+ self.assertTrue((root / "docs/catalog/dwd.customer_daily.md").exists())
579
+ # the generated runtime module takes exactly the arguments the engine reads
580
+ spec = importlib.util.spec_from_file_location("catalog_fields", root / config["fieldsModule"])
581
+ module = importlib.util.module_from_spec(spec)
582
+ sys.modules["catalog_fields"] = module # dataclasses resolve string annotations through sys.modules
583
+ self.addCleanup(sys.modules.pop, "catalog_fields", None)
584
+ spec.loader.exec_module(module)
585
+ f = module.Field("x", en="e", zh="z", source="s", window="7d", unit="USD")
586
+ self.assertEqual((f.name, f.zh, f.unit), ("x", "z", "USD"))
587
+ with self.assertRaises(TypeError):
588
+ module.Field("x", en="e", source="s")
589
+ # rerunning is a no-op, and the catalog check passes
590
+ code, out, _ = run(root, "init", "--write")
591
+ self.assertEqual(code, 0)
592
+ self.assertNotIn("write ", out.replace("would write", ""))
593
+ self.assertEqual(run(root, "generate", "--check")[0], 0)
594
+
595
+ def test_bad_language_code_is_refused(self):
596
+ root = make_repo({"jobs/a.py": "x = 1\n"})
597
+ code, _, err = run(root, "init", "--scripts", "jobs/*.py", "--languages", "English", "--write")
598
+ self.assertEqual(code, 2)
599
+ self.assertIn("`languages` must be codes", err)
600
+
601
+
602
+ if __name__ == "__main__":
603
+ unittest.main()
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: ios-foundation
3
3
  description: "Set up and prove an Expo / React Native native iOS workspace from a clean checkout to a manual Login smoke on an owned iOS Simulator: isolated worktree and private caches, Xcode/runtime/CocoaPods preflight, frozen install with `pod install --deployment`, platform-scoped prebuild, an ad-hoc-signed Debug simulator build, a private simulator by exact UDID, Metro on a free port, a disposable PostgreSQL backend with owned cleanup, and the evidence record (setup proof only; feature work reports the smoke result). Use for 'set up iOS', 'build the iOS app', 'run the app on the iOS simulator', 'add iOS to the React Native app', 'prove the iOS build', or when a pod install, xcodebuild, simulator boot, Metro connection or Keychain session fails. NOT for Android (android-foundation), signing, TestFlight or App Store release, or general workflow YAML (github-actions)."
4
- release-version: 0.36.0
4
+ release-version: 0.38.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # iOS foundation
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: monorepo
3
3
  description: Turborepo architecture for shared packages, pnpm workspaces, DAL boundaries, and pipelines.
4
- release-version: 0.36.0
4
+ release-version: 0.38.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Turborepo Monorepo Architecture
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: nextjs-fullstack
3
3
  description: "Build or debug Next.js App Router features with React 19 — pages/layouts/route handlers, Server/Client Components, Server Actions, forms, middleware, DAL, strict TypeScript, Zod — plus our conventions: nuqs URL state, SWR cache, Skeleton/Progress + Sonner feedback, drag-and-drop upload, the portal app shell, list pages as filter/sort/paginate tables. shadcn component mechanics: see the shadcn skill."
4
- release-version: 0.36.0
4
+ release-version: 0.38.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Next.js 16 Fullstack Development
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: prisma-setup
3
3
  description: Use when adding or debugging Prisma with PostgreSQL, Next.js, or Turborepo, including schema.prisma, prisma.config.ts, client generation, migrations, seeds, relations, adapters, and DAL/database-layer setup. Triggers on phrases like "Prisma schema", "prisma migrate", "schema.prisma", "seed the database", "PrismaClient error", "relation issue", or "database layer".
4
- release-version: 0.36.0
4
+ release-version: 0.38.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Prisma v7 Setup Guide
@@ -12,7 +12,7 @@ description: >
12
12
  Connect, spark-submit, Hudi commits, backfills, validate-only canaries,
13
13
  cluster reconciliation, or Spark event-log profiling — including warehouse
14
14
  job timeouts, empty Hudi writes, or EMR/Dataproc performance regressions.
15
- release-version: 0.36.0
15
+ release-version: 0.38.0
16
16
  ---
17
17
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
18
18
  # PySpark