@1aboveio/app-dev-skills 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -2
- package/package.json +34 -4
- package/skills/airflow/LICENSE +3 -0
- package/skills/airflow/SKILL.md +199 -0
- package/skills/airflow/release-check.mjs +93 -0
- package/skills/android-foundation/LICENSE +3 -0
- package/skills/android-foundation/SKILL.md +192 -0
- package/skills/android-foundation/assets/android-ci-job.yml +217 -0
- package/skills/android-foundation/references/ci.md +87 -0
- package/skills/android-foundation/references/gotchas.md +221 -0
- package/skills/android-foundation/references/setup.md +164 -0
- package/skills/android-foundation/release-check.mjs +93 -0
- package/skills/android-foundation/scripts/device-run.sh +176 -0
- package/skills/android-foundation/scripts/device-run.test.mjs +172 -0
- package/skills/app-debug/LICENSE +3 -0
- package/skills/app-debug/SKILL.md +110 -0
- package/skills/app-debug/references/common-errors.md +128 -0
- package/skills/app-debug/release-check.mjs +93 -0
- package/skills/better-auth/LICENSE +3 -0
- package/skills/better-auth/SKILL.md +598 -0
- package/skills/better-auth/references/feishu-api.md +270 -0
- package/skills/better-auth/release-check.mjs +93 -0
- package/skills/data-catalog/LICENSE +3 -0
- package/skills/data-catalog/SKILL.md +207 -0
- package/skills/data-catalog/agents/openai.yaml +3 -0
- package/skills/data-catalog/references/data-catalog-conventions.md +76 -0
- package/skills/data-catalog/references/field-definitions.md +123 -0
- package/skills/data-catalog/references/format.md +130 -0
- package/skills/data-catalog/release-check.mjs +93 -0
- package/skills/data-catalog/scripts/data-catalog.test.mjs +36 -0
- package/skills/data-catalog/scripts/data_catalog.py +1435 -0
- package/skills/data-catalog/scripts/persist-conventions-core.mjs +117 -0
- package/skills/data-catalog/scripts/persist-conventions.mjs +17 -0
- package/skills/data-catalog/scripts/test_data_catalog.py +603 -0
- package/skills/ios-foundation/LICENSE +3 -0
- package/skills/ios-foundation/SKILL.md +184 -0
- package/skills/ios-foundation/references/setup.md +243 -0
- package/skills/ios-foundation/release-check.mjs +93 -0
- package/skills/monorepo/LICENSE +3 -0
- package/skills/monorepo/SKILL.md +462 -0
- package/skills/monorepo/release-check.mjs +93 -0
- package/skills/nextjs-fullstack/LICENSE +3 -0
- package/skills/nextjs-fullstack/SKILL.md +211 -0
- package/skills/nextjs-fullstack/conventions.md +318 -0
- package/skills/nextjs-fullstack/frontend-conventions.md +61 -0
- package/skills/nextjs-fullstack/nextjs16.md +287 -0
- package/skills/nextjs-fullstack/release-check.mjs +93 -0
- package/skills/nextjs-fullstack/server-actions.md +409 -0
- package/skills/prisma-setup/LICENSE +3 -0
- package/skills/prisma-setup/SKILL.md +181 -0
- package/skills/prisma-setup/nextjs.md +258 -0
- package/skills/prisma-setup/release-check.mjs +93 -0
- package/skills/prisma-setup/turborepo.md +301 -0
- package/skills/pyspark/LICENSE +3 -0
- package/skills/pyspark/SKILL.md +127 -0
- package/skills/pyspark/assets/templates/etl.py +748 -0
- package/skills/pyspark/assets/templates/utils/__init__.py +1 -0
- package/skills/pyspark/assets/templates/utils/hudi_metadata.py +14 -0
- package/skills/pyspark/references/diagnosis-and-profiling.md +169 -0
- package/skills/pyspark/references/etl-contract.md +129 -0
- package/skills/pyspark/references/production-validation.md +131 -0
- package/skills/pyspark/references/reconciliation.md +38 -0
- package/skills/pyspark/references/transformation-design.md +190 -0
- package/skills/pyspark/references/velocity-feature-calculation.md +193 -0
- package/skills/pyspark/release-check.mjs +93 -0
- package/skills/pyspark/scripts/spark_eventlog_summary.py +223 -0
- package/skills/python-backend/LICENSE +3 -0
- package/skills/python-backend/SKILL.md +325 -0
- package/skills/python-backend/assets/uv-install.sh +48 -0
- package/skills/python-backend/references/uv-installs.md +83 -0
- package/skills/python-backend/release-check.mjs +93 -0
- package/skills/shadcn/LICENSE +3 -0
- package/skills/shadcn/SKILL.md +120 -0
- package/skills/shadcn/assets/shadcn-small.png +0 -0
- package/skills/shadcn/assets/shadcn.png +0 -0
- package/skills/shadcn/cli.md +411 -0
- package/skills/shadcn/customization.md +224 -0
- package/skills/shadcn/mcp.md +101 -0
- package/skills/shadcn/release-check.mjs +93 -0
- package/skills/shadcn/rules/base-vs-radix.md +323 -0
- package/skills/shadcn/rules/component-selection.md +67 -0
- package/skills/shadcn/rules/composition.md +195 -0
- package/skills/shadcn/rules/data-table.md +201 -0
- package/skills/shadcn/rules/forms.md +255 -0
- package/skills/shadcn/rules/icons.md +103 -0
- package/skills/shadcn/rules/styling.md +167 -0
- package/skills/zod-v4/LICENSE +3 -0
- package/skills/zod-v4/SKILL.md +286 -0
- package/skills/zod-v4/release-check.mjs +93 -0
|
@@ -0,0 +1,603 @@
|
|
|
1
|
+
"""Tests for data_catalog.py. Run: python3 test_data_catalog.py"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import contextlib
|
|
5
|
+
import importlib.util
|
|
6
|
+
import subprocess
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
import textwrap
|
|
12
|
+
import unittest
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
HERE = Path(__file__).resolve().parent
|
|
16
|
+
sys.path.insert(0, str(HERE))
|
|
17
|
+
import data_catalog as dc # noqa: E402
|
|
18
|
+
|
|
19
|
+
PY_SCRIPT = '''
|
|
20
|
+
"""Build dwd.customer_daily."""
|
|
21
|
+
from catalog_fields import Field, Table
|
|
22
|
+
|
|
23
|
+
TARGET_TABLE = "dwd.customer_daily"
|
|
24
|
+
WINDOW_NOTE = "[dt-7d, dt)"
|
|
25
|
+
|
|
26
|
+
CATALOG_TABLE = Table(
|
|
27
|
+
"dwd.customer_daily",
|
|
28
|
+
en="Daily customer collection rollup.",
|
|
29
|
+
zh="客户每日收款汇总。",
|
|
30
|
+
grain_en="one row per (dt, cust_id)",
|
|
31
|
+
grain_zh="每个 (dt, cust_id) 一行",
|
|
32
|
+
notes_en=("Amounts in USD via dim_forex.",),
|
|
33
|
+
notes_zh=("金额经 dim_forex 换算为 USD。",),
|
|
34
|
+
)
|
|
35
|
+
KEYS = (
|
|
36
|
+
Field("dt", en="Snapshot date.", zh="快照日期。", source="Run parameter `dt`."),
|
|
37
|
+
)
|
|
38
|
+
CATALOG_FIELDS = (
|
|
39
|
+
*KEYS,
|
|
40
|
+
Field(
|
|
41
|
+
"coll_amt_7d_usd",
|
|
42
|
+
en="Collection amount | last 7 days.",
|
|
43
|
+
zh="近 7 天收款金额。",
|
|
44
|
+
source="dwd_transaction POSTED collections, "
|
|
45
|
+
"sum of txn_amount_usd.",
|
|
46
|
+
window=WINDOW_NOTE,
|
|
47
|
+
unit="USD",
|
|
48
|
+
),
|
|
49
|
+
)
|
|
50
|
+
PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
|
|
51
|
+
'''
|
|
52
|
+
|
|
53
|
+
SQL_SCRIPT = """\
|
|
54
|
+
-- @table ads.merchant_monthly
|
|
55
|
+
-- en: Monthly merchant volume.
|
|
56
|
+
-- zh: 商户月度交易量。
|
|
57
|
+
-- grain_en: one row per (month, merchant_id)
|
|
58
|
+
-- grain_zh: 每个 (month, merchant_id) 一行
|
|
59
|
+
-- notes_en: Built monthly; Excludes test merchants
|
|
60
|
+
-- notes_zh: 每月构建;不含测试商户
|
|
61
|
+
-- @field merchant_id
|
|
62
|
+
-- en: Merchant id.
|
|
63
|
+
-- zh: 商户 ID。
|
|
64
|
+
-- source: dim_merchant.merchant_id
|
|
65
|
+
-- @field volume_usd
|
|
66
|
+
-- en: Settled volume in the month.
|
|
67
|
+
-- zh: 当月结算金额。
|
|
68
|
+
-- source: dwd_settlement settled rows,
|
|
69
|
+
-- summed amount_usd
|
|
70
|
+
-- unit: USD
|
|
71
|
+
|
|
72
|
+
INSERT OVERWRITE TABLE ads.merchant_monthly
|
|
73
|
+
WITH s AS (SELECT merchant_id, amount_usd FROM dwd_settlement WHERE status = 'SETTLED, final')
|
|
74
|
+
SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd
|
|
75
|
+
FROM s GROUP BY s.merchant_id;
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def make_repo(files: dict, config: dict | None = None) -> Path:
|
|
80
|
+
root = Path(tempfile.mkdtemp(prefix="data-catalog-test-"))
|
|
81
|
+
(root / ".git").mkdir()
|
|
82
|
+
for name, text in files.items():
|
|
83
|
+
path = root / name
|
|
84
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
path.write_text(textwrap.dedent(text) if name.endswith(".py") else text, encoding="utf-8")
|
|
86
|
+
if config is not None:
|
|
87
|
+
(root / dc.CONFIG_FILE).write_text(json.dumps({**{"version": 1, "output": "docs/catalog"}, **config}), encoding="utf-8")
|
|
88
|
+
return root
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def run(root: Path, *argv: str) -> tuple[int, str, str]:
|
|
92
|
+
out, err = io.StringIO(), io.StringIO()
|
|
93
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
94
|
+
code = dc.main(["--root", str(root), *argv])
|
|
95
|
+
return code, out.getvalue(), err.getvalue()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
BILINGUAL = {"scripts": ["jobs/**/*.py", "sql/**/*.sql"], "languages": ["en", "zh"]}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class GenerateTest(unittest.TestCase):
|
|
102
|
+
def test_python_and_sql_metadata_render_pages_and_index(self):
|
|
103
|
+
root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "sql/merchant_monthly.sql": SQL_SCRIPT}, BILINGUAL)
|
|
104
|
+
code, out, err = run(root, "generate")
|
|
105
|
+
self.assertEqual(code, 0, err)
|
|
106
|
+
page = (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8")
|
|
107
|
+
self.assertTrue(page.startswith(dc.GENERATED_MARK))
|
|
108
|
+
self.assertIn("- **Script:** [jobs/dwd/customer_daily.py](../../jobs/dwd/customer_daily.py)", page)
|
|
109
|
+
self.assertIn("| Field | English | 中文 | Source | Window | Unit |", page)
|
|
110
|
+
# starred KEYS come first, `|` is escaped, implicit concatenation and module constants resolve
|
|
111
|
+
self.assertIn("| `dt` | Snapshot date. | 快照日期。 | Run parameter `dt`. | | |", page)
|
|
112
|
+
self.assertIn("| `coll_amt_7d_usd` | Collection amount \\| last 7 days. | 近 7 天收款金额。 | "
|
|
113
|
+
"dwd_transaction POSTED collections, sum of txn_amount_usd. | [dt-7d, dt) | USD |", page)
|
|
114
|
+
sql_page = (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8")
|
|
115
|
+
self.assertIn("| `volume_usd` | Settled volume in the month. | 当月结算金额。 | "
|
|
116
|
+
"dwd_settlement settled rows, summed amount_usd | USD |", sql_page)
|
|
117
|
+
self.assertNotIn("Window", sql_page)
|
|
118
|
+
self.assertIn(" - Excludes test merchants", sql_page)
|
|
119
|
+
self.assertIn("## 中文", sql_page)
|
|
120
|
+
self.assertIn("- **粒度:** 每个 (month, merchant_id) 一行", sql_page)
|
|
121
|
+
self.assertIn(" - 不含测试商户", sql_page)
|
|
122
|
+
index = (root / "docs/catalog/README.md").read_text(encoding="utf-8")
|
|
123
|
+
self.assertLess(index.index("ads.merchant_monthly"), index.index("dwd.customer_daily"))
|
|
124
|
+
self.assertIn("| [`dwd.customer_daily`](dwd.customer_daily.md) | Daily customer collection rollup. | "
|
|
125
|
+
"one row per (dt, cust_id) | 2 |", index)
|
|
126
|
+
|
|
127
|
+
def test_check_passes_when_current_and_fails_after_a_definition_changes(self):
|
|
128
|
+
root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
|
|
129
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
130
|
+
self.assertEqual(run(root, "generate", "--check")[0], 0)
|
|
131
|
+
script = root / "jobs/dwd/customer_daily.py"
|
|
132
|
+
script.write_text(script.read_text(encoding="utf-8").replace("Snapshot date.", "Partition date."), encoding="utf-8")
|
|
133
|
+
code, _, err = run(root, "generate", "--check")
|
|
134
|
+
self.assertEqual(code, 1)
|
|
135
|
+
self.assertIn("stale: docs/catalog/dwd.customer_daily.md", err)
|
|
136
|
+
self.assertNotIn("Partition date.", (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8"))
|
|
137
|
+
|
|
138
|
+
def test_removing_metadata_makes_the_page_an_orphan_that_generate_removes(self):
|
|
139
|
+
root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
|
|
140
|
+
run(root, "generate")
|
|
141
|
+
(root / "docs/catalog/hand-written.md").write_text("# kept\n", encoding="utf-8")
|
|
142
|
+
(root / "jobs/dwd/customer_daily.py").write_text("x = 1\n", encoding="utf-8")
|
|
143
|
+
code, _, err = run(root, "generate", "--check")
|
|
144
|
+
self.assertEqual(code, 1)
|
|
145
|
+
self.assertIn("orphan: docs/catalog/dwd.customer_daily.md", err)
|
|
146
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
147
|
+
self.assertFalse((root / "docs/catalog/dwd.customer_daily.md").exists())
|
|
148
|
+
self.assertTrue((root / "docs/catalog/hand-written.md").exists())
|
|
149
|
+
|
|
150
|
+
def test_missing_language_text_fails_with_location_and_writes_nothing(self):
|
|
151
|
+
script = PY_SCRIPT.replace('zh="近 7 天收款金额。",', "")
|
|
152
|
+
root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
|
|
153
|
+
code, _, err = run(root, "generate")
|
|
154
|
+
self.assertEqual(code, 1)
|
|
155
|
+
self.assertRegex(err, r"jobs/dwd/customer_daily\.py:\d+: Field 'coll_amt_7d_usd' is missing `zh`")
|
|
156
|
+
self.assertFalse((root / "docs/catalog").exists())
|
|
157
|
+
|
|
158
|
+
def test_literal_contract_missing_a_field_is_an_error_and_extra_fields_only_warn(self):
|
|
159
|
+
script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
|
|
160
|
+
'PUBLISHED_COLUMNS = ("coll_amt_7d_usd", "payout_amt_7d_usd")')
|
|
161
|
+
root = make_repo({"jobs/dwd/customer_daily.py": script}, BILINGUAL)
|
|
162
|
+
code, _, err = run(root, "generate")
|
|
163
|
+
self.assertEqual(code, 1)
|
|
164
|
+
self.assertIn("PUBLISHED_COLUMNS has column(s) with no Field: payout_amt_7d_usd", err)
|
|
165
|
+
self.assertIn("Field(s) not in PUBLISHED_COLUMNS: dt", err)
|
|
166
|
+
|
|
167
|
+
def test_sql_select_is_the_contract_for_its_comment_block(self):
|
|
168
|
+
added = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, COUNT(*) AS txn_cnt")
|
|
169
|
+
root = make_repo({"sql/merchant_monthly.sql": added}, BILINGUAL)
|
|
170
|
+
code, _, err = run(root, "generate", "--check")
|
|
171
|
+
self.assertEqual(code, 1)
|
|
172
|
+
self.assertIn("sql/merchant_monthly.sql:1: the SELECT writing ads.merchant_monthly outputs column(s) "
|
|
173
|
+
"with no definition: txn_cnt", err)
|
|
174
|
+
dropped = SQL_SCRIPT.replace(", SUM(s.amount_usd) AS volume_usd", "")
|
|
175
|
+
root = make_repo({"sql/merchant_monthly.sql": dropped}, BILINGUAL)
|
|
176
|
+
code, _, err = run(root, "generate")
|
|
177
|
+
self.assertEqual(code, 1)
|
|
178
|
+
self.assertIn("definition(s) for column(s) the SELECT does not output: volume_usd", err)
|
|
179
|
+
|
|
180
|
+
def test_sql_inside_a_documented_python_script_without_a_contract_is_checked(self):
|
|
181
|
+
script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)", '''
|
|
182
|
+
SQL = """
|
|
183
|
+
INSERT OVERWRITE TABLE dwd.customer_daily
|
|
184
|
+
SELECT dt, SUM(amt) AS coll_amt_7d_usd, COUNT(*) AS coll_cnt_7d FROM dwd.txn GROUP BY dt
|
|
185
|
+
"""
|
|
186
|
+
''')
|
|
187
|
+
root = make_repo({"jobs/a.py": script}, BILINGUAL)
|
|
188
|
+
code, _, err = run(root, "generate")
|
|
189
|
+
self.assertEqual(code, 1)
|
|
190
|
+
self.assertIn("outputs column(s) with no definition: coll_cnt_7d", err)
|
|
191
|
+
|
|
192
|
+
def test_expression_metadata_and_imported_field_lists_are_rejected(self):
|
|
193
|
+
expr = PY_SCRIPT.replace('en="Snapshot date."', 'en=f"Snapshot {1}"')
|
|
194
|
+
root = make_repo({"jobs/a.py": expr}, BILINGUAL)
|
|
195
|
+
code, _, err = run(root, "generate")
|
|
196
|
+
self.assertEqual(code, 1)
|
|
197
|
+
self.assertIn("metadata must be a string literal", err)
|
|
198
|
+
imported = PY_SCRIPT.replace(" *KEYS,\n", " *other.KEYS,\n")
|
|
199
|
+
root = make_repo({"jobs/a.py": imported}, BILINGUAL)
|
|
200
|
+
self.assertIn("imported lists cannot be read", run(root, "generate")[2])
|
|
201
|
+
|
|
202
|
+
def test_table_grain_and_notes_are_per_language_with_matching_notes(self):
|
|
203
|
+
old = PY_SCRIPT.replace('grain_en="one row per (dt, cust_id)",', 'grain="one row per (dt, cust_id)",')
|
|
204
|
+
root = make_repo({"jobs/a.py": old}, BILINGUAL)
|
|
205
|
+
code, _, err = run(root, "generate")
|
|
206
|
+
self.assertEqual(code, 1)
|
|
207
|
+
self.assertIn("grain and notes are written per language: grain_en=..., notes_en=(...), grain_zh=...", err)
|
|
208
|
+
uneven = PY_SCRIPT.replace('notes_zh=("金额经 dim_forex 换算为 USD。",),', 'notes_zh=(),')
|
|
209
|
+
root = make_repo({"jobs/a.py": uneven}, BILINGUAL)
|
|
210
|
+
code, _, err = run(root, "generate")
|
|
211
|
+
self.assertEqual(code, 1)
|
|
212
|
+
self.assertIn("every language needs the same notes, got {'en': 1, 'zh': 0}", err)
|
|
213
|
+
|
|
214
|
+
def test_same_table_in_two_scripts_is_an_error(self):
|
|
215
|
+
root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT}, BILINGUAL)
|
|
216
|
+
code, _, err = run(root, "generate")
|
|
217
|
+
self.assertEqual(code, 1)
|
|
218
|
+
self.assertIn("table 'dwd.customer_daily' is also documented in jobs/a.py", err)
|
|
219
|
+
|
|
220
|
+
def test_hidden_directories_are_not_scanned(self):
|
|
221
|
+
root = make_repo({"jobs/a.py": PY_SCRIPT, ".worktree/jobs/a.py": PY_SCRIPT},
|
|
222
|
+
{**BILINGUAL, "scripts": ["**/jobs/*.py"]})
|
|
223
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class AuditTest(unittest.TestCase):
|
|
227
|
+
def test_reports_undocumented_writers_with_their_columns(self):
|
|
228
|
+
legacy = '''
|
|
229
|
+
TARGET_TABLE = "dwd.legacy"
|
|
230
|
+
PUBLISHED_COLUMNS = ("dt", "cust_id", "amt")
|
|
231
|
+
'''
|
|
232
|
+
root = make_repo({"jobs/legacy.py": legacy, "jobs/helper.py": "def f():\n return 1\n"})
|
|
233
|
+
code, out, _ = run(root, "audit", "--json", "jobs/*.py")
|
|
234
|
+
self.assertEqual(code, 0)
|
|
235
|
+
report = json.loads(out)
|
|
236
|
+
self.assertEqual((report["scanned"], report["writers"], report["tables"], report["documentedTables"]), (2, 1, 1, 0))
|
|
237
|
+
[entry] = report["scripts"]
|
|
238
|
+
self.assertEqual(entry["writes"], [{"table": "dwd.legacy", "evidence": "TARGET_TABLE"}])
|
|
239
|
+
self.assertEqual(entry["columns"], ["dt", "cust_id", "amt"])
|
|
240
|
+
kinds = [f["kind"] for f in entry["findings"]]
|
|
241
|
+
self.assertEqual(kinds, ["no-catalog", "undocumented"])
|
|
242
|
+
|
|
243
|
+
def test_a_table_documented_in_its_builder_does_not_flag_backfill_scripts(self):
|
|
244
|
+
backfill = '''
|
|
245
|
+
"""Backfill: INSERT INTO docs.only_in_docstring SELECT 1"""
|
|
246
|
+
TARGET_TABLE = "dwd.customer_daily"
|
|
247
|
+
'''
|
|
248
|
+
twin = '''
|
|
249
|
+
TARGET_TABLE = "dwd.undocumented"
|
|
250
|
+
'''
|
|
251
|
+
root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/backfill_a.py": backfill,
|
|
252
|
+
"jobs/b.py": twin, "jobs/backfill_b.py": twin}, BILINGUAL)
|
|
253
|
+
report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
|
|
254
|
+
self.assertEqual(report["jobs/backfill_a.py"]["findings"], [])
|
|
255
|
+
self.assertEqual([w["table"] for w in report["jobs/backfill_a.py"]["writes"]], ["dwd.customer_daily"])
|
|
256
|
+
self.assertEqual(report["jobs/b.py"]["findings"][0]["message"],
|
|
257
|
+
"writes dwd.undocumented but carries no CATALOG_TABLE/CATALOG_FIELDS (or -- @table block); "
|
|
258
|
+
"also written by jobs/backfill_b.py (put the metadata in the script that builds the table)")
|
|
259
|
+
|
|
260
|
+
def test_a_structtype_contract_is_read_for_its_column_names(self):
|
|
261
|
+
script = '''
|
|
262
|
+
L1A_PUBLISHED_SCHEMA = StructType([
|
|
263
|
+
StructField("dt", DateType(), False),
|
|
264
|
+
StructField("cust_id", StringType(), False),
|
|
265
|
+
])
|
|
266
|
+
TARGET_TABLE = "dwd.flow"
|
|
267
|
+
'''
|
|
268
|
+
root = make_repo({"jobs/flow.py": script})
|
|
269
|
+
[entry] = json.loads(run(root, "audit", "--json", "--contract", "L1A_PUBLISHED_SCHEMA", "jobs/*.py")[1])["scripts"]
|
|
270
|
+
self.assertEqual((entry["columns"], entry["columnsFrom"]), (["dt", "cust_id"], "L1A_PUBLISHED_SCHEMA"))
|
|
271
|
+
|
|
272
|
+
def test_class_attributes_name_the_table_with_their_schema(self):
|
|
273
|
+
script = '''
|
|
274
|
+
class OrdersEtl(Etl):
|
|
275
|
+
dst_db = "dwd"
|
|
276
|
+
dst_tbl = "orders"
|
|
277
|
+
|
|
278
|
+
class Other:
|
|
279
|
+
out_tbl = "scratch"
|
|
280
|
+
out_db = "tmp"
|
|
281
|
+
'''
|
|
282
|
+
root = make_repo({"jobs/orders.py": script})
|
|
283
|
+
[entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
|
|
284
|
+
self.assertEqual(entry["writes"], [{"table": "dwd.orders", "evidence": "dst_tbl"}])
|
|
285
|
+
[entry] = json.loads(run(root, "audit", "--json", "--table-name", "out_tbl", "--schema-name", "out_db",
|
|
286
|
+
"jobs/*.py")[1])["scripts"]
|
|
287
|
+
self.assertEqual([w["table"] for w in entry["writes"]], ["dwd.orders", "tmp.scratch"])
|
|
288
|
+
|
|
289
|
+
def test_sql_select_list_columns_ignore_cte_columns_comments_and_strings(self):
|
|
290
|
+
sql = """\
|
|
291
|
+
-- INSERT INTO not.this
|
|
292
|
+
INSERT OVERWRITE TABLE ads.daily PARTITION (dt)
|
|
293
|
+
WITH base AS (SELECT a, b AS inner_only FROM src)
|
|
294
|
+
SELECT DISTINCT base.a,
|
|
295
|
+
CASE WHEN b > 0 THEN 'x, y' ELSE 'z' END AS flag,
|
|
296
|
+
sum(b) total,
|
|
297
|
+
coalesce(c, 0),
|
|
298
|
+
t.*
|
|
299
|
+
FROM base JOIN t ON t.a = base.a;
|
|
300
|
+
"""
|
|
301
|
+
root = make_repo({"sql/daily.sql": textwrap.dedent(sql)})
|
|
302
|
+
report = json.loads(run(root, "audit", "--json", "sql/*.sql")[1])
|
|
303
|
+
[entry] = report["scripts"]
|
|
304
|
+
self.assertEqual([w["table"] for w in entry["writes"]], ["ads.daily"])
|
|
305
|
+
self.assertEqual(entry["columns"], ["a", "flag", "total"])
|
|
306
|
+
self.assertFalse(entry["columnsComplete"])
|
|
307
|
+
|
|
308
|
+
def test_sql_inside_python_strings_is_read(self):
|
|
309
|
+
script = '''
|
|
310
|
+
def run(spark, target):
|
|
311
|
+
spark.sql(f"""
|
|
312
|
+
INSERT INTO TABLE {target}
|
|
313
|
+
SELECT cust_id, count(*) AS txn_cnt FROM dwd.txn GROUP BY cust_id
|
|
314
|
+
""")
|
|
315
|
+
df.write.format("hudi").option("hoodie.table.name", "dwd_txn_cnt").save(path)
|
|
316
|
+
'''
|
|
317
|
+
root = make_repo({"jobs/cnt.py": script})
|
|
318
|
+
[entry] = json.loads(run(root, "audit", "--json", "jobs/*.py")[1])["scripts"]
|
|
319
|
+
self.assertEqual([w["table"] for w in entry["writes"]], ["dwd_txn_cnt", "{expr}"])
|
|
320
|
+
self.assertEqual(entry["columns"], ["cust_id", "txn_cnt"])
|
|
321
|
+
|
|
322
|
+
def test_partial_catalog_names_missing_columns_and_thin_definitions(self):
|
|
323
|
+
script = PY_SCRIPT.replace("PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)",
|
|
324
|
+
'PUBLISHED_COLUMNS = ("dt", "coll_amt_7d_usd", "payout_amt_7d_usd")')
|
|
325
|
+
script = script.replace('en="Snapshot date."', 'en="dt"').replace('zh="近 7 天收款金额。"', 'zh="同上,7 天窗口。"')
|
|
326
|
+
root = make_repo({"jobs/a.py": script}, BILINGUAL)
|
|
327
|
+
[entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
|
|
328
|
+
findings = {f["kind"]: f["message"] for f in entry["findings"]}
|
|
329
|
+
self.assertEqual(findings["undocumented"], "1 column(s) with no Field: payout_amt_7d_usd")
|
|
330
|
+
self.assertEqual(findings["contract-not-derived"], "PUBLISHED_COLUMNS is a literal list; derive it from CATALOG_FIELDS")
|
|
331
|
+
self.assertEqual(findings["thin-definition"], "definition or source only restates the name: dt")
|
|
332
|
+
self.assertEqual(findings["relative-definition"], 'definition or source points at another row ("same as above"); '
|
|
333
|
+
"each row must stand alone: coll_amt_7d_usd")
|
|
334
|
+
|
|
335
|
+
def test_writing_rule_leads_flag_bloated_rows_and_spare_plain_ones(self):
|
|
336
|
+
shared = "dwd.transaction where status = 'POSTED' and cust_type <> 'TEST' and txn_date < dt"
|
|
337
|
+
rows = "\n".join(
|
|
338
|
+
f''' Field("{name}", en="{en}", zh="{zh}", source="{shared}, {agg}."),'''
|
|
339
|
+
for name, en, zh, agg in [
|
|
340
|
+
("a_cnt", "Number of posted rows whose currency has a dim.forex rate on txn_date.", "近 7 天收款笔数。", "count rows"),
|
|
341
|
+
("b_cnt", "Number of distinct payer banks.", "不同付款银行数。", "count distinct payer_bank"),
|
|
342
|
+
("c_amt", " ".join(["word"] * 31), "近 7 天收款总额。", "sum of amount"),
|
|
343
|
+
])
|
|
344
|
+
script = f'''
|
|
345
|
+
CATALOG_TABLE = Table("dwd.t", en="Table.", zh="表。", grain_en="One row per id.", grain_zh="每个 id 一行。")
|
|
346
|
+
CATALOG_FIELDS = (
|
|
347
|
+
{rows}
|
|
348
|
+
)
|
|
349
|
+
'''
|
|
350
|
+
root = make_repo({"jobs/t.py": script}, BILINGUAL)
|
|
351
|
+
[entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
|
|
352
|
+
findings = {f["kind"]: f["message"] for f in entry["findings"]}
|
|
353
|
+
self.assertIn("appears in 3 of 3 rows", findings["repeated-text"])
|
|
354
|
+
self.assertTrue(findings["code-in-description"].endswith(": a_cnt"))
|
|
355
|
+
self.assertTrue(findings["long-definition"].endswith(": c_amt"))
|
|
356
|
+
|
|
357
|
+
def test_abbreviations_are_not_mistaken_for_table_names(self):
|
|
358
|
+
self.assertIsNone(dc.CODE_IN_TEXT_RE.search("Created between 0 a.m. and 6 a.m., e.g. at night."))
|
|
359
|
+
self.assertIsNotNone(dc.CODE_IN_TEXT_RE.search("Rate from dim.forex."))
|
|
360
|
+
|
|
361
|
+
def test_shared_vocabulary_and_plain_units_are_not_flagged(self):
|
|
362
|
+
script = '''
|
|
363
|
+
CATALOG_TABLE = Table("dwd.t", en="Table.", zh="表。", grain_en="One row per id.", grain_zh="每个 id 一行。",
|
|
364
|
+
notes_en=("All metrics count posted collections.", "Every metric is 0 if there were no collections."),
|
|
365
|
+
notes_zh=("所有指标只计已入账收款。", "若窗口内无收款,则指标为 0。"))
|
|
366
|
+
CATALOG_FIELDS = (
|
|
367
|
+
Field("a", en="Number of collections in the last 7 days.", zh="近 7 天收款笔数。",
|
|
368
|
+
source="count of dwd.transaction rows", window="[dt-7d, dt)"),
|
|
369
|
+
Field("b", en="Number of collections in the last 30 days.", zh="近 30 天收款笔数。",
|
|
370
|
+
source="count of dwd.transaction rows", window="[dt-30d, dt)"),
|
|
371
|
+
Field("c", en="Total collected in the last 7 days, in USD.", zh="近 7 天收款总额(USD)。",
|
|
372
|
+
source="sum of dwd.transaction.amount_usd", window="[dt-7d, dt)"),
|
|
373
|
+
)
|
|
374
|
+
PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
|
|
375
|
+
'''
|
|
376
|
+
root = make_repo({"jobs/t.py": script}, BILINGUAL)
|
|
377
|
+
[entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
|
|
378
|
+
self.assertEqual(entry["findings"], [])
|
|
379
|
+
|
|
380
|
+
def test_documented_script_with_derived_contract_has_no_findings(self):
|
|
381
|
+
root = make_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
|
|
382
|
+
report = json.loads(run(root, "audit", "--json")[1])
|
|
383
|
+
self.assertEqual((report["tables"], report["documentedTables"]), (1, 1))
|
|
384
|
+
self.assertEqual(report["scripts"][0]["findings"], [])
|
|
385
|
+
|
|
386
|
+
def test_audit_without_config_or_globs_explains_what_to_pass(self):
|
|
387
|
+
root = make_repo({})
|
|
388
|
+
code, _, err = run(root, "audit")
|
|
389
|
+
self.assertEqual(code, 2)
|
|
390
|
+
self.assertIn("pass globs to audit", err)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class ReviewFixTest(unittest.TestCase):
|
|
394
|
+
def test_trailing_slash_in_output_neither_fails_check_nor_deletes_pages(self):
|
|
395
|
+
root = make_repo({"sql/merchant_monthly.sql": SQL_SCRIPT}, {**BILINGUAL, "output": "docs/catalog/"})
|
|
396
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
397
|
+
code, _, err = run(root, "generate", "--check")
|
|
398
|
+
self.assertEqual(code, 0, err)
|
|
399
|
+
self.assertIn("0 written, 0 removed", run(root, "generate")[1])
|
|
400
|
+
self.assertTrue((root / "docs/catalog/ads.merchant_monthly.md").exists())
|
|
401
|
+
|
|
402
|
+
def test_plain_ctas_select_is_the_contract(self):
|
|
403
|
+
sql = SQL_SCRIPT.split("INSERT OVERWRITE")[0] + "CREATE TABLE ads.merchant_monthly AS SELECT merchant_id, volume_usd, extra FROM s;\n"
|
|
404
|
+
root = make_repo({"sql/m.sql": sql}, BILINGUAL)
|
|
405
|
+
code, _, err = run(root, "generate")
|
|
406
|
+
self.assertEqual(code, 1)
|
|
407
|
+
self.assertIn("outputs column(s) with no definition: extra", err)
|
|
408
|
+
|
|
409
|
+
def test_unaliased_expressions_never_fail_ci_on_a_guessed_name(self):
|
|
410
|
+
sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, s.price * s.qty, CAST(b AS INT) + 1")
|
|
411
|
+
root = make_repo({"sql/m.sql": sql}, BILINGUAL)
|
|
412
|
+
code, _, err = run(root, "generate")
|
|
413
|
+
self.assertEqual(code, 0, err)
|
|
414
|
+
self.assertIn("has `*` or unnamed items; only its named columns are checked", err)
|
|
415
|
+
|
|
416
|
+
def test_an_unrelated_statement_is_never_used_as_the_contract(self):
|
|
417
|
+
sql = SQL_SCRIPT.replace("SELECT s.merchant_id, SUM(s.amount_usd) AS volume_usd", "SELECT *") + \
|
|
418
|
+
"INSERT INTO tmp.stage SELECT zz FROM q;\n"
|
|
419
|
+
root = make_repo({"sql/m.sql": sql}, BILINGUAL)
|
|
420
|
+
code, _, err = run(root, "generate")
|
|
421
|
+
self.assertEqual(code, 0, err)
|
|
422
|
+
self.assertIn("no column contract and no readable SELECT writing ads.merchant_monthly", err)
|
|
423
|
+
|
|
424
|
+
def test_tables_mapping_to_one_page_or_the_index_are_errors(self):
|
|
425
|
+
a = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"ads.a_b",\n en=')
|
|
426
|
+
b = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"ads.a b",\n en=')
|
|
427
|
+
readme = PY_SCRIPT.replace('"dwd.customer_daily",\n en=', '"README",\n en=')
|
|
428
|
+
root = make_repo({"jobs/a.py": a, "jobs/b.py": b, "jobs/c.py": readme}, BILINGUAL)
|
|
429
|
+
code, _, err = run(root, "generate")
|
|
430
|
+
self.assertEqual(code, 1)
|
|
431
|
+
self.assertIn("map to the same page ads.a_b.md", err)
|
|
432
|
+
self.assertIn("would overwrite the catalog index", err)
|
|
433
|
+
|
|
434
|
+
def test_exclude_globs_match_root_level_files_like_script_globs(self):
|
|
435
|
+
root = make_repo({"a_legacy.py": PY_SCRIPT, "jobs/b.py": PY_SCRIPT.replace("dwd.customer_daily", "dwd.other")},
|
|
436
|
+
{**BILINGUAL, "scripts": ["**/*.py"], "exclude": ["**/*_legacy.py"]})
|
|
437
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
438
|
+
self.assertEqual(sorted(p.name for p in (root / "docs/catalog").iterdir()), ["README.md", "dwd.other.md"])
|
|
439
|
+
|
|
440
|
+
def test_same_short_name_in_another_schema_is_a_different_table(self):
|
|
441
|
+
ods = '''
|
|
442
|
+
TARGET_TABLE = "ods.customer_daily"
|
|
443
|
+
'''
|
|
444
|
+
root = make_repo({"jobs/a.py": PY_SCRIPT, "jobs/ods.py": ods}, BILINGUAL)
|
|
445
|
+
report = {e["script"]: e for e in json.loads(run(root, "audit", "--json")[1])["scripts"]}
|
|
446
|
+
self.assertEqual([f["kind"] for f in report["jobs/ods.py"]["findings"]], ["no-catalog"])
|
|
447
|
+
|
|
448
|
+
def test_audit_ignores_commented_writes_dynamic_names_and_directories(self):
|
|
449
|
+
script = PY_SCRIPT + '''
|
|
450
|
+
# df.write.insertInto("old.dropped_table")
|
|
451
|
+
def run(spark, db, tbl):
|
|
452
|
+
spark.sql(f"INSERT OVERWRITE TABLE {db}.{tbl} SELECT 1")
|
|
453
|
+
spark.sql("INSERT OVERWRITE LOCAL DIRECTORY '/tmp/x' SELECT 1")
|
|
454
|
+
'''
|
|
455
|
+
root = make_repo({"jobs/a.py": script}, BILINGUAL)
|
|
456
|
+
[entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
|
|
457
|
+
self.assertEqual(sorted(w["table"] for w in entry["writes"]), ["dwd.customer_daily", "{expr}.{expr}"])
|
|
458
|
+
self.assertEqual(entry["findings"], [])
|
|
459
|
+
|
|
460
|
+
def test_python_syntax_error_is_one_finding(self):
|
|
461
|
+
root = make_repo({"jobs/a.py": "CATALOG_TABLE = (\n"}, BILINGUAL)
|
|
462
|
+
[entry] = json.loads(run(root, "audit", "--json")[1])["scripts"]
|
|
463
|
+
self.assertEqual([f["kind"] for f in entry["findings"]], ["invalid-metadata"])
|
|
464
|
+
|
|
465
|
+
def test_sql_block_ends_at_code_and_malformed_directives_fail(self):
|
|
466
|
+
late = SQL_SCRIPT + "-- @field ghost\n-- en: x\n"
|
|
467
|
+
root = make_repo({"sql/m.sql": late}, BILINGUAL)
|
|
468
|
+
self.assertEqual(run(root, "generate")[0], 0)
|
|
469
|
+
self.assertNotIn("ghost", (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8"))
|
|
470
|
+
bad = SQL_SCRIPT.replace("-- @table ads.merchant_monthly", "-- @table ads.merchant_monthly (monthly)")
|
|
471
|
+
root = make_repo({"sql/m.sql": bad}, BILINGUAL)
|
|
472
|
+
code, _, err = run(root, "generate")
|
|
473
|
+
self.assertEqual(code, 1)
|
|
474
|
+
self.assertIn("sql/m.sql:1: write `-- @table <name>`", err)
|
|
475
|
+
|
|
476
|
+
def test_backslash_escaped_quote_keeps_the_select_readable(self):
|
|
477
|
+
sql = SQL_SCRIPT.replace("SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd) AS volume_usd, 'it\\'s' AS note")
|
|
478
|
+
root = make_repo({"sql/m.sql": sql}, BILINGUAL)
|
|
479
|
+
code, _, err = run(root, "generate")
|
|
480
|
+
self.assertEqual(code, 1)
|
|
481
|
+
self.assertIn("outputs column(s) with no definition: note", err)
|
|
482
|
+
|
|
483
|
+
def test_init_with_invalid_json_config_exits_2(self):
|
|
484
|
+
root = make_repo({"jobs/a.py": "x = 1\n"})
|
|
485
|
+
(root / dc.CONFIG_FILE).write_text("{oops", encoding="utf-8")
|
|
486
|
+
code, _, err = run(root, "init", "--write")
|
|
487
|
+
self.assertEqual(code, 2)
|
|
488
|
+
self.assertIn("invalid JSON", err)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def git_repo(files: dict, config: dict) -> Path:
|
|
492
|
+
root = make_repo(files, config)
|
|
493
|
+
(root / ".git").rmdir()
|
|
494
|
+
for cmd in (["init", "-q"], ["add", "-A"], ["-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "base"]):
|
|
495
|
+
subprocess.run(["git", *cmd], cwd=root, check=True)
|
|
496
|
+
return root
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def edit(root: Path, name: str, old: str, new: str) -> None:
|
|
500
|
+
path = root / name
|
|
501
|
+
text = path.read_text(encoding="utf-8")
|
|
502
|
+
assert old in text, old
|
|
503
|
+
path.write_text(text.replace(old, new), encoding="utf-8")
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
class DriftTest(unittest.TestCase):
|
|
507
|
+
def test_changed_expression_without_changed_definition_is_reported(self):
|
|
508
|
+
root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
|
|
509
|
+
self.assertEqual(run(root, "drift")[0], 0)
|
|
510
|
+
edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "SUM(s.amount_usd * 0.9) AS volume_usd")
|
|
511
|
+
code, out, _ = run(root, "drift")
|
|
512
|
+
self.assertEqual(code, 1)
|
|
513
|
+
self.assertIn("sql/m.sql (ads.merchant_monthly)\n - volume_usd: its SELECT expression changed; its definition did not", out)
|
|
514
|
+
self.assertNotIn("filters, joins", out)
|
|
515
|
+
edit(root, "sql/m.sql", "Settled volume in the month.", "Settled volume net of the 10% fee.")
|
|
516
|
+
self.assertEqual(run(root, "drift")[0], 0)
|
|
517
|
+
|
|
518
|
+
def test_changed_filter_flags_the_table_until_a_definition_changes(self):
|
|
519
|
+
root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
|
|
520
|
+
edit(root, "sql/m.sql", "'SETTLED, final'", "'SETTLED'")
|
|
521
|
+
code, out, _ = run(root, "drift")
|
|
522
|
+
self.assertEqual(code, 1)
|
|
523
|
+
self.assertIn("filters, joins or CTEs changed; no definition changed", out)
|
|
524
|
+
edit(root, "sql/m.sql", "dwd_settlement settled rows,", "dwd_settlement rows with status SETTLED,")
|
|
525
|
+
self.assertEqual(run(root, "drift")[0], 0)
|
|
526
|
+
|
|
527
|
+
def test_dataframe_script_change_without_definition_change_is_reported(self):
|
|
528
|
+
script = PY_SCRIPT + "\ndef build(df):\n return df.filter('status = 1')\n"
|
|
529
|
+
root = git_repo({"jobs/a.py": script}, BILINGUAL)
|
|
530
|
+
edit(root, "jobs/a.py", "status = 1", "status = 2")
|
|
531
|
+
code, out, _ = run(root, "drift")
|
|
532
|
+
self.assertEqual(code, 1)
|
|
533
|
+
self.assertIn("code changed and no definition changed; the columns could not be read", out)
|
|
534
|
+
|
|
535
|
+
def test_definition_only_edit_and_untracked_new_script_are_quiet(self):
|
|
536
|
+
root = git_repo({"jobs/a.py": PY_SCRIPT}, BILINGUAL)
|
|
537
|
+
edit(root, "jobs/a.py", "Snapshot date.", "Snapshot date (UTC).")
|
|
538
|
+
(root / "jobs/new.py").write_text(textwrap.dedent(PY_SCRIPT).replace("dwd.customer_daily", "dwd.new"), encoding="utf-8")
|
|
539
|
+
self.assertEqual(run(root, "drift")[0], 0)
|
|
540
|
+
|
|
541
|
+
def test_base_ref_compares_a_whole_branch(self):
|
|
542
|
+
root = git_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
|
|
543
|
+
edit(root, "sql/m.sql", "SUM(s.amount_usd) AS volume_usd", "MAX(s.amount_usd) AS volume_usd")
|
|
544
|
+
subprocess.run(["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "change"], cwd=root, check=True)
|
|
545
|
+
self.assertEqual(run(root, "drift")[0], 0)
|
|
546
|
+
self.assertEqual(run(root, "drift", "--base", "HEAD~1")[0], 1)
|
|
547
|
+
|
|
548
|
+
def test_outside_git_drift_exits_2(self):
|
|
549
|
+
root = make_repo({"sql/m.sql": SQL_SCRIPT}, BILINGUAL)
|
|
550
|
+
(root / ".git").rmdir()
|
|
551
|
+
code, _, err = run(root, "drift")
|
|
552
|
+
self.assertEqual(code, 2)
|
|
553
|
+
self.assertIn("is not a commit", err)
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
class InitTest(unittest.TestCase):
|
|
557
|
+
def test_dry_run_writes_nothing(self):
|
|
558
|
+
root = make_repo({"jobs/dwd/a.py": "x = 1\n"})
|
|
559
|
+
code, out, _ = run(root, "init", "--scripts", "jobs/**/*.py")
|
|
560
|
+
self.assertEqual(code, 0)
|
|
561
|
+
self.assertIn("would write data-catalog.json", out)
|
|
562
|
+
self.assertEqual(sorted(p.name for p in root.iterdir()), [".git", "jobs"])
|
|
563
|
+
|
|
564
|
+
def test_write_sets_up_repo_and_generated_module_accepts_documented_scripts(self):
|
|
565
|
+
root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT, "jobs/dwd/other.py": "x = 1\n",
|
|
566
|
+
"AGENTS.md": "# Agents\n\nExisting rules.\n"})
|
|
567
|
+
code, out, err = run(root, "init", "--scripts", "jobs/**/*.py", "--languages", "en,zh", "--write")
|
|
568
|
+
self.assertEqual(code, 0, err)
|
|
569
|
+
config = json.loads((root / dc.CONFIG_FILE).read_text(encoding="utf-8"))
|
|
570
|
+
self.assertEqual(config["fieldsModule"], "jobs/dwd/catalog_fields.py")
|
|
571
|
+
self.assertEqual((root / dc.ENGINE_PATH).read_text(encoding="utf-8"), (HERE / "data_catalog.py").read_text(encoding="utf-8"))
|
|
572
|
+
workflow = (root / dc.WORKFLOW_PATH).read_text(encoding="utf-8")
|
|
573
|
+
self.assertIn(" - 'jobs/**/*.py'\n", workflow)
|
|
574
|
+
self.assertIn("run: python3 scripts/data-catalog/data_catalog.py generate --check", workflow)
|
|
575
|
+
# agent pointers are persist-conventions.mjs's job; init leaves them alone
|
|
576
|
+
self.assertEqual((root / "AGENTS.md").read_text(encoding="utf-8"), "# Agents\n\nExisting rules.\n")
|
|
577
|
+
self.assertFalse((root / "CLAUDE.md").exists())
|
|
578
|
+
self.assertTrue((root / "docs/catalog/dwd.customer_daily.md").exists())
|
|
579
|
+
# the generated runtime module takes exactly the arguments the engine reads
|
|
580
|
+
spec = importlib.util.spec_from_file_location("catalog_fields", root / config["fieldsModule"])
|
|
581
|
+
module = importlib.util.module_from_spec(spec)
|
|
582
|
+
sys.modules["catalog_fields"] = module # dataclasses resolve string annotations through sys.modules
|
|
583
|
+
self.addCleanup(sys.modules.pop, "catalog_fields", None)
|
|
584
|
+
spec.loader.exec_module(module)
|
|
585
|
+
f = module.Field("x", en="e", zh="z", source="s", window="7d", unit="USD")
|
|
586
|
+
self.assertEqual((f.name, f.zh, f.unit), ("x", "z", "USD"))
|
|
587
|
+
with self.assertRaises(TypeError):
|
|
588
|
+
module.Field("x", en="e", source="s")
|
|
589
|
+
# rerunning is a no-op, and the catalog check passes
|
|
590
|
+
code, out, _ = run(root, "init", "--write")
|
|
591
|
+
self.assertEqual(code, 0)
|
|
592
|
+
self.assertNotIn("write ", out.replace("would write", ""))
|
|
593
|
+
self.assertEqual(run(root, "generate", "--check")[0], 0)
|
|
594
|
+
|
|
595
|
+
def test_bad_language_code_is_refused(self):
|
|
596
|
+
root = make_repo({"jobs/a.py": "x = 1\n"})
|
|
597
|
+
code, _, err = run(root, "init", "--scripts", "jobs/*.py", "--languages", "English", "--write")
|
|
598
|
+
self.assertEqual(code, 2)
|
|
599
|
+
self.assertIn("`languages` must be codes", err)
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
if __name__ == "__main__":
|
|
603
|
+
unittest.main()
|