@1aboveio/app-dev-skills 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,277 @@
1
+ """Tests for data_quality.py against a real repo and a real SQLite warehouse. Run: python3 test_data_quality.py"""
2
+ from __future__ import annotations
3
+
4
+ import contextlib
5
+ import io
6
+ import json
7
+ import shutil
8
+ import sqlite3
9
+ import sys
10
+ import tempfile
11
+ import textwrap
12
+ import unittest
13
+ from pathlib import Path
14
+
15
+ HERE = Path(__file__).resolve().parent
16
+ CATALOG_ENGINE = HERE.parent.parent / "data-catalog" / "scripts" / "data_catalog.py"
17
+ sys.path.insert(0, str(HERE))
18
+ import data_quality as dq # noqa: E402
19
+
20
+ SCRIPT = '''
21
+ from catalog_fields import Field, Table
22
+
23
+ CATALOG_TABLE = Table(
24
+ "customer_daily",
25
+ en="Daily customer metrics.",
26
+ grain_en="One row per (dt, cust_id).",
27
+ )
28
+ CATALOG_FIELDS = (
29
+ Field("dt", en="Snapshot date.", source="run parameter dt"),
30
+ Field("cust_id", en="Customer id.", source="dwd.transaction.cust_id"),
31
+ Field("coll_cnt_7d", en="Number of collections in the last 7 days.", source="count of rows", unit="count"),
32
+ Field("recall_ratio", en="Recalled share of collected amount, 0-1.", source="recall / collected", unit="ratio"),
33
+ Field("agent_risk", en="Agent risk: yes, no or undetermined.", source="agent list and recalls",
34
+ values=("是", "否", "无法判定")),
35
+ Field("result", en="Transaction result.", source="constant in v1", values=("SUCCESS", "FAILED")),
36
+ Field("cert_type", en="Payer certificate type.", source="dwd.transaction.debtor_cert_type"),
37
+ Field("legacy_flag", en="Legacy flag; always empty since v2.", source="none"),
38
+ Field("channel", en="Collection channel.", source="dwd.transaction.channel", values=("bank", "card")),
39
+ )
40
+ PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
41
+ '''
42
+
43
+ ROWS = [
44
+ # dt, cust_id, coll_cnt_7d, recall_ratio, agent_risk, result, cert_type, legacy_flag, channel
45
+ ("2026-10-01", "c1", 3, 0.1, "是", "SUCCESS", None, None, "bank"),
46
+ ("2026-10-01", "c2", 0, 1.4, "无法判定", "SUCCESS", None, None, "card"),
47
+ ("2026-10-02", "c1", -1, 0.0, "无法判定", "SUCCESS", None, None, "wallet"),
48
+ ("2026-10-02", "c3", 5, None, "是", "SUCCESS", None, None, "bank"),
49
+ ]
50
+
51
+
52
+ def make_repo(rows=ROWS, script=SCRIPT, extra_column=None) -> tuple[Path, Path]:
53
+ root = Path(tempfile.mkdtemp(prefix="data-quality-test-"))
54
+ (root / ".git").mkdir()
55
+ (root / "scripts/data-catalog").mkdir(parents=True)
56
+ shutil.copy(CATALOG_ENGINE, root / "scripts/data-catalog/data_catalog.py")
57
+ (root / "jobs").mkdir()
58
+ (root / "jobs/customer_daily.py").write_text(textwrap.dedent(script), encoding="utf-8")
59
+ (root / "data-catalog.json").write_text(json.dumps({"version": 1, "scripts": ["jobs/*.py"], "languages": ["en"]}),
60
+ encoding="utf-8")
61
+ db = root / "warehouse.db"
62
+ con = sqlite3.connect(db)
63
+ columns = ["dt TEXT", "cust_id TEXT", "coll_cnt_7d INTEGER", "recall_ratio REAL", "agent_risk TEXT", "result TEXT",
64
+ "cert_type TEXT", "legacy_flag TEXT", "channel TEXT"]
65
+ if extra_column:
66
+ columns.insert(6, extra_column)
67
+ con.execute(f"CREATE TABLE customer_daily ({', '.join(columns)})")
68
+ con.executemany(f"INSERT INTO customer_daily VALUES ({', '.join('?' * len(columns))})", rows)
69
+ con.commit()
70
+ con.close()
71
+ return root, db
72
+
73
+
74
+ def run(root: Path, *argv: str) -> tuple[int, str, str]:
75
+ out, err = io.StringIO(), io.StringIO()
76
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
77
+ code = dq.main(["--root", str(root), *argv])
78
+ return code, out.getvalue(), err.getvalue()
79
+
80
+
81
+ def findings(root: Path, db: Path, *extra: str) -> tuple[int, dict]:
82
+ code, out, err = run(root, "run", "--sqlite", str(db), "--json", "--constant-min-rows", "2", *extra)
83
+ assert out, err
84
+ report = json.loads(out)
85
+ return code, {(f["column"], f["kind"]): f for f in report["findings"]}
86
+
87
+
88
+ class RunTest(unittest.TestCase):
89
+ def test_each_check_fires_on_the_column_that_breaks_it(self):
90
+ root, db = make_repo()
91
+ code, found = findings(root, db)
92
+ self.assertEqual(code, 1)
93
+ self.assertEqual(found[("coll_cnt_7d", "negative-value")]["message"], "minimum -1, but the unit is count")
94
+ self.assertEqual(found[("recall_ratio", "out-of-range")]["message"],
95
+ "values from 0 to 1.4, outside 0–1 for unit ratio")
96
+ self.assertEqual(found[("agent_risk", "unseen-value")]["message"], "declared but never present: 否")
97
+ self.assertEqual(found[("channel", "undeclared-value")]["message"], "present but not declared: wallet (1)")
98
+ self.assertEqual(found[("result", "constant")]["message"], "one value in all 4 filled rows")
99
+ self.assertEqual(found[("result", "unseen-value")]["message"], "declared but never present: FAILED")
100
+ self.assertEqual(found[("cert_type", "never-filled")]["message"], "empty in all 4 rows")
101
+ self.assertFalse(found[("cert_type", "never-filled")]["documented"])
102
+ # the definition already says "always empty": reported, but as documented
103
+ self.assertTrue(found[("legacy_flag", "never-filled")]["documented"])
104
+ # clean columns raise nothing
105
+ self.assertFalse([k for k in found if k[0] in ("dt", "cust_id")])
106
+
107
+ def test_a_missing_flag_constant_at_zero_is_healthy_but_at_one_is_not(self):
108
+ script = SCRIPT.replace('Field("cert_type"', 'Field("fx_missing", en="1 if any amount lacked a rate.", source="flag"),\n Field("cert_type"')
109
+ for value, healthy in ((0, True), (1, False)):
110
+ rows = [r[:6] + (value,) + r[6:] for r in ROWS]
111
+ root, db = make_repo(rows, script, extra_column="fx_missing INTEGER")
112
+ _, found = findings(root, db)
113
+ self.assertEqual(found[("fx_missing", "constant")]["documented"], healthy, value)
114
+
115
+ def test_version_and_hash_columns_are_expected_to_be_constant(self):
116
+ script = SCRIPT.replace('Field("cert_type"', 'Field("rule_version", en="Rule set version.", source="RULE_VERSION"),\n Field("cert_type"')
117
+ rows = [r[:6] + ("v2",) + r[6:] for r in ROWS]
118
+ root = Path(tempfile.mkdtemp(prefix="data-quality-test-"))
119
+ root, db = make_repo(rows, script, extra_column="rule_version TEXT")
120
+ _, found = findings(root, db)
121
+ self.assertTrue(found[("rule_version", "constant")]["documented"])
122
+
123
+ def test_duplicate_grain_keys_are_counted(self):
124
+ rows = ROWS + [ROWS[0]] # (2026-10-01, c1) twice
125
+ root, db = make_repo(rows)
126
+ _, found = findings(root, db, "--key", "dt", "--key", "cust_id")
127
+ self.assertEqual(found[(None, "duplicate-key")]["message"],
128
+ "1 (dt, cust_id) value(s) repeat across 2 rows; the grain allows one each")
129
+ root, db = make_repo()
130
+ _, clean = findings(root, db, "--key", "dt", "--key", "cust_id")
131
+ self.assertNotIn((None, "duplicate-key"), clean)
132
+ code, _, err = run(root, "plan", "--key", "nope")
133
+ self.assertEqual(code, 2)
134
+ self.assertIn("--key column(s) not documented in customer_daily: nope", err)
135
+
136
+ def test_where_bounds_the_scan(self):
137
+ root, db = make_repo()
138
+ _, found = findings(root, db, "--where", "dt = '2026-10-01'")
139
+ self.assertNotIn(("coll_cnt_7d", "negative-value"), found)
140
+ self.assertNotIn(("channel", "undeclared-value"), found)
141
+ self.assertIn(("recall_ratio", "out-of-range"), found)
142
+
143
+ def test_where_columns_and_small_samples_are_not_called_constant(self):
144
+ root, db = make_repo()
145
+ _, found = findings(root, db, "--where", "dt = '2026-10-01'")
146
+ self.assertNotIn(("dt", "constant"), found)
147
+ code, out, _ = run(root, "run", "--sqlite", str(db), "--json") # default minimum: 30 rows
148
+ self.assertNotIn("constant", {f["kind"] for f in json.loads(out)["findings"]})
149
+
150
+ def test_an_empty_range_is_reported_once_and_judges_nothing_else(self):
151
+ root, db = make_repo()
152
+ code, found = findings(root, db, "--where", "dt = '1999-01-01'")
153
+ self.assertEqual(code, 1)
154
+ self.assertEqual(list(found), [(None, "empty-table")])
155
+
156
+ def test_clean_data_exits_zero(self):
157
+ clean = [(dt, c, n, ratio, a, r, cert, None, ch) for dt, c, n, ratio, a, r, cert, ch in [
158
+ ("2026-10-01", "c1", 3, 0.1, "是", "SUCCESS", "ID", "bank"),
159
+ ("2026-10-02", "c2", 0, 0.0, "否", "FAILED", "PASSPORT", "card"),
160
+ ("2026-10-03", "c3", 7, 1.0, "无法判定", "SUCCESS", "ID", "bank")]]
161
+ root, db = make_repo(clean)
162
+ code, found = findings(root, db)
163
+ self.assertEqual(code, 0, found)
164
+ self.assertEqual([k for k, f in found.items() if not f["documented"]], [])
165
+
166
+ def test_external_runner_gets_only_selects_and_its_csv_or_json_is_read(self):
167
+ root, db = make_repo()
168
+ runner = root / "runner.py"
169
+ log = root / "sql.log"
170
+ runner.write_text(textwrap.dedent(f'''
171
+ import csv, sqlite3, sys
172
+ sql = sys.stdin.read()
173
+ open({str(log)!r}, "a").write(sql + "\\n")
174
+ con = sqlite3.connect({str(db)!r})
175
+ cur = con.execute(sql)
176
+ w = csv.writer(sys.stdout)
177
+ w.writerow([d[0] for d in cur.description])
178
+ w.writerows(cur.fetchall())
179
+ '''), encoding="utf-8")
180
+ code, out, err = run(root, "run", "--runner", f"{sys.executable} {runner}", "--json")
181
+ self.assertEqual(code, 1, err)
182
+ kinds = {(f["column"], f["kind"]) for f in json.loads(out)["findings"]}
183
+ self.assertIn(("coll_cnt_7d", "negative-value"), kinds)
184
+ statements = [s for s in log.read_text(encoding="utf-8").split("\n") if s.strip()]
185
+ self.assertTrue(statements and all(s.startswith("SELECT ") for s in statements))
186
+
187
+
188
+ class PlanJudgeTest(unittest.TestCase):
189
+ def test_plan_then_judge_matches_run(self):
190
+ root, db = make_repo()
191
+ code, out, _ = run(root, "plan", "--dialect", "spark", "--skip", "channel")
192
+ self.assertEqual(code, 0)
193
+ queries = json.loads(out)
194
+ self.assertEqual([q["id"] for q in queries],
195
+ ["profile:customer_daily", "values:customer_daily.agent_risk", "values:customer_daily.result"])
196
+ self.assertIn("COUNT(DISTINCT `coll_cnt_7d`) AS d_2", queries[0]["sql"])
197
+ self.assertIn("MIN(`recall_ratio`) AS mn_3", queries[0]["sql"])
198
+ self.assertNotIn("channel", queries[0]["sql"])
199
+ con = sqlite3.connect(db)
200
+ results = {}
201
+ for q in queries:
202
+ cur = con.execute(q["sql"].replace("`", '"'))
203
+ results[q["id"]] = [dict(zip([d[0] for d in cur.description], r)) for r in cur.fetchall()]
204
+ path = root / "results.json"
205
+ path.write_text(json.dumps(results), encoding="utf-8")
206
+ code, out, _ = run(root, "judge", "--skip", "channel", "--results", str(path), "--json", "--constant-min-rows", "2")
207
+ kinds = {(f["column"], f["kind"]) for f in json.loads(out)["findings"]}
208
+ self.assertIn(("agent_risk", "unseen-value"), kinds)
209
+ self.assertNotIn(("channel", "undeclared-value"), kinds)
210
+
211
+ def test_profile_batches_cover_every_column_and_report_an_empty_range_once(self):
212
+ root, db = make_repo()
213
+ code, out, _ = run(root, "plan", "--batch", "4")
214
+ ids = [q["id"] for q in json.loads(out) if q["kind"] == "profile"]
215
+ self.assertEqual(ids, ["profile:customer_daily#1", "profile:customer_daily#2", "profile:customer_daily#3"])
216
+ cols = [c for q in json.loads(out) if q["kind"] == "profile" for c in q["columns"]]
217
+ self.assertEqual(len(cols), 9)
218
+ _, found = findings(root, db, "--batch", "4")
219
+ self.assertIn(("recall_ratio", "out-of-range"), found)
220
+ self.assertIn(("legacy_flag", "never-filled"), found)
221
+ _, empty = findings(root, db, "--batch", "4", "--where", "dt = 'none'")
222
+ self.assertEqual(list(empty), [(None, "empty-table")])
223
+
224
+ def test_a_missing_result_is_reported_not_skipped(self):
225
+ root, _ = make_repo()
226
+ path = root / "results.json"
227
+ path.write_text("{}", encoding="utf-8")
228
+ code, out, _ = run(root, "judge", "--results", str(path), "--json")
229
+ self.assertEqual(code, 1)
230
+ self.assertIn("not-run", {f["kind"] for f in json.loads(out)["findings"]})
231
+
232
+
233
+ class OldEngineTest(unittest.TestCase):
234
+ def test_fields_from_an_engine_copy_without_values_still_plan(self):
235
+ from types import SimpleNamespace as NS
236
+ old = NS(name="t", fields=[NS(name="a", text={"en": "x"}, source="s", window=None, unit="count", line=1)])
237
+ [query] = dq.plan_queries([old], None, "ansi", set())
238
+ self.assertEqual(query["sql"], 'SELECT COUNT(*) AS n, COUNT("a") AS nn_0, COUNT(DISTINCT "a") AS d_0, '
239
+ 'MIN("a") AS mn_0, MAX("a") AS mx_0 FROM t')
240
+
241
+
242
+ class RowsTest(unittest.TestCase):
243
+ def test_runner_output_formats(self):
244
+ self.assertEqual(dq.parse_rows('[{"n": 1}]'), [{"n": 1}])
245
+ self.assertEqual(dq.parse_rows('{"value": "a", "n": 2}\n{"value": "b", "n": 3}\n'),
246
+ [{"value": "a", "n": 2}, {"value": "b", "n": 3}])
247
+ self.assertEqual(dq.parse_rows("value,n\na,2\n"), [{"value": "a", "n": "2"}])
248
+ self.assertEqual(dq.parse_rows(""), [])
249
+
250
+
251
+ class SetupTest(unittest.TestCase):
252
+ def test_without_the_catalog_engine_it_explains_the_setup(self):
253
+ root, _ = make_repo()
254
+ (root / "scripts/data-catalog/data_catalog.py").unlink()
255
+ code, _, err = run(root, "plan")
256
+ self.assertEqual(code, 2)
257
+ self.assertIn("set the repo up with the data-catalog skill first", err)
258
+
259
+ def test_an_unknown_table_is_refused(self):
260
+ root, _ = make_repo()
261
+ code, _, err = run(root, "plan", "--table", "nope")
262
+ self.assertEqual(code, 2)
263
+ self.assertIn("not documented in the catalog: nope", err)
264
+
265
+ def test_statements_other_than_one_select_are_refused(self):
266
+ for sql in ("DELETE FROM t", "SELECT 1; DROP TABLE t"):
267
+ with self.assertRaises(dq.QualityError):
268
+ dq.assert_read_only(sql)
269
+
270
+ def test_the_sqlite_runner_cannot_write(self):
271
+ root, db = make_repo()
272
+ with self.assertRaises(sqlite3.OperationalError):
273
+ dq.run_sqlite(str(db), "INSERT INTO customer_daily (dt) VALUES ('x')")
274
+
275
+
276
+ if __name__ == "__main__":
277
+ unittest.main()
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: ios-foundation
3
3
  description: "Set up and prove an Expo / React Native native iOS workspace from a clean checkout to a manual Login smoke on an owned iOS Simulator: isolated worktree and private caches, Xcode/runtime/CocoaPods preflight, frozen install with `pod install --deployment`, platform-scoped prebuild, an ad-hoc-signed Debug simulator build, a private simulator by exact UDID, Metro on a free port, a disposable PostgreSQL backend with owned cleanup, and the evidence record (setup proof only; feature work reports the smoke result). Use for 'set up iOS', 'build the iOS app', 'run the app on the iOS simulator', 'add iOS to the React Native app', 'prove the iOS build', or when a pod install, xcodebuild, simulator boot, Metro connection or Keychain session fails. NOT for Android (android-foundation), signing, TestFlight or App Store release, or general workflow YAML (github-actions)."
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # iOS foundation
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: monorepo
3
3
  description: Turborepo architecture for shared packages, pnpm workspaces, DAL boundaries, and pipelines.
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Turborepo Monorepo Architecture
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: nextjs-fullstack
3
3
  description: "Build or debug Next.js App Router features with React 19 — pages/layouts/route handlers, Server/Client Components, Server Actions, forms, middleware, DAL, strict TypeScript, Zod — plus our conventions: nuqs URL state, SWR cache, Skeleton/Progress + Sonner feedback, drag-and-drop upload, the portal app shell, list pages as filter/sort/paginate tables. shadcn component mechanics: see the shadcn skill."
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Next.js 16 Fullstack Development
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: prisma-setup
3
3
  description: Use when adding or debugging Prisma with PostgreSQL, Next.js, or Turborepo, including schema.prisma, prisma.config.ts, client generation, migrations, seeds, relations, adapters, and DAL/database-layer setup. Triggers on phrases like "Prisma schema", "prisma migrate", "schema.prisma", "seed the database", "PrismaClient error", "relation issue", or "database layer".
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Prisma v7 Setup Guide
@@ -12,7 +12,7 @@ description: >
12
12
  Connect, spark-submit, Hudi commits, backfills, validate-only canaries,
13
13
  cluster reconciliation, or Spark event-log profiling — including warehouse
14
14
  job timeouts, empty Hudi writes, or EMR/Dataproc performance regressions.
15
- release-version: 0.38.0
15
+ release-version: 0.40.0
16
16
  ---
17
17
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
18
18
  # PySpark
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: python-backend
3
3
  description: "Python backend architecture for FastAPI, Tortoise ORM, Pydantic Settings, loguru, and uv. Also: uv.lock installs in CI/Docker via a proxy or mirror (`uv sync --frozen` bypasses it), slow uv installs on runners, relocking."
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Python Backend Conventions
@@ -2,7 +2,7 @@
2
2
  name: shadcn
3
3
  description: "Add or debug shadcn/ui in a React/Next.js project: components.json, shadcn init/add, presets, registries, Tailwind styling, base/radix/aria composition, component fixes, and list-index pages — where the default is the DiceUI data table with filter/sort/pagination in the URL via nuqs, not a bare shadcn Table."
4
4
  allowed-tools: Bash(npx shadcn@latest *), Bash(pnpm dlx shadcn@latest *), Bash(bunx --bun shadcn@latest *)
5
- release-version: 0.38.0
5
+ release-version: 0.40.0
6
6
  ---
7
7
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
8
8
  # shadcn/ui
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: zod-v4
3
3
  description: Zod v4 schema conventions and v3 migration for errors, validators, objects, enums, records, and async parsing.
4
- release-version: 0.38.0
4
+ release-version: 0.40.0
5
5
  ---
6
6
  **Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
7
7
  # Zod v3 to v4 Migration