@1aboveio/skills 1.0.1 → 1.0.3

package/skills/app-dev/data-catalog/references/data-catalog-conventions.md

@@ -52,7 +52,8 @@ table. `docs/catalog/` is generated from those definitions; never edit it by han
    collections.", not "0 when none".
 9. Never restate the column name or write "same as above".
 10. Cover every case: what the column holds for each kind of row, including when it
-    is empty; list an enumerated column's values; say so if it is always empty today.
+    is empty; list an enumerated column's values in the text and in `values=(...)`;
+    say so if it is always empty today.
 11. Write related columns as a set: a count and its amount, a value and its missing
     flag, a legacy column and its replacement use the same scope and wording, and any
     difference is stated in both rows.

package/skills/app-dev/data-catalog/references/field-definitions.md

@@ -50,7 +50,8 @@ column that says only what is particular to that column.
    `"同上，30 天窗口"` tell the reader nothing.
 10. **Cover every case.** Say what the column holds for every kind of row (each
     transaction type, each branch of the code), including when it is empty. List
-    the values an enumerated column can take, with a Chinese gloss for English codes.
+    the values an enumerated column can take, with a Chinese gloss for English codes,
+    and put the same list in `values=(...)` so the data can be checked against it.
     A column that is always empty or constant today says so.
     Bad: `"换汇交易为 COMPLETED，内部转账为空。"` (collections and payouts unsaid)
     Good: `"换汇交易为 COMPLETED；收款、付款和内部转账为空。"`

package/skills/app-dev/data-catalog/references/format.md

@@ -41,7 +41,10 @@ CATALOG_FIELDS = (
 PUBLISHED_COLUMNS = tuple(f.name for f in CATALOG_FIELDS)
 ```
 
-- `Field(name, <one keyword per language>, source, window=None, unit=None)`.
+- `Field(name, <one keyword per language>, source, window=None, unit=None, values=())`.
+  `values` lists the values an enumerated column can take, as distinct strings
+  (`values=("是", "否", "无法判定")`); the data-quality skill checks the data against it.
+  Never use it on identifiers or free text.
 - `Table(name, <one keyword per language>, grain_<lang>=..., notes_<lang>=(...))`: a
   grain for every language, and the same number of notes in every language (notes are
   optional).
@@ -77,7 +80,8 @@ SELECT ...
 ```
 
 - A directive line is `-- @table <name>` or `-- @field <name>`.
-- A key line is exactly two spaces after `-- `, then `key: value`.
+- A key line is exactly two spaces after `-- `, then `key: value`. `values:` takes a
+  list split on `;` or `；`.
 - A continuation line has four or more spaces and joins the previous key with a space.
 - `notes_<lang>` on `@table` splits on `;` or `；`.
 - The block ends at the first non-comment line. One `@table` per file.

package/skills/app-dev/data-catalog/scripts/data_catalog.py

@@ -49,7 +49,7 @@ DEFAULT_CONFIG = {
     "tableNames": [],
     "schemaNames": [],
 }
-FIELD_EXTRAS = ("source", "window", "unit")
+FIELD_EXTRAS = ("source", "window", "unit", "values")
 LANGUAGE_LABELS = {"en": "English", "zh": "中文", "zh_hant": "繁體中文", "ja": "日本語", "ko": "한국어"}
 LANGUAGE_RE = re.compile(r"^[a-z]{2}(_[a-z]+)?$")
 SKIP_DIRS = {"node_modules", "__pycache__", "venv"}
@@ -151,6 +151,7 @@ class FieldDoc:
     window: str | None
     unit: str | None
     line: int
+    values: tuple = ()   # the values an enumerated column can take; checked by data-quality
 
 
 @dataclass
@@ -258,7 +259,14 @@ def build_field(values: dict, languages: list, where: str, line: int) -> FieldDo
                            f"languages configured: {languages}")
     text = {lang: require_text(values, lang, where, line, f"Field {name!r}") for lang in languages}
     source = require_text(values, "source", where, line, f"Field {name!r}")
-    return FieldDoc(name, text, source, (values.get("window") or None), (values.get("unit") or None), line)
+    allowed = values.get("values") or ()
+    if isinstance(allowed, str):
+        allowed = tuple(v.strip() for v in re.split(r"[;；]", allowed))
+    if not isinstance(allowed, tuple) or not all(isinstance(v, str) and v != "" for v in allowed):
+        raise CatalogError(f"{where}:{line}: Field {name!r}: values must be a tuple of non-empty strings")
+    if len(set(allowed)) != len(allowed):
+        raise CatalogError(f"{where}:{line}: Field {name!r}: values lists a value twice")
+    return FieldDoc(name, text, source, (values.get("window") or None), (values.get("unit") or None), line, allowed)
 
 
 def build_table(values: dict, fields: list, languages: list, where: str, line: int, script: str) -> TableDoc:
@@ -545,15 +553,18 @@ def render_table(table: TableDoc, root: Path, config: dict) -> str:
         out += ["## Fields", ""]
     has_window = any(f.window for f in table.fields)
     has_unit = any(f.unit for f in table.fields)
+    has_values = any(f.values for f in table.fields)
     head = ["Field", *(label(lang) for lang in langs), "Source"]
     head += ["Window"] if has_window else []
     head += ["Unit"] if has_unit else []
+    head += ["Values"] if has_values else []
     out.append("| " + " | ".join(head) + " |")
     out.append("|" + "---|" * len(head))
     for f in table.fields:
         row = [f"`{f.name}`", *(cell(f.text[lang]) for lang in langs), cell(f.source)]
         row += [cell(f.window)] if has_window else []
         row += [cell(f.unit)] if has_unit else []
+        row += [cell(", ".join(f"`{v}`" for v in f.values))] if has_values else []
         out.append("| " + " | ".join(row) + " |")
     return "\n".join(out) + "\n"
 
@@ -1175,7 +1186,7 @@ def code_without_metadata(text: str, suffix: str) -> str:
 def definitions(table: TableDoc | None) -> tuple[dict, tuple]:
     if table is None:
         return {}, ()
-    fields = {f.name: (tuple(sorted(f.text.items())), f.source, f.window, f.unit) for f in table.fields}
+    fields = {f.name: (tuple(sorted(f.text.items())), f.source, f.window, f.unit, f.values) for f in table.fields}
     notes = tuple((lang, tuple(n)) for lang, n in sorted(table.notes.items()))
     return fields, (tuple(sorted(table.grain.items())), tuple(sorted(table.text.items())), notes)
 
@@ -1274,6 +1285,7 @@ class Field:
     source: str
     window: str | None = None
     unit: str | None = None
+    values: tuple[str, ...] = ()
 
 
 @dataclass(frozen=True)

package/skills/app-dev/data-catalog/scripts/persist-conventions-core.mjs

@@ -5,7 +5,8 @@
 // section, so no main-module guard here.
 //
 // Byte-identical copies live in test-quality (Testing conventions), github-flow
-// (CI conventions, Delivery) and data-catalog (Data catalog conventions);
+// (CI conventions, Delivery), data-catalog (Data catalog conventions) and
+// contributing (Issue tracking);
 // skills/conventions-sections.test.mjs keeps them identical and proves the sections
 // coexist in one file.
 //
@@ -22,7 +23,7 @@ const MENTIONS_CONTRIBUTING = /(?:^|[^\w-])CONTRIBUTING\.md/
 export const AGENTS_SECTION = [
   '## Contributing',
   '',
-  `Follow [${CONTRIBUTING_FILE}](${CONTRIBUTING_FILE}) when writing, reviewing, or delivering code: tests, CI, data definitions, and what to do after each commit.`,
+  `Follow [${CONTRIBUTING_FILE}](${CONTRIBUTING_FILE}) when writing, reviewing, or delivering code, or creating issues: tests, CI, data definitions, issue tracking, and what to do after each commit.`,
   '',
 ].join('\n')
 

package/skills/app-dev/data-catalog/scripts/test_data_catalog.py

@@ -124,6 +124,21 @@ class GenerateTest(unittest.TestCase):
         self.assertIn("| [`dwd.customer_daily`](dwd.customer_daily.md) | Daily customer collection rollup. | "
                       "one row per (dt, cust_id) | 2 |", index)
 
+    def test_values_render_and_must_be_distinct_strings(self):
+        script = PY_SCRIPT.replace('source="Run parameter `dt`."),', 'source="Run parameter `dt`.", values=("a", "b")),')
+        root = make_repo({"jobs/a.py": script}, BILINGUAL)
+        self.assertEqual(run(root, "generate")[0], 0)
+        page = (root / "docs/catalog/dwd.customer_daily.md").read_text(encoding="utf-8")
+        self.assertIn("| Unit | Values |", page)
+        self.assertIn("| `a`, `b` |", page)
+        twice = script.replace('values=("a", "b")', 'values=("a", "a")')
+        root = make_repo({"jobs/a.py": twice}, BILINGUAL)
+        self.assertIn("values lists a value twice", run(root, "generate")[2])
+        sql = SQL_SCRIPT.replace("--   source: dim_merchant.merchant_id", "--   source: dim_merchant.merchant_id\n--   values: m1；m2")
+        root = make_repo({"sql/m.sql": sql}, BILINGUAL)
+        self.assertEqual(run(root, "generate")[0], 0)
+        self.assertIn("`m1`, `m2`", (root / "docs/catalog/ads.merchant_monthly.md").read_text(encoding="utf-8"))
+
     def test_check_passes_when_current_and_fails_after_a_definition_changes(self):
         root = make_repo({"jobs/dwd/customer_daily.py": PY_SCRIPT}, BILINGUAL)
         self.assertEqual(run(root, "generate")[0], 0)

package/skills/app-dev/data-quality/LICENSE

@@ -0,0 +1,3 @@
+MIT License
+
+Copyright (c) 2026 1AboveIO

package/skills/app-dev/data-quality/SKILL.md

@@ -0,0 +1,96 @@
+---
+name: data-quality
+description: "Check a published warehouse table's data against its data-catalog definitions with read-only SELECTs: columns never filled or stuck at one value, counts below zero, ratios outside 0–1, declared values that never appear and undeclared values that do. Then classify each finding as a wrong definition, wrong code, or wrong data. Use when asked whether a table's data is right, to profile or sanity-check a dwd/ads table, to find dead or constant columns, to check a column's values against its documented list, after a pipeline change ships, or when a data-catalog definition and the data might disagree. NOT for validating a Spark change before it ships (pyspark canary and reconciliation), writing definitions (data-catalog), or monitoring dashboards."
+release-version: 0.40.0
+---
+**Updates:** Once/session run `node <skill-base-dir>/release-check.mjs`.
+# Data quality
+
+Ask the data whether it means what its definitions say. The definitions come from the
+data-catalog skill; this skill turns them into read-only queries, runs them, and
+reports where table, code, and documentation disagree.
+
+## Find your task
+
+| Task | Do |
+|---|---|
+| The repo has no `data-catalog.json` or the table has no definitions | Set the repo up and document the table with the data-catalog skill first |
+| Check one or more tables | [Run the checks](#run-the-checks) |
+| Decide what a finding means | [Classify each finding](#classify-each-finding) |
+| What each check does, and how to make a column checkable | [references/checks.md](references/checks.md) |
+| Connect to Presto, Trino, Spark SQL, or another engine | [references/checks.md#runners](references/checks.md#runners) |
+
+`<engine>` is `<skill-base-dir>/scripts/data_quality.py`. It reads definitions through
+the repo's own `scripts/data-catalog/data_catalog.py`.
+
+## Rules
+
+1. **Read only.** Every query is one SELECT built by the engine; the SQLite runner
+   opens the file read-only. Never add a write to a runner, and never run the checks
+   with credentials that can write when read-only ones exist.
+2. **Bound the scan.** Pass `--where` on the partition column for a recent range, and
+   widen only when a finding needs it.
+   Example: `--where "dt >= DATE '2026-10-01'"` scanned 333K rows in 18 seconds.
+3. **Aggregates only.** The checks return counts, distinct counts, minimums, maximums,
+   and counts per declared value. Never add a query that returns rows, and never print
+   a column's actual value when it could identify a customer.
+4. **A finding is a lead until you classify it.** Read the code that writes the column
+   before saying the data, the code, or the definition is wrong.
+   Example: `exchange_share` reached 25 while its unit says ratio: read the formula
+   before deciding whether the unit or the code is wrong.
+5. **Report; never fix in the same step.** List each finding with its classification
+   and evidence. Fixing a definition is a data-catalog change; fixing code or data is
+   the owner's call. File an issue only when the user asks.
+6. **Make columns checkable at the source.** A finding that needs a fact the catalog
+   lacks (allowed values, a unit) is fixed by adding it to the `Field`, not by
+   special-casing the check.
+   Example: add `values=("是", "否", "无法判定")` to `agent_risk_status`, then rerun.
+
+## Run the checks
+
+```bash
+python3 <engine> run --table usr.dwd_orders --where "dt >= DATE '2026-10-01'" \
+  --key dt --key order_id --runner "python3 tools/presto_query.py"   # or --sqlite warehouse.db
+python3 <engine> plan --table usr.dwd_orders --where "..."   # just the SQL, to run elsewhere
+python3 <engine> judge --results results.json               # judge rows you ran yourself
+```
+
+- Run from the repo root, or pass `--root`.
+- Pass the table's grain as `--key` (one per column): a join that repeats rows breaks
+  it, and nothing else catches that.
+- One table per run when tables use different partition columns.
+- Engine errors such as "exceeds the allowed maximum stages": lower `--batch`.
+- A column type the engine cannot count distinct values of (maps, some arrays): `--skip`.
+- Exit 0: no problems. Exit 1: problems found. Exit 2: setup or usage error.
+
+## Classify each finding
+
+For each problem, read the column's definition and the code that writes it, then give
+one of:
+
+| Class | Means | Example | Next step |
+|---|---|---|---|
+| definition | the code and data agree; the definition is wrong or incomplete | a retention ratio documented as 0–1 that is by design unbounded | fix the `Field` (data-catalog) |
+| code | the code does not compute what the definition says | `transaction_result` hard-coded to `SUCCESS` | report to the owner with the code location |
+| data | the code is right; the source data is wrong or missing | a source that never fills a certificate type | report to the source owner |
+| expected | correct as is, and the definition should say so | a payout-only column that is 0 in a collections-only period | add the reason to the definition so the next run marks it documented |
+
+A column documented as always empty whose `source` names a real upstream column is
+still a question for the owner: is the placeholder meant to stay, or is the code
+missing the read?
+
+Findings marked `(documented)` already match what the definition says (for example
+"always empty"), version and hash columns, and missing-flags that are 0 on every row:
+mention them, do not chase them.
+
+## Report
+
+Give, per table: the range checked, the row count, then each problem as
+`column — finding — class — evidence (file:line or query)`, most serious first.
+End with what you could not classify and why. Never claim a table is clean when a
+check was `not-run` or a column was skipped.
+
+## Done when
+
+- Every problem finding has a class and evidence, or is listed as unclassified.
+- Nothing was written to the warehouse, and no row-level value was printed.

package/skills/app-dev/data-quality/agents/openai.yaml

@@ -0,0 +1,3 @@
+interface:
+  display_name: "Data quality"
+  short_description: "Read-only checks of a warehouse table's data against its data-catalog definitions, with each finding classified"

package/skills/app-dev/data-quality/references/checks.md

@@ -0,0 +1,56 @@
+# Checks, findings, and runners
+
+## What each check reads
+
+The engine builds up to three kinds of query per documented table:
+
+- **Profile** (one per 40 columns, `--batch`): `COUNT(*)`, and per column `COUNT(col)`
+  and `COUNT(DISTINCT col)`; `MIN`/`MAX` for columns whose `unit` is numeric (count,
+  ratio, percent, a currency, seconds, days, score) or whose name looks like a flag
+  (`*_missing`, `*_flag`, `is_*`, `has_*`).
+- **Key** (with `--key`): counts the grain values that repeat, with a GROUP BY …
+  HAVING COUNT(*) > 1 subquery.
+- **Values** (one per column with `values=(...)`): `SELECT col AS value, COUNT(*) AS n …
+  GROUP BY col`. Only declare `values` on enumerated columns, never on identifiers.
+
+## Findings
+
+| Finding | Fires when | Marked documented when |
+|---|---|---|
+| `empty-table` | no rows in the `--where` range; nothing else is judged | — |
+| `never-filled` | the column is empty in every row | the definition says always empty / 始终为空 |
+| `mostly-empty` | filled in under 1% of rows | — |
+| `constant` | one distinct value in every filled row, with at least 30 rows (`--constant-min-rows`) and the column not named in `--where` | the definition says always/constant/恒为; the name is a version, hash, or source table; a flag column is constant at 0 or N |
+| `negative-value` | minimum below 0 for unit count, seconds, or days | — |
+| `out-of-range` | values outside 0–1 for unit `ratio`, or 0–100 for unit `percent` | — |
+| `unseen-value` | a declared value never appears (and the column has rows) | — |
+| `undeclared-value` | a value appears that `values` does not list (empty values are not counted) | — |
+| `duplicate-key` | a `--key` combination appears in more than one row | — |
+| `not-run` | a query has no result | — |
+
+## Making a column checkable
+
+Each check needs a fact in the column's data-catalog `Field`:
+
+| To check | Write in the `Field` |
+|---|---|
+| allowed values | `values=("是", "否", "无法判定")`, or `--   values: 是; 否; 无法判定` in SQL |
+| 0–1 range | `unit="ratio"`; use `"percent"` for 0–100; leave a ratio unbounded by design without that unit |
+| never negative | `unit="count"` (or seconds, days) |
+| expected empty or constant | say so in the text: "Always empty until the source fills it." |
+
+## Runners
+
+`run --runner CMD` sends one SELECT on stdin per query and reads the rows from stdout:
+a JSON array of objects, JSON lines (one object per line), or CSV with a header row.
+Exit non-zero with a message on stderr when a query fails.
+
+| Engine | Runner command shape |
+|---|---|
+| Trino / Presto CLI | `trino --server <url> --catalog hive --output-format JSON` (reads stdin) |
+| Presto REST | a small script that POSTs to `/v1/statement`, follows `nextUri`, prints a JSON array |
+| Spark SQL | a script that runs `spark.sql(stdin).toPandas().to_json(orient="records")` |
+| SQLite (local, tests) | built in: `--sqlite <file>`, opened read-only |
+
+Use `--dialect spark` with Spark SQL (backtick identifiers); the default quotes with
+double quotes. Use read-only credentials where the engine offers them.

package/skills/app-dev/data-quality/release-check.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const WEEK_MS = 7 * DAY_MS;
+const here = dirname(fileURLToPath(import.meta.url));
+
+function readJson(path, fallback = {}) {
+  try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return fallback; }
+}
+
+function frontmatterValue(text, key) {
+  const frontmatter = /^---\n([\s\S]*?)\n---/.exec(text)?.[1] ?? '';
+  const match = new RegExp('^' + key + ':\\s*([^\\n]+)$', 'm').exec(frontmatter);
+  return match?.[1].trim().replace(/^['"]|['"]$/g, '') ?? null;
+}
+
+function repositoryOf(source) {
+  const value = String(source ?? '').trim();
+  const match = /^(?:git@github\.com:|https:\/\/github\.com\/)?([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+?)(?:\.git)?$/.exec(value);
+  return match?.[1] ?? null;
+}
+
+function stableVersion(value) {
+  const match = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(String(value ?? '').trim());
+  return match ? match.slice(1).map(Number) : null;
+}
+
+function newer(left, right) {
+  for (let index = 0; index < 3; index += 1) {
+    if (left[index] !== right[index]) return left[index] > right[index];
+  }
+  return false;
+}
+
+function writeCache(path, value) {
+  try {
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, JSON.stringify(value));
+  } catch { /* A read-only cache must not block skill use. */ }
+}
+
+function main(now = Date.now()) {
+  if (process.env.ONEABOVEIO_NO_UPDATE_CHECK) return;
+  const skill = readFileSync(join(here, 'SKILL.md'), 'utf8');
+  const name = frontmatterValue(skill, 'name');
+  const localVersion = stableVersion(frontmatterValue(skill, 'release-version'));
+  if (!name || !localVersion) return;
+
+  const lock = readJson(join(process.env.HOME || homedir(), '.agents', '.skill-lock.json'));
+  const entry = lock.skills?.[name];
+  const repository = repositoryOf(entry?.sourceUrl || entry?.source);
+  if (!repository) return;
+
+  const cacheRoot = process.env.XDG_CACHE_HOME || join(process.env.HOME || homedir(), '.cache');
+  const cachePath = join(cacheRoot, '1aboveio-skills', 'release-check.json');
+  const all = readJson(cachePath);
+  const cache = all[repository] && typeof all[repository] === 'object' ? all[repository] : {};
+
+  if (!cache.lastCheck || now - cache.lastCheck >= DAY_MS) {
+    const result = spawnSync('gh', ['api', 'repos/' + repository + '/releases/latest', '--jq', '.tag_name'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+      timeout: 1500,
+    });
+    cache.lastCheck = now;
+    if (result.status === 0 && stableVersion(result.stdout)) cache.latestTag = result.stdout.trim();
+    else delete cache.latestTag;
+    all[repository] = cache;
+    writeCache(cachePath, all);
+  }
+
+  const latestVersion = stableVersion(cache.latestTag);
+  if (!latestVersion || !newer(latestVersion, localVersion)) return;
+  if (cache.notifiedTag === cache.latestTag && cache.notifiedAt && now - cache.notifiedAt < WEEK_MS) return;
+
+  cache.notifiedTag = cache.latestTag;
+  cache.notifiedAt = now;
+  all[repository] = cache;
+  writeCache(cachePath, all);
+  const command = `npx skills@1.5.23 add '${repository}#${cache.latestTag}' --global --agent universal claude-code --skill ${name} --yes`;
+  process.stdout.write(
+    `UPDATE_AVAILABLE: A newer 1AboveIO skills release (${cache.latestTag}) is available; this skill is `
+      + `v${localVersion.join('.')}. Ask once whether to refresh ${name}. On approval run \`${command}\`. `
+      + 'The refreshed instructions apply next session. Either way, continue the current task and do not ask again this session.\n',
+  );
+}
+
+try { main(); } catch { /* Update checks are advisory and always fail silent. */ }

package/skills/app-dev/data-quality/scripts/data-quality.test.mjs

@@ -0,0 +1,12 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import test from 'node:test';
+
+const scriptsDir = dirname(fileURLToPath(import.meta.url));
+
+test('data-quality test_data_quality.py', () => {
+  const result = spawnSync('python3', [join(scriptsDir, 'test_data_quality.py')], { encoding: 'utf8' });
+  assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
+});