@1aboveio/skills 0.17.0 → 0.18.0

package/README.md

@@ -68,7 +68,7 @@ planned same-named skill. The coordinator still owns group and new-member consen
 - `engineering-workflow` — Engineering Workflow: e2e-test, ensure-coverage, implement-and-pr, resolve-issues, resolve-release, review-pr, smoke, engineering-runtime
 - `delivery-infrastructure` — Delivery Infrastructure: cloud-build, cloud-debug, cloud-deploy, docker, google-cloud, mergify, podman
 - `backend` — Backend: app-debug, python-backend
-- `data-science` — Data Science: airflow-dag-develop, pyspark
+- `data-science` — Data Science: airflow, pyspark
 - `payment` — Payment: fraud-analysis, payment-analysis
 - `fullstack` — Fullstack: better-auth, monorepo, nextjs-fullstack, prisma-setup, shadcn, zod-v4
 
@@ -123,7 +123,7 @@ npx skills@1.5.22 remove --global --agent universal claude-code --skill cloud-bu
 # Backend
 npx skills@1.5.22 remove --global --agent universal claude-code --skill app-debug python-backend
 # Data Science
-npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop pyspark
+npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow pyspark
 # Payment
 npx skills@1.5.22 remove --global --agent universal claude-code --skill fraud-analysis payment-analysis
 # Fullstack

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1aboveio/skills",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Install the 1AboveIO first-party skill groups through the native Skills CLI.",
   "license": "MIT",
   "type": "module",

package/runtime/skills/distribution/generated/recipes.json

@@ -8,7 +8,7 @@
       "id": "first-party",
       "type": "first-party",
       "package": "@1aboveio/skills",
-      "version": "0.17.0",
+      "version": "0.18.0",
       "updatePolicy": "pinned-npm-version",
       "members": [
         {
@@ -114,8 +114,8 @@
           "licenseResponsibility": "1aboveio-mit"
         },
         {
-          "installName": "airflow-dag-develop",
-          "path": "skills/data-science/airflow-dag-develop",
+          "installName": "airflow",
+          "path": "skills/data-science/airflow",
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
@@ -196,7 +196,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -230,7 +230,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -274,7 +274,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -307,7 +307,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -350,7 +350,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -378,7 +378,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -416,17 +416,17 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
-              "airflow-dag-develop",
+              "airflow",
               "pyspark"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop pyspark"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow pyspark"
               }
             ],
             "onFailure": {
@@ -444,17 +444,17 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
-              "airflow-dag-develop",
+              "airflow",
               "pyspark"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop pyspark"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow pyspark"
               }
             ],
             "onFailure": {
@@ -465,7 +465,7 @@
         ]
       },
       "uninstall": {
-        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop pyspark",
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow pyspark",
         "removesDependencies": false
       }
     },
@@ -482,7 +482,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -510,7 +510,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -548,7 +548,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -580,7 +580,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -620,7 +620,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -646,7 +646,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.17.0",
+            "version": "0.18.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -678,7 +678,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.17.0",
+          "version": "0.18.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -699,7 +699,7 @@
             "podman",
             "app-debug",
             "python-backend",
-            "airflow-dag-develop",
+            "airflow",
             "pyspark",
             "fraud-analysis",
             "payment-analysis",
@@ -713,7 +713,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
             }
           ],
           "onFailure": {
@@ -731,7 +731,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.17.0",
+          "version": "0.18.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -752,7 +752,7 @@
             "podman",
             "app-debug",
             "python-backend",
-            "airflow-dag-develop",
+            "airflow",
             "pyspark",
             "fraud-analysis",
             "payment-analysis",
@@ -766,7 +766,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
             }
           ],
           "onFailure": {
@@ -777,7 +777,7 @@
       ]
     },
     "uninstall": {
-      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 harness-runtime"
+      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 harness-runtime"
     }
   }
 }

package/runtime/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs

@@ -97,7 +97,7 @@ export const WORKFLOW_TRUSTED_SOURCES = deepFreeze({
     id: 'first-party',
     type: 'first-party',
     package: '@1aboveio/skills',
-    version: '0.17.0',
+    version: '0.18.0',
   },
   matt: {
     id: 'matt-pocock',

package/skills/cicd-pipeline/mergify/references/watch-contract.md

@@ -24,7 +24,7 @@ So the watcher's memory *is* the verdict. Four pieces of state must survive acro
 
 1. **`seenQueued`** — has queue entry ever been observed? Every downstream transition depends on it. Set from the `queued` label **or** `mergify queue show`.
 2. **A re-read counter** (`missingActiveReads` / `absentStreak`) — one absence is suspicion, two consecutive *readable* absences settle `DEQUEUED`.
-3. **Readable vs. absent** — a failed `gh`/`mergify` call is *no information* and must not advance that counter.
+3. **Readable vs. absent** — a failed `gh`/`mergify` call is *no information*: it must not advance that counter or overwrite that source's last readable contribution to the stall-state snapshot. During a partial outage, the still-readable sibling remains informative and genuine progress from it still resets the change clock.
 4. **A change clock** (`lastChangeMs`) — the stall bound is *no observable change*, not elapsed time.
 
 ### Why a per-poll predicate cannot satisfy this
@@ -160,7 +160,7 @@ When the reason is not machine-readable, quote the Mergify status comment, queue
 ## Watcher Requirements
 
 - Run as one background task or one script invocation; do not spend foreground turns polling.
-- Retry transient `gh`, GraphQL, and Mergify CLI failures. EOF, TLS handshake timeouts, and temporary API failures are not dequeue signals — an unreadable read is *no information*, not a "not queued" observation, and must not advance the counter that settles `DEQUEUED`.
+- Retry transient `gh`, GraphQL, and Mergify CLI failures. EOF, TLS handshake timeouts, and temporary API failures are not dequeue signals — an unreadable read is *no information*, not a "not queued" observation, and must not advance the counter that settles `DEQUEUED`. Retain that source's last readable stall-state contribution instead; a readable sibling can still report genuine progress and reset the change clock.
 - Bound every subprocess read on its own timeout, in addition to the stall/ceiling bounds below — those are only evaluated after a poll returns, so an unbounded read can hang the watcher past both of them.
 - **Run the bundled script. Do not hand-roll a watcher** — see the progression section above for why the hand-rolled shape cannot hold the contract, whatever care goes into the code. (If you are reading a legacy shell watcher: avoid shell-reserved variable names such as `status`.)
 - Keep enqueue and watch separate. A watch observes only; it does not post `@mergifyio queue`, `@mergifyio dequeue`, or merge.

package/skills/cicd-pipeline/mergify/scripts/watch-pr-delivery-core.mjs

@@ -165,6 +165,50 @@ export function observationKey({ pr = {}, queue = {} } = {}) {
   ])
 }
 
+function stallPrSnapshot(pr = {}) {
+  return {
+    state: pr.state ?? null,
+    mergedAt: pr.mergedAt ?? null,
+    headRefOid: pr.headRefOid ?? null,
+    labels: Array.isArray(pr.labels) ? [...pr.labels] : [],
+    failedChecks: Array.isArray(pr.failedChecks) ? pr.failedChecks.map(({ name, status }) => ({ name, status })) : [],
+    pendingChecks: Array.isArray(pr.pendingChecks) ? pr.pendingChecks.map(({ name, status }) => ({ name, status })) : [],
+  }
+}
+
+function stallQueueSnapshot(queue = {}) {
+  return {
+    queued: queue.queued ?? null,
+    queueState: queue.queueState ?? queue.state ?? null,
+    position: queue.position ?? null,
+  }
+}
+
+// Retain each source's last readable contribution independently. A failed read carries no
+// replacement state, while a readable sibling can still move the effective stall snapshot.
+export function nextStallObservation({ cache = {}, pr = {}, queue = {} } = {}) {
+  const previous = cache && typeof cache === 'object' ? cache : {}
+  const prReadable = pr.readable !== false
+  const queueReadable = queue.readable !== false
+  const nextPr = prReadable ? stallPrSnapshot(pr) : (previous.pr ?? null)
+  const nextQueue = queueReadable ? stallQueueSnapshot(queue) : (previous.queue ?? null)
+
+  if (!prReadable && !queueReadable) {
+    return {
+      cache: { pr: nextPr, queue: nextQueue, key: previous.key ?? null },
+      key: previous.key ?? null,
+      changed: false,
+    }
+  }
+
+  const key = observationKey({ pr: nextPr ?? {}, queue: nextQueue ?? {} })
+  return {
+    cache: { pr: nextPr, queue: nextQueue, key },
+    key,
+    changed: key !== (previous.key ?? null),
+  }
+}
+
 // A failed subprocess read (`readable: false`) carries no information about queue membership
 // — it is not evidence the PR is still queued, and it is not evidence it left the queue
 // either. Only a clean read that reports absence should move the counter that eventually
@@ -609,7 +653,7 @@ export async function runCli(argv) {
   let seenQueued = args.seenQueued === true
   let everActive = false
   let missingActiveReads = 0
-  let lastKey = null
+  let stallObservationCache = null
   let lastChangeMs = startedMs
   let observations = 0
   let phases = []
@@ -675,10 +719,10 @@ export async function runCli(argv) {
       return verdict.outcome === 'MERGED' ? 0 : 4
     }
 
-    const key = observationKey({ pr, queue })
-    const changed = key !== lastKey
+    const stallObservation = nextStallObservation({ cache: stallObservationCache, pr, queue })
+    stallObservationCache = stallObservation.cache
+    const changed = stallObservation.changed
     if (changed) {
-      lastKey = key
       lastChangeMs = Date.now()
     }
 

package/skills/data-science/airflow/SKILL.md

@@ -0,0 +1,198 @@
+---
+name: airflow
+description: Airflow DAG architecture and conventions. Use when designing, writing, reviewing, testing, or debugging Airflow DAGs and tasks, including TaskGroups, data intervals, timezone-aware date parameters, CLI validation, environment-based scheduling, and pytest integrity tests.
+user-invocable: true
+---
+
+# Airflow DAG Development Patterns
+
+## 1. Organize Data Pipelines with TaskGroups
+
+Use `TaskGroup` to make meaningful pipeline stages visible in the Airflow graph. For data warehouse pipelines, group tasks by warehouse layer such as `stg`, `ods`, `dwd`, `dws`, and `ads`. For other pipelines, use domain-specific groups such as `features`, `exports`, `quality_checks`, or `notifications`.
+
+Keep groups aligned with real ownership or dependency boundaries; do not create a group solely to wrap one trivial task. Express dependencies between groups so the graph communicates the pipeline's high-level flow.
+
+```python
+from airflow.decorators import dag
+from airflow.utils.task_group import TaskGroup
+
+
+@dag(...)
+def customer_metrics():
+    with TaskGroup(group_id="stg") as stg:
+        load_stg_customers()
+        load_stg_orders()
+
+    with TaskGroup(group_id="dwd") as dwd:
+        build_dwd_customers()
+        build_dwd_orders()
+
+    with TaskGroup(group_id="ads") as ads:
+        build_ads_customer_metrics()
+
+    stg >> dwd >> ads
+
+
+customer_metrics()
+```
+
+## 2. Use Timezone-Aware Data Intervals
+
+Prefer Airflow's `data_interval_start` and `data_interval_end` as task date-range inputs. They are timezone-aware logical boundaries and behave correctly for scheduled runs, catchup, and daylight-saving transitions. Treat `data_interval_end` as an exclusive boundary.
+
+Prepare one shared set of templated date variables for the whole DAG, converting the interval boundaries to the DAG's business timezone. Pass those variables to every task that reads or writes partitioned data. Keep the conversion in Jinja so it happens for each DAG run, not while the scheduler parses the DAG file.
+
+```python
+import pendulum
+from airflow.decorators import dag, task
+from airflow.utils.task_group import TaskGroup
+
+LOCAL_TIMEZONE = "Asia/Shanghai"
+
+# These values are rendered separately for every DAG run. The end is exclusive.
+DATE_RANGE = {
+    "start_date": (
+        f"{{{{ data_interval_start.in_timezone('{LOCAL_TIMEZONE}') | ds }}}}"
+    ),
+    "end_date": (
+        f"{{{{ data_interval_end.in_timezone('{LOCAL_TIMEZONE}') | ds }}}}"
+    ),
+}
+
+
+@dag(
+    schedule="0 2 * * *",
+    start_date=pendulum.datetime(2024, 1, 1, tz=LOCAL_TIMEZONE),
+    catchup=True,
+)
+def daily_warehouse():
+    @task
+    def load_partition(start_date: str, end_date: str) -> None:
+        # Read records in the half-open range [start_date, end_date).
+        ...
+
+    with TaskGroup(group_id="stg"):
+        load_partition(**DATE_RANGE)
+
+
+daily_warehouse()
+```
+
+Use timestamps instead of dates when a task needs sub-day precision:
+
+```python
+INTERVAL_RANGE = {
+    "interval_start": "{{ data_interval_start.in_timezone('Asia/Shanghai').to_iso8601_string() }}",
+    "interval_end": "{{ data_interval_end.in_timezone('Asia/Shanghai').to_iso8601_string() }}",
+}
+```
+
+## 3. Operations: Wait for DAG Auto-Reload
+
+Airflow periodically scans the DAG directory and automatically reloads changed DAG files. The refresh cadence depends on the deployment's scheduler and DAG-processing configuration, and commonly takes a few minutes.
+
+After updating a DAG file, do not restart the Airflow services by default. Watch for the DAG to refresh in the Airflow UI or CLI after the configured scan interval. Investigate import errors or deployment synchronization only if the update does not appear after a reasonable interval.
+
+## 4. Quick DAG Validation with Airflow CLI
+
+Use these commands for fast feedback during development:
+
+```bash
+# List import errors (fastest check - catches syntax and import issues)
+airflow dags list-import-errors
+
+# Full lint check (comprehensive - catches best practice violations)
+airflow dags lint
+```
+
+**When to use:**
+- `list-import-errors`: During active development for quick syntax/import validation
+- `lint`: Before commits to catch style and best practice issues
+
+## 5. Environment-Based Schedule Override
+
+Disable schedules in local development to prevent accidental runs:
+
+```python
+import os
+from datetime import datetime
+from airflow import DAG
+
+def get_schedule(production_schedule: str) -> str | None:
+    """Return None for local dev, actual schedule for deployed environments.
+
+    Set AIRFLOW_ENV=local in .env for local development.
+    Cloud Run deployments set AIRFLOW_ENV=dev|staging|prod.
+    """
+    env = os.getenv("AIRFLOW_ENV", "local")
+    if env == "local":
+        return None
+    return production_schedule
+```
+
+**Usage in DAG:**
+```python
+with DAG(
+    dag_id="my_dag",
+    schedule=get_schedule("0 4 * * *"),  # Daily 4 AM in prod, None locally
+    start_date=datetime(2024, 1, 1),
+    catchup=False,
+) as dag:
+    ...
+```
+
+**Environment setup:**
+- **Local:** Add `AIRFLOW_ENV=local` to `.env`
+- **Cloud Run:** Set via cloudbuild.yaml substitutions (`_AIRFLOW_ENV: dev|staging|prod`)
+
+## 6. Pytest DAG Integrity Tests
+
+Standard pytest patterns for validating DAG quality:
+
+```python
+# tests/test_dag_integrity.py
+import pytest
+from airflow.models import DagBag
+
+@pytest.fixture(scope="session")
+def dagbag():
+    return DagBag(dag_folder="dags/", include_examples=False)
+
+def test_no_import_errors(dagbag):
+    """Verify all DAGs load without import errors."""
+    assert not dagbag.import_errors, f"DAG import errors: {dagbag.import_errors}"
+
+def test_dags_have_tags(dagbag):
+    """Verify all DAGs have at least one tag for organization."""
+    for dag_id, dag in dagbag.dags.items():
+        assert dag.tags, f"DAG {dag_id} has no tags"
+
+def test_dags_have_owner(dagbag):
+    """Verify all DAGs have an owner in default_args."""
+    for dag_id, dag in dagbag.dags.items():
+        assert dag.default_args.get("owner"), f"DAG {dag_id} missing owner"
+
+def test_no_cycles(dagbag):
+    """Verify DAGs don't have circular dependencies."""
+    for dag_id, dag in dagbag.dags.items():
+        # DagBag already validates this, but explicit test for clarity
+        assert dag.topo_sort(), f"DAG {dag_id} has cycles"
+```
+
+**Run tests:**
+```bash
+# Run all DAG tests
+pytest tests/test_dag_integrity.py -v
+
+# Run specific test
+pytest tests/test_dag_integrity.py::test_no_import_errors -v
+```
+
+## Quick Reference
+
+| Task | Command |
+|------|---------|
+| Check import errors | `airflow dags list-import-errors` |
+| Lint DAGs | `airflow dags lint` |
+| Run integrity tests | `pytest tests/test_dag_integrity.py -v` |
+| Trigger manual run | `airflow dags trigger <dag_id>` |

package/skills/data-science/pyspark/assets/templates/etl.py

@@ -344,6 +344,7 @@ class Etl(ABC):
             "hoodie.datasource.write.keygenerator.class": "org.apache.hudi.keygen.ComplexKeyGenerator",
             "hoodie.datasource.write.precombine.field": self.ts,
             "hoodie.datasource.write.operation": f"{operation}",
+            "hoodie.bulkinsert.sort.mode": "PARTITION_PATH_REPARTITION_AND_SORT",
             "hoodie.datasource.write.reconcile.schema": True,
             # Do not set hoodie.*.shuffle.parallelism or spark.sql.shuffle.partitions:
             # each shuffle task becomes a tiny Hudi file.

package/skills/data-science/pyspark/references/etl-contract.md

@@ -98,6 +98,9 @@ belong before the writer in the job process.
   `hoodie.bulkinsert.shuffle.parallelism`. A raised shuffle partition count
   becomes one small file per task on the write path. Size files with
   `HudiLayout` (default one ~128MB file per partition), not shuffle parallelism.
+- Keep bulk inserts physically grouped and sorted by partition with
+  `hoodie.bulkinsert.sort.mode=PARTITION_PATH_REPARTITION_AND_SORT`; override it
+  through `extra_hudi_options` only when the job contract requires another layout.
 
 ### Repair small files
 

package/skills/engineering/engineering-runtime/coherence/workflow.json

@@ -361,7 +361,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "eff6c7b5931bce5b2265a619bccddd371a89df6ce7bc2edc74f11d00060a71dd",
       "digestExcludes": []
@@ -374,7 +374,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "08230dc57a53d6526692b50138038abbd80abce1a067e01c52ad6acc182caf7e",
       "digestExcludes": []
@@ -387,7 +387,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "9eea7bfba348ddaea1b934c9a7fabdd9b101df8a146e1c93e947da4d335f1d98",
       "digestExcludes": []
@@ -400,7 +400,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "58b0556228a9271cf9e33727a05fc08a4fdfd29512ec0936b169f0a55d60e6ef",
       "digestExcludes": []
@@ -413,9 +413,9 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
-      "contentDigest": "0b2d8e0c4d0ae6336c733ff2102d6911a3bd4298394eddd9f40eefaeb884903c",
+      "contentDigest": "93241488e62da366d4ecef2a9188a7d0d89f2a5d6db78ad96f0f03bd05b43915",
       "digestExcludes": []
     },
     {
@@ -426,9 +426,9 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
-      "contentDigest": "b5af4dccf703362d4f41cac4fdff48305f652a00338d85975a2e5c35ec6bc61c",
+      "contentDigest": "0e770c0f20893330214a3e004dd56cad583707603e22c733e36e7276ef8a79ae",
       "digestExcludes": []
     },
     {
@@ -439,7 +439,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "f048fd00c69f2dc666fc7a3096cfeee6dfba73933bfb69602f3f0e26159798cc",
       "digestExcludes": []
@@ -452,7 +452,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
       "contentDigest": "90a7c4e1ad6da1e632ea2c5259e4967ffebb84353adccbca8dfc5d6bc50b60c1",
       "digestExcludes": []
@@ -465,15 +465,15 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       },
-      "contentDigest": "4d2242d41c7c57e24119b7cd974a88dd97cf526a8f76e6825385060ff86095c8",
+      "contentDigest": "2a4193c1b18220520723e145abbdd742f0ea6a93bba57a50b488bf69f95219d3",
       "digestExcludes": [
         "coherence/workflow.json"
       ]
     }
   ],
-  "releaseIdentity": "4a75f070c64c9f24bfbb250be0706a55f6fc3bf071d537200a0a5a91378a4959",
+  "releaseIdentity": "2368bd82f8a75798333be8a958c7ce11398839a3dcbe7ee4dbb68d621752a11a",
   "lifecycleAuthority": "native-skills-cli",
   "repairRecipe": {
     "id": "engineering-workflow-dependency-first",
@@ -531,7 +531,7 @@
         "sourceId": "first-party",
         "sourceType": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0",
+        "version": "0.18.0",
         "installPath": null,
         "members": [
           "harness-runtime",
@@ -547,7 +547,7 @@
         "commands": [
           {
             "transport": "npm",
-            "command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
+            "command": "npx @1aboveio/skills@0.18.0 install --group engineering-workflow --yes"
           }
         ],
         "onFailure": {

package/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs

@@ -97,7 +97,7 @@ export const WORKFLOW_TRUSTED_SOURCES = deepFreeze({
     id: 'first-party',
     type: 'first-party',
     package: '@1aboveio/skills',
-    version: '0.17.0',
+    version: '0.18.0',
   },
   matt: {
     id: 'matt-pocock',

package/skills/engineering/resolve-issues/generated/workflow-repair-policy.json

@@ -237,7 +237,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -248,7 +248,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -259,7 +259,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -270,7 +270,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -281,7 +281,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -292,7 +292,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -303,7 +303,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -314,7 +314,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     },
     {
@@ -325,7 +325,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0"
+        "version": "0.18.0"
       }
     }
   ],
@@ -426,7 +426,7 @@
         "sourceId": "first-party",
         "sourceType": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.17.0",
+        "version": "0.18.0",
         "installPath": null,
         "members": [
           "harness-runtime",
@@ -442,7 +442,7 @@
         "commands": [
           {
             "transport": "npm",
-            "command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
+            "command": "npx @1aboveio/skills@0.18.0 install --group engineering-workflow --yes"
           }
         ],
         "onFailure": {

package/skills/engineering/resolve-issues/scripts/run-state.mjs

@@ -3412,7 +3412,7 @@ export const WORKFLOW_PREFLIGHT_COMMANDS = Object.freeze([
 
 const WORKFLOW_VERIFIER_URL = new URL('../../engineering-runtime/scripts/workflow-coherence.mjs', import.meta.url)
 const WORKFLOW_FALLBACK_POLICY_URL = new URL('../generated/workflow-repair-policy.json', import.meta.url)
-export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = 'a6d80d500e43b660220f71f9764e5e6c905bf1955d63f8733d115b6c1a601448'
+export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = 'c3d9fac9486de96cb064da61f2bfcb07d4ebc9935be804cfac0c0b9f743c7a30'
 const WORKFLOW_REPAIR_RECIPE_REFERENCE = Object.freeze({
   id: 'engineering-workflow-dependency-first',
   generatedFrom: 'skills/distribution/generated/recipes.json',

package/skills/engineering/resolve-release/SKILL.md

@@ -1,6 +1,7 @@
 ---
 name: resolve-release
-description: "Release-lane orchestrator, from merge authorization to verified-in-production: assemble PRs into a release branch, cut version + CHANGELOG, promotion PR, deploy a candidate at 0% traffic, validate it, shift traffic, verify the live revision, tag and publish notes (or delete the candidate / roll back). Input is resolve-issues' handoff. NOT for driving issues to PRs (resolve-issues), pipeline debugging (cloud-build/cloud-deploy/cloud-debug), or broken production (app-debug)."
+description: "Slash/explicit-only (/resolve-release). Release-lane orchestrator, from merge authorization to verified-in-production: assemble PRs into a release branch, cut version + CHANGELOG, promotion PR, deploy a candidate at 0% traffic, validate it, shift traffic, verify the live revision, tag and publish notes (or delete the candidate / roll back). Do not auto-select. Input is resolve-issues' handoff. NOT for driving issues to PRs (resolve-issues), pipeline debugging (cloud-build/cloud-deploy/cloud-debug), or broken production (app-debug)."
+disable-model-invocation: true
 dependencies:
   - review-pr
   - e2e-test

package/skills/engineering/resolve-release/agents/openai.yaml

@@ -0,0 +1,9 @@
+interface:
+  display_name: "Resolve Release"
+  short_description: "Slash/explicit-only release lane: merge-ready PRs to verified production"
+
+policy:
+  # Codex counterpart to SKILL.md disable-model-invocation: true
+  # (Claude Code / Pi). Keeps $resolve-release / explicit invoke; blocks
+  # description-based auto-selection.
+  allow_implicit_invocation: false

package/skills/fullstack/zod-v4/SKILL.md

@@ -22,7 +22,7 @@ Use these skills by name when the Zod change is part of a larger stack-specific
 - `web-design-guidelines` for UI review after validation-driven form changes
 - `vercel-react-best-practices` for React/Next.js performance-sensitive form patterns
 - `app-debug` for runtime failures caused by validation or payload mismatches
-- `airflow-dag-develop` only when Zod-related config or payload contracts affect DAG tooling
+- `airflow` only when Zod-related config or payload contracts affect DAG tooling
 
 ---
 

package/skills/data-science/airflow-dag-develop/SKILL.md

@@ -1,111 +0,0 @@
----
-name: airflow-dag-develop
-description: Airflow DAG architecture and conventions. Read when designing, creating, or debugging DAGs — covers CLI validation, environment-based scheduling, and pytest integrity tests.
-user-invocable: true
----
-
-# Airflow DAG Development Patterns
-
-## 1. Quick DAG Validation with Airflow CLI
-
-Use these commands for fast feedback during development:
-
-```bash
-# List import errors (fastest check - catches syntax and import issues)
-airflow dags list-import-errors
-
-# Full lint check (comprehensive - catches best practice violations)
-airflow dags lint
-```
-
-**When to use:**
-- `list-import-errors`: During active development for quick syntax/import validation
-- `lint`: Before commits to catch style and best practice issues
-
-## 2. Environment-Based Schedule Override
-
-Disable schedules in local development to prevent accidental runs:
-
-```python
-import os
-from datetime import datetime
-from airflow import DAG
-
-def get_schedule(production_schedule: str) -> str | None:
-    """Return None for local dev, actual schedule for deployed environments.
-
-    Set AIRFLOW_ENV=local in .env for local development.
-    Cloud Run deployments set AIRFLOW_ENV=dev|staging|prod.
-    """
-    env = os.getenv("AIRFLOW_ENV", "local")
-    if env == "local":
-        return None
-    return production_schedule
-```
-
-**Usage in DAG:**
-```python
-with DAG(
-    dag_id="my_dag",
-    schedule=get_schedule("0 4 * * *"),  # Daily 4 AM in prod, None locally
-    start_date=datetime(2024, 1, 1),
-    catchup=False,
-) as dag:
-    ...
-```
-
-**Environment setup:**
-- **Local:** Add `AIRFLOW_ENV=local` to `.env`
-- **Cloud Run:** Set via cloudbuild.yaml substitutions (`_AIRFLOW_ENV: dev|staging|prod`)
-
-## 3. Pytest DAG Integrity Tests
-
-Standard pytest patterns for validating DAG quality:
-
-```python
-# tests/test_dag_integrity.py
-import pytest
-from airflow.models import DagBag
-
-@pytest.fixture(scope="session")
-def dagbag():
-    return DagBag(dag_folder="dags/", include_examples=False)
-
-def test_no_import_errors(dagbag):
-    """Verify all DAGs load without import errors."""
-    assert not dagbag.import_errors, f"DAG import errors: {dagbag.import_errors}"
-
-def test_dags_have_tags(dagbag):
-    """Verify all DAGs have at least one tag for organization."""
-    for dag_id, dag in dagbag.dags.items():
-        assert dag.tags, f"DAG {dag_id} has no tags"
-
-def test_dags_have_owner(dagbag):
-    """Verify all DAGs have an owner in default_args."""
-    for dag_id, dag in dagbag.dags.items():
-        assert dag.default_args.get("owner"), f"DAG {dag_id} missing owner"
-
-def test_no_cycles(dagbag):
-    """Verify DAGs don't have circular dependencies."""
-    for dag_id, dag in dagbag.dags.items():
-        # DagBag already validates this, but explicit test for clarity
-        assert dag.topo_sort(), f"DAG {dag_id} has cycles"
-```
-
-**Run tests:**
-```bash
-# Run all DAG tests
-pytest tests/test_dag_integrity.py -v
-
-# Run specific test
-pytest tests/test_dag_integrity.py::test_no_import_errors -v
-```
-
-## Quick Reference
-
-| Task | Command |
-|------|---------|
-| Check import errors | `airflow dags list-import-errors` |
-| Lint DAGs | `airflow dags lint` |
-| Run integrity tests | `pytest tests/test_dag_integrity.py -v` |
-| Trigger manual run | `airflow dags trigger <dag_id>` |