@1aboveio/skills 0.10.1 → 0.11.0

package/README.md

@@ -43,7 +43,7 @@ Declining leaves the existing content and its links exactly as they were.
 
 - `engineering-workflow` — Engineering Workflow: e2e-test, ensure-coverage, implement-and-pr, resolve-issues, resolve-release, review-pr, smoke, engineering-runtime
 - `delivery-infrastructure` — Delivery Infrastructure: cloud-build, cloud-debug, cloud-deploy, docker, google-cloud, mergify, podman
-- `backend` — Backend: airflow-dag-develop, app-debug, python-backend
+- `backend` — Backend: airflow-dag-develop, app-debug, pyspark, python-backend
 - `fullstack` — Fullstack: better-auth, monorepo, nextjs-fullstack, prisma-setup, shadcn, zod-v4, impeccable — impeccable is fetched from its upstream owner and not included in this package
 - `matt-pocock-toolkit` — Matt Pocock Toolkit: batch-grill-me, code-review, codebase-design, diagnosing-bugs, domain-modeling, grill-me, grill-with-docs, grilling, handoff, improve-codebase-architecture, prototype, research, resolving-merge-conflicts, setup-matt-pocock-skills, tdd, to-spec, to-tickets, triage, wayfinder — every member is fetched from its upstream owner and is not included in this package
 
@@ -69,7 +69,7 @@ npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test
 # Delivery Infrastructure
 npx skills@1.5.22 remove --global --agent universal claude-code --skill cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman
 # Backend
-npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug python-backend
+npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend
 # Fullstack
 npx skills@1.5.22 remove --global --agent universal claude-code --skill better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable
 # Matt Pocock Toolkit

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1aboveio/skills",
-  "version": "0.10.1",
+  "version": "0.11.0",
   "description": "Install the 1AboveIO first-party skill groups through the native Skills CLI.",
   "license": "MIT",
   "type": "module",

package/runtime/skills/distribution/generated/recipes.json

@@ -8,7 +8,7 @@
       "id": "first-party",
       "type": "first-party",
       "package": "@1aboveio/skills",
-      "version": "0.10.1",
+      "version": "0.11.0",
       "updatePolicy": "pinned-npm-version",
       "members": [
         {
@@ -113,6 +113,12 @@
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
+        {
+          "installName": "pyspark",
+          "path": "skills/backend/pyspark",
+          "role": "skill",
+          "licenseResponsibility": "1aboveio-mit"
+        },
         {
           "installName": "python-backend",
           "path": "skills/backend/python-backend",
@@ -448,7 +454,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -520,7 +526,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -564,7 +570,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -597,7 +603,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -640,18 +646,19 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
               "airflow-dag-develop",
               "app-debug",
+              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
               }
             ],
             "onFailure": {
@@ -669,18 +676,19 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
               "airflow-dag-develop",
               "app-debug",
+              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
               }
             ],
             "onFailure": {
@@ -691,7 +699,7 @@
         ]
       },
       "uninstall": {
-        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug python-backend",
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend",
         "removesDependencies": false
       }
     },
@@ -708,7 +716,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -760,7 +768,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -916,7 +924,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -942,7 +950,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.1",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -1012,7 +1020,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.10.1",
+          "version": "0.11.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -1033,6 +1041,7 @@
             "podman",
             "airflow-dag-develop",
             "app-debug",
+            "pyspark",
             "python-backend",
             "better-auth",
             "monorepo",
@@ -1044,7 +1053,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
             }
           ],
           "onFailure": {
@@ -1120,7 +1129,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.10.1",
+          "version": "0.11.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -1141,6 +1150,7 @@
             "podman",
             "airflow-dag-develop",
             "app-debug",
+            "pyspark",
             "python-backend",
             "better-auth",
             "monorepo",
@@ -1152,7 +1162,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
             }
           ],
           "onFailure": {
@@ -1183,7 +1193,7 @@
       ]
     },
     "uninstall": {
-      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable batch-grill-me code-review codebase-design diagnosing-bugs domain-modeling grill-me grill-with-docs grilling handoff improve-codebase-architecture prototype research resolving-merge-conflicts setup-matt-pocock-skills tdd to-spec to-tickets triage wayfinder harness-runtime"
+      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable batch-grill-me code-review codebase-design diagnosing-bugs domain-modeling grill-me grill-with-docs grilling handoff improve-codebase-architecture prototype research resolving-merge-conflicts setup-matt-pocock-skills tdd to-spec to-tickets triage wayfinder harness-runtime"
     }
   }
 }

package/skills/backend/pyspark/LICENSE

@@ -0,0 +1,3 @@
+MIT License
+
+Copyright (c) 2026 1AboveIO

package/skills/backend/pyspark/SKILL.md

@@ -0,0 +1,116 @@
+---
+name: pyspark
+description: Implement, debug, review, and optimize Apache Spark or PySpark batch jobs, ETL pipelines, feature calculations, and warehouse writes. Use whenever work involves Spark DataFrames or SQL, joins, windows, grouped state, UDFs, shuffle, skew, spill, slow stages, Spark Connect, spark-submit, Hudi publication, backfills, semantic parity, or production canaries, even when the request is framed only as a warehouse timeout or performance regression.
+---
+
+# PySpark
+
+Use this skill as the workflow and routing index. Read the referenced playbooks
+that match the task before editing code or running a production-scale job.
+
+## Workflow
+
+```text
+contract -> transform -> materialize -> test -> CI/CD
+```
+
+Follow all five steps in order.
+
+### 1. Contract
+
+- **Input:** declare all parameters, defaults, validation, source snapshot/date
+  scope, overlapping-run behavior, idempotency/single-writer rules, and exact
+  Python and PySpark versions.
+- **Output:** declare plain Spark versus Hudi table type, output identity/schema,
+  partition columns, record key, deterministic order or precombine field, and
+  append/upsert/overwrite write mode.
+- **Conventions:** declare native Spark versus Spark Connect and full rebuild
+  versus incremental processing, including the incremental watermark,
+  lookback/checkpoint, and restart boundary when applicable.
+
+Read [ETL Contract](references/etl-contract.md). When the repository has no ETL
+base module, adapt [`assets/templates/etl.py`](assets/templates/etl.py).
+
+### 2. Transform
+
+- Make `transform` accept DataFrames and return the complete candidate without
+  invoking validation or a writer.
+- Estimate rows after joins/explodes/groups and measure maximum rows per key.
+  Fix algorithmic expansion and hot keys before tuning Spark configuration.
+- Prefer built-in expressions and windows. Use exact bucketing or one sorted
+  grouped sweep only for state that built-ins cannot express cleanly.
+- Preserve complete row identity, ordering, output columns, and required
+  behavior.
+- Avoid explicit `.cache()` and `.persist()` by default.
+
+Read [Transformation Design](references/transformation-design.md). For temporal
+windows or FIFO paths, also read
+[Velocity Feature Calculation](references/velocity-feature-calculation.md).
+
+### 3. Materialize
+
+- Treat materialization as the Spark action that executes the lazy candidate,
+  not as caching.
+- Use one aggregate validation action before every writer. Reference every
+  derived output so Catalyst cannot prune untested calculations.
+- Check row count, distinct/null/duplicate identities, domain blockers, and
+  required date/partition scope. Fail closed on any blocker.
+- Bind validation and publication to the same immutable source snapshot because
+  Spark may recompute the candidate.
+- Keep the validate-only guard inside the job and before every writer. Do not
+  add `.count()`, `.cache()`, or `.persist()` to warm execution.
+
+Read [Production Validation](references/production-validation.md).
+
+### 4. Test
+
+1. **Local test:** run pure state, tiny real-Spark, plan-shape, focused repository,
+   and bidirectional parity tests. Cover boundaries, ties, nulls, arithmetic,
+   identity, and precedence.
+2. **Cluster canary test:** run the production entry point and source scope with
+   `--validate-only`. Require zero blockers, an explicit publication skip, zero
+   Spark output records/bytes, and readable driver/event logs.
+3. **Profiling:** measure an isolated baseline, parse the canary event log, and
+   compare both runs on the same source scope and resource profile. Record
+   wall/stage/task time, cardinality, shuffle, spill, skew, output, and target
+   plan patterns.
+
+```bash
+spark-submit <production-options> job.py --validate-only
+set -o pipefail
+hdfs dfs -cat "$EVENT_LOG" | \
+  python {baseDir}/scripts/spark_eventlog_summary.py > canary-summary.json
+```
+
+Reject missing evidence, incomparable scope, unexplained regressions, or a
+missed performance gate. Keep validate-only time separate from write time.
+
+Read [Parity Testing](references/parity-testing.md),
+[Diagnosis And Profiling](references/diagnosis-and-profiling.md), and
+[Production Validation](references/production-validation.md).
+
+### 5. CI/CD
+
+1. Open a PR with required-behavior, test, canary, profiling, and risk evidence.
+2. Run required CI and independent review; fix failures.
+3. Enqueue through the configured merge queue and merge without bypassing gates.
+4. Deploy only the merged commit through the normal deployment path.
+5. Run write-enabled production validation only with explicit authorization;
+   reconcile counts, identity, schema, coverage, blockers, readability, write
+   mode, and commit evidence independently.
+6. Restore scheduler/process ownership and report compute/validation,
+   publication, and end-to-end time separately.
+
+Read [Production Validation](references/production-validation.md).
+
+## Reference Index
+
+| Need | Read or run |
+|---|---|
+| Define or review an ETL job contract | [ETL Contract](references/etl-contract.md) and `assets/templates/etl.py` |
+| Diagnose or profile | [Diagnosis And Profiling](references/diagnosis-and-profiling.md) |
+| Design transformations | [Transformation Design](references/transformation-design.md) |
+| Optimize temporal/FIFO state | [Velocity Feature Calculation](references/velocity-feature-calculation.md) |
+| Prove parity | [Parity Testing](references/parity-testing.md) |
+| Run canaries or writes | [Production Validation](references/production-validation.md) |
+| Summarize an event log | `scripts/spark_eventlog_summary.py` |