@1aboveio/skills 0.10.0 → 0.11.0

package/README.md

@@ -32,6 +32,10 @@ Running `install` with no `--group` opens the interactive bundle picker. The aut
 - `--dry-run` — print the planned native commands, in order, and run none of them.
 - `--yes` — answer the native CLI's prompts, including the overwrite prompt, with yes.
 
+Selecting several groups (interactive picker, repeated `--group`, or `--all`) coalesces to one
+native `skills add` per upstream source, so the installation-method prompt appears once per
+source rather than once per group.
+
 Without `--yes`, an existing same-named skill produces the native overwrite summary and a prompt.
 Declining leaves the existing content and its links exactly as they were.
 
@@ -39,7 +43,7 @@ Declining leaves the existing content and its links exactly as they were.
 
 - `engineering-workflow` — Engineering Workflow: e2e-test, ensure-coverage, implement-and-pr, resolve-issues, resolve-release, review-pr, smoke, engineering-runtime
 - `delivery-infrastructure` — Delivery Infrastructure: cloud-build, cloud-debug, cloud-deploy, docker, google-cloud, mergify, podman
-- `backend` — Backend: airflow-dag-develop, app-debug, python-backend
+- `backend` — Backend: airflow-dag-develop, app-debug, pyspark, python-backend
 - `fullstack` — Fullstack: better-auth, monorepo, nextjs-fullstack, prisma-setup, shadcn, zod-v4, impeccable — impeccable is fetched from its upstream owner and not included in this package
 - `matt-pocock-toolkit` — Matt Pocock Toolkit: batch-grill-me, code-review, codebase-design, diagnosing-bugs, domain-modeling, grill-me, grill-with-docs, grilling, handoff, improve-codebase-architecture, prototype, research, resolving-merge-conflicts, setup-matt-pocock-skills, tdd, to-spec, to-tickets, triage, wayfinder — every member is fetched from its upstream owner and is not included in this package
 
@@ -65,7 +69,7 @@ npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test
 # Delivery Infrastructure
 npx skills@1.5.22 remove --global --agent universal claude-code --skill cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman
 # Backend
-npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug python-backend
+npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend
 # Fullstack
 npx skills@1.5.22 remove --global --agent universal claude-code --skill better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable
 # Matt Pocock Toolkit

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1aboveio/skills",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Install the 1AboveIO first-party skill groups through the native Skills CLI.",
   "license": "MIT",
   "type": "module",

package/runtime/skills/distribution/generated/recipes.json

@@ -8,7 +8,7 @@
       "id": "first-party",
       "type": "first-party",
       "package": "@1aboveio/skills",
-      "version": "0.10.0",
+      "version": "0.11.0",
       "updatePolicy": "pinned-npm-version",
       "members": [
         {
@@ -113,6 +113,12 @@
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
+        {
+          "installName": "pyspark",
+          "path": "skills/backend/pyspark",
+          "role": "skill",
+          "licenseResponsibility": "1aboveio-mit"
+        },
         {
           "installName": "python-backend",
           "path": "skills/backend/python-backend",
@@ -448,7 +454,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -520,7 +526,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -564,7 +570,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -597,7 +603,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -640,18 +646,19 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
               "airflow-dag-develop",
               "app-debug",
+              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
               }
             ],
             "onFailure": {
@@ -669,18 +676,19 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
               "airflow-dag-develop",
               "app-debug",
+              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
               }
             ],
             "onFailure": {
@@ -691,7 +699,7 @@
         ]
       },
       "uninstall": {
-        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug python-backend",
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend",
         "removesDependencies": false
       }
     },
@@ -708,7 +716,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -760,7 +768,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -916,7 +924,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -942,7 +950,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.10.0",
+            "version": "0.11.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -1012,7 +1020,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.10.0",
+          "version": "0.11.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -1033,6 +1041,7 @@
             "podman",
             "airflow-dag-develop",
             "app-debug",
+            "pyspark",
             "python-backend",
             "better-auth",
             "monorepo",
@@ -1044,7 +1053,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
             }
           ],
           "onFailure": {
@@ -1120,7 +1129,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.10.0",
+          "version": "0.11.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -1141,6 +1150,7 @@
             "podman",
             "airflow-dag-develop",
             "app-debug",
+            "pyspark",
             "python-backend",
             "better-auth",
             "monorepo",
@@ -1152,7 +1162,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm ci --ignore-scripts --prefix \"$HOME/.agents/skills/engineering-runtime/scripts\""
             }
           ],
           "onFailure": {
@@ -1183,7 +1193,7 @@
       ]
     },
     "uninstall": {
-      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable batch-grill-me code-review codebase-design diagnosing-bugs domain-modeling grill-me grill-with-docs grilling handoff improve-codebase-architecture prototype research resolving-merge-conflicts setup-matt-pocock-skills tdd to-spec to-tickets triage wayfinder harness-runtime"
+      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 impeccable batch-grill-me code-review codebase-design diagnosing-bugs domain-modeling grill-me grill-with-docs grilling handoff improve-codebase-architecture prototype research resolving-merge-conflicts setup-matt-pocock-skills tdd to-spec to-tickets triage wayfinder harness-runtime"
     }
   }
 }

package/runtime/skills/distribution/scripts/bundles.mjs

@@ -85,40 +85,108 @@ function localFirstPartyCommand(command, localRoot, groupId) {
   return command.replace(NATIVE_ADD_SOURCE, (_match, prefix) => `${prefix}${shellArgument(localRoot)}`);
 }
 
+// Native `skills add` prompts once per invocation for the installation method. Selecting several
+// bundles used to emit one add per group, so interactive installs asked the same question N times.
+// Coalesce by sourceId: one native add per upstream, with the union of --skill names, in the order
+// sources first appear across the selection. Identical full commands still collapse as before.
+const SKILL_TAIL = /^(.*? --skill )(.+?)( && .*)?$/;
+
+function skillsFromCommand(command) {
+  const match = SKILL_TAIL.exec(command);
+  if (!match) throw new Error(`Native add has no --skill list to coalesce: ${command}`);
+  return match[2].trim().split(/\s+/).filter(Boolean);
+}
+
+function withSkills(command, skills) {
+  const match = SKILL_TAIL.exec(command);
+  if (!match) throw new Error(`Native add has no --skill list to rewrite: ${command}`);
+  return `${match[1]}${skills.join(' ')}${match[3] || ''}`;
+}
+
+function withYesFlag(command) {
+  // --yes must bind to `skills`, not to a trailing `&& npm ci ...`.
+  const match = SKILL_TAIL.exec(command);
+  if (!match) return `${command} --yes`;
+  return `${match[1]}${match[2]} --yes${match[3] || ''}`;
+}
+
+function mergeSkillLists(existing, incoming) {
+  const seen = new Set(existing);
+  const merged = [...existing];
+  for (const skill of incoming) {
+    if (seen.has(skill)) continue;
+    seen.add(skill);
+    merged.push(skill);
+  }
+  return merged;
+}
+
 export function planBundleCommands(action, groupIds, { yes = false, ...transport } = {}) {
   if (!ACTIONS.has(action)) throw new Error(`Unsupported bundle action: ${action}`);
   const { recipesPath, localRoot } = resolveBundleTransport(transport);
   const recipes = readRecipes(recipesPath);
   assertLocalRoot(localRoot);
   const groups = new Map(recipes.groups.map((group) => [group.id, group]));
-  const commands = [];
-  const seen = new Set();
-
+  const requested = [];
   for (const groupId of groupIds) {
+    if (!groups.has(groupId) || SUPPORT_GROUPS.has(groupId)) throw new Error(`Unknown bundle: ${groupId}`);
+    if (!requested.includes(groupId)) requested.push(groupId);
+  }
+
+  // Always coalesce from the selected groups (not recipes.all). A public package may expose only a
+  // subset of groups while still shipping a full recipes.all from the private catalog; using that
+  // all-plan would install skills that are not in the selection / not in the package.
+  const bySource = new Map();
+  const sourceOrder = [];
+
+  for (const groupId of requested) {
     const group = groups.get(groupId);
-    if (!group || SUPPORT_GROUPS.has(groupId)) throw new Error(`Unknown bundle: ${groupId}`);
     for (const step of group[action].steps) {
       const generated = COMMAND_TRANSPORTS
-        .map((transport) => step.commands.find((candidate) => candidate.transport === transport))
+        .map((transportName) => step.commands.find((candidate) => candidate.transport === transportName))
         .find(Boolean);
       if (!generated) throw new Error(`Bundle ${groupId} has no supported command for source ${step.sourceId}`);
-      let command = step.sourceId === 'first-party'
+      const command = step.sourceId === 'first-party'
         ? localFirstPartyCommand(generated.command, localRoot, groupId)
         : generated.command;
-      if (yes) command += ' --yes';
-      if (seen.has(command)) continue;
-      seen.add(command);
-      commands.push({
-        groupId,
-        groupName: group.displayName,
-        sourceId: step.sourceId,
-        command,
-        cwd: localRoot,
-        failureMessage: step.onFailure.message,
-      });
+      const skills = skillsFromCommand(command);
+      const suffix = SKILL_TAIL.exec(command)?.[3] || '';
+      const existing = bySource.get(step.sourceId);
+      if (!existing) {
+        sourceOrder.push(step.sourceId);
+        bySource.set(step.sourceId, {
+          groupIds: [groupId],
+          groupNames: [group.displayName],
+          sourceId: step.sourceId,
+          template: command,
+          skills: [...skills],
+          suffix,
+          cwd: localRoot,
+          failureMessage: step.onFailure.message,
+        });
+        continue;
+      }
+      existing.groupIds.push(groupId);
+      if (!existing.groupNames.includes(group.displayName)) existing.groupNames.push(group.displayName);
+      existing.skills = mergeSkillLists(existing.skills, skills);
+      if (!existing.suffix && suffix) existing.suffix = suffix;
     }
   }
-  return commands;
+
+  return sourceOrder.map((sourceId) => {
+    const planned = bySource.get(sourceId);
+    let command = withSkills(planned.template, planned.skills);
+    if (planned.suffix && !command.includes(' && ')) command = `${command}${planned.suffix}`;
+    if (yes) command = withYesFlag(command);
+    return {
+      groupId: planned.groupIds.length === 1 ? planned.groupIds[0] : planned.groupIds.join('+'),
+      groupName: planned.groupNames.length === 1 ? planned.groupNames[0] : planned.groupNames.join(' + '),
+      sourceId: planned.sourceId,
+      command,
+      cwd: planned.cwd,
+      failureMessage: planned.failureMessage,
+    };
+  });
 }
 
 export function runBundleCommands(commands, {

package/skills/backend/pyspark/LICENSE

@@ -0,0 +1,3 @@
+MIT License
+
+Copyright (c) 2026 1AboveIO

package/skills/backend/pyspark/SKILL.md

@@ -0,0 +1,116 @@
+---
+name: pyspark
+description: Implement, debug, review, and optimize Apache Spark or PySpark batch jobs, ETL pipelines, feature calculations, and warehouse writes. Use whenever work involves Spark DataFrames or SQL, joins, windows, grouped state, UDFs, shuffle, skew, spill, slow stages, Spark Connect, spark-submit, Hudi publication, backfills, semantic parity, or production canaries, even when the request is framed only as a warehouse timeout or performance regression.
+---
+
+# PySpark
+
+Use this skill as the workflow and routing index. Read the referenced playbooks
+that match the task before editing code or running a production-scale job.
+
+## Workflow
+
+```text
+contract -> transform -> materialize -> test -> CI/CD
+```
+
+Follow all five steps in order.
+
+### 1. Contract
+
+- **Input:** declare all parameters, defaults, validation, source snapshot/date
+  scope, overlapping-run behavior, idempotency/single-writer rules, and exact
+  Python and PySpark versions.
+- **Output:** declare plain Spark versus Hudi table type, output identity/schema,
+  partition columns, record key, deterministic order or precombine field, and
+  append/upsert/overwrite write mode.
+- **Conventions:** declare native Spark versus Spark Connect and full rebuild
+  versus incremental processing, including the incremental watermark,
+  lookback/checkpoint, and restart boundary when applicable.
+
+Read [ETL Contract](references/etl-contract.md). When the repository has no ETL
+base module, adapt [`assets/templates/etl.py`](assets/templates/etl.py).
+
+### 2. Transform
+
+- Make `transform` accept DataFrames and return the complete candidate without
+  invoking validation or a writer.
+- Estimate rows after joins/explodes/groups and measure maximum rows per key.
+  Fix algorithmic expansion and hot keys before tuning Spark configuration.
+- Prefer built-in expressions and windows. Use exact bucketing or one sorted
+  grouped sweep only for state that built-ins cannot express cleanly.
+- Preserve complete row identity, ordering, output columns, and required
+  behavior.
+- Avoid explicit `.cache()` and `.persist()` by default.
+
+Read [Transformation Design](references/transformation-design.md). For temporal
+windows or FIFO paths, also read
+[Velocity Feature Calculation](references/velocity-feature-calculation.md).
+
+### 3. Materialize
+
+- Treat materialization as the Spark action that executes the lazy candidate,
+  not as caching.
+- Use one aggregate validation action before every writer. Reference every
+  derived output so Catalyst cannot prune untested calculations.
+- Check row count, distinct/null/duplicate identities, domain blockers, and
+  required date/partition scope. Fail closed on any blocker.
+- Bind validation and publication to the same immutable source snapshot because
+  Spark may recompute the candidate.
+- Keep the validate-only guard inside the job and before every writer. Do not
+  add `.count()`, `.cache()`, or `.persist()` to warm execution.
+
+Read [Production Validation](references/production-validation.md).
+
+### 4. Test
+
+1. **Local test:** run pure state, tiny real-Spark, plan-shape, focused repository,
+   and bidirectional parity tests. Cover boundaries, ties, nulls, arithmetic,
+   identity, and precedence.
+2. **Cluster canary test:** run the production entry point and source scope with
+   `--validate-only`. Require zero blockers, an explicit publication skip, zero
+   Spark output records/bytes, and readable driver/event logs.
+3. **Profiling:** measure an isolated baseline, parse the canary event log, and
+   compare both runs on the same source scope and resource profile. Record
+   wall/stage/task time, cardinality, shuffle, spill, skew, output, and target
+   plan patterns.
+
+```bash
+spark-submit <production-options> job.py --validate-only
+set -o pipefail
+hdfs dfs -cat "$EVENT_LOG" | \
+  python {baseDir}/scripts/spark_eventlog_summary.py > canary-summary.json
+```
+
+Reject missing evidence, incomparable scope, unexplained regressions, or a
+missed performance gate. Keep validate-only time separate from write time.
+
+Read [Parity Testing](references/parity-testing.md),
+[Diagnosis And Profiling](references/diagnosis-and-profiling.md), and
+[Production Validation](references/production-validation.md).
+
+### 5. CI/CD
+
+1. Open a PR with required-behavior, test, canary, profiling, and risk evidence.
+2. Run required CI and independent review; fix failures.
+3. Enqueue through the configured merge queue and merge without bypassing gates.
+4. Deploy only the merged commit through the normal deployment path.
+5. Run write-enabled production validation only with explicit authorization;
+   reconcile counts, identity, schema, coverage, blockers, readability, write
+   mode, and commit evidence independently.
+6. Restore scheduler/process ownership and report compute/validation,
+   publication, and end-to-end time separately.
+
+Read [Production Validation](references/production-validation.md).
+
+## Reference Index
+
+| Need | Read or run |
+|---|---|
+| Define or review an ETL job contract | [ETL Contract](references/etl-contract.md) and `assets/templates/etl.py` |
+| Diagnose or profile | [Diagnosis And Profiling](references/diagnosis-and-profiling.md) |
+| Design transformations | [Transformation Design](references/transformation-design.md) |
+| Optimize temporal/FIFO state | [Velocity Feature Calculation](references/velocity-feature-calculation.md) |
+| Prove parity | [Parity Testing](references/parity-testing.md) |
+| Run canaries or writes | [Production Validation](references/production-validation.md) |
+| Summarize an event log | `scripts/spark_eventlog_summary.py` |