@1aboveio/skills 0.14.0 → 0.16.1

package/README.md

@@ -67,7 +67,9 @@ planned same-named skill. The coordinator still owns group and new-member consen
 
 - `engineering-workflow` — Engineering Workflow: e2e-test, ensure-coverage, implement-and-pr, resolve-issues, resolve-release, review-pr, smoke, engineering-runtime
 - `delivery-infrastructure` — Delivery Infrastructure: cloud-build, cloud-debug, cloud-deploy, docker, google-cloud, mergify, podman
-- `backend` — Backend: airflow-dag-develop, app-debug, pyspark, python-backend
+- `backend` — Backend: app-debug, python-backend
+- `data-science` — Data Science: airflow-dag-develop, pyspark
+- `payment` — Payment: fraud-analysis, payment-analysis
 - `fullstack` — Fullstack: better-auth, monorepo, nextjs-fullstack, prisma-setup, shadcn, zod-v4
 
 Internal dependencies, installed with the groups above and not selectable on their own: `harness-runtime`.
@@ -119,7 +121,11 @@ npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test
 # Delivery Infrastructure
 npx skills@1.5.22 remove --global --agent universal claude-code --skill cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman
 # Backend
-npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend
+npx skills@1.5.22 remove --global --agent universal claude-code --skill app-debug python-backend
+# Data Science
+npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop pyspark
+# Payment
+npx skills@1.5.22 remove --global --agent universal claude-code --skill fraud-analysis payment-analysis
 # Fullstack
 npx skills@1.5.22 remove --global --agent universal claude-code --skill better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1aboveio/skills",
-  "version": "0.14.0",
+  "version": "0.16.1",
   "description": "Install the 1AboveIO first-party skill groups through the native Skills CLI.",
   "license": "MIT",
   "type": "module",

package/runtime/skills/distribution/generated/recipes.json

@@ -8,7 +8,7 @@
       "id": "first-party",
       "type": "first-party",
       "package": "@1aboveio/skills",
-      "version": "0.14.0",
+      "version": "0.16.1",
       "updatePolicy": "pinned-npm-version",
       "members": [
         {
@@ -102,26 +102,38 @@
           "licenseResponsibility": "1aboveio-mit"
         },
         {
-          "installName": "airflow-dag-develop",
-          "path": "skills/backend/airflow-dag-develop",
+          "installName": "app-debug",
+          "path": "skills/backend/app-debug",
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
         {
-          "installName": "app-debug",
-          "path": "skills/backend/app-debug",
+          "installName": "python-backend",
+          "path": "skills/backend/python-backend",
+          "role": "skill",
+          "licenseResponsibility": "1aboveio-mit"
+        },
+        {
+          "installName": "airflow-dag-develop",
+          "path": "skills/data-science/airflow-dag-develop",
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
         {
           "installName": "pyspark",
-          "path": "skills/backend/pyspark",
+          "path": "skills/data-science/pyspark",
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
         {
-          "installName": "python-backend",
-          "path": "skills/backend/python-backend",
+          "installName": "fraud-analysis",
+          "path": "skills/payment/fraud-analysis",
+          "role": "skill",
+          "licenseResponsibility": "1aboveio-mit"
+        },
+        {
+          "installName": "payment-analysis",
+          "path": "skills/payment/payment-analysis",
           "role": "skill",
           "licenseResponsibility": "1aboveio-mit"
         },
@@ -184,7 +196,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -218,7 +230,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -262,7 +274,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -295,7 +307,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -338,19 +350,17 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
-              "airflow-dag-develop",
               "app-debug",
-              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime app-debug python-backend"
               }
             ],
             "onFailure": {
@@ -368,19 +378,149 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
-              "airflow-dag-develop",
               "app-debug",
-              "pyspark",
               "python-backend"
             ],
             "commands": [
               {
                 "transport": "local",
-                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop app-debug pyspark python-backend"
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime app-debug python-backend"
+              }
+            ],
+            "onFailure": {
+              "action": "stop",
+              "message": "Source first-party fetch failed; stop this recipe before running another source."
+            }
+          }
+        ]
+      },
+      "uninstall": {
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill app-debug python-backend",
+        "removesDependencies": false
+      }
+    },
+    {
+      "id": "data-science",
+      "displayName": "Data Science",
+      "dependsOn": [
+        "harness-runtime"
+      ],
+      "install": {
+        "stopOnFailure": true,
+        "steps": [
+          {
+            "sourceId": "first-party",
+            "sourceType": "first-party",
+            "package": "@1aboveio/skills",
+            "version": "0.16.1",
+            "installPath": null,
+            "members": [
+              "harness-runtime",
+              "airflow-dag-develop",
+              "pyspark"
+            ],
+            "commands": [
+              {
+                "transport": "local",
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop pyspark"
+              }
+            ],
+            "onFailure": {
+              "action": "stop",
+              "message": "Source first-party fetch failed; stop this recipe before running another source."
+            }
+          }
+        ]
+      },
+      "update": {
+        "mode": "rerun-install-recipe",
+        "stopOnFailure": true,
+        "steps": [
+          {
+            "sourceId": "first-party",
+            "sourceType": "first-party",
+            "package": "@1aboveio/skills",
+            "version": "0.16.1",
+            "installPath": null,
+            "members": [
+              "harness-runtime",
+              "airflow-dag-develop",
+              "pyspark"
+            ],
+            "commands": [
+              {
+                "transport": "local",
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime airflow-dag-develop pyspark"
+              }
+            ],
+            "onFailure": {
+              "action": "stop",
+              "message": "Source first-party fetch failed; stop this recipe before running another source."
+            }
+          }
+        ]
+      },
+      "uninstall": {
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop pyspark",
+        "removesDependencies": false
+      }
+    },
+    {
+      "id": "payment",
+      "displayName": "Payment",
+      "dependsOn": [
+        "harness-runtime"
+      ],
+      "install": {
+        "stopOnFailure": true,
+        "steps": [
+          {
+            "sourceId": "first-party",
+            "sourceType": "first-party",
+            "package": "@1aboveio/skills",
+            "version": "0.16.1",
+            "installPath": null,
+            "members": [
+              "harness-runtime",
+              "fraud-analysis",
+              "payment-analysis"
+            ],
+            "commands": [
+              {
+                "transport": "local",
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime fraud-analysis payment-analysis"
+              }
+            ],
+            "onFailure": {
+              "action": "stop",
+              "message": "Source first-party fetch failed; stop this recipe before running another source."
+            }
+          }
+        ]
+      },
+      "update": {
+        "mode": "rerun-install-recipe",
+        "stopOnFailure": true,
+        "steps": [
+          {
+            "sourceId": "first-party",
+            "sourceType": "first-party",
+            "package": "@1aboveio/skills",
+            "version": "0.16.1",
+            "installPath": null,
+            "members": [
+              "harness-runtime",
+              "fraud-analysis",
+              "payment-analysis"
+            ],
+            "commands": [
+              {
+                "transport": "local",
+                "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime fraud-analysis payment-analysis"
               }
             ],
             "onFailure": {
@@ -391,7 +531,7 @@
         ]
       },
       "uninstall": {
-        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill airflow-dag-develop app-debug pyspark python-backend",
+        "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill fraud-analysis payment-analysis",
         "removesDependencies": false
       }
     },
@@ -408,7 +548,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -440,7 +580,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -480,7 +620,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -506,7 +646,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.14.0",
+            "version": "0.16.1",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -538,7 +678,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.14.0",
+          "version": "0.16.1",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -557,10 +697,12 @@
             "google-cloud",
             "mergify",
             "podman",
-            "airflow-dag-develop",
             "app-debug",
-            "pyspark",
             "python-backend",
+            "airflow-dag-develop",
+            "pyspark",
+            "fraud-analysis",
+            "payment-analysis",
             "better-auth",
             "monorepo",
             "nextjs-fullstack",
@@ -571,7 +713,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
             }
           ],
           "onFailure": {
@@ -589,7 +731,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.14.0",
+          "version": "0.16.1",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -608,10 +750,12 @@
             "google-cloud",
             "mergify",
             "podman",
-            "airflow-dag-develop",
             "app-debug",
-            "pyspark",
             "python-backend",
+            "airflow-dag-develop",
+            "pyspark",
+            "fraud-analysis",
+            "payment-analysis",
             "better-auth",
             "monorepo",
             "nextjs-fullstack",
@@ -622,7 +766,7 @@
           "commands": [
             {
               "transport": "local",
-              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
+              "command": "npx skills@1.5.22 add . --global --agent universal claude-code --skill harness-runtime e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 && npm --prefix \"$(realpath \"$HOME/.agents/skills/engineering-runtime/scripts\")\" ci --ignore-scripts"
             }
           ],
           "onFailure": {
@@ -633,7 +777,7 @@
       ]
     },
     "uninstall": {
-      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman airflow-dag-develop app-debug pyspark python-backend better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 harness-runtime"
+      "command": "npx skills@1.5.22 remove --global --agent universal claude-code --skill e2e-test ensure-coverage implement-and-pr resolve-issues resolve-release review-pr smoke engineering-runtime cloud-build cloud-debug cloud-deploy docker google-cloud mergify podman app-debug python-backend airflow-dag-develop pyspark fraud-analysis payment-analysis better-auth monorepo nextjs-fullstack prisma-setup shadcn zod-v4 harness-runtime"
     }
   }
 }

package/runtime/skills/distribution/scripts/bundles.mjs

@@ -38,6 +38,10 @@ const COMMAND_TRANSPORTS = ['local', 'https'];
 // is never presented as something a user picks. The picker, `list`, the plan and the generated
 // package README all derive their answer from here rather than each re-listing the ids.
 export const SUPPORT_GROUPS = new Set(['harness-runtime']);
+// External toolkits stay in the manifest for standalone docs/canary, but the coordinator — checkout
+// and the public package alike — does not offer or install them.
+export const EXTERNAL_ONLY_GROUPS = new Set(['matt-pocock-toolkit']);
+const HIDDEN_GROUPS = new Set([...SUPPORT_GROUPS, ...EXTERNAL_ONLY_GROUPS]);
 const ACTIONS = new Set(['install', 'update']);
 
 // Selection memory (ADR 0002/0004 amendment, epic #1124): the remembered user-facing group
@@ -113,7 +117,10 @@ function writeSelectionState(state, env = process.env) {
 // user-selected) are dropped with a warning; memory never resurrects them and never falls back to
 // filesystem inference. Nothing left is an error, not an empty plan.
 function rememberedGroupIds(state, catalog, stderr) {
-  const known = new Set(catalog.map((bundle) => bundle.id));
+  const known = new Set([
+    ...catalog.map((bundle) => bundle.id),
+    ...EXTERNAL_ONLY_GROUPS,
+  ]);
   const kept = [];
   const dropped = [];
   for (const group of state.groups) {
@@ -256,7 +263,7 @@ function assertLocalRoot(localRoot) {
 
 export function bundleCatalog(transport = {}) {
   return readRecipes(resolveBundleTransport(transport).recipesPath).groups
-    .filter((group) => !SUPPORT_GROUPS.has(group.id))
+    .filter((group) => !HIDDEN_GROUPS.has(group.id))
     .map((group) => ({
       id: group.id,
       displayName: group.displayName,
@@ -744,7 +751,8 @@ async function consentNewMembers(candidates, { yes, stdin, stdout, stderr }) {
 
 function selectionIncludesTdd(groupIds, catalog) {
   const selected = new Set(groupIds);
-  return catalog.some((bundle) => selected.has(bundle.id) && bundle.members.includes('tdd'));
+  return catalog.some((bundle) => selected.has(bundle.id) && bundle.members.includes('tdd'))
+    || groupIds.includes('matt-pocock-toolkit');
 }
 
 // Dependencies are injectable so the coordinator seam can be tested black-box: a stubbed native

package/runtime/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs

@@ -97,7 +97,7 @@ export const WORKFLOW_TRUSTED_SOURCES = deepFreeze({
     id: 'first-party',
     type: 'first-party',
     package: '@1aboveio/skills',
-    version: '0.14.0',
+    version: '0.16.1',
   },
   matt: {
     id: 'matt-pocock',

package/skills/{backend → data-science}/pyspark/SKILL.md

@@ -1,17 +1,27 @@
 ---
 name: pyspark
-description: Implement, debug, review, and optimize Apache Spark or PySpark batch jobs, ETL pipelines, feature calculations, and Hudi/warehouse writes. Use whenever the task involves Spark DataFrames or Spark SQL, joins, windows, grouped state, UDFs, shuffle, skew, spill, slow stages, driver/executor OOM, Spark Connect, spark-submit, Hudi commits, backfills, bidirectional parity tests, validate-only canaries, or Spark event-log profiling — including when the user only mentions a warehouse job timeout, empty Hudi write, or performance regression on an EMR/Dataproc job.
+description: >
+  Implement, debug, review, and optimize Apache Spark or PySpark batch jobs,
+  ETL pipelines, feature calculations, and Hudi/warehouse writes. Follow
+  contract → transform → materialize → local test → production validation
+  (one-month canary, optional profile if that canary exceeds 5 minutes without
+  resource constraints, authorized write, cluster reconciliation). Use whenever
+  the task involves Spark DataFrames or Spark SQL, joins, windows, grouped
+  state, UDFs, shuffle, skew, spill, slow stages, driver/executor OOM, Spark
+  Connect, spark-submit, Hudi commits, backfills, validate-only canaries,
+  cluster reconciliation, or Spark event-log profiling — including warehouse
+  job timeouts, empty Hudi writes, or EMR/Dataproc performance regressions.
 ---
 
 # PySpark
 
-Use this skill as the workflow and routing index. Read the referenced playbooks
-that match the task before editing code or running a production-scale job.
+Use this skill as the workflow and routing index. Read the playbook for the
+step you are on before editing code or submitting to the cluster.
 
 ## Workflow
 
 ```text
-contract -> transform -> materialize -> test -> CI/CD
+contract -> transform -> materialize -> test -> production validation
 ```
 
 Follow all five steps in order.
@@ -60,48 +70,40 @@ windows or FIFO paths, also read
 - Keep the validate-only guard inside the job and before every writer. Do not
   add `.count()`, `.cache()`, or `.persist()` to warm execution.
 
-Read [Production Validation](references/production-validation.md).
-
 ### 4. Test
 
-1. **Local test:** run pure state, tiny real-Spark, plan-shape, focused repository,
-   and bidirectional parity tests. Cover boundaries, ties, nulls, arithmetic,
-   identity, and precedence.
-2. **Cluster canary test:** run the production entry point and source scope with
-   `--validate-only`. Require zero blockers, an explicit publication skip, zero
-   Spark output records/bytes, and readable driver/event logs.
-3. **Profiling:** measure an isolated baseline, parse the canary event log, and
-   compare both runs on the same source scope and resource profile. Record
-   wall/stage/task time, cardinality, shuffle, spill, skew, output, and target
-   plan patterns.
+Run **local** tests only. Do not treat a laptop Spark session as cluster
+evidence.
 
-```bash
-spark-submit <production-options> job.py --validate-only
-set -o pipefail
-hdfs dfs -cat "$EVENT_LOG" | \
-  python {baseDir}/scripts/spark_eventlog_summary.py > canary-summary.json
-```
+- Pure state, tiny real-Spark, plan-shape, and focused repository tests.
+- Cover boundaries, ties, nulls, arithmetic, identity, and precedence.
 
-Reject missing evidence, incomparable scope, unexplained regressions, or a
-missed performance gate. Keep validate-only time separate from write time.
+### 5. Production validation
 
-Read [Parity Testing](references/parity-testing.md),
-[Diagnosis And Profiling](references/diagnosis-and-profiling.md), and
-[Production Validation](references/production-validation.md).
+Run the production entry point on the cluster against a **small sample**,
+typically **one month** of source data. Local tests do not substitute.
 
-### 5. CI/CD
+1. **Canary:** `spark-submit` with `--validate-only` on that sample and the
+   production runtime. Require zero blockers, an explicit publication skip,
+   zero Spark output records/bytes, and readable driver/event logs.
+2. **Profiling (optional):** do this only when the small-sample canary wall
+   time exceeds **5 minutes** **and** the run was not resource-constrained
+   (queued, under-provisioned executors, competing warehouse work). Otherwise
+   skip. When required, follow
+   [Diagnosis And Profiling](references/diagnosis-and-profiling.md).
+3. **Write (authorized):** re-check admission, then publish the same pinned
+   sample. Do not present canary time as write performance.
+4. **Reconciliation:** independently prove the published sample. Follow
+   [Reconciliation](references/reconciliation.md).
+
+```bash
+spark-submit <production-options> job.py --validate-only
+```
 
-1. Open a PR with required-behavior, test, canary, profiling, and risk evidence.
-2. Run required CI and independent review; fix failures.
-3. Enqueue through the configured merge queue and merge without bypassing gates.
-4. Deploy only the merged commit through the normal deployment path.
-5. Run write-enabled production validation only with explicit authorization;
-   reconcile counts, identity, schema, coverage, blockers, readability, write
-   mode, and commit evidence independently.
-6. Restore scheduler/process ownership and report compute/validation,
-   publication, and end-to-end time separately.
+Restore scheduler/process ownership after the run.
 
-Read [Production Validation](references/production-validation.md).
+Read [Production Validation](references/production-validation.md) for admission,
+canary evidence, and write rules.
 
 ## Reference Index
 
@@ -111,6 +113,6 @@ Read [Production Validation](references/production-validation.md).
 | Diagnose or profile | [Diagnosis And Profiling](references/diagnosis-and-profiling.md) |
 | Design transformations | [Transformation Design](references/transformation-design.md) |
 | Optimize temporal/FIFO state | [Velocity Feature Calculation](references/velocity-feature-calculation.md) |
-| Prove parity | [Parity Testing](references/parity-testing.md) |
-| Run canaries or writes | [Production Validation](references/production-validation.md) |
+| Reconcile published sample vs previous path | [Reconciliation](references/reconciliation.md) |
+| Run production validation (canary, optional profile, write, reconciliation) | [Production Validation](references/production-validation.md) |
 | Summarize an event log | `scripts/spark_eventlog_summary.py` |

package/skills/{backend → data-science}/pyspark/references/diagnosis-and-profiling.md

@@ -1,9 +1,24 @@
 # Diagnosis And Profiling
 
 Use this reference to decide whether a Spark job is fast enough, identify the
-largest performance gap, and determine whether a fix is ready to ship. Focus on
-the required evidence and outcome; choose environment-specific commands and
-APIs from the available runtime.
+largest performance gap, and determine whether a canary is ready for write and
+reconciliation. Focus on evidence and outcome; choose environment-specific
+commands from the available runtime.
+
+## When Profiling Is Required
+
+Profiling is optional during production validation. It is **required** only when
+the small-sample canary (typically one month) wall time exceeds **5 minutes**
+and the run was **not** resource-constrained (queued, under-provisioned
+executors, competing warehouse work). Then parse the canary event log and
+compare an isolated baseline on the same sample and resource profile. Keep
+validate-only time separate from write time.
+
+```bash
+set -o pipefail
+hdfs dfs -cat "$EVENT_LOG" | \
+  python {baseDir}/scripts/spark_eventlog_summary.py > canary-summary.json
+```
 
 ## Performance Target
 
@@ -114,13 +129,12 @@ settings merely because the slow work becomes visible at a later action.
 3. **Make the fix:** reduce the responsible logical or physical work without
    tuning parameters, adding resources, or weakening output identity, ordering,
    boundaries, arithmetic, history, validation, or write behavior.
-4. **Validate locally:** prove required behavior, bidirectional parity, boundary
-   cases, and the intended physical-plan change with focused tests.
-5. **Canary and profile in the cluster:** run the production entry point in
-   validate-only mode on the comparable isolated snapshot; retain logs, prove
-   zero writes, and compare the same metrics with the baseline.
-6. **Ship:** open and review the PR, pass CI, enqueue and merge, deploy the merged
-   commit, then run authorized production validation and reconciliation.
+4. **Validate locally:** prove required behavior, boundary cases, and the
+   intended physical-plan change with focused tests. Do not treat local Spark as
+   cluster parity.
+5. **Production validation:** one-month (typical) sample: canary, optional
+   profile (only if that canary exceeds 5 minutes without resource constraints),
+   authorized write, then reconciliation.
 
 Repeat the loop when the target is still missed or the dominant gap moves.
 
@@ -132,9 +146,10 @@ Repeat the loop when the target is still missed or the dominant gap moves.
   confirmation of the same runtime, resources, and isolation conditions.
 - **Performance:** wall time, dominant stage, maximum task, cardinality,
   shuffle/spill, logic change, and pass/fail against the 5-minute target.
-- **Correctness:** local parity/plan results and canary blocker/output status.
-- **Delivery:** deployed hash, publication/reconciliation result when applicable,
-  and remaining limitations.
+- **Correctness:** local plan/boundary results, canary blocker/output status,
+  and cluster reconciliation (aggregates + bidirectional parity).
+- **Delivery:** cluster application IDs, published hash/commit, reconciliation
+  result when a write ran, and remaining limitations.
 
 Keep evidence aggregate-only. Never print credentials or entity-level rows.