@1aboveio/skills 0.16.1 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1aboveio/skills",
-  "version": "0.16.1",
+  "version": "0.17.0",
   "description": "Install the 1AboveIO first-party skill groups through the native Skills CLI.",
   "license": "MIT",
   "type": "module",

package/runtime/skills/distribution/generated/recipes.json

@@ -8,7 +8,7 @@
       "id": "first-party",
       "type": "first-party",
       "package": "@1aboveio/skills",
-      "version": "0.16.1",
+      "version": "0.17.0",
       "updatePolicy": "pinned-npm-version",
       "members": [
         {
@@ -196,7 +196,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -230,7 +230,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -274,7 +274,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -307,7 +307,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -350,7 +350,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -378,7 +378,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -416,7 +416,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -444,7 +444,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -482,7 +482,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -510,7 +510,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -548,7 +548,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -580,7 +580,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime",
@@ -620,7 +620,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -646,7 +646,7 @@
             "sourceId": "first-party",
             "sourceType": "first-party",
             "package": "@1aboveio/skills",
-            "version": "0.16.1",
+            "version": "0.17.0",
             "installPath": null,
             "members": [
               "harness-runtime"
@@ -678,7 +678,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.16.1",
+          "version": "0.17.0",
           "installPath": null,
           "members": [
             "harness-runtime",
@@ -731,7 +731,7 @@
           "sourceId": "first-party",
           "sourceType": "first-party",
           "package": "@1aboveio/skills",
-          "version": "0.16.1",
+          "version": "0.17.0",
           "installPath": null,
           "members": [
             "harness-runtime",

package/runtime/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs

@@ -97,7 +97,7 @@ export const WORKFLOW_TRUSTED_SOURCES = deepFreeze({
     id: 'first-party',
     type: 'first-party',
     package: '@1aboveio/skills',
-    version: '0.16.1',
+    version: '0.17.0',
   },
   matt: {
     id: 'matt-pocock',

package/skills/data-science/pyspark/SKILL.md

@@ -5,7 +5,8 @@ description: >
   ETL pipelines, feature calculations, and Hudi/warehouse writes. Follow
   contract → transform → materialize → local test → production validation
   (one-month canary, optional profile if that canary exceeds 5 minutes without
-  resource constraints, authorized write, cluster reconciliation). Use whenever
+  resource constraints or any job exceeds 30 minutes, authorized write, cluster
+  reconciliation). Use whenever
   the task involves Spark DataFrames or Spark SQL, joins, windows, grouped
   state, UDFs, shuffle, skew, spill, slow stages, driver/executor OOM, Spark
   Connect, spark-submit, Hudi commits, backfills, validate-only canaries,
@@ -69,6 +70,11 @@ windows or FIFO paths, also read
   Spark may recompute the candidate.
 - Keep the validate-only guard inside the job and before every writer. Do not
   add `.count()`, `.cache()`, or `.persist()` to warm execution.
+- Do not set Spark or Hudi shuffle parallelism (`spark.sql.shuffle.partitions`,
+  `hoodie.*.shuffle.parallelism`). Those knobs split the write into many tiny
+  files. Leave Spark/Hudi defaults; size writes with `HudiLayout`. Repair an
+  existing small-file table with `Etl.rewrite_hudi_layout` (see
+  [ETL Contract](references/etl-contract.md)).
 
 ### 4. Test
 
@@ -86,10 +92,12 @@ typically **one month** of source data. Local tests do not substitute.
 1. **Canary:** `spark-submit` with `--validate-only` on that sample and the
    production runtime. Require zero blockers, an explicit publication skip,
    zero Spark output records/bytes, and readable driver/event logs.
-2. **Profiling (optional):** do this only when the small-sample canary wall
-   time exceeds **5 minutes** **and** the run was not resource-constrained
-   (queued, under-provisioned executors, competing warehouse work). Otherwise
-   skip. When required, follow
+2. **Profiling (optional):** required when **either** (a) the small-sample
+   canary wall time exceeds **5 minutes** **and** the run was not
+   resource-constrained (queued, under-provisioned executors, competing
+   warehouse work), **or** (b) **any** job (canary, write, or scheduled run)
+   exceeds **30 minutes** — a hard gate, even if the cluster was constrained.
+   Otherwise skip. When required, follow
    [Diagnosis And Profiling](references/diagnosis-and-profiling.md).
 3. **Write (authorized):** re-check admission, then publish the same pinned
    sample. Do not present canary time as write performance.

package/skills/data-science/pyspark/assets/templates/etl.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser
 from contextlib import contextmanager
+from dataclasses import dataclass
 import enum
 import logging
 import os
@@ -22,6 +23,25 @@ from utils.hudi_metadata import hudi_metadata_write_options
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class HudiLayout:
+    """Target on-disk layout. Do not pair with Spark/Hudi shuffle parallelism."""
+
+    files_per_partition: int = 1
+    target_file_size_mb: int = 128
+
+    def write_options(self) -> Dict[str, Union[str, int]]:
+        target_bytes = int(self.target_file_size_mb) * 1024 * 1024
+        group_bytes = target_bytes * max(1, int(self.files_per_partition))
+        return {
+            "hoodie.parquet.max.file.size": str(target_bytes),
+            "hoodie.parquet.small.file.limit": str(target_bytes),
+            "hoodie.copyonwrite.insert.split.size": str(target_bytes),
+            "hoodie.clustering.plan.strategy.target.file.max.bytes": str(target_bytes),
+            "hoodie.clustering.plan.strategy.max.bytes.per.group": str(group_bytes),
+        }
+
+
 _SPARK_LEASE_LOCK = threading.RLock()
 _SPARK_LEASES = {}
 
@@ -157,6 +177,10 @@ class Etl(ABC):
         "SINGLE_WRITER"
     )
     extra_hudi_options: Dict[str, Union[str, bool, int]] = {}
+    hudi_layout = HudiLayout(
+        files_per_partition=1,
+        target_file_size_mb=128,
+    )
     zookeeper: str = None
 
     def __init__(
@@ -321,6 +345,8 @@ class Etl(ABC):
             "hoodie.datasource.write.precombine.field": self.ts,
             "hoodie.datasource.write.operation": f"{operation}",
             "hoodie.datasource.write.reconcile.schema": True,
+            # Do not set hoodie.*.shuffle.parallelism or spark.sql.shuffle.partitions:
+            # each shuffle task becomes a tiny Hudi file.
             "hoodie.schema.on.read.enable": True,
             # ---------------------------------------------------------------
             # Hudi 0.15 ComplexKeyGenerator regression fix
@@ -350,6 +376,7 @@ class Etl(ABC):
             "path": self.path,
         }
         hudi_options.update(self.extra_hudi_options)
+        hudi_options.update(self.hudi_layout.write_options())
         # Applied last, deliberately. A table inside the daily v3 metadata
         # listing scope must have its Hudi metadata index maintained by every
         # writer; an `extra_hudi_options` opt-out would leave the index stale
@@ -361,6 +388,30 @@ class Etl(ABC):
 
         df.write.format("hudi").options(**hudi_options).mode("append").save()
 
+    def rewrite_hudi_layout(self, partition_filter: Optional[str] = None) -> None:
+        """Repair small files on an existing Hudi table.
+
+        Reads the published path, drops `_hoodie_*` meta columns, and
+        ``insert_overwrite``s with ``HudiLayout``. Do not set Spark/Hudi
+        shuffle parallelism. Pass a Spark SQL predicate to limit the rewrite
+        (for example ``dt = '2026-08-01'``).
+        """
+        if self.table_type != "hudi_table":
+            raise ValueError("rewrite_hudi_layout is for hudi_table only")
+        if not self._is_existing_hudi_table():
+            raise ValueError(f"no Hudi table at {self.path}")
+        source = self.spark.read.format("hudi").load(self.path)
+        if partition_filter:
+            source = source.where(partition_filter)
+        meta = [c for c in source.columns if c.startswith("_hoodie_")]
+        source = source.drop(*meta) if meta else source
+        previous = self.hudi_mode
+        try:
+            self.hudi_mode = "insert_overwrite"
+            self.load_hudi(source)
+        finally:
+            self.hudi_mode = previous
+
     def load_spark(self, df: DataFrame):
         output = df.repartition(*self.repartition) if self.repartition else df
         try:

package/skills/data-science/pyspark/references/diagnosis-and-profiling.md

@@ -7,12 +7,19 @@ commands from the available runtime.
 
 ## When Profiling Is Required
 
-Profiling is optional during production validation. It is **required** only when
-the small-sample canary (typically one month) wall time exceeds **5 minutes**
-and the run was **not** resource-constrained (queued, under-provisioned
-executors, competing warehouse work). Then parse the canary event log and
-compare an isolated baseline on the same sample and resource profile. Keep
-validate-only time separate from write time.
+Profiling is optional during production validation. It is **required** when
+**either**:
+
+1. The small-sample canary (typically one month) wall time exceeds **5 minutes**
+   **and** the run was **not** resource-constrained (queued, under-provisioned
+   executors, competing warehouse work).
+2. **Any** job — canary, authorized write, or a scheduled production run —
+   exceeds **30 minutes**. This is a hard gate. Resource contention does not
+   waive it.
+
+Otherwise skip. When required, parse the event log and compare an isolated
+baseline on the same sample and resource profile. Keep validate-only time
+separate from write time.
 
 ```bash
 set -o pipefail
@@ -65,7 +72,9 @@ Apply these rules to every performance change:
   hot-key algorithms, and write amplification in the job logic or data layout.
 - Do not claim a performance fix from changing shuffle partitions, executor
   count/cores/memory, timeouts, broadcast thresholds or hints, AQE switches,
-  repartition counts, cache/persistence, or retry settings.
+  repartition counts, cache/persistence, or retry settings. Raising Spark or
+  Hudi shuffle parallelism also creates small files on the write path; leave
+  those settings unset.
 - Do not meet the target by dropping history, rows, columns, validations,
   formulas, or publication work required by the job.
 - Classify a proven cluster, storage, or configuration defect as an operational
@@ -133,7 +142,7 @@ settings merely because the slow work becomes visible at a later action.
    intended physical-plan change with focused tests. Do not treat local Spark as
    cluster parity.
 5. **Production validation:** one-month (typical) sample: canary, optional
-   profile (only if that canary exceeds 5 minutes without resource constraints),
+   profile (5-minute unconstrained canary **or** any job over 30 minutes),
    authorized write, then reconciliation.
 
 Repeat the loop when the target is still missed or the dominant gap moves.

package/skills/data-science/pyspark/references/etl-contract.md

@@ -93,6 +93,25 @@ belong before the writer in the job process.
 - Reject incremental mode without a filter/watermark and exact restart boundary.
 - Reject optimistic concurrency without a working lock provider.
 - Do not rely on scheduler timing as the single-writer guarantee.
+- Do not set `spark.sql.shuffle.partitions` or Hudi
+  `hoodie.insert.shuffle.parallelism` / `hoodie.upsert.shuffle.parallelism` /
+  `hoodie.bulkinsert.shuffle.parallelism`. A raised shuffle partition count
+  becomes one small file per task on the write path. Size files with
+  `HudiLayout` (default one ~128MB file per partition), not shuffle parallelism.
+
+### Repair small files
+
+Existing tables written with high shuffle parallelism stay small-file until
+rewritten. Use `Etl.rewrite_hudi_layout` (same keys, partitions, locks, and
+`HudiLayout`). Restrict with a partition predicate when the table is large.
+Do not set shuffle parallelism on the rewrite either.
+
+```python
+job = MyEtl(start_date="2026-08-01", end_date="2026-08-02")
+job.rewrite_hudi_layout("dt = '2026-08-01'")  # one partition
+# job.rewrite_hudi_layout()                   # whole table
+```
+
 - Do not present `repartition`, JDBC partition count, fetch size, or writer
   option changes as logic optimization; classify required configuration fixes
   as separate operational work.

package/skills/data-science/pyspark/references/production-validation.md

@@ -102,8 +102,9 @@ Do not add `.cache()` or `.persist()` to warm the canary.
 
 ## 2. Profiling (optional)
 
-After the canary, profile only when wall time exceeds **5 minutes** and the run
-was not resource-constrained. Otherwise skip. When required, follow
+After the canary, profile when **either** the small-sample canary exceeds
+**5 minutes** without resource constraints, **or any job exceeds 30 minutes**
+(hard gate). Otherwise skip. When required, follow
 [Diagnosis And Profiling](diagnosis-and-profiling.md).
 
 ## 3. Write

package/skills/engineering/engineering-runtime/coherence/workflow.json

@@ -361,7 +361,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "eff6c7b5931bce5b2265a619bccddd371a89df6ce7bc2edc74f11d00060a71dd",
       "digestExcludes": []
@@ -374,7 +374,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "08230dc57a53d6526692b50138038abbd80abce1a067e01c52ad6acc182caf7e",
       "digestExcludes": []
@@ -387,7 +387,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "9eea7bfba348ddaea1b934c9a7fabdd9b101df8a146e1c93e947da4d335f1d98",
       "digestExcludes": []
@@ -400,7 +400,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "58b0556228a9271cf9e33727a05fc08a4fdfd29512ec0936b169f0a55d60e6ef",
       "digestExcludes": []
@@ -413,9 +413,9 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
-      "contentDigest": "a3743e0b48f0ddb54cd98d1dad22907c5f5613d13525d758c4915f1661788062",
+      "contentDigest": "0b2d8e0c4d0ae6336c733ff2102d6911a3bd4298394eddd9f40eefaeb884903c",
       "digestExcludes": []
     },
     {
@@ -426,7 +426,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "b5af4dccf703362d4f41cac4fdff48305f652a00338d85975a2e5c35ec6bc61c",
       "digestExcludes": []
@@ -439,7 +439,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "f048fd00c69f2dc666fc7a3096cfeee6dfba73933bfb69602f3f0e26159798cc",
       "digestExcludes": []
@@ -452,7 +452,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
       "contentDigest": "90a7c4e1ad6da1e632ea2c5259e4967ffebb84353adccbca8dfc5d6bc50b60c1",
       "digestExcludes": []
@@ -465,15 +465,15 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       },
-      "contentDigest": "1d85a6268819b2c2d4622355baeb4f8576e05a9725341339785badae8fe887d3",
+      "contentDigest": "4d2242d41c7c57e24119b7cd974a88dd97cf526a8f76e6825385060ff86095c8",
       "digestExcludes": [
         "coherence/workflow.json"
       ]
     }
   ],
-  "releaseIdentity": "e4cd408398f14ccc6847f86e4f0deb5a733b95ff23773ab6a4669c1c7a336130",
+  "releaseIdentity": "4a75f070c64c9f24bfbb250be0706a55f6fc3bf071d537200a0a5a91378a4959",
   "lifecycleAuthority": "native-skills-cli",
   "repairRecipe": {
     "id": "engineering-workflow-dependency-first",
@@ -531,7 +531,7 @@
         "sourceId": "first-party",
         "sourceType": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1",
+        "version": "0.17.0",
         "installPath": null,
         "members": [
           "harness-runtime",
@@ -547,7 +547,7 @@
         "commands": [
           {
             "transport": "npm",
-            "command": "npx @1aboveio/skills@0.16.1 install --group engineering-workflow --yes"
+            "command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
           }
         ],
         "onFailure": {

package/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs

@@ -97,7 +97,7 @@ export const WORKFLOW_TRUSTED_SOURCES = deepFreeze({
     id: 'first-party',
     type: 'first-party',
     package: '@1aboveio/skills',
-    version: '0.16.1',
+    version: '0.17.0',
   },
   matt: {
     id: 'matt-pocock',

package/skills/engineering/resolve-issues/generated/workflow-repair-policy.json

@@ -237,7 +237,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -248,7 +248,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -259,7 +259,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -270,7 +270,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -281,7 +281,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -292,7 +292,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -303,7 +303,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -314,7 +314,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     },
     {
@@ -325,7 +325,7 @@
         "id": "first-party",
         "type": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1"
+        "version": "0.17.0"
       }
     }
   ],
@@ -426,7 +426,7 @@
         "sourceId": "first-party",
         "sourceType": "first-party",
         "package": "@1aboveio/skills",
-        "version": "0.16.1",
+        "version": "0.17.0",
         "installPath": null,
         "members": [
           "harness-runtime",
@@ -442,7 +442,7 @@
         "commands": [
           {
             "transport": "npm",
-            "command": "npx @1aboveio/skills@0.16.1 install --group engineering-workflow --yes"
+            "command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
           }
         ],
         "onFailure": {

package/skills/engineering/resolve-issues/scripts/run-state.mjs

@@ -3412,7 +3412,7 @@ export const WORKFLOW_PREFLIGHT_COMMANDS = Object.freeze([
 
 const WORKFLOW_VERIFIER_URL = new URL('../../engineering-runtime/scripts/workflow-coherence.mjs', import.meta.url)
 const WORKFLOW_FALLBACK_POLICY_URL = new URL('../generated/workflow-repair-policy.json', import.meta.url)
-export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = '41c079a5d5a346693847ab6f6067b409ba48a5c92acbe9eab46a1633ced6494e'
+export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = 'a6d80d500e43b660220f71f9764e5e6c905bf1955d63f8733d115b6c1a601448'
 const WORKFLOW_REPAIR_RECIPE_REFERENCE = Object.freeze({
   id: 'engineering-workflow-dependency-first',
   generatedFrom: 'skills/distribution/generated/recipes.json',