@1aboveio/skills 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -2
- package/package.json +1 -1
- package/runtime/skills/distribution/generated/recipes.json +178 -34
- package/runtime/skills/distribution/scripts/bundles.mjs +11 -3
- package/runtime/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs +1 -1
- package/skills/data-science/pyspark/SKILL.md +126 -0
- package/skills/{backend → data-science}/pyspark/assets/templates/etl.py +51 -0
- package/skills/{backend → data-science}/pyspark/references/diagnosis-and-profiling.md +38 -14
- package/skills/{backend → data-science}/pyspark/references/etl-contract.md +19 -0
- package/skills/data-science/pyspark/references/production-validation.md +131 -0
- package/skills/data-science/pyspark/references/reconciliation.md +38 -0
- package/skills/{backend → data-science}/pyspark/references/transformation-design.md +30 -2
- package/skills/engineering/engineering-runtime/coherence/workflow.json +14 -14
- package/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs +1 -1
- package/skills/engineering/resolve-issues/generated/workflow-repair-policy.json +11 -11
- package/skills/engineering/resolve-issues/scripts/run-state.mjs +1 -1
- package/skills/payment/fraud-analysis/LICENSE +3 -0
- package/skills/payment/fraud-analysis/SKILL.md +113 -0
- package/skills/payment/fraud-analysis/evals/evals.json +40 -0
- package/skills/payment/fraud-analysis/references/archetypes/authorized-payment-scam.md +41 -0
- package/skills/payment/fraud-analysis/references/archetypes/first-party-fraud.md +44 -0
- package/skills/payment/fraud-analysis/references/archetypes/third-party-fraud.md +27 -0
- package/skills/payment/fraud-analysis/references/contexts/bank-transfer.md +24 -0
- package/skills/payment/fraud-analysis/references/contexts/card-payment.md +30 -0
- package/skills/payment/fraud-analysis/references/contexts/payment-collection.md +20 -0
- package/skills/payment/fraud-analysis/references/contexts/payout.md +20 -0
- package/skills/payment/fraud-analysis/references/feature-engineering.md +158 -0
- package/skills/payment/fraud-analysis/references/mechanisms/account-takeover.md +36 -0
- package/skills/payment/fraud-analysis/references/report-rationale.md +45 -0
- package/skills/payment/fraud-analysis/references/report-template.md +190 -0
- package/skills/payment/fraud-analysis/references/review-checklist.md +175 -0
- package/skills/payment/fraud-analysis/references/taxonomy.md +79 -0
- package/skills/payment/fraud-analysis/references/terminology.md +108 -0
- package/skills/payment/fraud-analysis/references/workflow.md +175 -0
- package/skills/payment/payment-analysis/LICENSE +3 -0
- package/skills/payment/payment-analysis/SKILL.md +127 -0
- package/skills/payment/payment-analysis/references/auth-rate-actions.md +30 -0
- package/skills/payment/payment-analysis/references/chargebacks.md +88 -0
- package/skills/payment/payment-analysis/references/event-layers.md +79 -0
- package/skills/payment/payment-analysis/references/fx.md +59 -0
- package/skills/payment/payment-analysis/references/journey.md +78 -0
- package/skills/payment/payment-analysis/references/metrics.md +62 -0
- package/skills/payment/payment-analysis/references/report-template.md +98 -0
- package/skills/payment/payment-analysis/references/terminology.md +85 -0
- package/skills/payment/payment-analysis/references/visualization.md +47 -0
- package/skills/backend/pyspark/SKILL.md +0 -116
- package/skills/backend/pyspark/references/parity-testing.md +0 -83
- package/skills/backend/pyspark/references/production-validation.md +0 -166
- /package/skills/{backend → data-science}/airflow-dag-develop/LICENSE +0 -0
- /package/skills/{backend → data-science}/airflow-dag-develop/SKILL.md +0 -0
- /package/skills/{backend → data-science}/pyspark/LICENSE +0 -0
- /package/skills/{backend → data-science}/pyspark/assets/templates/utils/__init__.py +0 -0
- /package/skills/{backend → data-science}/pyspark/assets/templates/utils/hudi_metadata.py +0 -0
- /package/skills/{backend → data-science}/pyspark/references/velocity-feature-calculation.md +0 -0
- /package/skills/{backend → data-science}/pyspark/scripts/spark_eventlog_summary.py +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from argparse import ArgumentParser
|
|
3
3
|
from contextlib import contextmanager
|
|
4
|
+
from dataclasses import dataclass
|
|
4
5
|
import enum
|
|
5
6
|
import logging
|
|
6
7
|
import os
|
|
@@ -22,6 +23,25 @@ from utils.hudi_metadata import hudi_metadata_write_options
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
23
24
|
|
|
24
25
|
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class HudiLayout:
|
|
28
|
+
"""Target on-disk layout. Do not pair with Spark/Hudi shuffle parallelism."""
|
|
29
|
+
|
|
30
|
+
files_per_partition: int = 1
|
|
31
|
+
target_file_size_mb: int = 128
|
|
32
|
+
|
|
33
|
+
def write_options(self) -> Dict[str, Union[str, int]]:
|
|
34
|
+
target_bytes = int(self.target_file_size_mb) * 1024 * 1024
|
|
35
|
+
group_bytes = target_bytes * max(1, int(self.files_per_partition))
|
|
36
|
+
return {
|
|
37
|
+
"hoodie.parquet.max.file.size": str(target_bytes),
|
|
38
|
+
"hoodie.parquet.small.file.limit": str(target_bytes),
|
|
39
|
+
"hoodie.copyonwrite.insert.split.size": str(target_bytes),
|
|
40
|
+
"hoodie.clustering.plan.strategy.target.file.max.bytes": str(target_bytes),
|
|
41
|
+
"hoodie.clustering.plan.strategy.max.bytes.per.group": str(group_bytes),
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
25
45
|
_SPARK_LEASE_LOCK = threading.RLock()
|
|
26
46
|
_SPARK_LEASES = {}
|
|
27
47
|
|
|
@@ -157,6 +177,10 @@ class Etl(ABC):
|
|
|
157
177
|
"SINGLE_WRITER"
|
|
158
178
|
)
|
|
159
179
|
extra_hudi_options: Dict[str, Union[str, bool, int]] = {}
|
|
180
|
+
hudi_layout = HudiLayout(
|
|
181
|
+
files_per_partition=1,
|
|
182
|
+
target_file_size_mb=128,
|
|
183
|
+
)
|
|
160
184
|
zookeeper: str = None
|
|
161
185
|
|
|
162
186
|
def __init__(
|
|
@@ -321,6 +345,8 @@ class Etl(ABC):
|
|
|
321
345
|
"hoodie.datasource.write.precombine.field": self.ts,
|
|
322
346
|
"hoodie.datasource.write.operation": f"{operation}",
|
|
323
347
|
"hoodie.datasource.write.reconcile.schema": True,
|
|
348
|
+
# Do not set hoodie.*.shuffle.parallelism or spark.sql.shuffle.partitions:
|
|
349
|
+
# each shuffle task becomes a tiny Hudi file.
|
|
324
350
|
"hoodie.schema.on.read.enable": True,
|
|
325
351
|
# ---------------------------------------------------------------
|
|
326
352
|
# Hudi 0.15 ComplexKeyGenerator regression fix
|
|
@@ -350,6 +376,7 @@ class Etl(ABC):
|
|
|
350
376
|
"path": self.path,
|
|
351
377
|
}
|
|
352
378
|
hudi_options.update(self.extra_hudi_options)
|
|
379
|
+
hudi_options.update(self.hudi_layout.write_options())
|
|
353
380
|
# Applied last, deliberately. A table inside the daily v3 metadata
|
|
354
381
|
# listing scope must have its Hudi metadata index maintained by every
|
|
355
382
|
# writer; an `extra_hudi_options` opt-out would leave the index stale
|
|
@@ -361,6 +388,30 @@ class Etl(ABC):
|
|
|
361
388
|
|
|
362
389
|
df.write.format("hudi").options(**hudi_options).mode("append").save()
|
|
363
390
|
|
|
391
|
+
def rewrite_hudi_layout(self, partition_filter: Optional[str] = None) -> None:
|
|
392
|
+
"""Repair small files on an existing Hudi table.
|
|
393
|
+
|
|
394
|
+
Reads the published path, drops `_hoodie_*` meta columns, and
|
|
395
|
+
``insert_overwrite``s with ``HudiLayout``. Do not set Spark/Hudi
|
|
396
|
+
shuffle parallelism. Pass a Spark SQL predicate to limit the rewrite
|
|
397
|
+
(for example ``dt = '2026-08-01'``).
|
|
398
|
+
"""
|
|
399
|
+
if self.table_type != "hudi_table":
|
|
400
|
+
raise ValueError("rewrite_hudi_layout is for hudi_table only")
|
|
401
|
+
if not self._is_existing_hudi_table():
|
|
402
|
+
raise ValueError(f"no Hudi table at {self.path}")
|
|
403
|
+
source = self.spark.read.format("hudi").load(self.path)
|
|
404
|
+
if partition_filter:
|
|
405
|
+
source = source.where(partition_filter)
|
|
406
|
+
meta = [c for c in source.columns if c.startswith("_hoodie_")]
|
|
407
|
+
source = source.drop(*meta) if meta else source
|
|
408
|
+
previous = self.hudi_mode
|
|
409
|
+
try:
|
|
410
|
+
self.hudi_mode = "insert_overwrite"
|
|
411
|
+
self.load_hudi(source)
|
|
412
|
+
finally:
|
|
413
|
+
self.hudi_mode = previous
|
|
414
|
+
|
|
364
415
|
def load_spark(self, df: DataFrame):
|
|
365
416
|
output = df.repartition(*self.repartition) if self.repartition else df
|
|
366
417
|
try:
|
|
@@ -1,9 +1,31 @@
|
|
|
1
1
|
# Diagnosis And Profiling
|
|
2
2
|
|
|
3
3
|
Use this reference to decide whether a Spark job is fast enough, identify the
|
|
4
|
-
largest performance gap, and determine whether a
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
largest performance gap, and determine whether a canary is ready for write and
|
|
5
|
+
reconciliation. Focus on evidence and outcome; choose environment-specific
|
|
6
|
+
commands from the available runtime.
|
|
7
|
+
|
|
8
|
+
## When Profiling Is Required
|
|
9
|
+
|
|
10
|
+
Profiling is optional during production validation. It is **required** when
|
|
11
|
+
**either**:
|
|
12
|
+
|
|
13
|
+
1. The small-sample canary (typically one month) wall time exceeds **5 minutes**
|
|
14
|
+
**and** the run was **not** resource-constrained (queued, under-provisioned
|
|
15
|
+
executors, competing warehouse work).
|
|
16
|
+
2. **Any** job — canary, authorized write, or a scheduled production run —
|
|
17
|
+
exceeds **30 minutes**. This is a hard gate. Resource contention does not
|
|
18
|
+
waive it.
|
|
19
|
+
|
|
20
|
+
Otherwise skip. When required, parse the event log and compare an isolated
|
|
21
|
+
baseline on the same sample and resource profile. Keep validate-only time
|
|
22
|
+
separate from write time.
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
set -o pipefail
|
|
26
|
+
hdfs dfs -cat "$EVENT_LOG" | \
|
|
27
|
+
python {baseDir}/scripts/spark_eventlog_summary.py > canary-summary.json
|
|
28
|
+
```
|
|
7
29
|
|
|
8
30
|
## Performance Target
|
|
9
31
|
|
|
@@ -50,7 +72,9 @@ Apply these rules to every performance change:
|
|
|
50
72
|
hot-key algorithms, and write amplification in the job logic or data layout.
|
|
51
73
|
- Do not claim a performance fix from changing shuffle partitions, executor
|
|
52
74
|
count/cores/memory, timeouts, broadcast thresholds or hints, AQE switches,
|
|
53
|
-
repartition counts, cache/persistence, or retry settings.
|
|
75
|
+
repartition counts, cache/persistence, or retry settings. Raising Spark or
|
|
76
|
+
Hudi shuffle parallelism also creates small files on the write path; leave
|
|
77
|
+
those settings unset.
|
|
54
78
|
- Do not meet the target by dropping history, rows, columns, validations,
|
|
55
79
|
formulas, or publication work required by the job.
|
|
56
80
|
- Classify a proven cluster, storage, or configuration defect as an operational
|
|
@@ -114,13 +138,12 @@ settings merely because the slow work becomes visible at a later action.
|
|
|
114
138
|
3. **Make the fix:** reduce the responsible logical or physical work without
|
|
115
139
|
tuning parameters, adding resources, or weakening output identity, ordering,
|
|
116
140
|
boundaries, arithmetic, history, validation, or write behavior.
|
|
117
|
-
4. **Validate locally:** prove required behavior,
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
commit, then run authorized production validation and reconciliation.
|
|
141
|
+
4. **Validate locally:** prove required behavior, boundary cases, and the
|
|
142
|
+
intended physical-plan change with focused tests. Do not treat local Spark as
|
|
143
|
+
cluster parity.
|
|
144
|
+
5. **Production validation:** one-month (typical) sample: canary, optional
|
|
145
|
+
profile (5-minute unconstrained canary **or** any job over 30 minutes),
|
|
146
|
+
authorized write, then reconciliation.
|
|
124
147
|
|
|
125
148
|
Repeat the loop when the target is still missed or the dominant gap moves.
|
|
126
149
|
|
|
@@ -132,9 +155,10 @@ Repeat the loop when the target is still missed or the dominant gap moves.
|
|
|
132
155
|
confirmation of the same runtime, resources, and isolation conditions.
|
|
133
156
|
- **Performance:** wall time, dominant stage, maximum task, cardinality,
|
|
134
157
|
shuffle/spill, logic change, and pass/fail against the 5-minute target.
|
|
135
|
-
- **Correctness:** local
|
|
136
|
-
|
|
137
|
-
|
|
158
|
+
- **Correctness:** local plan/boundary results, canary blocker/output status,
|
|
159
|
+
and cluster reconciliation (aggregates + bidirectional parity).
|
|
160
|
+
- **Delivery:** cluster application IDs, published hash/commit, reconciliation
|
|
161
|
+
result when a write ran, and remaining limitations.
|
|
138
162
|
|
|
139
163
|
Keep evidence aggregate-only. Never print credentials or entity-level rows.
|
|
140
164
|
|
|
@@ -93,6 +93,25 @@ belong before the writer in the job process.
|
|
|
93
93
|
- Reject incremental mode without a filter/watermark and exact restart boundary.
|
|
94
94
|
- Reject optimistic concurrency without a working lock provider.
|
|
95
95
|
- Do not rely on scheduler timing as the single-writer guarantee.
|
|
96
|
+
- Do not set `spark.sql.shuffle.partitions` or Hudi
|
|
97
|
+
`hoodie.insert.shuffle.parallelism` / `hoodie.upsert.shuffle.parallelism` /
|
|
98
|
+
`hoodie.bulkinsert.shuffle.parallelism`. A raised shuffle partition count
|
|
99
|
+
becomes one small file per task on the write path. Size files with
|
|
100
|
+
`HudiLayout` (default one ~128MB file per partition), not shuffle parallelism.
|
|
101
|
+
|
|
102
|
+
### Repair small files
|
|
103
|
+
|
|
104
|
+
Existing tables written with high shuffle parallelism stay small-file until
|
|
105
|
+
rewritten. Use `Etl.rewrite_hudi_layout` (same keys, partitions, locks, and
|
|
106
|
+
`HudiLayout`). Restrict with a partition predicate when the table is large.
|
|
107
|
+
Do not set shuffle parallelism on the rewrite either.
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
job = MyEtl(start_date="2026-08-01", end_date="2026-08-02")
|
|
111
|
+
job.rewrite_hudi_layout("dt = '2026-08-01'") # one partition
|
|
112
|
+
# job.rewrite_hudi_layout() # whole table
|
|
113
|
+
```
|
|
114
|
+
|
|
96
115
|
- Do not present `repartition`, JDBC partition count, fetch size, or writer
|
|
97
116
|
option changes as logic optimization; classify required configuration fixes
|
|
98
117
|
as separate operational work.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# Production Validation
|
|
2
|
+
|
|
3
|
+
Playbook for workflow step 5: **canary → (optional) profile → write →
|
|
4
|
+
reconciliation** on the cluster. Use a **small sample**, typically **one month**
|
|
5
|
+
of source data. Bind canary and write to the same pinned snapshot. Local tests
|
|
6
|
+
do not substitute.
|
|
7
|
+
|
|
8
|
+
## Fail-Closed Admission
|
|
9
|
+
|
|
10
|
+
Do not launch until all applicable facts are recorded and valid:
|
|
11
|
+
|
|
12
|
+
- exact commit/module hash;
|
|
13
|
+
- production Spark, Python, Java, and table-format runtimes;
|
|
14
|
+
- source snapshot or date boundary (the one-month sample);
|
|
15
|
+
- scheduler pause/ownership state;
|
|
16
|
+
- zero competing warehouse work;
|
|
17
|
+
- explicit validate-only or write-enabled mode;
|
|
18
|
+
- durable writable log and event-log destination;
|
|
19
|
+
- restart/recovery plan for writes.
|
|
20
|
+
|
|
21
|
+
Classify existing processes and working-tree changes before touching them. Do
|
|
22
|
+
not kill or overwrite another operator's work.
|
|
23
|
+
|
|
24
|
+
## 1. Canary
|
|
25
|
+
|
|
26
|
+
Implement the validate-only guard **in the Spark job**, not only in a wrapper.
|
|
27
|
+
Validate with one aggregate action, then return before any writer.
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
import json
|
|
31
|
+
|
|
32
|
+
from pyspark.sql import functions as F
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def validate_candidate(candidate):
|
|
36
|
+
# first() is a Spark action: it executes the lazy transform plan.
|
|
37
|
+
metrics = candidate.agg(
|
|
38
|
+
F.count("*").alias("row_count"),
|
|
39
|
+
F.countDistinct("event_id").alias("distinct_event_count"),
|
|
40
|
+
F.sum(
|
|
41
|
+
F.when(F.col("event_id").isNull(), 1).otherwise(0)
|
|
42
|
+
).alias("null_event_id_count"),
|
|
43
|
+
F.sum(
|
|
44
|
+
F.when(F.col("event_count_1h") < 0, 1).otherwise(0)
|
|
45
|
+
).alias("invalid_event_count_1h"),
|
|
46
|
+
F.count("amount_usd_1h").alias("non_null_amount_usd_1h_count"),
|
|
47
|
+
).first().asDict()
|
|
48
|
+
|
|
49
|
+
blockers = []
|
|
50
|
+
if metrics["row_count"] != metrics["distinct_event_count"]:
|
|
51
|
+
blockers.append("event_id is not unique")
|
|
52
|
+
if metrics["null_event_id_count"]:
|
|
53
|
+
blockers.append("event_id contains nulls")
|
|
54
|
+
if metrics["invalid_event_count_1h"]:
|
|
55
|
+
blockers.append("event_count_1h contains negative values")
|
|
56
|
+
if blockers:
|
|
57
|
+
raise RuntimeError(f"validation blocked: {blockers}")
|
|
58
|
+
return metrics
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run(validate_only, source_snapshot):
|
|
62
|
+
source = extract(source_snapshot)
|
|
63
|
+
candidate = transform(source)
|
|
64
|
+
|
|
65
|
+
metrics = validate_candidate(candidate)
|
|
66
|
+
print(json.dumps(metrics, sort_keys=True))
|
|
67
|
+
|
|
68
|
+
if validate_only:
|
|
69
|
+
print("validate-only: skipping publication")
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
publish(candidate)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Pin every source read to the same immutable snapshot. If the source cannot be
|
|
76
|
+
pinned, the canary does not prove what the writer will publish. Reference every
|
|
77
|
+
derived output in at least one validation aggregate so Catalyst cannot prune it.
|
|
78
|
+
|
|
79
|
+
Launch through the production `spark-submit` path and runtimes:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
export PYSPARK_DRIVER_PYTHON="$DRIVER_PYTHON"
|
|
83
|
+
export PYSPARK_PYTHON="$EXECUTOR_PYTHON"
|
|
84
|
+
|
|
85
|
+
spark-submit \
|
|
86
|
+
--name "$CANARY_NAME" \
|
|
87
|
+
--conf spark.pyspark.driver.python="$DRIVER_PYTHON" \
|
|
88
|
+
--conf spark.pyspark.python="$EXECUTOR_PYTHON" \
|
|
89
|
+
--conf spark.eventLog.enabled=true \
|
|
90
|
+
job.py --validate-only
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Do not replace another operator's scheduler files in place.
|
|
94
|
+
|
|
95
|
+
Prove the canary did not write:
|
|
96
|
+
|
|
97
|
+
1. Validation completed with zero blockers.
|
|
98
|
+
2. The terminal log says publication was skipped.
|
|
99
|
+
3. The event log reports zero Spark output records and bytes.
|
|
100
|
+
|
|
101
|
+
Do not add `.cache()` or `.persist()` to warm the canary.
|
|
102
|
+
|
|
103
|
+
## 2. Profiling (optional)
|
|
104
|
+
|
|
105
|
+
After the canary, profile when **either** the small-sample canary exceeds
|
|
106
|
+
**5 minutes** without resource constraints, **or any job exceeds 30 minutes**
|
|
107
|
+
(hard gate). Otherwise skip. When required, follow
|
|
108
|
+
[Diagnosis And Profiling](diagnosis-and-profiling.md).
|
|
109
|
+
|
|
110
|
+
## 3. Write
|
|
111
|
+
|
|
112
|
+
Write-enabled only after a successful canary and **explicit authorization**.
|
|
113
|
+
Re-check admission. Publish the same pinned sample. Do not present canary time
|
|
114
|
+
as write performance.
|
|
115
|
+
|
|
116
|
+
If a write is cancelled, prove from logs/source that publication was not
|
|
117
|
+
reached. YARN `KILLED` alone does not prove no commit occurred.
|
|
118
|
+
|
|
119
|
+
## 4. Reconciliation
|
|
120
|
+
|
|
121
|
+
Independent cluster read of the published scope — not a reuse of canary
|
|
122
|
+
aggregates. Follow [Reconciliation](reconciliation.md). Fail closed on mismatch.
|
|
123
|
+
|
|
124
|
+
## Evidence And Ownership
|
|
125
|
+
|
|
126
|
+
Safe evidence: counts, null rates, distinct counts, min/max dates, hashes,
|
|
127
|
+
stage metrics, plan-node counts, commit IDs. Do not print credentials or
|
|
128
|
+
entity-level rows.
|
|
129
|
+
|
|
130
|
+
After the run: confirm processes are terminal, restore scheduler ownership,
|
|
131
|
+
and report compute/validation, publication, and end-to-end time separately.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Reconciliation
|
|
2
|
+
|
|
3
|
+
Cluster check after an authorized write in production validation. Use the **same
|
|
4
|
+
one-month sample** as the canary. Local fixtures do not prove this.
|
|
5
|
+
|
|
6
|
+
Independently read the published table or Hudi commit. Do not reuse canary
|
|
7
|
+
aggregates. Fail closed on mismatch.
|
|
8
|
+
|
|
9
|
+
## What To Reconcile
|
|
10
|
+
|
|
11
|
+
Against the canary metrics and the previous path/oracle:
|
|
12
|
+
|
|
13
|
+
- row count and distinct identity count
|
|
14
|
+
- schema and required columns
|
|
15
|
+
- date/partition coverage of the sample
|
|
16
|
+
- write mode, commit success, and downstream readability
|
|
17
|
+
- validation blockers still zero on the published scope
|
|
18
|
+
|
|
19
|
+
## Bidirectional Row Parity
|
|
20
|
+
|
|
21
|
+
Counts alone miss wrong values, swapped identities, and duplicate multiplicity.
|
|
22
|
+
Compare full rows both ways with `exceptAll`, not `subtract`:
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
expected = old_build(source).select(*OUTPUT_COLUMNS)
|
|
26
|
+
actual = published.select(*OUTPUT_COLUMNS)
|
|
27
|
+
|
|
28
|
+
assert expected.exceptAll(actual).count() == 0
|
|
29
|
+
assert actual.exceptAll(expected).count() == 0
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Pin `source` to the same snapshot the writer used. Keep the previous
|
|
33
|
+
implementation only as this oracle while parity is being proven.
|
|
34
|
+
|
|
35
|
+
## Evidence
|
|
36
|
+
|
|
37
|
+
Safe evidence: counts, distinct counts, null rates, min/max dates, hashes, and
|
|
38
|
+
commit IDs. Do not print credentials or entity-level production rows.
|
|
@@ -158,5 +158,33 @@ assert candidate.columns == OUTPUT_COLUMNS
|
|
|
158
158
|
```
|
|
159
159
|
|
|
160
160
|
For composite identity, list every component, for example
|
|
161
|
-
`IDENTITY_COLUMNS = ["transaction_id", "transaction_leg_id"]`.
|
|
162
|
-
|
|
161
|
+
`IDENTITY_COLUMNS = ["transaction_id", "transaction_leg_id"]`.
|
|
162
|
+
|
|
163
|
+
## Local proof
|
|
164
|
+
|
|
165
|
+
These checks belong in **local tests** (workflow step 4). Cluster row parity is
|
|
166
|
+
[Reconciliation](reconciliation.md) after a write.
|
|
167
|
+
|
|
168
|
+
Keep a bounded previous-path oracle on **synthetic** fixtures. Assert critical
|
|
169
|
+
rules directly so oracle and candidate cannot share the same mistake:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
by_id = {row.event_id: row for row in actual.collect()}
|
|
173
|
+
assert by_id["at-24h"].count_24h == 2 # inclusive lower boundary
|
|
174
|
+
assert by_id["after-24h"].count_24h == 1 # one unit outside
|
|
175
|
+
assert by_id["same-time-b"].count_1h == 2 # stable tie ordering
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Cover empty and single-event groups; exact boundaries and one unit beyond;
|
|
179
|
+
same-timestamp ties; null key/time/amount; open/close/eviction; floating-point
|
|
180
|
+
residue; one dense hot key; duplicate and missing identities.
|
|
181
|
+
|
|
182
|
+
Assert the expensive plan shape is absent without snapshotting the whole plan:
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
plan = actual._jdf.queryExecution().executedPlan().toString().lower()
|
|
186
|
+
assert not ("join" in plan and "event_time" in plan and "interval" in plan)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Mutation-prove high-risk tests (cutoff direction, tie-breaker, pair join,
|
|
190
|
+
required column) so they fail when the rule is broken.
|
|
@@ -361,7 +361,7 @@
|
|
|
361
361
|
"id": "first-party",
|
|
362
362
|
"type": "first-party",
|
|
363
363
|
"package": "@1aboveio/skills",
|
|
364
|
-
"version": "0.
|
|
364
|
+
"version": "0.17.0"
|
|
365
365
|
},
|
|
366
366
|
"contentDigest": "eff6c7b5931bce5b2265a619bccddd371a89df6ce7bc2edc74f11d00060a71dd",
|
|
367
367
|
"digestExcludes": []
|
|
@@ -374,7 +374,7 @@
|
|
|
374
374
|
"id": "first-party",
|
|
375
375
|
"type": "first-party",
|
|
376
376
|
"package": "@1aboveio/skills",
|
|
377
|
-
"version": "0.
|
|
377
|
+
"version": "0.17.0"
|
|
378
378
|
},
|
|
379
379
|
"contentDigest": "08230dc57a53d6526692b50138038abbd80abce1a067e01c52ad6acc182caf7e",
|
|
380
380
|
"digestExcludes": []
|
|
@@ -387,7 +387,7 @@
|
|
|
387
387
|
"id": "first-party",
|
|
388
388
|
"type": "first-party",
|
|
389
389
|
"package": "@1aboveio/skills",
|
|
390
|
-
"version": "0.
|
|
390
|
+
"version": "0.17.0"
|
|
391
391
|
},
|
|
392
392
|
"contentDigest": "9eea7bfba348ddaea1b934c9a7fabdd9b101df8a146e1c93e947da4d335f1d98",
|
|
393
393
|
"digestExcludes": []
|
|
@@ -400,7 +400,7 @@
|
|
|
400
400
|
"id": "first-party",
|
|
401
401
|
"type": "first-party",
|
|
402
402
|
"package": "@1aboveio/skills",
|
|
403
|
-
"version": "0.
|
|
403
|
+
"version": "0.17.0"
|
|
404
404
|
},
|
|
405
405
|
"contentDigest": "58b0556228a9271cf9e33727a05fc08a4fdfd29512ec0936b169f0a55d60e6ef",
|
|
406
406
|
"digestExcludes": []
|
|
@@ -413,9 +413,9 @@
|
|
|
413
413
|
"id": "first-party",
|
|
414
414
|
"type": "first-party",
|
|
415
415
|
"package": "@1aboveio/skills",
|
|
416
|
-
"version": "0.
|
|
416
|
+
"version": "0.17.0"
|
|
417
417
|
},
|
|
418
|
-
"contentDigest": "
|
|
418
|
+
"contentDigest": "0b2d8e0c4d0ae6336c733ff2102d6911a3bd4298394eddd9f40eefaeb884903c",
|
|
419
419
|
"digestExcludes": []
|
|
420
420
|
},
|
|
421
421
|
{
|
|
@@ -426,7 +426,7 @@
|
|
|
426
426
|
"id": "first-party",
|
|
427
427
|
"type": "first-party",
|
|
428
428
|
"package": "@1aboveio/skills",
|
|
429
|
-
"version": "0.
|
|
429
|
+
"version": "0.17.0"
|
|
430
430
|
},
|
|
431
431
|
"contentDigest": "b5af4dccf703362d4f41cac4fdff48305f652a00338d85975a2e5c35ec6bc61c",
|
|
432
432
|
"digestExcludes": []
|
|
@@ -439,7 +439,7 @@
|
|
|
439
439
|
"id": "first-party",
|
|
440
440
|
"type": "first-party",
|
|
441
441
|
"package": "@1aboveio/skills",
|
|
442
|
-
"version": "0.
|
|
442
|
+
"version": "0.17.0"
|
|
443
443
|
},
|
|
444
444
|
"contentDigest": "f048fd00c69f2dc666fc7a3096cfeee6dfba73933bfb69602f3f0e26159798cc",
|
|
445
445
|
"digestExcludes": []
|
|
@@ -452,7 +452,7 @@
|
|
|
452
452
|
"id": "first-party",
|
|
453
453
|
"type": "first-party",
|
|
454
454
|
"package": "@1aboveio/skills",
|
|
455
|
-
"version": "0.
|
|
455
|
+
"version": "0.17.0"
|
|
456
456
|
},
|
|
457
457
|
"contentDigest": "90a7c4e1ad6da1e632ea2c5259e4967ffebb84353adccbca8dfc5d6bc50b60c1",
|
|
458
458
|
"digestExcludes": []
|
|
@@ -465,15 +465,15 @@
|
|
|
465
465
|
"id": "first-party",
|
|
466
466
|
"type": "first-party",
|
|
467
467
|
"package": "@1aboveio/skills",
|
|
468
|
-
"version": "0.
|
|
468
|
+
"version": "0.17.0"
|
|
469
469
|
},
|
|
470
|
-
"contentDigest": "
|
|
470
|
+
"contentDigest": "4d2242d41c7c57e24119b7cd974a88dd97cf526a8f76e6825385060ff86095c8",
|
|
471
471
|
"digestExcludes": [
|
|
472
472
|
"coherence/workflow.json"
|
|
473
473
|
]
|
|
474
474
|
}
|
|
475
475
|
],
|
|
476
|
-
"releaseIdentity": "
|
|
476
|
+
"releaseIdentity": "4a75f070c64c9f24bfbb250be0706a55f6fc3bf071d537200a0a5a91378a4959",
|
|
477
477
|
"lifecycleAuthority": "native-skills-cli",
|
|
478
478
|
"repairRecipe": {
|
|
479
479
|
"id": "engineering-workflow-dependency-first",
|
|
@@ -531,7 +531,7 @@
|
|
|
531
531
|
"sourceId": "first-party",
|
|
532
532
|
"sourceType": "first-party",
|
|
533
533
|
"package": "@1aboveio/skills",
|
|
534
|
-
"version": "0.
|
|
534
|
+
"version": "0.17.0",
|
|
535
535
|
"installPath": null,
|
|
536
536
|
"members": [
|
|
537
537
|
"harness-runtime",
|
|
@@ -547,7 +547,7 @@
|
|
|
547
547
|
"commands": [
|
|
548
548
|
{
|
|
549
549
|
"transport": "npm",
|
|
550
|
-
"command": "npx @1aboveio/skills@0.
|
|
550
|
+
"command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
|
|
551
551
|
}
|
|
552
552
|
],
|
|
553
553
|
"onFailure": {
|
|
@@ -237,7 +237,7 @@
|
|
|
237
237
|
"id": "first-party",
|
|
238
238
|
"type": "first-party",
|
|
239
239
|
"package": "@1aboveio/skills",
|
|
240
|
-
"version": "0.
|
|
240
|
+
"version": "0.17.0"
|
|
241
241
|
}
|
|
242
242
|
},
|
|
243
243
|
{
|
|
@@ -248,7 +248,7 @@
|
|
|
248
248
|
"id": "first-party",
|
|
249
249
|
"type": "first-party",
|
|
250
250
|
"package": "@1aboveio/skills",
|
|
251
|
-
"version": "0.
|
|
251
|
+
"version": "0.17.0"
|
|
252
252
|
}
|
|
253
253
|
},
|
|
254
254
|
{
|
|
@@ -259,7 +259,7 @@
|
|
|
259
259
|
"id": "first-party",
|
|
260
260
|
"type": "first-party",
|
|
261
261
|
"package": "@1aboveio/skills",
|
|
262
|
-
"version": "0.
|
|
262
|
+
"version": "0.17.0"
|
|
263
263
|
}
|
|
264
264
|
},
|
|
265
265
|
{
|
|
@@ -270,7 +270,7 @@
|
|
|
270
270
|
"id": "first-party",
|
|
271
271
|
"type": "first-party",
|
|
272
272
|
"package": "@1aboveio/skills",
|
|
273
|
-
"version": "0.
|
|
273
|
+
"version": "0.17.0"
|
|
274
274
|
}
|
|
275
275
|
},
|
|
276
276
|
{
|
|
@@ -281,7 +281,7 @@
|
|
|
281
281
|
"id": "first-party",
|
|
282
282
|
"type": "first-party",
|
|
283
283
|
"package": "@1aboveio/skills",
|
|
284
|
-
"version": "0.
|
|
284
|
+
"version": "0.17.0"
|
|
285
285
|
}
|
|
286
286
|
},
|
|
287
287
|
{
|
|
@@ -292,7 +292,7 @@
|
|
|
292
292
|
"id": "first-party",
|
|
293
293
|
"type": "first-party",
|
|
294
294
|
"package": "@1aboveio/skills",
|
|
295
|
-
"version": "0.
|
|
295
|
+
"version": "0.17.0"
|
|
296
296
|
}
|
|
297
297
|
},
|
|
298
298
|
{
|
|
@@ -303,7 +303,7 @@
|
|
|
303
303
|
"id": "first-party",
|
|
304
304
|
"type": "first-party",
|
|
305
305
|
"package": "@1aboveio/skills",
|
|
306
|
-
"version": "0.
|
|
306
|
+
"version": "0.17.0"
|
|
307
307
|
}
|
|
308
308
|
},
|
|
309
309
|
{
|
|
@@ -314,7 +314,7 @@
|
|
|
314
314
|
"id": "first-party",
|
|
315
315
|
"type": "first-party",
|
|
316
316
|
"package": "@1aboveio/skills",
|
|
317
|
-
"version": "0.
|
|
317
|
+
"version": "0.17.0"
|
|
318
318
|
}
|
|
319
319
|
},
|
|
320
320
|
{
|
|
@@ -325,7 +325,7 @@
|
|
|
325
325
|
"id": "first-party",
|
|
326
326
|
"type": "first-party",
|
|
327
327
|
"package": "@1aboveio/skills",
|
|
328
|
-
"version": "0.
|
|
328
|
+
"version": "0.17.0"
|
|
329
329
|
}
|
|
330
330
|
}
|
|
331
331
|
],
|
|
@@ -426,7 +426,7 @@
|
|
|
426
426
|
"sourceId": "first-party",
|
|
427
427
|
"sourceType": "first-party",
|
|
428
428
|
"package": "@1aboveio/skills",
|
|
429
|
-
"version": "0.
|
|
429
|
+
"version": "0.17.0",
|
|
430
430
|
"installPath": null,
|
|
431
431
|
"members": [
|
|
432
432
|
"harness-runtime",
|
|
@@ -442,7 +442,7 @@
|
|
|
442
442
|
"commands": [
|
|
443
443
|
{
|
|
444
444
|
"transport": "npm",
|
|
445
|
-
"command": "npx @1aboveio/skills@0.
|
|
445
|
+
"command": "npx @1aboveio/skills@0.17.0 install --group engineering-workflow --yes"
|
|
446
446
|
}
|
|
447
447
|
],
|
|
448
448
|
"onFailure": {
|
|
@@ -3412,7 +3412,7 @@ export const WORKFLOW_PREFLIGHT_COMMANDS = Object.freeze([
|
|
|
3412
3412
|
|
|
3413
3413
|
const WORKFLOW_VERIFIER_URL = new URL('../../engineering-runtime/scripts/workflow-coherence.mjs', import.meta.url)
|
|
3414
3414
|
const WORKFLOW_FALLBACK_POLICY_URL = new URL('../generated/workflow-repair-policy.json', import.meta.url)
|
|
3415
|
-
export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = '
|
|
3415
|
+
export const WORKFLOW_TRUSTED_FALLBACK_POLICY_SHA256 = 'a6d80d500e43b660220f71f9764e5e6c905bf1955d63f8733d115b6c1a601448'
|
|
3416
3416
|
const WORKFLOW_REPAIR_RECIPE_REFERENCE = Object.freeze({
|
|
3417
3417
|
id: 'engineering-workflow-dependency-first',
|
|
3418
3418
|
generatedFrom: 'skills/distribution/generated/recipes.json',
|