claude-autopm 2.8.2 → 2.8.4

package/packages/plugin-data/rules/etl-pipeline-standards.md

@@ -0,0 +1,255 @@
+# ETL Pipeline Standards
+
+**Priority:** HIGH
+**Applies To:** commands, agents
+**Enforces On:** airflow-orchestration-expert, kedro-pipeline-expert
+
+ETL/ELT pipeline best practices with Context7-verified patterns from Airflow, Kedro, dbt, and pandas.
+
+## Context7 Documentation Requirements
+
+**MANDATORY**: Query Context7 before implementing ETL pipelines for framework-specific patterns.
+
+## Core ETL Principles
+
+### 1. Idempotency
+
+**Rule:** All ETL operations must be idempotent - safe to re-run multiple times.
+
+```python
+# ✅ CORRECT: Idempotent load using upsert
+def load_to_warehouse(df: pd.DataFrame, table: str):
+    """
+    Idempotent load using MERGE/UPSERT.
+
+    Context7 Pattern: Prevents duplicates on retry
+    """
+    # Use MERGE/UPSERT instead of INSERT
+    connection.execute(f"""
+        MERGE INTO {table} AS target
+        USING temp_table AS source
+        ON target.id = source.id
+        WHEN MATCHED THEN UPDATE SET ...
+        WHEN NOT MATCHED THEN INSERT ...
+    """)
+```
+
+### 2. Incremental Processing
+
+**Rule:** Process only new/changed data when possible.
+
+```python
+# ✅ CORRECT: Incremental extraction
+@task
+def extract_incremental(last_run_timestamp: str) -> Dict:
+    """
+    Extract only new records since last run.
+
+    Context7 Pattern (Airflow): TaskFlow with state tracking
+    """
+    query = f"""
+        SELECT * FROM source_table
+        WHERE updated_at > '{last_run_timestamp}'
+    """
+    data = execute_query(query)
+    return {"data": data, "timestamp": datetime.utcnow().isoformat()}
+```
+
+### 3. Error Handling and Retries
+
+**Rule:** Implement proper error handling with exponential backoff.
+
+```python
+# ✅ CORRECT: Airflow retry configuration
+default_args = {
+    "retries": 3,
+    "retry_delay": timedelta(minutes=5),
+    "retry_exponential_backoff": True,
+    "max_retry_delay": timedelta(hours=1),
+}
+```
+
+### 4. Data Lineage Tracking
+
+**Rule:** Track data provenance and transformations.
+
+```python
+# ✅ CORRECT: Kedro pipeline with clear lineage
+def create_pipeline(**kwargs) -> Pipeline:
+    return pipeline([
+        node(
+            func=extract_raw_data,
+            inputs=None,
+            outputs="raw_data",  # Clear lineage
+            name="extract",
+        ),
+        node(
+            func=transform_data,
+            inputs="raw_data",
+            outputs="transformed_data",  # Tracks transformation
+            name="transform",
+        ),
+        node(
+            func=load_data,
+            inputs="transformed_data",
+            outputs=None,
+            name="load",
+        ),
+    ])
+```
+
+### 5. Monitoring and Logging
+
+**Rule:** Log all stages with metrics and timing.
+
+```python
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+def etl_stage_wrapper(func):
+    """Decorator for ETL stage logging."""
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        logger.info(f"Starting {func.__name__}")
+
+        try:
+            result = func(*args, **kwargs)
+            duration = time.time() - start_time
+
+            logger.info(
+                f"Completed {func.__name__} in {duration:.2f}s"
+            )
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed {func.__name__}: {e}")
+            raise
+
+    return wrapper
+```
+
+## Pipeline Patterns
+
+### Extract Phase
+
+```python
+@task
+def extract_from_api(endpoint: str) -> Dict:
+    """
+    Extract data from API with retry logic.
+
+    Context7 Pattern: Robust extraction
+    """
+    max_retries = 3
+    for attempt in range(max_retries):
+        try:
+            response = requests.get(endpoint, timeout=30)
+            response.raise_for_status()
+            return response.json()
+        except RequestException as e:
+            if attempt == max_retries - 1:
+                raise
+            time.sleep(2 ** attempt)  # Exponential backoff
+```
+
+### Transform Phase
+
+```python
+def transform_data(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Transform data with Context7 pandas patterns.
+
+    Context7 Pattern: Vectorized operations
+    """
+    # ✅ CORRECT: Vectorization over apply
+    df["total"] = df["quantity"] * df["price"]
+
+    # ✅ CORRECT: GroupBy aggregations
+    aggregated = df.groupby("customer_id").agg({
+        "total": ["sum", "mean", "count"],
+        "order_date": "max"
+    })
+
+    # ✅ CORRECT: Efficient merging
+    result = df.merge(
+        customers,
+        on="customer_id",
+        how="left",
+        validate="many_to_one"
+    )
+
+    return result
+```
+
+### Load Phase
+
+```python
+def load_to_warehouse(
+    df: pd.DataFrame,
+    table: str,
+    mode: str = "append"
+):
+    """
+    Load data to warehouse with proper handling.
+
+    Args:
+        df: Data to load
+        table: Target table
+        mode: 'append' or 'replace'
+    """
+    # Validate before loading
+    assert len(df) > 0, "Empty DataFrame"
+    assert not df.isnull().all().any(), "Column with all nulls"
+
+    # Load with transaction
+    with engine.begin() as conn:
+        df.to_sql(
+            table,
+            conn,
+            if_exists=mode,
+            index=False,
+            method="multi",  # Batch inserts
+            chunksize=1000
+        )
+
+    logger.info(f"Loaded {len(df)} rows to {table}")
+```
+
+## Testing Standards
+
+```python
+def test_etl_pipeline():
+    """Test ETL pipeline end-to-end."""
+    # Arrange: Setup test data
+    test_data = create_test_data()
+
+    # Act: Run pipeline
+    result = run_pipeline(test_data)
+
+    # Assert: Validate results
+    assert len(result) == expected_count
+    assert result["total"].sum() == expected_total
+    assert not result.isnull().any().any()
+```
+
+## Enforcement Checklist
+
+- [ ] Context7 documentation queried
+- [ ] Idempotent operations
+- [ ] Incremental processing where applicable
+- [ ] Error handling with retries
+- [ ] Data validation at each stage
+- [ ] Lineage tracking
+- [ ] Monitoring and logging
+- [ ] Tests cover all stages
+- [ ] Documentation complete
+
+## Related Resources
+
+- Rule: `data-quality-standards.md`
+- Command: `airflow-dag-scaffold.md`
+- Command: `kafka-pipeline-scaffold.md`
+- Script: `airflow-dag-example.py`
+- Script: `pandas-etl-example.py`

package/packages/plugin-data/scripts/examples/airflow-dag-example.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+Airflow DAG Example - Context7 Best Practices
+
+Demonstrates production-ready Airflow patterns from Context7:
+- TaskFlow API with @task decorators
+- Task dependencies with bitshift operators
+- Sensors for external dependencies
+- Error handling and retries
+- TaskGroups for organization
+
+Source: /apache/airflow (5,854 snippets, trust 9.1)
+"""
+
+from datetime import datetime, timedelta
+from typing import Dict, List
+
+from airflow.decorators import dag, task
+from airflow.operators.python import PythonOperator
+from airflow.sensors.external_task import ExternalTaskSensor
+from airflow.utils.task_group import TaskGroup
+
+# ✅ CORRECT: Proper default arguments with error handling
+default_args = {
+    "owner": "data_team",
+    "depends_on_past": False,
+    "email": ["alerts@example.com"],
+    "email_on_failure": True,
+    "email_on_retry": False,
+    "retries": 3,
+    "retry_delay": timedelta(minutes=5),
+    "retry_exponential_backoff": True,
+    "max_retry_delay": timedelta(hours=1),
+}
+
+
+@dag(
+    dag_id="example_etl_pipeline",
+    default_args=default_args,
+    description="Production ETL pipeline with Context7 patterns",
+    schedule_interval="@daily",
+    start_date=datetime(2025, 1, 1),
+    catchup=False,
+    tags=["example", "etl", "context7"],
+)
+def example_etl_pipeline():
+    """
+    Example ETL pipeline demonstrating Context7 patterns.
+
+    Stages:
+    1. Wait for upstream dependencies (sensors)
+    2. Extract data from multiple sources (parallel)
+    3. Transform and validate data
+    4. Load to warehouse
+    5. Send notifications
+    """
+
+    # ===================================================================
+    # DEPENDENCY CHECKS
+    # ===================================================================
+
+    # ✅ CORRECT: ExternalTaskSensor for upstream DAG dependency
+    wait_for_upstream = ExternalTaskSensor(
+        task_id="wait_for_upstream_dag",
+        external_dag_id="upstream_data_collection",
+        external_task_id="final_task",
+        allowed_states=["success"],
+        failed_states=["failed", "skipped"],
+        timeout=3600,
+        poke_interval=300,
+    )
+
+    # ===================================================================
+    # EXTRACTION PHASE (PARALLEL)
+    # ===================================================================
+
+    @task
+    def extract_from_database() -> Dict:
+        """
+        Extract data from database.
+
+        Context7 Pattern: TaskFlow with @task decorator
+        Returns data dict for XCom passing
+        """
+        print("Extracting from database...")
+
+        # Simulated extraction
+        data = {
+            "source": "database",
+            "records": list(range(100)),
+            "count": 100,
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+
+        print(f"✓ Extracted {data['count']} records from database")
+        return data
+
+    @task
+    def extract_from_api() -> Dict:
+        """
+        Extract data from external API.
+
+        Context7 Pattern: Parallel extraction with independent @task
+        """
+        print("Extracting from API...")
+
+        # Simulated API call
+        data = {
+            "source": "api",
+            "records": list(range(100, 150)),
+            "count": 50,
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+
+        print(f"✓ Extracted {data['count']} records from API")
+        return data
+
+    # ===================================================================
+    # TRANSFORMATION PHASE
+    # ===================================================================
+
+    @task
+    def transform_and_combine(db_data: Dict, api_data: Dict) -> Dict:
+        """
+        Transform and combine data from multiple sources.
+
+        Context7 Pattern: TaskFlow automatically handles XCom passing
+        Parameters match upstream task outputs
+        """
+        print("Transforming and combining data...")
+
+        combined_records = db_data["records"] + api_data["records"]
+        total_count = db_data["count"] + api_data["count"]
+
+        # Apply transformations
+        transformed = {
+            "records": [x * 2 for x in combined_records],  # Example transformation
+            "count": total_count,
+            "sources": [db_data["source"], api_data["source"]],
+            "transformation": "doubled values",
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+
+        print(f"✓ Transformed {transformed['count']} total records")
+        return transformed
+
+    @task
+    def validate_data(data: Dict) -> Dict:
+        """
+        Validate transformed data quality.
+
+        Context7 Pattern: Data quality checks in pipeline
+        """
+        print("Validating data quality...")
+
+        # Validation checks
+        assert data["count"] > 0, "No records to validate"
+        assert len(data["records"]) == data["count"], "Record count mismatch"
+        assert all(isinstance(x, int) for x in data["records"]), "Invalid record types"
+
+        print(f"✓ Validation passed for {data['count']} records")
+        return data
+
+    # ===================================================================
+    # LOADING PHASE
+    # ===================================================================
+
+    @task
+    def load_to_warehouse(data: Dict) -> Dict:
+        """
+        Load validated data to data warehouse.
+
+        Context7 Pattern: Final load task with success confirmation
+        """
+        print(f"Loading {data['count']} records to warehouse...")
+
+        # Simulated load operation
+        loaded_data = {
+            **data,
+            "loaded_at": datetime.utcnow().isoformat(),
+            "status": "success",
+        }
+
+        print(f"✓ Successfully loaded {data['count']} records")
+        return loaded_data
+
+    @task
+    def send_success_notification(data: Dict) -> None:
+        """
+        Send notification on pipeline success.
+
+        Context7 Pattern: Notification task for monitoring
+        """
+        message = (
+            f"ETL Pipeline completed successfully:\n"
+            f"- Records processed: {data['count']}\n"
+            f"- Sources: {', '.join(data['sources'])}\n"
+            f"- Loaded at: {data['loaded_at']}"
+        )
+
+        print(f"📧 Notification: {message}")
+        # In production: send email, Slack message, etc.
+
+    # ===================================================================
+    # DAG STRUCTURE WITH CONTEXT7 PATTERNS
+    # ===================================================================
+
+    # Context7 Pattern: Sensor before extraction
+    sensor_task = wait_for_upstream
+
+    # Context7 Pattern: Parallel extraction (TaskFlow resolves automatically)
+    db_data = extract_from_database()
+    api_data = extract_from_api()
+
+    # Context7 Pattern: Pass multiple inputs to transformation
+    transformed = transform_and_combine(db_data, api_data)
+
+    # Context7 Pattern: Sequential validation and loading
+    validated = validate_data(transformed)
+    loaded = load_to_warehouse(validated)
+    notify = send_success_notification(loaded)
+
+    # Context7 Pattern: Bitshift operator for explicit dependencies
+    sensor_task >> [db_data, api_data]
+    loaded >> notify
+
+
+# Instantiate DAG
+dag_instance = example_etl_pipeline()
+
+if __name__ == "__main__":
+    print("Airflow DAG Example - Context7 Best Practices")
+    print("=" * 60)
+    print("")
+    print("Context7 Patterns Demonstrated:")
+    print("1. ✅ TaskFlow API with @task decorators")
+    print("2. ✅ Parallel extraction from multiple sources")
+    print("3. ✅ ExternalTaskSensor for upstream dependencies")
+    print("4. ✅ Automatic XCom passing between tasks")
+    print("5. ✅ Bitshift operators for dependencies (>>, <<)")
+    print("6. ✅ Error handling with retries and exponential backoff")
+    print("7. ✅ Data validation in pipeline")
+    print("8. ✅ Notification on completion")
+    print("")
+    print("Source: /apache/airflow (5,854 snippets, trust 9.1)")