aigroup-workflow 2.1.1 → 2.2.0

package/skills/ai-ml/ml-pipeline/references/pipeline-orchestration.md

@@ -0,0 +1,907 @@
+# Pipeline Orchestration
+
+---
+
+## Overview
+
+Pipeline orchestration automates the end-to-end ML workflow from data ingestion through model deployment. Orchestrators manage dependencies, handle failures, enable scheduling, and provide observability across complex multi-step pipelines.
+
+## When to Use This Reference
+
+- Building Kubeflow Pipelines for ML workflows
+- Creating Airflow DAGs for data and ML pipelines
+- Implementing Prefect flows for modern orchestration
+- Designing pipeline DAGs and component dependencies
+- Setting up scheduled retraining workflows
+
+## When NOT to Use
+
+- Simple linear scripts without dependencies
+- One-off data processing tasks
+- Interactive development and experimentation
+
+---
+
+## Kubeflow Pipelines
+
+### Pipeline Definition (KFP v2)
+
+```python
+from kfp import dsl
+from kfp.dsl import Input, Output, Artifact, Dataset, Model, Metrics
+from kfp import compiler
+from typing import NamedTuple
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn"],
+)
+def load_data(
+    data_path: str,
+    output_dataset: Output[Dataset],
+) -> None:
+    """Load and validate raw data."""
+    import pandas as pd
+
+    df = pd.read_parquet(data_path)
+
+    # Basic validation
+    assert len(df) > 0, "Dataset is empty"
+    assert "target" in df.columns, "Missing target column"
+
+    df.to_parquet(output_dataset.path)
+    output_dataset.metadata["num_rows"] = len(df)
+    output_dataset.metadata["num_features"] = len(df.columns) - 1
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn"],
+)
+def preprocess_data(
+    input_dataset: Input[Dataset],
+    train_dataset: Output[Dataset],
+    test_dataset: Output[Dataset],
+    test_size: float = 0.2,
+    random_state: int = 42,
+) -> None:
+    """Preprocess and split data."""
+    import pandas as pd
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    df = pd.read_parquet(input_dataset.path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=random_state
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+    train_df.to_parquet(train_dataset.path)
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+    test_df.to_parquet(test_dataset.path)
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn", "joblib"],
+)
+def train_model(
+    train_dataset: Input[Dataset],
+    model_artifact: Output[Model],
+    n_estimators: int = 100,
+    max_depth: int = 10,
+) -> None:
+    """Train RandomForest model."""
+    import pandas as pd
+    from sklearn.ensemble import RandomForestClassifier
+    import joblib
+
+    df = pd.read_parquet(train_dataset.path)
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    model = RandomForestClassifier(
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+        random_state=42,
+    )
+    model.fit(X, y)
+
+    joblib.dump(model, model_artifact.path)
+    model_artifact.metadata["n_estimators"] = n_estimators
+    model_artifact.metadata["max_depth"] = max_depth
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn", "joblib"],
+)
+def evaluate_model(
+    model_artifact: Input[Model],
+    test_dataset: Input[Dataset],
+    metrics: Output[Metrics],
+    threshold: float = 0.8,
+) -> NamedTuple("Outputs", [("passed", bool), ("accuracy", float)]):
+    """Evaluate model and check threshold."""
+    import pandas as pd
+    from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
+    import joblib
+    from collections import namedtuple
+
+    model = joblib.load(model_artifact.path)
+    df = pd.read_parquet(test_dataset.path)
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    predictions = model.predict(X)
+
+    accuracy = accuracy_score(y, predictions)
+    precision = precision_score(y, predictions, average="weighted")
+    recall = recall_score(y, predictions, average="weighted")
+    f1 = f1_score(y, predictions, average="weighted")
+
+    metrics.log_metric("accuracy", accuracy)
+    metrics.log_metric("precision", precision)
+    metrics.log_metric("recall", recall)
+    metrics.log_metric("f1_score", f1)
+
+    passed = accuracy >= threshold
+
+    Outputs = namedtuple("Outputs", ["passed", "accuracy"])
+    return Outputs(passed, accuracy)
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["google-cloud-storage"],
+)
+def deploy_model(
+    model_artifact: Input[Model],
+    model_name: str,
+    endpoint: str,
+) -> str:
+    """Deploy model to serving endpoint."""
+    from google.cloud import storage
+    import shutil
+
+    # Copy model to GCS
+    bucket_name = endpoint.split("/")[2]
+    model_path = f"models/{model_name}/model.joblib"
+
+    client = storage.Client()
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(model_path)
+    blob.upload_from_filename(model_artifact.path)
+
+    return f"gs://{bucket_name}/{model_path}"
+
+@dsl.pipeline(
+    name="ml-training-pipeline",
+    description="End-to-end ML training pipeline",
+)
+def ml_pipeline(
+    data_path: str,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+    accuracy_threshold: float = 0.8,
+    model_name: str = "classifier",
+    endpoint: str = "gs://ml-models/serving",
+) -> None:
+    """Complete ML training pipeline."""
+
+    load_task = load_data(data_path=data_path)
+
+    preprocess_task = preprocess_data(
+        input_dataset=load_task.outputs["output_dataset"],
+    )
+
+    train_task = train_model(
+        train_dataset=preprocess_task.outputs["train_dataset"],
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+    )
+
+    evaluate_task = evaluate_model(
+        model_artifact=train_task.outputs["model_artifact"],
+        test_dataset=preprocess_task.outputs["test_dataset"],
+        threshold=accuracy_threshold,
+    )
+
+    with dsl.If(evaluate_task.outputs["passed"] == True):
+        deploy_model(
+            model_artifact=train_task.outputs["model_artifact"],
+            model_name=model_name,
+            endpoint=endpoint,
+        )
+
+# Compile pipeline
+if __name__ == "__main__":
+    compiler.Compiler().compile(
+        ml_pipeline,
+        "ml_pipeline.yaml",
+    )
+```
+
+### Running Kubeflow Pipelines
+
+```python
+from kfp.client import Client
+
+def run_pipeline(
+    pipeline_file: str,
+    experiment_name: str,
+    run_name: str,
+    parameters: dict,
+) -> str:
+    """Submit pipeline run to Kubeflow."""
+    client = Client(host="https://kubeflow.example.com/pipeline")
+
+    # Create or get experiment
+    experiment = client.create_experiment(name=experiment_name)
+
+    # Submit run
+    run = client.create_run_from_pipeline_package(
+        pipeline_file=pipeline_file,
+        experiment_id=experiment.experiment_id,
+        run_name=run_name,
+        arguments=parameters,
+    )
+
+    return run.run_id
+
+def schedule_pipeline(
+    pipeline_file: str,
+    experiment_name: str,
+    schedule_name: str,
+    cron_expression: str,
+    parameters: dict,
+) -> str:
+    """Create recurring pipeline run."""
+    client = Client(host="https://kubeflow.example.com/pipeline")
+
+    experiment = client.create_experiment(name=experiment_name)
+
+    # Create recurring run
+    job = client.create_recurring_run(
+        experiment_id=experiment.experiment_id,
+        job_name=schedule_name,
+        pipeline_package_path=pipeline_file,
+        cron_expression=cron_expression,
+        enabled=True,
+        parameters=parameters,
+    )
+
+    return job.id
+```
+
+---
+
+## Apache Airflow
+
+### ML Pipeline DAG
+
+```python
+from airflow import DAG
+from airflow.operators.python import PythonOperator, BranchPythonOperator
+from airflow.operators.empty import EmptyOperator
+from airflow.providers.amazon.aws.operators.s3 import S3CreateObjectOperator
+from airflow.utils.trigger_rule import TriggerRule
+from datetime import datetime, timedelta
+import json
+
+default_args = {
+    "owner": "ml-team",
+    "depends_on_past": False,
+    "email_on_failure": True,
+    "email_on_retry": False,
+    "retries": 2,
+    "retry_delay": timedelta(minutes=5),
+    "execution_timeout": timedelta(hours=2),
+}
+
+def load_data(**context):
+    """Load data from source."""
+    import pandas as pd
+
+    data_path = context["params"]["data_path"]
+    df = pd.read_parquet(data_path)
+
+    # Push to XCom for downstream tasks
+    output_path = f"/tmp/data_{context['run_id']}.parquet"
+    df.to_parquet(output_path)
+
+    context["ti"].xcom_push(key="data_path", value=output_path)
+    context["ti"].xcom_push(key="num_rows", value=len(df))
+
+    return output_path
+
+def preprocess_data(**context):
+    """Preprocess and split data."""
+    import pandas as pd
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    input_path = context["ti"].xcom_pull(key="data_path", task_ids="load_data")
+    df = pd.read_parquet(input_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    # Save processed data
+    train_path = f"/tmp/train_{context['run_id']}.parquet"
+    test_path = f"/tmp/test_{context['run_id']}.parquet"
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+    train_df.to_parquet(train_path)
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+    test_df.to_parquet(test_path)
+
+    context["ti"].xcom_push(key="train_path", value=train_path)
+    context["ti"].xcom_push(key="test_path", value=test_path)
+
+def train_model(**context):
+    """Train ML model."""
+    import pandas as pd
+    from sklearn.ensemble import RandomForestClassifier
+    import joblib
+
+    train_path = context["ti"].xcom_pull(key="train_path", task_ids="preprocess_data")
+    df = pd.read_parquet(train_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    params = context["params"]
+    model = RandomForestClassifier(
+        n_estimators=params.get("n_estimators", 100),
+        max_depth=params.get("max_depth", 10),
+        random_state=42,
+    )
+    model.fit(X, y)
+
+    model_path = f"/tmp/model_{context['run_id']}.joblib"
+    joblib.dump(model, model_path)
+
+    context["ti"].xcom_push(key="model_path", value=model_path)
+
+def evaluate_model(**context):
+    """Evaluate model and return metrics."""
+    import pandas as pd
+    from sklearn.metrics import accuracy_score, precision_score, recall_score
+    import joblib
+
+    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
+    test_path = context["ti"].xcom_pull(key="test_path", task_ids="preprocess_data")
+
+    model = joblib.load(model_path)
+    df = pd.read_parquet(test_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    predictions = model.predict(X)
+
+    metrics = {
+        "accuracy": accuracy_score(y, predictions),
+        "precision": precision_score(y, predictions, average="weighted"),
+        "recall": recall_score(y, predictions, average="weighted"),
+    }
+
+    context["ti"].xcom_push(key="metrics", value=metrics)
+
+    return metrics
+
+def check_metrics_threshold(**context):
+    """Branch based on model performance."""
+    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
+    threshold = context["params"].get("accuracy_threshold", 0.8)
+
+    if metrics["accuracy"] >= threshold:
+        return "deploy_model"
+    return "skip_deployment"
+
+def deploy_model(**context):
+    """Deploy model to production."""
+    import shutil
+
+    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
+    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
+
+    # In production, this would upload to model registry/serving
+    deploy_path = f"/models/production/model_{context['run_id']}.joblib"
+    shutil.copy(model_path, deploy_path)
+
+    return {
+        "model_path": deploy_path,
+        "metrics": metrics,
+        "deployed_at": datetime.utcnow().isoformat(),
+    }
+
+with DAG(
+    dag_id="ml_training_pipeline",
+    default_args=default_args,
+    description="End-to-end ML training pipeline",
+    schedule_interval="0 2 * * *",  # Daily at 2 AM
+    start_date=datetime(2024, 1, 1),
+    catchup=False,
+    tags=["ml", "training", "production"],
+    params={
+        "data_path": "s3://data-bucket/training_data.parquet",
+        "n_estimators": 100,
+        "max_depth": 10,
+        "accuracy_threshold": 0.8,
+    },
+) as dag:
+
+    start = EmptyOperator(task_id="start")
+
+    load = PythonOperator(
+        task_id="load_data",
+        python_callable=load_data,
+    )
+
+    preprocess = PythonOperator(
+        task_id="preprocess_data",
+        python_callable=preprocess_data,
+    )
+
+    train = PythonOperator(
+        task_id="train_model",
+        python_callable=train_model,
+    )
+
+    evaluate = PythonOperator(
+        task_id="evaluate_model",
+        python_callable=evaluate_model,
+    )
+
+    check_threshold = BranchPythonOperator(
+        task_id="check_metrics_threshold",
+        python_callable=check_metrics_threshold,
+    )
+
+    deploy = PythonOperator(
+        task_id="deploy_model",
+        python_callable=deploy_model,
+    )
+
+    skip = EmptyOperator(task_id="skip_deployment")
+
+    end = EmptyOperator(
+        task_id="end",
+        trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS,
+    )
+
+    start >> load >> preprocess >> train >> evaluate >> check_threshold
+    check_threshold >> [deploy, skip] >> end
+```
+
+---
+
+## Prefect
+
+### Modern Flow-Based Pipeline
+
+```python
+from prefect import flow, task, get_run_logger
+from prefect.artifacts import create_markdown_artifact
+from prefect.tasks import task_input_hash
+from datetime import timedelta
+import pandas as pd
+
+@task(
+    retries=3,
+    retry_delay_seconds=60,
+    cache_key_fn=task_input_hash,
+    cache_expiration=timedelta(hours=1),
+)
+def load_data(data_path: str) -> pd.DataFrame:
+    """Load data with caching."""
+    logger = get_run_logger()
+    logger.info(f"Loading data from {data_path}")
+
+    df = pd.read_parquet(data_path)
+    logger.info(f"Loaded {len(df)} rows")
+
+    return df
+
+@task(retries=2)
+def preprocess_data(
+    df: pd.DataFrame,
+    test_size: float = 0.2,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Preprocess and split data."""
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    logger = get_run_logger()
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=42
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+
+    logger.info(f"Train: {len(train_df)}, Test: {len(test_df)}")
+
+    return train_df, test_df
+
+@task
+def train_model(
+    train_df: pd.DataFrame,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+):
+    """Train RandomForest model."""
+    from sklearn.ensemble import RandomForestClassifier
+
+    logger = get_run_logger()
+
+    X = train_df.drop("target", axis=1)
+    y = train_df["target"]
+
+    model = RandomForestClassifier(
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+        random_state=42,
+        n_jobs=-1,
+    )
+
+    logger.info("Training model...")
+    model.fit(X, y)
+    logger.info("Training complete")
+
+    return model
+
+@task
+def evaluate_model(model, test_df: pd.DataFrame) -> dict:
+    """Evaluate model and create artifact."""
+    from sklearn.metrics import (
+        accuracy_score, precision_score, recall_score,
+        f1_score, classification_report
+    )
+
+    logger = get_run_logger()
+
+    X = test_df.drop("target", axis=1)
+    y = test_df["target"]
+
+    predictions = model.predict(X)
+
+    metrics = {
+        "accuracy": accuracy_score(y, predictions),
+        "precision": precision_score(y, predictions, average="weighted"),
+        "recall": recall_score(y, predictions, average="weighted"),
+        "f1_score": f1_score(y, predictions, average="weighted"),
+    }
+
+    logger.info(f"Metrics: {metrics}")
+
+    # Create markdown artifact for Prefect UI
+    report = classification_report(y, predictions)
+    markdown = f"""
+# Model Evaluation Report
+
+## Metrics
+| Metric | Value |
+|--------|-------|
+| Accuracy | {metrics['accuracy']:.4f} |
+| Precision | {metrics['precision']:.4f} |
+| Recall | {metrics['recall']:.4f} |
+| F1 Score | {metrics['f1_score']:.4f} |
+
+## Classification Report
+```
+{report}
+```
+"""
+    create_markdown_artifact(
+        key="model-evaluation",
+        markdown=markdown,
+        description="Model evaluation metrics",
+    )
+
+    return metrics
+
+@task
+def deploy_model(model, metrics: dict, threshold: float) -> bool:
+    """Deploy model if metrics pass threshold."""
+    import joblib
+    from datetime import datetime
+
+    logger = get_run_logger()
+
+    if metrics["accuracy"] < threshold:
+        logger.warning(
+            f"Model accuracy {metrics['accuracy']:.4f} below threshold {threshold}"
+        )
+        return False
+
+    # Save model
+    model_path = f"/models/model_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.joblib"
+    joblib.dump(model, model_path)
+    logger.info(f"Model deployed to {model_path}")
+
+    return True
+
+@flow(
+    name="ml-training-pipeline",
+    description="End-to-end ML training pipeline",
+    retries=1,
+    retry_delay_seconds=300,
+)
+def ml_training_flow(
+    data_path: str,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+    accuracy_threshold: float = 0.8,
+) -> dict:
+    """Main ML training flow."""
+    logger = get_run_logger()
+    logger.info("Starting ML training pipeline")
+
+    # Load and preprocess
+    df = load_data(data_path)
+    train_df, test_df = preprocess_data(df)
+
+    # Train and evaluate
+    model = train_model(train_df, n_estimators, max_depth)
+    metrics = evaluate_model(model, test_df)
+
+    # Deploy if threshold met
+    deployed = deploy_model(model, metrics, accuracy_threshold)
+
+    return {
+        "metrics": metrics,
+        "deployed": deployed,
+    }
+
+# Deployment configuration
+if __name__ == "__main__":
+    from prefect.deployments import Deployment
+    from prefect.server.schemas.schedules import CronSchedule
+
+    deployment = Deployment.build_from_flow(
+        flow=ml_training_flow,
+        name="daily-training",
+        schedule=CronSchedule(cron="0 2 * * *"),
+        parameters={
+            "data_path": "s3://data/training.parquet",
+            "n_estimators": 100,
+            "max_depth": 10,
+            "accuracy_threshold": 0.8,
+        },
+        tags=["ml", "production"],
+        work_queue_name="ml-queue",
+    )
+
+    deployment.apply()
+```
+
+---
+
+## DAG Design Patterns
+
+### Parallel Processing Pattern
+
+```python
+from prefect import flow, task, unmapped
+from typing import List
+
+@task
+def process_partition(partition_id: int, data_path: str) -> dict:
+    """Process single data partition."""
+    # Process partition
+    return {"partition_id": partition_id, "records_processed": 1000}
+
+@task
+def aggregate_results(results: List[dict]) -> dict:
+    """Aggregate parallel processing results."""
+    total_records = sum(r["records_processed"] for r in results)
+    return {"total_records": total_records}
+
+@flow
+def parallel_processing_flow(data_path: str, num_partitions: int = 4):
+    """Process data in parallel partitions."""
+
+    # Map over partitions
+    partition_results = process_partition.map(
+        partition_id=range(num_partitions),
+        data_path=unmapped(data_path),
+    )
+
+    # Aggregate results
+    final_result = aggregate_results(partition_results)
+
+    return final_result
+```
+
+### Conditional Branching Pattern
+
+```python
+from prefect import flow, task
+
+@task
+def check_data_quality(df) -> bool:
+    """Check if data meets quality standards."""
+    null_ratio = df.isnull().sum().sum() / df.size
+    return null_ratio < 0.1
+
+@task
+def handle_poor_quality(df):
+    """Handle data that fails quality checks."""
+    # Impute, clean, or alert
+    pass
+
+@task
+def process_good_quality(df):
+    """Process data that passes quality checks."""
+    pass
+
+@flow
+def conditional_flow(data_path: str):
+    """Flow with conditional branching."""
+    df = load_data(data_path)
+    quality_ok = check_data_quality(df)
+
+    if quality_ok:
+        result = process_good_quality(df)
+    else:
+        result = handle_poor_quality(df)
+
+    return result
+```
+
+### Error Handling Pattern
+
+```python
+from prefect import flow, task
+from prefect.states import Failed
+
+@task
+def risky_operation():
+    """Operation that might fail."""
+    import random
+    if random.random() < 0.3:
+        raise ValueError("Random failure")
+    return "success"
+
+@task
+def fallback_operation():
+    """Fallback when primary fails."""
+    return "fallback_result"
+
+@task
+def send_alert(error: Exception):
+    """Send alert on failure."""
+    # Send to Slack, PagerDuty, etc.
+    pass
+
+@flow
+def resilient_flow():
+    """Flow with error handling."""
+    try:
+        result = risky_operation()
+    except Exception as e:
+        send_alert(e)
+        result = fallback_operation()
+
+    return result
+```
+
+---
+
+## Best Practices
+
+### Pipeline Configuration
+
+```yaml
+# pipeline_config.yaml
+pipeline:
+  name: ml-training
+  version: "1.0.0"
+  description: "Production ML training pipeline"
+
+stages:
+  - name: load_data
+    timeout: 300
+    retries: 3
+
+  - name: preprocess
+    timeout: 600
+    retries: 2
+    depends_on: [load_data]
+
+  - name: train
+    timeout: 3600
+    retries: 1
+    depends_on: [preprocess]
+    resources:
+      cpu: 4
+      memory: 16Gi
+      gpu: 1
+
+  - name: evaluate
+    timeout: 300
+    depends_on: [train]
+
+  - name: deploy
+    timeout: 300
+    depends_on: [evaluate]
+    condition: "evaluate.metrics.accuracy >= 0.8"
+
+schedule:
+  cron: "0 2 * * *"
+  timezone: "UTC"
+
+notifications:
+  on_failure:
+    - slack: "#ml-alerts"
+    - email: ml-team@company.com
+  on_success:
+    - slack: "#ml-notifications"
+```
+
+### Idempotency Guidelines
+
+```python
+# Good: Idempotent operations
+def process_data(run_id: str, data_path: str):
+    """Idempotent data processing."""
+    output_path = f"s3://processed/{run_id}/data.parquet"
+
+    # Check if already processed
+    if file_exists(output_path):
+        return output_path
+
+    # Process and save
+    df = pd.read_parquet(data_path)
+    processed = transform(df)
+    processed.to_parquet(output_path)
+
+    return output_path
+```
+
+---
+
+## Related References
+
+- `training-pipelines.md` - Training components for pipelines
+- `experiment-tracking.md` - Logging pipeline runs
+- `feature-engineering.md` - Feature pipeline components
+- `model-validation.md` - Validation stages in pipelines
+
+## Cross-Reference Skills
+
+- **DevOps Engineer** - CI/CD for pipeline deployment
+- **Kubernetes Specialist** - Running pipelines on K8s
+- **Cloud Architect** - Cloud infrastructure for orchestration