aigroup-workflow 2.2.1 → 2.2.2

package/skills/ai-ml/ml-pipeline/references/pipeline-orchestration.md

@@ -1,907 +1,907 @@
-# Pipeline Orchestration
-
----
-
-## Overview
-
-Pipeline orchestration automates the end-to-end ML workflow from data ingestion through model deployment. Orchestrators manage dependencies, handle failures, enable scheduling, and provide observability across complex multi-step pipelines.
-
-## When to Use This Reference
-
-- Building Kubeflow Pipelines for ML workflows
-- Creating Airflow DAGs for data and ML pipelines
-- Implementing Prefect flows for modern orchestration
-- Designing pipeline DAGs and component dependencies
-- Setting up scheduled retraining workflows
-
-## When NOT to Use
-
-- Simple linear scripts without dependencies
-- One-off data processing tasks
-- Interactive development and experimentation
-
----
-
-## Kubeflow Pipelines
-
-### Pipeline Definition (KFP v2)
-
-```python
-from kfp import dsl
-from kfp.dsl import Input, Output, Artifact, Dataset, Model, Metrics
-from kfp import compiler
-from typing import NamedTuple
-
-@dsl.component(
-    base_image="python:3.11-slim",
-    packages_to_install=["pandas", "scikit-learn"],
-)
-def load_data(
-    data_path: str,
-    output_dataset: Output[Dataset],
-) -> None:
-    """Load and validate raw data."""
-    import pandas as pd
-
-    df = pd.read_parquet(data_path)
-
-    # Basic validation
-    assert len(df) > 0, "Dataset is empty"
-    assert "target" in df.columns, "Missing target column"
-
-    df.to_parquet(output_dataset.path)
-    output_dataset.metadata["num_rows"] = len(df)
-    output_dataset.metadata["num_features"] = len(df.columns) - 1
-
-@dsl.component(
-    base_image="python:3.11-slim",
-    packages_to_install=["pandas", "scikit-learn"],
-)
-def preprocess_data(
-    input_dataset: Input[Dataset],
-    train_dataset: Output[Dataset],
-    test_dataset: Output[Dataset],
-    test_size: float = 0.2,
-    random_state: int = 42,
-) -> None:
-    """Preprocess and split data."""
-    import pandas as pd
-    from sklearn.model_selection import train_test_split
-    from sklearn.preprocessing import StandardScaler
-
-    df = pd.read_parquet(input_dataset.path)
-
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=test_size, random_state=random_state
-    )
-
-    scaler = StandardScaler()
-    X_train_scaled = scaler.fit_transform(X_train)
-    X_test_scaled = scaler.transform(X_test)
-
-    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
-    train_df["target"] = y_train.values
-    train_df.to_parquet(train_dataset.path)
-
-    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
-    test_df["target"] = y_test.values
-    test_df.to_parquet(test_dataset.path)
-
-@dsl.component(
-    base_image="python:3.11-slim",
-    packages_to_install=["pandas", "scikit-learn", "joblib"],
-)
-def train_model(
-    train_dataset: Input[Dataset],
-    model_artifact: Output[Model],
-    n_estimators: int = 100,
-    max_depth: int = 10,
-) -> None:
-    """Train RandomForest model."""
-    import pandas as pd
-    from sklearn.ensemble import RandomForestClassifier
-    import joblib
-
-    df = pd.read_parquet(train_dataset.path)
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    model = RandomForestClassifier(
-        n_estimators=n_estimators,
-        max_depth=max_depth,
-        random_state=42,
-    )
-    model.fit(X, y)
-
-    joblib.dump(model, model_artifact.path)
-    model_artifact.metadata["n_estimators"] = n_estimators
-    model_artifact.metadata["max_depth"] = max_depth
-
-@dsl.component(
-    base_image="python:3.11-slim",
-    packages_to_install=["pandas", "scikit-learn", "joblib"],
-)
-def evaluate_model(
-    model_artifact: Input[Model],
-    test_dataset: Input[Dataset],
-    metrics: Output[Metrics],
-    threshold: float = 0.8,
-) -> NamedTuple("Outputs", [("passed", bool), ("accuracy", float)]):
-    """Evaluate model and check threshold."""
-    import pandas as pd
-    from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
-    import joblib
-    from collections import namedtuple
-
-    model = joblib.load(model_artifact.path)
-    df = pd.read_parquet(test_dataset.path)
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    predictions = model.predict(X)
-
-    accuracy = accuracy_score(y, predictions)
-    precision = precision_score(y, predictions, average="weighted")
-    recall = recall_score(y, predictions, average="weighted")
-    f1 = f1_score(y, predictions, average="weighted")
-
-    metrics.log_metric("accuracy", accuracy)
-    metrics.log_metric("precision", precision)
-    metrics.log_metric("recall", recall)
-    metrics.log_metric("f1_score", f1)
-
-    passed = accuracy >= threshold
-
-    Outputs = namedtuple("Outputs", ["passed", "accuracy"])
-    return Outputs(passed, accuracy)
-
-@dsl.component(
-    base_image="python:3.11-slim",
-    packages_to_install=["google-cloud-storage"],
-)
-def deploy_model(
-    model_artifact: Input[Model],
-    model_name: str,
-    endpoint: str,
-) -> str:
-    """Deploy model to serving endpoint."""
-    from google.cloud import storage
-    import shutil
-
-    # Copy model to GCS
-    bucket_name = endpoint.split("/")[2]
-    model_path = f"models/{model_name}/model.joblib"
-
-    client = storage.Client()
-    bucket = client.bucket(bucket_name)
-    blob = bucket.blob(model_path)
-    blob.upload_from_filename(model_artifact.path)
-
-    return f"gs://{bucket_name}/{model_path}"
-
-@dsl.pipeline(
-    name="ml-training-pipeline",
-    description="End-to-end ML training pipeline",
-)
-def ml_pipeline(
-    data_path: str,
-    n_estimators: int = 100,
-    max_depth: int = 10,
-    accuracy_threshold: float = 0.8,
-    model_name: str = "classifier",
-    endpoint: str = "gs://ml-models/serving",
-) -> None:
-    """Complete ML training pipeline."""
-
-    load_task = load_data(data_path=data_path)
-
-    preprocess_task = preprocess_data(
-        input_dataset=load_task.outputs["output_dataset"],
-    )
-
-    train_task = train_model(
-        train_dataset=preprocess_task.outputs["train_dataset"],
-        n_estimators=n_estimators,
-        max_depth=max_depth,
-    )
-
-    evaluate_task = evaluate_model(
-        model_artifact=train_task.outputs["model_artifact"],
-        test_dataset=preprocess_task.outputs["test_dataset"],
-        threshold=accuracy_threshold,
-    )
-
-    with dsl.If(evaluate_task.outputs["passed"] == True):
-        deploy_model(
-            model_artifact=train_task.outputs["model_artifact"],
-            model_name=model_name,
-            endpoint=endpoint,
-        )
-
-# Compile pipeline
-if __name__ == "__main__":
-    compiler.Compiler().compile(
-        ml_pipeline,
-        "ml_pipeline.yaml",
-    )
-```
-
-### Running Kubeflow Pipelines
-
-```python
-from kfp.client import Client
-
-def run_pipeline(
-    pipeline_file: str,
-    experiment_name: str,
-    run_name: str,
-    parameters: dict,
-) -> str:
-    """Submit pipeline run to Kubeflow."""
-    client = Client(host="https://kubeflow.example.com/pipeline")
-
-    # Create or get experiment
-    experiment = client.create_experiment(name=experiment_name)
-
-    # Submit run
-    run = client.create_run_from_pipeline_package(
-        pipeline_file=pipeline_file,
-        experiment_id=experiment.experiment_id,
-        run_name=run_name,
-        arguments=parameters,
-    )
-
-    return run.run_id
-
-def schedule_pipeline(
-    pipeline_file: str,
-    experiment_name: str,
-    schedule_name: str,
-    cron_expression: str,
-    parameters: dict,
-) -> str:
-    """Create recurring pipeline run."""
-    client = Client(host="https://kubeflow.example.com/pipeline")
-
-    experiment = client.create_experiment(name=experiment_name)
-
-    # Create recurring run
-    job = client.create_recurring_run(
-        experiment_id=experiment.experiment_id,
-        job_name=schedule_name,
-        pipeline_package_path=pipeline_file,
-        cron_expression=cron_expression,
-        enabled=True,
-        parameters=parameters,
-    )
-
-    return job.id
-```
-
----
-
-## Apache Airflow
-
-### ML Pipeline DAG
-
-```python
-from airflow import DAG
-from airflow.operators.python import PythonOperator, BranchPythonOperator
-from airflow.operators.empty import EmptyOperator
-from airflow.providers.amazon.aws.operators.s3 import S3CreateObjectOperator
-from airflow.utils.trigger_rule import TriggerRule
-from datetime import datetime, timedelta
-import json
-
-default_args = {
-    "owner": "ml-team",
-    "depends_on_past": False,
-    "email_on_failure": True,
-    "email_on_retry": False,
-    "retries": 2,
-    "retry_delay": timedelta(minutes=5),
-    "execution_timeout": timedelta(hours=2),
-}
-
-def load_data(**context):
-    """Load data from source."""
-    import pandas as pd
-
-    data_path = context["params"]["data_path"]
-    df = pd.read_parquet(data_path)
-
-    # Push to XCom for downstream tasks
-    output_path = f"/tmp/data_{context['run_id']}.parquet"
-    df.to_parquet(output_path)
-
-    context["ti"].xcom_push(key="data_path", value=output_path)
-    context["ti"].xcom_push(key="num_rows", value=len(df))
-
-    return output_path
-
-def preprocess_data(**context):
-    """Preprocess and split data."""
-    import pandas as pd
-    from sklearn.model_selection import train_test_split
-    from sklearn.preprocessing import StandardScaler
-
-    input_path = context["ti"].xcom_pull(key="data_path", task_ids="load_data")
-    df = pd.read_parquet(input_path)
-
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=0.2, random_state=42
-    )
-
-    scaler = StandardScaler()
-    X_train_scaled = scaler.fit_transform(X_train)
-    X_test_scaled = scaler.transform(X_test)
-
-    # Save processed data
-    train_path = f"/tmp/train_{context['run_id']}.parquet"
-    test_path = f"/tmp/test_{context['run_id']}.parquet"
-
-    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
-    train_df["target"] = y_train.values
-    train_df.to_parquet(train_path)
-
-    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
-    test_df["target"] = y_test.values
-    test_df.to_parquet(test_path)
-
-    context["ti"].xcom_push(key="train_path", value=train_path)
-    context["ti"].xcom_push(key="test_path", value=test_path)
-
-def train_model(**context):
-    """Train ML model."""
-    import pandas as pd
-    from sklearn.ensemble import RandomForestClassifier
-    import joblib
-
-    train_path = context["ti"].xcom_pull(key="train_path", task_ids="preprocess_data")
-    df = pd.read_parquet(train_path)
-
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    params = context["params"]
-    model = RandomForestClassifier(
-        n_estimators=params.get("n_estimators", 100),
-        max_depth=params.get("max_depth", 10),
-        random_state=42,
-    )
-    model.fit(X, y)
-
-    model_path = f"/tmp/model_{context['run_id']}.joblib"
-    joblib.dump(model, model_path)
-
-    context["ti"].xcom_push(key="model_path", value=model_path)
-
-def evaluate_model(**context):
-    """Evaluate model and return metrics."""
-    import pandas as pd
-    from sklearn.metrics import accuracy_score, precision_score, recall_score
-    import joblib
-
-    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
-    test_path = context["ti"].xcom_pull(key="test_path", task_ids="preprocess_data")
-
-    model = joblib.load(model_path)
-    df = pd.read_parquet(test_path)
-
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    predictions = model.predict(X)
-
-    metrics = {
-        "accuracy": accuracy_score(y, predictions),
-        "precision": precision_score(y, predictions, average="weighted"),
-        "recall": recall_score(y, predictions, average="weighted"),
-    }
-
-    context["ti"].xcom_push(key="metrics", value=metrics)
-
-    return metrics
-
-def check_metrics_threshold(**context):
-    """Branch based on model performance."""
-    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
-    threshold = context["params"].get("accuracy_threshold", 0.8)
-
-    if metrics["accuracy"] >= threshold:
-        return "deploy_model"
-    return "skip_deployment"
-
-def deploy_model(**context):
-    """Deploy model to production."""
-    import shutil
-
-    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
-    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
-
-    # In production, this would upload to model registry/serving
-    deploy_path = f"/models/production/model_{context['run_id']}.joblib"
-    shutil.copy(model_path, deploy_path)
-
-    return {
-        "model_path": deploy_path,
-        "metrics": metrics,
-        "deployed_at": datetime.utcnow().isoformat(),
-    }
-
-with DAG(
-    dag_id="ml_training_pipeline",
-    default_args=default_args,
-    description="End-to-end ML training pipeline",
-    schedule_interval="0 2 * * *",  # Daily at 2 AM
-    start_date=datetime(2024, 1, 1),
-    catchup=False,
-    tags=["ml", "training", "production"],
-    params={
-        "data_path": "s3://data-bucket/training_data.parquet",
-        "n_estimators": 100,
-        "max_depth": 10,
-        "accuracy_threshold": 0.8,
-    },
-) as dag:
-
-    start = EmptyOperator(task_id="start")
-
-    load = PythonOperator(
-        task_id="load_data",
-        python_callable=load_data,
-    )
-
-    preprocess = PythonOperator(
-        task_id="preprocess_data",
-        python_callable=preprocess_data,
-    )
-
-    train = PythonOperator(
-        task_id="train_model",
-        python_callable=train_model,
-    )
-
-    evaluate = PythonOperator(
-        task_id="evaluate_model",
-        python_callable=evaluate_model,
-    )
-
-    check_threshold = BranchPythonOperator(
-        task_id="check_metrics_threshold",
-        python_callable=check_metrics_threshold,
-    )
-
-    deploy = PythonOperator(
-        task_id="deploy_model",
-        python_callable=deploy_model,
-    )
-
-    skip = EmptyOperator(task_id="skip_deployment")
-
-    end = EmptyOperator(
-        task_id="end",
-        trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS,
-    )
-
-    start >> load >> preprocess >> train >> evaluate >> check_threshold
-    check_threshold >> [deploy, skip] >> end
-```
-
----
-
-## Prefect
-
-### Modern Flow-Based Pipeline
-
-```python
-from prefect import flow, task, get_run_logger
-from prefect.artifacts import create_markdown_artifact
-from prefect.tasks import task_input_hash
-from datetime import timedelta
-import pandas as pd
-
-@task(
-    retries=3,
-    retry_delay_seconds=60,
-    cache_key_fn=task_input_hash,
-    cache_expiration=timedelta(hours=1),
-)
-def load_data(data_path: str) -> pd.DataFrame:
-    """Load data with caching."""
-    logger = get_run_logger()
-    logger.info(f"Loading data from {data_path}")
-
-    df = pd.read_parquet(data_path)
-    logger.info(f"Loaded {len(df)} rows")
-
-    return df
-
-@task(retries=2)
-def preprocess_data(
-    df: pd.DataFrame,
-    test_size: float = 0.2,
-) -> tuple[pd.DataFrame, pd.DataFrame]:
-    """Preprocess and split data."""
-    from sklearn.model_selection import train_test_split
-    from sklearn.preprocessing import StandardScaler
-
-    logger = get_run_logger()
-
-    X = df.drop("target", axis=1)
-    y = df["target"]
-
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=test_size, random_state=42
-    )
-
-    scaler = StandardScaler()
-    X_train_scaled = scaler.fit_transform(X_train)
-    X_test_scaled = scaler.transform(X_test)
-
-    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
-    train_df["target"] = y_train.values
-
-    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
-    test_df["target"] = y_test.values
-
-    logger.info(f"Train: {len(train_df)}, Test: {len(test_df)}")
-
-    return train_df, test_df
-
-@task
-def train_model(
-    train_df: pd.DataFrame,
-    n_estimators: int = 100,
-    max_depth: int = 10,
-):
-    """Train RandomForest model."""
-    from sklearn.ensemble import RandomForestClassifier
-
-    logger = get_run_logger()
-
-    X = train_df.drop("target", axis=1)
-    y = train_df["target"]
-
-    model = RandomForestClassifier(
-        n_estimators=n_estimators,
-        max_depth=max_depth,
-        random_state=42,
-        n_jobs=-1,
-    )
-
-    logger.info("Training model...")
-    model.fit(X, y)
-    logger.info("Training complete")
-
-    return model
-
-@task
-def evaluate_model(model, test_df: pd.DataFrame) -> dict:
-    """Evaluate model and create artifact."""
-    from sklearn.metrics import (
-        accuracy_score, precision_score, recall_score,
-        f1_score, classification_report
-    )
-
-    logger = get_run_logger()
-
-    X = test_df.drop("target", axis=1)
-    y = test_df["target"]
-
-    predictions = model.predict(X)
-
-    metrics = {
-        "accuracy": accuracy_score(y, predictions),
-        "precision": precision_score(y, predictions, average="weighted"),
-        "recall": recall_score(y, predictions, average="weighted"),
-        "f1_score": f1_score(y, predictions, average="weighted"),
-    }
-
-    logger.info(f"Metrics: {metrics}")
-
-    # Create markdown artifact for Prefect UI
-    report = classification_report(y, predictions)
-    markdown = f"""
-# Model Evaluation Report
-
-## Metrics
-| Metric | Value |
-|--------|-------|
-| Accuracy | {metrics['accuracy']:.4f} |
-| Precision | {metrics['precision']:.4f} |
-| Recall | {metrics['recall']:.4f} |
-| F1 Score | {metrics['f1_score']:.4f} |
-
-## Classification Report
-```
-{report}
-```
-"""
-    create_markdown_artifact(
-        key="model-evaluation",
-        markdown=markdown,
-        description="Model evaluation metrics",
-    )
-
-    return metrics
-
-@task
-def deploy_model(model, metrics: dict, threshold: float) -> bool:
-    """Deploy model if metrics pass threshold."""
-    import joblib
-    from datetime import datetime
-
-    logger = get_run_logger()
-
-    if metrics["accuracy"] < threshold:
-        logger.warning(
-            f"Model accuracy {metrics['accuracy']:.4f} below threshold {threshold}"
-        )
-        return False
-
-    # Save model
-    model_path = f"/models/model_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.joblib"
-    joblib.dump(model, model_path)
-    logger.info(f"Model deployed to {model_path}")
-
-    return True
-
-@flow(
-    name="ml-training-pipeline",
-    description="End-to-end ML training pipeline",
-    retries=1,
-    retry_delay_seconds=300,
-)
-def ml_training_flow(
-    data_path: str,
-    n_estimators: int = 100,
-    max_depth: int = 10,
-    accuracy_threshold: float = 0.8,
-) -> dict:
-    """Main ML training flow."""
-    logger = get_run_logger()
-    logger.info("Starting ML training pipeline")
-
-    # Load and preprocess
-    df = load_data(data_path)
-    train_df, test_df = preprocess_data(df)
-
-    # Train and evaluate
-    model = train_model(train_df, n_estimators, max_depth)
-    metrics = evaluate_model(model, test_df)
-
-    # Deploy if threshold met
-    deployed = deploy_model(model, metrics, accuracy_threshold)
-
-    return {
-        "metrics": metrics,
-        "deployed": deployed,
-    }
-
-# Deployment configuration
-if __name__ == "__main__":
-    from prefect.deployments import Deployment
-    from prefect.server.schemas.schedules import CronSchedule
-
-    deployment = Deployment.build_from_flow(
-        flow=ml_training_flow,
-        name="daily-training",
-        schedule=CronSchedule(cron="0 2 * * *"),
-        parameters={
-            "data_path": "s3://data/training.parquet",
-            "n_estimators": 100,
-            "max_depth": 10,
-            "accuracy_threshold": 0.8,
-        },
-        tags=["ml", "production"],
-        work_queue_name="ml-queue",
-    )
-
-    deployment.apply()
-```
-
----
-
-## DAG Design Patterns
-
-### Parallel Processing Pattern
-
-```python
-from prefect import flow, task, unmapped
-from typing import List
-
-@task
-def process_partition(partition_id: int, data_path: str) -> dict:
-    """Process single data partition."""
-    # Process partition
-    return {"partition_id": partition_id, "records_processed": 1000}
-
-@task
-def aggregate_results(results: List[dict]) -> dict:
-    """Aggregate parallel processing results."""
-    total_records = sum(r["records_processed"] for r in results)
-    return {"total_records": total_records}
-
-@flow
-def parallel_processing_flow(data_path: str, num_partitions: int = 4):
-    """Process data in parallel partitions."""
-
-    # Map over partitions
-    partition_results = process_partition.map(
-        partition_id=range(num_partitions),
-        data_path=unmapped(data_path),
-    )
-
-    # Aggregate results
-    final_result = aggregate_results(partition_results)
-
-    return final_result
-```
-
-### Conditional Branching Pattern
-
-```python
-from prefect import flow, task
-
-@task
-def check_data_quality(df) -> bool:
-    """Check if data meets quality standards."""
-    null_ratio = df.isnull().sum().sum() / df.size
-    return null_ratio < 0.1
-
-@task
-def handle_poor_quality(df):
-    """Handle data that fails quality checks."""
-    # Impute, clean, or alert
-    pass
-
-@task
-def process_good_quality(df):
-    """Process data that passes quality checks."""
-    pass
-
-@flow
-def conditional_flow(data_path: str):
-    """Flow with conditional branching."""
-    df = load_data(data_path)
-    quality_ok = check_data_quality(df)
-
-    if quality_ok:
-        result = process_good_quality(df)
-    else:
-        result = handle_poor_quality(df)
-
-    return result
-```
-
-### Error Handling Pattern
-
-```python
-from prefect import flow, task
-from prefect.states import Failed
-
-@task
-def risky_operation():
-    """Operation that might fail."""
-    import random
-    if random.random() < 0.3:
-        raise ValueError("Random failure")
-    return "success"
-
-@task
-def fallback_operation():
-    """Fallback when primary fails."""
-    return "fallback_result"
-
-@task
-def send_alert(error: Exception):
-    """Send alert on failure."""
-    # Send to Slack, PagerDuty, etc.
-    pass
-
-@flow
-def resilient_flow():
-    """Flow with error handling."""
-    try:
-        result = risky_operation()
-    except Exception as e:
-        send_alert(e)
-        result = fallback_operation()
-
-    return result
-```
-
----
-
-## Best Practices
-
-### Pipeline Configuration
-
-```yaml
-# pipeline_config.yaml
-pipeline:
-  name: ml-training
-  version: "1.0.0"
-  description: "Production ML training pipeline"
-
-stages:
-  - name: load_data
-    timeout: 300
-    retries: 3
-
-  - name: preprocess
-    timeout: 600
-    retries: 2
-    depends_on: [load_data]
-
-  - name: train
-    timeout: 3600
-    retries: 1
-    depends_on: [preprocess]
-    resources:
-      cpu: 4
-      memory: 16Gi
-      gpu: 1
-
-  - name: evaluate
-    timeout: 300
-    depends_on: [train]
-
-  - name: deploy
-    timeout: 300
-    depends_on: [evaluate]
-    condition: "evaluate.metrics.accuracy >= 0.8"
-
-schedule:
-  cron: "0 2 * * *"
-  timezone: "UTC"
-
-notifications:
-  on_failure:
-    - slack: "#ml-alerts"
-    - email: ml-team@company.com
-  on_success:
-    - slack: "#ml-notifications"
-```
-
-### Idempotency Guidelines
-
-```python
-# Good: Idempotent operations
-def process_data(run_id: str, data_path: str):
-    """Idempotent data processing."""
-    output_path = f"s3://processed/{run_id}/data.parquet"
-
-    # Check if already processed
-    if file_exists(output_path):
-        return output_path
-
-    # Process and save
-    df = pd.read_parquet(data_path)
-    processed = transform(df)
-    processed.to_parquet(output_path)
-
-    return output_path
-```
-
----
-
-## Related References
-
-- `training-pipelines.md` - Training components for pipelines
-- `experiment-tracking.md` - Logging pipeline runs
-- `feature-engineering.md` - Feature pipeline components
-- `model-validation.md` - Validation stages in pipelines
-
-## Cross-Reference Skills
-
-- **DevOps Engineer** - CI/CD for pipeline deployment
-- **Kubernetes Specialist** - Running pipelines on K8s
-- **Cloud Architect** - Cloud infrastructure for orchestration
+# Pipeline Orchestration
+
+---
+
+## Overview
+
+Pipeline orchestration automates the end-to-end ML workflow from data ingestion through model deployment. Orchestrators manage dependencies, handle failures, enable scheduling, and provide observability across complex multi-step pipelines.
+
+## When to Use This Reference
+
+- Building Kubeflow Pipelines for ML workflows
+- Creating Airflow DAGs for data and ML pipelines
+- Implementing Prefect flows for modern orchestration
+- Designing pipeline DAGs and component dependencies
+- Setting up scheduled retraining workflows
+
+## When NOT to Use
+
+- Simple linear scripts without dependencies
+- One-off data processing tasks
+- Interactive development and experimentation
+
+---
+
+## Kubeflow Pipelines
+
+### Pipeline Definition (KFP v2)
+
+```python
+from kfp import dsl
+from kfp.dsl import Input, Output, Artifact, Dataset, Model, Metrics
+from kfp import compiler
+from typing import NamedTuple
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn"],
+)
+def load_data(
+    data_path: str,
+    output_dataset: Output[Dataset],
+) -> None:
+    """Load and validate raw data."""
+    import pandas as pd
+
+    df = pd.read_parquet(data_path)
+
+    # Basic validation
+    assert len(df) > 0, "Dataset is empty"
+    assert "target" in df.columns, "Missing target column"
+
+    df.to_parquet(output_dataset.path)
+    output_dataset.metadata["num_rows"] = len(df)
+    output_dataset.metadata["num_features"] = len(df.columns) - 1
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn"],
+)
+def preprocess_data(
+    input_dataset: Input[Dataset],
+    train_dataset: Output[Dataset],
+    test_dataset: Output[Dataset],
+    test_size: float = 0.2,
+    random_state: int = 42,
+) -> None:
+    """Preprocess and split data."""
+    import pandas as pd
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    df = pd.read_parquet(input_dataset.path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=random_state
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+    train_df.to_parquet(train_dataset.path)
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+    test_df.to_parquet(test_dataset.path)
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn", "joblib"],
+)
+def train_model(
+    train_dataset: Input[Dataset],
+    model_artifact: Output[Model],
+    n_estimators: int = 100,
+    max_depth: int = 10,
+) -> None:
+    """Train RandomForest model."""
+    import pandas as pd
+    from sklearn.ensemble import RandomForestClassifier
+    import joblib
+
+    df = pd.read_parquet(train_dataset.path)
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    model = RandomForestClassifier(
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+        random_state=42,
+    )
+    model.fit(X, y)
+
+    joblib.dump(model, model_artifact.path)
+    model_artifact.metadata["n_estimators"] = n_estimators
+    model_artifact.metadata["max_depth"] = max_depth
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["pandas", "scikit-learn", "joblib"],
+)
+def evaluate_model(
+    model_artifact: Input[Model],
+    test_dataset: Input[Dataset],
+    metrics: Output[Metrics],
+    threshold: float = 0.8,
+) -> NamedTuple("Outputs", [("passed", bool), ("accuracy", float)]):
+    """Evaluate model and check threshold."""
+    import pandas as pd
+    from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
+    import joblib
+    from collections import namedtuple
+
+    model = joblib.load(model_artifact.path)
+    df = pd.read_parquet(test_dataset.path)
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    predictions = model.predict(X)
+
+    accuracy = accuracy_score(y, predictions)
+    precision = precision_score(y, predictions, average="weighted")
+    recall = recall_score(y, predictions, average="weighted")
+    f1 = f1_score(y, predictions, average="weighted")
+
+    metrics.log_metric("accuracy", accuracy)
+    metrics.log_metric("precision", precision)
+    metrics.log_metric("recall", recall)
+    metrics.log_metric("f1_score", f1)
+
+    passed = accuracy >= threshold
+
+    Outputs = namedtuple("Outputs", ["passed", "accuracy"])
+    return Outputs(passed, accuracy)
+
+@dsl.component(
+    base_image="python:3.11-slim",
+    packages_to_install=["google-cloud-storage"],
+)
+def deploy_model(
+    model_artifact: Input[Model],
+    model_name: str,
+    endpoint: str,
+) -> str:
+    """Deploy model to serving endpoint."""
+    from google.cloud import storage
+    import shutil
+
+    # Copy model to GCS
+    bucket_name = endpoint.split("/")[2]
+    model_path = f"models/{model_name}/model.joblib"
+
+    client = storage.Client()
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(model_path)
+    blob.upload_from_filename(model_artifact.path)
+
+    return f"gs://{bucket_name}/{model_path}"
+
+@dsl.pipeline(
+    name="ml-training-pipeline",
+    description="End-to-end ML training pipeline",
+)
+def ml_pipeline(
+    data_path: str,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+    accuracy_threshold: float = 0.8,
+    model_name: str = "classifier",
+    endpoint: str = "gs://ml-models/serving",
+) -> None:
+    """Complete ML training pipeline."""
+
+    load_task = load_data(data_path=data_path)
+
+    preprocess_task = preprocess_data(
+        input_dataset=load_task.outputs["output_dataset"],
+    )
+
+    train_task = train_model(
+        train_dataset=preprocess_task.outputs["train_dataset"],
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+    )
+
+    evaluate_task = evaluate_model(
+        model_artifact=train_task.outputs["model_artifact"],
+        test_dataset=preprocess_task.outputs["test_dataset"],
+        threshold=accuracy_threshold,
+    )
+
+    with dsl.If(evaluate_task.outputs["passed"] == True):
+        deploy_model(
+            model_artifact=train_task.outputs["model_artifact"],
+            model_name=model_name,
+            endpoint=endpoint,
+        )
+
+# Compile pipeline
+if __name__ == "__main__":
+    compiler.Compiler().compile(
+        ml_pipeline,
+        "ml_pipeline.yaml",
+    )
+```
+
+### Running Kubeflow Pipelines
+
+```python
+from kfp.client import Client
+
+def run_pipeline(
+    pipeline_file: str,
+    experiment_name: str,
+    run_name: str,
+    parameters: dict,
+) -> str:
+    """Submit pipeline run to Kubeflow."""
+    client = Client(host="https://kubeflow.example.com/pipeline")
+
+    # Create or get experiment
+    experiment = client.create_experiment(name=experiment_name)
+
+    # Submit run
+    run = client.create_run_from_pipeline_package(
+        pipeline_file=pipeline_file,
+        experiment_id=experiment.experiment_id,
+        run_name=run_name,
+        arguments=parameters,
+    )
+
+    return run.run_id
+
+def schedule_pipeline(
+    pipeline_file: str,
+    experiment_name: str,
+    schedule_name: str,
+    cron_expression: str,
+    parameters: dict,
+) -> str:
+    """Create recurring pipeline run."""
+    client = Client(host="https://kubeflow.example.com/pipeline")
+
+    experiment = client.create_experiment(name=experiment_name)
+
+    # Create recurring run
+    job = client.create_recurring_run(
+        experiment_id=experiment.experiment_id,
+        job_name=schedule_name,
+        pipeline_package_path=pipeline_file,
+        cron_expression=cron_expression,
+        enabled=True,
+        parameters=parameters,
+    )
+
+    return job.id
+```
+
+---
+
+## Apache Airflow
+
+### ML Pipeline DAG
+
+```python
+from airflow import DAG
+from airflow.operators.python import PythonOperator, BranchPythonOperator
+from airflow.operators.empty import EmptyOperator
+from airflow.providers.amazon.aws.operators.s3 import S3CreateObjectOperator
+from airflow.utils.trigger_rule import TriggerRule
+from datetime import datetime, timedelta
+import json
+
+default_args = {
+    "owner": "ml-team",
+    "depends_on_past": False,
+    "email_on_failure": True,
+    "email_on_retry": False,
+    "retries": 2,
+    "retry_delay": timedelta(minutes=5),
+    "execution_timeout": timedelta(hours=2),
+}
+
+def load_data(**context):
+    """Load data from source."""
+    import pandas as pd
+
+    data_path = context["params"]["data_path"]
+    df = pd.read_parquet(data_path)
+
+    # Push to XCom for downstream tasks
+    output_path = f"/tmp/data_{context['run_id']}.parquet"
+    df.to_parquet(output_path)
+
+    context["ti"].xcom_push(key="data_path", value=output_path)
+    context["ti"].xcom_push(key="num_rows", value=len(df))
+
+    return output_path
+
+def preprocess_data(**context):
+    """Preprocess and split data."""
+    import pandas as pd
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    input_path = context["ti"].xcom_pull(key="data_path", task_ids="load_data")
+    df = pd.read_parquet(input_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    # Save processed data
+    train_path = f"/tmp/train_{context['run_id']}.parquet"
+    test_path = f"/tmp/test_{context['run_id']}.parquet"
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+    train_df.to_parquet(train_path)
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+    test_df.to_parquet(test_path)
+
+    context["ti"].xcom_push(key="train_path", value=train_path)
+    context["ti"].xcom_push(key="test_path", value=test_path)
+
+def train_model(**context):
+    """Train ML model."""
+    import pandas as pd
+    from sklearn.ensemble import RandomForestClassifier
+    import joblib
+
+    train_path = context["ti"].xcom_pull(key="train_path", task_ids="preprocess_data")
+    df = pd.read_parquet(train_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    params = context["params"]
+    model = RandomForestClassifier(
+        n_estimators=params.get("n_estimators", 100),
+        max_depth=params.get("max_depth", 10),
+        random_state=42,
+    )
+    model.fit(X, y)
+
+    model_path = f"/tmp/model_{context['run_id']}.joblib"
+    joblib.dump(model, model_path)
+
+    context["ti"].xcom_push(key="model_path", value=model_path)
+
+def evaluate_model(**context):
+    """Evaluate model and return metrics."""
+    import pandas as pd
+    from sklearn.metrics import accuracy_score, precision_score, recall_score
+    import joblib
+
+    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
+    test_path = context["ti"].xcom_pull(key="test_path", task_ids="preprocess_data")
+
+    model = joblib.load(model_path)
+    df = pd.read_parquet(test_path)
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    predictions = model.predict(X)
+
+    metrics = {
+        "accuracy": accuracy_score(y, predictions),
+        "precision": precision_score(y, predictions, average="weighted"),
+        "recall": recall_score(y, predictions, average="weighted"),
+    }
+
+    context["ti"].xcom_push(key="metrics", value=metrics)
+
+    return metrics
+
+def check_metrics_threshold(**context):
+    """Branch based on model performance."""
+    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
+    threshold = context["params"].get("accuracy_threshold", 0.8)
+
+    if metrics["accuracy"] >= threshold:
+        return "deploy_model"
+    return "skip_deployment"
+
+def deploy_model(**context):
+    """Deploy model to production."""
+    import shutil
+
+    model_path = context["ti"].xcom_pull(key="model_path", task_ids="train_model")
+    metrics = context["ti"].xcom_pull(key="metrics", task_ids="evaluate_model")
+
+    # In production, this would upload to model registry/serving
+    deploy_path = f"/models/production/model_{context['run_id']}.joblib"
+    shutil.copy(model_path, deploy_path)
+
+    return {
+        "model_path": deploy_path,
+        "metrics": metrics,
+        "deployed_at": datetime.utcnow().isoformat(),
+    }
+
+with DAG(
+    dag_id="ml_training_pipeline",
+    default_args=default_args,
+    description="End-to-end ML training pipeline",
+    schedule_interval="0 2 * * *",  # Daily at 2 AM
+    start_date=datetime(2024, 1, 1),
+    catchup=False,
+    tags=["ml", "training", "production"],
+    params={
+        "data_path": "s3://data-bucket/training_data.parquet",
+        "n_estimators": 100,
+        "max_depth": 10,
+        "accuracy_threshold": 0.8,
+    },
+) as dag:
+
+    start = EmptyOperator(task_id="start")
+
+    load = PythonOperator(
+        task_id="load_data",
+        python_callable=load_data,
+    )
+
+    preprocess = PythonOperator(
+        task_id="preprocess_data",
+        python_callable=preprocess_data,
+    )
+
+    train = PythonOperator(
+        task_id="train_model",
+        python_callable=train_model,
+    )
+
+    evaluate = PythonOperator(
+        task_id="evaluate_model",
+        python_callable=evaluate_model,
+    )
+
+    check_threshold = BranchPythonOperator(
+        task_id="check_metrics_threshold",
+        python_callable=check_metrics_threshold,
+    )
+
+    deploy = PythonOperator(
+        task_id="deploy_model",
+        python_callable=deploy_model,
+    )
+
+    skip = EmptyOperator(task_id="skip_deployment")
+
+    end = EmptyOperator(
+        task_id="end",
+        trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS,
+    )
+
+    start >> load >> preprocess >> train >> evaluate >> check_threshold
+    check_threshold >> [deploy, skip] >> end
+```
+
+---
+
+## Prefect
+
+### Modern Flow-Based Pipeline
+
+```python
+from prefect import flow, task, get_run_logger
+from prefect.artifacts import create_markdown_artifact
+from prefect.tasks import task_input_hash
+from datetime import timedelta
+import pandas as pd
+
+@task(
+    retries=3,
+    retry_delay_seconds=60,
+    cache_key_fn=task_input_hash,
+    cache_expiration=timedelta(hours=1),
+)
+def load_data(data_path: str) -> pd.DataFrame:
+    """Load data with caching."""
+    logger = get_run_logger()
+    logger.info(f"Loading data from {data_path}")
+
+    df = pd.read_parquet(data_path)
+    logger.info(f"Loaded {len(df)} rows")
+
+    return df
+
+@task(retries=2)
+def preprocess_data(
+    df: pd.DataFrame,
+    test_size: float = 0.2,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Preprocess and split data."""
+    from sklearn.model_selection import train_test_split
+    from sklearn.preprocessing import StandardScaler
+
+    logger = get_run_logger()
+
+    X = df.drop("target", axis=1)
+    y = df["target"]
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=42
+    )
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    train_df = pd.DataFrame(X_train_scaled, columns=X.columns)
+    train_df["target"] = y_train.values
+
+    test_df = pd.DataFrame(X_test_scaled, columns=X.columns)
+    test_df["target"] = y_test.values
+
+    logger.info(f"Train: {len(train_df)}, Test: {len(test_df)}")
+
+    return train_df, test_df
+
+@task
+def train_model(
+    train_df: pd.DataFrame,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+):
+    """Train RandomForest model."""
+    from sklearn.ensemble import RandomForestClassifier
+
+    logger = get_run_logger()
+
+    X = train_df.drop("target", axis=1)
+    y = train_df["target"]
+
+    model = RandomForestClassifier(
+        n_estimators=n_estimators,
+        max_depth=max_depth,
+        random_state=42,
+        n_jobs=-1,
+    )
+
+    logger.info("Training model...")
+    model.fit(X, y)
+    logger.info("Training complete")
+
+    return model
+
+@task
+def evaluate_model(model, test_df: pd.DataFrame) -> dict:
+    """Evaluate model and create artifact."""
+    from sklearn.metrics import (
+        accuracy_score, precision_score, recall_score,
+        f1_score, classification_report
+    )
+
+    logger = get_run_logger()
+
+    X = test_df.drop("target", axis=1)
+    y = test_df["target"]
+
+    predictions = model.predict(X)
+
+    metrics = {
+        "accuracy": accuracy_score(y, predictions),
+        "precision": precision_score(y, predictions, average="weighted"),
+        "recall": recall_score(y, predictions, average="weighted"),
+        "f1_score": f1_score(y, predictions, average="weighted"),
+    }
+
+    logger.info(f"Metrics: {metrics}")
+
+    # Create markdown artifact for Prefect UI
+    report = classification_report(y, predictions)
+    markdown = f"""
+# Model Evaluation Report
+
+## Metrics
+| Metric | Value |
+|--------|-------|
+| Accuracy | {metrics['accuracy']:.4f} |
+| Precision | {metrics['precision']:.4f} |
+| Recall | {metrics['recall']:.4f} |
+| F1 Score | {metrics['f1_score']:.4f} |
+
+## Classification Report
+```
+{report}
+```
+"""
+    create_markdown_artifact(
+        key="model-evaluation",
+        markdown=markdown,
+        description="Model evaluation metrics",
+    )
+
+    return metrics
+
+@task
+def deploy_model(model, metrics: dict, threshold: float) -> bool:
+    """Deploy model if metrics pass threshold."""
+    import joblib
+    from datetime import datetime
+
+    logger = get_run_logger()
+
+    if metrics["accuracy"] < threshold:
+        logger.warning(
+            f"Model accuracy {metrics['accuracy']:.4f} below threshold {threshold}"
+        )
+        return False
+
+    # Save model
+    model_path = f"/models/model_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.joblib"
+    joblib.dump(model, model_path)
+    logger.info(f"Model deployed to {model_path}")
+
+    return True
+
+@flow(
+    name="ml-training-pipeline",
+    description="End-to-end ML training pipeline",
+    retries=1,
+    retry_delay_seconds=300,
+)
+def ml_training_flow(
+    data_path: str,
+    n_estimators: int = 100,
+    max_depth: int = 10,
+    accuracy_threshold: float = 0.8,
+) -> dict:
+    """Main ML training flow."""
+    logger = get_run_logger()
+    logger.info("Starting ML training pipeline")
+
+    # Load and preprocess
+    df = load_data(data_path)
+    train_df, test_df = preprocess_data(df)
+
+    # Train and evaluate
+    model = train_model(train_df, n_estimators, max_depth)
+    metrics = evaluate_model(model, test_df)
+
+    # Deploy if threshold met
+    deployed = deploy_model(model, metrics, accuracy_threshold)
+
+    return {
+        "metrics": metrics,
+        "deployed": deployed,
+    }
+
+# Deployment configuration
+if __name__ == "__main__":
+    from prefect.deployments import Deployment
+    from prefect.server.schemas.schedules import CronSchedule
+
+    deployment = Deployment.build_from_flow(
+        flow=ml_training_flow,
+        name="daily-training",
+        schedule=CronSchedule(cron="0 2 * * *"),
+        parameters={
+            "data_path": "s3://data/training.parquet",
+            "n_estimators": 100,
+            "max_depth": 10,
+            "accuracy_threshold": 0.8,
+        },
+        tags=["ml", "production"],
+        work_queue_name="ml-queue",
+    )
+
+    deployment.apply()
+```
+
+---
+
+## DAG Design Patterns
+
+### Parallel Processing Pattern
+
+```python
+from prefect import flow, task, unmapped
+from typing import List
+
+@task
+def process_partition(partition_id: int, data_path: str) -> dict:
+    """Process single data partition."""
+    # Process partition
+    return {"partition_id": partition_id, "records_processed": 1000}
+
+@task
+def aggregate_results(results: List[dict]) -> dict:
+    """Aggregate parallel processing results."""
+    total_records = sum(r["records_processed"] for r in results)
+    return {"total_records": total_records}
+
+@flow
+def parallel_processing_flow(data_path: str, num_partitions: int = 4):
+    """Process data in parallel partitions."""
+
+    # Map over partitions
+    partition_results = process_partition.map(
+        partition_id=range(num_partitions),
+        data_path=unmapped(data_path),
+    )
+
+    # Aggregate results
+    final_result = aggregate_results(partition_results)
+
+    return final_result
+```
+
+### Conditional Branching Pattern
+
+```python
+from prefect import flow, task
+
+@task
+def check_data_quality(df) -> bool:
+    """Check if data meets quality standards."""
+    null_ratio = df.isnull().sum().sum() / df.size
+    return null_ratio < 0.1
+
+@task
+def handle_poor_quality(df):
+    """Handle data that fails quality checks."""
+    # Impute, clean, or alert
+    pass
+
+@task
+def process_good_quality(df):
+    """Process data that passes quality checks."""
+    pass
+
+@flow
+def conditional_flow(data_path: str):
+    """Flow with conditional branching."""
+    df = load_data(data_path)
+    quality_ok = check_data_quality(df)
+
+    if quality_ok:
+        result = process_good_quality(df)
+    else:
+        result = handle_poor_quality(df)
+
+    return result
+```
+
+### Error Handling Pattern
+
+```python
+from prefect import flow, task
+from prefect.states import Failed
+
+@task
+def risky_operation():
+    """Operation that might fail."""
+    import random
+    if random.random() < 0.3:
+        raise ValueError("Random failure")
+    return "success"
+
+@task
+def fallback_operation():
+    """Fallback when primary fails."""
+    return "fallback_result"
+
+@task
+def send_alert(error: Exception):
+    """Send alert on failure."""
+    # Send to Slack, PagerDuty, etc.
+    pass
+
+@flow
+def resilient_flow():
+    """Flow with error handling."""
+    try:
+        result = risky_operation()
+    except Exception as e:
+        send_alert(e)
+        result = fallback_operation()
+
+    return result
+```
+
+---
+
+## Best Practices
+
+### Pipeline Configuration
+
+```yaml
+# pipeline_config.yaml
+pipeline:
+  name: ml-training
+  version: "1.0.0"
+  description: "Production ML training pipeline"
+
+stages:
+  - name: load_data
+    timeout: 300
+    retries: 3
+
+  - name: preprocess
+    timeout: 600
+    retries: 2
+    depends_on: [load_data]
+
+  - name: train
+    timeout: 3600
+    retries: 1
+    depends_on: [preprocess]
+    resources:
+      cpu: 4
+      memory: 16Gi
+      gpu: 1
+
+  - name: evaluate
+    timeout: 300
+    depends_on: [train]
+
+  - name: deploy
+    timeout: 300
+    depends_on: [evaluate]
+    condition: "evaluate.metrics.accuracy >= 0.8"
+
+schedule:
+  cron: "0 2 * * *"
+  timezone: "UTC"
+
+notifications:
+  on_failure:
+    - slack: "#ml-alerts"
+    - email: ml-team@company.com
+  on_success:
+    - slack: "#ml-notifications"
+```
+
+### Idempotency Guidelines
+
+```python
+# Good: Idempotent operations
+def process_data(run_id: str, data_path: str):
+    """Idempotent data processing."""
+    output_path = f"s3://processed/{run_id}/data.parquet"
+
+    # Check if already processed
+    if file_exists(output_path):
+        return output_path
+
+    # Process and save
+    df = pd.read_parquet(data_path)
+    processed = transform(df)
+    processed.to_parquet(output_path)
+
+    return output_path
+```
+
+---
+
+## Related References
+
+- `training-pipelines.md` - Training components for pipelines
+- `experiment-tracking.md` - Logging pipeline runs
+- `feature-engineering.md` - Feature pipeline components
+- `model-validation.md` - Validation stages in pipelines
+
+## Cross-Reference Skills
+
+- **DevOps Engineer** - CI/CD for pipeline deployment
+- **Kubernetes Specialist** - Running pipelines on K8s
+- **Cloud Architect** - Cloud infrastructure for orchestration