omgkit 2.13.0 → 2.15.0

package/plugin/skills/ai-ml/experiment-tracking/SKILL.md

@@ -0,0 +1,338 @@
+# Experiment Tracking
+
+MLflow, Weights & Biases, experiment versioning, hyperparameter tracking, and model registry integration.
+
+## Overview
+
+Experiment tracking provides systematic recording and comparison of ML experiments including parameters, metrics, artifacts, and model versions.
+
+## Core Concepts
+
+### What to Track
+- **Parameters**: Hyperparameters, configurations
+- **Metrics**: Loss, accuracy, custom metrics
+- **Artifacts**: Models, plots, datasets
+- **Code**: Git commit, environment
+- **Metadata**: Tags, notes, lineage
+
+### Tracking Hierarchy
+```
+Project
+└── Experiment (logical grouping)
+    └── Run (single training execution)
+        ├── Parameters
+        ├── Metrics
+        ├── Artifacts
+        └── Tags
+```
+
+## MLflow Implementation
+
+### Basic Tracking
+```python
+import mlflow
+from mlflow.tracking import MlflowClient
+
+# Set tracking URI
+mlflow.set_tracking_uri("http://mlflow-server:5000")
+
+# Set experiment
+mlflow.set_experiment("recommendation-model")
+
+# Start run
+with mlflow.start_run(run_name="baseline-v1") as run:
+    # Log parameters
+    mlflow.log_params({
+        "learning_rate": 0.001,
+        "batch_size": 32,
+        "epochs": 100,
+        "optimizer": "adam",
+        "model_type": "transformer"
+    })
+
+    # Training loop
+    for epoch in range(100):
+        train_loss, val_loss = train_epoch(model, data)
+
+        # Log metrics
+        mlflow.log_metrics({
+            "train_loss": train_loss,
+            "val_loss": val_loss,
+            "epoch": epoch
+        }, step=epoch)
+
+    # Log model
+    mlflow.pytorch.log_model(model, "model")
+
+    # Log artifacts
+    mlflow.log_artifact("confusion_matrix.png")
+    mlflow.log_artifact("feature_importance.json")
+
+    # Set tags
+    mlflow.set_tags({
+        "team": "ml-platform",
+        "version": "1.0.0",
+        "stage": "development"
+    })
+```
+
+### Model Registry
+```python
+from mlflow.tracking import MlflowClient
+
+client = MlflowClient()
+
+# Register model
+model_uri = f"runs:/{run.info.run_id}/model"
+model_version = mlflow.register_model(model_uri, "RecommendationModel")
+
+# Transition to staging
+client.transition_model_version_stage(
+    name="RecommendationModel",
+    version=model_version.version,
+    stage="Staging"
+)
+
+# Add description
+client.update_model_version(
+    name="RecommendationModel",
+    version=model_version.version,
+    description="Baseline transformer model with 85% accuracy"
+)
+
+# Load model by stage
+model = mlflow.pyfunc.load_model(
+    model_uri="models:/RecommendationModel/Production"
+)
+```
+
+### Autologging
+```python
+# Enable autologging for framework
+mlflow.pytorch.autolog()
+mlflow.sklearn.autolog()
+mlflow.tensorflow.autolog()
+
+# All parameters, metrics, and models logged automatically
+with mlflow.start_run():
+    model.fit(X_train, y_train)
+    # Automatically logs: parameters, metrics, model, feature importance
+```
+
+## Weights & Biases
+
+### Basic Tracking
+```python
+import wandb
+
+# Initialize
+wandb.init(
+    project="recommendation-model",
+    name="baseline-v1",
+    config={
+        "learning_rate": 0.001,
+        "batch_size": 32,
+        "epochs": 100,
+        "architecture": "transformer"
+    }
+)
+
+# Training loop
+for epoch in range(100):
+    train_loss, val_loss = train_epoch(model, data)
+
+    # Log metrics
+    wandb.log({
+        "train_loss": train_loss,
+        "val_loss": val_loss,
+        "epoch": epoch
+    })
+
+# Log artifacts
+wandb.log({"confusion_matrix": wandb.Image("confusion_matrix.png")})
+wandb.log({"model": wandb.Artifact("model", type="model")})
+
+# Finish
+wandb.finish()
+```
+
+### Hyperparameter Sweeps
+```python
+# Define sweep configuration
+sweep_config = {
+    "method": "bayes",
+    "metric": {"name": "val_loss", "goal": "minimize"},
+    "parameters": {
+        "learning_rate": {"min": 0.0001, "max": 0.1, "distribution": "log_uniform_values"},
+        "batch_size": {"values": [16, 32, 64, 128]},
+        "hidden_size": {"min": 64, "max": 512, "distribution": "int_uniform"},
+        "dropout": {"min": 0.1, "max": 0.5}
+    }
+}
+
+# Create sweep
+sweep_id = wandb.sweep(sweep_config, project="recommendation-model")
+
+# Define training function
+def train():
+    wandb.init()
+    config = wandb.config
+
+    model = create_model(
+        hidden_size=config.hidden_size,
+        dropout=config.dropout
+    )
+
+    optimizer = Adam(model.parameters(), lr=config.learning_rate)
+
+    for epoch in range(100):
+        loss = train_epoch(model, config.batch_size)
+        wandb.log({"val_loss": loss})
+
+# Run sweep
+wandb.agent(sweep_id, train, count=50)
+```
+
+### Tables and Visualizations
+```python
+# Create table for predictions
+columns = ["image", "prediction", "ground_truth", "confidence"]
+table = wandb.Table(columns=columns)
+
+for img, pred, gt, conf in predictions:
+    table.add_data(
+        wandb.Image(img),
+        pred,
+        gt,
+        conf
+    )
+
+wandb.log({"predictions": table})
+
+# Custom charts
+wandb.log({
+    "pr_curve": wandb.plot.pr_curve(y_true, y_scores),
+    "roc_curve": wandb.plot.roc_curve(y_true, y_scores),
+    "confusion_matrix": wandb.plot.confusion_matrix(
+        y_true=y_true,
+        preds=y_pred,
+        class_names=class_names
+    )
+})
+```
+
+## Experiment Comparison
+
+### MLflow Query API
+```python
+from mlflow.tracking import MlflowClient
+
+client = MlflowClient()
+
+# Search runs
+runs = client.search_runs(
+    experiment_ids=["1"],
+    filter_string="metrics.val_accuracy > 0.8 AND params.model_type = 'transformer'",
+    order_by=["metrics.val_accuracy DESC"],
+    max_results=10
+)
+
+# Compare runs
+for run in runs:
+    print(f"Run: {run.info.run_id}")
+    print(f"  Accuracy: {run.data.metrics['val_accuracy']}")
+    print(f"  Learning Rate: {run.data.params['learning_rate']}")
+```
+
+### Reproducibility
+```python
+import mlflow
+import random
+import numpy as np
+import torch
+
+def set_seed(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+with mlflow.start_run():
+    seed = 42
+    set_seed(seed)
+
+    # Log environment
+    mlflow.log_params({
+        "seed": seed,
+        "python_version": sys.version,
+        "torch_version": torch.__version__,
+        "cuda_version": torch.version.cuda
+    })
+
+    # Log git commit
+    mlflow.set_tag("git_commit", get_git_commit())
+
+    # Log requirements
+    mlflow.log_artifact("requirements.txt")
+```
+
+## Best Practices
+
+1. **Consistent Naming**: Use clear experiment/run names
+2. **Version Control**: Link to git commits
+3. **Environment Logging**: Capture dependencies
+4. **Artifact Organization**: Structure artifacts logically
+5. **Tagging Strategy**: Use tags for filtering
+
+## CI/CD Integration
+
+### GitHub Actions
+```yaml
+name: ML Experiment
+on:
+  push:
+    paths:
+      - 'models/**'
+      - 'experiments/**'
+
+jobs:
+  train:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Train model
+        env:
+          MLFLOW_TRACKING_URI: ${{ secrets.MLFLOW_URI }}
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          pip install -r requirements.txt
+          python train.py --experiment-name "ci-${GITHUB_SHA}"
+```
+
+## Anti-Patterns
+
+- Not logging all hyperparameters
+- Missing environment information
+- Inconsistent metric names
+- Not tagging experiments properly
+- Ignoring failed runs
+
+## When to Use
+
+- Model development and iteration
+- Hyperparameter optimization
+- Team collaboration on ML
+- Reproducibility requirements
+- Model governance
+
+## When NOT to Use
+
+- Simple one-off scripts
+- No iteration expected
+- Strict air-gapped environments

package/plugin/skills/ai-ml/feature-stores/SKILL.md

@@ -0,0 +1,340 @@
+# Feature Stores
+
+Feast, Tecton, feature engineering pipelines, online/offline feature serving, and feature versioning.
+
+## Overview
+
+Feature stores provide centralized management of ML features, enabling consistent feature computation, storage, and serving across training and inference.
+
+## Core Concepts
+
+### Feature Store Components
+- **Feature Registry**: Metadata and definitions
+- **Offline Store**: Historical features for training
+- **Online Store**: Low-latency features for inference
+- **Feature Transformation**: Computation pipelines
+- **Feature Serving**: API for retrieving features
+
+### Feature Types
+```
+Point-in-Time Features:
+- User age at transaction time
+- Account balance when order placed
+
+Aggregated Features:
+- 7-day rolling average spend
+- Count of logins last 30 days
+
+Real-time Features:
+- Current session duration
+- Items in cart right now
+```
+
+## Feast Implementation
+
+### Feature Definitions
+```python
+from feast import Entity, Feature, FeatureView, FileSource, ValueType
+from feast.types import Float32, Int64, String
+from datetime import timedelta
+
+# Define entity
+customer = Entity(
+    name="customer_id",
+    value_type=ValueType.STRING,
+    description="Unique customer identifier"
+)
+
+# Define data source
+customer_stats_source = FileSource(
+    path="s3://bucket/customer_stats.parquet",
+    timestamp_field="event_timestamp",
+    created_timestamp_column="created_timestamp"
+)
+
+# Define feature view
+customer_stats_fv = FeatureView(
+    name="customer_stats",
+    entities=["customer_id"],
+    ttl=timedelta(days=1),
+    schema=[
+        Feature(name="total_transactions", dtype=Int64),
+        Feature(name="avg_transaction_amount", dtype=Float32),
+        Feature(name="days_since_last_purchase", dtype=Int64),
+        Feature(name="customer_segment", dtype=String),
+    ],
+    online=True,
+    source=customer_stats_source,
+    tags={"team": "ml-platform", "owner": "data-team"}
+)
+```
+
+### Feature Registration
+```python
+from feast import FeatureStore
+
+# Initialize store
+store = FeatureStore(repo_path="feature_repo/")
+
+# Apply definitions
+store.apply([customer, customer_stats_fv])
+
+# Materialize to online store
+store.materialize_incremental(end_date=datetime.now())
+```
+
+### Training Data Retrieval
+```python
+from feast import FeatureStore
+import pandas as pd
+
+store = FeatureStore(repo_path="feature_repo/")
+
+# Entity dataframe with timestamps
+entity_df = pd.DataFrame({
+    "customer_id": ["C001", "C002", "C003"],
+    "event_timestamp": [
+        datetime(2024, 1, 15),
+        datetime(2024, 1, 16),
+        datetime(2024, 1, 17)
+    ]
+})
+
+# Get historical features (point-in-time correct)
+training_df = store.get_historical_features(
+    entity_df=entity_df,
+    features=[
+        "customer_stats:total_transactions",
+        "customer_stats:avg_transaction_amount",
+        "customer_stats:days_since_last_purchase",
+        "customer_stats:customer_segment"
+    ]
+).to_df()
+```
+
+### Online Feature Retrieval
+```python
+# Get features for inference
+feature_vector = store.get_online_features(
+    features=[
+        "customer_stats:total_transactions",
+        "customer_stats:avg_transaction_amount",
+        "customer_stats:customer_segment"
+    ],
+    entity_rows=[
+        {"customer_id": "C001"},
+        {"customer_id": "C002"}
+    ]
+).to_dict()
+```
+
+## On-Demand Features
+
+### Real-time Transformations
+```python
+from feast import on_demand_feature_view, Field
+from feast.types import Float32
+import pandas as pd
+
+@on_demand_feature_view(
+    sources=[customer_stats_fv],
+    schema=[
+        Field(name="transaction_velocity", dtype=Float32),
+        Field(name="is_high_value", dtype=Int64)
+    ]
+)
+def customer_derived_features(inputs: pd.DataFrame) -> pd.DataFrame:
+    df = pd.DataFrame()
+    df["transaction_velocity"] = (
+        inputs["total_transactions"] /
+        (inputs["days_since_last_purchase"] + 1)
+    )
+    df["is_high_value"] = (inputs["avg_transaction_amount"] > 100).astype(int)
+    return df
+```
+
+## Stream Features
+
+### Kafka Integration
+```python
+from feast import KafkaSource, FeatureView
+from feast.data_format import JsonFormat
+
+# Kafka source for real-time features
+click_stream_source = KafkaSource(
+    name="click_stream",
+    kafka_bootstrap_servers="kafka:9092",
+    topic="user_clicks",
+    timestamp_field="event_timestamp",
+    message_format=JsonFormat(
+        schema_json="""
+        {
+            "type": "record",
+            "name": "click",
+            "fields": [
+                {"name": "user_id", "type": "string"},
+                {"name": "page_id", "type": "string"},
+                {"name": "event_timestamp", "type": "long"}
+            ]
+        }
+        """
+    )
+)
+
+# Stream feature view
+click_features = FeatureView(
+    name="user_click_features",
+    entities=["user_id"],
+    ttl=timedelta(minutes=30),
+    schema=[
+        Feature(name="click_count_30m", dtype=Int64),
+        Feature(name="unique_pages_30m", dtype=Int64)
+    ],
+    source=click_stream_source,
+    online=True
+)
+```
+
+## Feature Engineering Pipelines
+
+### Batch Pipeline (Spark)
+```python
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+from pyspark.sql.window import Window
+
+def compute_customer_features(spark: SparkSession, date: str):
+    transactions = spark.read.parquet(f"s3://data/transactions/date={date}")
+
+    # Aggregation window
+    window_30d = Window.partitionBy("customer_id").orderBy("timestamp").rangeBetween(-30*86400, 0)
+
+    features = transactions.groupBy("customer_id").agg(
+        F.count("*").alias("total_transactions"),
+        F.avg("amount").alias("avg_transaction_amount"),
+        F.sum("amount").alias("total_spend"),
+        F.max("timestamp").alias("last_transaction"),
+        F.countDistinct("merchant_id").alias("unique_merchants")
+    )
+
+    # Compute derived features
+    features = features.withColumn(
+        "days_since_last_purchase",
+        F.datediff(F.current_date(), F.col("last_transaction"))
+    )
+
+    # Write to feature store
+    features.write.mode("overwrite").parquet(
+        f"s3://feature-store/customer_stats/date={date}"
+    )
+
+    return features
+```
+
+### Streaming Pipeline (Flink)
+```python
+from pyflink.datastream import StreamExecutionEnvironment
+from pyflink.table import StreamTableEnvironment
+
+def create_streaming_features():
+    env = StreamExecutionEnvironment.get_execution_environment()
+    t_env = StreamTableEnvironment.create(env)
+
+    # Read from Kafka
+    t_env.execute_sql("""
+        CREATE TABLE clicks (
+            user_id STRING,
+            page_id STRING,
+            event_time TIMESTAMP(3),
+            WATERMARK FOR event_time AS event_time - INTERVAL '5' SECOND
+        ) WITH (
+            'connector' = 'kafka',
+            'topic' = 'user_clicks',
+            'properties.bootstrap.servers' = 'kafka:9092',
+            'format' = 'json'
+        )
+    """)
+
+    # Windowed aggregation
+    t_env.execute_sql("""
+        CREATE TABLE click_features (
+            user_id STRING,
+            window_start TIMESTAMP(3),
+            click_count BIGINT,
+            unique_pages BIGINT,
+            PRIMARY KEY (user_id) NOT ENFORCED
+        ) WITH (
+            'connector' = 'upsert-kafka',
+            'topic' = 'click_features',
+            'properties.bootstrap.servers' = 'kafka:9092',
+            'key.format' = 'json',
+            'value.format' = 'json'
+        )
+    """)
+
+    t_env.execute_sql("""
+        INSERT INTO click_features
+        SELECT
+            user_id,
+            TUMBLE_START(event_time, INTERVAL '30' MINUTE) as window_start,
+            COUNT(*) as click_count,
+            COUNT(DISTINCT page_id) as unique_pages
+        FROM clicks
+        GROUP BY user_id, TUMBLE(event_time, INTERVAL '30' MINUTE)
+    """)
+```
+
+## Best Practices
+
+1. **Point-in-Time Correctness**: Prevent data leakage
+2. **Feature Documentation**: Clear descriptions
+3. **Version Control**: Track feature definitions
+4. **Monitoring**: Feature freshness and quality
+5. **Naming Conventions**: Consistent feature names
+
+## Feature Quality Monitoring
+
+```python
+from great_expectations import DataContext
+
+def validate_features(feature_df):
+    context = DataContext()
+
+    expectation_suite = context.create_expectation_suite("feature_validation")
+
+    # Add expectations
+    validator = context.get_validator(
+        batch_request=batch_request,
+        expectation_suite_name="feature_validation"
+    )
+
+    validator.expect_column_values_to_not_be_null("customer_id")
+    validator.expect_column_values_to_be_between("avg_transaction_amount", 0, 10000)
+    validator.expect_column_values_to_be_in_set("customer_segment", ["bronze", "silver", "gold"])
+
+    results = validator.validate()
+    return results.success
+```
+
+## Anti-Patterns
+
+- Training/serving skew
+- Missing point-in-time correctness
+- Feature definitions in notebooks only
+- No feature documentation
+- Ignoring feature freshness
+
+## When to Use
+
+- Multiple models share features
+- Need training/serving consistency
+- Feature computation is expensive
+- Team collaboration on features
+- Regulatory requirements
+
+## When NOT to Use
+
+- Single simple model
+- Features change frequently
+- No reuse expected
+- Small team, simple pipeline