javi-forge 0.1.0

package/ai-config/skills/data-ai/mlflow/SKILL.md

@@ -0,0 +1,302 @@
+---
+name: mlflow
+description: >
+  ML experiment tracking, model registry, and deployment with MLflow.
+  Trigger: mlflow, experiment tracking, model registry, mlops, model versioning
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [mlflow, mlops, experiment-tracking, model-registry]
+  updated: "2026-02"
+---
+
+# MLflow Skill
+
+ML experiment tracking, model registry, and deployment.
+
+## Stack
+
+```yaml
+mlflow: 2.10+
+mlflow-skinny: 2.10+  # client only
+boto3: 1.34+  # S3 artifacts
+psycopg2: 2.9+  # PostgreSQL backend
+```
+
+## Docker Setup
+
+```yaml
+# docker-compose.mlflow.yml
+services:
+  mlflow:
+    image: ghcr.io/mlflow/mlflow:v2.10.0
+    ports:
+      - "5000:5000"
+    environment:
+      - MLFLOW_BACKEND_STORE_URI=postgresql://mlflow:mlflow@postgres:5432/mlflow
+      - MLFLOW_DEFAULT_ARTIFACT_ROOT=s3://mlflow-artifacts
+    command: >
+      mlflow server
+      --backend-store-uri postgresql://mlflow:mlflow@postgres:5432/mlflow
+      --default-artifact-root s3://mlflow-artifacts
+      --host 0.0.0.0 --port 5000
+    depends_on:
+      - postgres
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: mlflow
+      POSTGRES_PASSWORD: mlflow
+      POSTGRES_DB: mlflow
+    volumes:
+      - mlflow_db:/var/lib/postgresql/data
+
+volumes:
+  mlflow_db:
+```
+
+## Client Configuration
+
+```python
+import os
+import mlflow
+
+def setup_mlflow():
+    mlflow.set_tracking_uri(os.getenv("MLFLOW_TRACKING_URI", "http://localhost:5000"))
+    mlflow.set_experiment("my-project")
+    mlflow.sklearn.autolog()  # Enable autologging
+    mlflow.pytorch.autolog()
+```
+
+## Experiment Tracking
+
+```python
+import mlflow
+from mlflow.models import infer_signature
+from datetime import datetime
+
+def train_with_tracking(model, X_train, y_train, X_test, y_test, model_name: str, tags: dict = None):
+    with mlflow.start_run(run_name=f"{model_name}_{datetime.now().strftime('%Y%m%d_%H%M')}"):
+        # Tags
+        mlflow.set_tags({
+            "model_type": model_name,
+            "environment": os.getenv("ENVIRONMENT", "dev"),
+            **(tags or {})
+        })
+
+        # Parameters
+        if hasattr(model, 'get_params'):
+            mlflow.log_params(model.get_params())
+
+        # Train
+        model.fit(X_train, y_train)
+        y_pred = model.predict(X_test)
+
+        # Metrics
+        from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
+        mlflow.log_metrics({
+            "accuracy": accuracy_score(y_test, y_pred),
+            "f1_score": f1_score(y_test, y_pred, average='weighted'),
+            "precision": precision_score(y_test, y_pred, average='weighted'),
+            "recall": recall_score(y_test, y_pred, average='weighted'),
+        })
+
+        # Log model with signature
+        signature = infer_signature(X_train, y_pred)
+        mlflow.sklearn.log_model(model, "model", signature=signature, registered_model_name=model_name)
+
+        # Log confusion matrix
+        log_confusion_matrix(y_test, y_pred)
+
+        return mlflow.active_run().info.run_id
+
+def log_confusion_matrix(y_true, y_pred):
+    import matplotlib.pyplot as plt
+    import seaborn as sns
+    from sklearn.metrics import confusion_matrix
+
+    cm = confusion_matrix(y_true, y_pred)
+    fig, ax = plt.subplots(figsize=(8, 6))
+    sns.heatmap(cm, annot=True, fmt='d', ax=ax)
+    ax.set_xlabel('Predicted')
+    ax.set_ylabel('Actual')
+    plt.savefig('/tmp/confusion_matrix.png')
+    mlflow.log_artifact('/tmp/confusion_matrix.png')
+    plt.close()
+```
+
+## PyTorch Lightning Integration
+
+```python
+import lightning as L
+from lightning.pytorch.loggers import MLFlowLogger
+from lightning.pytorch.callbacks import ModelCheckpoint, EarlyStopping
+
+def train_with_lightning(model: L.LightningModule, datamodule: L.LightningDataModule, experiment_name: str):
+    mlflow_logger = MLFlowLogger(
+        experiment_name=experiment_name,
+        tracking_uri=os.getenv("MLFLOW_TRACKING_URI"),
+        log_model=True
+    )
+
+    trainer = L.Trainer(
+        max_epochs=100,
+        accelerator="auto",
+        logger=mlflow_logger,
+        callbacks=[
+            ModelCheckpoint(monitor="val_loss", mode="min", save_top_k=3),
+            EarlyStopping(monitor="val_loss", patience=10)
+        ]
+    )
+
+    mlflow_logger.log_hyperparams(model.hparams)
+    trainer.fit(model, datamodule)
+    trainer.test(model, datamodule)
+    return mlflow_logger.run_id
+```
+
+## Model Registry
+
+```python
+from mlflow.tracking import MlflowClient
+
+def register_model(run_id: str, model_name: str, description: str = None, tags: dict = None):
+    client = MlflowClient()
+    model_uri = f"runs:/{run_id}/model"
+    result = mlflow.register_model(model_uri, model_name)
+
+    if description:
+        client.update_registered_model(name=model_name, description=description)
+
+    if tags:
+        for key, value in tags.items():
+            client.set_model_version_tag(name=model_name, version=result.version, key=key, value=value)
+
+    print(f"Registered {model_name} v{result.version}")
+    return result.version
+
+def promote_model(model_name: str, version: int, stage: str):  # "Staging", "Production", "Archived"
+    client = MlflowClient()
+    client.transition_model_version_stage(
+        name=model_name,
+        version=version,
+        stage=stage,
+        archive_existing_versions=(stage == "Production")
+    )
+
+def get_production_model(model_name: str):
+    client = MlflowClient()
+    versions = client.get_latest_versions(model_name, stages=["Production"])
+    if not versions:
+        raise ValueError(f"No production version for {model_name}")
+    return versions[0]
+```
+
+## Model Loading
+
+```python
+def load_model(model_name: str, stage: str = "Production"):
+    return mlflow.sklearn.load_model(f"models:/{model_name}/{stage}")
+
+def load_model_version(model_name: str, version: int):
+    return mlflow.sklearn.load_model(f"models:/{model_name}/{version}")
+
+class CachedModelLoader:
+    def __init__(self):
+        self._cache = {}
+        self._client = MlflowClient()
+
+    def get_model(self, model_name: str, stage: str = "Production"):
+        cache_key = f"{model_name}:{stage}"
+        versions = self._client.get_latest_versions(model_name, stages=[stage])
+        if not versions:
+            raise ValueError(f"No {stage} version for {model_name}")
+
+        current_version = versions[0].version
+        if cache_key in self._cache:
+            cached_version, model = self._cache[cache_key]
+            if cached_version == current_version:
+                return model
+
+        model = load_model(model_name, stage)
+        self._cache[cache_key] = (current_version, model)
+        return model
+```
+
+## Model Comparison
+
+```python
+import pandas as pd
+
+def compare_runs(experiment_name: str, metric: str = "f1_score", top_n: int = 10) -> pd.DataFrame:
+    client = MlflowClient()
+    experiment = client.get_experiment_by_name(experiment_name)
+
+    runs = client.search_runs(
+        experiment_ids=[experiment.experiment_id],
+        order_by=[f"metrics.{metric} DESC"],
+        max_results=top_n
+    )
+
+    data = []
+    for run in runs:
+        row = {
+            "run_id": run.info.run_id,
+            "run_name": run.info.run_name,
+            "duration_min": (run.info.end_time - run.info.start_time) / 60000 if run.info.end_time else None,
+        }
+        row.update(run.data.params)
+        row.update(run.data.metrics)
+        data.append(row)
+
+    return pd.DataFrame(data)
+
+def get_best_run(experiment_name: str, metric: str = "f1_score", mode: str = "max") -> str:
+    client = MlflowClient()
+    experiment = client.get_experiment_by_name(experiment_name)
+    order = "DESC" if mode == "max" else "ASC"
+
+    runs = client.search_runs(
+        experiment_ids=[experiment.experiment_id],
+        order_by=[f"metrics.{metric} {order}"],
+        max_results=1
+    )
+    return runs[0].info.run_id if runs else None
+```
+
+## Serving
+
+```bash
+# REST API serving
+mlflow models serve -m "models:/anomaly_detector/Production" -p 5001
+
+# Docker deployment
+mlflow models build-docker -m "models:/anomaly_detector/Production" -n ml-model
+docker run -p 5001:8080 ml-model
+
+# Request prediction
+curl -X POST http://localhost:5001/invocations \
+  -H "Content-Type: application/json" \
+  -d '{"inputs": [[1.0, 2.0, 3.0, 4.0, 5.0]]}'
+```
+
+## Best Practices
+
+1. **Always use experiments** - `mlflow.set_experiment("project-name")`
+2. **Log everything** - params, metrics, artifacts, model signature
+3. **Use signatures** - `infer_signature(X_train, y_pred)` for input validation
+4. **Tag runs** - model_type, dataset_version, author for filtering
+5. **Version artifacts** - `mlflow.log_artifact("config.yaml", artifact_path="config")`
+
+## Related Skills
+
+- `pytorch`: Deep learning experiments
+- `scikit-learn`: Classical ML experiments
+- `kubernetes`: Model deployment at scale
+- `docker-containers`: Containerized tracking

package/ai-config/skills/data-ai/onnx-inference/SKILL.md

@@ -0,0 +1,290 @@
+---
+name: onnx-inference
+description: >
+  Deploy ML models with ONNX Runtime for cross-platform inference (Python, Rust, Go).
+  Trigger: onnx, model deployment, inference, edge ml, quantization
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [onnx, ml, inference, edge, deployment]
+  updated: "2026-02"
+---
+
+# ONNX Inference Skill
+
+Deploy ML models with ONNX Runtime for cross-platform, high-performance inference.
+
+## Stack
+
+```yaml
+# Python
+onnx: 1.15+
+onnxruntime: 1.17+
+torch: 2.2+
+
+# Rust
+ort: 2.0+
+
+# Go
+onnxruntime_go: 1.17+
+```
+
+## Model Export
+
+### PyTorch to ONNX
+
+```python
+import torch
+import torch.onnx
+
+def export_model(model: torch.nn.Module, output_path: str, input_shape: tuple):
+    model.eval()
+    dummy_input = torch.randn(*input_shape)
+
+    torch.onnx.export(
+        model,
+        dummy_input,
+        output_path,
+        export_params=True,
+        opset_version=17,
+        do_constant_folding=True,
+        input_names=['input'],
+        output_names=['output'],
+        dynamic_axes={
+            'input': {0: 'batch_size'},
+            'output': {0: 'batch_size'}
+        }
+    )
+
+# Validate
+def validate_model(model_path: str):
+    import onnx
+    model = onnx.load(model_path)
+    onnx.checker.check_model(model)
+    print(f"Inputs: {[i.name for i in model.graph.input]}")
+    print(f"Outputs: {[o.name for o in model.graph.output]}")
+```
+
+### Scikit-learn to ONNX
+
+```python
+from skl2onnx import convert_sklearn
+from skl2onnx.common.data_types import FloatTensorType
+
+def export_sklearn(model, output_path: str, input_features: int):
+    initial_type = [('input', FloatTensorType([None, input_features]))]
+    onnx_model = convert_sklearn(
+        model,
+        initial_types=initial_type,
+        target_opset=17,
+        options={id(model): {'zipmap': False}}
+    )
+    with open(output_path, 'wb') as f:
+        f.write(onnx_model.SerializeToString())
+```
+
+## Python Inference
+
+```python
+import onnxruntime as ort
+import numpy as np
+
+class ONNXPredictor:
+    def __init__(self, model_path: str, providers: list = None):
+        if providers is None:
+            providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
+
+        sess_options = ort.SessionOptions()
+        sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        sess_options.intra_op_num_threads = 4
+
+        self.session = ort.InferenceSession(model_path, sess_options, providers=providers)
+        self.input_name = self.session.get_inputs()[0].name
+        self.output_names = [o.name for o in self.session.get_outputs()]
+
+    def predict(self, data: np.ndarray) -> np.ndarray:
+        if data.dtype != np.float32:
+            data = data.astype(np.float32)
+        outputs = self.session.run(self.output_names, {self.input_name: data})
+        return outputs[0]
+
+    def predict_batch(self, data: np.ndarray, batch_size: int = 32) -> np.ndarray:
+        results = []
+        for i in range(0, len(data), batch_size):
+            batch = data[i:i + batch_size]
+            results.append(self.predict(batch))
+        return np.concatenate(results, axis=0)
+```
+
+## Rust Inference
+
+```rust
+// Cargo.toml: ort = { version = "2.0", features = ["load-dynamic"] }
+use ort::{Environment, Session, SessionBuilder, Value};
+use ndarray::{Array3, ArrayD, IxDyn};
+use std::sync::Arc;
+
+pub struct Predictor {
+    session: Session,
+}
+
+impl Predictor {
+    pub fn new(model_path: &str) -> Result<Self, ort::Error> {
+        let environment = Arc::new(
+            Environment::builder()
+                .with_name("inference")
+                .with_execution_providers([ort::ExecutionProvider::CPU(Default::default())])
+                .build()?
+        );
+
+        let session = SessionBuilder::new(&environment)?
+            .with_optimization_level(ort::GraphOptimizationLevel::Level3)?
+            .with_intra_threads(4)?
+            .with_model_from_file(model_path)?;
+
+        Ok(Self { session })
+    }
+
+    pub fn predict(&self, input: Array3<f32>) -> Result<Vec<f32>, ort::Error> {
+        let shape = input.shape().to_vec();
+        let data: Vec<f32> = input.into_iter().collect();
+        let tensor = Value::from_array(
+            self.session.allocator(),
+            &ArrayD::from_shape_vec(IxDyn(&shape), data)?
+        )?;
+
+        let outputs = self.session.run(vec![tensor])?;
+        let result: ArrayD<f32> = outputs[0].try_extract()?.view().to_owned();
+        Ok(result.into_iter().collect())
+    }
+}
+```
+
+## Go Inference
+
+```go
+package ml
+
+import "github.com/yalue/onnxruntime_go"
+
+type Predictor struct {
+    session *onnxruntime_go.AdvancedSession
+}
+
+func NewPredictor(modelPath string) (*Predictor, error) {
+    onnxruntime_go.InitializeEnvironment()
+    session, err := onnxruntime_go.NewAdvancedSession(
+        modelPath,
+        []string{"input"},
+        []string{"output"},
+        nil,
+    )
+    if err != nil {
+        return nil, err
+    }
+    return &Predictor{session: session}, nil
+}
+
+func (p *Predictor) Predict(input []float32, shape []int64) ([]float32, error) {
+    inputTensor, _ := onnxruntime_go.NewTensor(onnxruntime_go.NewShape(shape...), input)
+    defer inputTensor.Destroy()
+
+    outputSize := int64(1)
+    for _, d := range shape[1:] { outputSize *= d }
+    outputData := make([]float32, outputSize)
+    outputTensor, _ := onnxruntime_go.NewTensor(onnxruntime_go.NewShape(shape[0], outputSize), outputData)
+    defer outputTensor.Destroy()
+
+    p.session.Run()
+    return outputTensor.GetData(), nil
+}
+
+func (p *Predictor) Close() {
+    p.session.Destroy()
+    onnxruntime_go.DestroyEnvironment()
+}
+```
+
+## Model Optimization
+
+### Quantization
+
+```python
+from onnxruntime.quantization import quantize_dynamic, QuantType
+
+def quantize_model(input_path: str, output_path: str):
+    quantize_dynamic(input_path, output_path, weight_type=QuantType.QInt8)
+
+def optimize_model(input_path: str, output_path: str):
+    import onnxoptimizer
+    import onnx
+
+    model = onnx.load(input_path)
+    passes = [
+        'eliminate_identity',
+        'eliminate_deadend',
+        'fuse_bn_into_conv',
+        'fuse_matmul_add_bias_into_gemm',
+    ]
+    optimized = onnxoptimizer.optimize(model, passes)
+    onnx.save(optimized, output_path)
+
+# Full pipeline
+def prepare_for_edge(model_path: str):
+    base = model_path.replace('.onnx', '')
+    optimize_model(model_path, f"{base}_optimized.onnx")
+    quantize_model(f"{base}_optimized.onnx", f"{base}_quantized.onnx")
+```
+
+## Model Registry
+
+```python
+import hashlib
+import json
+from pathlib import Path
+from datetime import datetime
+
+class ModelRegistry:
+    def __init__(self, registry_path: str = "models/registry.json"):
+        self.path = Path(registry_path)
+        self.registry = json.loads(self.path.read_text()) if self.path.exists() else {"models": {}}
+
+    def register(self, name: str, model_path: str, version: str, metrics: dict = None):
+        model_hash = hashlib.sha256(Path(model_path).read_bytes()).hexdigest()[:16]
+        if name not in self.registry["models"]:
+            self.registry["models"][name] = {"versions": {}}
+
+        self.registry["models"][name]["versions"][version] = {
+            "path": model_path,
+            "hash": model_hash,
+            "registered_at": datetime.utcnow().isoformat(),
+            "metrics": metrics or {}
+        }
+        self.registry["models"][name]["latest"] = version
+        self.path.write_text(json.dumps(self.registry, indent=2))
+
+    def get_path(self, name: str, version: str = "latest") -> str:
+        if version == "latest":
+            version = self.registry["models"][name]["latest"]
+        return self.registry["models"][name]["versions"][version]["path"]
+```
+
+## Best Practices
+
+1. **Dynamic axes** - Enable batch dimension flexibility
+2. **Quantize for edge** - INT8 reduces size ~4x with minimal accuracy loss
+3. **Validate exports** - Always run `onnx.checker.check_model()`
+4. **Use execution providers** - CUDA > TensorRT > CPU fallback
+5. **Version models** - Track hash, metrics, and metadata
+
+## Related Skills
+
+- `pytorch`: Model training source
+- `scikit-learn`: Classical ML models
+- `ai-ml`: Full pipeline patterns
+- `fastapi`: Model serving API