@defai.digital/cli 13.4.3 → 13.4.5

package/bundled/workflows/ml-model-evaluation.yaml

@@ -0,0 +1,234 @@
+workflowId: ml-model-evaluation
+name: Model Evaluation Pipeline
+description: Comprehensive model evaluation with performance, fairness, robustness, and production-readiness checks
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - evaluation
+  - fairness
+  - robustness
+  - production
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+    - statistical-analysis
+    - data-analysis
+  estimatedDuration: 600
+  complexity: high
+
+steps:
+  - stepId: performance-metrics
+    name: Calculate Performance Metrics
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Evaluate model performance on the test dataset.
+
+        ## Classification Metrics (if applicable)
+        - Accuracy
+        - Precision (macro, micro, weighted)
+        - Recall (macro, micro, weighted)
+        - F1 Score (macro, micro, weighted)
+        - AUC-ROC (one-vs-rest for multiclass)
+        - AUC-PR (Precision-Recall curve)
+        - Log Loss
+        - Confusion Matrix
+        - Per-class metrics breakdown
+
+        ## Regression Metrics (if applicable)
+        - Mean Squared Error (MSE)
+        - Root Mean Squared Error (RMSE)
+        - Mean Absolute Error (MAE)
+        - R-squared (Coefficient of Determination)
+        - Mean Absolute Percentage Error (MAPE)
+        - Residual analysis
+
+        ## Confidence Intervals
+        - Bootstrap 95% CI for primary metric
+        - Report standard deviation across folds
+
+        Provide structured output with all applicable metrics.
+
+  - stepId: fairness-audit
+    name: Fairness & Bias Audit
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Conduct a comprehensive fairness and bias audit.
+
+        ## Demographic Analysis
+        For each protected attribute (gender, age, race, etc.):
+
+        1. **Demographic Parity**:
+           - Selection rate across groups
+           - Disparate impact ratio (should be > 0.8)
+
+        2. **Equal Opportunity**:
+           - True positive rate across groups
+           - False negative rate disparity
+
+        3. **Equalized Odds**:
+           - TPR and FPR across groups
+           - Maximum disparity
+
+        4. **Calibration**:
+           - Predicted probability vs actual outcome by group
+
+        ## Bias Detection
+        - Identify any systematic prediction bias
+        - Check for proxy discrimination
+        - Analyze error distribution across groups
+
+        ## Mitigation Recommendations
+        If bias is detected:
+        - Recommend specific mitigation strategies
+        - Estimate impact of mitigation on overall performance
+
+        ## Compliance Check
+        - Flag any potential regulatory concerns (GDPR, CCPA, ECOA)
+
+  - stepId: robustness-testing
+    name: Robustness & Edge Case Testing
+    type: prompt
+    timeout: 180000
+    config:
+      agent: ml-engineer
+      task: |
+        Test model robustness and behavior on edge cases.
+
+        ## Input Perturbation Testing
+        1. **Noise Sensitivity**:
+           - Add Gaussian noise to numerical features
+           - Measure performance degradation curve
+
+        2. **Missing Values**:
+           - Simulate missing data patterns
+           - Test model behavior with incomplete inputs
+
+        3. **Out-of-Range Values**:
+           - Test with extreme values
+           - Check for catastrophic failures
+
+        ## Distribution Shift Testing
+        1. **Covariate Shift**:
+           - Test on data with shifted feature distributions
+           - Measure performance drop
+
+        2. **Label Shift**:
+           - Test with different class proportions
+           - Check calibration under shift
+
+        ## Edge Cases
+        1. Boundary conditions
+        2. Rare class handling
+        3. Adversarial-like examples (if applicable)
+
+        ## Out-of-Distribution Detection
+        - Can the model identify when inputs are OOD?
+        - Confidence calibration on OOD samples
+
+  - stepId: latency-profiling
+    name: Latency & Resource Profiling
+    type: prompt
+    timeout: 120000
+    config:
+      agent: ml-engineer
+      task: |
+        Profile model for production deployment.
+
+        ## Inference Latency
+        - Single sample latency (p50, p95, p99)
+        - Batch inference latency by batch size
+        - Cold start latency
+        - Warm cache latency
+
+        ## Resource Requirements
+        - Memory footprint (model size)
+        - Peak memory during inference
+        - CPU utilization pattern
+        - GPU utilization (if applicable)
+        - GPU memory requirements
+
+        ## Scalability Analysis
+        - Throughput (requests/second)
+        - Scaling behavior under load
+        - Recommended instance type
+        - Cost per 1M predictions
+
+        ## Optimization Opportunities
+        - Quantization potential
+        - Pruning opportunities
+        - Distillation candidates
+        - Caching strategies
+
+  - stepId: evaluation-report
+    name: Generate Evaluation Report
+    type: prompt
+    timeout: 120000
+    config:
+      agent: data-scientist
+      task: |
+        Create a comprehensive model evaluation report.
+
+        ## Executive Summary
+        - Model name and version
+        - One-paragraph assessment
+        - Go/No-Go recommendation with confidence level
+
+        ## Performance Summary
+        - Primary metric with confidence interval
+        - Comparison to baseline
+        - Key strengths and weaknesses
+
+        ## Fairness Summary
+        - Overall fairness assessment (Pass/Fail/Warning)
+        - Groups with highest disparity
+        - Required mitigations before deployment
+
+        ## Robustness Summary
+        - Robustness score (1-10)
+        - Critical failure modes identified
+        - Recommended guardrails
+
+        ## Production Readiness
+        - Latency requirements: Met/Not Met
+        - Resource requirements: Acceptable/High/Excessive
+        - Scaling assessment
+
+        ## Recommendations
+        1. **If Go**: Deployment checklist
+        2. **If No-Go**: Required improvements ranked by priority
+        3. **Monitoring requirements** for production
+
+        ## Appendix
+        - Full metrics tables
+        - Visualization references
+        - Test dataset description
+
+  - stepId: store-evaluation
+    name: Store Evaluation Results
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-evaluations
+      key: "{{model_name}}/{{model_version}}/evaluation"
+      ttl: 15552000
+      value:
+        model_name: "{{model_name}}"
+        model_version: "{{model_version}}"
+        evaluation_id: "{{evaluation_id}}"
+        recommendation: "{{go_no_go}}"
+        performance_metrics: "{{performance_metrics}}"
+        fairness_results: "{{fairness_results}}"
+        robustness_score: "{{robustness_score}}"
+        latency_profile: "{{latency_profile}}"
+        full_report: "{{evaluation_report}}"
+        evaluated_at: "{{timestamp}}"
+        evaluated_by: "{{user}}"

package/bundled/workflows/ml-model-monitoring.yaml

@@ -0,0 +1,227 @@
+workflowId: ml-model-monitoring
+name: Production Model Monitoring
+description: Detect drift, degradation, and anomalies in production models
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - monitoring
+  - drift
+  - production
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+    - statistical-analysis
+    - data-analysis
+  estimatedDuration: 300
+  complexity: high
+  schedule: "0 */6 * * *"  # Every 6 hours
+
+steps:
+  - stepId: fetch-baseline
+    name: Fetch Baseline Statistics
+    type: tool
+    timeout: 10000
+    tool: memory_retrieve
+    config:
+      namespace: ml-model-registry
+      key: "{{model_name}}/versions/{{production_version}}/baseline_stats"
+
+  - stepId: fetch-production-data
+    name: Fetch Recent Production Data
+    type: tool
+    timeout: 30000
+    tool: memory_search
+    config:
+      namespace: ml-production-predictions
+      query: "model:{{model_name}} AND timestamp:[{{window_start}} TO {{window_end}}]"
+      limit: 10000
+
+  - stepId: data-drift-detection
+    name: Detect Data Drift
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Analyze production data for drift compared to training data.
+
+        ## Feature Distribution Analysis
+        For each feature:
+
+        1. **Statistical Tests**:
+           - Kolmogorov-Smirnov test (continuous features)
+           - Chi-squared test (categorical features)
+           - Population Stability Index (PSI)
+
+        2. **Drift Severity**:
+           - No Drift: PSI < 0.1
+           - Moderate Drift: 0.1 <= PSI < 0.25
+           - Significant Drift: PSI >= 0.25
+
+        3. **Per-Feature Report**:
+           - Feature name
+           - Drift score
+           - Baseline mean/distribution
+           - Current mean/distribution
+           - Visual: distribution comparison
+
+        ## Overall Assessment
+        - Total features with significant drift
+        - Most drifted features (top 5)
+        - Drift trend (increasing/stable/decreasing)
+
+        ## Alert Level
+        - GREEN: No significant drift
+        - YELLOW: Moderate drift, monitor closely
+        - RED: Significant drift, action required
+
+  - stepId: prediction-drift
+    name: Detect Prediction Drift
+    type: prompt
+    timeout: 120000
+    config:
+      agent: data-scientist
+      task: |
+        Analyze prediction distribution for drift.
+
+        ## Prediction Distribution Analysis
+
+        1. **Output Distribution**:
+           - Compare current vs baseline prediction distribution
+           - For classification: class probability distributions
+           - For regression: prediction value distribution
+
+        2. **Confidence Scores**:
+           - Average confidence: baseline vs current
+           - Confidence distribution shift
+           - Low-confidence prediction rate
+
+        3. **Prediction Patterns**:
+           - Class balance (classification)
+           - Prediction range (regression)
+           - Unusual prediction clusters
+
+        ## Anomaly Detection
+        - Identify sudden prediction shifts
+        - Detect prediction patterns not seen in training
+        - Flag potential model failures
+
+  - stepId: performance-degradation
+    name: Check Performance Degradation
+    type: prompt
+    timeout: 120000
+    config:
+      agent: ml-engineer
+      task: |
+        Compare current model performance against baseline.
+
+        ## Metric Comparison (if ground truth available)
+
+        | Metric | Baseline | Current | Delta | Status |
+        |--------|----------|---------|-------|--------|
+        | Accuracy | X | Y | Z% | OK/WARN/ALERT |
+        | Precision | X | Y | Z% | OK/WARN/ALERT |
+        | Recall | X | Y | Z% | OK/WARN/ALERT |
+
+        ## Thresholds
+        - OK: Within 5% of baseline
+        - WARN: 5-10% degradation
+        - ALERT: >10% degradation
+
+        ## Latency Analysis
+        - P50 latency: baseline vs current
+        - P99 latency: baseline vs current
+        - Timeout rate
+
+        ## Error Analysis
+        - Error rate trend
+        - Error type distribution
+        - New error patterns
+
+  - stepId: retraining-decision
+    name: Retraining Recommendation
+    type: prompt
+    timeout: 60000
+    config:
+      agent: ml-engineer
+      task: |
+        Based on drift and degradation analysis, recommend action.
+
+        ## Decision Matrix
+
+        | Data Drift | Performance Drop | Recommendation |
+        |------------|------------------|----------------|
+        | None | None | NO_ACTION |
+        | Moderate | None | MONITOR |
+        | Significant | None | INVESTIGATE |
+        | None | Moderate | INVESTIGATE |
+        | Moderate | Moderate | RETRAIN |
+        | Significant | Any | RETRAIN_URGENT |
+        | Any | Significant | RETRAIN_URGENT |
+
+        ## Recommendation Output
+        - **Action**: NO_ACTION / MONITOR / INVESTIGATE / RETRAIN / RETRAIN_URGENT
+        - **Confidence**: HIGH / MEDIUM / LOW
+        - **Reasoning**: Explanation
+
+        ## If RETRAIN Recommended
+        - Suggested training data window
+        - Features to focus on
+        - Hyperparameter adjustments to consider
+
+        ## If INVESTIGATE Recommended
+        - Likely root causes to check
+        - Data quality checks to run
+        - External factors to consider
+
+  - stepId: generate-alerts
+    name: Generate Monitoring Alerts
+    type: conditional
+    config:
+      condition: "{{action}} != 'NO_ACTION'"
+      then:
+        - stepId: store-alert
+          type: tool
+          tool: memory_store
+          config:
+            namespace: ml-alerts
+            key: "{{model_name}}/{{timestamp}}"
+            value:
+              alert_id: "{{alert_id}}"
+              model_name: "{{model_name}}"
+              model_version: "{{production_version}}"
+              severity: "{{severity}}"
+              action_required: "{{action}}"
+              drift_summary:
+                data_drift_level: "{{data_drift_level}}"
+                prediction_drift_level: "{{prediction_drift_level}}"
+                top_drifted_features: "{{top_drifted_features}}"
+              performance_summary:
+                degradation_detected: "{{degradation_detected}}"
+                metrics_affected: "{{metrics_affected}}"
+              recommendation: "{{recommendation}}"
+              created_at: "{{timestamp}}"
+              acknowledged: false
+
+  - stepId: store-monitoring-report
+    name: Store Monitoring Report
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-monitoring-reports
+      key: "{{model_name}}/{{report_date}}"
+      ttl: 2592000
+      value:
+        model_name: "{{model_name}}"
+        model_version: "{{production_version}}"
+        report_period:
+          start: "{{window_start}}"
+          end: "{{window_end}}"
+        data_drift_report: "{{data_drift_report}}"
+        prediction_drift_report: "{{prediction_drift_report}}"
+        performance_report: "{{performance_report}}"
+        action_taken: "{{action}}"
+        generated_at: "{{timestamp}}"

package/bundled/workflows/ml-model-registry.yaml

@@ -0,0 +1,232 @@
+workflowId: ml-model-registry
+name: Model Registry Management
+description: Version, store, and manage ML models through their lifecycle
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - model-registry
+  - versioning
+  - deployment
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+  estimatedDuration: 120
+  complexity: low
+
+steps:
+  - stepId: validate-model
+    name: Validate Model Artifact
+    type: prompt
+    timeout: 60000
+    config:
+      agent: ml-engineer
+      task: |
+        Validate the model artifact before registration.
+
+        ## Validation Checks
+        1. **Artifact Integrity**:
+           - Model file exists and is readable
+           - Checksum matches expected value
+
+        2. **Schema Validation**:
+           - Input schema is defined
+           - Output schema is defined
+           - Feature names match training data
+
+        3. **Metadata Completeness**:
+           - Training dataset version recorded
+           - Hyperparameters recorded
+           - Evaluation metrics recorded
+           - Experiment ID linked
+
+        4. **Dependency Check**:
+           - Required libraries documented
+           - Version compatibility verified
+
+        Return validation status: PASS or FAIL with details.
+
+  - stepId: register-model
+    name: Register Model Version
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-model-registry
+      key: "{{model_name}}/versions/{{version}}"
+      value:
+        model_name: "{{model_name}}"
+        version: "{{version}}"
+        description: "{{description}}"
+
+        # Lineage
+        experiment_id: "{{experiment_id}}"
+        training_dataset: "{{dataset_version}}"
+        parent_model: "{{parent_version}}"
+
+        # Artifact
+        artifact_path: "{{artifact_path}}"
+        artifact_checksum: "{{checksum}}"
+        artifact_size_bytes: "{{size}}"
+
+        # Schemas
+        input_schema: "{{input_schema}}"
+        output_schema: "{{output_schema}}"
+        feature_names: "{{feature_names}}"
+
+        # Performance
+        metrics: "{{metrics}}"
+        evaluation_id: "{{evaluation_id}}"
+
+        # Dependencies
+        framework: "{{framework}}"
+        framework_version: "{{framework_version}}"
+        dependencies: "{{dependencies}}"
+
+        # Lifecycle
+        status: "staged"
+        created_at: "{{timestamp}}"
+        created_by: "{{user}}"
+
+        # Tags
+        tags: "{{tags}}"
+
+  - stepId: update-model-index
+    name: Update Model Index
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-model-registry
+      key: "{{model_name}}/index"
+      value:
+        model_name: "{{model_name}}"
+        latest_version: "{{version}}"
+        production_version: "{{production_version}}"
+        all_versions: "{{version_list}}"
+        updated_at: "{{timestamp}}"
+
+  - stepId: compare-versions
+    name: Compare Model Versions
+    type: prompt
+    timeout: 120000
+    config:
+      agent: data-scientist
+      task: |
+        Compare the new model version against the current production version.
+
+        ## Comparison Dimensions
+
+        1. **Performance Delta**:
+           - Metric-by-metric comparison
+           - Statistical significance of differences
+           - Confidence intervals
+
+        2. **Schema Compatibility**:
+           - Input schema changes (breaking/non-breaking)
+           - Output schema changes
+           - Feature additions/removals
+
+        3. **Resource Changes**:
+           - Model size difference
+           - Inference latency change
+           - Memory requirement change
+
+        4. **Risk Assessment**:
+           - Breaking changes identified
+           - Rollback complexity
+           - Required client updates
+
+        ## Recommendation
+        - **PROMOTE**: Ready for production rollout
+        - **CANARY**: Recommend gradual rollout with monitoring
+        - **HOLD**: Needs more evaluation
+        - **REJECT**: Does not meet requirements
+
+  - stepId: handle-promote
+    name: Promote to Production
+    type: conditional
+    config:
+      condition: "{{recommendation}} == 'PROMOTE'"
+      then:
+        - stepId: update-production-status
+          type: tool
+          tool: memory_store
+          config:
+            namespace: ml-model-registry
+            key: "{{model_name}}/versions/{{version}}"
+            merge: true
+            value:
+              status: "production"
+              promoted_at: "{{timestamp}}"
+              promoted_by: "{{user}}"
+
+  - stepId: handle-canary
+    name: Set Canary Status
+    type: conditional
+    config:
+      condition: "{{recommendation}} == 'CANARY'"
+      then:
+        - stepId: update-canary-status
+          type: tool
+          tool: memory_store
+          config:
+            namespace: ml-model-registry
+            key: "{{model_name}}/versions/{{version}}"
+            merge: true
+            value:
+              status: "canary"
+              canary_started_at: "{{timestamp}}"
+              canary_traffic_percent: 10
+
+  - stepId: notify-promotion
+    name: Generate Promotion Notification
+    type: prompt
+    timeout: 60000
+    config:
+      agent: writer
+      task: |
+        Create a model promotion notification for stakeholders.
+
+        ## Notification Content
+
+        **Subject**: Model Promotion: {{model_name}} v{{version}}
+
+        **Summary**:
+        - Previous production version: {{old_version}}
+        - New production version: {{version}}
+        - Promotion type: {{promotion_type}} (immediate/canary)
+
+        **Key Changes**:
+        - Performance improvement: {{performance_delta}}
+        - Notable changes: {{changes}}
+
+        **Rollback Plan**:
+        - Rollback command/procedure
+        - Rollback criteria
+        - On-call contact
+
+        **Timeline**:
+        - Promotion start: {{timestamp}}
+        - Full rollout (if canary): {{full_rollout_date}}
+        - Monitoring period: {{monitoring_period}}
+
+  - stepId: store-promotion-event
+    name: Log Promotion Event
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-model-events
+      key: "{{model_name}}/promotions/{{timestamp}}"
+      ttl: 31536000
+      value:
+        event_type: "promotion"
+        model_name: "{{model_name}}"
+        from_version: "{{old_production_version}}"
+        to_version: "{{version}}"
+        promotion_type: "{{promotion_type}}"
+        initiated_by: "{{user}}"
+        timestamp: "{{timestamp}}"
+        notification_sent: true