@defai.digital/cli 13.4.4 → 13.4.7

package/bundled/workflows/ml-ab-testing.yaml

@@ -0,0 +1,311 @@
+workflowId: ml-ab-testing
+name: Model A/B Testing
+description: Statistical comparison of model variants in production
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - ab-testing
+  - experimentation
+  - statistics
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+    - statistical-analysis
+    - data-analysis
+  estimatedDuration: 300
+  complexity: high
+
+steps:
+  - stepId: experiment-design
+    name: Design A/B Experiment
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Design a statistically rigorous A/B test for model comparison.
+
+        ## Hypothesis Definition
+
+        1. **Null Hypothesis (H0)**:
+           - State: "There is no difference between model A and model B"
+
+        2. **Alternative Hypothesis (H1)**:
+           - State: "Model B performs better than model A"
+           - Test type: one-tailed or two-tailed
+
+        ## Sample Size Calculation
+
+        Given:
+        - Baseline conversion rate / metric: {{baseline_rate}}
+        - Minimum detectable effect (MDE): {{mde}}
+        - Significance level (alpha): 0.05
+        - Statistical power (1-beta): 0.80
+
+        Calculate:
+        - Required sample size per variant
+        - Estimated test duration based on traffic
+
+        ## Metric Definition
+
+        1. **Primary Metric**:
+           - Metric name
+           - Definition
+           - Success direction (higher/lower is better)
+
+        2. **Secondary Metrics**:
+           - List of supporting metrics
+
+        3. **Guardrail Metrics**:
+           - Metrics that must not degrade
+           - Degradation threshold
+
+        ## Randomization Strategy
+        - Randomization unit (user, session, request)
+        - Stratification variables (if any)
+        - Holdout percentage
+
+  - stepId: traffic-allocation
+    name: Define Traffic Allocation
+    type: prompt
+    timeout: 60000
+    config:
+      agent: ml-engineer
+      task: |
+        Plan the traffic allocation strategy.
+
+        ## Traffic Split
+
+        | Variant | Description | Traffic % |
+        |---------|-------------|-----------|
+        | Control | Current production model | {{control_pct}}% |
+        | Treatment | New model candidate | {{treatment_pct}}% |
+        | Holdout | No model (baseline) | {{holdout_pct}}% |
+
+        ## Ramping Strategy
+
+        - **Day 1-2**: 1% treatment (burn-in period)
+        - **Day 3-7**: 10% treatment (early signal)
+        - **Day 8-14**: 50% treatment (full test)
+
+        ## Safeguards
+
+        1. **Auto-Rollback Triggers**:
+           - Error rate > {{error_threshold}}%
+           - Latency P99 > {{latency_threshold}}ms
+           - Guardrail metric degradation > {{guardrail_threshold}}%
+
+        2. **Manual Review Triggers**:
+           - Unusual traffic patterns
+           - Unexpected metric movements
+
+  - stepId: store-experiment-config
+    name: Store Experiment Configuration
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-ab-experiments
+      key: "{{experiment_id}}/config"
+      value:
+        experiment_id: "{{experiment_id}}"
+        experiment_name: "{{experiment_name}}"
+        status: "configured"
+        hypothesis:
+          null_hypothesis: "{{h0}}"
+          alternative_hypothesis: "{{h1}}"
+        variants:
+          control:
+            model_name: "{{control_model}}"
+            model_version: "{{control_version}}"
+            traffic_percent: "{{control_pct}}"
+          treatment:
+            model_name: "{{treatment_model}}"
+            model_version: "{{treatment_version}}"
+            traffic_percent: "{{treatment_pct}}"
+        metrics:
+          primary: "{{primary_metric}}"
+          secondary: "{{secondary_metrics}}"
+          guardrails: "{{guardrail_metrics}}"
+        statistical_params:
+          significance_level: 0.05
+          power: 0.80
+          mde: "{{mde}}"
+          required_sample_size: "{{sample_size}}"
+        schedule:
+          start_date: "{{start_date}}"
+          estimated_end_date: "{{end_date}}"
+          ramping_schedule: "{{ramp_schedule}}"
+        created_at: "{{timestamp}}"
+        created_by: "{{user}}"
+
+  - stepId: monitor-experiment
+    name: Monitor Running Experiment
+    type: prompt
+    timeout: 120000
+    config:
+      agent: data-scientist
+      task: |
+        Monitor the running A/B experiment for issues.
+
+        ## Health Checks
+
+        1. **Sample Ratio Mismatch (SRM)**:
+           - Expected ratio: {{expected_ratio}}
+           - Observed ratio: {{observed_ratio}}
+           - Chi-squared test for SRM
+           - ALERT if p-value < 0.001
+
+        2. **Traffic Distribution**:
+           - Verify randomization is working
+           - Check for selection bias
+
+        3. **Guardrail Metrics**:
+           - Current values vs baseline
+           - Flag any degradation
+
+        4. **Data Quality**:
+           - Missing data rate
+           - Logging issues
+
+        ## Early Stopping Check
+
+        - **Sequential testing**: Can we stop early?
+        - Current confidence level
+        - Projected final sample size
+
+        ## Issues Found
+        - List any anomalies
+        - Recommended actions
+
+  - stepId: analyze-results
+    name: Analyze Experiment Results
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Perform statistical analysis of A/B test results.
+
+        ## Primary Metric Analysis
+
+        1. **Descriptive Statistics**:
+           | Variant | N | Mean | Std | Median |
+           |---------|---|------|-----|--------|
+           | Control | | | | |
+           | Treatment | | | | |
+
+        2. **Effect Size**:
+           - Absolute difference: {{treatment_mean}} - {{control_mean}}
+           - Relative lift: (T - C) / C * 100%
+           - 95% Confidence Interval: [{{ci_lower}}, {{ci_upper}}]
+
+        3. **Statistical Significance**:
+           - Test used: t-test / Mann-Whitney / Chi-squared
+           - Test statistic: {{test_stat}}
+           - P-value: {{p_value}}
+           - Significant at alpha=0.05: Yes/No
+
+        4. **Practical Significance**:
+           - Is the effect size meaningful?
+           - Does it meet MDE threshold?
+
+        ## Secondary Metrics
+
+        | Metric | Control | Treatment | Lift | P-value | Significant |
+        |--------|---------|-----------|------|---------|-------------|
+
+        ## Guardrail Metrics
+
+        | Metric | Control | Treatment | Change | Status |
+        |--------|---------|-----------|--------|--------|
+
+        ## Segment Analysis
+
+        Analyze effect by key segments:
+        - New vs returning users
+        - Mobile vs desktop
+        - Geographic regions
+        - User cohorts
+
+        ## Novelty/Primacy Effects
+
+        - Plot metric over time
+        - Check for effect decay
+        - Estimate steady-state effect
+
+  - stepId: make-decision
+    name: Make Experiment Decision
+    type: prompt
+    timeout: 120000
+    config:
+      agent: ml-engineer
+      task: |
+        Based on analysis, make a decision about the experiment.
+
+        ## Decision Framework
+
+        | Significance | Practical Impact | Guardrails | Decision |
+        |--------------|------------------|------------|----------|
+        | Yes | Positive | Pass | SHIP |
+        | Yes | Negative | - | ROLLBACK |
+        | No | - | Pass | EXTEND or NO_SHIP |
+        | - | - | Fail | ROLLBACK |
+
+        ## Decision
+
+        **Recommendation**: {{decision}}
+
+        **Confidence**: HIGH / MEDIUM / LOW
+
+        **Reasoning**:
+        - Statistical evidence: {{stat_reasoning}}
+        - Business impact: {{business_reasoning}}
+        - Risk assessment: {{risk_reasoning}}
+
+        ## If SHIP
+        - Rollout plan (gradual vs immediate)
+        - Monitoring requirements
+        - Success criteria for full rollout
+
+        ## If ROLLBACK
+        - Root cause hypothesis
+        - Recommended investigation
+        - Next experiment suggestions
+
+        ## If EXTEND
+        - Additional sample size needed
+        - New end date
+        - Any design changes
+
+  - stepId: store-results
+    name: Store Experiment Results
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-ab-experiments
+      key: "{{experiment_id}}/results"
+      ttl: 31536000
+      value:
+        experiment_id: "{{experiment_id}}"
+        status: "completed"
+        results:
+          primary_metric:
+            control_mean: "{{control_mean}}"
+            treatment_mean: "{{treatment_mean}}"
+            relative_lift: "{{lift}}"
+            confidence_interval: "{{ci}}"
+            p_value: "{{p_value}}"
+            significant: "{{significant}}"
+          secondary_metrics: "{{secondary_results}}"
+          guardrail_metrics: "{{guardrail_results}}"
+          segment_analysis: "{{segment_results}}"
+        decision:
+          recommendation: "{{decision}}"
+          confidence: "{{confidence}}"
+          reasoning: "{{reasoning}}"
+        completed_at: "{{timestamp}}"
+        decided_by: "{{user}}"

package/bundled/workflows/ml-experiment-tracker.yaml

@@ -0,0 +1,150 @@
+workflowId: ml-experiment-tracker
+name: ML Experiment Tracker
+description: Track, compare, and manage ML experiments for reproducibility
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - experiments
+  - tracking
+  - reproducibility
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+    - statistical-analysis
+    - data-analysis
+  estimatedDuration: 300
+  complexity: medium
+
+steps:
+  - stepId: define-experiment
+    name: Define Experiment
+    type: prompt
+    timeout: 60000
+    config:
+      agent: ml-engineer
+      task: |
+        Define the ML experiment with the following structure:
+
+        ## Experiment Definition
+
+        1. **Hypothesis**: What are we testing?
+        2. **Baseline Model**: What is the current best model?
+        3. **Metrics to Track**:
+           - Primary metric (optimization target)
+           - Secondary metrics (guardrails)
+        4. **Success Criteria**: What improvement is significant?
+        5. **Dataset Version**: Which data split are we using?
+
+        Provide structured output for logging.
+
+  - stepId: log-parameters
+    name: Log Experiment Parameters
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-experiments
+      key: "{{experiment_id}}/parameters"
+      value:
+        experiment_id: "{{experiment_id}}"
+        experiment_name: "{{experiment_name}}"
+        hypothesis: "{{hypothesis}}"
+        model_type: "{{model_type}}"
+        hyperparameters: "{{hyperparameters}}"
+        dataset_version: "{{dataset_version}}"
+        baseline_model: "{{baseline_model}}"
+        success_criteria: "{{success_criteria}}"
+        created_at: "{{timestamp}}"
+        created_by: "{{user}}"
+        status: "running"
+
+  - stepId: log-metrics
+    name: Log Training Metrics
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-experiments
+      key: "{{experiment_id}}/metrics"
+      value:
+        experiment_id: "{{experiment_id}}"
+        metrics:
+          accuracy: "{{accuracy}}"
+          precision: "{{precision}}"
+          recall: "{{recall}}"
+          f1_score: "{{f1_score}}"
+          auc_roc: "{{auc_roc}}"
+          loss: "{{loss}}"
+        training_metrics:
+          training_time_seconds: "{{training_time}}"
+          epochs_completed: "{{epochs}}"
+          early_stopping_epoch: "{{early_stop_epoch}}"
+        resource_usage:
+          peak_memory_mb: "{{peak_memory}}"
+          gpu_utilization: "{{gpu_util}}"
+        logged_at: "{{timestamp}}"
+
+  - stepId: compare-experiments
+    name: Compare with Baseline
+    type: prompt
+    timeout: 120000
+    config:
+      agent: data-scientist
+      task: |
+        Compare experiment {{experiment_id}} against the baseline model.
+
+        ## Analysis Required
+
+        1. **Metric Comparison**:
+           - Calculate absolute and relative improvement
+           - For each metric: baseline vs experiment
+
+        2. **Statistical Significance**:
+           - Is the improvement statistically significant?
+           - Calculate p-value if applicable
+           - Report confidence intervals
+
+        3. **Trade-off Analysis**:
+           - Accuracy vs inference latency
+           - Model complexity vs performance
+           - Training cost vs improvement
+
+        4. **Recommendation**:
+           - PROMOTE: Significant improvement, ready for production
+           - ITERATE: Promising but needs refinement
+           - REJECT: No improvement or regression
+
+        Provide structured recommendation with justification.
+
+  - stepId: update-status
+    name: Update Experiment Status
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-experiments
+      key: "{{experiment_id}}/status"
+      value:
+        experiment_id: "{{experiment_id}}"
+        status: "{{recommendation}}"
+        comparison_summary: "{{comparison_summary}}"
+        statistical_significance: "{{p_value}}"
+        recommendation_rationale: "{{rationale}}"
+        completed_at: "{{timestamp}}"
+
+  - stepId: store-comparison-report
+    name: Store Comparison Report
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-experiment-reports
+      key: "{{experiment_id}}/comparison"
+      ttl: 7776000
+      value:
+        experiment_id: "{{experiment_id}}"
+        baseline_id: "{{baseline_model}}"
+        full_report: "{{comparison_report}}"
+        created_at: "{{timestamp}}"

package/bundled/workflows/ml-feature-engineering.yaml

@@ -0,0 +1,242 @@
+workflowId: ml-feature-engineering
+name: Feature Engineering Pipeline
+description: Systematic feature creation, validation, and selection
+version: "1.0.0"
+category: machine-learning
+tags:
+  - ml
+  - features
+  - feature-engineering
+  - feature-selection
+
+metadata:
+  requiredAbilities:
+    - machine-learning
+    - data-analysis
+    - feature-engineering
+  estimatedDuration: 600
+  complexity: high
+
+steps:
+  - stepId: feature-ideation
+    name: Feature Ideation
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Given the prediction target and available data, brainstorm candidate features.
+
+        ## Feature Categories to Consider
+
+        1. **Raw Features**:
+           - Direct columns from source data
+           - Basic transformations (log, sqrt, power)
+
+        2. **Domain Knowledge Features**:
+           - Business logic features
+           - Industry-specific indicators
+           - Expert-derived calculations
+
+        3. **Interaction Features**:
+           - Feature products (A * B)
+           - Feature ratios (A / B)
+           - Feature differences (A - B)
+
+        4. **Time-Based Features** (if temporal data):
+           - Lag features
+           - Rolling statistics (mean, std, min, max)
+           - Time since event
+           - Seasonal indicators
+           - Trend features
+
+        5. **Aggregation Features**:
+           - Group-by statistics
+           - Entity-level aggregations
+           - Window aggregations
+
+        6. **Text Features** (if text data):
+           - TF-IDF
+           - Word embeddings
+           - Sentiment scores
+           - Named entities
+
+        7. **Categorical Encodings**:
+           - One-hot encoding
+           - Target encoding
+           - Frequency encoding
+           - Embedding encoding
+
+        ## Output Format
+        For each candidate feature:
+        - Feature name
+        - Description
+        - Calculation logic
+        - Expected predictive value (hypothesis)
+        - Implementation complexity (low/medium/high)
+
+  - stepId: feature-implementation
+    name: Implement Features
+    type: prompt
+    timeout: 300000
+    config:
+      agent: data-scientist
+      task: |
+        Implement the candidate features with production-quality code.
+
+        ## For Each Feature
+
+        1. **Transformation Code**:
+           ```python
+           def compute_feature_name(df: pd.DataFrame) -> pd.Series:
+               '''
+               Description of what this feature represents.
+
+               Args:
+                   df: Input dataframe with required columns
+
+               Returns:
+                   pd.Series: Computed feature values
+               '''
+               # Implementation
+               pass
+           ```
+
+        2. **Missing Value Handling**:
+           - Strategy: drop / fill_mean / fill_median / fill_mode / fill_constant
+           - Justification for chosen strategy
+
+        3. **Edge Case Handling**:
+           - Division by zero
+           - Negative values for log transforms
+           - Null propagation
+
+        4. **Data Type**:
+           - Output dtype
+           - Value range
+
+        ## Feature Pipeline
+        Create a combined feature pipeline function that:
+        - Takes raw data
+        - Applies all transformations
+        - Returns feature matrix
+
+  - stepId: feature-validation
+    name: Validate Features
+    type: prompt
+    timeout: 180000
+    config:
+      agent: data-scientist
+      task: |
+        Validate feature quality and check for common issues.
+
+        ## Data Leakage Check
+
+        1. **Target Leakage**:
+           - Does any feature contain information from the target?
+           - Is any feature computed using future data?
+           - Check correlation with target on train vs test
+
+        2. **Train-Test Leakage**:
+           - Are features computed using test data statistics?
+           - Is any global normalization applied before split?
+
+        ## Feature Quality Checks
+
+        1. **Missing Values**:
+           - Missing rate per feature
+           - Flag features with >50% missing
+
+        2. **Cardinality**:
+           - Unique value count
+           - Flag high-cardinality categoricals
+
+        3. **Distribution**:
+           - Check for extreme skewness
+           - Identify outliers
+           - Check for constant features
+
+        4. **Correlation Analysis**:
+           - Correlation with target
+           - Inter-feature correlation matrix
+           - Flag highly correlated pairs (>0.95)
+
+        ## Validation Report
+        For each feature:
+        - Leakage risk: SAFE / WARNING / DANGEROUS
+        - Quality score: 1-10
+        - Issues found
+        - Recommendations
+
+  - stepId: feature-selection
+    name: Select Final Features
+    type: prompt
+    timeout: 180000
+    config:
+      agent: ml-engineer
+      task: |
+        Select the final feature set for modeling.
+
+        ## Selection Methods
+
+        1. **Importance-Based Selection**:
+           - SHAP values from baseline model
+           - Permutation importance
+           - Random forest feature importance
+
+        2. **Statistical Selection**:
+           - Mutual information with target
+           - Chi-squared test (categorical)
+           - ANOVA F-value (continuous)
+
+        3. **Iterative Selection**:
+           - Forward selection results
+           - Backward elimination results
+           - Recursive feature elimination
+
+        ## Selection Criteria
+
+        - Remove features with:
+          - Near-zero variance
+          - High missing rate (>50%)
+          - High correlation with other features (>0.95)
+          - Low importance (bottom 10%)
+          - Data leakage risk
+
+        - Keep features with:
+          - High target correlation
+          - Domain importance
+          - Unique information
+
+        ## Final Feature Set
+
+        | Feature | Importance Rank | Rationale |
+        |---------|-----------------|-----------|
+        | feature_1 | 1 | ... |
+        | feature_2 | 2 | ... |
+
+        Total features selected: X out of Y candidates
+
+  - stepId: store-feature-definitions
+    name: Store Feature Definitions
+    type: tool
+    timeout: 10000
+    tool: memory_store
+    config:
+      namespace: ml-feature-store
+      key: "{{feature_set_name}}/{{version}}"
+      value:
+        feature_set_name: "{{feature_set_name}}"
+        version: "{{version}}"
+        description: "{{description}}"
+        target_variable: "{{target}}"
+        features: "{{feature_definitions}}"
+        validation_results:
+          leakage_check: "{{leakage_status}}"
+          quality_scores: "{{quality_scores}}"
+        selection_summary:
+          total_candidates: "{{total_candidates}}"
+          selected_count: "{{selected_count}}"
+          selection_method: "{{selection_method}}"
+        created_at: "{{timestamp}}"
+        created_by: "{{user}}"