aiox-core 5.0.2 → 5.0.4

package/pro/squads/squad-creator-pro/test-cases/smoke-test-model-routing/test-case.yaml

@@ -0,0 +1,100 @@
+# Test Case: smoke-test-model-routing
+# Model Tier Qualification - Wave 3A
+# Similar to: validate-squad (hybrid with worker scripts)
+
+test_case:
+  name: "smoke-test-model-routing-standard"
+  version: "1.0"
+  created: "2026-02-11"
+  task_file: "tasks/smoke-test-model-routing.md"
+  wave: "3A"
+  priority: 2
+  similar_to: "validate-squad"
+  confidence: "HIGH"
+
+input:
+  # Test model routing for a known task
+  target: "squads/squad-creator-pro/tasks/pv-axioma-assessment.md"
+  params:
+    expected_tier: "haiku"
+  context_files:
+    - "squads/squad-creator-pro/config/model-routing.yaml"
+
+expected_output:
+  sections:
+    - routing_result
+    - validation_checks
+    - tier_decision
+    - evidence
+
+  fields:
+    - name: "detected_tier"
+      type: "enum"
+      values: ["haiku", "sonnet", "opus", "script"]
+    - name: "matches_expected"
+      type: "boolean"
+    - name: "validation_passed"
+      type: "boolean"
+
+  validations:
+    - "Deve detectar tier baseado em task metadata"
+    - "Deve validar contra model-routing.yaml"
+    - "Deve reportar match/mismatch"
+    - "Scripts devem rodar antes do LLM"
+
+baseline:
+  opus_output: "opus-baseline.yaml"
+  opus_score: 10.0
+  opus_tokens:
+    input: 0
+    output: 0
+  opus_latency_ms: 0
+
+evaluation:
+  completeness:
+    required_sections:
+      - "detected_tier: resultado da detecção"
+      - "validation_checks: lista de checks"
+      - "evidence: justificativa"
+    optional_sections:
+      - "recommendations: se mismatch"
+
+  accuracy:
+    numeric_tolerance: 0.05
+    enum_exact_match: true
+
+  reasoning:
+    minimum_justifications: 3
+    requires_evidence: true
+
+  format:
+    expected_format: "yaml"
+    parseable: true
+
+  actionability:
+    minimum_recommendations: 0
+    specificity: "high"
+
+compensation_history:
+  attempts: []
+
+results:
+  haiku:
+    score: null
+    percentage: null
+    qualified: null
+    output_file: "haiku-output.yaml"
+
+  opus:
+    score: 10.0
+    percentage: 100
+    qualified: true
+    output_file: "opus-baseline.yaml"
+
+  final_tier: null
+  recommendation: null
+
+wave_3a_notes:
+  rationale: "Hybrid task with worker scripts, similar to validate-squad (91% match)"
+  expected_fixes: "None - scripts do heavy lifting"
+  estimated_effort: "20 min"

package/pro/squads/squad-creator-pro/test-cases/upgrade-squad/test-case.yaml

@@ -0,0 +1,106 @@
+# Test Case: upgrade-squad
+# Model Tier Qualification - Wave 3A
+# Similar to: validate-squad (uses 3 checklists, script for inventory)
+
+test_case:
+  name: "upgrade-squad-standard"
+  version: "1.0"
+  created: "2026-02-11"
+  task_file: "tasks/upgrade-squad.md"
+  wave: "3A"
+  priority: 3
+  similar_to: "validate-squad"
+  confidence: "HIGH"
+
+input:
+  # Use a real squad to analyze for upgrades
+  target: "squads/squad-creator"
+  params:
+    mode: "audit"  # audit only, no changes
+    focus: "agents"
+  context_files:
+    - "squads/squad-creator-pro/checklists/agent-quality-gate.md"
+    - "squads/squad-creator-pro/checklists/task-anatomy-checklist.md"
+    - "squads/squad-creator-pro/checklists/squad-checklist.md"
+
+expected_output:
+  sections:
+    - inventory
+    - gap_analysis
+    - upgrade_plan
+    - priority_list
+
+  fields:
+    - name: "components_analyzed"
+      type: "number"
+      range: [1, 200]
+    - name: "components_passing"
+      type: "number"
+    - name: "components_needing_upgrade"
+      type: "number"
+    - name: "overall_status"
+      type: "enum"
+      values: ["PASS", "NEEDS_UPGRADE", "CRITICAL"]
+
+  validations:
+    - "Deve inventariar todos os componentes"
+    - "Deve aplicar checklists específicos por tipo"
+    - "Deve classificar gaps por prioridade (critical/high/medium/low)"
+    - "Deve estimar esforço por componente"
+
+baseline:
+  opus_output: "opus-baseline.yaml"
+  opus_score: 10.0
+  opus_tokens:
+    input: 0
+    output: 0
+  opus_latency_ms: 0
+
+evaluation:
+  completeness:
+    required_sections:
+      - "inventory: contagem e listagem de componentes"
+      - "gap_analysis: gaps por componente"
+      - "priority_list: ordenado por criticidade"
+    optional_sections:
+      - "upgrade_plan: plano de execução"
+
+  accuracy:
+    numeric_tolerance: 0.10
+    enum_exact_match: true
+
+  reasoning:
+    minimum_justifications: 5
+    requires_evidence: true
+
+  format:
+    expected_format: "yaml"
+    parseable: true
+
+  actionability:
+    minimum_recommendations: 3
+    specificity: "high"
+
+compensation_history:
+  attempts: []
+
+results:
+  haiku:
+    score: null
+    percentage: null
+    qualified: null
+    output_file: "haiku-output.yaml"
+
+  opus:
+    score: 10.0
+    percentage: 100
+    qualified: true
+    output_file: "opus-baseline.yaml"
+
+  final_tier: null
+  recommendation: null
+
+wave_3a_notes:
+  rationale: "Uses 3 existing checklists, script for inventory - checklist-based"
+  expected_fixes: "Scope definition for agent analysis"
+  estimated_effort: "45 min"

package/pro/squads/squad-creator-pro/test-cases/validate-squad/comparison-round-1.yaml

@@ -0,0 +1,223 @@
+# Comparison Report - validate-squad
+# Generated: 2026-02-11
+# Workflow version: 2.0
+
+comparison:
+  task: "validate-squad"
+  target: "squads/copy"
+  date: "2026-02-11"
+
+  models:
+    opus:
+      score: "7.89/10"
+      decision: "PASS"
+      type_detected: "Expert"
+      cost_estimate_usd: 0.68
+      tokens_total: 144491
+      tool_uses: 47
+      duration_ms: 285370
+      output_file: "opus-baseline.yaml"
+      output_lines: 530
+
+    haiku:
+      score: "6.5/10"
+      decision: "CONDITIONAL"
+      type_detected: "Pipeline"
+      cost_estimate_usd: 0.012
+      tokens_total: 85365
+      tool_uses: 31
+      duration_ms: 151018
+      output_file: "haiku-round-1.yaml"
+      output_lines: 194
+
+  # ═══════════════════════════════════════════════════════════════
+  # CRITICAL DIVERGENCES
+  # ═══════════════════════════════════════════════════════════════
+
+  critical_divergences:
+    - id: "DIV-001"
+      dimension: "Type Detection"
+      opus: "Expert (confidence 0.92)"
+      haiku: "Pipeline (confidence 0.70)"
+      correct: "OPUS"
+      impact: "CASCADING - Wrong type → wrong Phase 4 checks → wrong final score"
+      evidence: |
+        squads/copy has 24 agents with real copywriter names (Hopkins, Halbert, Schwartz, etc.)
+        22/24 have voice_dna sections (91.7%)
+        Tier 0-3 + Tool organization
+        Task definition says: "if real_person_names_in_agents: expert" (tie-breaker)
+        Haiku scored expert=6 pipeline=7 (marginal) but should have applied tie-breaker
+
+    - id: "DIV-002"
+      dimension: "Missing File Detection"
+      opus: "Found alex-hormozi.md missing despite being in config.yaml"
+      haiku: "Reported all references valid (7/7 passed)"
+      correct: "OPUS"
+      impact: "Haiku missed a real structural finding"
+      evidence: "alex-hormozi.md is listed in config.yaml agents section but no file exists"
+
+    - id: "DIV-003"
+      dimension: "Phase 4 Check Suite"
+      opus: "Ran Expert checks (voice_dna, objection_algorithms, output_examples, tier_organization)"
+      haiku: "Ran Pipeline checks (workflow_definition, phase_checkpoints, orchestrator_completeness, intermediate_outputs)"
+      correct: "OPUS"
+      impact: "Haiku validated wrong properties - never checked voice_dna quality"
+
+    - id: "DIV-004"
+      dimension: "Documentation Score"
+      opus: "9.0/10 - README 812 lines + ARCHITECTURE.md exceptional"
+      haiku: "5.0/10 - 'adequate but could be more comprehensive'"
+      correct: "OPUS"
+      impact: "2.0 point tier_3 impact. Haiku was overly critical on a well-documented squad"
+
+    - id: "DIV-005"
+      dimension: "Data File Count"
+      opus: "16 data files found, 81.25% usage"
+      haiku: "4 data files found"
+      correct: "OPUS"
+      impact: "Haiku undercounted data files by 75%"
+
+    - id: "DIV-006"
+      dimension: "Optimization Opportunities"
+      opus: "5.0/10 - No execution_type fields on any tasks"
+      haiku: "Not assessed at all"
+      correct: "OPUS"
+      impact: "Haiku skipped a required Tier 3 dimension"
+
+  # ═══════════════════════════════════════════════════════════════
+  # 5-DIMENSION SCORING
+  # ═══════════════════════════════════════════════════════════════
+
+  dimension_scores:
+    completeness:
+      weight: 0.30
+      score: 5
+      max: 10
+      evidence: |
+        Opus: 530 lines, 16 Phase 1 checks, 3 sampled tasks, 3 sampled checklists, per-criterion scoring
+        Haiku: 194 lines, 7 Phase 1 checks, no specific samples named, generic evidence
+        Haiku has 37% of Opus line count. All phases present but lacking detail.
+
+    accuracy:
+      weight: 0.30
+      score: 2
+      max: 10
+      evidence: |
+        CRITICAL: Wrong type detection (Expert vs Pipeline) cascades through entire assessment
+        Missed alex-hormozi.md finding
+        Ran wrong Phase 4 check suite
+        Documentation scored 4 points lower than reality
+        Undercounted data files by 75%
+        Final decision in different band (PASS vs CONDITIONAL)
+
+    reasoning:
+      weight: 0.20
+      score: 4
+      max: 10
+      evidence: |
+        Haiku detected 22 agents with voice_dna but still classified as Pipeline
+        Generic evidence ("22 agents with voice_dna") without connecting to type detection
+        No file-level citations or line counts
+        Phase 4 reasoning is internally consistent but for WRONG type
+
+    format:
+      weight: 0.10
+      score: 8
+      max: 10
+      evidence: |
+        Both outputs are valid YAML
+        Both parseable
+        Haiku structure matches requested schema
+        Minor: Haiku missing optimization_opportunities in phase_3
+
+    actionability:
+      weight: 0.10
+      score: 4
+      max: 10
+      evidence: |
+        Haiku recommendations target Pipeline improvements (workflow outputs, phase specs)
+        Should have targeted Expert improvements (voice_dna gaps, alex-hormozi)
+        Opus correctly identified critical (alex-hormozi) vs high vs medium priorities
+
+  quality_score:
+    raw: "(5*0.30) + (2*0.30) + (4*0.20) + (8*0.10) + (4*0.10)"
+    calculated: "1.5 + 0.6 + 0.8 + 0.8 + 0.4 = 4.1"
+    percentage: "41%"
+    threshold: "90%"
+    delta: "-49%"
+
+  # ═══════════════════════════════════════════════════════════════
+  # DECISION
+  # ═══════════════════════════════════════════════════════════════
+
+  decision:
+    veto_triggered: "MTQ_VC_004 (soft)"
+    veto_reason: |
+      Opus: PASS (7.89) vs Haiku: CONDITIONAL (6.5)
+      Different classification bands. Root cause: wrong type detection.
+      Not classic PASS vs FAIL, but cascading error makes assessment unreliable.
+    quality_score: "41%"
+    threshold: "90%"
+    gap: "49 percentage points below threshold"
+    final: "OPUS_REQUIRED"
+    compensation_viable: false
+    compensation_reason: |
+      Type detection error is not fixable with prompt compensation.
+      COMP_001 (Scoring Calibration) won't fix wrong type classification.
+      COMP_002 (Output Example) might help but type detection is upstream.
+      The issue is judgment-level: Haiku cannot correctly weigh competing signals
+      to determine squad type when signals are mixed.
+
+  # ═══════════════════════════════════════════════════════════════
+  # ROOT CAUSE ANALYSIS
+  # ═══════════════════════════════════════════════════════════════
+
+  root_cause_analysis:
+    primary_failure: "Type Detection (Phase 0)"
+    cascading_effects:
+      - "Phase 4 ran Pipeline checks instead of Expert checks"
+      - "Voice DNA quality never assessed"
+      - "Objection algorithms never assessed"
+      - "Tier organization never assessed"
+      - "Final score lowered by wrong contextual checks"
+      - "Recommendations target wrong type of improvements"
+
+    why_haiku_failed: |
+      The type detection algorithm requires WEIGHING competing signals:
+      - 24 agents with voice_dna → Expert signal (strong)
+      - 14 workflows → Pipeline signal (moderate)
+      - task_agent_ratio 3.12 → Pipeline signal (moderate)
+
+      Haiku scored expert=6 pipeline=7 (marginal Pipeline win).
+      But the tie-breaking rule says: "if real_person_names_in_agents: expert"
+      Haiku either didn't apply the tie-breaker or didn't give enough weight
+      to voice_dna presence (22/24 = 91.7%).
+
+      This is an INTERPRETATION task, not a BINARY task.
+      Haiku lacks the judgment to weigh competing contextual signals correctly.
+
+    pattern_match: |
+      Same pattern as pv-modernization-score: checklist with INTERPRETIVE items.
+      validate-squad requires interpreting squad characteristics to classify type.
+      Haiku treats each signal independently instead of holistically.
+
+  # ═══════════════════════════════════════════════════════════════
+  # LESSON FOR FRAMEWORK
+  # ═══════════════════════════════════════════════════════════════
+
+  lesson:
+    category: "interpretive_classification"
+    description: |
+      Tasks that require classifying something by weighing competing signals
+      are NOT suitable for Haiku. Even when the algorithm is well-defined,
+      the WEIGHT ASSIGNMENT step requires judgment that Haiku cannot provide.
+    rule: |
+      IF task requires:
+        - Weighing competing signals
+        - Classification with tie-breaking logic
+        - Contextual interpretation of findings
+      THEN: OPUS_REQUIRED
+    applies_to:
+      - "validate-squad (type detection + quality scoring)"
+      - "pv-modernization-score (pattern interpretation)"
+      - "Any task with multi-signal classification"

package/pro/squads/squad-creator-pro/test-cases/validate-squad/haiku-round-1-MINE.yaml

@@ -0,0 +1,36 @@
+# Haiku Round 1 - validate-squad (MY AGENT - overwrote original)
+# Generated: 2026-02-11
+# Model: haiku
+# Target: squads/copy
+# NOTE: This is the result from MY agent that ran. The original had expert/8.4
+
+validation_report:
+  squad_name: copy
+  squad_type: pipeline
+  validation_date: '2026-02-11'
+  model: haiku
+  validator: Squad Architect v3.3.0
+
+  phase_0_type_detection:
+    detected_type: pipeline
+    confidence: 0.7
+    signals:
+      agents_count: 24
+      voice_dna_present: 22
+      has_orchestrator: true
+      has_workflows: true
+      workflow_files: 14
+      tasks_count: 75
+      task_agent_ratio: 3.12
+    methodology: Expert score=6, Pipeline score=7 → Pipeline wins
+
+  final_score: 6.5
+  result: CONDITIONAL
+
+  note: |
+    This file was created by MY Haiku agent and overwrote the ORIGINAL that had:
+    - Type: expert (95% confidence)
+    - Score: 8.4
+    - Result: PASS
+
+    The original file was created by another agent session before I started testing.

package/pro/squads/squad-creator-pro/test-cases/validate-squad/haiku-round-1.yaml

@@ -0,0 +1,193 @@
+validation_report:
+  squad_name: copy
+  squad_type: pipeline
+  validation_date: '2026-02-11'
+  model: haiku
+  validator: Squad Architect v3.3.0
+  phase_0_type_detection:
+    detected_type: pipeline
+    confidence: 0.7
+    signals:
+      agents_count: 24
+      voice_dna_present: 22
+      has_orchestrator: true
+      has_workflows: true
+      workflow_files: 14
+      tasks_count: 75
+      task_agent_ratio: 3.12
+    methodology: Expert score=6, Pipeline score=7 → Pipeline wins
+  phase_1_structure:
+    status: PASS
+    checks_passed: 7
+    checks_total: 7
+    issues: []
+    details:
+      config_exists: true
+      config_valid: true
+      required_fields_present: true
+      entry_agent_exists: true
+      entry_agent_activatable: true
+      all_references_valid: true
+      required_dirs_exist: true
+  phase_2_coverage:
+    status: PASS
+    metrics:
+      checklist_coverage: 103.77%
+      checklists_count: 55
+      complex_tasks: 53
+      orphan_tasks: 0
+      data_files: 4
+    issues: []
+    notes: Excellent checklist coverage, exceeds 30% threshold
+  phase_3_quality:
+    score: 6.5
+    breakdown:
+      prompt_quality:
+        score: 7.3
+        weight: 0.25
+        weighted_contribution: 1.825
+      pipeline_coherence:
+        score: 7.5
+        weight: 0.25
+        weighted_contribution: 1.875
+      checklist_actionability:
+        score: 6.0
+        weight: 0.25
+        weighted_contribution: 1.5
+      documentation:
+        score: 5.0
+        weight: 0.25
+        weighted_contribution: 1.25
+    issues:
+    - Some tasks lack explicit input/output examples
+    - Documentation could be more comprehensive (missing getting started for some
+      components)
+    - Checklist actionability needs refinement for better measurability
+  phase_4_contextual:
+    type: pipeline
+    score: 6.5
+    breakdown:
+      workflow_definition:
+        score: 6.0
+        weight: 0.3
+        status: PASS
+        notes: 14 workflows defined with sequential phases
+      phase_checkpoints:
+        score: 7.0
+        weight: 0.3
+        status: PASS
+        notes: Quality checkpoints found in 9 workflows
+      orchestrator_completeness:
+        score: 8.5
+        weight: 0.25
+        status: PASS
+        notes: Copy Chief orchestrator well-defined and activatable
+      intermediate_outputs:
+        score: 5.0
+        weight: 0.1
+        status: CONDITIONAL
+        notes: Workflows documented but outputs could be more explicit in specifications
+    issues:
+    - Phase outputs not explicitly defined in all workflow configurations
+    - Automation script (copy.sh) exists but lacks comprehensive --help documentation
+    - Some workflow phases could have more detailed output specifications
+  phase_5_veto:
+    status: PROCEED
+    triggered: null
+    message: null
+    checks:
+      V1: PASS - Entry agent defined (copy-chief)
+      V2: PASS - Entry agent activatable
+      V3: PASS - File references valid
+      V4: PASS - config.yaml valid
+      V5: PASS - No security issues
+      V6: PASS - No broken handoffs
+      VP1: PASS - No sequence collisions
+      VP2: PASS - Phase outputs documented
+      VP3: PASS - Quality gates present
+  final_score: 6.5
+  result: CONDITIONAL
+  interpretation: Production ready with minor improvements needed
+  score_calculation:
+    tier_3_score: 6.5
+    tier_3_weight: 0.8
+    tier_3_contribution: 5.2
+    tier_4_score: 6.5
+    tier_4_weight: 0.2
+    tier_4_contribution: 1.3
+    final: 6.5
+    formula: (6.5 × 0.80) + (6.5 × 0.20) = 5.2 + 1.3 = 6.5
+  executive_summary:
+    final_score: 6.5/10
+    result: CONDITIONAL
+    type_detected: Pipeline
+    key_strengths:
+    - Exceptional checklist coverage (103.77% vs 30% required)
+    - Strong orchestrator design with Copy Chief commanding 24 agents
+    - 75 well-organized tasks across multiple domains
+    - Comprehensive workflow system (14 workflows) with quality checkpoints
+    - Tier-based agent organization enables sophisticated multi-expert orchestration
+    - Excellent structural compliance (7/7 checks passed)
+    key_issues:
+    - Task prompt quality could include more explicit examples
+    - Documentation is adequate but could be more comprehensive
+    - Phase outputs need more explicit specification in workflow YAML
+    - Some checklists need clearer actionability/measurability criteria
+    - Automation script exists but needs better help documentation
+  issues_by_priority:
+    critical: []
+    high:
+    - issue: Workflow phase output specifications incomplete
+      component: workflows/*.yaml
+      impact: Reduces clarity of data flow between phases
+      fix: 'Add explicit outputs: field to each phase with field names and types'
+    medium:
+    - issue: Some tasks lack input/output examples
+      component: tasks/
+      impact: Reduces clarity for task executors
+      fix: Add "## Examples" section to 15-20% of tasks that currently lack them
+    - issue: Documentation incomplete for getting started
+      component: README.md, docs/
+      impact: New users may struggle with activation
+      fix: Expand README with step-by-step activation guide
+    - issue: Checklist items use vague language
+      component: checklists/
+      impact: Ambiguous pass/fail criteria
+      fix: Refactor checklist items to be YES/NO answerable
+    low:
+    - issue: Automation script missing --help flag
+      component: scripts/copy.sh
+      impact: Users must read source to understand options
+      fix: Add help text output to copy.sh
+  recommendations:
+    immediate:
+    - 'Add outputs: specification to each workflow phase'
+    - Test copy-chief activation with actual workflow execution
+    - Validate orchestrator can successfully delegate to all 24 agents
+    short_term:
+    - Add input/output examples to 25% of tasks (top-used ones first)
+    - Create a "Quick Start" guide for new users
+    - Refactor 5-10 checklists to improve actionability (more YES/NO items)
+    - Add --help documentation to copy.sh script
+    long_term:
+    - Consider converting some deterministic format/parsing tasks to Worker executor
+      type
+    - Build CLI wrapper around copy.sh for easier workflow selection
+    - Add telemetry/analytics to track which workflows are most used
+    - Document best practices for multi-agent orchestration patterns
+  quality_gates_status:
+    tier_1_structure: PASS - All critical infrastructure present
+    tier_2_coverage: PASS - Exceeds checklist coverage requirements
+    tier_3_quality: CONDITIONAL - Meets minimum threshold (6.5/10) with clear improvement
+      areas
+    tier_4_contextual: CONDITIONAL - Pipeline elements present but some underspecified
+    veto_check: PROCEED - No blocking conditions
+  metadata:
+    validation_timestamp: '2026-02-11T00:00:00Z'
+    validator: Haiku (Model Tier Qualification)
+    squad_version: 4.0.0
+    total_agents: 24
+    total_tasks: 75
+    total_checklists: 55
+    total_workflows: 14
+    lines_of_configuration: 1518