agentic-qe 3.7.8 → 3.7.10

package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml

@@ -1,459 +1,459 @@
-# =============================================================================
-# AQE Skill Evaluation Test Suite: QE Code Intelligence v1.0.0
-# =============================================================================
-#
-# Comprehensive evaluation suite for the qe-code-intelligence skill.
-# Tests knowledge graph construction, semantic code search, dependency mapping,
-# and intelligent context retrieval with 80% token reduction.
-#
-# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
-# Validator: .claude/skills/qe-code-intelligence/scripts/validate-config.json
-#
-# Coverage:
-# - Codebase indexing and knowledge graph construction
-# - Semantic search with relevance ranking
-# - Dependency analysis and mapping
-# - Intelligent context retrieval with token optimization
-# - Entity relationship extraction
-#
-# =============================================================================
-
-skill: qe-code-intelligence
-version: 1.0.0
-description: >
-  Comprehensive evaluation suite for the qe-code-intelligence skill.
-  Tests knowledge graph construction, semantic code search, dependency mapping,
-  context-aware code understanding, and intelligent token budget optimization.
-
-# =============================================================================
-# Multi-Model Configuration
-# =============================================================================
-
-models_to_test:
-  - claude-3.5-sonnet
-  - claude-3-haiku
-
-# =============================================================================
-# MCP Integration Configuration
-# =============================================================================
-
-mcp_integration:
-  enabled: true
-  namespace: skill-validation
-
-  query_patterns: true
-  track_outcomes: true
-  store_patterns: true
-  share_learning: true
-  update_quality_gate: true
-
-  target_agents:
-    - qe-learning-coordinator
-    - qe-queen-coordinator
-    - qe-knowledge-graph
-    - qe-semantic-searcher
-
-# =============================================================================
-# ReasoningBank Learning Configuration
-# =============================================================================
-
-learning:
-  store_success_patterns: true
-  store_failure_patterns: true
-  pattern_ttl_days: 90
-  min_confidence_to_store: 0.7
-  cross_model_comparison: true
-
-# =============================================================================
-# Result Format Configuration
-# =============================================================================
-
-result_format:
-  json_output: true
-  markdown_report: true
-  include_raw_output: false
-  include_timing: true
-  include_token_usage: true
-
-# =============================================================================
-# Environment Setup
-# =============================================================================
-
-setup:
-  required_tools:
-    - jq
-  environment_variables:
-    KG_CACHE_ENABLED: "true"
-    SEARCH_LIMIT: "20"
-  fixtures: []
-
-# =============================================================================
-# TEST CASES
-# =============================================================================
-
-test_cases:
-  # ---------------------------------------------------------------------------
-  # CATEGORY: Knowledge Graph Construction
-  # ---------------------------------------------------------------------------
-
-  - id: tc001_kg_codebase_indexing
-    description: "Build knowledge graph from TypeScript codebase with entity extraction"
-    category: knowledge_graph
-    priority: critical
-
-    input:
-      prompt: |
-        Index a TypeScript codebase with the following structure:
-        - Extract all classes, interfaces, functions, and types
-        - Map relationships: imports, calls, inheritance, implementation
-        - Generate embeddings for semantic search
-        - Support incremental indexing for changed files only
-
-        What entities and relationships should be extracted?
-      context:
-        language: typescript
-        scope: src/
-        incremental: true
-
-    expected_output:
-      must_contain:
-        - "entities"
-        - "classes"
-        - "functions"
-        - "relationships"
-        - "imports"
-        - "embeddings"
-      must_not_contain:
-        - "error"
-        - "cannot parse"
-      severity_classification: critical
-      finding_count:
-        min: 1
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.8
-      reasoning_quality_min: 0.75
-
-  - id: tc002_entity_relationship_mapping
-    description: "Extract and validate entity-relationship metadata from code"
-    category: knowledge_graph
-    priority: high
-
-    input:
-      prompt: |
-        For this UserService class, extract:
-        1. Class metadata (name, file, line, complexity)
-        2. Method signatures
-        3. Dependencies (what it imports)
-        4. Relationships (what imports it)
-
-        ```typescript
-        export class UserService {
-          constructor(private db: Database) {}
-          async getUserById(id: string): Promise<User> { ... }
-        }
-        ```
-      context:
-        file: src/services/UserService.ts
-        extract_complexity: true
-
-    expected_output:
-      must_contain:
-        - "UserService"
-        - "Database"
-        - "complexity"
-        - "dependencies"
-        - "methods"
-      must_not_contain:
-        - "parse error"
-      severity_classification: high
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.75
-
-  # ---------------------------------------------------------------------------
-  # CATEGORY: Semantic Code Search
-  # ---------------------------------------------------------------------------
-
-  - id: tc003_semantic_search_relevance
-    description: "Search code semantically and rank results by relevance"
-    category: semantic_search
-    priority: critical
-
-    input:
-      prompt: |
-        Perform semantic search in a codebase for "payment processing with stripe".
-        Return top 10 results ranked by relevance score (0-1).
-
-        Results should include:
-        - File location and line number
-        - Code snippet
-        - Relevance explanation
-
-        How would you score relevance?
-      context:
-        query: "payment processing with stripe"
-        limit: 10
-        threshold: 0.7
-
-    expected_output:
-      must_contain:
-        - "relevance"
-        - "score"
-        - "ranking"
-        - "snippet"
-        - "payment"
-        - "stripe"
-      must_not_contain:
-        - "keyword matching"
-      severity_classification: critical
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.8
-      reasoning_quality_min: 0.75
-
-  - id: tc004_search_with_context_retrieval
-    description: "Return semantic search results with intelligent context"
-    category: semantic_search
-    priority: high
-
-    input:
-      prompt: |
-        Search for "authentication middleware" and return results with:
-        - Before/after code context (surrounding lines)
-        - Related entities (what it calls, what calls it)
-        - Usage examples if available
-
-        How would you prioritize context when token budget is limited?
-      context:
-        query: "authentication middleware"
-        include_context: true
-        max_tokens: 2000
-
-    expected_output:
-      must_contain:
-        - "authentication"
-        - "middleware"
-        - "context"
-        - "related"
-        - "usage"
-      finding_count:
-        min: 1
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.75
-
-  # ---------------------------------------------------------------------------
-  # CATEGORY: Dependency Analysis
-  # ---------------------------------------------------------------------------
-
-  - id: tc005_dependency_mapping
-    description: "Map dependencies for a service with depth analysis"
-    category: dependencies
-    priority: critical
-
-    input:
-      prompt: |
-        Analyze dependencies for OrderService at depth 3:
-        - Direct dependencies (1 level)
-        - Transitive dependencies (2 levels)
-        - Deep dependencies (3 levels)
-
-        Calculate coupling metrics:
-        - Afferent coupling (incoming dependencies)
-        - Efferent coupling (outgoing dependencies)
-        - Instability score
-      context:
-        entry_point: src/services/OrderService.ts
-        depth: 3
-        direction: both
-
-    expected_output:
-      must_contain:
-        - "dependencies"
-        - "coupling"
-        - "afferent"
-        - "efferent"
-        - "instability"
-      must_not_contain:
-        - "unable"
-        - "cannot resolve"
-      severity_classification: critical
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.8
-
-  - id: tc006_circular_dependency_detection
-    description: "Detect circular dependencies that could cause issues"
-    category: dependencies
-    priority: high
-
-    input:
-      prompt: |
-        How would you detect circular dependencies in a codebase?
-        A -> B -> C -> A
-
-        What are the implications of circular dependencies?
-        How would you report and fix them?
-      context:
-        analysis_type: circular_detection
-        fix_suggestions: true
-
-    expected_output:
-      must_contain:
-        - "circular"
-        - "dependencies"
-        - "cycle"
-        - "implications"
-        - "fix"
-      severity_classification: high
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.75
-
-  # ---------------------------------------------------------------------------
-  # CATEGORY: Token Optimization
-  # ---------------------------------------------------------------------------
-
-  - id: tc007_intelligent_context_optimization
-    description: "Retrieve context with 80% token reduction"
-    category: optimization
-    priority: critical
-
-    input:
-      prompt: |
-        Retrieve context for "implement user registration" with:
-        - Token budget: 4000 tokens
-        - Include function signatures
-        - Include relevant examples (top 3)
-        - Summarize implementations when necessary
-        - Deduplicate related code
-
-        Achieve 80% token reduction compared to full context.
-        How would you measure the reduction?
-      context:
-        query: "implement user registration"
-        budget_tokens: 4000
-        target_reduction: 0.8
-
-    expected_output:
-      must_contain:
-        - "token"
-        - "optimization"
-        - "budget"
-        - "signature"
-        - "example"
-        - "reduction"
-      must_not_contain:
-        - "full context"
-      severity_classification: critical
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.8
-      reasoning_quality_min: 0.75
-
-  - id: tc008_context_prioritization
-    description: "Prioritize context elements by relevance and importance"
-    category: optimization
-    priority: high
-
-    input:
-      prompt: |
-        When token budget is limited (2000 tokens), what would you prioritize?
-
-        1. Function signatures (essential)
-        2. Comments/documentation (helpful)
-        3. Full implementations (verbose)
-        4. Examples (useful)
-        5. Related entities (context)
-
-        How would you rank these for maximum usefulness?
-      context:
-        budget_tokens: 2000
-        prioritization: true
-
-    expected_output:
-      must_contain:
-        - "prioritize"
-        - "signatures"
-        - "relevant"
-        - "essential"
-        - "examples"
-      finding_count:
-        min: 1
-
-    validation:
-      schema_check: true
-      keyword_match_threshold: 0.75
-
-  # ---------------------------------------------------------------------------
-  # CATEGORY: Negative Tests
-  # ---------------------------------------------------------------------------
-
-  - id: tc009_graceful_handling_missing_code
-    description: "Handle missing or unparseable code gracefully"
-    category: negative
-    priority: high
-
-    input:
-      prompt: |
-        How should the knowledge graph handle:
-        1. Files with syntax errors
-        2. Generated code (node_modules, dist/)
-        3. Non-supported languages
-        4. Binary files
-        5. Empty files
-
-        What error recovery strategies would you use?
-      context:
-        include_error_handling: true
-        exclude_patterns: ["node_modules", "dist", "*.min.js"]
-
-    expected_output:
-      must_contain:
-        - "graceful"
-        - "error"
-        - "handle"
-        - "skip"
-        - "recovery"
-      must_not_contain:
-        - "crash"
-        - "fail"
-      finding_count:
-        max: 2
-
-    validation:
-      schema_check: true
-      allow_partial: true
-
-# =============================================================================
-# SUCCESS CRITERIA
-# =============================================================================
-
-success_criteria:
-  pass_rate: 0.8
-  critical_pass_rate: 1.0
-  avg_reasoning_quality: 0.75
-  max_execution_time_ms: 300000
-  cross_model_variance: 0.15
-
-# =============================================================================
-# METADATA
-# =============================================================================
-
-metadata:
-  author: "qe-knowledge-graph"
-  created: "2026-02-02"
-  last_updated: "2026-02-02"
-  coverage_target: >
-    Knowledge graph construction with entity/relationship extraction, semantic
-    code search with relevance ranking, dependency analysis with coupling metrics,
-    circular dependency detection, and intelligent token budget optimization
-    achieving 80% token reduction.
+# =============================================================================
+# AQE Skill Evaluation Test Suite: QE Code Intelligence v1.0.0
+# =============================================================================
+#
+# Comprehensive evaluation suite for the qe-code-intelligence skill.
+# Tests knowledge graph construction, semantic code search, dependency mapping,
+# and intelligent context retrieval with 80% token reduction.
+#
+# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
+# Validator: .claude/skills/qe-code-intelligence/scripts/validate-config.json
+#
+# Coverage:
+# - Codebase indexing and knowledge graph construction
+# - Semantic search with relevance ranking
+# - Dependency analysis and mapping
+# - Intelligent context retrieval with token optimization
+# - Entity relationship extraction
+#
+# =============================================================================
+
+skill: qe-code-intelligence
+version: 1.0.0
+description: >
+  Comprehensive evaluation suite for the qe-code-intelligence skill.
+  Tests knowledge graph construction, semantic code search, dependency mapping,
+  context-aware code understanding, and intelligent token budget optimization.
+
+# =============================================================================
+# Multi-Model Configuration
+# =============================================================================
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+# =============================================================================
+# MCP Integration Configuration
+# =============================================================================
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  share_learning: true
+  update_quality_gate: true
+
+  target_agents:
+    - qe-learning-coordinator
+    - qe-queen-coordinator
+    - qe-knowledge-graph
+    - qe-semantic-searcher
+
+# =============================================================================
+# ReasoningBank Learning Configuration
+# =============================================================================
+
+learning:
+  store_success_patterns: true
+  store_failure_patterns: true
+  pattern_ttl_days: 90
+  min_confidence_to_store: 0.7
+  cross_model_comparison: true
+
+# =============================================================================
+# Result Format Configuration
+# =============================================================================
+
+result_format:
+  json_output: true
+  markdown_report: true
+  include_raw_output: false
+  include_timing: true
+  include_token_usage: true
+
+# =============================================================================
+# Environment Setup
+# =============================================================================
+
+setup:
+  required_tools:
+    - jq
+  environment_variables:
+    KG_CACHE_ENABLED: "true"
+    SEARCH_LIMIT: "20"
+  fixtures: []
+
+# =============================================================================
+# TEST CASES
+# =============================================================================
+
+test_cases:
+  # ---------------------------------------------------------------------------
+  # CATEGORY: Knowledge Graph Construction
+  # ---------------------------------------------------------------------------
+
+  - id: tc001_kg_codebase_indexing
+    description: "Build knowledge graph from TypeScript codebase with entity extraction"
+    category: knowledge_graph
+    priority: critical
+
+    input:
+      prompt: |
+        Index a TypeScript codebase with the following structure:
+        - Extract all classes, interfaces, functions, and types
+        - Map relationships: imports, calls, inheritance, implementation
+        - Generate embeddings for semantic search
+        - Support incremental indexing for changed files only
+
+        What entities and relationships should be extracted?
+      context:
+        language: typescript
+        scope: src/
+        incremental: true
+
+    expected_output:
+      must_contain:
+        - "entities"
+        - "classes"
+        - "functions"
+        - "relationships"
+        - "imports"
+        - "embeddings"
+      must_not_contain:
+        - "error"
+        - "cannot parse"
+      severity_classification: critical
+      finding_count:
+        min: 1
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+      reasoning_quality_min: 0.75
+
+  - id: tc002_entity_relationship_mapping
+    description: "Extract and validate entity-relationship metadata from code"
+    category: knowledge_graph
+    priority: high
+
+    input:
+      prompt: |
+        For this UserService class, extract:
+        1. Class metadata (name, file, line, complexity)
+        2. Method signatures
+        3. Dependencies (what it imports)
+        4. Relationships (what imports it)
+
+        ```typescript
+        export class UserService {
+          constructor(private db: Database) {}
+          async getUserById(id: string): Promise<User> { ... }
+        }
+        ```
+      context:
+        file: src/services/UserService.ts
+        extract_complexity: true
+
+    expected_output:
+      must_contain:
+        - "UserService"
+        - "Database"
+        - "complexity"
+        - "dependencies"
+        - "methods"
+      must_not_contain:
+        - "parse error"
+      severity_classification: high
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.75
+
+  # ---------------------------------------------------------------------------
+  # CATEGORY: Semantic Code Search
+  # ---------------------------------------------------------------------------
+
+  - id: tc003_semantic_search_relevance
+    description: "Search code semantically and rank results by relevance"
+    category: semantic_search
+    priority: critical
+
+    input:
+      prompt: |
+        Perform semantic search in a codebase for "payment processing with stripe".
+        Return top 10 results ranked by relevance score (0-1).
+
+        Results should include:
+        - File location and line number
+        - Code snippet
+        - Relevance explanation
+
+        How would you score relevance?
+      context:
+        query: "payment processing with stripe"
+        limit: 10
+        threshold: 0.7
+
+    expected_output:
+      must_contain:
+        - "relevance"
+        - "score"
+        - "ranking"
+        - "snippet"
+        - "payment"
+        - "stripe"
+      must_not_contain:
+        - "keyword matching"
+      severity_classification: critical
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+      reasoning_quality_min: 0.75
+
+  - id: tc004_search_with_context_retrieval
+    description: "Return semantic search results with intelligent context"
+    category: semantic_search
+    priority: high
+
+    input:
+      prompt: |
+        Search for "authentication middleware" and return results with:
+        - Before/after code context (surrounding lines)
+        - Related entities (what it calls, what calls it)
+        - Usage examples if available
+
+        How would you prioritize context when token budget is limited?
+      context:
+        query: "authentication middleware"
+        include_context: true
+        max_tokens: 2000
+
+    expected_output:
+      must_contain:
+        - "authentication"
+        - "middleware"
+        - "context"
+        - "related"
+        - "usage"
+      finding_count:
+        min: 1
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.75
+
+  # ---------------------------------------------------------------------------
+  # CATEGORY: Dependency Analysis
+  # ---------------------------------------------------------------------------
+
+  - id: tc005_dependency_mapping
+    description: "Map dependencies for a service with depth analysis"
+    category: dependencies
+    priority: critical
+
+    input:
+      prompt: |
+        Analyze dependencies for OrderService at depth 3:
+        - Direct dependencies (1 level)
+        - Transitive dependencies (2 levels)
+        - Deep dependencies (3 levels)
+
+        Calculate coupling metrics:
+        - Afferent coupling (incoming dependencies)
+        - Efferent coupling (outgoing dependencies)
+        - Instability score
+      context:
+        entry_point: src/services/OrderService.ts
+        depth: 3
+        direction: both
+
+    expected_output:
+      must_contain:
+        - "dependencies"
+        - "coupling"
+        - "afferent"
+        - "efferent"
+        - "instability"
+      must_not_contain:
+        - "unable"
+        - "cannot resolve"
+      severity_classification: critical
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc006_circular_dependency_detection
+    description: "Detect circular dependencies that could cause issues"
+    category: dependencies
+    priority: high
+
+    input:
+      prompt: |
+        How would you detect circular dependencies in a codebase?
+        A -> B -> C -> A
+
+        What are the implications of circular dependencies?
+        How would you report and fix them?
+      context:
+        analysis_type: circular_detection
+        fix_suggestions: true
+
+    expected_output:
+      must_contain:
+        - "circular"
+        - "dependencies"
+        - "cycle"
+        - "implications"
+        - "fix"
+      severity_classification: high
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.75
+
+  # ---------------------------------------------------------------------------
+  # CATEGORY: Token Optimization
+  # ---------------------------------------------------------------------------
+
+  - id: tc007_intelligent_context_optimization
+    description: "Retrieve context with 80% token reduction"
+    category: optimization
+    priority: critical
+
+    input:
+      prompt: |
+        Retrieve context for "implement user registration" with:
+        - Token budget: 4000 tokens
+        - Include function signatures
+        - Include relevant examples (top 3)
+        - Summarize implementations when necessary
+        - Deduplicate related code
+
+        Achieve 80% token reduction compared to full context.
+        How would you measure the reduction?
+      context:
+        query: "implement user registration"
+        budget_tokens: 4000
+        target_reduction: 0.8
+
+    expected_output:
+      must_contain:
+        - "token"
+        - "optimization"
+        - "budget"
+        - "signature"
+        - "example"
+        - "reduction"
+      must_not_contain:
+        - "full context"
+      severity_classification: critical
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+      reasoning_quality_min: 0.75
+
+  - id: tc008_context_prioritization
+    description: "Prioritize context elements by relevance and importance"
+    category: optimization
+    priority: high
+
+    input:
+      prompt: |
+        When token budget is limited (2000 tokens), what would you prioritize?
+
+        1. Function signatures (essential)
+        2. Comments/documentation (helpful)
+        3. Full implementations (verbose)
+        4. Examples (useful)
+        5. Related entities (context)
+
+        How would you rank these for maximum usefulness?
+      context:
+        budget_tokens: 2000
+        prioritization: true
+
+    expected_output:
+      must_contain:
+        - "prioritize"
+        - "signatures"
+        - "relevant"
+        - "essential"
+        - "examples"
+      finding_count:
+        min: 1
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.75
+
+  # ---------------------------------------------------------------------------
+  # CATEGORY: Negative Tests
+  # ---------------------------------------------------------------------------
+
+  - id: tc009_graceful_handling_missing_code
+    description: "Handle missing or unparseable code gracefully"
+    category: negative
+    priority: high
+
+    input:
+      prompt: |
+        How should the knowledge graph handle:
+        1. Files with syntax errors
+        2. Generated code (node_modules, dist/)
+        3. Non-supported languages
+        4. Binary files
+        5. Empty files
+
+        What error recovery strategies would you use?
+      context:
+        include_error_handling: true
+        exclude_patterns: ["node_modules", "dist", "*.min.js"]
+
+    expected_output:
+      must_contain:
+        - "graceful"
+        - "error"
+        - "handle"
+        - "skip"
+        - "recovery"
+      must_not_contain:
+        - "crash"
+        - "fail"
+      finding_count:
+        max: 2
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+# =============================================================================
+# SUCCESS CRITERIA
+# =============================================================================
+
+success_criteria:
+  pass_rate: 0.8
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.75
+  max_execution_time_ms: 300000
+  cross_model_variance: 0.15
+
+# =============================================================================
+# METADATA
+# =============================================================================
+
+metadata:
+  author: "qe-knowledge-graph"
+  created: "2026-02-02"
+  last_updated: "2026-02-02"
+  coverage_target: >
+    Knowledge graph construction with entity/relationship extraction, semantic
+    code search with relevance ranking, dependency analysis with coupling metrics,
+    circular dependency detection, and intelligent token budget optimization
+    achieving 80% token reduction.