pangea 0.0.42 → 0.0.45

data/scripts/synthesis_coverage_report.rb

@@ -0,0 +1,284 @@
+#!/usr/bin/env ruby
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require 'find'
+require 'json'
+
+# Simple color output without external dependencies
+class String
+  def colorize(color)
+    colors = {
+      red: 31,
+      green: 32,
+      yellow: 33,
+      blue: 34,
+      magenta: 35,
+      cyan: 36,
+      light_blue: 94
+    }
+    "\e[#{colors[color] || 0}m#{self}\e[0m"
+  end
+end
+
+# Script to analyze synthesis test coverage for all resources
+class SynthesisCoverageAnalyzer
+  RESOURCE_DIR = File.join(__dir__, '..', 'lib', 'pangea', 'resources')
+  SPEC_DIR = File.join(__dir__, '..', 'spec', 'resources')
+  
+  def initialize
+    @resources = {}
+    @coverage_report = {}
+  end
+  
+  def run
+    puts "=== Pangea Synthesis Test Coverage Report ===".colorize(:yellow)
+    puts
+    
+    discover_resources
+    analyze_coverage
+    generate_report
+  end
+  
+  private
+  
+  def discover_resources
+    Dir.glob(File.join(RESOURCE_DIR, 'aws_*')).each do |resource_path|
+      next unless File.directory?(resource_path)
+      
+      resource_name = File.basename(resource_path)
+      resource_file = File.join(resource_path, 'resource.rb')
+      
+      # Only consider directories that have a resource.rb file with actual implementation
+      next unless File.exist?(resource_file)
+      
+      # Check if the resource.rb file contains an actual function definition
+      content = File.read(resource_file)
+      next unless content.include?("def #{resource_name}(")
+      
+      @resources[resource_name] = {
+        path: resource_path,
+        resource_file: resource_file,
+        has_types: File.exist?(File.join(resource_path, 'types.rb')),
+        has_docs: File.exist?(File.join(resource_path, 'CLAUDE.md'))
+      }
+    end
+    
+    puts "Found #{@resources.size} implemented resources to analyze".colorize(:green)
+  end
+  
+  def analyze_coverage
+    @resources.each do |resource_name, info|
+      spec_dir = File.join(SPEC_DIR, resource_name)
+      
+      @coverage_report[resource_name] = {
+        has_spec_dir: Dir.exist?(spec_dir),
+        resource_spec: nil,
+        synthesis_spec: nil,
+        total_tests: 0,
+        synthesis_tests: 0,
+        synthesis_coverage: []
+      }
+      
+      if Dir.exist?(spec_dir)
+        # Check for resource spec
+        resource_spec_file = File.join(spec_dir, 'complete_resource_spec.rb')
+        if File.exist?(resource_spec_file)
+          @coverage_report[resource_name][:resource_spec] = analyze_spec_file(resource_spec_file)
+        end
+        
+        # Check for synthesis spec
+        synthesis_spec_file = File.join(spec_dir, 'complete_synthesis_spec.rb')
+        if File.exist?(synthesis_spec_file)
+          @coverage_report[resource_name][:synthesis_spec] = analyze_synthesis_spec(synthesis_spec_file)
+          @coverage_report[resource_name][:synthesis_tests] = @coverage_report[resource_name][:synthesis_spec][:test_count]
+          @coverage_report[resource_name][:synthesis_coverage] = extract_synthesis_coverage(synthesis_spec_file)
+        end
+        
+        # Total test count
+        @coverage_report[resource_name][:total_tests] = 
+          (@coverage_report[resource_name][:resource_spec]&.dig(:test_count) || 0) +
+          (@coverage_report[resource_name][:synthesis_spec]&.dig(:test_count) || 0)
+      end
+    end
+  end
+  
+  def analyze_spec_file(file_path)
+    content = File.read(file_path)
+    
+    {
+      test_count: content.scan(/\bit\s+["']/).count,
+      describe_blocks: content.scan(/describe\s+["']/).count,
+      file_size: File.size(file_path)
+    }
+  end
+  
+  def analyze_synthesis_spec(file_path)
+    content = File.read(file_path)
+    
+    # Look for specific synthesis patterns
+    synthesis_patterns = {
+      basic_synthesis: content.include?('synthesizes basic') || content.include?('synthesizes minimal'),
+      tag_synthesis: content.include?('synthesizes tags'),
+      conditional_synthesis: content.include?('conditional attributes') || content.include?('handles conditional'),
+      output_validation: content.include?('terraform reference outputs') || content.include?('validates terraform'),
+      complex_synthesis: content.scan(/synthesizes\s+\w+\s+with/).count,
+      mock_synthesizer: content.include?('mock_synthesizer') || content.include?('MockSynthesizer')
+    }
+    
+    {
+      test_count: content.scan(/\bit\s+["']/).count,
+      describe_blocks: content.scan(/describe\s+["']/).count,
+      file_size: File.size(file_path),
+      patterns: synthesis_patterns
+    }
+  end
+  
+  def extract_synthesis_coverage(file_path)
+    content = File.read(file_path)
+    coverage = []
+    
+    # Extract test descriptions
+    content.scan(/it\s+["']([^"']+)["']/) do |match|
+      test_name = match[0]
+      if test_name.downcase.include?('synthesis') || test_name.downcase.include?('terraform')
+        coverage << test_name
+      end
+    end
+    
+    coverage
+  end
+  
+  def generate_report
+    puts "\n=== Coverage Summary ===".colorize(:yellow)
+    
+    # Sort resources by name
+    sorted_resources = @coverage_report.sort_by { |name, _| name }
+    
+    # Summary stats
+    total_resources = @resources.size
+    resources_with_tests = @coverage_report.count { |_, data| data[:has_spec_dir] }
+    resources_with_synthesis = @coverage_report.count { |_, data| data[:synthesis_tests] > 0 }
+    total_tests = @coverage_report.sum { |_, data| data[:total_tests] }
+    total_synthesis_tests = @coverage_report.sum { |_, data| data[:synthesis_tests] }
+    
+    puts "Total Resources: #{total_resources}".colorize(:blue)
+    puts "Resources with Tests: #{resources_with_tests} (#{(resources_with_tests.to_f / total_resources * 100).round(1)}%)".colorize(:green)
+    puts "Resources with Synthesis Tests: #{resources_with_synthesis} (#{(resources_with_synthesis.to_f / total_resources * 100).round(1)}%)".colorize(:green)
+    puts "Total Tests: #{total_tests}".colorize(:blue)
+    puts "Total Synthesis Tests: #{total_synthesis_tests}".colorize(:blue)
+    
+    puts "\n=== Detailed Resource Coverage ===".colorize(:yellow)
+    
+    sorted_resources.each do |resource_name, data|
+      puts "\n#{resource_name}:".colorize(:cyan)
+      
+      if !data[:has_spec_dir]
+        puts "  ❌ No test directory found".colorize(:red)
+      else
+        if data[:resource_spec]
+          puts "  ✅ Resource tests: #{data[:resource_spec][:test_count]} tests".colorize(:green)
+        else
+          puts "  ❌ No resource tests found".colorize(:red)
+        end
+        
+        if data[:synthesis_spec]
+          puts "  ✅ Synthesis tests: #{data[:synthesis_tests]} tests".colorize(:green)
+          
+          # Show synthesis patterns
+          patterns = data[:synthesis_spec][:patterns]
+          puts "    Synthesis coverage:".colorize(:light_blue)
+          puts "      #{patterns[:basic_synthesis] ? '✓' : '✗'} Basic synthesis".colorize(patterns[:basic_synthesis] ? :green : :red)
+          puts "      #{patterns[:tag_synthesis] ? '✓' : '✗'} Tag synthesis".colorize(patterns[:tag_synthesis] ? :green : :red)
+          puts "      #{patterns[:conditional_synthesis] ? '✓' : '✗'} Conditional attributes".colorize(patterns[:conditional_synthesis] ? :green : :red)
+          puts "      #{patterns[:output_validation] ? '✓' : '✗'} Output validation".colorize(patterns[:output_validation] ? :green : :red)
+          puts "      #{patterns[:mock_synthesizer] ? '✓' : '✗'} Mock synthesizer usage".colorize(patterns[:mock_synthesizer] ? :green : :red)
+          
+          if patterns[:complex_synthesis] > 0
+            puts "      ✓ Complex synthesis tests: #{patterns[:complex_synthesis]}".colorize(:green)
+          end
+        else
+          puts "  ❌ No synthesis tests found".colorize(:red)
+        end
+        
+        puts "  Total: #{data[:total_tests]} tests"
+      end
+    end
+    
+    # Recommendations
+    puts "\n=== Recommendations ===".colorize(:yellow)
+    
+    missing_synthesis = sorted_resources.select { |_, data| data[:has_spec_dir] && data[:synthesis_tests] == 0 }
+    if missing_synthesis.any?
+      puts "\nResources missing synthesis tests:".colorize(:red)
+      missing_synthesis.each do |name, _|
+        puts "  - #{name}"
+      end
+    end
+    
+    low_synthesis_coverage = sorted_resources.select do |_, data| 
+      data[:synthesis_tests] > 0 && data[:synthesis_tests] < 10
+    end
+    
+    if low_synthesis_coverage.any?
+      puts "\nResources with low synthesis test coverage (<10 tests):".colorize(:yellow)
+      low_synthesis_coverage.each do |name, data|
+        puts "  - #{name}: #{data[:synthesis_tests]} tests"
+      end
+    end
+    
+    # Missing critical patterns
+    puts "\nResources missing critical synthesis patterns:".colorize(:yellow)
+    sorted_resources.each do |name, data|
+      next unless data[:synthesis_spec]
+      
+      patterns = data[:synthesis_spec][:patterns]
+      missing = []
+      
+      missing << "basic synthesis" unless patterns[:basic_synthesis]
+      missing << "tag synthesis" unless patterns[:tag_synthesis]
+      missing << "output validation" unless patterns[:output_validation]
+      
+      if missing.any?
+        puts "  - #{name}: missing #{missing.join(', ')}"
+      end
+    end
+    
+    # Export detailed report
+    export_json_report
+  end
+  
+  def export_json_report
+    report_file = File.join(__dir__, 'synthesis_coverage_report.json')
+    
+    report_data = {
+      generated_at: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ'),
+      summary: {
+        total_resources: @resources.size,
+        resources_with_tests: @coverage_report.count { |_, data| data[:has_spec_dir] },
+        resources_with_synthesis: @coverage_report.count { |_, data| data[:synthesis_tests] > 0 },
+        total_tests: @coverage_report.sum { |_, data| data[:total_tests] },
+        total_synthesis_tests: @coverage_report.sum { |_, data| data[:synthesis_tests] }
+      },
+      resources: @coverage_report
+    }
+    
+    File.write(report_file, JSON.pretty_generate(report_data))
+    puts "\n💾 Detailed report saved to: #{report_file}".colorize(:green)
+  end
+end
+
+# Run the analyzer
+SynthesisCoverageAnalyzer.new.run

data/scripts/synthesis_coverage_summary.rb

@@ -0,0 +1,233 @@
+#!/usr/bin/env ruby
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require 'json'
+
+# Script to summarize synthesis test coverage for the 15 tested resources
+class SynthesisCoverageSummary
+  TESTED_RESOURCES = %w[
+    aws_vpc
+    aws_subnet
+    aws_security_group
+    aws_instance
+    aws_internet_gateway
+    aws_route_table
+    aws_nat_gateway
+    aws_launch_template
+    aws_autoscaling_group
+    aws_lb_target_group
+    aws_lb
+    aws_s3_bucket
+    aws_iam_role
+    aws_db_instance
+    aws_eip
+  ].freeze
+  
+  def run
+    puts "=== Synthesis Coverage Summary for Tested Resources ==="
+    puts
+    
+    analyze_coverage
+    generate_enhancement_tasks
+  end
+  
+  private
+  
+  def analyze_coverage
+    puts "Resource Coverage Status:"
+    puts "=" * 80
+    
+    total_resource_tests = 0
+    total_synthesis_tests = 0
+    missing_patterns = {}
+    
+    TESTED_RESOURCES.each do |resource|
+      spec_dir = File.join(__dir__, '..', 'spec', 'resources', resource)
+      
+      if Dir.exist?(spec_dir)
+        resource_spec = File.join(spec_dir, 'complete_resource_spec.rb')
+        synthesis_spec = File.join(spec_dir, 'complete_synthesis_spec.rb')
+        
+        resource_tests = 0
+        synthesis_tests = 0
+        patterns = []
+        
+        if File.exist?(resource_spec)
+          content = File.read(resource_spec)
+          resource_tests = content.scan(/\bit\s+["']/).count
+          total_resource_tests += resource_tests
+        end
+        
+        if File.exist?(synthesis_spec)
+          content = File.read(synthesis_spec)
+          synthesis_tests = content.scan(/\bit\s+["']/).count
+          total_synthesis_tests += synthesis_tests
+          
+          # Check for specific patterns
+          missing = []
+          missing << "tag synthesis" unless content.include?('synthesizes tags') || content.include?('synthesizes a') && content.include?('with tags')
+          missing << "conditional attributes" unless content.include?('conditional attributes') || content.include?('handles conditional')
+          missing << "complex scenarios" unless content.scan(/synthesizes\s+\w+\s+with/).count > 3
+          
+          missing_patterns[resource] = missing if missing.any?
+        end
+        
+        status = if synthesis_tests >= 10
+          "✅ GOOD"
+        elsif synthesis_tests > 0
+          "⚠️  LOW"
+        else
+          "❌ NONE"
+        end
+        
+        puts "#{resource.ljust(25)} | Resource: #{resource_tests.to_s.rjust(2)} | Synthesis: #{synthesis_tests.to_s.rjust(2)} | #{status}"
+      else
+        puts "#{resource.ljust(25)} | ❌ No test directory"
+      end
+    end
+    
+    puts "=" * 80
+    puts "Total Resource Tests: #{total_resource_tests}"
+    puts "Total Synthesis Tests: #{total_synthesis_tests}"
+    puts "Average Synthesis Tests per Resource: #{(total_synthesis_tests.to_f / TESTED_RESOURCES.size).round(1)}"
+    
+    if missing_patterns.any?
+      puts "\nResources Missing Key Synthesis Patterns:"
+      puts "=" * 80
+      missing_patterns.each do |resource, patterns|
+        puts "#{resource}: #{patterns.join(', ')}"
+      end
+    end
+  end
+  
+  def generate_enhancement_tasks
+    puts "\n=== Synthesis Enhancement Tasks ==="
+    puts "=" * 80
+    
+    tasks = []
+    
+    # Task 1: Add missing tag synthesis tests
+    needs_tag_tests = %w[
+      aws_vpc
+      aws_subnet
+      aws_security_group
+      aws_instance
+      aws_internet_gateway
+      aws_route_table
+      aws_nat_gateway
+      aws_launch_template
+      aws_autoscaling_group
+      aws_lb_target_group
+      aws_lb
+    ]
+    
+    tasks << {
+      id: "SYNTH-ENHANCE-001",
+      priority: "high",
+      title: "Add tag synthesis tests for resources missing them",
+      resources: needs_tag_tests,
+      description: "Add explicit tag synthesis tests to ensure tag blocks are properly generated"
+    }
+    
+    # Task 2: Add conditional attribute tests
+    needs_conditional_tests = %w[
+      aws_subnet
+      aws_security_group
+      aws_instance
+      aws_internet_gateway
+      aws_nat_gateway
+      aws_route_table
+    ]
+    
+    tasks << {
+      id: "SYNTH-ENHANCE-002", 
+      priority: "medium",
+      title: "Add conditional attribute synthesis tests",
+      resources: needs_conditional_tests,
+      description: "Test that optional attributes are only synthesized when present"
+    }
+    
+    # Task 3: Increase synthesis coverage for low-coverage resources
+    low_coverage = %w[
+      aws_vpc
+      aws_subnet
+      aws_security_group
+      aws_instance
+      aws_internet_gateway
+    ]
+    
+    tasks << {
+      id: "SYNTH-ENHANCE-003",
+      priority: "medium", 
+      title: "Increase synthesis test coverage to 10+ tests",
+      resources: low_coverage,
+      description: "Add more synthesis scenarios to reach minimum 10 tests per resource"
+    }
+    
+    # Task 4: Add complex synthesis scenarios
+    needs_complex = %w[
+      aws_vpc
+      aws_subnet
+      aws_security_group
+      aws_route_table
+      aws_nat_gateway
+    ]
+    
+    tasks << {
+      id: "SYNTH-ENHANCE-004",
+      priority: "low",
+      title: "Add complex synthesis scenarios",
+      resources: needs_complex,
+      description: "Add tests for resources with multiple complex attributes set"
+    }
+    
+    # Task 5: Add cross-reference synthesis tests
+    tasks << {
+      id: "SYNTH-ENHANCE-005",
+      priority: "low",
+      title: "Add cross-reference synthesis tests",
+      resources: ["all"],
+      description: "Test synthesis with terraform variable and resource references"
+    }
+    
+    # Print tasks
+    tasks.each do |task|
+      puts "\n#{task[:id]} - #{task[:title]} [#{task[:priority].upcase}]"
+      puts "-" * 60
+      puts "Description: #{task[:description]}"
+      puts "Resources: #{task[:resources].join(', ')}"
+    end
+    
+    # Export tasks to JSON
+    export_tasks(tasks)
+  end
+  
+  def export_tasks(tasks)
+    report_file = File.join(__dir__, 'synthesis_enhancement_tasks.json')
+    
+    task_data = {
+      generated_at: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ'),
+      total_tasks: tasks.size,
+      tasks: tasks
+    }
+    
+    File.write(report_file, JSON.pretty_generate(task_data))
+    puts "\n\n💾 Enhancement tasks saved to: #{report_file}"
+  end
+end
+
+# Run the summary
+SynthesisCoverageSummary.new.run

data/scripts/synthesis_enhancement_tasks.json

@@ -0,0 +1,74 @@
+{
+  "generated_at": "2025-09-02T18:50:10Z",
+  "total_tasks": 5,
+  "tasks": [
+    {
+      "id": "SYNTH-ENHANCE-001",
+      "priority": "high",
+      "title": "Add tag synthesis tests for resources missing them",
+      "resources": [
+        "aws_vpc",
+        "aws_subnet",
+        "aws_security_group",
+        "aws_instance",
+        "aws_internet_gateway",
+        "aws_route_table",
+        "aws_nat_gateway",
+        "aws_launch_template",
+        "aws_autoscaling_group",
+        "aws_lb_target_group",
+        "aws_lb"
+      ],
+      "description": "Add explicit tag synthesis tests to ensure tag blocks are properly generated"
+    },
+    {
+      "id": "SYNTH-ENHANCE-002",
+      "priority": "medium",
+      "title": "Add conditional attribute synthesis tests",
+      "resources": [
+        "aws_subnet",
+        "aws_security_group",
+        "aws_instance",
+        "aws_internet_gateway",
+        "aws_nat_gateway",
+        "aws_route_table"
+      ],
+      "description": "Test that optional attributes are only synthesized when present"
+    },
+    {
+      "id": "SYNTH-ENHANCE-003",
+      "priority": "medium",
+      "title": "Increase synthesis test coverage to 10+ tests",
+      "resources": [
+        "aws_vpc",
+        "aws_subnet",
+        "aws_security_group",
+        "aws_instance",
+        "aws_internet_gateway"
+      ],
+      "description": "Add more synthesis scenarios to reach minimum 10 tests per resource"
+    },
+    {
+      "id": "SYNTH-ENHANCE-004",
+      "priority": "low",
+      "title": "Add complex synthesis scenarios",
+      "resources": [
+        "aws_vpc",
+        "aws_subnet",
+        "aws_security_group",
+        "aws_route_table",
+        "aws_nat_gateway"
+      ],
+      "description": "Add tests for resources with multiple complex attributes set"
+    },
+    {
+      "id": "SYNTH-ENHANCE-005",
+      "priority": "low",
+      "title": "Add cross-reference synthesis tests",
+      "resources": [
+        "all"
+      ],
+      "description": "Test synthesis with terraform variable and resource references"
+    }
+  ]
+}