aws_recon 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02f62713767ee1d437543e7684f844a1a9a922179bf6be3688ef7ccb114de345
-  data.tar.gz: d71ef31099b1fbee477b482a9aa84bfe6c9e091aacf2772678cbdf3b9dbfb7ca
+  metadata.gz: 7481b13d21571402935b0ce2b67a7cdaaf3d3fc245b49f5569ab249b00a80769
+  data.tar.gz: d755e86dbe27036c6db5aec7a10497f1cf85c4ad64265f673ec10fd1490d9566
 SHA512:
-  metadata.gz: 9215bf848adbd54d2652b35429897ac23e5f7140d6c7aa79db941622c95dee3468bd0354aedcdb3378086592740ea496b435a32adc138ce491a999b56ea4fc59
-  data.tar.gz: f151740b1e793abcae34a948f6375f8ff3a496d52a4df596cd115f59260b0afbbc1710400646c77eece2220fe399aef6ef5f181d2ad5a6cf326ebf51b4ea75d9
+  metadata.gz: a97a2b0b84fd34a79be57dac06caefa77231a7098d2ee221e3d6587d57c51aa181aff4aefeb13bfcfd52578b2f30285ce3e1a2f5f70ed8cff6c37d426f2daaa4
+  data.tar.gz: 7fdab7b7ddebb23fd28d28721966ff1d29a7b3a07c351e6319d3c6cdeb08fc4788869b8c30e2ea5f38a180bfe4cbf55dda05206a9985e568d0ea564d1c7eb19b

data/.github/workflows/check-aws-regions.yml

@@ -0,0 +1,17 @@
+name: check-service-regions
+
+on:
+  schedule:
+    - cron: '40 15 * * *'
+
+jobs:
+  region-check:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Set version tag
+        run: |
+          cd utils/aws ; ruby check_region_exclusions.rb

data/lib/aws_recon/collectors/iam.rb

@@ -50,7 +50,7 @@ class IAM < Mapper
                                          policy_document: p.policy_document.parse_policy
                                        }
                                      end
-                                    end
+                                   end
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/options.rb

@@ -36,8 +36,8 @@ class Parser
         aws_regions = ['global'].concat(Aws::EC2::Client.new.describe_regions.regions.map(&:region_name))
       end
     rescue Aws::Errors::ServiceError => e
-      puts "\nAWS Error: #{e.code}\n\n"
-      exit
+      warn "\nAWS Error: #{e.code}\n\n"
+      exit(1)
     end
 
     aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)

data/lib/aws_recon/services.yaml

@@ -106,7 +106,7 @@
 - name: SecretsManager
   alias: sm
 - name: SecurityHub
-  alias: sh
+  alias: securityhub
   excluded_regions:
     - ap-northeast-3
 - name: Support

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

data/utils/aws/check_region_exclusions.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+#
+# Check regional service availability against services.yaml exclusions.
+#
+require 'net/http'
+require 'json'
+require 'yaml'
+
+TS = Time.now.to_i
+URL = "https://api.regional-table.region-services.aws.a2z.com/index.json?timestamp=#{TS}000"
+
+region_exclusion_mistmatch = nil
+
+#
+# load current AWS Recon regions
+#
+recon_services = YAML.safe_load(File.read('../../lib/aws_recon/services.yaml'))
+abort('Errors loading AWS Recon services') unless recon_services.is_a?(Array)
+
+#
+# load current AWS regions (non-gov, non-cn)
+#
+regions = YAML.safe_load(File.read('regions.yaml'))
+abort('Errors loading regions') unless regions['Regions']
+
+all_regions = regions['Regions'].map { |r| r['RegionName'] }
+
+#
+# get service/price list from AWS
+#
+uri = URI(URL)
+res = Net::HTTP.get_response(uri)
+abort('Error loading AWS services from API') unless res.code == '200'
+
+map = {}
+
+#
+# load service region availability
+#
+data = res.body
+json = JSON.parse(data)
+
+# iterate through AWS provided services & regions
+json['prices'].each do |p|
+  at = p['attributes']
+  service_name = at['aws:serviceName']
+  service_id, service_region = p['id'].split(':')
+
+  # skip this service unless AWS Recon already has exclusions
+  next unless recon_services.filter { |s| s['alias'] == service_id }&.length&.positive?
+
+  if map.key?(service_name)
+    map[service_name]['regions'] << service_region
+  else
+    map[service_name] = {
+      'id' => service_id,
+      'regions' => [service_region]
+    }
+  end
+end
+
+# iterate through the services AWS Recon knows about
+map.sort.each do |k, v|
+  service_excluded_regions = all_regions.reject { |r| v['regions'].include?(r) }
+
+  aws_recon_service = recon_services.filter { |s| s['alias'] == v['id'] }&.first
+  aws_recon_service_excluded_regions = aws_recon_service['excluded_regions'] || []
+
+  # move on if AWS Recon region exclusions match AWS service region exclusions
+  next unless service_excluded_regions.sort != aws_recon_service_excluded_regions.sort
+
+  region_exclusion_mistmatch = true
+
+  puts "#{k} (#{v['id']})"
+
+  # determine the direction of the exclusion mismatch
+  if (service_excluded_regions - aws_recon_service_excluded_regions).length.positive?
+    puts " + missing region exclusion: #{(service_excluded_regions - aws_recon_service_excluded_regions).join(', ')}"
+  else
+    puts " - unnecessary region exclusion: #{(aws_recon_service_excluded_regions - service_excluded_regions).join(', ')}"
+  end
+end
+
+# exit code 1 if we have any mismatches
+exit 1 if region_exclusion_mistmatch

data/utils/aws/regions.yaml

@@ -0,0 +1,43 @@
+Regions:
+- Endpoint: ec2.af-south-1.amazonaws.com
+  RegionName: af-south-1
+- Endpoint: ec2.eu-north-1.amazonaws.com
+  RegionName: eu-north-1
+- Endpoint: ec2.ap-south-1.amazonaws.com
+  RegionName: ap-south-1
+- Endpoint: ec2.eu-west-3.amazonaws.com
+  RegionName: eu-west-3
+- Endpoint: ec2.eu-west-2.amazonaws.com
+  RegionName: eu-west-2
+- Endpoint: ec2.eu-south-1.amazonaws.com
+  RegionName: eu-south-1
+- Endpoint: ec2.eu-west-1.amazonaws.com
+  RegionName: eu-west-1
+- Endpoint: ec2.ap-northeast-3.amazonaws.com
+  RegionName: ap-northeast-3
+- Endpoint: ec2.ap-northeast-2.amazonaws.com
+  RegionName: ap-northeast-2
+- Endpoint: ec2.me-south-1.amazonaws.com
+  RegionName: me-south-1
+- Endpoint: ec2.ap-northeast-1.amazonaws.com
+  RegionName: ap-northeast-1
+- Endpoint: ec2.sa-east-1.amazonaws.com
+  RegionName: sa-east-1
+- Endpoint: ec2.ca-central-1.amazonaws.com
+  RegionName: ca-central-1
+- Endpoint: ec2.ap-east-1.amazonaws.com
+  RegionName: ap-east-1
+- Endpoint: ec2.ap-southeast-1.amazonaws.com
+  RegionName: ap-southeast-1
+- Endpoint: ec2.ap-southeast-2.amazonaws.com
+  RegionName: ap-southeast-2
+- Endpoint: ec2.eu-central-1.amazonaws.com
+  RegionName: eu-central-1
+- Endpoint: ec2.us-east-1.amazonaws.com
+  RegionName: us-east-1
+- Endpoint: ec2.us-east-2.amazonaws.com
+  RegionName: us-east-2
+- Endpoint: ec2.us-west-1.amazonaws.com
+  RegionName: us-west-1
+- Endpoint: ec2.us-west-2.amazonaws.com
+  RegionName: us-west-2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-07 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -163,6 +163,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/stale.yml"
+- ".github/workflows/check-aws-regions.yml"
 - ".github/workflows/docker-build.yml"
 - ".github/workflows/smoke-test.yml"
 - ".gitignore"
@@ -245,6 +246,8 @@ files:
 - lib/aws_recon/services.yaml
 - lib/aws_recon/version.rb
 - readme.md
+- utils/aws/check_region_exclusions.rb
+- utils/aws/regions.yaml
 - utils/cloudformation/aws-recon-cfn-template.yml
 - utils/terraform/cloudwatch.tf
 - utils/terraform/ecs.tf