aws_recon 0.5.0 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b98e94e11398d8f2aec76174c8d660be040521e44439ca70ca7b5b680e6a631
-  data.tar.gz: e8920ba7308df8491ae6ce6271188fedf3e57e499bc51f6d9d772728277ef6a6
+  metadata.gz: d7574ac2a2ffd096dae78183fe03998f59540d1f9672724b2e705894fc0fb5da
+  data.tar.gz: 1af2a00fa0e5a656f36b3966ada67b1c898763ecf2018a0dd087f2d7214d0506
 SHA512:
-  metadata.gz: b4c38506bd2a3f6a84d1bbe430f9812e5989bf80d6e8198bfee55ebb1b7f6b8010ee998b896e4202443224046a5fd33c1ed1913fa43d59a25d2cfdf4028d94cc
-  data.tar.gz: a91bd12229dead116b77d412043ecfcef306af8a2ac80772e2c6596574c488cd9e9c97e662334d76a97acb50b95bc578c51ccfcd5dbb3c7b53145e832eea9989
+  metadata.gz: 5e00e5416344bd5964e43b81677c90e483b522d5b116c7607f08215d83037088f67d8bd614979d3d056506d1c8a940108cf97ef2e2dddd0566c593d52fb1f56b
+  data.tar.gz: 49903877096060c68333b68022a7badde671183775b08b03efcc16e4c8967ab148903bd69e9e9eb7a0860350acafbb051d3aee2186e68667c6ed9b4075212f2b

data/.github/workflows/check-aws-regions.yml

@@ -0,0 +1,18 @@
+name: check-service-regions
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 10 * * *'
+
+jobs:
+  region-check:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Check AWS service regions
+        run: |
+          cd utils/aws ; ruby check_region_exclusions.rb

data/aws_recon.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
   spec.name          = 'aws_recon'
   spec.version       = AwsRecon::VERSION
   spec.authors       = ['Josh Larsen', 'Darkbit']
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.6.0'
   spec.summary       = 'A multi-threaded AWS security-focused inventory collection tool.'
   spec.description   = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
   spec.homepage      = 'https://github.com/darkbitio/aws-recon'

data/lib/aws_recon/collectors/ec2.rb

@@ -29,7 +29,7 @@ class EC2 < Mapper
         struct = OpenStruct.new
         struct.attributes = response.account_attributes.map(&:to_h)
         struct.type = 'account'
-        struct.arn = "arn:aws:ec2::#{@account}/account_attributes"
+        struct.arn = "arn:aws:ec2::#{@account}:attributes/account_attributes"
 
         resources.push(struct.to_h)
       end
@@ -45,7 +45,7 @@ class EC2 < Mapper
 
         struct = OpenStruct.new(response.to_h)
         struct.type = 'ebs_encryption_settings'
-        struct.arn = "arn:aws:ec2:#{@region}:#{@account}/ebs_encryption_settings"
+        struct.arn = "arn:aws:ec2:#{@region}:#{@account}:settings/ebs_encryption_settings"
 
         resources.push(struct.to_h)
       end
@@ -64,7 +64,7 @@ class EC2 < Mapper
           reservation.instances.each do |instance|
             struct = OpenStruct.new(instance.to_h)
             struct.type = 'instance'
-            struct.arn = instance.instance_id # no true ARN
+            struct.arn = "arn:aws:ec2:#{@region}:#{@account}:instance/#{instance.instance_id}" # no true ARN
             struct.reservation_id = reservation.reservation_id
 
             # collect instance user_data
@@ -96,7 +96,7 @@ class EC2 < Mapper
         response.vpcs.each do |vpc|
           struct = OpenStruct.new(vpc.to_h)
           struct.type = 'vpc'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{vpc.vpc_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:vpc/#{vpc.vpc_id}" # no true ARN
           struct.flow_logs = @client
                              .describe_flow_logs({ filter: [{ name: 'resource-id', values: [vpc.vpc_id] }] })
                              .flow_logs.first.to_h
@@ -114,7 +114,7 @@ class EC2 < Mapper
         response.security_groups.each do |security_group|
           struct = OpenStruct.new(security_group.to_h)
           struct.type = 'security_group'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{security_group.group_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:security_group/#{security_group.group_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -129,7 +129,7 @@ class EC2 < Mapper
         response.network_interfaces.each do |network_interface|
           struct = OpenStruct.new(network_interface.to_h)
           struct.type = 'network_interface'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{network_interface.network_interface_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:network_interface/#{network_interface.network_interface_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -144,7 +144,7 @@ class EC2 < Mapper
         response.network_acls.each do |network_acl|
           struct = OpenStruct.new(network_acl.to_h)
           struct.type = 'network_acl'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{network_acl.network_acl_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:network_acl/#{network_acl.network_acl_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -159,7 +159,7 @@ class EC2 < Mapper
         response.subnets.each do |subnet|
           struct = OpenStruct.new(subnet.to_h)
           struct.type = 'subnet'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{subnet.subnet_arn}" # no true ARN
+          struct.arn = subnet.subnet_arn
 
           resources.push(struct.to_h)
         end
@@ -174,7 +174,7 @@ class EC2 < Mapper
         response.addresses.each do |address|
           struct = OpenStruct.new(address.to_h)
           struct.type = 'eip_address'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{address.allocation_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:eip_address/#{address.allocation_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -189,7 +189,7 @@ class EC2 < Mapper
         response.nat_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'nat_gateway'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.nat_gateway_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:nat_gateway/#{gateway.nat_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -204,7 +204,7 @@ class EC2 < Mapper
         response.internet_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'internet_gateway'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.internet_gateway_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:internet_gateway/#{gateway.internet_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -219,7 +219,7 @@ class EC2 < Mapper
         response.route_tables.each do |table|
           struct = OpenStruct.new(table.to_h)
           struct.type = 'route_table'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{table.route_table_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:route_table/#{table.route_table_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -234,7 +234,7 @@ class EC2 < Mapper
         response.images.each do |image|
           struct = OpenStruct.new(image.to_h)
           struct.type = 'image'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{image.image_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:image/#{image.image_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -249,7 +249,7 @@ class EC2 < Mapper
         response.snapshots.each do |snapshot|
           struct = OpenStruct.new(snapshot.to_h)
           struct.type = 'snapshot'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{snapshot.snapshot_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:snapshot/#{snapshot.snapshot_id}" # no true ARN
           struct.create_volume_permissions = @client.describe_snapshot_attribute({
                                                                                    attribute: 'createVolumePermission',
                                                                                    snapshot_id: snapshot.snapshot_id
@@ -268,7 +268,7 @@ class EC2 < Mapper
         response.flow_logs.each do |flow_log|
           struct = OpenStruct.new(flow_log.to_h)
           struct.type = 'flow_log'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{flow_log.flow_log_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:flow_log/#{flow_log.flow_log_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -283,7 +283,7 @@ class EC2 < Mapper
         response.volumes.each do |volume|
           struct = OpenStruct.new(volume.to_h)
           struct.type = 'volume'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{volume.volume_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:volume/#{volume.volume_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -298,7 +298,7 @@ class EC2 < Mapper
         response.vpn_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'vpn_gateway'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.vpn_gateway_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:vpn_gateway/#{gateway.vpn_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -313,7 +313,7 @@ class EC2 < Mapper
         response.vpc_peering_connections.each do |peer|
           struct = OpenStruct.new(peer.to_h)
           struct.type = 'peering_connection'
-          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{peer.vpc_peering_connection_id}" # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}:peering_connection/#{peer.vpc_peering_connection_id}" # no true ARN
 
           resources.push(struct.to_h)
         end

data/lib/aws_recon/collectors/iam.rb

@@ -50,7 +50,7 @@ class IAM < Mapper
                                          policy_document: p.policy_document.parse_policy
                                        }
                                      end
-                                    end
+                                   end
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/options.rb

@@ -36,8 +36,8 @@ class Parser
         aws_regions = ['global'].concat(Aws::EC2::Client.new.describe_regions.regions.map(&:region_name))
       end
     rescue Aws::Errors::ServiceError => e
-      puts "\nAWS Error: #{e.code}\n\n"
-      exit
+      warn "\nAWS Error: #{e.code}\n\n"
+      exit(1)
     end
 
     aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)

data/lib/aws_recon/services.yaml

@@ -37,8 +37,6 @@
   alias: ecs
 - name: ElasticLoadBalancing
   alias: elb
-  excluded_regions:
-    - ap-southeast-1
 - name: ElasticLoadBalancingV2
   alias: elbv2
 - name: ElastiCache
@@ -85,15 +83,15 @@
 - name: Shield
   global: true
   alias: shield
+  excluded_regions:
+    - ap-northeast-3
 - name: CloudFormation
   alias: cloudformation
 - name: SES
   alias: ses
   excluded_regions:
-    - af-south-1
     - ap-east-1
     - ap-northeast-3
-    - eu-south-1
 - name: CloudWatch
   alias: cloudwatch
 - name: CloudWatchLogs
@@ -106,9 +104,7 @@
 - name: SecretsManager
   alias: sm
 - name: SecurityHub
-  alias: sh
-  excluded_regions:
-    - ap-northeast-3
+  alias: securityhub
 - name: Support
   global: true
   alias: support
@@ -116,16 +112,12 @@
   alias: ssm
 - name: GuardDuty
   alias: guardduty
-  excluded_regions:
-    - ap-northeast-3
 - name: Athena
   alias: athena
   excluded_regions:
     - ap-northeast-3
 - name: EFS
   alias: efs
-  excluded_regions:
-    - ap-northeast-3
 - name: Firehose
   alias: firehose
 - name: Lightsail
@@ -145,7 +137,6 @@
     - af-south-1
     - ap-east-1
     - ap-northeast-3
-    - ap-south-1
     - eu-north-1
     - eu-south-1
     - eu-west-3

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.5.0"
+  VERSION = "0.5.5"
 end

data/readme.md

@@ -1,6 +1,6 @@
 [![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/darkbitio/aws-recon/smoke-test/main)](https://github.com/darkbitio/aws-recon/actions?query=branch%3Amain)
 [![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://rubygems.org/gems/aws_recon)
-
+[![AWS Service Regions](https://github.com/darkbitio/aws-recon/actions/workflows/check-aws-regions.yml/badge.svg?branch=main&event=schedule)](https://github.com/darkbitio/aws-recon/actions/workflows/check-aws-regions.yml)
 # AWS Recon
 
 A multi-threaded AWS security-focused inventory collection tool written in Ruby.
@@ -54,13 +54,13 @@ To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.4.5.gem
+Fetching aws_recon-0.5.2.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.20.1.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.20.1
-Successfully installed aws_recon-0.4.5
+Successfully installed aws_recon-0.5.2
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -72,7 +72,7 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel-1.20.1
-Using aws_recon 0.4.5
+Using aws_recon 0.5.2
 ```
 
 ## Usage
@@ -249,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.4.5)
+AWS Recon - AWS Inventory Collector (0.5.2)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)

data/utils/aws/check_region_exclusions.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+#
+# Check regional service availability against services.yaml exclusions.
+#
+# AWS updates the regional service table daily. By checking regional service
+# coverage, we can identify regions that should be excluded from AWS Recon
+# checks. We exclude non-supported regions because service APIs handle non-
+# availability differently. Some will respond with an error that can be handled
+# by the errors defined in the AWS Ruby SDK client. Others will fail at the
+# network level (i.e. there is no API endpoint even available). We could handle
+# those errors and silently fail, but we choose not to so we can identify cases
+# where there is a lack of service availability in a particular region.
+#
+require 'net/http'
+require 'json'
+require 'yaml'
+
+TS = Time.now.to_i
+# AWS Regional services table
+URL = "https://api.regional-table.region-services.aws.a2z.com/index.json?timestamp=#{TS}000"
+
+service_to_query = ARGV[0]
+region_exclusion_mistmatch = nil
+
+#
+# load current AWS Recon regions
+#
+recon_services = YAML.safe_load(File.read('../../lib/aws_recon/services.yaml'))
+abort('Errors loading AWS Recon services') unless recon_services.is_a?(Array)
+
+#
+# load current AWS regions (non-gov, non-cn)
+#
+regions = YAML.safe_load(File.read('regions.yaml'))
+abort('Errors loading regions') unless regions['Regions']
+
+all_regions = regions['Regions'].map { |r| r['RegionName'] }
+
+#
+# get service/price list from AWS
+#
+uri = URI(URL)
+res = Net::HTTP.get_response(uri)
+abort('Error loading AWS services from API') unless res.code == '200'
+
+map = {}
+
+#
+# load service region availability
+#
+data = res.body
+json = JSON.parse(data)
+
+#
+# query regions for a single service
+#
+if service_to_query
+  single_service_regions = []
+
+  json['prices'].each do |p|
+    single_service_regions << p['id'].split(':').last
+  end
+
+  single_service_regions.uniq.sort.each { |r| puts r }
+
+  exit 0
+end
+
+# iterate through AWS provided services & regions
+json['prices'].each do |p|
+  at = p['attributes']
+  service_name = at['aws:serviceName']
+  service_id, service_region = p['id'].split(':')
+
+  # skip this service unless AWS Recon already has exclusions
+  next unless recon_services.filter { |s| s['alias'] == service_id }&.length&.positive?
+
+  if map.key?(service_name)
+    map[service_name]['regions'] << service_region
+  else
+    map[service_name] = {
+      'id' => service_id,
+      'regions' => [service_region]
+    }
+  end
+end
+
+# iterate through the services AWS Recon knows about
+map.sort.each do |k, v|
+  service_excluded_regions = all_regions.reject { |r| v['regions'].include?(r) }
+
+  aws_recon_service = recon_services.filter { |s| s['alias'] == v['id'] }&.first
+  aws_recon_service_excluded_regions = aws_recon_service['excluded_regions'] || []
+
+  # move on if AWS Recon region exclusions match AWS service region exclusions
+  next unless service_excluded_regions.sort != aws_recon_service_excluded_regions.sort
+
+  region_exclusion_mistmatch = true
+
+  puts "#{k} (#{v['id']})"
+
+  # determine the direction of the exclusion mismatch
+  if (service_excluded_regions - aws_recon_service_excluded_regions).length.positive?
+    puts " + missing region exclusion: #{(service_excluded_regions - aws_recon_service_excluded_regions).join(', ')}"
+  else
+    puts " - unnecessary region exclusion: #{(aws_recon_service_excluded_regions - service_excluded_regions).join(', ')}"
+  end
+end
+
+# exit code 1 if we have any mismatches
+exit 1 if region_exclusion_mistmatch

data/utils/aws/regions.yaml

@@ -0,0 +1,43 @@
+Regions:
+- Endpoint: ec2.af-south-1.amazonaws.com
+  RegionName: af-south-1
+- Endpoint: ec2.eu-north-1.amazonaws.com
+  RegionName: eu-north-1
+- Endpoint: ec2.ap-south-1.amazonaws.com
+  RegionName: ap-south-1
+- Endpoint: ec2.eu-west-3.amazonaws.com
+  RegionName: eu-west-3
+- Endpoint: ec2.eu-west-2.amazonaws.com
+  RegionName: eu-west-2
+- Endpoint: ec2.eu-south-1.amazonaws.com
+  RegionName: eu-south-1
+- Endpoint: ec2.eu-west-1.amazonaws.com
+  RegionName: eu-west-1
+- Endpoint: ec2.ap-northeast-3.amazonaws.com
+  RegionName: ap-northeast-3
+- Endpoint: ec2.ap-northeast-2.amazonaws.com
+  RegionName: ap-northeast-2
+- Endpoint: ec2.me-south-1.amazonaws.com
+  RegionName: me-south-1
+- Endpoint: ec2.ap-northeast-1.amazonaws.com
+  RegionName: ap-northeast-1
+- Endpoint: ec2.sa-east-1.amazonaws.com
+  RegionName: sa-east-1
+- Endpoint: ec2.ca-central-1.amazonaws.com
+  RegionName: ca-central-1
+- Endpoint: ec2.ap-east-1.amazonaws.com
+  RegionName: ap-east-1
+- Endpoint: ec2.ap-southeast-1.amazonaws.com
+  RegionName: ap-southeast-1
+- Endpoint: ec2.ap-southeast-2.amazonaws.com
+  RegionName: ap-southeast-2
+- Endpoint: ec2.eu-central-1.amazonaws.com
+  RegionName: eu-central-1
+- Endpoint: ec2.us-east-1.amazonaws.com
+  RegionName: us-east-1
+- Endpoint: ec2.us-east-2.amazonaws.com
+  RegionName: us-east-2
+- Endpoint: ec2.us-west-1.amazonaws.com
+  RegionName: us-west-1
+- Endpoint: ec2.us-west-2.amazonaws.com
+  RegionName: us-west-2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.5
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-06 00:00:00.000000000 Z
+date: 2021-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -163,6 +163,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/stale.yml"
+- ".github/workflows/check-aws-regions.yml"
 - ".github/workflows/docker-build.yml"
 - ".github/workflows/smoke-test.yml"
 - ".gitignore"
@@ -245,6 +246,8 @@ files:
 - lib/aws_recon/services.yaml
 - lib/aws_recon/version.rb
 - readme.md
+- utils/aws/check_region_exclusions.rb
+- utils/aws/regions.yaml
 - utils/cloudformation/aws-recon-cfn-template.yml
 - utils/terraform/cloudwatch.tf
 - utils/terraform/ecs.tf
@@ -267,7 +270,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="