aws_recon 0.2.20 → 0.2.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b945e5c44370658e13ee470fee26d7e0b46fb6102a661485be9513af305f45d
-  data.tar.gz: 969330b10b9a8264bdb5759b1c910b74ccf05e855f8f380cdebdac8b3694fb11
+  metadata.gz: 8786af60d08967c2203b1f4417480b135ae32b9a40ec05810afc9c2ae126f49d
+  data.tar.gz: fbd4741002a54dc15f2b2716dee6e1b6efcec48edfdf3e101d9c7bcb19105a4e
 SHA512:
-  metadata.gz: a7d26039b60a21370bd7000d7ccaca93cfc5efc7c0dba88ae5d7ea12d63d3a7b8f973b06d3a1a2633fb82804a716f163ffda23f96a00ee73251eb85956f14dfd
-  data.tar.gz: 61b1aa696e9ec44fd6b93d4ced975be8a2bb514537ed232db8a11cb840045bc52132a215a21a99e573f4df0b557b1612eb4dda458c249c0e5f86939b83049680
+  metadata.gz: 2c11e60d61cf9fcf894b8e38cb136e934b03fd7e9ef1bd349ce27c41c9943e75cb146893d14374167d7f2eb73b40d97a901fbaebd6e6aee5ac93ef8095b1d0b4
+  data.tar.gz: 9510a49029fbc382fefc50ad8ae0de0351b6f8648a1044abbebdf695459ef5a4845014057618387549546876c10ea1eb09cba55f4692a0405f41b96810c0dce5

data/lib/aws_recon/collectors/backup.rb

@@ -0,0 +1,25 @@
+class Backup < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # list_backup_plans
+    #
+    @client.list_protected_resources.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.results.each do |resource|
+        struct = OpenStruct.new(resource.to_h)
+        struct.type = 'protected_resource'
+        struct.arn = resource.resource_arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/codepipeline.rb

@@ -8,20 +8,37 @@ class CodePipeline < Mapper
     #
     # list_pipelines
     #
-    @client.list_pipelines.each_with_index do |response, page|
-      log(response.context.operation_name, page)
+    begin
+      @client.list_pipelines.each_with_index do |response, page|
+        log(response.context.operation_name, page)
 
-      # get_pipeline
-      response.pipelines.each do |pipeline|
-        resp = @client.get_pipeline(name: pipeline.name)
-        struct = OpenStruct.new(resp.pipeline.to_h)
-        struct.type = 'pipeline'
-        struct.arn = resp.metadata.pipeline_arn
+        # get_pipeline
+        response.pipelines.each do |pipeline|
+          resp = @client.get_pipeline(name: pipeline.name)
+          struct = OpenStruct.new(resp.pipeline.to_h)
+          struct.type = 'pipeline'
+          struct.arn = resp.metadata.pipeline_arn
 
-        resources.push(struct.to_h)
+          resources.push(struct.to_h)
+        end
+      end
+    rescue Aws::CodePipeline::Errors::ServiceError => e
+      log_error(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
       end
     end
 
     resources
   end
+
+  private
+
+  # not an error
+  def suppressed_errors
+    %w[
+      AccessDeniedException
+    ]
+  end
 end

data/lib/aws_recon/collectors/dms.rb

@@ -14,7 +14,7 @@ class DatabaseMigrationService < Mapper
       response.replication_instances.each do |instance|
         struct = OpenStruct.new(instance.to_h)
         struct.type = 'replication_instance'
-        struct.arb = "arn:aws:#{@service}:#{@region}::replication_instance/#{instance.replication_instance_identifier}"
+        struct.arn = "arn:aws:#{@service}:#{@region}::replication_instance/#{instance.replication_instance_identifier}"
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/dynamodb.rb

@@ -29,6 +29,7 @@ class DynamoDB < Mapper
         struct = OpenStruct.new(@client.describe_table({ table_name: table_name }).table.to_h)
         struct.type = 'table'
         struct.arn = struct.table_arn
+        struct.continuous_backups_description = @client.describe_continuous_backups({ table_name: table_name }).continuous_backups_description.to_h
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/ecr.rb

@@ -16,10 +16,14 @@ class ECR < Mapper
         struct.type = 'repository'
         struct.arn = repo.repository_arn
         struct.policy = @client
-                        .get_repository_policy({ repository_name: repo.repository_name }).to_h
+                        .get_repository_policy({ repository_name: repo.repository_name }).policy_text.parse_policy
 
       rescue Aws::ECR::Errors::ServiceError => e
-        raise e unless suppressed_errors.include?(e.code)
+        log_error(e.code)
+
+        unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+          raise e
+        end
       ensure
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/emr.rb

@@ -0,0 +1,39 @@
+class EMR < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # get_block_public_access_configuration
+    #
+    @client.get_block_public_access_configuration.each do |response|
+      log(response.context.operation_name)
+
+      struct = OpenStruct.new(response.block_public_access_configuration.to_h)
+      struct.type = 'configuration'
+
+      resources.push(struct.to_h)
+    end
+
+    #
+    # list_clusters
+    #
+    @client.list_clusters.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.clusters.each do |cluster|
+        log(response.context.operation_name, cluster.id)
+
+        struct = OpenStruct.new(@client.describe_cluster({ cluster_id: cluster.id }).cluster.to_h)
+        struct.type = 'cluster'
+        struct.arn = cluster.cluster_arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/iam.rb

@@ -89,14 +89,22 @@ class IAM < Mapper
     #
     # get_account_password_policy
     #
-    @client.get_account_password_policy.each do |response|
-      log(response.context.operation_name)
+    begin
+      @client.get_account_password_policy.each do |response|
+        log(response.context.operation_name)
 
-      struct = OpenStruct.new(response.password_policy.to_h)
-      struct.type = 'password_policy'
-      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
+        struct = OpenStruct.new(response.password_policy.to_h)
+        struct.type = 'password_policy'
+        struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
-      resources.push(struct.to_h)
+        resources.push(struct.to_h)
+      end
+    rescue Aws::IAM::Errors::ServiceError => e
+      log_error(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
+      end
     end
 
     #
@@ -178,7 +186,10 @@ class IAM < Mapper
       end
     rescue Aws::IAM::Errors::ServiceError => e
       log_error(e.code)
-      raise e unless suppressed_errors.include?(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
+      end
     end
 
     resources
@@ -190,6 +201,7 @@ class IAM < Mapper
   def suppressed_errors
     %w[
       ReportNotPresent
+      NoSuchEntity
     ]
   end
 end

data/lib/aws_recon/collectors/kms.rb

@@ -29,7 +29,10 @@ class KMS < Mapper
                                     .key_rotation_enabled
         rescue Aws::KMS::Errors::ServiceError => e
           log_error(e.code)
-          raise e unless suppressed_errors.include?(e.code)
+
+          unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+            raise e
+          end
         end
 
         # list_grants

data/lib/aws_recon/collectors/organizations.rb

@@ -34,18 +34,35 @@ class Organizations < Mapper
     #
     # list_policies
     #
-    @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page|
-      log(response.context.operation_name, page)
+    begin
+      @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page|
+        log(response.context.operation_name, page)
 
-      response.policies.each do |policy|
-        struct = OpenStruct.new(policy.to_h)
-        struct.type = 'service_control_policy'
-        struct.content = @client.describe_policy({ policy_id: policy.id }).policy.content.parse_policy
+        response.policies.each do |policy|
+          struct = OpenStruct.new(policy.to_h)
+          struct.type = 'service_control_policy'
+          struct.content = @client.describe_policy({ policy_id: policy.id }).policy.content.parse_policy
 
-        resources.push(struct.to_h)
+          resources.push(struct.to_h)
+        end
+      end
+    rescue Aws::Organizations::Errors::ServiceError => e
+      log_error(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
       end
     end
 
     resources
   end
+
+  private
+
+  # not an error
+  def suppressed_errors
+    %w[
+      AccessDeniedException
+    ]
+  end
 end

data/lib/aws_recon/collectors/rds.rb

@@ -38,6 +38,8 @@ class RDS < Mapper
         struct.arn = instance.db_instance_arn
         struct.parent_id = instance.db_cluster_identifier
 
+        # TODO: describe_db_snapshots here (with public flag)
+
         resources.push(struct.to_h)
       end
     end

data/lib/aws_recon/collectors/redshift.rb

@@ -15,6 +15,7 @@ class Redshift < Mapper
         struct = OpenStruct.new(cluster.to_h)
         struct.type = 'cluster'
         struct.arn = cluster.cluster_identifier
+        struct.logging_status = @client.describe_logging_status({ cluster_identifier: cluster.cluster_identifier }).to_h
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/s3.rb

@@ -61,7 +61,11 @@ class S3 < Mapper
                            end
 
         rescue Aws::S3::Errors::ServiceError => e
-          raise e unless suppressed_errors.include?(e.code)
+          log_error(e.code)
+
+          unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+            raise e
+          end
         end
 
         resources.push(struct.to_h)

data/lib/aws_recon/collectors/securityhub.rb

@@ -8,16 +8,33 @@ class SecurityHub < Mapper
     #
     # describe_hub
     #
-    @client.describe_hub.each do |response|
-      log(response.context.operation_name)
+    begin
+      @client.describe_hub.each do |response|
+        log(response.context.operation_name)
 
-      struct = OpenStruct.new(response.to_h)
-      struct.type = 'hub'
-      struct.arn = response.hub_arn
+        struct = OpenStruct.new(response.to_h)
+        struct.type = 'hub'
+        struct.arn = response.hub_arn
 
-      resources.push(struct.to_h)
+        resources.push(struct.to_h)
+      end
+    rescue Aws::SecurityHub::Errors::ServiceError => e
+      log_error(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
+      end
     end
 
     resources
   end
+
+  private
+
+  # not an error
+  def suppressed_errors
+    %w[
+      InvalidAccessException
+    ]
+  end
 end

data/lib/aws_recon/collectors/servicequotas.rb

@@ -27,7 +27,10 @@ class ServiceQuotas < Mapper
       end
     rescue Aws::ServiceQuotas::Errors::ServiceError => e
       log_error(e.code, service)
-      raise e unless suppressed_errors.include?(e.code)
+
+      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+        raise e
+      end
     end
 
     resources

data/lib/aws_recon/collectors/shield.rb

@@ -51,7 +51,10 @@ class Shield < Mapper
     resources
   rescue Aws::Shield::Errors::ServiceError => e
     log_error(e.code)
-    raise e unless suppressed_errors.include?(e.code)
+
+    unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+      raise e
+    end
 
     [] # no access or service isn't enabled
   end

data/lib/aws_recon/collectors/support.rb

@@ -27,7 +27,10 @@ class Support < Mapper
     resources
   rescue Aws::Support::Errors::ServiceError => e
     log_error(e.code)
-    raise e unless suppressed_errors.include?(e.code)
+
+    unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
+      raise e
+    end
 
     [] # no Support subscription
   end
@@ -37,6 +40,7 @@ class Support < Mapper
   # not an error
   def suppressed_errors
     %w[
+      AccessDeniedException
       SubscriptionRequiredException
     ]
   end

data/lib/aws_recon/options.rb

@@ -20,6 +20,7 @@ class Parser
     :skip_credential_report,
     :stream_output,
     :verbose,
+    :quit_on_exception,
     :debug
   )
 
@@ -47,6 +48,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -135,6 +137,11 @@ class Parser
         args.verbose = true unless args.stream_output
       end
 
+      # re-raise exceptions
+      opts.on('-q', '--quit-on-exception', 'Stop collection if an API error is encountered (default: false)') do
+        args.quit_on_exception = true
+      end
+
       # debug
       opts.on('-d', '--debug', 'Output debug with wire trace info') do
         unless args.stream_output

data/lib/aws_recon/services.yaml

@@ -6,12 +6,19 @@
   alias: aa
 - name: ApplicationAutoScaling
   alias: aas
+- name: Backup
+  alias: backup
 - name: ConfigService
   alias: config
 - name: CodeBuild
   alias: codebuild
+  excluded_regions:
+    - af-south-1
 - name: CodePipeline
   alias: codepipeline
+  excluded_regions:
+    - af-south-1
+    - me-south-1
 - name: AutoScaling
   alias: autoscaling
 - name: CloudTrail
@@ -37,6 +44,13 @@
     - ap-southeast-1
 - name: ElastiCache
   alias: elasticache
+- name: EMR
+  alias: emr
+  excluded_regions:
+    - ap-east-1
+    - af-south-1
+    - eu-south-1
+    - me-south-1
 - name: IAM
   global: true
   alias: iam
@@ -85,12 +99,17 @@
     - eu-north-1
     - eu-west-3
     - us-west-1
+    - ap-east-1
+    - af-south-1
+    - eu-south-1
 - name: CloudWatch
   alias: cloudwatch
 - name: CloudWatchLogs
   alias: cloudwatchlogs
 - name: Kafka
   alias: kafka
+  excluded_regions:
+    - af-south-1
 - name: SecretsManager
   alias: sm
 - name: SecurityHub
@@ -116,6 +135,10 @@
     - eu-north-1
     - us-west-1
     - sa-east-1
+    - ap-east-1
+    - af-south-1
+    - eu-south-1
+    - me-south-1
 - name: WorkSpaces
   alias: workspaces
   excluded_regions:
@@ -124,12 +147,21 @@
     - eu-west-3
     - us-east-2
     - us-west-1
+    - ap-east-1
+    - af-south-1
+    - eu-south-1
+    - me-south-1
 - name: SageMaker
   alias: sagemaker
 - name: ServiceQuotas
   alias: servicequotas
 - name: Transfer
   alias: transfer
+  excluded_regions:
+    - ap-east-1
+    - af-south-1
+    - eu-south-1
+    - me-south-1
 - name: DirectConnect
   alias: dc
 - name: DirectoryService

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.20"
+  VERSION = "0.2.25"
 end

data/readme.md

@@ -215,6 +215,20 @@ Usage: aws_recon [options]
 
 Output is always some form of JSON - either JSON lines or plain JSON. The output is either written to a file (the default), or written to stdout (with `-j`).
 
+## Support for Manually Enabled Regions
+
+If you have enabled manually enabled regions:
+
+- me-south-1 - Middle East (Bahrain)
+- af-south-1 - Africa (Cape Town)
+- ap-east-1 - Asia Pacific (Hong Kong)
+- eu-south-1 - Europe (Milan)
+
+and you are using STS to assume a role into an account, you will need to [enable v2 STS tokens](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) in the account you are assuming the role **from** to be able to run AWS Recon against those regions.
+
+> Version 1 tokens are valid only in AWS Regions that are available by default. These tokens do not work in manually enabled Regions, such as Asia Pacific (Hong Kong). Version 2 tokens are valid in all Regions. However, version 2 tokens are longer and might affect systems where you temporarily store tokens.
+
+If you are using a static access key/secret, you can collect from these regions with either `v1` or `v2` STS tokens.
 
 ## Supported Services & Resources
 
@@ -226,6 +240,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] AdvancedShield
 - [x] ApplicationAutoScaling
 - [x] Athena
+- [x] Backup
 - [x] GuardDuty
 - [ ] Macie
 - [x] Systems Manager
@@ -249,8 +264,9 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] ECR
 - [x] ECS
 - [x] EFS
-- [x] ELB
 - [x] EKS
+- [x] ELB
+- [x] EMR
 - [x] Elasticsearch
 - [x] ElastiCache
 - [x] Firehose

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.2.20
+  version: 0.2.25
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-26 00:00:00.000000000 Z
+date: 2020-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -187,6 +187,7 @@ files:
 - lib/aws_recon/collectors/applicationautoscaling.rb
 - lib/aws_recon/collectors/athena.rb
 - lib/aws_recon/collectors/autoscaling.rb
+- lib/aws_recon/collectors/backup.rb
 - lib/aws_recon/collectors/cloudformation.rb
 - lib/aws_recon/collectors/cloudfront.rb
 - lib/aws_recon/collectors/cloudtrail.rb
@@ -208,6 +209,7 @@ files:
 - lib/aws_recon/collectors/elasticloadbalancing.rb
 - lib/aws_recon/collectors/elasticloadbalancingv2.rb
 - lib/aws_recon/collectors/elasticsearch.rb
+- lib/aws_recon/collectors/emr.rb
 - lib/aws_recon/collectors/firehose.rb
 - lib/aws_recon/collectors/guardduty.rb
 - lib/aws_recon/collectors/iam.rb