aws_recon 0.2.25 → 0.2.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8786af60d08967c2203b1f4417480b135ae32b9a40ec05810afc9c2ae126f49d
-  data.tar.gz: fbd4741002a54dc15f2b2716dee6e1b6efcec48edfdf3e101d9c7bcb19105a4e
+  metadata.gz: 818784fdc3d39b16bc30cafd043aec9396f3a8bb0e1fb62b696deb40a861d953
+  data.tar.gz: ebd7c961414a16e7cfeb60e8f3f871051bfd0705cc7a5408cc8fdd054d5f7925
 SHA512:
-  metadata.gz: 2c11e60d61cf9fcf894b8e38cb136e934b03fd7e9ef1bd349ce27c41c9943e75cb146893d14374167d7f2eb73b40d97a901fbaebd6e6aee5ac93ef8095b1d0b4
-  data.tar.gz: 9510a49029fbc382fefc50ad8ae0de0351b6f8648a1044abbebdf695459ef5a4845014057618387549546876c10ea1eb09cba55f4692a0405f41b96810c0dce5
+  metadata.gz: 52fc427274d6ac70c8565eb139681a3c0e0b243db9ee1acaced741fcc688496ce383629755d6e3aa49ec1bfdf47891adce77b88a2a095f000245461ea2508f0d
+  data.tar.gz: e6f4fe0b53ea000b766c94f8eaafa7dddf33086916419ae14fd65db7218ecb853d85d94e546c1be086ec48d4837b5fa64708326f7c2e8517a04602ff7b4dd9e7

data/.rubocop.yml

@@ -9,4 +9,20 @@
 #
 # See https://docs.rubocop.org/rubocop/configuration
 Layout/LineLength:
-  Max: 80
+  Max: 100
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: always_true
+  Safe: true
+  SafeAutoCorrect: true
+Style/ClassAndModuleChildren:
+  Enabled: false
+Metrics/BlockLength:
+  Enabled: false
+Metrics/MethodLength:
+  Enabled: false
+Metrics/PerceivedComplexity:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Metrics/AbcSize:
+  Enabled: false

data/lib/aws_recon/collectors/accessanalyzer.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect AccessAnalyzer resources
+#
 class AccessAnalyzer < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/acm.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ACM resources
+#
 class ACM < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/apigateway.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect API Gateway resources
+#
 class APIGateway < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/apigatewayv2.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect API Gateway v2 resources
+#
 class ApiGatewayV2 < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/applicationautoscaling.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ApplicationAutoScaling resources
+#
 class ApplicationAutoScaling < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/athena.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Athena resources
+#
 class Athena < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/autoscaling.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect AutoScaling resources
+#
 class AutoScaling < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/backup.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Backup resources
+#
 class Backup < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/cloudformation.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CloudFormation resources
+#
 class CloudFormation < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/cloudfront.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CloudFront resources
+#
 class CloudFront < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/cloudtrail.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CloudTrail resources
+#
 class CloudTrail < Mapper
   #
   # Returns an array of resources.
@@ -19,7 +24,7 @@ class CloudTrail < Mapper
                  end
 
         struct = OpenStruct.new(trail.to_h)
-        struct.tags = client.list_tags({ resource_id_list: [trail.trail_arn] }).resource_tag_list.first.tags_list
+        struct.tags = client.list_tags({ resource_id_list: [trail.trail_arn] }).resource_tag_list.first.tags_list.map(&:to_h)
         struct.type = 'cloud_trail'
         struct.event_selectors = client.get_event_selectors({ trail_name: trail.name }).to_h
         struct.status = client.get_trail_status({ name: trail.name }).to_h

data/lib/aws_recon/collectors/cloudwatch.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CloudWatch resources
+#
 class CloudWatch < Mapper
   #
   # Returns an array of resources.
@@ -23,6 +28,7 @@ class CloudWatch < Mapper
         struct = OpenStruct.new(alarm.to_h)
         struct.type = 'metric_alarm'
         struct.arn = alarm.alarm_arn
+        struct.state_reason_data = alarm.state_reason_data&.parse_policy
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/cloudwatchlogs.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CloudWatchLogs resources
+#
 class CloudWatchLogs < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/codebuild.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CodeBuild resources
+#
 class CodeBuild < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/codepipeline.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect CodePipeline resources
+#
 class CodePipeline < Mapper
   #
   # Returns an array of resources.
@@ -25,9 +30,7 @@ class CodePipeline < Mapper
     rescue Aws::CodePipeline::Errors::ServiceError => e
       log_error(e.code)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     resources

data/lib/aws_recon/collectors/configservice.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Config resources
+#
 class ConfigService < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/directconnect.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect DirectConnect resources
+#
 class DirectConnect < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/{directyservice.rb → directoryservice.rb}

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect DirectoryService resources
+#
 class DirectoryService < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/dms.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect DMS resources
+#
 class DatabaseMigrationService < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/dynamodb.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect DynamodDB resources
+#
 class DynamoDB < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/ec2.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect EC2 resources
+#
 class EC2 < Mapper
   #
   # Returns an array of resources.
@@ -71,9 +76,7 @@ class EC2 < Mapper
               if user_data_raw
                 user_data = Base64.decode64(user_data_raw)
 
-                if user_data.force_encoding('UTF-8').ascii_only?
-                  struct.user_data = user_data
-                end
+                struct.user_data = user_data if user_data.force_encoding('UTF-8').ascii_only?
               end
             end
 

data/lib/aws_recon/collectors/ecr.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ECR resources
+#
 class ECR < Mapper
   #
   # Returns an array of resources.
@@ -21,9 +26,7 @@ class ECR < Mapper
       rescue Aws::ECR::Errors::ServiceError => e
         log_error(e.code)
 
-        unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-          raise e
-        end
+        raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
       ensure
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/ecs.rb

@@ -1,31 +1,34 @@
+# frozen_string_literal: true
+
+#
+# Collect ECS resources
+#
 class ECS < Mapper
   #
   # Returns an array of resources.
   #
-  # TODO: test live
-  #
   def collect
     resources = []
 
     #
-    # describe_clusters
+    # list_clusters
     #
-    @client.describe_clusters.each_with_index do |response, page|
+    @client.list_clusters.each_with_index do |response, page|
       log(response.context.operation_name, page)
 
-      response.clusters.each do |cluster|
-        struct = OpenStruct.new(cluster.to_h)
+      response.cluster_arns.each do |cluster|
+        struct = OpenStruct.new(@client.describe_clusters({ clusters: [cluster] }).clusters.first.to_h)
         struct.type = 'cluster'
-        struct.arn = cluster.cluster_arn
+        struct.arn = cluster
         struct.tasks = []
 
         # list_tasks
-        @client.list_tasks({ cluster: cluster.cluster_arn }).each_with_index do |response, page|
+        @client.list_tasks({ cluster: cluster }).each_with_index do |response, page|
           log(response.context.operation_name, 'list_tasks', page)
 
           # describe_tasks
           response.task_arns.each do |task_arn|
-            @client.describe_tasks({ cluster: cluster.cluster_arn, tasks: [task_arn] }).tasks.each do |task|
+            @client.describe_tasks({ cluster: cluster, tasks: [task_arn] }).tasks.each do |task|
               struct.tasks.push(task)
             end
           end

data/lib/aws_recon/collectors/efs.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect EFS resources
+#
 class EFS < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/eks.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect EKS resources
+#
 class EKS < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/elasticache.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ElastiCache resources
+#
 class ElastiCache < Mapper
   def collect
     resources = []

data/lib/aws_recon/collectors/elasticloadbalancing.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ELB resources
+#
 class ElasticLoadBalancing < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/elasticloadbalancingv2.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ELBv2 resources
+#
 class ElasticLoadBalancingV2 < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/elasticsearch.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ElasticSearch resources
+#
 class ElasticsearchService < Mapper
   #
   # Returns an array of resources.
@@ -17,6 +22,7 @@ class ElasticsearchService < Mapper
         # describe_elasticsearch_domains
         struct = OpenStruct.new(@client.describe_elasticsearch_domain({ domain_name: domain.domain_name }).domain_status.to_h)
         struct.type = 'domain'
+        struct.access_policies = struct.access_policies&.parse_policy
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/emr.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect EMR resources
+#
 class EMR < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/firehose.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Firehose resources
+#
 class Firehose < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/guardduty.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
+#
+# Collect GuardDuty resources
+#
 class GuardDuty < Mapper
   #
   # Returns an array of resources.
   #
-  # TODO: test live
-  #
   def collect
     resources = []
 

data/lib/aws_recon/collectors/iam.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect IAM resources
+#
 class IAM < Mapper
   #
   # Returns an array of resources.
@@ -102,9 +107,7 @@ class IAM < Mapper
     rescue Aws::IAM::Errors::ServiceError => e
       log_error(e.code)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     #
@@ -187,9 +190,7 @@ class IAM < Mapper
     rescue Aws::IAM::Errors::ServiceError => e
       log_error(e.code)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     resources

data/lib/aws_recon/collectors/kafka.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Kafka resources
+#
 class Kafka < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/kinesis.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Kinesis resources
+#
 class Kinesis < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/kms.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect KMS resources
+#
 class KMS < Mapper
   #
   # Returns an array of resources.
@@ -30,9 +35,7 @@ class KMS < Mapper
         rescue Aws::KMS::Errors::ServiceError => e
           log_error(e.code)
 
-          unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-            raise e
-          end
+          raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
         end
 
         # list_grants
@@ -48,7 +51,7 @@ class KMS < Mapper
         log(response.context.operation_name, 'get_key_policy')
         struct.policy = @client
                         .get_key_policy({ key_id: key.key_id, policy_name: 'default' })
-                        .policy
+                        .policy.parse_policy
 
         # list_aliases
         log(response.context.operation_name, 'list_aliases')

data/lib/aws_recon/collectors/lambda.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Lambda resources
+#
 class Lambda < Mapper
   def collect
     resources = []

data/lib/aws_recon/collectors/lightsail.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Lightsail resources
+#
 class Lightsail < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/organizations.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Org resources
+#
 class Organizations < Mapper
   #
   # Returns an array of resources.
@@ -49,9 +54,7 @@ class Organizations < Mapper
     rescue Aws::Organizations::Errors::ServiceError => e
       log_error(e.code)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     resources

data/lib/aws_recon/collectors/rds.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect RDS Resources
+#
 class RDS < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/redshift.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Redshift resources
+#
 class Redshift < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/route53.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Route53 resources
+#
 class Route53 < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/route53domains.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Route53 Domain resources
+#
 class Route53Domains < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/s3.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect S3 Resources
+#
 class S3 < Mapper
   #
   # Returns an array of resources.
@@ -63,9 +68,7 @@ class S3 < Mapper
         rescue Aws::S3::Errors::ServiceError => e
           log_error(e.code)
 
-          unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-            raise e
-          end
+          raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
         end
 
         resources.push(struct.to_h)

data/lib/aws_recon/collectors/sagemaker.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect SageMaker Resources
+#
 class SageMaker < Mapper
   #
   # Returns an array of resources.
@@ -12,7 +17,9 @@ class SageMaker < Mapper
       log(response.context.operation_name, page)
 
       response.notebook_instances.each do |instance|
-        struct = OpenStruct.new(instance.to_h)
+        struct = OpenStruct.new(@client.describe_notebook_instance({
+                                                                     notebook_instance_name: instance.notebook_instance_name
+                                                                   }).to_h)
         struct.type = 'notebook_instance'
         struct.arn = instance.notebook_instance_arn
 
@@ -20,6 +27,23 @@ class SageMaker < Mapper
       end
     end
 
+    #
+    # list_endpoints
+    #
+    @client.list_endpoints.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.endpoints.each do |instance|
+        struct = OpenStruct.new(@client.describe_endpoint({
+                                                            endpoint_name: instance.endpoint_name
+                                                          }).to_h)
+        struct.type = 'endpoint'
+        struct.arn = instance.endpoint_arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
     resources
   end
 end

data/lib/aws_recon/collectors/secretsmanager.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Secrets Manager resources
+#
 class SecretsManager < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/securityhub.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Security Hub resources
+#
 class SecurityHub < Mapper
   #
   # Returns an array of resources.
@@ -21,9 +26,7 @@ class SecurityHub < Mapper
     rescue Aws::SecurityHub::Errors::ServiceError => e
       log_error(e.code)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     resources

data/lib/aws_recon/collectors/servicequotas.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect ServiceQuota resources
+#
 class ServiceQuotas < Mapper
   #
   # Returns an array of resources.
@@ -28,9 +33,7 @@ class ServiceQuotas < Mapper
     rescue Aws::ServiceQuotas::Errors::ServiceError => e
       log_error(e.code, service)
 
-      unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-        raise e
-      end
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     resources

data/lib/aws_recon/collectors/ses.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect SES resources
+#
 class SES < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/shield.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Shield resources
+#
 class Shield < Mapper
   #
   # Returns an array of resources.
@@ -27,7 +32,7 @@ class Shield < Mapper
       struct = OpenStruct.new
       struct.type = 'contact_list'
       struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
-      struct.contacts = response.emergency_contact_list.map(&:to_h)
+      struct.contacts = response&.emergency_contact_list&.map(&:to_h)
 
       resources.push(struct.to_h)
     end
@@ -52,9 +57,7 @@ class Shield < Mapper
   rescue Aws::Shield::Errors::ServiceError => e
     log_error(e.code)
 
-    unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-      raise e
-    end
+    raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
 
     [] # no access or service isn't enabled
   end

data/lib/aws_recon/collectors/sns.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect SNS resources
+#
 class SNS < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/sqs.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect SQS resources
+#
 class SQS < Mapper
   #
   # Returns an array of resources.
@@ -18,7 +23,7 @@ class SQS < Mapper
         struct = OpenStruct.new(@client.get_queue_attributes({ queue_url: queue, attribute_names: ['All'] }).attributes.to_h)
         struct.type = 'queue'
         struct.arn = struct.QueueArn
-        struct.policy = struct.delete_field('Policy').parse_policy
+        struct.policy = struct.Policy ? struct.delete_field('Policy').parse_policy : nil
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/collectors/ssm.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect SSM resources
+#
 class SSM < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/support.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Support resources
+#
 class Support < Mapper
   #
   # Returns an array of resources.
@@ -28,9 +33,7 @@ class Support < Mapper
   rescue Aws::Support::Errors::ServiceError => e
     log_error(e.code)
 
-    unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
-      raise e
-    end
+    raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
 
     [] # no Support subscription
   end

data/lib/aws_recon/collectors/transfer.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect Transfer resources
+#
 class Transfer < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/wafv2.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect WAFv2 resources
+#
 class WAFV2 < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/workspaces.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect WorkSpaces resources
+#
 class WorkSpaces < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/collectors/xray.rb

@@ -1,3 +1,8 @@
+# frozen_string_literal: true
+
+#
+# Collect XRay resources
+#
 class XRay < Mapper
   #
   # Returns an array of resources.

data/lib/aws_recon/lib/mapper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Generic wrapper for service clients.
 #
@@ -64,14 +66,14 @@ class Mapper
   end
 
   def log(*msg)
-    if @options.verbose
-      puts _msg(msg).map { |x| "\x1b[32m#{x}\x1b[0m" }.join("\x1b[35m.\x1b[0m")
-    end
+    return unless @options.verbose
+
+    puts _msg(msg).map { |x| "\x1b[32m#{x}\x1b[0m" }.join("\x1b[35m.\x1b[0m")
   end
 
   def log_error(*msg)
-    if @options.verbose
-      puts _msg(msg).map { |x| "\x1b[35m#{x}\x1b[0m" }.join("\x1b[32m.\x1b[0m")
-    end
+    return unless @options.verbose
+
+    puts _msg(msg).map { |x| "\x1b[35m#{x}\x1b[0m" }.join("\x1b[32m.\x1b[0m")
   end
 end

data/lib/aws_recon/lib/patch.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Parse and unescape AWS policy document string
 #

data/lib/aws_recon/options.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+#
+# Command line options parser
+#
 class Parser
   DEFAULT_CONFIG_FILE = nil
   DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze
@@ -97,16 +100,12 @@ class Parser
 
       # output format
       opts.on('-f', '--format [FORMAT]', 'Specify output format (default: aws)') do |file|
-        if %w[aws custom].include?(file.downcase)
-          args.output_format = file.downcase
-        end
+        args.output_format = file.downcase if %w[aws custom].include?(file.downcase)
       end
 
       # threads
       opts.on('-t', '--threads [THREADS]', "Specify max threads (default: #{Parser::DEFAULT_THREADS}, max: 128)") do |threads|
-        if (0..Parser::MAX_THREADS).include?(threads.to_i)
-          args.threads = threads.to_i
-        end
+        args.threads = threads.to_i if (0..Parser::MAX_THREADS).include?(threads.to_i)
       end
 
       # collect EC2 instance user data

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.25"
+  VERSION = "0.2.30"
 end

data/readme.md

@@ -7,9 +7,9 @@ A multi-threaded AWS inventory collection tool.
 
 The [creators](https://darkbit.io) of this tool have a recurring need to be able to efficiently collect a large amount of AWS resource attributes and metadata to help clients understand their cloud security posture.
 
-There are a handful of tools (e.g. [AWS Config](https://aws.amazon.com/config), [CloudMapper](https://github.com/duo-labs/cloudmapper), [CloudSploit](https://github.com/cloudsploit/scans), [Prowler](https://github.com/toniblyx/prowler)) that do some form of resource collection to support other functions. But we found we needed broader coverage and more details at a per-service level. We also needed a consistent and structured format that allowed for integration with our other systems and tooling.
+Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity we needed. We also needed a tool that produced consistent output that was easily consumed by other tools/systems.
 
-Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain Ruby. Though most AWS tooling tends to be dominated by Python, the [Ruby SDK](https://aws.amazon.com/sdk-for-ruby/) is quite mature and capable. The maintainers of the Ruby SDK have done a fantastic job making it easy to handle automatic retries, paging of large responses, and threading huge numbers of requests.
+Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain Ruby. Though Python tends to dominate the AWS tooling landscape, the [Ruby SDK](https://aws.amazon.com/sdk-for-ruby/) has a few convenient advantages over the [other](https://aws.amazon.com/sdk-for-node-js/) [AWS](https://aws.amazon.com/sdk-for-python/) [SDKs](https://aws.amazon.com/sdk-for-go/) we tested. Specifically, easy handling of automatic retries, paging of large responses, and - with some help - threading huge numbers of requests.
 
 ## Project Goals
 
@@ -23,24 +23,44 @@ Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain R
 
 ### Requirements
 
-Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
+AWS Recon needs an AWS account role or credentials with `ReadOnlyAccess`. Full `AdministratorAccess` is over-privileged, but will work as well. The `SecurityAudit` policy is **not** sufficient as it omits access to many services.
+
+#### Running via Docker
+
+Use Docker version 19.x or above to run the pre-built image without having to install anything.
+
+#### Running locally via Ruby
+
+If you already have Ruby installed (2.5.x or 2.6.x), you may want to install the Ruby gem.
 
 ### Installation
 
-AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+AWS Recon can be run locally via a Docker container or by installing the Ruby gem.
+
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
+```
 
 To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.8.gem
-Fetching aws-sdk-resources-3.76.0.gem
+Fetching aws_recon-0.2.28.gem
 Fetching aws-sdk-3.0.1.gem
-Fetching parallel-1.19.2.gem
+Fetching parallel-1.20.1.gem
 ...
 Successfully installed aws-sdk-3.0.1
-Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.8
+Successfully installed parallel-1.20.1
+Successfully installed aws_recon-0.2.28
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -51,27 +71,13 @@ Fetching gem metadata from https://rubygems.org/
 Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
-Using parallel 1.19.2
-Using aws_recon 0.2.8
+Using parallel-1.20.1
+Using aws_recon 0.2.28
 ```
 
-To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
-
-```
-$ docker run -t --rm \
-  -e AWS_REGION \
-  -e AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY \
-  -e AWS_SESSION_TOKEN \
-  -v $(pwd)/output.json:/recon/output.json \
-  darkbitio/aws_recon:latest \
-  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
-```
-
-
 ## Usage
 
-AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
+AWS Recon will leverage any AWS credentials (see [requirements](#requirements)) currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials.
 
 ```
 $ aws-vault exec profile -- aws_recon
@@ -95,7 +101,7 @@ $ aws-vault exec <vault_profile> -- docker run -t --rm \
   aws_recon -j -s EC2 -r global,us-east-1,us-east-2
 ```
 
-To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
+To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because only that one file is mounted into the Docker container at run time. For example:
 
 Create an empty file.
 
@@ -159,22 +165,44 @@ $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
-Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted output.
+Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted (NDJSON) output.
 
 ```
-$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2 -f custom > output.json
+$ AWS_PROFILE=<profile> aws_recon -j \
+  -s S3,EC2 \
+  -r global,us-east-1,us-east-2 \
+  -f custom > output.json
 ```
 
 #### Errors
 
-An exception will be raised on `AccessDeniedException` errors. This typically means your user/role doesn't have the necessary permissions to get/list/describe for that service. These exceptions are raised so troubleshooting access issues is easier.
+API exceptions related to permissions are silently ignored in most cases. These errors are usually due to one of these cases:
+
+- using a role without sufficient permissions
+- querying an account with SCPs in place that prevent usage of certain services
+- trying to query a service that isn't enabled/available in your region/account
+
+In `verbose` mode, you will see exception logs in the output:
+
+```
+t2.us-east-1.EC2.describe_subnets.0
+t4.us-east-1.SSM.describe_instance_information.0
+t6.us-east-1.SecurityHub.InvalidAccessException   <-----
+t2.us-east-1.EC2.describe_addresses.0
+t4.us-east-1.SSM.describe_parameters.0
+t1.us-east-1.GuardDuty.list_detectors.0
+```
+
+Use the `-q` command line option to re-raise these exceptions so troubleshooting access issues is easier.
 
 ```
 Traceback (most recent call last):
-arn:aws:sts::1234567890:assumed-role/role/9876543210 is not authorized to perform: codepipeline:GetPipeline on resource: arn:aws:codepipeline:us-west-2:876543210123:pipeline (Aws::CodePipeline::Errors::AccessDeniedException)
+arn:aws:sts::1234567890:assumed-role/role/my-audit-role is not authorized to perform:
+ codepipeline:GetPipeline on resource: arn:aws:codepipeline:us-west-2:1234567890:pipeline
+ (Aws::CodePipeline::Errors::AccessDeniedException)
 ```
 
-The exact API operation that triggered the exception is indicated on the last line of the stack trace. If you can't resolve the necessary access, you should exclude those services with `-x` or `--not-services` so the collection can continue.
+The exact API operation that triggered the exception is indicated on the last line of the stack trace. If you can't resolve the necessary access, you should exclude those services with `-x` or `--not-services`, or leave off the `-q` option so the collection can continue.
 
 ### Threads
 
@@ -184,14 +212,20 @@ For global services like IAM, Shield, and Support, requests are not multi-thread
 
 For regional services, a thread (up to the thread limit) is spawned for each service in a region. By default, up to 8 threads will be used. If your account has resources spread across many regions, you may see a speed improvement by increasing threads with `-t X`, where `X` is the number of threads.
 
+### Performance
+
+AWS Recon will make a minimum of ~2,000 API calls in a new/empty account, just to query the supported services in all 20 standard (non-GovCloud, non-China) regions. It is very likely to encounter API rate-limiting (throttling) on large accounts if you enable more threads than the default (8).
+
+Recon will automatically backoff and respect the retry limits in the API response. If you observe long pauses during collection, this is likely what is happening. Retry collection with the `-d` or `--debug` option to observe the wire trace and see if you're being throttled. Consider using fewer threads or requesting higher rate limits from AWS if you are regularly getting rate-limited.
+
 ### Options
 
-Most users will want to limit collection to relevant services and regions. Running without any options will attempt to collect all resources from all 16 regular regions.
+Most users will want to limit collection to relevant services and regions. Running without any exclusions will attempt to collect all resources from all regions enabled for the account.
 
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.2.8)
+AWS Recon - AWS Inventory Collector (0.2.28)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -204,8 +238,10 @@ Usage: aws_recon [options]
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
     -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
+    -g, --skip-credential-report     Skip generating IAM credential report (default: false)
     -j, --stream-output              Stream JSON lines to stdout (default: false)
     -v, --verbose                    Output client progress and current operation
+    -q, --quit-on-exception          Stop collection if an API error is encountered (default: false)
     -d, --debug                      Output debug with wire trace info
     -h, --help                       Print this help information
 
@@ -217,7 +253,7 @@ Output is always some form of JSON - either JSON lines or plain JSON. The output
 
 ## Support for Manually Enabled Regions
 
-If you have enabled manually enabled regions:
+If you have enabled **manually enabled regions**:
 
 - me-south-1 - Middle East (Bahrain)
 - af-south-1 - Africa (Cape Town)
@@ -228,7 +264,7 @@ and you are using STS to assume a role into an account, you will need to [enable
 
 > Version 1 tokens are valid only in AWS Regions that are available by default. These tokens do not work in manually enabled Regions, such as Asia Pacific (Hong Kong). Version 2 tokens are valid in all Regions. However, version 2 tokens are longer and might affect systems where you temporarily store tokens.
 
-If you are using a static access key/secret, you can collect from these regions with either `v1` or `v2` STS tokens.
+If you are using a static access key/secret, you can collect from these regions regardless of STS token version.
 
 ## Supported Services & Resources
 
@@ -324,11 +360,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ### TODO
 
-- [ ] Optionally suppress AWS API errors instead of re-raising them
-- [x] Package as a gem
 - [ ] Test coverage with AWS SDK stubbed resources
 
-
 ## Kudos
 
 AWS Recon was inspired by the excellent work of the people and teams behind these tools:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.2.25
+  version: 0.2.30
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-06 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -197,7 +197,7 @@ files:
 - lib/aws_recon/collectors/codepipeline.rb
 - lib/aws_recon/collectors/configservice.rb
 - lib/aws_recon/collectors/directconnect.rb
-- lib/aws_recon/collectors/directyservice.rb
+- lib/aws_recon/collectors/directoryservice.rb
 - lib/aws_recon/collectors/dms.rb
 - lib/aws_recon/collectors/dynamodb.rb
 - lib/aws_recon/collectors/ec2.rb