aws_recon 0.2.1

data/lib/aws_recon/lib/formatter.rb

@@ -0,0 +1,32 @@
+#
+# Customize resource format/shape
+#
+class Formatter
+  #
+  # Custom
+  #
+  def custom(account_id, region, service, resource)
+    {
+      account: account_id,
+      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      service: service.name,
+      region: region,
+      asset_type: resource[:type],
+      resource: { data: resource, version: 'v1' },
+      timestamp: Time.now.utc
+    }
+  end
+
+  #
+  # Standard AWS
+  #
+  def aws(account_id, region, service, resource)
+    {
+      account: account_id,
+      service: service.name,
+      region: region,
+      resource: resource,
+      timestamp: Time.now.utc
+    }
+  end
+end

data/lib/aws_recon/lib/mapper.rb

@@ -0,0 +1,69 @@
+#
+# Generic wrapper for service clients.
+#
+# Paging
+# ------
+# The AWS Ruby SDK has a built-in enumerator in the client response
+# object that automatically handles paging large requests. Confirm
+# the AWS SDK client call returns a Aws::PageableResponse to take
+# advantage of automatic paging.
+#
+# Retries
+# -------
+# The AWS Ruby SDK performs retries automatically. We change the
+# retry_limit to 9 (10 total requests) and the retry_backoff
+# to add 5 seconds delay on each retry for a total max of 55 seconds.
+#
+class Mapper
+  def initialize(service, region, options)
+    @service = service
+    @region = region
+    @options = options
+    @thread = Parallel.worker_number || 0
+
+    # build the client interface
+    module_name = "Aws::#{service}::Client"
+
+    # incremental delay on retries (seconds)
+    retry_delay = 5
+
+    # default is 3 retries, with 15 second sleep in between
+    # reset to 9 retries, with incremental backoff
+    client_options = {
+      retry_mode: 'legacy', # legacy, standard, or adaptive
+      retry_limit: 9, # only legacy
+      retry_backoff: ->(context) { sleep(retry_delay * context.retries + 1) }, # only legacy
+      http_read_timeout: 10
+    }
+
+    # regional service
+    client_options.merge!({ region: region }) unless region == 'global'
+
+    # organizations only uses us-east-1 in non cn/gov regions
+    client_options.merge!({ region: 'us-east-1' }) if service.downcase == 'organizations' # rubocop:disable Layout/LineLength
+
+    # debug with wire trace
+    client_options.merge!({ http_wire_trace: true }) if @options.debug
+
+    @client = Object.const_get(module_name).new(client_options)
+  end
+
+  private
+
+  def _msg(msg)
+    base_msg = ["t#{@thread}", @region, @service]
+    base_msg.concat(msg)
+  end
+
+  def log(*msg)
+    if @options.verbose
+      puts _msg(msg).map { |x| "\x1b[32m#{x}\x1b[0m" }.join("\x1b[35m.\x1b[0m")
+    end
+  end
+
+  def log_error(*msg)
+    if @options.verbose
+      puts _msg(msg).map { |x| "\x1b[35m#{x}\x1b[0m" }.join("\x1b[32m.\x1b[0m")
+    end
+  end
+end

data/lib/aws_recon/options.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+class Parser
+  DEFAULT_CONFIG_FILE = nil
+  DEFAULT_OUTPUT_FILE = File.join(File.dirname(__FILE__), '../output.json').freeze
+  SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
+  DEFAULT_FORMAT = 'aws'
+  DEFAULT_THREADS = 8
+  MAX_THREADS = 128
+
+  Options = Struct.new(
+    :regions,
+    :services,
+    :config_file,
+    :output_file,
+    :output_format,
+    :threads,
+    :skip_slow,
+    :stream_output,
+    :verbose,
+    :debug
+  )
+
+  def self.parse(options)
+    begin
+      unless (options & ['-h', '--help']).any?
+        aws_regions = ['global'].concat(Aws::EC2::Client.new.describe_regions.regions.map(&:region_name))
+      end
+    rescue Aws::Errors::ServiceError => e
+      puts "\nAWS Error: #{e.code}\n\n"
+      exit
+    end
+
+    aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
+
+    args = Options.new(
+      aws_regions,
+      aws_services.map { |service| service[:name] },
+      DEFAULT_CONFIG_FILE,
+      DEFAULT_OUTPUT_FILE,
+      DEFAULT_FORMAT,
+      DEFAULT_THREADS,
+      false,
+      false,
+      false,
+      false
+    )
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector\n\nUsage: aws_recon [options]"
+
+      # regions
+      opts.on('-r', '--regions [REGIONS]', 'Regions to scan, separated by comma (default: all)') do |regions|
+        next if regions.downcase == 'all'
+
+        args.regions = args.regions.filter { |region| regions.split(',').include?(region) }
+      end
+
+      # regions to skip
+      opts.on('-n', '--not-regions [REGIONS]', 'Regions to skip, separated by comma (default: none)') do |regions|
+        next if regions.downcase == 'all'
+
+        args.regions = args.regions.filter { |region| !regions.split(',').include?(region) }
+      end
+
+      # services
+      opts.on('-s', '--services [SERVICES]', 'Services to scan, separated by comma (default: all)') do |services|
+        next if services.downcase == 'all'
+
+        svcs = services.split(',')
+        args.services = aws_services.map { |service| service[:name] if svcs.include?(service[:name]) || svcs.include?(service[:alias]) }.compact # rubocop:disable Layout/LineLength
+      end
+
+      # services to skip
+      opts.on('-x', '--not-services [SERVICES]', 'Services to skip, separated by comma (default: none)') do |services|
+        next if services.downcase == 'all'
+
+        svcs = services.split(',')
+        args.services = aws_services.map { |service| service[:name] unless svcs.include?(service[:name]) || svcs.include?(service[:alias]) }.compact # rubocop:disable Layout/LineLength
+      end
+
+      # config file
+      opts.on('-c', '--config [CONFIG]', 'Specify config file for services & regions (e.g. config.yaml)') do |config|
+        args.config_file = config
+      end
+
+      # output file
+      opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output|
+        args.output_file = output
+      end
+
+      # output format
+      opts.on('-f', '--format [FORMAT]', 'Specify output format (default: aws)') do |file|
+        if %w[aws custom].include?(file.downcase)
+          args.output_format = file.downcase
+        end
+      end
+
+      # threads
+      opts.on('-t', '--threads [THREADS]', "Specify max threads (default: #{Parser::DEFAULT_THREADS}, max: 128)") do |threads|
+        if (0..Parser::MAX_THREADS).include?(threads.to_i)
+          args.threads = threads.to_i
+        end
+      end
+
+      # skip slow operations
+      opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do
+        args.skip_slow = true
+      end
+
+      # stream output (forces JSON lines, doesn't output handled warnings or errors )
+      opts.on('-j', '--stream-output', 'Stream JSON lines to stdout (default: false)') do
+        args.output_file = nil
+        args.verbose = false
+        args.debug = false
+        args.stream_output = true
+      end
+
+      # verbose
+      opts.on('-v', '--verbose', 'Output client progress and current operation') do
+        args.verbose = true unless args.stream_output
+      end
+
+      # debug
+      opts.on('-d', '--debug', 'Output debug with wire trace info') do
+        unless args.stream_output
+          args.debug = true
+          args.verbose = true
+        end
+      end
+
+      opts.on('-h', '--help', 'Print this help information') do
+        puts opts
+        exit
+      end
+    end
+
+    opt_parser.parse!(options)
+    args
+  end
+end

data/lib/aws_recon/services.yaml

@@ -0,0 +1,134 @@
+---
+- name: Organizations
+  global: true
+  alias: organizations
+- name: ConfigService
+  alias: config
+- name: CodeBuild
+  alias: codebuild
+- name: CodePipeline
+  alias: codepipeline
+- name: AutoScaling
+  alias: autoscaling
+- name: CloudTrail
+  alias: cloudtrail
+- name: CloudFront
+  alias: cloudfront
+- name: EC2
+  global: true
+  alias: ec2
+- name: EC2
+  alias: ec2
+- name: EKS
+  alias: eks
+  excluded_regions:
+    - us-west-1
+- name: ECS
+  alias: ecs
+- name: ElasticLoadBalancing
+  alias: elb
+  excluded_regions:
+    - ap-southeast-1
+- name: ElasticLoadBalancingV2
+  alias: elbv2
+  excluded_regions:
+    - ap-southeast-1
+- name: IAM
+  global: true
+  alias: iam
+- name: Lambda
+  alias: lambda
+- name: S3
+  global: true
+  alias: s3
+- name: RDS
+  alias: rds
+- name: ECR
+  alias: ecr
+- name: DynamoDB
+  alias: ddb
+- name: KMS
+  alias: kms
+- name: Kinesis
+  alias: kinesis
+- name: Redshift
+  alias: redshift
+- name: ElasticsearchService
+  alias: es
+- name: APIGateway
+  alias: apigateway
+- name: ApiGatewayV2
+  alias: apigatewayv2
+- name: Route53
+  alias: route53
+- name: Route53Domains
+  global: true
+  alias: route53domains
+- name: SQS
+  alias: sqs
+- name: ACM
+  alias: acm
+- name: SNS
+  alias: sns
+- name: Shield
+  global: true
+  alias: shield
+- name: CloudFormation
+  alias: cloudformation
+- name: SES
+  alias: ses
+  excluded_regions:
+    - eu-north-1
+    - eu-west-3
+    - us-west-1
+- name: CloudWatch
+  alias: cloudwatch
+- name: CloudWatchLogs
+  alias: cloudwatchlogs
+- name: Kafka
+  alias: kafka
+- name: Support
+  global: true
+  alias: support
+- name: SSM
+  alias: ssm
+  excluded_regions:
+    - ap-southeast-1
+- name: GuardDuty
+  alias: guardduty
+- name: Athena
+  alias: athena
+- name: EFS
+  alias: efs
+- name: Firehose
+  alias: firehose
+- name: Lightsail
+  alias: lightsail
+  excluded_regions:
+    - eu-north-1
+    - us-west-1
+    - sa-east-1
+- name: WorkSpaces
+  alias: workspaces
+  excluded_regions:
+    - eu-north-1
+    - ap-south-1
+    - eu-west-3
+    - us-east-2
+    - us-west-1
+- name: SageMaker
+  alias: sagemaker
+- name: ServiceQuotas
+  alias: servicequotas
+- name: Transfer
+  alias: transfer
+- name: DirectConnect
+  alias: dc
+- name: DirectoryService
+  alias: ds
+- name: DatabaseMigrationService
+  alias: dms
+- name: XRay
+  alias: xray
+- name: WAFV2
+  alias: wafv2

data/lib/aws_recon/version.rb

@@ -0,0 +1,3 @@
+module AwsRecon
+  VERSION = "0.2.1"
+end

data/readme.md

@@ -0,0 +1,226 @@
+# AWS Recon
+
+A multi-threaded AWS inventory collection tool.
+
+The [creators](https://darkbit.io) of this tool have a recurring need to be able to efficiently collect a large amount of AWS resource attributes and metadata to help clients understand their cloud security posture.
+
+There are a handful of tools (e.g. [AWS Config](https://aws.amazon.com/config), [CloudMapper](https://github.com/duo-labs/cloudmapper), [CloudSploit](https://github.com/cloudsploit/scans), [Prowler](https://github.com/toniblyx/prowler)) that do some form of resource collection to support other functions. But we found we needed broader coverage and more details at a per-service level. We also needed a consistent and structured format that allowed for integration with our other systems and tooling.
+
+Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain Ruby. Though most AWS tooling tends to be dominated by Python, the [Ruby SDK](https://aws.amazon.com/sdk-for-ruby/) is quite mature and capable. The maintainers of the Ruby SDK have done a fantastic job making it easy to handle automatic retries, paging of large responses, and threading huge numbers of requests.
+
+## Project Goals
+
+- More complete resource coverage than available tools (especially for ECS & EKS)
+- More granular resource detail, including nested related resources in the output
+- Flexible output (console, JSON lines, plain JSON, file, standard out)
+- Efficient (multi-threaded, rate limited, automatic retries, and automatic result paging)
+- Easy to maintain and extend
+
+## Setup
+
+### Requirements
+
+Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
+
+### Installation
+
+Clone this repository, then install the required gems using `bundle`:
+
+```
+$ git clone git@github.com:darkbitio/aws-recon.git
+$ cd aws-recon
+$ bundle
+...
+Using aws-sdk-core 3.103.0
+...
+Bundle complete! 5 Gemfile dependencies, 259 gems now installed.
+Use `bundle info [gemname]` to see where a bundled gem is installed.
+```
+
+## Usage
+
+AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
+
+```
+$ aws-vault exec profile -- ./recon.rb
+```
+
+Plain environment variables will work fine too.
+
+```
+$ AWS_PROFILE=<profile> ./recon.rb
+```
+
+You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
+
+In verbose mode, the console output will show:
+
+```
+<thread>.<region>.<service>.<operation>
+```
+
+The `t` prefix indicates which thread a particular request is running under. Region, service, and operation indicate which request operation is currently in progress and where.
+
+```
+$ ./recon.rb -v
+
+t0.global.EC2.describe_account_attributes
+t2.global.S3.list_buckets
+t3.global.Support.describe_trusted_advisor_checks
+t2.global.S3.list_buckets.acl
+t5.ap-southeast-1.WorkSpaces.describe_workspaces
+t6.ap-northeast-1.Lightsail.get_instances
+...
+t2.us-west-2.WorkSpaces.describe_workspaces
+t1.us-east-2.Lightsail.get_instances
+t4.ap-southeast-1.Firehose.list_delivery_streams
+t7.ap-southeast-1.Lightsail.get_instances
+t0.ap-south-1.Lightsail.get_instances
+t1.us-east-2.Lightsail.get_load_balancers
+t7.ap-southeast-2.WorkSpaces.describe_workspaces
+t2.eu-west-3.SageMaker.list_notebook_instances
+t3.eu-west-2.SageMaker.list_notebook_instances
+
+Finished in 46 seconds. Saving resources to output.json.
+```
+
+#### Example command line options
+
+```
+$ AWS_PROFILE=<profile> ./recon.rb -s S3,EC2 -r global,us-east-1,us-east-2
+```
+
+```
+$ AWS_PROFILE=<profile> ./recon.rb --services S3,EC2 --regions global,us-east-1,us-east-2
+```
+
+#### Errors
+
+An exception will be raised on `AccessDeniedException` errors. This typically means your user/role doesn't have the necessary permissions to get/list/describe for that service. These exceptions are raised so troubleshooting access issues is easier.
+
+```
+Traceback (most recent call last):
+arn:aws:sts::1234567890:assumed-role/role/9876543210 is not authorized to perform: codepipeline:GetPipeline on resource: arn:aws:codepipeline:us-west-2:876543210123:pipeline (Aws::CodePipeline::Errors::AccessDeniedException)
+```
+
+The exact API operation that triggered the exception is indicated on the last line of the stack trace. If you can't resolve the necessary access, you should exclude those services with `-x` or `--not-services` so the collection can continue.
+
+### Threads
+
+AWS Recon uses multiple threads to try to overcome some of the I/O challenges of performing many API calls to endpoints all over the world.
+
+For global services like IAM, Shield, and Support, requests are not multi-threaded. The S3 module is multi-threaded since each bucket requires several additional calls to collect complete metadata.
+
+For regional services, a thread (up to the thread limit) is spawned for each service in a region. By default, up to 8 threads will be used. If your account has resources spread across many regions, you may see a speed improvement by increasing threads with `-t X`, where `X` is the number of threads.
+
+### Options
+
+Most users will want to limit collection to relevant services and regions. Running without any options will attempt to collect all resources from all 16 regular regions.
+
+```
+$ ./recon.rb -h
+
+AWS Recon - AWS Inventory Collector
+
+Usage: ./recon.rb [options]
+    -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
+    -n, --not-regions [REGIONS]      Regions to skip, separated by comma (default: none)
+    -s, --services [SERVICES]        Services to scan, separated by comma (default: all)
+    -x, --not-services [SERVICES]    Services to skip, separated by comma (default: none)
+    -c, --config [CONFIG]            Specify config file for services & regions (e.g. config.yaml)
+    -o, --output [OUTPUT]            Specify output file (default: output.json)
+    -f, --format [FORMAT]            Specify output format (default: aws)
+    -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -z, --skip-slow                  Skip slow operations (default: false)
+    -j, --stream-output              Stream JSON lines to stdout (default: false)
+    -v, --verbose                    Output client progress and current operation
+    -d, --debug                      Output debug with wire trace info
+    -h, --help                       Print this help information
+
+```
+
+#### Output
+
+Output is always some form of JSON - either JSON lines or plain JSON. The output is either written to a file (the default), or written to stdout (with `-j`).
+
+
+## Supported Services & Resources
+
+Current "coverage" by service is listed below. The services without coverage will eventually be added. PRs are certainly welcome. :)
+
+AWS Recon aims to collect all resources and metadata that are relevant in determining the security posture of your AWS account(s). However, it does not actually examine the resources for security posture - that is the job of other tools that take the output of AWS Recon as input.
+
+- [x] AdvancedShield
+- [x] Athena
+- [x] GuardDuty
+- [ ] Macie
+- [x] Systems Manager
+- [x] Trusted Advisor
+- [x] ACM
+- [x] API Gateway
+- [x] AutoScaling
+- [x] CodePipeline
+- [x] CodeBuild
+- [x] CloudFormation
+- [x] CloudFront
+- [x] CloudWatch
+- [x] CloudWatch Logs
+- [x] CloudTrail
+- [x] Config
+- [x] DirectoryService
+- [x] DirectConnect
+- [x] DMS
+- [x] DynamoDB
+- [x] EC2
+- [x] ECR
+- [x] ECS
+- [x] EFS
+- [x] ELB
+- [x] EKS
+- [x] Elasticsearch
+- [x] Firehose
+- [ ] FMS
+- [ ] Glacier
+- [x] IAM
+- [x] KMS
+- [x] Kafka
+- [x] Kinesis
+- [x] Lambda
+- [x] Lightsail
+- [x] Organizations
+- [x] RDS
+- [x] Redshift
+- [x] Route53
+- [x] Route53Domains
+- [x] S3
+- [x] SageMaker
+- [x] SES
+- [x] ServiceQuotas
+- [x] Shield
+- [x] SNS
+- [x] SQS
+- [x] Transfer
+- [x] VPC
+- [ ] WAF
+- [x] WAFv2
+- [x] Workspaces
+- [x] Xray
+
+### Additional Coverage
+
+One of the primary motivations for AWS Recon was to build a tool that is easy to maintain and extend. If you feel like coverage could be improved for a particular service, we would welcome PRs to that effect. Anyone with a moderate familiarity with Ruby will be able to mimic the pattern used by the existing collectors to query a specific service and add the results to the resource collection.
+
+### TODO
+
+- [ ] Optionally suppress AWS API errors instead of re-raising them
+- [ ] Package as a gem
+- [ ] Test coverage with AWS SDK stubbed resources
+
+
+## Kudos
+
+AWS Recon was inspired by the excellent work of the people and teams behind these tools:
+
+- CloudMapper [https://github.com/duo-labs/cloudmapper](https://github.com/duo-labs/cloudmapper)
+- Prowler [https://github.com/toniblyx/prowler](https://github.com/toniblyx/prowler)
+- CloudSploit [https://github.com/cloudsploit/scans](https://github.com/cloudsploit/scans)