aws_recon 0.2.1 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8cd90cd73ab80c766882e88a2bbd5f03c106a7365805b7e50847997decd98de
-  data.tar.gz: 1a605e550317fdbf2295f72c0df9df6bdb02b1cb70e940d83db3c779d165d7d3
+  metadata.gz: 2707134a4d4be23f0f1cd3320fda7e75463aae49113b2f4b802257aa02b0de52
+  data.tar.gz: 85452e37a3657d315fae2b67852bc521786a95fc386db53f4a3212801e140ee4
 SHA512:
-  metadata.gz: f3f8600403036263543e0be3d9fee779d40a9cedbed38a982f8144e645c1184d0e4c135691d87d0e7809033c55e371dab4eaaba8ff80b6ba27baab7663d75ec0
-  data.tar.gz: 7e4af190e9451570c40e90fdd222899bc11c072da15f496d7739cf057da5ee04615d6a6b0cfda375308fda88a9dd91796bb0ab8e31e6d3289f7cf4daf7286e3c
+  metadata.gz: 5e1a919375c34b51a72a3bb470ce178b39c178e8e3a569fb47f618c8ac122512e14c2564e42bd72aab3e5a550041f8e9490c3f3d6245ac2d3c933e77a9053d18
+  data.tar.gz: e93eaad39853ffdd5c934572b2d9fb2add88efc0150bc48cbc3cf7cef542c25a98a2ebd1c92e6b64404dde122d86d8d2d38638f6bd5fde0e11974c2c52e5cdc1

data/.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 30
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 5
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

data/.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 *.json
+Gemfile.lock
 .rvmrc
 .ruby-gemset
 .ruby-version

data/aws_recon.gemspec

@@ -7,10 +7,10 @@ require 'aws_recon/version'
 Gem::Specification.new do |spec|
   spec.name          = 'aws_recon'
   spec.version       = AwsRecon::VERSION
-  spec.authors       = ['Josh Larsen']
+  spec.authors       = ['Josh Larsen', 'Darkbit']
   spec.required_ruby_version = '>= 2.5.0'
-  spec.summary       = 'A multi-threaded AWS inventory collection tool.'
-  spec.description   = spec.summary
+  spec.summary       = 'A multi-threaded AWS inventory collection cli tool.'
+  spec.description   = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
   spec.homepage      = 'https://github.com/darkbitio/aws-recon'
   spec.license       = 'MIT'
 
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'gem-release', '~> 2.1'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'

data/bin/aws_recon

@@ -1,5 +1,8 @@
 #!/usr/bin/env ruby
 
+# for local testing
+$LOAD_PATH.unshift(File.expand_path(File.join('..', '..', 'lib'), __FILE__))
+
 require 'aws_recon'
 
 AwsRecon::CLI.new.start(ARGV)

data/lib/aws_recon.rb

@@ -8,12 +8,11 @@ require 'ostruct'
 require 'optparse'
 require 'yaml'
 require 'csv'
-require 'pry'
 require 'aws-sdk'
 require 'aws_recon/options.rb'
 require 'aws_recon/lib/mapper.rb'
 require 'aws_recon/lib/formatter.rb'
-require 'aws_recon/collectors/collectors.rb'
+require 'aws_recon/collectors.rb'
 
 require 'aws_recon/version'
 require 'aws_recon/aws_recon'

data/lib/aws_recon/aws_recon.rb

@@ -4,37 +4,39 @@ SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
 
 module AwsRecon
   class CLI
-    # parse options
-    @options = Parser.parse ARGV.length < 1 ? %w[-h] : ARGV
+    def initialize
+      # parse options
+      @options = Parser.parse ARGV.length < 1 ? %w[-h] : ARGV
 
-    # timing
-    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      # timing
+      @starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
-    # AWS account id
-    @account_id = Aws::STS::Client.new.get_caller_identity.account
+      # AWS account id
+      @account_id = Aws::STS::Client.new.get_caller_identity.account
 
-    # AWS services
-    aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
+      # AWS services
+      @aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
 
-    # User config services
-    if @options.config_file
-      user_config = YAML.load(File.read(@options.config_file), symbolize_names: true)
+      # User config services
+      if @options.config_file
+        user_config = YAML.load(File.read(@options.config_file), symbolize_names: true)
 
-      services = user_config[:services]
-      regions = user_config[:regions]
-    else
-      regions = @options.regions
-      services = @options.services
-    end
+        @services = user_config[:services]
+        @regions = user_config[:regions]
+      else
+        @regions = @options.regions
+        @services = @options.services
+      end
 
-    # collection
-    @resources = []
+      # collection
+      @resources = []
 
-    # formatter
-    @formatter = Formatter.new
+      # formatter
+      @formatter = Formatter.new
 
-    unless @options.stream_output
-      puts "\nStarting collection with #{@options.threads} threads...\n"
+      unless @options.stream_output
+        puts "\nStarting collection with #{@options.threads} threads...\n"
+      end
     end
 
     #
@@ -66,16 +68,16 @@ module AwsRecon
     #
     # main wrapper
     #
-    def start
+    def start(_args)
       #
       # global services
       #
-      aws_services.map { |x| OpenStruct.new(x) }.filter { |s| s.global }.each do |service|
+      @aws_services.map { |x| OpenStruct.new(x) }.filter { |s| s.global }.each do |service|
         # user included this service in the args
-        next unless services.include?(service.name)
+        next unless @services.include?(service.name)
 
         # user did not exclude 'global'
-        next unless regions.include?('global')
+        next unless @regions.include?('global')
 
         collect(service, 'global')
       end
@@ -83,28 +85,28 @@ module AwsRecon
       #
       # regional services
       #
-      regions.filter { |x| x != 'global' }.each do |region|
-        Parallel.map(aws_services.map { |x| OpenStruct.new(x) }.filter { |s| !s.global }.each, in_threads: @options.threads) do |service|
+      @regions.filter { |x| x != 'global' }.each do |region|
+        Parallel.map(@aws_services.map { |x| OpenStruct.new(x) }.filter { |s| !s.global }.each, in_threads: @options.threads) do |service|
           # some services aren't available in some regions
           skip_region = service&.excluded_regions&.include?(region)
 
           # user included this region in the args
-          next unless regions.include?(region) && !skip_region
+          next unless @regions.include?(region) && !skip_region
 
           # user included this service in the args
-          next unless services.include?(service.name) || services.include?(service.alias) # rubocop:disable Layout/LineLength
+          next unless @services.include?(service.name) || @services.include?(service.alias) # rubocop:disable Layout/LineLength
 
           collect(service, region)
         end
       end
     rescue Interrupt # ctrl-c
-      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
 
       puts "\nStopped early after \x1b[32m#{elapsed.to_i}\x1b[0m seconds.\n"
     ensure
       # write output file
       if @options.output_file
-        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
 
         puts "\nFinished in \x1b[32m#{elapsed.to_i}\x1b[0m seconds. Saving resources to \x1b[32m#{@options.output_file}\x1b[0m.\n\n"
 

data/lib/aws_recon/collectors.rb

@@ -0,0 +1,2 @@
+# require all collectors
+Dir[File.join(__dir__, 'collectors', '*.rb')].each { |file| require file }

data/lib/aws_recon/collectors/ec2.rb

@@ -48,6 +48,23 @@ class EC2 < Mapper
             struct.arn = instance.instance_id # no true ARN
             struct.reservation_id = reservation.reservation_id
 
+            # collect instance user_data
+            if @options.collect_user_data
+              user_data_raw = @client.describe_instance_attribute({
+                                                                    attribute: 'userData',
+                                                                    instance_id: instance.instance_id
+                                                                  }).user_data.to_h[:value]
+
+              # don't save non-string user_data
+              if user_data_raw
+                user_data = Base64.decode64(user_data_raw)
+
+                if user_data.force_encoding('UTF-8').ascii_only?
+                  struct.user_data = user_data
+                end
+              end
+            end
+
             resources.push(struct.to_h)
           end
         end

data/lib/aws_recon/collectors/elasticache.rb

@@ -0,0 +1,22 @@
+class ElastiCache < Mapper
+  def collect
+    resources = []
+
+    #
+    # describe_cache_clusters
+    #
+    @client.describe_cache_clusters.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.cache_clusters.each do |cluster|
+        struct = OpenStruct.new(cluster.to_h)
+        struct.type = 'cluster'
+        struct.arn = cluster.arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/lambda.rb

@@ -1,6 +1,5 @@
 class Lambda < Mapper
   def collect
-    service = self.class.to_s.downcase
     resources = []
 
     #

data/lib/aws_recon/collectors/s3.rb

@@ -15,8 +15,6 @@ class S3 < Mapper
       log(response.context.operation_name, page)
 
       Parallel.map(response.buckets.each, in_threads: @options.threads) do |bucket|
-        # use shared client instance
-        client = @client
         @thread = Parallel.worker_number
         log(response.context.operation_name, bucket.name)
 
@@ -27,10 +25,14 @@ class S3 < Mapper
         # check bucket region constraint
         location = @client.get_bucket_location({ bucket: bucket.name }).location_constraint
 
-        # reset client if needed
-        unless location.empty?
-          client = Aws::S3::Client.new({ region: location })
-        end
+        # if you use a region other than the us-east-1 endpoint
+        # to create a bucket, you must set the location_constraint
+        # bucket parameter to the same region. (https://docs.aws.amazon.com/general/latest/gr/s3.html)
+        client = if location.empty?
+                   @client
+                 else
+                   Aws::S3::Client.new({ region: location })
+                 end
 
         operations = [
           { func: 'get_bucket_acl', key: 'acl', field: nil },

data/lib/aws_recon/lib/mapper.rb

@@ -15,6 +15,13 @@
 # to add 5 seconds delay on each retry for a total max of 55 seconds.
 #
 class Mapper
+  # Services that use us-east-1 endpoint only:
+  #   Organizations
+  #   Route53Domains
+  #   Shield
+  #   S3 (unless the bucket was created in another region)
+  SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
+
   def initialize(service, region, options)
     @service = service
     @region = region
@@ -39,8 +46,8 @@ class Mapper
     # regional service
     client_options.merge!({ region: region }) unless region == 'global'
 
-    # organizations only uses us-east-1 in non cn/gov regions
-    client_options.merge!({ region: 'us-east-1' }) if service.downcase == 'organizations' # rubocop:disable Layout/LineLength
+    # single region services
+    client_options.merge!({ region: 'us-east-1' }) if SINGLE_REGION_SERVICES.include?(service.downcase) # rubocop:disable Layout/LineLength
 
     # debug with wire trace
     client_options.merge!({ http_wire_trace: true }) if @options.debug

data/lib/aws_recon/options.rb

@@ -2,7 +2,7 @@
 
 class Parser
   DEFAULT_CONFIG_FILE = nil
-  DEFAULT_OUTPUT_FILE = File.join(File.dirname(__FILE__), '../output.json').freeze
+  DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze
   SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
   DEFAULT_FORMAT = 'aws'
   DEFAULT_THREADS = 8
@@ -15,6 +15,7 @@ class Parser
     :output_file,
     :output_format,
     :threads,
+    :collect_user_data,
     :skip_slow,
     :stream_output,
     :verbose,
@@ -43,11 +44,12 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
     opt_parser = OptionParser.new do |opts|
-      opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector\n\nUsage: aws_recon [options]"
+      opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector (#{AwsRecon::VERSION})\n\nUsage: aws_recon [options]"
 
       # regions
       opts.on('-r', '--regions [REGIONS]', 'Regions to scan, separated by comma (default: all)') do |regions|
@@ -86,7 +88,7 @@ class Parser
 
       # output file
       opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output|
-        args.output_file = output
+        args.output_file = File.expand_path(File.join(Dir.pwd, output))
       end
 
       # output format
@@ -103,6 +105,11 @@ class Parser
         end
       end
 
+      # collect EC2 instance user data
+      opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do
+        args.collect_user_data = true
+      end
+
       # skip slow operations
       opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do
         args.skip_slow = true

data/lib/aws_recon/services.yaml

@@ -33,6 +33,8 @@
   alias: elbv2
   excluded_regions:
     - ap-southeast-1
+- name: ElastiCache
+  alias: elasticache
 - name: IAM
   global: true
   alias: iam

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.1"
+  VERSION = "0.2.6"
 end

data/readme.md

@@ -1,3 +1,5 @@
+[![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://badge.fury.io/rb/aws_recon)
+
 # AWS Recon
 
 A multi-threaded AWS inventory collection tool.
@@ -24,17 +26,30 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Clone this repository, then install the required gems using `bundle`:
+Install the gem:
 
 ```
-$ git clone git@github.com:darkbitio/aws-recon.git
-$ cd aws-recon
-$ bundle
+$ gem install aws_recon
+Fetching aws_recon-0.2.6.gem
+Fetching aws-sdk-resources-3.76.0.gem
+Fetching aws-sdk-3.0.1.gem
+Fetching parallel-1.19.2.gem
 ...
-Using aws-sdk-core 3.103.0
+Successfully installed aws-sdk-3.0.1
+Successfully installed parallel-1.19.2
+Successfully installed aws_recon-0.2.6
+```
+
+Or add it to your Gemfile using `bundle`:
+
+```
+$ bundle add aws_recon
+Fetching gem metadata from https://rubygems.org/
+Resolving dependencies...
 ...
-Bundle complete! 5 Gemfile dependencies, 259 gems now installed.
-Use `bundle info [gemname]` to see where a bundled gem is installed.
+Using aws-sdk 3.0.1
+Using parallel 1.19.2
+Using aws_recon 0.2.2
 ```
 
 ## Usage
@@ -42,13 +57,13 @@ Use `bundle info [gemname]` to see where a bundled gem is installed.
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
 
 ```
-$ aws-vault exec profile -- ./recon.rb
+$ aws-vault exec profile -- aws_recon
 ```
 
 Plain environment variables will work fine too.
 
 ```
-$ AWS_PROFILE=<profile> ./recon.rb
+$ AWS_PROFILE=<profile> aws_recon
 ```
 
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
@@ -62,7 +77,7 @@ In verbose mode, the console output will show:
 The `t` prefix indicates which thread a particular request is running under. Region, service, and operation indicate which request operation is currently in progress and where.
 
 ```
-$ ./recon.rb -v
+$ aws_recon -v
 
 t0.global.EC2.describe_account_attributes
 t2.global.S3.list_buckets
@@ -87,11 +102,11 @@ Finished in 46 seconds. Saving resources to output.json.
 #### Example command line options
 
 ```
-$ AWS_PROFILE=<profile> ./recon.rb -s S3,EC2 -r global,us-east-1,us-east-2
+$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 ```
 
 ```
-$ AWS_PROFILE=<profile> ./recon.rb --services S3,EC2 --regions global,us-east-1,us-east-2
+$ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
 #### Errors
@@ -118,11 +133,11 @@ For regional services, a thread (up to the thread limit) is spawned for each ser
 Most users will want to limit collection to relevant services and regions. Running without any options will attempt to collect all resources from all 16 regular regions.
 
 ```
-$ ./recon.rb -h
+$ aws_recon -h
 
-AWS Recon - AWS Inventory Collector
+AWS Recon - AWS Inventory Collector (0.2.6)
 
-Usage: ./recon.rb [options]
+Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
     -n, --not-regions [REGIONS]      Regions to skip, separated by comma (default: none)
     -s, --services [SERVICES]        Services to scan, separated by comma (default: all)
@@ -131,6 +146,7 @@ Usage: ./recon.rb [options]
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
     -j, --stream-output              Stream JSON lines to stdout (default: false)
     -v, --verbose                    Output client progress and current operation
@@ -178,6 +194,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] ELB
 - [x] EKS
 - [x] Elasticsearch
+- [x] ElastiCache
 - [x] Firehose
 - [ ] FMS
 - [ ] Glacier
@@ -210,10 +227,29 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 
 One of the primary motivations for AWS Recon was to build a tool that is easy to maintain and extend. If you feel like coverage could be improved for a particular service, we would welcome PRs to that effect. Anyone with a moderate familiarity with Ruby will be able to mimic the pattern used by the existing collectors to query a specific service and add the results to the resource collection.
 
+### Development
+
+Clone this repository:
+
+```
+$ git clone git@github.com:darkbitio/aws-recon.git
+$ cd aws-recon
+```
+
+Create a sticky gemset if using RVM:
+
+```
+$ rvm use 2.6.5@aws_recon_dev --create --ruby-version
+```
+
+Run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ### TODO
 
 - [ ] Optionally suppress AWS API errors instead of re-raising them
-- [ ] Package as a gem
+- [x] Package as a gem
 - [ ] Test coverage with AWS SDK stubbed resources