aws_recon 0.2.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Darkbit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/aws_recon.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aws_recon/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'aws_recon'
+  spec.version       = AwsRecon::VERSION
+  spec.authors       = ['Josh Larsen']
+  spec.required_ruby_version = '>= 2.5.0'
+  spec.summary       = 'A multi-threaded AWS inventory collection tool.'
+  spec.description   = spec.summary
+  spec.homepage      = 'https://github.com/darkbitio/aws-recon'
+  spec.license       = 'MIT'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-sdk', '~> 3.0'
+  spec.add_dependency 'parallel', '~> 1.19'
+
+  spec.add_development_dependency 'bundler', '~> 1.17'
+  spec.add_development_dependency 'gem-release', '~> 2.1'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'solargraph', '~> 0.39.11'
+  spec.add_development_dependency 'rubocop', '~> 0.87.1'
+  spec.add_development_dependency 'pry', '~> 0.13.1'
+end

data/bin/aws_recon

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'aws_recon'
+
+AwsRecon::CLI.new.start(ARGV)

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "aws_recon"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/aws_recon.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module AwsRecon
+end
+
+require 'parallel'
+require 'ostruct'
+require 'optparse'
+require 'yaml'
+require 'csv'
+require 'pry'
+require 'aws-sdk'
+require 'aws_recon/options.rb'
+require 'aws_recon/lib/mapper.rb'
+require 'aws_recon/lib/formatter.rb'
+require 'aws_recon/collectors/collectors.rb'
+
+require 'aws_recon/version'
+require 'aws_recon/aws_recon'

data/lib/aws_recon/aws_recon.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
+
+module AwsRecon
+  class CLI
+    # parse options
+    @options = Parser.parse ARGV.length < 1 ? %w[-h] : ARGV
+
+    # timing
+    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    # AWS account id
+    @account_id = Aws::STS::Client.new.get_caller_identity.account
+
+    # AWS services
+    aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
+
+    # User config services
+    if @options.config_file
+      user_config = YAML.load(File.read(@options.config_file), symbolize_names: true)
+
+      services = user_config[:services]
+      regions = user_config[:regions]
+    else
+      regions = @options.regions
+      services = @options.services
+    end
+
+    # collection
+    @resources = []
+
+    # formatter
+    @formatter = Formatter.new
+
+    unless @options.stream_output
+      puts "\nStarting collection with #{@options.threads} threads...\n"
+    end
+
+    #
+    # collector wrapper
+    #
+    def collect(service, region)
+      mapper = Object.const_get(service.name)
+      resources = mapper.new(service.name, region, @options)
+
+      collection = resources.collect.map do |resource|
+        if @options.output_format == 'custom'
+          @formatter.custom(@account_id, region, service, resource)
+        else
+          @formatter.aws(@account_id, region, service, resource)
+        end
+      end
+
+      # write resources to stdout
+      if @options.stream_output
+        collection.each do |item|
+          puts item.to_json
+        end
+      end
+
+      # add resources to resources array for output to file
+      @resources.concat(collection) if @options.output_file
+    end
+
+    #
+    # main wrapper
+    #
+    def start
+      #
+      # global services
+      #
+      aws_services.map { |x| OpenStruct.new(x) }.filter { |s| s.global }.each do |service|
+        # user included this service in the args
+        next unless services.include?(service.name)
+
+        # user did not exclude 'global'
+        next unless regions.include?('global')
+
+        collect(service, 'global')
+      end
+
+      #
+      # regional services
+      #
+      regions.filter { |x| x != 'global' }.each do |region|
+        Parallel.map(aws_services.map { |x| OpenStruct.new(x) }.filter { |s| !s.global }.each, in_threads: @options.threads) do |service|
+          # some services aren't available in some regions
+          skip_region = service&.excluded_regions&.include?(region)
+
+          # user included this region in the args
+          next unless regions.include?(region) && !skip_region
+
+          # user included this service in the args
+          next unless services.include?(service.name) || services.include?(service.alias) # rubocop:disable Layout/LineLength
+
+          collect(service, region)
+        end
+      end
+    rescue Interrupt # ctrl-c
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
+
+      puts "\nStopped early after \x1b[32m#{elapsed.to_i}\x1b[0m seconds.\n"
+    ensure
+      # write output file
+      if @options.output_file
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
+
+        puts "\nFinished in \x1b[32m#{elapsed.to_i}\x1b[0m seconds. Saving resources to \x1b[32m#{@options.output_file}\x1b[0m.\n\n"
+
+        File.write(@options.output_file, @resources.to_json)
+      end
+    end
+  end
+end

data/lib/aws_recon/collectors/acm.rb

@@ -0,0 +1,32 @@
+class ACM < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # list_certificates
+    #
+    @client.list_certificates.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.certificate_summary_list.each do |cert|
+        log(response.context.operation_name, cert.domain_name, page)
+
+        struct = OpenStruct.new(cert.to_h)
+        struct.type = 'certificate'
+        struct.arn = cert.certificate_arn
+
+        # describe_certificate
+        struct.details = @client
+                         .describe_certificate({ certificate_arn: cert.certificate_arn })
+                         .certificate.to_h
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/apigateway.rb

@@ -0,0 +1,50 @@
+class APIGateway < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # get_rest_apis
+    #
+    @client.get_rest_apis.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.items.each do |api|
+        struct = OpenStruct.new(api.to_h)
+        struct.type = 'api'
+        struct.arn = api.id
+
+        # get_authorizers
+        struct.authorizers = @client.get_authorizers({ rest_api_id: api.id }).items.map(&:to_h)
+
+        # get_stages
+        struct.stages = @client.get_stages({ rest_api_id: api.id }).item.map(&:to_h)
+
+        # get_models
+        struct.models = @client.get_models({ rest_api_id: api.id }).items.map(&:to_h)
+
+        # get_resources
+        struct.resources = @client.get_resources({ rest_api_id: api.id }).items.map(&:to_h)
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    # get_domain_names
+    @client.get_domain_names.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.items.each do |domain|
+        struct = OpenStruct.new(domain.to_h)
+        struct.type = 'domain'
+        struct.arn = domain.domain_name
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/apigatewayv2.rb

@@ -0,0 +1,37 @@
+class ApiGatewayV2 < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # get_apis
+    #
+    @client.get_apis.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.items.each do |api|
+        struct = OpenStruct.new(api.to_h)
+        struct.type = 'api'
+        struct.arn = api.api_id
+
+        # get_authorizers
+        struct.authorizers = @client.get_authorizers({ api_id: api.api_id }).items.map(&:to_h)
+
+        # get_stages
+        struct.stages = @client.get_stages({ api_id: api.api_id }).items.map(&:to_h)
+
+        # get_models
+        struct.models = @client.get_models({ api_id: api.api_id }).items.map(&:to_h)
+
+        # get_deployments
+        struct.deployments = @client.get_deployments({ api_id: api.api_id }).items.map(&:to_h)
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/athena.rb

@@ -0,0 +1,28 @@
+class Athena < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # list_work_groups
+    #
+    @client.list_work_groups.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.work_groups.each do |workgroup|
+        struct = OpenStruct.new(workgroup.to_h)
+        struct.type = 'workgroup'
+        struct.arn = "arn:aws:athena:#{@region}::workgroup/#{workgroup.name}"
+
+        # get_work_group
+        struct.details = @client.get_work_group({ work_group: workgroup.name }).to_h
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/autoscaling.rb

@@ -0,0 +1,35 @@
+class AutoScaling < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # describe_auto_scaling_groups
+    #
+    @client.describe_auto_scaling_groups.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.auto_scaling_groups.each do |asg|
+        struct = OpenStruct.new(asg.to_h)
+        struct.type = 'auto_scaling_group'
+        struct.arn = asg.auto_scaling_group_arn
+        struct.policies = []
+
+        # describe_policies
+        @client.describe_policies({ auto_scaling_group_name: asg.auto_scaling_group_name }).each_with_index do |response, page|
+          log(response.context.operation_name, page)
+
+          response.scaling_policies.each do |policy|
+            struct.policies.push(policy.to_h)
+          end
+        end
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/cloudformation.rb

@@ -0,0 +1,29 @@
+class CloudFormation < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # describe_stacks
+    #
+    @client.describe_stacks.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.stacks.each do |stack|
+        struct = OpenStruct.new(stack.to_h)
+        struct.type = 'stack'
+        struct.arn = stack.stack_id
+
+        # get_template
+        struct.tempate = @client.get_template({ stack_name: stack.stack_name }).to_h
+        log(response.context.operation_name, 'get_template', stack.stack_name)
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end