aws_recon 0.2.4 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ea30bb9feaa68c1aa5a90c8888ff341f6d67f38f7a113a43d80cadfc25e6ec1
-  data.tar.gz: 7c417745acb025337f8f9d1843691acc37569bacdaf83e6d5423a218c9af6eae
+  metadata.gz: 748e27233aa80e92e6f74f5685c9620315a649cc425c96186d234650b7c56242
+  data.tar.gz: ebb4fd703ffa348040d6659b9095c5a8c5ac49e60707f08958a84f43fa46440c
 SHA512:
-  metadata.gz: e9eb11461d22069eb2f26da22dece1ad3d641d253b7aa231b107807e3d001fa86e5049b50d9ee2b9df4052519935d156bed2dfddef15caf24e6f94ed9948394b
-  data.tar.gz: 9e311bddbd3632829c109815ef6ff8a65eff147a855eaae347147b2af301d33c5837c9423536c119058a6af0864cf4ffd02a599b6d98f51a0193fdbf5a71ae9e
+  metadata.gz: 8bf432c66917846ca2982d566b570d8b1930dff31168847732132ca033c679c9595b2c0729e6d48da13814ef84e76ec4809af288ebb6097e27183b224d8cf30e
+  data.tar.gz: cd2694d0a363bf37ad38e0ec1c3d0f1dfd9488e85541169ba2cf3c56a078ce9088406e4f58d8dee2b3b87240969532d8dfaf87fa666413cf2c55da69eeeaea9d

data/.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 30
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 5
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

data/.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 *.json
+Gemfile.lock
 .rvmrc
 .ruby-gemset
 .ruby-version

data/Dockerfile

@@ -0,0 +1,34 @@
+ARG RUBY_VERSION=2.6.6
+FROM ruby:${RUBY_VERSION}-alpine
+
+LABEL maintainer="Darkbit <info@darkbit.io>"
+
+ARG USER=recon
+ARG GEM=aws_recon
+ARG VERSION=0.2.8
+ARG BUNDLER_VERSION=2.1.4
+
+# Install new Bundler version
+RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
+    gem uninstall bundler && \
+    gem install bundler -v ${BUNDLER_VERSION}
+
+# Install gem
+RUN gem install ${GEM} -v ${VERSION}
+
+# Create non-root user
+RUN addgroup -S ${USER} && \
+    adduser -S ${USER} \
+    -G ${USER} \
+    -s /bin/ash \
+    -h /${USER}
+
+# Copy binstub
+COPY binstub/${GEM} /usr/local/bundle/bin/
+RUN chmod +x /usr/local/bundle/bin/${GEM}
+
+# Switch user
+USER ${USER}
+WORKDIR /${USER}
+
+CMD ["ash"]

data/LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Darkbit
+Copyright (c) 2020 Darkbit, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/aws_recon.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'gem-release', '~> 2.1'
-  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'

data/binstub/aws_recon

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#
+# Manually generated binstub
+#
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("aws_recon", "aws_recon")

data/lib/aws_recon/aws_recon.rb

@@ -44,7 +44,7 @@ module AwsRecon
     #
     def collect(service, region)
       mapper = Object.const_get(service.name)
-      resources = mapper.new(service.name, region, @options)
+      resources = mapper.new(@account_id, service.name, region, @options)
 
       collection = resources.collect.map do |resource|
         if @options.output_format == 'custom'

data/lib/aws_recon/collectors/ec2.rb

@@ -48,6 +48,23 @@ class EC2 < Mapper
             struct.arn = instance.instance_id # no true ARN
             struct.reservation_id = reservation.reservation_id
 
+            # collect instance user_data
+            if @options.collect_user_data
+              user_data_raw = @client.describe_instance_attribute({
+                                                                    attribute: 'userData',
+                                                                    instance_id: instance.instance_id
+                                                                  }).user_data.to_h[:value]
+
+              # don't save non-string user_data
+              if user_data_raw
+                user_data = Base64.decode64(user_data_raw)
+
+                if user_data.force_encoding('UTF-8').ascii_only?
+                  struct.user_data = user_data
+                end
+              end
+            end
+
             resources.push(struct.to_h)
           end
         end

data/lib/aws_recon/collectors/elasticache.rb

@@ -0,0 +1,22 @@
+class ElastiCache < Mapper
+  def collect
+    resources = []
+
+    #
+    # describe_cache_clusters
+    #
+    @client.describe_cache_clusters.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.cache_clusters.each do |cluster|
+        struct = OpenStruct.new(cluster.to_h)
+        struct.type = 'cluster'
+        struct.arn = cluster.arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/iam.rb

@@ -10,7 +10,10 @@ class IAM < Mapper
     #   list_mfa_devices
     #   list_ssh_public_keys
     #
-    @client.get_account_authorization_details.each_with_index do |response, page|
+    opts = {
+      filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy]
+    }
+    @client.get_account_authorization_details(opts).each_with_index do |response, page|
       log(response.context.operation_name, page)
 
       # users
@@ -19,6 +22,14 @@ class IAM < Mapper
         struct.type = 'user'
         struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h)
         struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h)
+        struct.user_policy_list = if user.user_policy_list
+                                    user.user_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -27,6 +38,14 @@ class IAM < Mapper
       response.group_detail_list.each do |group|
         struct = OpenStruct.new(group.to_h)
         struct.type = 'group'
+        struct.group_policy_list = if group.group_policy_list
+                                     group.group_policy_list.map do |p|
+                                       {
+                                         policy_name: p.policy_name,
+                                         policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                       }
+                                     end
+                                    end
 
         resources.push(struct.to_h)
       end
@@ -35,6 +54,15 @@ class IAM < Mapper
       response.role_detail_list.each do |role|
         struct = OpenStruct.new(role.to_h)
         struct.type = 'role'
+        struct.assume_role_policy_document = JSON.parse(CGI.unescape(role.assume_role_policy_document))
+        struct.role_policy_list = if role.role_policy_list
+                                    role.role_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -43,6 +71,16 @@ class IAM < Mapper
       response.policies.each do |policy|
         struct = OpenStruct.new(policy.to_h)
         struct.type = 'policy'
+        struct.policy_version_list = if policy.policy_version_list
+                                       policy.policy_version_list.map do |p|
+                                         {
+                                           version_id: p.version_id,
+                                           document: JSON.parse(CGI.unescape(p.document)),
+                                           is_default_version: p.is_default_version,
+                                           create_date: p.create_date
+                                         }
+                                       end
+                                      end
 
         resources.push(struct.to_h)
       end
@@ -56,6 +94,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.password_policy.to_h)
       struct.type = 'password_policy'
+      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
       resources.push(struct.to_h)
     end
@@ -68,6 +107,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.summary_map)
       struct.type = 'account_summary'
+      struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
 
       resources.push(struct.to_h)
     end
@@ -111,6 +151,7 @@ class IAM < Mapper
 
         struct = OpenStruct.new
         struct.type = 'credential_report'
+        struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
         struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
         struct.report_format = response.report_format
         struct.generated_time = response.generated_time

data/lib/aws_recon/collectors/shield.rb

@@ -13,7 +13,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new(response.subscription.to_h)
       struct.type = 'subscription'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
 
       resources.push(struct.to_h)
     end
@@ -26,7 +26,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new
       struct.type = 'contact_list'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
       struct.contacts = response.emergency_contact_list.map(&:to_h)
 
       resources.push(struct.to_h)

data/lib/aws_recon/lib/formatter.rb

@@ -8,7 +8,7 @@ class Formatter
   def custom(account_id, region, service, resource)
     {
       account: account_id,
-      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      name: resource[:arn],
       service: service.name,
       region: region,
       asset_type: resource[:type],

data/lib/aws_recon/lib/mapper.rb

@@ -22,7 +22,8 @@ class Mapper
   #   S3 (unless the bucket was created in another region)
   SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
 
-  def initialize(service, region, options)
+  def initialize(account, service, region, options)
+    @account = account
     @service = service
     @region = region
     @options = options

data/lib/aws_recon/options.rb

@@ -15,6 +15,7 @@ class Parser
     :output_file,
     :output_format,
     :threads,
+    :collect_user_data,
     :skip_slow,
     :stream_output,
     :verbose,
@@ -43,6 +44,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -103,6 +105,11 @@ class Parser
         end
       end
 
+      # collect EC2 instance user data
+      opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do
+        args.collect_user_data = true
+      end
+
       # skip slow operations
       opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do
         args.skip_slow = true

data/lib/aws_recon/services.yaml

@@ -21,8 +21,6 @@
   alias: ec2
 - name: EKS
   alias: eks
-  excluded_regions:
-    - us-west-1
 - name: ECS
   alias: ecs
 - name: ElasticLoadBalancing
@@ -33,6 +31,8 @@
   alias: elbv2
   excluded_regions:
     - ap-southeast-1
+- name: ElastiCache
+  alias: elasticache
 - name: IAM
   global: true
   alias: iam

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.4"
+  VERSION = "0.2.9"
 end

data/readme.md

@@ -26,18 +26,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Install the gem:
+AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+
+To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.2.gem
+Fetching aws_recon-0.2.8.gem
 Fetching aws-sdk-resources-3.76.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.19.2.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.2
+Successfully installed aws_recon-0.2.8
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -49,9 +51,23 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel 1.19.2
-Using aws_recon 0.2.2
+Using aws_recon 0.2.8
+```
+
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
 ```
 
+
 ## Usage
 
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
@@ -66,6 +82,39 @@ Plain environment variables will work fine too.
 $ AWS_PROFILE=<profile> aws_recon
 ```
 
+To run from a Docker container using `aws-vault` managed credentials (output to stdout):
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  darkbitio/aws_recon:latest \
+  aws_recon -j -s EC2 -r global,us-east-1,us-east-2
+```
+
+To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
+
+Create an empty file.
+
+```
+$ touch output.json
+```
+
+Run the `aws_recon` container, specifying the output file.
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -s EC2 -v -r global,us-east-1,us-east-2
+```
+
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
 
 In verbose mode, the console output will show:
@@ -135,7 +184,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector
+AWS Recon - AWS Inventory Collector (0.2.8)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -146,6 +195,7 @@ Usage: aws_recon [options]
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
     -j, --stream-output              Stream JSON lines to stdout (default: false)
     -v, --verbose                    Output client progress and current operation
@@ -193,6 +243,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] ELB
 - [x] EKS
 - [x] Elasticsearch
+- [x] ElastiCache
 - [x] Firehose
 - [ ] FMS
 - [ ] Glacier