aws_recon 0.2.5 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a7f47022478c23d46ab98689197a4f986b4dd9b0c31f8b735196e7cb6367f43
-  data.tar.gz: '09b8c94ee155aec24c9b33912cc2a547cb0be6e022e4a5bbdcce6cce3ed72877'
+  metadata.gz: 9ed0fd50b4c2a0542194d2844f4e95c98e2f09ce3802878c0463f78cec8a8f92
+  data.tar.gz: 5112bb3ec8e07f61a4ba0f673044486f8423fb61710893650c11cc0e793bd9e1
 SHA512:
-  metadata.gz: 98f874c9b86214db1e351aeba46d23c7fbf1c33e88bd3164f1002a17b4643a3b46f65b6702efef41a90083c38631ac737661f98f867049165c8ba4c66a3b28dc
-  data.tar.gz: c2d70e2d05ee8e9161a8f90d2d066744205e55b0c383c1c8246f827aa1a8f75fa4651745612471f2d6baf0474a46b8e28d1a50bb336f759f3e9437cf36f4ce91
+  metadata.gz: 2dc80a5605f4c8673efb9f026e7485a66f7d190fb57e9eb636fd67a8e31db39b857f1d2045b9e56b9ced86dd31f186290f94eac2ef87de3671e4709e604a2d20
+  data.tar.gz: a045657734baf60b898d8f78dc94eef3ae69645fcccae85288beddfb2cd46ed1d330edc773fd88e058b1ddaf5031e939df36523398c8b3cc8cdf42e17cb74e64

data/.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 *.json
+Gemfile.lock
 .rvmrc
 .ruby-gemset
 .ruby-version

data/Dockerfile

@@ -0,0 +1,34 @@
+ARG RUBY_VERSION=2.6.6
+FROM ruby:${RUBY_VERSION}-alpine
+
+LABEL maintainer="Darkbit <info@darkbit.io>"
+
+ARG USER=recon
+ARG GEM=aws_recon
+ARG VERSION=0.2.9
+ARG BUNDLER_VERSION=2.1.4
+
+# Install new Bundler version
+RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
+    gem uninstall bundler && \
+    gem install bundler -v ${BUNDLER_VERSION}
+
+# Install gem
+RUN gem install ${GEM} -v ${VERSION}
+
+# Create non-root user
+RUN addgroup -S ${USER} && \
+    adduser -S ${USER} \
+    -G ${USER} \
+    -s /bin/ash \
+    -h /${USER}
+
+# Copy binstub
+COPY binstub/${GEM} /usr/local/bundle/bin/
+RUN chmod +x /usr/local/bundle/bin/${GEM}
+
+# Switch user
+USER ${USER}
+WORKDIR /${USER}
+
+CMD ["ash"]

data/LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Darkbit
+Copyright (c) 2020 Darkbit, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/aws_recon.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'gem-release', '~> 2.1'
-  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'

data/binstub/aws_recon

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#
+# Manually generated binstub
+#
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("aws_recon", "aws_recon")

data/lib/aws_recon/aws_recon.rb

@@ -44,7 +44,7 @@ module AwsRecon
     #
     def collect(service, region)
       mapper = Object.const_get(service.name)
-      resources = mapper.new(service.name, region, @options)
+      resources = mapper.new(@account_id, service.name, region, @options)
 
       collection = resources.collect.map do |resource|
         if @options.output_format == 'custom'

data/lib/aws_recon/collectors/elasticache.rb

@@ -0,0 +1,22 @@
+class ElastiCache < Mapper
+  def collect
+    resources = []
+
+    #
+    # describe_cache_clusters
+    #
+    @client.describe_cache_clusters.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.cache_clusters.each do |cluster|
+        struct = OpenStruct.new(cluster.to_h)
+        struct.type = 'cluster'
+        struct.arn = cluster.arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/iam.rb

@@ -10,7 +10,10 @@ class IAM < Mapper
     #   list_mfa_devices
     #   list_ssh_public_keys
     #
-    @client.get_account_authorization_details.each_with_index do |response, page|
+    opts = {
+      filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy]
+    }
+    @client.get_account_authorization_details(opts).each_with_index do |response, page|
       log(response.context.operation_name, page)
 
       # users
@@ -19,6 +22,14 @@ class IAM < Mapper
         struct.type = 'user'
         struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h)
         struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h)
+        struct.user_policy_list = if user.user_policy_list
+                                    user.user_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -27,6 +38,14 @@ class IAM < Mapper
       response.group_detail_list.each do |group|
         struct = OpenStruct.new(group.to_h)
         struct.type = 'group'
+        struct.group_policy_list = if group.group_policy_list
+                                     group.group_policy_list.map do |p|
+                                       {
+                                         policy_name: p.policy_name,
+                                         policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                       }
+                                     end
+                                    end
 
         resources.push(struct.to_h)
       end
@@ -35,6 +54,15 @@ class IAM < Mapper
       response.role_detail_list.each do |role|
         struct = OpenStruct.new(role.to_h)
         struct.type = 'role'
+        struct.assume_role_policy_document = JSON.parse(CGI.unescape(role.assume_role_policy_document))
+        struct.role_policy_list = if role.role_policy_list
+                                    role.role_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -43,6 +71,16 @@ class IAM < Mapper
       response.policies.each do |policy|
         struct = OpenStruct.new(policy.to_h)
         struct.type = 'policy'
+        struct.policy_version_list = if policy.policy_version_list
+                                       policy.policy_version_list.map do |p|
+                                         {
+                                           version_id: p.version_id,
+                                           document: JSON.parse(CGI.unescape(p.document)),
+                                           is_default_version: p.is_default_version,
+                                           create_date: p.create_date
+                                         }
+                                       end
+                                      end
 
         resources.push(struct.to_h)
       end
@@ -56,6 +94,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.password_policy.to_h)
       struct.type = 'password_policy'
+      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
       resources.push(struct.to_h)
     end
@@ -68,6 +107,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.summary_map)
       struct.type = 'account_summary'
+      struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
 
       resources.push(struct.to_h)
     end
@@ -111,6 +151,7 @@ class IAM < Mapper
 
         struct = OpenStruct.new
         struct.type = 'credential_report'
+        struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
         struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
         struct.report_format = response.report_format
         struct.generated_time = response.generated_time

data/lib/aws_recon/collectors/organizations.rb

@@ -31,6 +31,21 @@ class Organizations < Mapper
       end
     end
 
+    #
+    # list_policies
+    #
+    @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.policies.each do |policy|
+        struct = OpenStruct.new(policy.to_h)
+        struct.type = 'service_control_policy'
+        struct.content = JSON.parse(CGI.unescape(@client.describe_policy({ policy_id: policy.id }).policy.content))
+
+        resources.push(struct.to_h)
+      end
+    end
+
     resources
   end
 end

data/lib/aws_recon/collectors/shield.rb

@@ -13,7 +13,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new(response.subscription.to_h)
       struct.type = 'subscription'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
 
       resources.push(struct.to_h)
     end
@@ -26,7 +26,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new
       struct.type = 'contact_list'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
       struct.contacts = response.emergency_contact_list.map(&:to_h)
 
       resources.push(struct.to_h)

data/lib/aws_recon/collectors/sqs.rb

@@ -18,6 +18,7 @@ class SQS < Mapper
         struct = OpenStruct.new(@client.get_queue_attributes({ queue_url: queue, attribute_names: ['All'] }).attributes.to_h)
         struct.type = 'queue'
         struct.arn = struct.QueueArn
+        struct.Policy = JSON.parse(CGI.unescape(struct.Policy))
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/lib/formatter.rb

@@ -8,7 +8,7 @@ class Formatter
   def custom(account_id, region, service, resource)
     {
       account: account_id,
-      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      name: resource[:arn],
       service: service.name,
       region: region,
       asset_type: resource[:type],

data/lib/aws_recon/lib/mapper.rb

@@ -22,7 +22,8 @@ class Mapper
   #   S3 (unless the bucket was created in another region)
   SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
 
-  def initialize(service, region, options)
+  def initialize(account, service, region, options)
+    @account = account
     @service = service
     @region = region
     @options = options

data/lib/aws_recon/services.yaml

@@ -21,8 +21,6 @@
   alias: ec2
 - name: EKS
   alias: eks
-  excluded_regions:
-    - us-west-1
 - name: ECS
   alias: ecs
 - name: ElasticLoadBalancing
@@ -33,6 +31,8 @@
   alias: elbv2
   excluded_regions:
     - ap-southeast-1
+- name: ElastiCache
+  alias: elasticache
 - name: IAM
   global: true
   alias: iam

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.5"
+  VERSION = "0.2.10"
 end

data/readme.md

@@ -26,18 +26,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Install the gem:
+AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+
+To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.2.gem
+Fetching aws_recon-0.2.8.gem
 Fetching aws-sdk-resources-3.76.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.19.2.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.2
+Successfully installed aws_recon-0.2.8
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -49,9 +51,23 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel 1.19.2
-Using aws_recon 0.2.2
+Using aws_recon 0.2.8
+```
+
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
 ```
 
+
 ## Usage
 
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
@@ -66,6 +82,39 @@ Plain environment variables will work fine too.
 $ AWS_PROFILE=<profile> aws_recon
 ```
 
+To run from a Docker container using `aws-vault` managed credentials (output to stdout):
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  darkbitio/aws_recon:latest \
+  aws_recon -j -s EC2 -r global,us-east-1,us-east-2
+```
+
+To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
+
+Create an empty file.
+
+```
+$ touch output.json
+```
+
+Run the `aws_recon` container, specifying the output file.
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -s EC2 -v -r global,us-east-1,us-east-2
+```
+
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
 
 In verbose mode, the console output will show:
@@ -109,6 +158,12 @@ $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
+Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted output.
+
+```
+$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2 -f custom > output.json
+```
+
 #### Errors
 
 An exception will be raised on `AccessDeniedException` errors. This typically means your user/role doesn't have the necessary permissions to get/list/describe for that service. These exceptions are raised so troubleshooting access issues is easier.
@@ -135,7 +190,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector
+AWS Recon - AWS Inventory Collector (0.2.8)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -146,6 +201,7 @@ Usage: aws_recon [options]
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
     -j, --stream-output              Stream JSON lines to stdout (default: false)
     -v, --verbose                    Output client progress and current operation
@@ -193,6 +249,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] ELB
 - [x] EKS
 - [x] Elasticsearch
+- [x] ElastiCache
 - [x] Firehose
 - [ ] FMS
 - [ ] Glacier

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.10
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -71,16 +71,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -152,14 +152,15 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- Dockerfile
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - Rakefile
 - aws_recon.gemspec
 - bin/aws_recon
 - bin/console
 - bin/setup
+- binstub/aws_recon
 - lib/aws_recon.rb
 - lib/aws_recon/aws_recon.rb
 - lib/aws_recon/collectors.rb
@@ -185,6 +186,7 @@ files:
 - lib/aws_recon/collectors/ecs.rb
 - lib/aws_recon/collectors/efs.rb
 - lib/aws_recon/collectors/eks.rb
+- lib/aws_recon/collectors/elasticache.rb
 - lib/aws_recon/collectors/elasticloadbalancing.rb
 - lib/aws_recon/collectors/elasticloadbalancingv2.rb
 - lib/aws_recon/collectors/elasticsearch.rb