aws_recon 0.2.6 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2707134a4d4be23f0f1cd3320fda7e75463aae49113b2f4b802257aa02b0de52
-  data.tar.gz: 85452e37a3657d315fae2b67852bc521786a95fc386db53f4a3212801e140ee4
+  metadata.gz: 342238f4c9197721172d05451ed5c3346475b52ae807f27607b83036d15280dd
+  data.tar.gz: 3f26ec0ad41491836331076bac849ab967db5640e9295413db30b21f1567a324
 SHA512:
-  metadata.gz: 5e1a919375c34b51a72a3bb470ce178b39c178e8e3a569fb47f618c8ac122512e14c2564e42bd72aab3e5a550041f8e9490c3f3d6245ac2d3c933e77a9053d18
-  data.tar.gz: e93eaad39853ffdd5c934572b2d9fb2add88efc0150bc48cbc3cf7cef542c25a98a2ebd1c92e6b64404dde122d86d8d2d38638f6bd5fde0e11974c2c52e5cdc1
+  metadata.gz: b1591c03b3d1608d9a4f38987ff2f7ffce879d685966214184eaa7e8bdc6bb5f39cc2e7e0a99235e48ee6f2b25d927189bc8f8873b693b212ff2671819bd641b
+  data.tar.gz: 407bb475473ba08694681a77a7c8212b4406b8587ba668dfdff0947f514c48fba065cfd5f94e08d8a5cc0567886af944a31c3d0518d1a15537bb8682049078b2

data/Dockerfile

@@ -0,0 +1,34 @@
+ARG RUBY_VERSION=2.6.6
+FROM ruby:${RUBY_VERSION}-alpine
+
+LABEL maintainer="Darkbit <info@darkbit.io>"
+
+ARG USER=recon
+ARG GEM=aws_recon
+ARG VERSION=0.2.10
+ARG BUNDLER_VERSION=2.1.4
+
+# Install new Bundler version
+RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
+    gem uninstall bundler && \
+    gem install bundler -v ${BUNDLER_VERSION}
+
+# Install gem
+RUN gem install ${GEM} -v ${VERSION}
+
+# Create non-root user
+RUN addgroup -S ${USER} && \
+    adduser -S ${USER} \
+    -G ${USER} \
+    -s /bin/ash \
+    -h /${USER}
+
+# Copy binstub
+COPY binstub/${GEM} /usr/local/bundle/bin/
+RUN chmod +x /usr/local/bundle/bin/${GEM}
+
+# Switch user
+USER ${USER}
+WORKDIR /${USER}
+
+CMD ["ash"]

data/LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Darkbit
+Copyright (c) 2020 Darkbit, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/aws_recon.gemspec

@@ -28,9 +28,10 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'gem-release', '~> 2.1'
-  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'
   spec.add_development_dependency 'pry', '~> 0.13.1'
+  spec.add_development_dependency 'byebug', '~> 11.1'
 end

data/binstub/aws_recon

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#
+# Manually generated binstub
+#
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("aws_recon", "aws_recon")

data/lib/aws_recon/aws_recon.rb

@@ -44,7 +44,7 @@ module AwsRecon
     #
     def collect(service, region)
       mapper = Object.const_get(service.name)
-      resources = mapper.new(service.name, region, @options)
+      resources = mapper.new(@account_id, service.name, region, @options)
 
       collection = resources.collect.map do |resource|
         if @options.output_format == 'custom'

data/lib/aws_recon/collectors/iam.rb

@@ -10,7 +10,10 @@ class IAM < Mapper
     #   list_mfa_devices
     #   list_ssh_public_keys
     #
-    @client.get_account_authorization_details.each_with_index do |response, page|
+    opts = {
+      filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy]
+    }
+    @client.get_account_authorization_details(opts).each_with_index do |response, page|
       log(response.context.operation_name, page)
 
       # users
@@ -19,6 +22,14 @@ class IAM < Mapper
         struct.type = 'user'
         struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h)
         struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h)
+        struct.user_policy_list = if user.user_policy_list
+                                    user.user_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -27,6 +38,14 @@ class IAM < Mapper
       response.group_detail_list.each do |group|
         struct = OpenStruct.new(group.to_h)
         struct.type = 'group'
+        struct.group_policy_list = if group.group_policy_list
+                                     group.group_policy_list.map do |p|
+                                       {
+                                         policy_name: p.policy_name,
+                                         policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                       }
+                                     end
+                                    end
 
         resources.push(struct.to_h)
       end
@@ -35,6 +54,15 @@ class IAM < Mapper
       response.role_detail_list.each do |role|
         struct = OpenStruct.new(role.to_h)
         struct.type = 'role'
+        struct.assume_role_policy_document = JSON.parse(CGI.unescape(role.assume_role_policy_document))
+        struct.role_policy_list = if role.role_policy_list
+                                    role.role_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -43,6 +71,16 @@ class IAM < Mapper
       response.policies.each do |policy|
         struct = OpenStruct.new(policy.to_h)
         struct.type = 'policy'
+        struct.policy_version_list = if policy.policy_version_list
+                                       policy.policy_version_list.map do |p|
+                                         {
+                                           version_id: p.version_id,
+                                           document: JSON.parse(CGI.unescape(p.document)),
+                                           is_default_version: p.is_default_version,
+                                           create_date: p.create_date
+                                         }
+                                       end
+                                      end
 
         resources.push(struct.to_h)
       end
@@ -56,6 +94,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.password_policy.to_h)
       struct.type = 'password_policy'
+      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
       resources.push(struct.to_h)
     end
@@ -68,6 +107,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.summary_map)
       struct.type = 'account_summary'
+      struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
 
       resources.push(struct.to_h)
     end
@@ -102,6 +142,24 @@ class IAM < Mapper
       end
     end
 
+    #
+    # generate_credential_report
+    #
+    unless @options.skip_credential_report
+      status = 'STARTED'
+      interval = 5
+
+      # wait for report to generate
+      while status != 'COMPLETE'
+        @client.generate_credential_report.each do |response|
+          log(response.context.operation_name)
+          status = response.state
+        end
+
+        sleep interval unless status == 'COMPLETE'
+      end
+    end
+
     #
     # get_credential_report
     #
@@ -111,6 +169,7 @@ class IAM < Mapper
 
         struct = OpenStruct.new
         struct.type = 'credential_report'
+        struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
         struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
         struct.report_format = response.report_format
         struct.generated_time = response.generated_time

data/lib/aws_recon/collectors/organizations.rb

@@ -31,6 +31,21 @@ class Organizations < Mapper
       end
     end
 
+    #
+    # list_policies
+    #
+    @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.policies.each do |policy|
+        struct = OpenStruct.new(policy.to_h)
+        struct.type = 'service_control_policy'
+        struct.content = JSON.parse(CGI.unescape(@client.describe_policy({ policy_id: policy.id }).policy.content))
+
+        resources.push(struct.to_h)
+      end
+    end
+
     resources
   end
 end

data/lib/aws_recon/collectors/shield.rb

@@ -13,7 +13,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new(response.subscription.to_h)
       struct.type = 'subscription'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
 
       resources.push(struct.to_h)
     end
@@ -26,7 +26,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new
       struct.type = 'contact_list'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
       struct.contacts = response.emergency_contact_list.map(&:to_h)
 
       resources.push(struct.to_h)

data/lib/aws_recon/collectors/sqs.rb

@@ -18,6 +18,7 @@ class SQS < Mapper
         struct = OpenStruct.new(@client.get_queue_attributes({ queue_url: queue, attribute_names: ['All'] }).attributes.to_h)
         struct.type = 'queue'
         struct.arn = struct.QueueArn
+        struct.Policy = JSON.parse(CGI.unescape(struct.Policy))
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/lib/formatter.rb

@@ -8,7 +8,7 @@ class Formatter
   def custom(account_id, region, service, resource)
     {
       account: account_id,
-      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      name: resource[:arn],
       service: service.name,
       region: region,
       asset_type: resource[:type],

data/lib/aws_recon/lib/mapper.rb

@@ -22,7 +22,8 @@ class Mapper
   #   S3 (unless the bucket was created in another region)
   SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
 
-  def initialize(service, region, options)
+  def initialize(account, service, region, options)
+    @account = account
     @service = service
     @region = region
     @options = options

data/lib/aws_recon/options.rb

@@ -17,6 +17,7 @@ class Parser
     :threads,
     :collect_user_data,
     :skip_slow,
+    :skip_credential_report,
     :stream_output,
     :verbose,
     :debug
@@ -45,6 +46,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -115,6 +117,11 @@ class Parser
         args.skip_slow = true
       end
 
+      # skip generating IAM credential report
+      opts.on('-g', '--skip-credential-report', 'Skip generating IAM credential report (default: false)') do
+        args.skip_credential_report = true
+      end
+
       # stream output (forces JSON lines, doesn't output handled warnings or errors )
       opts.on('-j', '--stream-output', 'Stream JSON lines to stdout (default: false)') do
         args.output_file = nil

data/lib/aws_recon/services.yaml

@@ -21,8 +21,6 @@
   alias: ec2
 - name: EKS
   alias: eks
-  excluded_regions:
-    - us-west-1
 - name: ECS
   alias: ecs
 - name: ElasticLoadBalancing

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.6"
+  VERSION = "0.2.11"
 end

data/readme.md

@@ -26,18 +26,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Install the gem:
+AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+
+To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.6.gem
+Fetching aws_recon-0.2.8.gem
 Fetching aws-sdk-resources-3.76.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.19.2.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.6
+Successfully installed aws_recon-0.2.8
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -49,9 +51,23 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel 1.19.2
-Using aws_recon 0.2.2
+Using aws_recon 0.2.8
+```
+
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
 ```
 
+
 ## Usage
 
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
@@ -66,6 +82,39 @@ Plain environment variables will work fine too.
 $ AWS_PROFILE=<profile> aws_recon
 ```
 
+To run from a Docker container using `aws-vault` managed credentials (output to stdout):
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  darkbitio/aws_recon:latest \
+  aws_recon -j -s EC2 -r global,us-east-1,us-east-2
+```
+
+To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
+
+Create an empty file.
+
+```
+$ touch output.json
+```
+
+Run the `aws_recon` container, specifying the output file.
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -s EC2 -v -r global,us-east-1,us-east-2
+```
+
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
 
 In verbose mode, the console output will show:
@@ -109,6 +158,12 @@ $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
+Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted output.
+
+```
+$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2 -f custom > output.json
+```
+
 #### Errors
 
 An exception will be raised on `AccessDeniedException` errors. This typically means your user/role doesn't have the necessary permissions to get/list/describe for that service. These exceptions are raised so troubleshooting access issues is easier.
@@ -135,7 +190,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.2.6)
+AWS Recon - AWS Inventory Collector (0.2.8)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.11
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-19 00:00:00.000000000 Z
+date: 2020-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -71,16 +71,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -137,6 +137,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.13.1
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
 description: AWS Recon is a command line tool to collect resources from an Amazon
   Web Services (AWS) account. The tool outputs JSON suitable for processing with other
   tools.
@@ -152,6 +166,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - Rakefile
@@ -159,6 +174,7 @@ files:
 - bin/aws_recon
 - bin/console
 - bin/setup
+- binstub/aws_recon
 - lib/aws_recon.rb
 - lib/aws_recon/aws_recon.rb
 - lib/aws_recon/collectors.rb