aws_recon 0.2.7 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5378dc5f65acecaf982ff59b6d4330561a03d3ed68b8ec56051126e64dff9b09
-  data.tar.gz: d4c1c151b7a96c66e0bf541fc198f479622d06ced30f1197dd912afd965122fd
+  metadata.gz: 91c04c23df1fe4fb4ca28ae164c15bc2994a63b8acf0b33b07aedeeb7a11f021
+  data.tar.gz: 148087c3e082a71c78efccdf6628a9d70115a4f409ac9c7dfdb503218e11d168
 SHA512:
-  metadata.gz: 65bfec760bd658d331a505d9faa37c3bf5c27b6e36d6b963c962ae963fa7da738e7ba4b2d48f584c94b70fba6d1ce441de3c4e23a594fb0d81e0cb59066c8b07
-  data.tar.gz: cf33e89b9faf71b70dbf555e14d2bbf8190fbc81aa8c2504f43e5773dc614fc3a5a2b086360d02491ffdf626abf8a91e3bcdf019e6f54637aafe36ec9f0caf83
+  metadata.gz: 53d87f172f074567da9b2d31ff9fad2e3507bef9e95da5a419ff30e3fbb70c00c46bc70b8b9b1af57da400aceb8a524b79c042dda648cd6098a9792b5eb628ea
+  data.tar.gz: 90396f3003e5d34b8dd21a582fc2bab93f11b28744cac6c207658e16c4f43376fa42932d03e3e8438b73fd80400036af172b7573da82fa38043e9ba78e29137c

data/.github/workflows/docker-build.yml

@@ -0,0 +1,38 @@
+name: docker-build
+
+on:
+  push:
+    branches: build
+    paths:
+      - 'lib/aws_recon/version.rb '
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set version tag
+        run: |
+          echo "VERSION_TAG=$(grep VERSION lib/aws_recon/version.rb | awk -F\" '{print $2}')" >> $GITHUB_ENV
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          build-args: |
+            VERSION=${{ env.VERSION_TAG }}
+          tags: |
+            darkbitio/aws_recon:${{ env.VERSION_TAG }}
+            darkbitio/aws_recon:latest

data/.github/workflows/smoke-test.yml

@@ -0,0 +1,23 @@
+name: smoke-test
+
+on:
+  push:
+    branches: main
+
+jobs:
+  smoke-test:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Set version tag
+        run: |
+          echo "VERSION_TAG=$(grep VERSION lib/aws_recon/version.rb | awk -F\" '{print $2}')" >> $GITHUB_ENV
+      - name: Smoke Test :${{ env.VERSION_TAG }}
+        run: |
+          docker run -t --rm darkbitio/aws_recon:${{ env.VERSION_TAG }} aws_recon
+      - name: Smoke Test :latest
+        run: |
+          docker run -t --rm darkbitio/aws_recon:latest aws_recon

data/Dockerfile

@@ -0,0 +1,35 @@
+ARG RUBY_VERSION=2.6.6
+FROM ruby:${RUBY_VERSION}-alpine
+
+LABEL maintainer="Darkbit <info@darkbit.io>"
+
+# Supply AWS Recon version at build time
+ARG VERSION
+ARG USER=recon
+ARG GEM=aws_recon
+ARG BUNDLER_VERSION=2.1.4
+
+# Install new Bundler version
+RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
+    gem uninstall bundler && \
+    gem install bundler -v ${BUNDLER_VERSION}
+
+# Install gem
+RUN gem install ${GEM} -v ${VERSION}
+
+# Create non-root user
+RUN addgroup -S ${USER} && \
+    adduser -S ${USER} \
+    -G ${USER} \
+    -s /bin/ash \
+    -h /${USER}
+
+# Copy binstub
+COPY binstub/${GEM} /usr/local/bundle/bin/
+RUN chmod +x /usr/local/bundle/bin/${GEM}
+
+# Switch user
+USER ${USER}
+WORKDIR /${USER}
+
+CMD ["ash"]

data/LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Darkbit
+Copyright (c) 2020 Darkbit, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/aws_recon.gemspec

@@ -33,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'
   spec.add_development_dependency 'pry', '~> 0.13.1'
+  spec.add_development_dependency 'byebug', '~> 11.1'
 end

data/binstub/aws_recon

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#
+# Manually generated binstub
+#
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("aws_recon", "aws_recon")

data/lib/aws_recon/aws_recon.rb

@@ -44,7 +44,7 @@ module AwsRecon
     #
     def collect(service, region)
       mapper = Object.const_get(service.name)
-      resources = mapper.new(service.name, region, @options)
+      resources = mapper.new(@account_id, service.name, region, @options)
 
       collection = resources.collect.map do |resource|
         if @options.output_format == 'custom'

data/lib/aws_recon/collectors/accessanalyzer.rb

@@ -0,0 +1,24 @@
+class AccessAnalyzer < Mapper
+  #
+  # Returns an array of resources.
+  #
+  def collect
+    resources = []
+
+    #
+    # list_analyzers
+    #
+    @client.list_analyzers.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      # analyzers
+      response.analyzers.each do |analyzer|
+        struct = OpenStruct.new(analyzer.to_h)
+        struct.type = 'analyzer'
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/ec2.rb

@@ -31,6 +31,18 @@ class EC2 < Mapper
 
     # regional calls
     if @region != 'global'
+      #
+      # get_ebs_encryption_by_default
+      #
+      @client.get_ebs_encryption_by_default.each do |response|
+        log(response.context.operation_name)
+
+        struct = OpenStruct.new(response.to_h)
+        struct.type = 'ebs_encryption_settings'
+
+        resources.push(struct.to_h)
+      end
+
       #
       # describe_instances
       #

data/lib/aws_recon/collectors/iam.rb

@@ -10,7 +10,10 @@ class IAM < Mapper
     #   list_mfa_devices
     #   list_ssh_public_keys
     #
-    @client.get_account_authorization_details.each_with_index do |response, page|
+    opts = {
+      filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy]
+    }
+    @client.get_account_authorization_details(opts).each_with_index do |response, page|
       log(response.context.operation_name, page)
 
       # users
@@ -19,6 +22,14 @@ class IAM < Mapper
         struct.type = 'user'
         struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h)
         struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h)
+        struct.user_policy_list = if user.user_policy_list
+                                    user.user_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -27,6 +38,14 @@ class IAM < Mapper
       response.group_detail_list.each do |group|
         struct = OpenStruct.new(group.to_h)
         struct.type = 'group'
+        struct.group_policy_list = if group.group_policy_list
+                                     group.group_policy_list.map do |p|
+                                       {
+                                         policy_name: p.policy_name,
+                                         policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                       }
+                                     end
+                                    end
 
         resources.push(struct.to_h)
       end
@@ -35,6 +54,15 @@ class IAM < Mapper
       response.role_detail_list.each do |role|
         struct = OpenStruct.new(role.to_h)
         struct.type = 'role'
+        struct.assume_role_policy_document = JSON.parse(CGI.unescape(role.assume_role_policy_document))
+        struct.role_policy_list = if role.role_policy_list
+                                    role.role_policy_list.map do |p|
+                                      {
+                                        policy_name: p.policy_name,
+                                        policy_document: JSON.parse(CGI.unescape(p.policy_document))
+                                      }
+                                    end
+                                  end
 
         resources.push(struct.to_h)
       end
@@ -43,6 +71,16 @@ class IAM < Mapper
       response.policies.each do |policy|
         struct = OpenStruct.new(policy.to_h)
         struct.type = 'policy'
+        struct.policy_version_list = if policy.policy_version_list
+                                       policy.policy_version_list.map do |p|
+                                         {
+                                           version_id: p.version_id,
+                                           document: JSON.parse(CGI.unescape(p.document)),
+                                           is_default_version: p.is_default_version,
+                                           create_date: p.create_date
+                                         }
+                                       end
+                                      end
 
         resources.push(struct.to_h)
       end
@@ -56,6 +94,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.password_policy.to_h)
       struct.type = 'password_policy'
+      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
       resources.push(struct.to_h)
     end
@@ -68,6 +107,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.summary_map)
       struct.type = 'account_summary'
+      struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
 
       resources.push(struct.to_h)
     end
@@ -102,6 +142,24 @@ class IAM < Mapper
       end
     end
 
+    #
+    # generate_credential_report
+    #
+    unless @options.skip_credential_report
+      status = 'STARTED'
+      interval = 5
+
+      # wait for report to generate
+      while status != 'COMPLETE'
+        @client.generate_credential_report.each do |response|
+          log(response.context.operation_name)
+          status = response.state
+        end
+
+        sleep interval unless status == 'COMPLETE'
+      end
+    end
+
     #
     # get_credential_report
     #
@@ -111,6 +169,7 @@ class IAM < Mapper
 
         struct = OpenStruct.new
         struct.type = 'credential_report'
+        struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
         struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
         struct.report_format = response.report_format
         struct.generated_time = response.generated_time

data/lib/aws_recon/collectors/organizations.rb

@@ -31,6 +31,21 @@ class Organizations < Mapper
       end
     end
 
+    #
+    # list_policies
+    #
+    @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.policies.each do |policy|
+        struct = OpenStruct.new(policy.to_h)
+        struct.type = 'service_control_policy'
+        struct.content = JSON.parse(CGI.unescape(@client.describe_policy({ policy_id: policy.id }).policy.content))
+
+        resources.push(struct.to_h)
+      end
+    end
+
     resources
   end
 end

data/lib/aws_recon/collectors/shield.rb

@@ -13,7 +13,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new(response.subscription.to_h)
       struct.type = 'subscription'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
 
       resources.push(struct.to_h)
     end
@@ -26,7 +26,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new
       struct.type = 'contact_list'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
       struct.contacts = response.emergency_contact_list.map(&:to_h)
 
       resources.push(struct.to_h)

data/lib/aws_recon/collectors/sqs.rb

@@ -18,6 +18,7 @@ class SQS < Mapper
         struct = OpenStruct.new(@client.get_queue_attributes({ queue_url: queue, attribute_names: ['All'] }).attributes.to_h)
         struct.type = 'queue'
         struct.arn = struct.QueueArn
+        struct.Policy = JSON.parse(CGI.unescape(struct.Policy))
 
         resources.push(struct.to_h)
       end

data/lib/aws_recon/lib/formatter.rb

@@ -8,7 +8,7 @@ class Formatter
   def custom(account_id, region, service, resource)
     {
       account: account_id,
-      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      name: resource[:arn],
       service: service.name,
       region: region,
       asset_type: resource[:type],

data/lib/aws_recon/lib/mapper.rb

@@ -22,7 +22,8 @@ class Mapper
   #   S3 (unless the bucket was created in another region)
   SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
 
-  def initialize(service, region, options)
+  def initialize(account, service, region, options)
+    @account = account
     @service = service
     @region = region
     @options = options

data/lib/aws_recon/options.rb

@@ -17,6 +17,7 @@ class Parser
     :threads,
     :collect_user_data,
     :skip_slow,
+    :skip_credential_report,
     :stream_output,
     :verbose,
     :debug
@@ -45,6 +46,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -115,6 +117,11 @@ class Parser
         args.skip_slow = true
       end
 
+      # skip generating IAM credential report
+      opts.on('-g', '--skip-credential-report', 'Skip generating IAM credential report (default: false)') do
+        args.skip_credential_report = true
+      end
+
       # stream output (forces JSON lines, doesn't output handled warnings or errors )
       opts.on('-j', '--stream-output', 'Stream JSON lines to stdout (default: false)') do
         args.output_file = nil

data/lib/aws_recon/services.yaml

@@ -2,6 +2,8 @@
 - name: Organizations
   global: true
   alias: organizations
+- name: AccessAnalyzer
+  alias: aa
 - name: ConfigService
   alias: config
 - name: CodeBuild

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.7"
+  VERSION = "0.2.12"
 end

data/readme.md

@@ -1,3 +1,4 @@
+![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/darkbitio/aws-recon/smoke-test/main)
 [![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://badge.fury.io/rb/aws_recon)
 
 # AWS Recon
@@ -26,18 +27,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Install the gem:
+AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+
+To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.7.gem
+Fetching aws_recon-0.2.8.gem
 Fetching aws-sdk-resources-3.76.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.19.2.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.7
+Successfully installed aws_recon-0.2.8
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -49,9 +52,23 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel 1.19.2
-Using aws_recon 0.2.2
+Using aws_recon 0.2.8
+```
+
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -v -s EC2 -r global,us-east-1,us-east-2
 ```
 
+
 ## Usage
 
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
@@ -66,6 +83,39 @@ Plain environment variables will work fine too.
 $ AWS_PROFILE=<profile> aws_recon
 ```
 
+To run from a Docker container using `aws-vault` managed credentials (output to stdout):
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  darkbitio/aws_recon:latest \
+  aws_recon -j -s EC2 -r global,us-east-1,us-east-2
+```
+
+To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
+
+Create an empty file.
+
+```
+$ touch output.json
+```
+
+Run the `aws_recon` container, specifying the output file.
+
+```
+$ aws-vault exec <vault_profile> -- docker run -t --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  darkbitio/aws_recon:latest \
+  aws_recon -s EC2 -v -r global,us-east-1,us-east-2
+```
+
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
 
 In verbose mode, the console output will show:
@@ -109,6 +159,12 @@ $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
+Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted output.
+
+```
+$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2 -f custom > output.json
+```
+
 #### Errors
 
 An exception will be raised on `AccessDeniedException` errors. This typically means your user/role doesn't have the necessary permissions to get/list/describe for that service. These exceptions are raised so troubleshooting access issues is easier.
@@ -135,7 +191,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.2.7)
+AWS Recon - AWS Inventory Collector (0.2.8)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.12
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-21 00:00:00.000000000 Z
+date: 2020-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -137,6 +137,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.13.1
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
 description: AWS Recon is a command line tool to collect resources from an Amazon
   Web Services (AWS) account. The tool outputs JSON suitable for processing with other
   tools.
@@ -149,9 +163,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/stale.yml"
+- ".github/workflows/docker-build.yml"
+- ".github/workflows/smoke-test.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - Rakefile
@@ -159,9 +176,11 @@ files:
 - bin/aws_recon
 - bin/console
 - bin/setup
+- binstub/aws_recon
 - lib/aws_recon.rb
 - lib/aws_recon/aws_recon.rb
 - lib/aws_recon/collectors.rb
+- lib/aws_recon/collectors/accessanalyzer.rb
 - lib/aws_recon/collectors/acm.rb
 - lib/aws_recon/collectors/apigateway.rb
 - lib/aws_recon/collectors/apigatewayv2.rb