aws_recon 0.2.3 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac6fae11753e715682d656eba8c922267831fe7324d9a44bd96296543cf9653e
-  data.tar.gz: 900bf49cc999b1fd9d227067609ad85c3b6a462886156c94e7dfb5d5e1c0a982
+  metadata.gz: 939b12091dee8bd4c6b36877a9954ba43372267edda3b4a1d93d3c5695bfde5b
+  data.tar.gz: fe6dbac4e8001bd82d21bbcb8b22d904f91e864afff58d12a6f86b54d4789d2c
 SHA512:
-  metadata.gz: 6c76cf3f96cbf58501c61861361be27e76cc95e1c60cf83ae141d2e4aa0a4d12efe7ed9b1dae74094fff2a5a157a2d17a49ef37ecc471de7f985156b4228e608
-  data.tar.gz: 93341c804a9daf7c4f849927ff429a34b72af74acdbe741e324829e938d01aa805550e7a41b0aeeeb68c2dc80ec37e14cb83e7c5ef1f9406c7c94815eca6757e
+  metadata.gz: 120629a6ac6f8839b4f5dea1a0e269133ded6e1679f4e3ca3411965e34cf901d851638edda80450423c6de86ba1701cd00d13b9ea7292cb75881189a98cf4238
+  data.tar.gz: 3d16a17670a9326d3668e7eb37e9fdf883d5dcce1290b368722fc205d37d0a60b58fe850b21211db435f790b1650a44312ede568db41b03537a5fca679373387

data/.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 30
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 5
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

data/.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 *.json
+Gemfile.lock
 .rvmrc
 .ruby-gemset
 .ruby-version

data/Dockerfile

@@ -0,0 +1,34 @@
+ARG RUBY_VERSION=2.6.6
+FROM ruby:${RUBY_VERSION}-alpine
+
+LABEL maintainer="Darkbit <info@darkbit.io>"
+
+ARG USER=recon
+ARG GEM=aws_recon
+ARG VERSION=0.2.8
+ARG BUNDLER_VERSION=2.1.4
+
+# Install new Bundler version
+RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
+    gem uninstall bundler && \
+    gem install bundler -v $BUNDLER_VERSION
+
+# Install gem
+RUN gem install ${GEM} -v ${VERSION}
+
+# Create non-root user
+RUN addgroup -S ${USER} && \
+    adduser -S ${USER} \
+    -G ${USER} \
+    -s /bin/ash \
+    -h /${USER}
+
+# Copy binstub
+COPY binstub/${GEM} /usr/local/bundle/bin/
+RUN chmod +x /usr/local/bundle/bin/${GEM}
+
+# Switch user
+USER ${USER}
+WORKDIR /${USER}
+
+CMD ["ash"]

data/aws_recon.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'gem-release', '~> 2.1'
-  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'solargraph', '~> 0.39.11'
   spec.add_development_dependency 'rubocop', '~> 0.87.1'

data/binstub/aws_recon

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#
+# Manually generated binstub
+#
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("aws_recon", "aws_recon")

data/lib/aws_recon.rb

@@ -12,7 +12,7 @@ require 'aws-sdk'
 require 'aws_recon/options.rb'
 require 'aws_recon/lib/mapper.rb'
 require 'aws_recon/lib/formatter.rb'
-require 'aws_recon/collectors/collectors.rb'
+require 'aws_recon/collectors.rb'
 
 require 'aws_recon/version'
 require 'aws_recon/aws_recon'

data/lib/aws_recon/aws_recon.rb

@@ -44,7 +44,7 @@ module AwsRecon
     #
     def collect(service, region)
       mapper = Object.const_get(service.name)
-      resources = mapper.new(service.name, region, @options)
+      resources = mapper.new(@account_id, service.name, region, @options)
 
       collection = resources.collect.map do |resource|
         if @options.output_format == 'custom'

data/lib/aws_recon/collectors.rb

@@ -0,0 +1,2 @@
+# require all collectors
+Dir[File.join(__dir__, 'collectors', '*.rb')].each { |file| require file }

data/lib/aws_recon/collectors/ec2.rb

@@ -48,6 +48,23 @@ class EC2 < Mapper
             struct.arn = instance.instance_id # no true ARN
             struct.reservation_id = reservation.reservation_id
 
+            # collect instance user_data
+            if @options.collect_user_data
+              user_data_raw = @client.describe_instance_attribute({
+                                                                    attribute: 'userData',
+                                                                    instance_id: instance.instance_id
+                                                                  }).user_data.to_h[:value]
+
+              # don't save non-string user_data
+              if user_data_raw
+                user_data = Base64.decode64(user_data_raw)
+
+                if user_data.force_encoding('UTF-8').ascii_only?
+                  struct.user_data = user_data
+                end
+              end
+            end
+
             resources.push(struct.to_h)
           end
         end

data/lib/aws_recon/collectors/elasticache.rb

@@ -0,0 +1,22 @@
+class ElastiCache < Mapper
+  def collect
+    resources = []
+
+    #
+    # describe_cache_clusters
+    #
+    @client.describe_cache_clusters.each_with_index do |response, page|
+      log(response.context.operation_name, page)
+
+      response.cache_clusters.each do |cluster|
+        struct = OpenStruct.new(cluster.to_h)
+        struct.type = 'cluster'
+        struct.arn = cluster.arn
+
+        resources.push(struct.to_h)
+      end
+    end
+
+    resources
+  end
+end

data/lib/aws_recon/collectors/iam.rb

@@ -48,6 +48,21 @@ class IAM < Mapper
       end
     end
 
+    #
+    # list_policies
+    #
+    @client.list_policies.each do |response|
+      log(response.context.operation_name)
+
+      # managed policies
+      response.policies.each do |policy|
+        struct = OpenStruct.new(policy.to_h)
+        struct.type = 'managed_policy'
+
+        resources.push(struct.to_h)
+      end
+    end
+
     #
     # get_account_password_policy
     #
@@ -56,6 +71,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.password_policy.to_h)
       struct.type = 'password_policy'
+      struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
 
       resources.push(struct.to_h)
     end
@@ -68,6 +84,7 @@ class IAM < Mapper
 
       struct = OpenStruct.new(response.summary_map)
       struct.type = 'account_summary'
+      struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
 
       resources.push(struct.to_h)
     end
@@ -111,6 +128,7 @@ class IAM < Mapper
 
         struct = OpenStruct.new
         struct.type = 'credential_report'
+        struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
         struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
         struct.report_format = response.report_format
         struct.generated_time = response.generated_time

data/lib/aws_recon/collectors/lambda.rb

@@ -1,6 +1,5 @@
 class Lambda < Mapper
   def collect
-    service = self.class.to_s.downcase
     resources = []
 
     #

data/lib/aws_recon/collectors/shield.rb

@@ -13,7 +13,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new(response.subscription.to_h)
       struct.type = 'subscription'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
 
       resources.push(struct.to_h)
     end
@@ -26,7 +26,7 @@ class Shield < Mapper
 
       struct = OpenStruct.new
       struct.type = 'contact_list'
-      struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
+      struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
       struct.contacts = response.emergency_contact_list.map(&:to_h)
 
       resources.push(struct.to_h)

data/lib/aws_recon/lib/formatter.rb

@@ -8,7 +8,7 @@ class Formatter
   def custom(account_id, region, service, resource)
     {
       account: account_id,
-      name: resource[:arn] || "#{account_id}_#{region}_#{service.name}_#{resource[:type]}",
+      name: resource[:arn],
       service: service.name,
       region: region,
       asset_type: resource[:type],

data/lib/aws_recon/lib/mapper.rb

@@ -22,7 +22,8 @@ class Mapper
   #   S3 (unless the bucket was created in another region)
   SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
 
-  def initialize(service, region, options)
+  def initialize(account, service, region, options)
+    @account = account
     @service = service
     @region = region
     @options = options

data/lib/aws_recon/options.rb

@@ -15,6 +15,7 @@ class Parser
     :output_file,
     :output_format,
     :threads,
+    :collect_user_data,
     :skip_slow,
     :stream_output,
     :verbose,
@@ -43,6 +44,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -103,6 +105,11 @@ class Parser
         end
       end
 
+      # collect EC2 instance user data
+      opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do
+        args.collect_user_data = true
+      end
+
       # skip slow operations
       opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do
         args.skip_slow = true

data/lib/aws_recon/services.yaml

@@ -21,8 +21,6 @@
   alias: ec2
 - name: EKS
   alias: eks
-  excluded_regions:
-    - us-west-1
 - name: ECS
   alias: ecs
 - name: ElasticLoadBalancing
@@ -33,6 +31,8 @@
   alias: elbv2
   excluded_regions:
     - ap-southeast-1
+- name: ElastiCache
+  alias: elasticache
 - name: IAM
   global: true
   alias: iam

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.2.3"
+  VERSION = "0.2.8"
 end

data/readme.md

@@ -26,18 +26,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
 
 ### Installation
 
-Install the gem:
+AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
+
+To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.2.2.gem
+Fetching aws_recon-0.2.7.gem
 Fetching aws-sdk-resources-3.76.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.19.2.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.19.2
-Successfully installed aws_recon-0.2.2
+Successfully installed aws_recon-0.2.7
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -52,6 +54,20 @@ Using parallel 1.19.2
 Using aws_recon 0.2.2
 ```
 
+To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
+
+```
+$ docker run --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  aws_recon:latest \
+  aws_recon -v -s EC2 -r us-east-1,us-east-2
+```
+
+
 ## Usage
 
 AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 
@@ -66,6 +82,31 @@ Plain environment variables will work fine too.
 $ AWS_PROFILE=<profile> aws_recon
 ```
 
+To run from a Docker container using `aws-vault` managed credentials (output to file):
+
+```
+$ aws-vault exec darkbit -- docker run --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  -v $(pwd)/output.json:/recon/output.json \
+  aws_recon:latest \
+  aws_recon -s EC2 -v -r us-east-1,us-east-2
+```
+
+To run from a Docker container using `aws-vault` managed credentials (output to stdout):
+
+```
+$ aws-vault exec darkbit -- docker run --rm \
+  -e AWS_REGION \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  aws_recon:latest \
+  aws_recon -j -s EC2 -r us-east-1,us-east-2
+```
+
 You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 
 
 In verbose mode, the console output will show:
@@ -135,7 +176,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector
+AWS Recon - AWS Inventory Collector (0.2.7)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -146,6 +187,7 @@ Usage: aws_recon [options]
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
     -j, --stream-output              Stream JSON lines to stdout (default: false)
     -v, --verbose                    Output client progress and current operation
@@ -193,6 +235,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
 - [x] ELB
 - [x] EKS
 - [x] Elasticsearch
+- [x] ElastiCache
 - [x] Firehose
 - [ ] FMS
 - [ ] Glacier
@@ -227,23 +270,27 @@ One of the primary motivations for AWS Recon was to build a tool that is easy to
 
 ### Development
 
-Clone this repository, then install the required gems using `bundle`:
+Clone this repository:
 
 ```
 $ git clone git@github.com:darkbitio/aws-recon.git
 $ cd aws-recon
-$ bundle
-...
-Using aws-sdk-core 3.103.0
-...
-Bundle complete! 5 Gemfile dependencies, 259 gems now installed.
-Use `bundle info [gemname]` to see where a bundled gem is installed.
 ```
 
+Create a sticky gemset if using RVM:
+
+```
+$ rvm use 2.6.5@aws_recon_dev --create --ruby-version
+```
+
+Run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ### TODO
 
 - [ ] Optionally suppress AWS API errors instead of re-raising them
-- [ ] Package as a gem
+- [x] Package as a gem
 - [ ] Test coverage with AWS SDK stubbed resources