aws_recon 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12b30d8e1939333bd6a2f94ba0bfa5a8b9aa381e0330546425158360cda8e099
-  data.tar.gz: dea36844f6fc06403b563fd0dc6938d222c0dc8757b7b88c0cdd03e0b5df79e5
+  metadata.gz: 9647cee32c4df1624a99cf180258f43e664e7ee62b8ceee5fe00b8d6b31a2803
+  data.tar.gz: aca9a6383263ca7add32682df25a633ad9db031a50d2f38f319f903b6dd75c45
 SHA512:
-  metadata.gz: 50fa5ec78c7bbedc8f89321cbd0a679945e2509a2ba8d51c0fd6e95d15a3e7cdf29db051f0b9816d85bf49af0d7375e2f8ada6796b7c9a4ddb3b403ebda4b598
-  data.tar.gz: d1d1cb453321a8dcb669839b3e9597af0a11882863c15f9a44c23f6dda77efaa0c302ed3208032c12a12f97b09f9eff0776f4346e579b588ab10c1ba4d2b713e
+  metadata.gz: 426acf5937d26974c7a7c5bc26e84487fe8b76cc28bf0f51a7302d05897356e7007b877e26a47cc3af7b54044ac45bec23b409b1b66bef0336aaa1216d340437
+  data.tar.gz: 790fd25c65d4fe49c2e3857f65cddb590ecfe9f481b923bcdf8def9007ddf1ccb0b16f616164e7472b43a75eebe77698146d3d2e46204af1085b276afedf2099

data/.rubocop.yml

@@ -12,7 +12,6 @@ Layout/LineLength:
   Max: 100
 Style/FrozenStringLiteralComment:
   EnforcedStyle: always_true
-  Safe: true
   SafeAutoCorrect: true
 Style/ClassAndModuleChildren:
   Enabled: false

data/lib/aws_recon/aws_recon.rb

@@ -34,9 +34,9 @@ module AwsRecon
       # formatter
       @formatter = Formatter.new
 
-      unless @options.stream_output
-        puts "\nStarting collection with #{@options.threads} threads...\n"
-      end
+      return unless @options.stream_output
+
+      puts "\nStarting collection with #{@options.threads} threads...\n"
     end
 
     #
@@ -72,7 +72,7 @@ module AwsRecon
       #
       # global services
       #
-      @aws_services.map { |x| OpenStruct.new(x) }.filter { |s| s.global }.each do |service|
+      @aws_services.map { |x| OpenStruct.new(x) }.filter(&:global).each do |service|
         # user included this service in the args
         next unless @services.include?(service.name)
 
@@ -104,14 +104,45 @@ module AwsRecon
 
       puts "\nStopped early after \x1b[32m#{elapsed.to_i}\x1b[0m seconds.\n"
     ensure
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
+
+      puts "\nFinished in \x1b[32m#{elapsed.to_i}\x1b[0m seconds.\n\n"
+
       # write output file
       if @options.output_file
-        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
-
-        puts "\nFinished in \x1b[32m#{elapsed.to_i}\x1b[0m seconds. Saving resources to \x1b[32m#{@options.output_file}\x1b[0m.\n\n"
+        puts "Saving resources to \x1b[32m#{@options.output_file}\x1b[0m.\n\n"
 
         File.write(@options.output_file, @resources.to_json)
       end
+
+      # write output file to S3 bucket
+      if @options.s3
+        t = Time.now.utc
+
+        s3_full_object_path = "AWSRecon/#{t.year}/#{t.month}/#{t.day}/#{@account_id}_aws_recon_#{t.to_i}.json.gz"
+
+        begin
+          # get bucket name (and region if not us-east-1)
+          s3_bucket, s3_region = @options.s3.split(':')
+
+          # build IO object and gzip it
+          io = StringIO.new
+          gzip_data = Zlib::GzipWriter.new(io)
+          gzip_data.write(@resources.to_json)
+          gzip_data.close
+
+          # send it to S3
+          s3_client = Aws::S3::Client.new(region: s3_region || 'us-east-1')
+          s3_resource = Aws::S3::Resource.new(client: s3_client)
+          obj = s3_resource.bucket(s3_bucket).object(s3_full_object_path)
+          obj.put(body: io.string)
+
+          puts "Saving resources to S3 \x1b[32ms3://#{s3_bucket}/#{s3_full_object_path}\x1b[0m\n\n"
+        rescue Aws::S3::Errors::ServiceError => e
+          puts "\x1b[35mError!\x1b[0m - could not save output S3 bucket\n\n"
+          puts "#{e.message} - #{e.code}\n"
+        end
+      end
     end
   end
 end

data/lib/aws_recon/collectors.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 # require all collectors
-Dir[File.join(__dir__, 'collectors', '*.rb')].each { |file| require file }
+Dir[File.join(__dir__, 'collectors', '*.rb')].sort.each { |file| require file }

data/lib/aws_recon/options.rb

@@ -6,6 +6,7 @@
 class Parser
   DEFAULT_CONFIG_FILE = nil
   DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze
+  DEFAULT_S3_PATH = nil
   SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
   DEFAULT_FORMAT = 'aws'
   DEFAULT_THREADS = 8
@@ -15,6 +16,7 @@ class Parser
     :regions,
     :services,
     :config_file,
+    :s3,
     :output_file,
     :output_format,
     :threads,
@@ -43,6 +45,7 @@ class Parser
       aws_regions,
       aws_services.map { |service| service[:name] },
       DEFAULT_CONFIG_FILE,
+      DEFAULT_S3_PATH,
       DEFAULT_OUTPUT_FILE,
       DEFAULT_FORMAT,
       DEFAULT_THREADS,
@@ -93,6 +96,11 @@ class Parser
         args.config_file = config
       end
 
+      # write output file to S3 bucket
+      opts.on('-b', '--s3-bucket [BUCKET:REGION]', 'Write output file to S3 bucket (default: \'\')') do |bucket_with_region|
+        args.s3 = bucket_with_region
+      end
+
       # output file
       opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output|
         args.output_file = File.expand_path(File.join(Dir.pwd, output))

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.3.5"
+  VERSION = "0.4.0"
 end

data/readme.md

@@ -5,9 +5,9 @@
 
 A multi-threaded AWS inventory collection tool.
 
-The [creators](https://darkbit.io) of this tool have a recurring need to be able to efficiently collect a large amount of AWS resource attributes and metadata to help clients understand their cloud security posture.
+This tool was created to facilitate efficient collection of a large amount of AWS resource attributes and metadata. It aims to collect nearly everything that is relevant to the security configuration and posture of an AWS environment.
 
-Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity we needed. We also needed a tool that produced consistent output that was easily consumed by other tools/systems.
+Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity to accurately measure security posture (e.g. detailed attribute data and full policy documents).
 
 Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain Ruby. Though Python tends to dominate the AWS tooling landscape, the [Ruby SDK](https://aws.amazon.com/sdk-for-ruby/) has a few convenient advantages over the [other](https://aws.amazon.com/sdk-for-node-js/) [AWS](https://aws.amazon.com/sdk-for-python/) [SDKs](https://aws.amazon.com/sdk-for-go/) we tested. Specifically, easy handling of automatic retries, paging of large responses, and - with some help - threading huge numbers of requests.
 
@@ -15,7 +15,7 @@ Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain R
 
 - More complete resource coverage than available tools (especially for ECS & EKS)
 - More granular resource detail, including nested related resources in the output
-- Flexible output (console, JSON lines, plain JSON, file, standard out)
+- Flexible output (console, JSON lines, plain JSON, file, S3 bucket, and standard out)
 - Efficient (multi-threaded, rate limited, automatic retries, and automatic result paging)
 - Easy to maintain and extend
 
@@ -54,13 +54,13 @@ To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.3.0.gem
+Fetching aws_recon-0.4.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.20.1.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.20.1
-Successfully installed aws_recon-0.3.0
+Successfully installed aws_recon-0.4.0
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -72,7 +72,7 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel-1.20.1
-Using aws_recon 0.3.0
+Using aws_recon 0.4.0
 ```
 
 ## Usage
@@ -158,13 +158,37 @@ Finished in 46 seconds. Saving resources to output.json.
 #### Example command line options
 
 ```
+# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
+
 $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 ```
 
 ```
+# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
+
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
+```
+# save output to S3 bucket
+
+$ AWS_PROFILE=<profile> aws_recon \
+  --services S3,EC2 \
+  --regions global,us-east-1,us-east-2 \
+  --verbose \
+  --s3-bucket my-recon-bucket
+```
+
+```
+# save output to S3 bucket with a home region other than us-east-1
+
+$ AWS_PROFILE=<profile> aws_recon \
+  --services S3,EC2 \
+  --regions global,us-east-1,us-east-2 \
+  --verbose \
+  --s3-bucket my-recon-bucket:us-west-2
+```
+
 Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted (NDJSON) output.
 
 ```
@@ -225,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.3.0)
+AWS Recon - AWS Inventory Collector (0.4.0)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -233,6 +257,7 @@ Usage: aws_recon [options]
     -s, --services [SERVICES]        Services to scan, separated by comma (default: all)
     -x, --not-services [SERVICES]    Services to skip, separated by comma (default: none)
     -c, --config [CONFIG]            Specify config file for services & regions (e.g. config.yaml)
+    -b, --s3-bucket [BUCKET:REGION]  Write output file to S3 bucket (default: '')
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-25 00:00:00.000000000 Z
+date: 2021-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk