aws_recon 0.3.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cbcf69491befc3f3fb506855f5f11597ac6f7325e697f9e77161e80eeceea08
-  data.tar.gz: 00d9aaa7ca5c58f1ad05c892d50f90b8468d37b4b7b20396a19c09a84d408b30
+  metadata.gz: 212cb7795c7ff6e28ef56336bdd26de0e4d174e71b85f841fb71d60584e6967f
+  data.tar.gz: 2c25dacdbf4124361ae3a76726d72557ba5cb6ac16fbdffd0ae636f8d8ef5f86
 SHA512:
-  metadata.gz: 8e60879ad2fd773f359d5b75fdac5f846cb9c1708d223396f28c448f4a47b79e683b43135e2700171e5cbc68eefef43f75d9ff3e1a1885076fbd24e97b605c54
-  data.tar.gz: c58078ba779cb25cfa5600549c31381e533ae8bcbc1479581d49c5c3462e9354e0df3714d796ca0d678c8fdc2059a041c08d953c99d4ed04e6f3ee1744d2fc89
+  metadata.gz: '08a247b20671f56f119101e26e257489ae71c81461e5cf59d0ccf9538c1f0a81d72bedae93eed1a5f3e9e18de846f4a9000e781b0a69c355355f8fd4195ba129'
+  data.tar.gz: e479cb51db2afc92493b06f17928a9ae6549b8799e345484835219e177c191d29360d3309eec32ba1ea9b5ae3a5e84cfa9c31c61c480c422f0022497d6e46a9b

data/.gitignore

@@ -12,3 +12,5 @@ Gemfile.lock
 /pkg/
 /spec/reports/
 /tmp/
+.terraform*
+terraform.tfstate*

data/.rubocop.yml

@@ -12,7 +12,6 @@ Layout/LineLength:
   Max: 100
 Style/FrozenStringLiteralComment:
   EnforcedStyle: always_true
-  Safe: true
   SafeAutoCorrect: true
 Style/ClassAndModuleChildren:
   Enabled: false

data/aws_recon.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.version       = AwsRecon::VERSION
   spec.authors       = ['Josh Larsen', 'Darkbit']
   spec.required_ruby_version = '>= 2.5.0'
-  spec.summary       = 'A multi-threaded AWS inventory collection cli tool.'
+  spec.summary       = 'A multi-threaded AWS security-focused inventory collection tool.'
   spec.description   = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
   spec.homepage      = 'https://github.com/darkbitio/aws-recon'
   spec.license       = 'MIT'

data/lib/aws_recon/aws_recon.rb

@@ -6,7 +6,7 @@ module AwsRecon
   class CLI
     def initialize
       # parse options
-      @options = Parser.parse ARGV.length < 1 ? %w[-h] : ARGV
+      @options = Parser.parse ARGV.empty? ? %w[-h] : ARGV
 
       # timing
       @starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -15,11 +15,11 @@ module AwsRecon
       @account_id = Aws::STS::Client.new.get_caller_identity.account
 
       # AWS services
-      @aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
+      @aws_services = YAML.safe_load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
 
       # User config services
       if @options.config_file
-        user_config = YAML.load(File.read(@options.config_file), symbolize_names: true)
+        user_config = YAML.safe_load(File.read(@options.config_file), symbolize_names: true)
 
         @services = user_config[:services]
         @regions = user_config[:regions]
@@ -34,9 +34,9 @@ module AwsRecon
       # formatter
       @formatter = Formatter.new
 
-      unless @options.stream_output
-        puts "\nStarting collection with #{@options.threads} threads...\n"
-      end
+      return unless @options.stream_output
+
+      puts "\nStarting collection with #{@options.threads} threads...\n"
     end
 
     #
@@ -72,7 +72,7 @@ module AwsRecon
       #
       # global services
       #
-      @aws_services.map { |x| OpenStruct.new(x) }.filter { |s| s.global }.each do |service|
+      @aws_services.map { |x| OpenStruct.new(x) }.filter(&:global).each do |service|
         # user included this service in the args
         next unless @services.include?(service.name)
 
@@ -94,7 +94,7 @@ module AwsRecon
           next unless @regions.include?(region) && !skip_region
 
           # user included this service in the args
-          next unless @services.include?(service.name) || @services.include?(service.alias) # rubocop:disable Layout/LineLength
+          next unless @services.include?(service.name) || @services.include?(service.alias)
 
           collect(service, region)
         end
@@ -102,16 +102,47 @@ module AwsRecon
     rescue Interrupt # ctrl-c
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
 
-      puts "\nStopped early after \x1b[32m#{elapsed.to_i}\x1b[0m seconds.\n"
+      puts "\nStopped early after #{elapsed.to_i} seconds.\n"
     ensure
-      # write output file
-      if @options.output_file
-        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
 
-        puts "\nFinished in \x1b[32m#{elapsed.to_i}\x1b[0m seconds. Saving resources to \x1b[32m#{@options.output_file}\x1b[0m.\n\n"
+      puts "\nFinished in #{elapsed.to_i} seconds.\n\n"
+
+      # write output file
+      if @options.output_file && !@options.s3
+        puts "Saving resources to #{@options.output_file}.\n\n"
 
         File.write(@options.output_file, @resources.to_json)
       end
+
+      # write output file to S3 bucket
+      if @options.s3
+        t = Time.now.utc
+
+        s3_full_object_path = "AWSRecon/#{t.year}/#{t.month}/#{t.day}/#{@account_id}_aws_recon_#{t.to_i}.json.gz"
+
+        begin
+          # get bucket name (and region if not us-east-1)
+          s3_bucket, s3_region = @options.s3.split(':')
+
+          # build IO object and gzip it
+          io = StringIO.new
+          gzip_data = Zlib::GzipWriter.new(io)
+          gzip_data.write(@resources.to_json)
+          gzip_data.close
+
+          # send it to S3
+          s3_client = Aws::S3::Client.new(region: s3_region || 'us-east-1')
+          s3_resource = Aws::S3::Resource.new(client: s3_client)
+          obj = s3_resource.bucket(s3_bucket).object(s3_full_object_path)
+          obj.put(body: io.string)
+
+          puts "Saving resources to S3 s3://#{s3_bucket}/#{s3_full_object_path}\n\n"
+        rescue Aws::S3::Errors::ServiceError => e
+          puts "Error! - could not save output S3 bucket\n\n"
+          puts "#{e.message} - #{e.code}\n"
+        end
+      end
     end
   end
 end

data/lib/aws_recon/collectors.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 # require all collectors
-Dir[File.join(__dir__, 'collectors', '*.rb')].each { |file| require file }
+Dir[File.join(__dir__, 'collectors', '*.rb')].sort.each { |file| require file }

data/lib/aws_recon/collectors/emr.rb

@@ -13,14 +13,20 @@ class EMR < Mapper
     #
     # get_block_public_access_configuration
     #
-    @client.get_block_public_access_configuration.each do |response|
-      log(response.context.operation_name)
+    begin
+      @client.get_block_public_access_configuration.each do |response|
+        log(response.context.operation_name)
 
-      struct = OpenStruct.new(response.block_public_access_configuration.to_h)
-      struct.type = 'configuration'
-      struct.arn = "arn:aws:emr:#{@region}:#{@account}/block_public_access_configuration"
+        struct = OpenStruct.new(response.block_public_access_configuration.to_h)
+        struct.type = 'configuration'
+        struct.arn = "arn:aws:emr:#{@region}:#{@account}/block_public_access_configuration"
 
-      resources.push(struct.to_h)
+        resources.push(struct.to_h)
+      end
+    rescue Aws::EMR::Errors::ServiceError => e
+      log_error(e.code)
+
+      raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
     end
 
     #
@@ -42,4 +48,12 @@ class EMR < Mapper
 
     resources
   end
+
+  private
+
+  def suppressed_errors
+    %w[
+      InvalidRequestException
+    ]
+  end
 end

data/lib/aws_recon/collectors/s3.rb

@@ -48,6 +48,7 @@ class S3 < Mapper
           { func: 'get_bucket_policy', key: 'policy', field: 'policy' },
           { func: 'get_bucket_policy_status', key: 'public', field: 'policy_status' },
           { func: 'get_public_access_block', key: 'public_access_block', field: 'public_access_block_configuration' },
+          { func: 'get_object_lock_configuration', key: 'object_lock_configuration', field: 'object_lock_configuration' },
           { func: 'get_bucket_tagging', key: 'tagging', field: nil },
           { func: 'get_bucket_logging', key: 'logging', field: 'logging_enabled' },
           { func: 'get_bucket_versioning', key: 'versioning', field: nil },
@@ -66,7 +67,7 @@ class S3 < Mapper
                            end
 
         rescue Aws::S3::Errors::ServiceError => e
-          log_error(e.code)
+          log_error(bucket.name, op.func, e.code)
 
           raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception
         end
@@ -90,6 +91,7 @@ class S3 < Mapper
       NoSuchWebsiteConfiguration
       ReplicationConfigurationNotFoundError
       NoSuchPublicAccessBlockConfiguration
+      ObjectLockConfigurationNotFoundError
     ]
   end
 end

data/lib/aws_recon/lib/mapper.rb

@@ -68,12 +68,12 @@ class Mapper
   def log(*msg)
     return unless @options.verbose
 
-    puts _msg(msg).map { |x| "\x1b[32m#{x}\x1b[0m" }.join("\x1b[35m.\x1b[0m")
+    puts _msg(msg).map(&:to_s).join('.')
   end
 
   def log_error(*msg)
     return unless @options.verbose
 
-    puts _msg(msg).map { |x| "\x1b[35m#{x}\x1b[0m" }.join("\x1b[32m.\x1b[0m")
+    puts _msg(msg).map(&:to_s).join('.')
   end
 end

data/lib/aws_recon/options.rb

@@ -6,6 +6,7 @@
 class Parser
   DEFAULT_CONFIG_FILE = nil
   DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze
+  DEFAULT_S3_PATH = nil
   SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
   DEFAULT_FORMAT = 'aws'
   DEFAULT_THREADS = 8
@@ -15,6 +16,7 @@ class Parser
     :regions,
     :services,
     :config_file,
+    :s3,
     :output_file,
     :output_format,
     :threads,
@@ -43,6 +45,7 @@ class Parser
       aws_regions,
       aws_services.map { |service| service[:name] },
       DEFAULT_CONFIG_FILE,
+      DEFAULT_S3_PATH,
       DEFAULT_OUTPUT_FILE,
       DEFAULT_FORMAT,
       DEFAULT_THREADS,
@@ -93,6 +96,11 @@ class Parser
         args.config_file = config
       end
 
+      # write output file to S3 bucket
+      opts.on('-b', '--s3-bucket [BUCKET:REGION]', 'Write output file to S3 bucket (default: \'\')') do |bucket_with_region|
+        args.s3 = bucket_with_region
+      end
+
       # output file
       opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output|
         args.output_file = File.expand_path(File.join(Dir.pwd, output))

data/lib/aws_recon/services.yaml

@@ -13,11 +13,12 @@
 - name: CodeBuild
   alias: codebuild
   excluded_regions:
-    - af-south-1
+    - ap-northeast-3
 - name: CodePipeline
   alias: codepipeline
   excluded_regions:
     - af-south-1
+    - ap-northeast-3
     - me-south-1
 - name: AutoScaling
   alias: autoscaling
@@ -40,17 +41,10 @@
     - ap-southeast-1
 - name: ElasticLoadBalancingV2
   alias: elbv2
-  excluded_regions:
-    - ap-southeast-1
 - name: ElastiCache
   alias: elasticache
 - name: EMR
   alias: emr
-  excluded_regions:
-    - ap-east-1
-    - af-south-1
-    - eu-south-1
-    - me-south-1
 - name: IAM
   global: true
   alias: iam
@@ -96,11 +90,9 @@
 - name: SES
   alias: ses
   excluded_regions:
-    - eu-north-1
-    - eu-west-3
-    - us-west-1
-    - ap-east-1
     - af-south-1
+    - ap-east-1
+    - ap-northeast-3
     - eu-south-1
 - name: CloudWatch
   alias: cloudwatch
@@ -110,65 +102,78 @@
   alias: kafka
   excluded_regions:
     - af-south-1
+    - ap-northeast-3
 - name: SecretsManager
   alias: sm
 - name: SecurityHub
   alias: sh
+  excluded_regions:
+    - ap-northeast-3
 - name: Support
   global: true
   alias: support
 - name: SSM
   alias: ssm
-  excluded_regions:
-    - ap-southeast-1
 - name: GuardDuty
   alias: guardduty
+  excluded_regions:
+    - ap-northeast-3
 - name: Athena
   alias: athena
+  excluded_regions:
+    - ap-northeast-3
 - name: EFS
   alias: efs
+  excluded_regions:
+    - ap-northeast-3
 - name: Firehose
   alias: firehose
 - name: Lightsail
   alias: lightsail
   excluded_regions:
-    - eu-north-1
-    - us-west-1
-    - sa-east-1
-    - ap-east-1
     - af-south-1
+    - ap-east-1
+    - ap-northeast-3
+    - eu-north-1
     - eu-south-1
     - me-south-1
+    - sa-east-1
+    - us-west-1
 - name: WorkSpaces
   alias: workspaces
   excluded_regions:
-    - eu-north-1
+    - af-south-1
+    - ap-east-1
+    - ap-northeast-3
     - ap-south-1
+    - eu-north-1
+    - eu-south-1
     - eu-west-3
+    - me-south-1
     - us-east-2
     - us-west-1
-    - ap-east-1
-    - af-south-1
-    - eu-south-1
-    - me-south-1
 - name: SageMaker
   alias: sagemaker
+  excluded_regions:
+    - ap-northeast-3
 - name: ServiceQuotas
   alias: servicequotas
 - name: Transfer
   alias: transfer
   excluded_regions:
-    - ap-east-1
-    - af-south-1
+    - ap-northeast-3
     - eu-south-1
-    - me-south-1
 - name: DirectConnect
   alias: dc
 - name: DirectoryService
   alias: ds
+  excluded_regions:
+    - ap-northeast-3
 - name: DatabaseMigrationService
   alias: dms
 - name: XRay
   alias: xray
 - name: WAFV2
   alias: wafv2
+  excluded_regions:
+    - ap-northeast-3

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.3.2"
+  VERSION = "0.4.1"
 end

data/readme.md

@@ -3,19 +3,19 @@
 
 # AWS Recon
 
-A multi-threaded AWS inventory collection tool.
+A multi-threaded AWS security-focused inventory collection tool written in Ruby.
 
-The [creators](https://darkbit.io) of this tool have a recurring need to be able to efficiently collect a large amount of AWS resource attributes and metadata to help clients understand their cloud security posture.
+This tool was created to facilitate efficient collection of a large amount of AWS resource attributes and metadata. It aims to collect nearly everything that is relevant to the security configuration and posture of an AWS environment.
 
-Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity we needed. We also needed a tool that produced consistent output that was easily consumed by other tools/systems.
+Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity to accurately measure security posture (e.g. detailed resource attribute data, fully parsed policy documents, and nested resource relationships).
 
-Enter AWS Recon, multi-threaded AWS inventory collection tool written in plain Ruby. Though Python tends to dominate the AWS tooling landscape, the [Ruby SDK](https://aws.amazon.com/sdk-for-ruby/) has a few convenient advantages over the [other](https://aws.amazon.com/sdk-for-node-js/) [AWS](https://aws.amazon.com/sdk-for-python/) [SDKs](https://aws.amazon.com/sdk-for-go/) we tested. Specifically, easy handling of automatic retries, paging of large responses, and - with some help - threading huge numbers of requests.
+AWS Recon handles collection from large accounts by taking advantage of automatic retries (either due to network reliability or API throttling), automatic paging of large responses (> 100 resources per API call), and multi-threading parallel requests to speed up collection.
 
 ## Project Goals
 
 - More complete resource coverage than available tools (especially for ECS & EKS)
 - More granular resource detail, including nested related resources in the output
-- Flexible output (console, JSON lines, plain JSON, file, standard out)
+- Flexible output (console, JSON lines, plain JSON, file, S3 bucket, and standard out)
 - Efficient (multi-threaded, rate limited, automatic retries, and automatic result paging)
 - Easy to maintain and extend
 
@@ -31,7 +31,7 @@ Use Docker version 19.x or above to run the pre-built image without having to in
 
 #### Running locally via Ruby
 
-If you already have Ruby installed (2.5.x or 2.6.x), you may want to install the Ruby gem.
+If you already have Ruby installed (2.6.x or 2.7.x), you may want to install the Ruby gem.
 
 ### Installation
 
@@ -54,13 +54,13 @@ To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.3.0.gem
+Fetching aws_recon-0.4.0.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.20.1.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.20.1
-Successfully installed aws_recon-0.3.0
+Successfully installed aws_recon-0.4.0
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -72,7 +72,7 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel-1.20.1
-Using aws_recon 0.3.0
+Using aws_recon 0.4.0
 ```
 
 ## Usage
@@ -158,13 +158,37 @@ Finished in 46 seconds. Saving resources to output.json.
 #### Example command line options
 
 ```
+# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
+
 $ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
 ```
 
 ```
+# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
+
 $ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
 ```
 
+```
+# save output to S3 bucket
+
+$ AWS_PROFILE=<profile> aws_recon \
+  --services S3,EC2 \
+  --regions global,us-east-1,us-east-2 \
+  --verbose \
+  --s3-bucket my-recon-bucket
+```
+
+```
+# save output to S3 bucket with a home region other than us-east-1
+
+$ AWS_PROFILE=<profile> aws_recon \
+  --services S3,EC2 \
+  --regions global,us-east-1,us-east-2 \
+  --verbose \
+  --s3-bucket my-recon-bucket:us-west-2
+```
+
 Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted (NDJSON) output.
 
 ```
@@ -225,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.3.0)
+AWS Recon - AWS Inventory Collector (0.4.0)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -233,6 +257,7 @@ Usage: aws_recon [options]
     -s, --services [SERVICES]        Services to scan, separated by comma (default: all)
     -x, --not-services [SERVICES]    Services to skip, separated by comma (default: none)
     -c, --config [CONFIG]            Specify config file for services & regions (e.g. config.yaml)
+    -b, --s3-bucket [BUCKET:REGION]  Write output file to S3 bucket (default: '')
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
@@ -251,6 +276,8 @@ Usage: aws_recon [options]
 
 Output is always some form of JSON - either JSON lines or plain JSON. The output is either written to a file (the default), or written to stdout (with `-j`).
 
+When writing to an S3 bucket, the JSON output is automatically compressed with `gzip`.
+
 ## Support for Manually Enabled Regions
 
 If you have enabled **manually enabled regions**:
@@ -351,7 +378,7 @@ $ cd aws-recon
 Create a sticky gemset if using RVM:
 
 ```
-$ rvm use 2.6.5@aws_recon_dev --create --ruby-version
+$ rvm use 2.7.2@aws_recon_dev --create --ruby-version
 ```
 
 Run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/terraform/cloudwatch.tf

@@ -0,0 +1,30 @@
+# https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_rule.html
+resource "aws_cloudwatch_event_rule" "default" {
+  name                = "${var.aws_recon_base_name}-${random_id.rule.hex}"
+  description         = "AWS Recon scheduled task"
+  schedule_expression = var.schedule_expression
+}
+
+# https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_target.html
+resource "aws_cloudwatch_event_target" "default" {
+  target_id = aws_ecs_task_definition.aws_recon_task.id
+  arn       = aws_ecs_cluster.aws_recon.arn
+  rule      = aws_cloudwatch_event_rule.default.name
+  role_arn  = aws_iam_role.cw_events.arn
+
+  ecs_target {
+    launch_type         = "FARGATE"
+    task_definition_arn = aws_ecs_task_definition.aws_recon_task.arn
+    platform_version    = "LATEST"
+
+    network_configuration {
+      assign_public_ip = true
+      security_groups  = [aws_security_group.sg.id]
+      subnets          = [aws_subnet.subnet.id]
+    }
+  }
+}
+
+resource "random_id" "rule" {
+  byte_length = 4
+}

data/terraform/ecs.tf

@@ -0,0 +1,51 @@
+resource "aws_ecs_cluster" "aws_recon" {
+  name               = "${var.aws_recon_base_name}-${random_id.cluster.hex}"
+  capacity_providers = [local.ecs_task_provider]
+}
+
+resource "random_id" "cluster" {
+  byte_length = 4
+}
+
+resource "aws_ecs_task_definition" "aws_recon_task" {
+  family                   = "${var.aws_recon_base_name}-${random_id.cluster.hex}"
+  task_role_arn            = aws_iam_role.aws_recon_role.arn
+  execution_role_arn       = aws_iam_role.ecs_task_execution.arn
+  requires_compatibilities = [local.ecs_task_provider]
+  network_mode             = "awsvpc"
+  cpu                      = 1024
+  memory                   = 2048
+
+  container_definitions = jsonencode([
+    {
+      name             = "${var.aws_recon_base_name}-${random_id.cluster.hex}"
+      image            = "${var.aws_recon_container_name}:${var.aws_recon_container_version}"
+      assign_public_ip = true
+      entryPoint = [
+        "aws_recon",
+        "--verbose",
+        "--s3-bucket",
+        "${aws_s3_bucket.aws_recon.bucket}:${data.aws_region.current.name}",
+        "--regions",
+        join(",", var.aws_regions)
+      ]
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.aws_recon.name,
+          awslogs-region        = data.aws_region.current.name,
+          awslogs-stream-prefix = "ecs"
+        }
+      }
+    }
+  ])
+}
+
+resource "aws_cloudwatch_log_group" "aws_recon" {
+  name              = "/ecs/${var.aws_recon_base_name}-${random_id.cluster.hex}"
+  retention_in_days = var.retention_period
+}
+
+locals {
+  ecs_task_provider = "FARGATE"
+}

data/terraform/iam.tf

@@ -0,0 +1,125 @@
+#
+# IAM policies and roles for ECS and CloudWatch execution
+#
+resource "aws_iam_role" "aws_recon_role" {
+  name               = local.aws_recon_task_role_name
+  assume_role_policy = data.aws_iam_policy_document.aws_recon_task_execution_assume_role_policy.json
+}
+
+data "aws_iam_policy_document" "aws_recon_task_execution_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type = "Service"
+      identifiers = [
+        "ecs.amazonaws.com",
+        "ecs-tasks.amazonaws.com"
+      ]
+    }
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "aws_recon_task_execution" {
+  role       = aws_iam_role.aws_recon_role.name
+  policy_arn = data.aws_iam_policy.aws_recon_task_execution.arn
+}
+
+resource "aws_iam_role_policy" "aws_recon" {
+  name = local.bucket_write_policy_name
+  role = aws_iam_role.aws_recon_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Id      = "${var.aws_recon_base_name}-bucket-write"
+    Statement = [
+      {
+        Sid    = "AWSReconS3PutObject"
+        Effect = "Allow"
+        Action = "s3:PutObject"
+        Resource = [
+          "${aws_s3_bucket.aws_recon.arn}/*"
+        ]
+      }
+    ]
+  })
+}
+
+data "aws_iam_policy" "aws_recon_task_execution" {
+  arn = "arn:aws:iam::aws:policy/ReadOnlyAccess"
+}
+
+resource "aws_iam_role" "ecs_task_execution" {
+  name               = local.ecs_task_execution_role_name
+  assume_role_policy = data.aws_iam_policy_document.ecs_task_execution_assume_role_policy.json
+
+  tags = {
+    Name = local.ecs_task_execution_role_name
+  }
+}
+
+data "aws_iam_policy_document" "ecs_task_execution_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ecs-tasks.amazonaws.com"]
+    }
+  }
+}
+
+# ECS task execution
+resource "aws_iam_policy" "ecs_task_execution" {
+  name   = local.ecs_task_execution_policy_name
+  policy = data.aws_iam_policy.ecs_task_execution.policy
+}
+
+resource "aws_iam_role_policy_attachment" "ecs_task_execution" {
+  role       = aws_iam_role.ecs_task_execution.name
+  policy_arn = aws_iam_policy.ecs_task_execution.arn
+}
+
+data "aws_iam_policy" "ecs_task_execution" {
+  arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+}
+
+# CloudWatch Events
+resource "aws_iam_role" "cw_events" {
+  name               = local.cw_events_role_name
+  assume_role_policy = data.aws_iam_policy_document.cw_events_assume_role_policy.json
+}
+
+data "aws_iam_policy_document" "cw_events_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_policy" "cw_events" {
+  name   = local.cw_events_policy_name
+  policy = data.aws_iam_policy.cw_events.policy
+}
+
+resource "aws_iam_role_policy_attachment" "cw_events" {
+  role       = aws_iam_role.cw_events.name
+  policy_arn = aws_iam_policy.cw_events.arn
+}
+
+data "aws_iam_policy" "cw_events" {
+  arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
+}
+
+locals {
+  bucket_write_policy_name       = "${var.aws_recon_base_name}-bucket-write-policy"
+  ecs_task_execution_role_name   = "${var.aws_recon_base_name}-ecs-task-execution-role"
+  ecs_task_execution_policy_name = "${var.aws_recon_base_name}-ecs-task-execution-policy"
+  cw_events_policy_name          = "${var.aws_recon_base_name}-cw-events-policy"
+  cw_events_role_name            = "${var.aws_recon_base_name}-cw-events-role"
+  aws_recon_task_role_name       = "${var.aws_recon_base_name}-exec-role"
+}

data/terraform/main.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  region = "us-east-2"
+}

data/terraform/output.tf

@@ -0,0 +1,13 @@
+output "aws_recon_s3_bucket" {
+  value = aws_s3_bucket.aws_recon.bucket
+}
+
+output "aws_recon_ecs_cluster" {
+  value = aws_ecs_cluster.aws_recon.name
+}
+
+output "aws_recon_ecs_scheduled_task" {
+  value = aws_cloudwatch_event_rule.default.name
+}
+
+

data/terraform/readme.md

@@ -0,0 +1,20 @@
+## Terraform Setup
+
+This is an example module that can be used in its current form or modified for your specific environment. It builds the minimum components necessary to collect inventory on a schedule running AWS Recon as a Fargate scheduled task.
+
+### Requirements
+
+Before running this Terraform module, adjust your region accordingly in `main.tf`.
+
+### What is created?
+
+This Terraform example will deploy the following resources:
+
+- an S3 bucket to store compressed JSON output files
+- an IAM role for ECS task execution
+- a Security Group for the ECS cluster/task
+- a VPC and NGW for the ECS cluster/task
+- an ECS/Fargate cluster
+- an ECS task definition to run AWS Recon collection
+- a CloudWatch event rule to trigger the ECS task
+- a CloudTrail log group for ECS task logs

data/terraform/s3.tf

@@ -0,0 +1,19 @@
+resource "aws_s3_bucket" "aws_recon" {
+  bucket = "${var.aws_recon_base_name}-${random_id.bucket.hex}-${data.aws_iam_account_alias.current.id}"
+  acl    = "private"
+
+  lifecycle_rule {
+    id      = "expire-after-${var.retention_period}-days"
+    enabled = true
+
+    expiration {
+      days = var.retention_period
+    }
+  }
+}
+
+resource "random_id" "bucket" {
+  byte_length = 4
+}
+
+data "aws_iam_account_alias" "current" {}

data/terraform/vars.tf

@@ -0,0 +1,57 @@
+variable "aws_recon_base_name" {
+  type    = string
+  default = "aws-recon"
+}
+
+variable "aws_recon_container_name" {
+  type    = string
+  default = "darkbitio/aws_recon"
+}
+
+variable "aws_recon_container_version" {
+  type    = string
+  default = "latest"
+}
+
+variable "aws_regions" {
+  type = list(any)
+  default = [
+    "global",
+    # "af-south-1",
+    # "ap-east-1",
+    # "ap-northeast-1",
+    # "ap-northeast-2",
+    # "ap-northeast-3",
+    # "ap-south-1",
+    # "ap-southeast-1",
+    # "ap-southeast-2",
+    # "ca-central-1",
+    # "eu-central-1",
+    # "eu-north-1",
+    # "eu-south-1",
+    # "eu-west-1",
+    # "eu-west-2",
+    # "eu-west-3",
+    # "me-south-1",
+    # "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+  ]
+}
+
+variable "retention_period" {
+  type    = number
+  default = 30
+}
+
+variable "schedule_expression" {
+  type    = string
+  default = "cron(4 * * * ? *)"
+}
+
+variable "base_subnet_cidr" {
+  type    = string
+  default = "10.76.0.0/16"
+}

data/terraform/vpc.tf

@@ -0,0 +1,78 @@
+
+# Create a VPC
+resource "aws_vpc" "vpc" {
+  cidr_block = local.cidr_block
+  tags = {
+    Name = "${var.aws_recon_base_name}-${random_id.vpc.hex}"
+  }
+}
+
+# Create subnet
+resource "aws_subnet" "subnet" {
+  vpc_id                  = aws_vpc.vpc.id
+  cidr_block              = local.subnet_cidr_block
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "${var.aws_recon_base_name}-${random_id.vpc.hex}-public"
+  }
+}
+
+resource "aws_security_group" "sg" {
+  name        = "${var.aws_recon_base_name}-${random_id.vpc.hex}"
+  description = "Allow AWS Recon collection egress"
+  vpc_id      = aws_vpc.vpc.id
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "${var.aws_recon_base_name}-${random_id.vpc.hex}"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.vpc.id
+
+  tags = {
+    Name = "${var.aws_recon_base_name}-${random_id.vpc.hex}"
+  }
+}
+
+resource "aws_route_table" "rt" {
+  vpc_id = aws_vpc.vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "${var.aws_recon_base_name}-${random_id.vpc.hex}"
+  }
+}
+
+resource "aws_route_table_association" "rt_association" {
+  subnet_id      = aws_subnet.subnet.id
+  route_table_id = aws_route_table.rt.id
+}
+
+locals {
+  cidr_block        = var.base_subnet_cidr
+  subnet_cidr_block = cidrsubnet(local.cidr_block, 8, 0)
+}
+
+resource "random_id" "vpc" {
+  byte_length = 4
+}
+
+data "aws_region" "current" {}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.1
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -167,7 +167,6 @@ files:
 - ".github/workflows/smoke-test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - Dockerfile
 - Gemfile
 - LICENSE.txt
@@ -245,6 +244,15 @@ files:
 - lib/aws_recon/services.yaml
 - lib/aws_recon/version.rb
 - readme.md
+- terraform/cloudwatch.tf
+- terraform/ecs.tf
+- terraform/iam.tf
+- terraform/main.tf
+- terraform/output.tf
+- terraform/readme.md
+- terraform/s3.tf
+- terraform/vars.tf
+- terraform/vpc.tf
 homepage: https://github.com/darkbitio/aws-recon
 licenses:
 - MIT
@@ -267,5 +275,5 @@ requirements: []
 rubygems_version: 3.0.8
 signing_key:
 specification_version: 4
-summary: A multi-threaded AWS inventory collection cli tool.
+summary: A multi-threaded AWS security-focused inventory collection tool.
 test_files: []

data/.travis.yml

@@ -1,7 +0,0 @@
----
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.5
-before_install: gem install bundler -v 1.17.3