aws_recon 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8166ac8e24580b32dc3bd5144571d1ba499b7d0e193ab1051c07414f1dfc444b
-  data.tar.gz: 43a7207373c94f1aef7ff94e94a62d542595ba0f246aea7ae4c945a0c143aa9f
+  metadata.gz: 9b98e94e11398d8f2aec76174c8d660be040521e44439ca70ca7b5b680e6a631
+  data.tar.gz: e8920ba7308df8491ae6ce6271188fedf3e57e499bc51f6d9d772728277ef6a6
 SHA512:
-  metadata.gz: 13f36944733b6d7c3a387243713cd77e7d2880875bbd3f02fdb2deea663c11d74857a67985e988a40a3a196e5592048219c22e8a3c6ebddc350d49e13d7ef695
-  data.tar.gz: e818d1cde999ebb9e667710af47373d10b3369d52d0c53535396745cdea4ed7040f4331c95f697658d1a1afc897926542a9ae22fdfac6dd78ae28e451ff376b5
+  metadata.gz: b4c38506bd2a3f6a84d1bbe430f9812e5989bf80d6e8198bfee55ebb1b7f6b8010ee998b896e4202443224046a5fd33c1ed1913fa43d59a25d2cfdf4028d94cc
+  data.tar.gz: a91bd12229dead116b77d412043ecfcef306af8a2ac80772e2c6596574c488cd9e9c97e662334d76a97acb50b95bc578c51ccfcd5dbb3c7b53145e832eea9989

data/.solargraph.yml

@@ -0,0 +1,15 @@
+---
+include:
+- "**/*.rb"
+exclude:
+- spec/**/*
+- test/**/*
+- vendor/**/*
+- ".bundle/**/*"
+require: []
+domains: []
+reporters:
+- rubocop
+require_paths: []
+plugins: []
+max_files: 5000

data/lib/aws_recon/aws_recon.rb

@@ -65,6 +65,17 @@ module AwsRecon
       @resources.concat(collection) if @options.output_file
     end
 
+    #
+    # Format @resources as either
+    #
+    def formatted_json
+      if @options.jsonl
+        @resources.map { |r| JSON.generate(r) }.join("\n")
+      else
+        @resources.to_json
+      end
+    end
+
     #
     # main wrapper
     #
@@ -112,7 +123,7 @@ module AwsRecon
       if @options.output_file && !@options.s3
         puts "Saving resources to #{@options.output_file}.\n\n"
 
-        File.write(@options.output_file, @resources.to_json)
+        File.write(@options.output_file, formatted_json)
       end
 
       # write output file to S3 bucket
@@ -128,7 +139,7 @@ module AwsRecon
           # build IO object and gzip it
           io = StringIO.new
           gzip_data = Zlib::GzipWriter.new(io)
-          gzip_data.write(@resources.to_json)
+          gzip_data.write(formatted_json)
           gzip_data.close
 
           # send it to S3

data/lib/aws_recon/collectors/dynamodb.rb

@@ -18,7 +18,7 @@ class DynamoDB < Mapper
 
       struct = OpenStruct.new(response)
       struct.type = 'limits'
-      struct.arn = "arn:aws:dynamodb:#{@region}:#{@account}:limits"
+      struct.arn = "arn:aws:dynamodb:#{@region}:#{@account}/limits"
 
       resources.push(struct.to_h)
     end

data/lib/aws_recon/collectors/ec2.rb

@@ -29,7 +29,7 @@ class EC2 < Mapper
         struct = OpenStruct.new
         struct.attributes = response.account_attributes.map(&:to_h)
         struct.type = 'account'
-        struct.arn = "arn:aws::#{@account}"
+        struct.arn = "arn:aws:ec2::#{@account}/account_attributes"
 
         resources.push(struct.to_h)
       end
@@ -45,6 +45,7 @@ class EC2 < Mapper
 
         struct = OpenStruct.new(response.to_h)
         struct.type = 'ebs_encryption_settings'
+        struct.arn = "arn:aws:ec2:#{@region}:#{@account}/ebs_encryption_settings"
 
         resources.push(struct.to_h)
       end
@@ -95,7 +96,7 @@ class EC2 < Mapper
         response.vpcs.each do |vpc|
           struct = OpenStruct.new(vpc.to_h)
           struct.type = 'vpc'
-          struct.arn = vpc.vpc_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{vpc.vpc_id}" # no true ARN
           struct.flow_logs = @client
                              .describe_flow_logs({ filter: [{ name: 'resource-id', values: [vpc.vpc_id] }] })
                              .flow_logs.first.to_h
@@ -113,7 +114,7 @@ class EC2 < Mapper
         response.security_groups.each do |security_group|
           struct = OpenStruct.new(security_group.to_h)
           struct.type = 'security_group'
-          struct.arn = security_group.group_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{security_group.group_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -128,7 +129,7 @@ class EC2 < Mapper
         response.network_interfaces.each do |network_interface|
           struct = OpenStruct.new(network_interface.to_h)
           struct.type = 'network_interface'
-          struct.arn = network_interface.network_interface_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{network_interface.network_interface_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -143,7 +144,7 @@ class EC2 < Mapper
         response.network_acls.each do |network_acl|
           struct = OpenStruct.new(network_acl.to_h)
           struct.type = 'network_acl'
-          struct.arn = network_acl.network_acl_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{network_acl.network_acl_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -158,7 +159,7 @@ class EC2 < Mapper
         response.subnets.each do |subnet|
           struct = OpenStruct.new(subnet.to_h)
           struct.type = 'subnet'
-          struct.arn = subnet.subnet_arn
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{subnet.subnet_arn}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -173,7 +174,7 @@ class EC2 < Mapper
         response.addresses.each do |address|
           struct = OpenStruct.new(address.to_h)
           struct.type = 'eip_address'
-          struct.arn = address.allocation_id
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{address.allocation_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -188,7 +189,7 @@ class EC2 < Mapper
         response.nat_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'nat_gateway'
-          struct.arn = gateway.nat_gateway_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.nat_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -203,7 +204,7 @@ class EC2 < Mapper
         response.internet_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'internet_gateway'
-          struct.arn = gateway.internet_gateway_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.internet_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -218,7 +219,7 @@ class EC2 < Mapper
         response.route_tables.each do |table|
           struct = OpenStruct.new(table.to_h)
           struct.type = 'route_table'
-          struct.arn = table.route_table_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{table.route_table_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -233,7 +234,7 @@ class EC2 < Mapper
         response.images.each do |image|
           struct = OpenStruct.new(image.to_h)
           struct.type = 'image'
-          struct.arn = image.image_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{image.image_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -248,7 +249,7 @@ class EC2 < Mapper
         response.snapshots.each do |snapshot|
           struct = OpenStruct.new(snapshot.to_h)
           struct.type = 'snapshot'
-          struct.arn = snapshot.snapshot_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{snapshot.snapshot_id}" # no true ARN
           struct.create_volume_permissions = @client.describe_snapshot_attribute({
                                                                                    attribute: 'createVolumePermission',
                                                                                    snapshot_id: snapshot.snapshot_id
@@ -267,7 +268,7 @@ class EC2 < Mapper
         response.flow_logs.each do |flow_log|
           struct = OpenStruct.new(flow_log.to_h)
           struct.type = 'flow_log'
-          struct.arn = flow_log.flow_log_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{flow_log.flow_log_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -282,7 +283,7 @@ class EC2 < Mapper
         response.volumes.each do |volume|
           struct = OpenStruct.new(volume.to_h)
           struct.type = 'volume'
-          struct.arn = volume.volume_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{volume.volume_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -297,7 +298,7 @@ class EC2 < Mapper
         response.vpn_gateways.each do |gateway|
           struct = OpenStruct.new(gateway.to_h)
           struct.type = 'vpn_gateway'
-          struct.arn = gateway.vpn_gateway_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{gateway.vpn_gateway_id}" # no true ARN
 
           resources.push(struct.to_h)
         end
@@ -312,7 +313,7 @@ class EC2 < Mapper
         response.vpc_peering_connections.each do |peer|
           struct = OpenStruct.new(peer.to_h)
           struct.type = 'peering_connection'
-          struct.arn = peer.vpc_peering_connection_id # no true ARN
+          struct.arn = "arn:aws:ec2:#{@region}:#{@account}/#{peer.vpc_peering_connection_id}" # no true ARN
 
           resources.push(struct.to_h)
         end

data/lib/aws_recon/collectors/route53.rb

@@ -19,7 +19,7 @@ class Route53 < Mapper
       response.hosted_zones.each do |zone|
         struct = OpenStruct.new(zone.to_h)
         struct.type = 'zone'
-        struct.arn = "aws:route53:#{@region}:#{@account}:zone/#{zone.name}"
+        struct.arn = "arn:aws:route53:#{@region}:#{@account}:zone/#{zone.name}"
         struct.logging_config = @client
                                 .list_query_logging_configs({ hosted_zone_id: zone.id })
                                 .query_logging_configs.first.to_h

data/lib/aws_recon/collectors/s3.rb

@@ -73,6 +73,9 @@ class S3 < Mapper
         end
 
         resources.push(struct.to_h)
+
+      rescue Aws::S3::Errors::NoSuchBucket
+        # skip missing bucket
       end
     end
 

data/lib/aws_recon/options.rb

@@ -20,6 +20,7 @@ class Parser
     :output_file,
     :output_format,
     :threads,
+    :jsonl,
     :collect_user_data,
     :skip_slow,
     :skip_credential_report,
@@ -55,6 +56,7 @@ class Parser
       false,
       false,
       false,
+      false,
       false
     )
 
@@ -116,6 +118,11 @@ class Parser
         args.threads = threads.to_i if (0..Parser::MAX_THREADS).include?(threads.to_i)
       end
 
+      # output NDJSON/JSONL format
+      opts.on('-l', '--json-lines', 'Output NDJSON/JSONL format (default: false)') do
+        args.jsonl = true
+      end
+
       # collect EC2 instance user data
       opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do
         args.collect_user_data = true

data/lib/aws_recon/version.rb

@@ -1,3 +1,3 @@
 module AwsRecon
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/readme.md

@@ -54,13 +54,13 @@ To run locally, first install the gem:
 
 ```
 $ gem install aws_recon
-Fetching aws_recon-0.4.0.gem
+Fetching aws_recon-0.4.5.gem
 Fetching aws-sdk-3.0.1.gem
 Fetching parallel-1.20.1.gem
 ...
 Successfully installed aws-sdk-3.0.1
 Successfully installed parallel-1.20.1
-Successfully installed aws_recon-0.4.0
+Successfully installed aws_recon-0.4.5
 ```
 
 Or add it to your Gemfile using `bundle`:
@@ -72,7 +72,7 @@ Resolving dependencies...
 ...
 Using aws-sdk 3.0.1
 Using parallel-1.20.1
-Using aws_recon 0.4.0
+Using aws_recon 0.4.5
 ```
 
 ## Usage
@@ -249,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
 ```
 $ aws_recon -h
 
-AWS Recon - AWS Inventory Collector (0.4.0)
+AWS Recon - AWS Inventory Collector (0.4.5)
 
 Usage: aws_recon [options]
     -r, --regions [REGIONS]          Regions to scan, separated by comma (default: all)
@@ -261,6 +261,7 @@ Usage: aws_recon [options]
     -o, --output [OUTPUT]            Specify output file (default: output.json)
     -f, --format [FORMAT]            Specify output format (default: aws)
     -t, --threads [THREADS]          Specify max threads (default: 8, max: 128)
+    -l, --json-lines                 Output NDJSON/JSONL format (default: false)
     -u, --user-data                  Collect EC2 instance user data (default: false)
     -z, --skip-slow                  Skip slow operations (default: false)
     -g, --skip-credential-report     Skip generating IAM credential report (default: false)

data/utils/cloudformation/aws-recon-cfn-template.yml

@@ -0,0 +1,151 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: 'Deploys AWS Recon inventory collection resources, scheduled ECS task and corresponding IAM roles and policies.'
+Resources:
+  AWSReconVPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: '10.75.0.0/27'
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+  AWSReconSubnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      CidrBlock: '10.75.0.0/28'
+      VpcId: !Ref AWSReconVPC
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+    DependsOn: AWSReconVPC
+  AWSReconSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: AWS Recon collection egress
+      VpcId: !Ref AWSReconVPC
+      SecurityGroupEgress:
+        - IpProtocol: -1
+          FromPort: 0
+          ToPort: 0
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+  AWSReconInternetGateway:
+    Type: AWS::EC2::InternetGateway
+    Properties:
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+  AWSReconInternetGatewayAttachment:
+    Type: AWS::EC2::VPCGatewayAttachment
+    Properties:
+      InternetGatewayId: !Ref AWSReconInternetGateway
+      VpcId: !Ref AWSReconVPC
+  AWSReconEgressRouteTable:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref AWSReconVPC
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+  AWSReconSubnetRouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      SubnetId: !Ref AWSReconSubnet
+      RouteTableId: !Ref AWSReconEgressRouteTable
+  AWSReconEgressRoute:
+    Type: AWS::EC2::Route
+    Properties:
+      DestinationCidrBlock: '0.0.0.0/0'
+      GatewayId: !Ref AWSReconInternetGateway
+      RouteTableId: !Ref AWSReconEgressRouteTable
+  AWSReconECSCluster:
+    Type: AWS::ECS::Cluster
+    Properties:
+      ClusterName: aws-recon-CFN
+      CapacityProviders:
+        - FARGATE
+      Tags:
+        - Key: Name
+          Value: aws-recon-CFN
+    DependsOn: AWSReconSubnet
+  AWSReconECSTask:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      Family: aws-recon-CFN
+      RequiresCompatibilities:
+        - FARGATE
+      NetworkMode: awsvpc
+      Cpu: 1024
+      Memory: 2048
+      TaskRoleArn: !Ref AWSReconECSTaskRole
+      ExecutionRoleArn: !Ref AWSReconECSExecutionRole
+      ContainerDefinitions:
+        - Name: aws-recon-CFN
+          Image: 'darkbitio/aws_recon:latest'
+          EntryPoint:
+            - 'aws_recon'
+            - '--verbose'
+            - '--format'
+            - 'custom'
+  AWSReconECSTaskRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: aws-recon-ecs-task-role
+      ManagedPolicyArns:
+        - 'arn:aws:iam::aws:policy/ReadOnlyAccess'
+      Policies:
+        - PolicyName: AWSReconECSTaskRole
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action: 's3:PutObject'
+                Resource: 'arn:aws:s3:::CHANGEME/*'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - ecs.amazonaws.com
+                - ecs-tasks.amazonaws.com
+            Action: 'sts:AssumeRole'
+  AWSReconECSExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: aws-recon-ecs-execution-role
+      Policies:
+        - PolicyName: AWSReconECSTaskExecutionPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'ecr:GetAuthorizationToken'
+                  - 'ecr:BatchCheckLayerAvailability'
+                  - 'ecr:GetDownloadUrlForLayer'
+                  - 'ecr:BatchGetImage'
+                  - 'logs:CreateLogStream'
+                  - 'logs:PutLogEvents'
+                Resource: '*'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - ecs-tasks.amazonaws.com
+            Action: 'sts:AssumeRole'
+  AWSReconCloudWatchEventsRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: aws-recon-events-role
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - events.amazonaws.com
+            Action: 'sts:AssumeRole'

data/{terraform → utils/terraform}/ecs.tf

@@ -20,6 +20,9 @@ resource "aws_ecs_task_definition" "aws_recon_task" {
       entryPoint = [
         "aws_recon",
         "--verbose",
+        "--format",
+        "custom",
+        "--json-lines",
         "--s3-bucket",
         "${aws_s3_bucket.aws_recon.bucket}:${data.aws_region.current.name}",
         "--regions",

data/utils/terraform/output.tf

@@ -0,0 +1,15 @@
+output "aws_recon_ecs_cluster" {
+  value = aws_ecs_cluster.aws_recon.name
+}
+
+output "aws_recon_ecs_scheduled_task" {
+  value = aws_cloudwatch_event_rule.default.name
+}
+
+output "aws_recon_s3_bucket" {
+  value = aws_s3_bucket.aws_recon.bucket
+}
+
+output "aws_recon_task_manual_run_command" {
+  value = "\nOne-off task run command:\n\naws ecs run-task --task-definition ${aws_ecs_task_definition.aws_recon_task.family} --cluster ${aws_ecs_cluster.aws_recon.name} --launch-type FARGATE --network-configuration \"awsvpcConfiguration={subnets=[${aws_subnet.subnet.id}],securityGroups=[${aws_security_group.sg.id}],assignPublicIp=ENABLED}\"\n"
+}

data/{terraform → utils/terraform}/vars.tf

@@ -41,6 +41,7 @@ variable "aws_regions" {
   ]
 }
 
+# must be one of: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365
 variable "retention_period" {
   type    = number
   default = 30

data/{terraform → utils/terraform}/vpc.tf

@@ -9,10 +9,9 @@ resource "aws_vpc" "vpc" {
 
 # Create subnet
 resource "aws_subnet" "subnet" {
-  vpc_id                  = aws_vpc.vpc.id
-  cidr_block              = local.subnet_cidr_block
-  availability_zone       = data.aws_availability_zones.available.names[0]
-  map_public_ip_on_launch = true
+  vpc_id            = aws_vpc.vpc.id
+  cidr_block        = local.subnet_cidr_block
+  availability_zone = data.aws_availability_zones.available.names[0]
 
   tags = {
     Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}-public"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_recon
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Josh Larsen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-02 00:00:00.000000000 Z
+date: 2021-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -167,6 +167,7 @@ files:
 - ".github/workflows/smoke-test.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".solargraph.yml"
 - Dockerfile
 - Gemfile
 - LICENSE.txt
@@ -244,15 +245,16 @@ files:
 - lib/aws_recon/services.yaml
 - lib/aws_recon/version.rb
 - readme.md
-- terraform/cloudwatch.tf
-- terraform/ecs.tf
-- terraform/iam.tf
-- terraform/main.tf
-- terraform/output.tf
-- terraform/readme.md
-- terraform/s3.tf
-- terraform/vars.tf
-- terraform/vpc.tf
+- utils/cloudformation/aws-recon-cfn-template.yml
+- utils/terraform/cloudwatch.tf
+- utils/terraform/ecs.tf
+- utils/terraform/iam.tf
+- utils/terraform/main.tf
+- utils/terraform/output.tf
+- utils/terraform/readme.md
+- utils/terraform/s3.tf
+- utils/terraform/vars.tf
+- utils/terraform/vpc.tf
 homepage: https://github.com/darkbitio/aws-recon
 licenses:
 - MIT

data/terraform/output.tf

@@ -1,11 +0,0 @@
-output "aws_recon_ecs_cluster" {
-  value = aws_ecs_cluster.aws_recon.name
-}
-
-output "aws_recon_ecs_scheduled_task" {
-  value = aws_cloudwatch_event_rule.default.name
-}
-
-output "aws_recon_s3_bucket" {
-  value = aws_s3_bucket.aws_recon.bucket
-}