aws_recon 0.3.5 → 0.4.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rubocop.yml +0 -1
- data/aws_recon.gemspec +1 -1
- data/lib/aws_recon/aws_recon.rb +40 -9
- data/lib/aws_recon/collectors.rb +3 -1
- data/lib/aws_recon/collectors/route53.rb +1 -1
- data/lib/aws_recon/collectors/s3.rb +3 -0
- data/lib/aws_recon/lib/mapper.rb +2 -2
- data/lib/aws_recon/options.rb +8 -0
- data/lib/aws_recon/version.rb +1 -1
- data/readme.md +38 -11
- data/utils/cloudformation/aws-recon-cfn-template.yml +151 -0
- data/utils/terraform/cloudwatch.tf +26 -0
- data/utils/terraform/ecs.tf +49 -0
- data/utils/terraform/iam.tf +125 -0
- data/utils/terraform/main.tf +13 -0
- data/utils/terraform/output.tf +15 -0
- data/utils/terraform/readme.md +20 -0
- data/utils/terraform/s3.tf +20 -0
- data/utils/terraform/vars.tf +58 -0
- data/utils/terraform/vpc.tf +73 -0
- metadata +13 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 48ccf03d964fff678dc732fcf74d4dcd9c7c06293845bbb9a4f54e9e1ce61a24
|
4
|
+
data.tar.gz: 7c39747a7845fe497be052baae41af8ee45014166d97114abb797d5f6c06e5b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b6c26ff3b38dd58c63313b70a609e6b08fe6d16762afb9651bbc81d6cef21d7c46e8b11ad73d40caf0e6abbfd877ef5d5f2fde1536425b70a25afe82b7a4a78
|
7
|
+
data.tar.gz: a74af8ad1b868895d997110a4688bc88266536ad2187c235ba27a0720e3ccce0d18d6959981ab3e82e40bb738944d1402380761f9c3c65ce33cc1649bfddbc4e
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/aws_recon.gemspec
CHANGED
@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.version = AwsRecon::VERSION
|
10
10
|
spec.authors = ['Josh Larsen', 'Darkbit']
|
11
11
|
spec.required_ruby_version = '>= 2.5.0'
|
12
|
-
spec.summary = 'A multi-threaded AWS inventory collection
|
12
|
+
spec.summary = 'A multi-threaded AWS security-focused inventory collection tool.'
|
13
13
|
spec.description = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
|
14
14
|
spec.homepage = 'https://github.com/darkbitio/aws-recon'
|
15
15
|
spec.license = 'MIT'
|
data/lib/aws_recon/aws_recon.rb
CHANGED
@@ -34,9 +34,9 @@ module AwsRecon
|
|
34
34
|
# formatter
|
35
35
|
@formatter = Formatter.new
|
36
36
|
|
37
|
-
unless @options.stream_output
|
38
|
-
|
39
|
-
|
37
|
+
return unless @options.stream_output
|
38
|
+
|
39
|
+
puts "\nStarting collection with #{@options.threads} threads...\n"
|
40
40
|
end
|
41
41
|
|
42
42
|
#
|
@@ -72,7 +72,7 @@ module AwsRecon
|
|
72
72
|
#
|
73
73
|
# global services
|
74
74
|
#
|
75
|
-
@aws_services.map { |x| OpenStruct.new(x) }.filter
|
75
|
+
@aws_services.map { |x| OpenStruct.new(x) }.filter(&:global).each do |service|
|
76
76
|
# user included this service in the args
|
77
77
|
next unless @services.include?(service.name)
|
78
78
|
|
@@ -102,16 +102,47 @@ module AwsRecon
|
|
102
102
|
rescue Interrupt # ctrl-c
|
103
103
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
|
104
104
|
|
105
|
-
puts "\nStopped early after
|
105
|
+
puts "\nStopped early after #{elapsed.to_i} seconds.\n"
|
106
106
|
ensure
|
107
|
-
|
108
|
-
if @options.output_file
|
109
|
-
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
|
107
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting
|
110
108
|
|
111
|
-
|
109
|
+
puts "\nFinished in #{elapsed.to_i} seconds.\n\n"
|
110
|
+
|
111
|
+
# write output file
|
112
|
+
if @options.output_file && !@options.s3
|
113
|
+
puts "Saving resources to #{@options.output_file}.\n\n"
|
112
114
|
|
113
115
|
File.write(@options.output_file, @resources.to_json)
|
114
116
|
end
|
117
|
+
|
118
|
+
# write output file to S3 bucket
|
119
|
+
if @options.s3
|
120
|
+
t = Time.now.utc
|
121
|
+
|
122
|
+
s3_full_object_path = "AWSRecon/#{t.year}/#{t.month}/#{t.day}/#{@account_id}_aws_recon_#{t.to_i}.json.gz"
|
123
|
+
|
124
|
+
begin
|
125
|
+
# get bucket name (and region if not us-east-1)
|
126
|
+
s3_bucket, s3_region = @options.s3.split(':')
|
127
|
+
|
128
|
+
# build IO object and gzip it
|
129
|
+
io = StringIO.new
|
130
|
+
gzip_data = Zlib::GzipWriter.new(io)
|
131
|
+
gzip_data.write(@resources.to_json)
|
132
|
+
gzip_data.close
|
133
|
+
|
134
|
+
# send it to S3
|
135
|
+
s3_client = Aws::S3::Client.new(region: s3_region || 'us-east-1')
|
136
|
+
s3_resource = Aws::S3::Resource.new(client: s3_client)
|
137
|
+
obj = s3_resource.bucket(s3_bucket).object(s3_full_object_path)
|
138
|
+
obj.put(body: io.string)
|
139
|
+
|
140
|
+
puts "Saving resources to S3 s3://#{s3_bucket}/#{s3_full_object_path}\n\n"
|
141
|
+
rescue Aws::S3::Errors::ServiceError => e
|
142
|
+
puts "Error! - could not save output S3 bucket\n\n"
|
143
|
+
puts "#{e.message} - #{e.code}\n"
|
144
|
+
end
|
145
|
+
end
|
115
146
|
end
|
116
147
|
end
|
117
148
|
end
|
data/lib/aws_recon/collectors.rb
CHANGED
@@ -19,7 +19,7 @@ class Route53 < Mapper
|
|
19
19
|
response.hosted_zones.each do |zone|
|
20
20
|
struct = OpenStruct.new(zone.to_h)
|
21
21
|
struct.type = 'zone'
|
22
|
-
struct.arn = "aws:route53:#{@region}:#{@account}:zone/#{zone.name}"
|
22
|
+
struct.arn = "arn:aws:route53:#{@region}:#{@account}:zone/#{zone.name}"
|
23
23
|
struct.logging_config = @client
|
24
24
|
.list_query_logging_configs({ hosted_zone_id: zone.id })
|
25
25
|
.query_logging_configs.first.to_h
|
data/lib/aws_recon/lib/mapper.rb
CHANGED
@@ -68,12 +68,12 @@ class Mapper
|
|
68
68
|
def log(*msg)
|
69
69
|
return unless @options.verbose
|
70
70
|
|
71
|
-
puts _msg(msg).map
|
71
|
+
puts _msg(msg).map(&:to_s).join('.')
|
72
72
|
end
|
73
73
|
|
74
74
|
def log_error(*msg)
|
75
75
|
return unless @options.verbose
|
76
76
|
|
77
|
-
puts _msg(msg).map
|
77
|
+
puts _msg(msg).map(&:to_s).join('.')
|
78
78
|
end
|
79
79
|
end
|
data/lib/aws_recon/options.rb
CHANGED
@@ -6,6 +6,7 @@
|
|
6
6
|
class Parser
|
7
7
|
DEFAULT_CONFIG_FILE = nil
|
8
8
|
DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze
|
9
|
+
DEFAULT_S3_PATH = nil
|
9
10
|
SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze
|
10
11
|
DEFAULT_FORMAT = 'aws'
|
11
12
|
DEFAULT_THREADS = 8
|
@@ -15,6 +16,7 @@ class Parser
|
|
15
16
|
:regions,
|
16
17
|
:services,
|
17
18
|
:config_file,
|
19
|
+
:s3,
|
18
20
|
:output_file,
|
19
21
|
:output_format,
|
20
22
|
:threads,
|
@@ -43,6 +45,7 @@ class Parser
|
|
43
45
|
aws_regions,
|
44
46
|
aws_services.map { |service| service[:name] },
|
45
47
|
DEFAULT_CONFIG_FILE,
|
48
|
+
DEFAULT_S3_PATH,
|
46
49
|
DEFAULT_OUTPUT_FILE,
|
47
50
|
DEFAULT_FORMAT,
|
48
51
|
DEFAULT_THREADS,
|
@@ -93,6 +96,11 @@ class Parser
|
|
93
96
|
args.config_file = config
|
94
97
|
end
|
95
98
|
|
99
|
+
# write output file to S3 bucket
|
100
|
+
opts.on('-b', '--s3-bucket [BUCKET:REGION]', 'Write output file to S3 bucket (default: \'\')') do |bucket_with_region|
|
101
|
+
args.s3 = bucket_with_region
|
102
|
+
end
|
103
|
+
|
96
104
|
# output file
|
97
105
|
opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output|
|
98
106
|
args.output_file = File.expand_path(File.join(Dir.pwd, output))
|
data/lib/aws_recon/version.rb
CHANGED
data/readme.md
CHANGED
@@ -3,19 +3,19 @@
|
|
3
3
|
|
4
4
|
# AWS Recon
|
5
5
|
|
6
|
-
A multi-threaded AWS inventory collection tool.
|
6
|
+
A multi-threaded AWS security-focused inventory collection tool written in Ruby.
|
7
7
|
|
8
|
-
|
8
|
+
This tool was created to facilitate efficient collection of a large amount of AWS resource attributes and metadata. It aims to collect nearly everything that is relevant to the security configuration and posture of an AWS environment.
|
9
9
|
|
10
|
-
Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity
|
10
|
+
Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity to accurately measure security posture (e.g. detailed resource attribute data, fully parsed policy documents, and nested resource relationships).
|
11
11
|
|
12
|
-
|
12
|
+
AWS Recon handles collection from large accounts by taking advantage of automatic retries (either due to network reliability or API throttling), automatic paging of large responses (> 100 resources per API call), and multi-threading parallel requests to speed up collection.
|
13
13
|
|
14
14
|
## Project Goals
|
15
15
|
|
16
16
|
- More complete resource coverage than available tools (especially for ECS & EKS)
|
17
17
|
- More granular resource detail, including nested related resources in the output
|
18
|
-
- Flexible output (console, JSON lines, plain JSON, file, standard out)
|
18
|
+
- Flexible output (console, JSON lines, plain JSON, file, S3 bucket, and standard out)
|
19
19
|
- Efficient (multi-threaded, rate limited, automatic retries, and automatic result paging)
|
20
20
|
- Easy to maintain and extend
|
21
21
|
|
@@ -31,7 +31,7 @@ Use Docker version 19.x or above to run the pre-built image without having to in
|
|
31
31
|
|
32
32
|
#### Running locally via Ruby
|
33
33
|
|
34
|
-
If you already have Ruby installed (2.
|
34
|
+
If you already have Ruby installed (2.6.x or 2.7.x), you may want to install the Ruby gem.
|
35
35
|
|
36
36
|
### Installation
|
37
37
|
|
@@ -54,13 +54,13 @@ To run locally, first install the gem:
|
|
54
54
|
|
55
55
|
```
|
56
56
|
$ gem install aws_recon
|
57
|
-
Fetching aws_recon-0.
|
57
|
+
Fetching aws_recon-0.4.0.gem
|
58
58
|
Fetching aws-sdk-3.0.1.gem
|
59
59
|
Fetching parallel-1.20.1.gem
|
60
60
|
...
|
61
61
|
Successfully installed aws-sdk-3.0.1
|
62
62
|
Successfully installed parallel-1.20.1
|
63
|
-
Successfully installed aws_recon-0.
|
63
|
+
Successfully installed aws_recon-0.4.0
|
64
64
|
```
|
65
65
|
|
66
66
|
Or add it to your Gemfile using `bundle`:
|
@@ -72,7 +72,7 @@ Resolving dependencies...
|
|
72
72
|
...
|
73
73
|
Using aws-sdk 3.0.1
|
74
74
|
Using parallel-1.20.1
|
75
|
-
Using aws_recon 0.
|
75
|
+
Using aws_recon 0.4.0
|
76
76
|
```
|
77
77
|
|
78
78
|
## Usage
|
@@ -158,13 +158,37 @@ Finished in 46 seconds. Saving resources to output.json.
|
|
158
158
|
#### Example command line options
|
159
159
|
|
160
160
|
```
|
161
|
+
# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
|
162
|
+
|
161
163
|
$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
|
162
164
|
```
|
163
165
|
|
164
166
|
```
|
167
|
+
# collect S3 and EC2 global resources, as well as us-east-1 and us-east-2
|
168
|
+
|
165
169
|
$ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
|
166
170
|
```
|
167
171
|
|
172
|
+
```
|
173
|
+
# save output to S3 bucket
|
174
|
+
|
175
|
+
$ AWS_PROFILE=<profile> aws_recon \
|
176
|
+
--services S3,EC2 \
|
177
|
+
--regions global,us-east-1,us-east-2 \
|
178
|
+
--verbose \
|
179
|
+
--s3-bucket my-recon-bucket
|
180
|
+
```
|
181
|
+
|
182
|
+
```
|
183
|
+
# save output to S3 bucket with a home region other than us-east-1
|
184
|
+
|
185
|
+
$ AWS_PROFILE=<profile> aws_recon \
|
186
|
+
--services S3,EC2 \
|
187
|
+
--regions global,us-east-1,us-east-2 \
|
188
|
+
--verbose \
|
189
|
+
--s3-bucket my-recon-bucket:us-west-2
|
190
|
+
```
|
191
|
+
|
168
192
|
Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted (NDJSON) output.
|
169
193
|
|
170
194
|
```
|
@@ -225,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
|
|
225
249
|
```
|
226
250
|
$ aws_recon -h
|
227
251
|
|
228
|
-
AWS Recon - AWS Inventory Collector (0.
|
252
|
+
AWS Recon - AWS Inventory Collector (0.4.0)
|
229
253
|
|
230
254
|
Usage: aws_recon [options]
|
231
255
|
-r, --regions [REGIONS] Regions to scan, separated by comma (default: all)
|
@@ -233,6 +257,7 @@ Usage: aws_recon [options]
|
|
233
257
|
-s, --services [SERVICES] Services to scan, separated by comma (default: all)
|
234
258
|
-x, --not-services [SERVICES] Services to skip, separated by comma (default: none)
|
235
259
|
-c, --config [CONFIG] Specify config file for services & regions (e.g. config.yaml)
|
260
|
+
-b, --s3-bucket [BUCKET:REGION] Write output file to S3 bucket (default: '')
|
236
261
|
-o, --output [OUTPUT] Specify output file (default: output.json)
|
237
262
|
-f, --format [FORMAT] Specify output format (default: aws)
|
238
263
|
-t, --threads [THREADS] Specify max threads (default: 8, max: 128)
|
@@ -251,6 +276,8 @@ Usage: aws_recon [options]
|
|
251
276
|
|
252
277
|
Output is always some form of JSON - either JSON lines or plain JSON. The output is either written to a file (the default), or written to stdout (with `-j`).
|
253
278
|
|
279
|
+
When writing to an S3 bucket, the JSON output is automatically compressed with `gzip`.
|
280
|
+
|
254
281
|
## Support for Manually Enabled Regions
|
255
282
|
|
256
283
|
If you have enabled **manually enabled regions**:
|
@@ -351,7 +378,7 @@ $ cd aws-recon
|
|
351
378
|
Create a sticky gemset if using RVM:
|
352
379
|
|
353
380
|
```
|
354
|
-
$ rvm use 2.
|
381
|
+
$ rvm use 2.7.2@aws_recon_dev --create --ruby-version
|
355
382
|
```
|
356
383
|
|
357
384
|
Run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -0,0 +1,151 @@
|
|
1
|
+
AWSTemplateFormatVersion: '2010-09-09'
|
2
|
+
Description: 'Deploys AWS Recon inventory collection resources, scheduled ECS task and corresponding IAM roles and policies.'
|
3
|
+
Resources:
|
4
|
+
AWSReconVPC:
|
5
|
+
Type: AWS::EC2::VPC
|
6
|
+
Properties:
|
7
|
+
CidrBlock: '10.75.0.0/27'
|
8
|
+
Tags:
|
9
|
+
- Key: Name
|
10
|
+
Value: aws-recon-CFN
|
11
|
+
AWSReconSubnet:
|
12
|
+
Type: AWS::EC2::Subnet
|
13
|
+
Properties:
|
14
|
+
CidrBlock: '10.75.0.0/28'
|
15
|
+
VpcId: !Ref AWSReconVPC
|
16
|
+
Tags:
|
17
|
+
- Key: Name
|
18
|
+
Value: aws-recon-CFN
|
19
|
+
DependsOn: AWSReconVPC
|
20
|
+
AWSReconSecurityGroup:
|
21
|
+
Type: AWS::EC2::SecurityGroup
|
22
|
+
Properties:
|
23
|
+
GroupDescription: AWS Recon collection egress
|
24
|
+
VpcId: !Ref AWSReconVPC
|
25
|
+
SecurityGroupEgress:
|
26
|
+
- IpProtocol: -1
|
27
|
+
FromPort: 0
|
28
|
+
ToPort: 0
|
29
|
+
CidrIp: 0.0.0.0/0
|
30
|
+
Tags:
|
31
|
+
- Key: Name
|
32
|
+
Value: aws-recon-CFN
|
33
|
+
AWSReconInternetGateway:
|
34
|
+
Type: AWS::EC2::InternetGateway
|
35
|
+
Properties:
|
36
|
+
Tags:
|
37
|
+
- Key: Name
|
38
|
+
Value: aws-recon-CFN
|
39
|
+
AWSReconInternetGatewayAttachment:
|
40
|
+
Type: AWS::EC2::VPCGatewayAttachment
|
41
|
+
Properties:
|
42
|
+
InternetGatewayId: !Ref AWSReconInternetGateway
|
43
|
+
VpcId: !Ref AWSReconVPC
|
44
|
+
AWSReconEgressRouteTable:
|
45
|
+
Type: AWS::EC2::RouteTable
|
46
|
+
Properties:
|
47
|
+
VpcId: !Ref AWSReconVPC
|
48
|
+
Tags:
|
49
|
+
- Key: Name
|
50
|
+
Value: aws-recon-CFN
|
51
|
+
AWSReconSubnetRouteTableAssociation:
|
52
|
+
Type: AWS::EC2::SubnetRouteTableAssociation
|
53
|
+
Properties:
|
54
|
+
SubnetId: !Ref AWSReconSubnet
|
55
|
+
RouteTableId: !Ref AWSReconEgressRouteTable
|
56
|
+
AWSReconEgressRoute:
|
57
|
+
Type: AWS::EC2::Route
|
58
|
+
Properties:
|
59
|
+
DestinationCidrBlock: '0.0.0.0/0'
|
60
|
+
GatewayId: !Ref AWSReconInternetGateway
|
61
|
+
RouteTableId: !Ref AWSReconEgressRouteTable
|
62
|
+
AWSReconECSCluster:
|
63
|
+
Type: AWS::ECS::Cluster
|
64
|
+
Properties:
|
65
|
+
ClusterName: aws-recon-CFN
|
66
|
+
CapacityProviders:
|
67
|
+
- FARGATE
|
68
|
+
Tags:
|
69
|
+
- Key: Name
|
70
|
+
Value: aws-recon-CFN
|
71
|
+
DependsOn: AWSReconSubnet
|
72
|
+
AWSReconECSTask:
|
73
|
+
Type: AWS::ECS::TaskDefinition
|
74
|
+
Properties:
|
75
|
+
Family: aws-recon-CFN
|
76
|
+
RequiresCompatibilities:
|
77
|
+
- FARGATE
|
78
|
+
NetworkMode: awsvpc
|
79
|
+
Cpu: 1024
|
80
|
+
Memory: 2048
|
81
|
+
TaskRoleArn: !Ref AWSReconECSTaskRole
|
82
|
+
ExecutionRoleArn: !Ref AWSReconECSExecutionRole
|
83
|
+
ContainerDefinitions:
|
84
|
+
- Name: aws-recon-CFN
|
85
|
+
Image: 'darkbitio/aws_recon:latest'
|
86
|
+
EntryPoint:
|
87
|
+
- 'aws_recon'
|
88
|
+
- '--verbose'
|
89
|
+
- '--format'
|
90
|
+
- 'custom'
|
91
|
+
AWSReconECSTaskRole:
|
92
|
+
Type: AWS::IAM::Role
|
93
|
+
Properties:
|
94
|
+
RoleName: aws-recon-ecs-task-role
|
95
|
+
ManagedPolicyArns:
|
96
|
+
- 'arn:aws:iam::aws:policy/ReadOnlyAccess'
|
97
|
+
Policies:
|
98
|
+
- PolicyName: AWSReconECSTaskRole
|
99
|
+
PolicyDocument:
|
100
|
+
Version: '2012-10-17'
|
101
|
+
Statement:
|
102
|
+
- Effect: Allow
|
103
|
+
Action: 's3:PutObject'
|
104
|
+
Resource: 'arn:aws:s3:::CHANGEME/*'
|
105
|
+
AssumeRolePolicyDocument:
|
106
|
+
Version: '2012-10-17'
|
107
|
+
Statement:
|
108
|
+
- Effect: Allow
|
109
|
+
Principal:
|
110
|
+
Service:
|
111
|
+
- ecs.amazonaws.com
|
112
|
+
- ecs-tasks.amazonaws.com
|
113
|
+
Action: 'sts:AssumeRole'
|
114
|
+
AWSReconECSExecutionRole:
|
115
|
+
Type: AWS::IAM::Role
|
116
|
+
Properties:
|
117
|
+
RoleName: aws-recon-ecs-execution-role
|
118
|
+
Policies:
|
119
|
+
- PolicyName: AWSReconECSTaskExecutionPolicy
|
120
|
+
PolicyDocument:
|
121
|
+
Version: '2012-10-17'
|
122
|
+
Statement:
|
123
|
+
- Effect: Allow
|
124
|
+
Action:
|
125
|
+
- 'ecr:GetAuthorizationToken'
|
126
|
+
- 'ecr:BatchCheckLayerAvailability'
|
127
|
+
- 'ecr:GetDownloadUrlForLayer'
|
128
|
+
- 'ecr:BatchGetImage'
|
129
|
+
- 'logs:CreateLogStream'
|
130
|
+
- 'logs:PutLogEvents'
|
131
|
+
Resource: '*'
|
132
|
+
AssumeRolePolicyDocument:
|
133
|
+
Version: '2012-10-17'
|
134
|
+
Statement:
|
135
|
+
- Effect: Allow
|
136
|
+
Principal:
|
137
|
+
Service:
|
138
|
+
- ecs-tasks.amazonaws.com
|
139
|
+
Action: 'sts:AssumeRole'
|
140
|
+
AWSReconCloudWatchEventsRole:
|
141
|
+
Type: AWS::IAM::Role
|
142
|
+
Properties:
|
143
|
+
RoleName: aws-recon-events-role
|
144
|
+
AssumeRolePolicyDocument:
|
145
|
+
Version: '2012-10-17'
|
146
|
+
Statement:
|
147
|
+
- Effect: Allow
|
148
|
+
Principal:
|
149
|
+
Service:
|
150
|
+
- events.amazonaws.com
|
151
|
+
Action: 'sts:AssumeRole'
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_rule.html
|
2
|
+
resource "aws_cloudwatch_event_rule" "default" {
|
3
|
+
name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
4
|
+
description = "AWS Recon scheduled task"
|
5
|
+
schedule_expression = var.schedule_expression
|
6
|
+
}
|
7
|
+
|
8
|
+
# https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_target.html
|
9
|
+
resource "aws_cloudwatch_event_target" "default" {
|
10
|
+
target_id = aws_ecs_task_definition.aws_recon_task.id
|
11
|
+
arn = aws_ecs_cluster.aws_recon.arn
|
12
|
+
rule = aws_cloudwatch_event_rule.default.name
|
13
|
+
role_arn = aws_iam_role.cw_events.arn
|
14
|
+
|
15
|
+
ecs_target {
|
16
|
+
launch_type = "FARGATE"
|
17
|
+
task_definition_arn = aws_ecs_task_definition.aws_recon_task.arn
|
18
|
+
platform_version = "LATEST"
|
19
|
+
|
20
|
+
network_configuration {
|
21
|
+
assign_public_ip = true
|
22
|
+
security_groups = [aws_security_group.sg.id]
|
23
|
+
subnets = [aws_subnet.subnet.id]
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
@@ -0,0 +1,49 @@
|
|
1
|
+
resource "aws_ecs_cluster" "aws_recon" {
|
2
|
+
name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
3
|
+
capacity_providers = [local.ecs_task_provider]
|
4
|
+
}
|
5
|
+
|
6
|
+
resource "aws_ecs_task_definition" "aws_recon_task" {
|
7
|
+
family = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
8
|
+
task_role_arn = aws_iam_role.aws_recon_role.arn
|
9
|
+
execution_role_arn = aws_iam_role.ecs_task_execution.arn
|
10
|
+
requires_compatibilities = [local.ecs_task_provider]
|
11
|
+
network_mode = "awsvpc"
|
12
|
+
cpu = 1024
|
13
|
+
memory = 2048
|
14
|
+
|
15
|
+
container_definitions = jsonencode([
|
16
|
+
{
|
17
|
+
name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
18
|
+
image = "${var.aws_recon_container_name}:${var.aws_recon_container_version}"
|
19
|
+
assign_public_ip = true
|
20
|
+
entryPoint = [
|
21
|
+
"aws_recon",
|
22
|
+
"--verbose",
|
23
|
+
"--format",
|
24
|
+
"custom",
|
25
|
+
"--s3-bucket",
|
26
|
+
"${aws_s3_bucket.aws_recon.bucket}:${data.aws_region.current.name}",
|
27
|
+
"--regions",
|
28
|
+
join(",", var.aws_regions)
|
29
|
+
]
|
30
|
+
logConfiguration = {
|
31
|
+
logDriver = "awslogs"
|
32
|
+
options = {
|
33
|
+
awslogs-group = aws_cloudwatch_log_group.aws_recon.name,
|
34
|
+
awslogs-region = data.aws_region.current.name,
|
35
|
+
awslogs-stream-prefix = "ecs"
|
36
|
+
}
|
37
|
+
}
|
38
|
+
}
|
39
|
+
])
|
40
|
+
}
|
41
|
+
|
42
|
+
resource "aws_cloudwatch_log_group" "aws_recon" {
|
43
|
+
name = "/ecs/${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
44
|
+
retention_in_days = var.retention_period
|
45
|
+
}
|
46
|
+
|
47
|
+
locals {
|
48
|
+
ecs_task_provider = "FARGATE"
|
49
|
+
}
|
@@ -0,0 +1,125 @@
|
|
1
|
+
#
|
2
|
+
# IAM policies and roles for ECS and CloudWatch execution
|
3
|
+
#
|
4
|
+
resource "aws_iam_role" "aws_recon_role" {
|
5
|
+
name = local.aws_recon_task_role_name
|
6
|
+
assume_role_policy = data.aws_iam_policy_document.aws_recon_task_execution_assume_role_policy.json
|
7
|
+
}
|
8
|
+
|
9
|
+
data "aws_iam_policy_document" "aws_recon_task_execution_assume_role_policy" {
|
10
|
+
statement {
|
11
|
+
actions = ["sts:AssumeRole"]
|
12
|
+
|
13
|
+
principals {
|
14
|
+
type = "Service"
|
15
|
+
identifiers = [
|
16
|
+
"ecs.amazonaws.com",
|
17
|
+
"ecs-tasks.amazonaws.com"
|
18
|
+
]
|
19
|
+
}
|
20
|
+
}
|
21
|
+
}
|
22
|
+
|
23
|
+
resource "aws_iam_role_policy_attachment" "aws_recon_task_execution" {
|
24
|
+
role = aws_iam_role.aws_recon_role.name
|
25
|
+
policy_arn = data.aws_iam_policy.aws_recon_task_execution.arn
|
26
|
+
}
|
27
|
+
|
28
|
+
resource "aws_iam_role_policy" "aws_recon" {
|
29
|
+
name = local.bucket_write_policy_name
|
30
|
+
role = aws_iam_role.aws_recon_role.id
|
31
|
+
|
32
|
+
policy = jsonencode({
|
33
|
+
Version = "2012-10-17"
|
34
|
+
Id = "${var.aws_recon_base_name}-bucket-write"
|
35
|
+
Statement = [
|
36
|
+
{
|
37
|
+
Sid = "AWSReconS3PutObject"
|
38
|
+
Effect = "Allow"
|
39
|
+
Action = "s3:PutObject"
|
40
|
+
Resource = [
|
41
|
+
"${aws_s3_bucket.aws_recon.arn}/*"
|
42
|
+
]
|
43
|
+
}
|
44
|
+
]
|
45
|
+
})
|
46
|
+
}
|
47
|
+
|
48
|
+
data "aws_iam_policy" "aws_recon_task_execution" {
|
49
|
+
arn = "arn:aws:iam::aws:policy/ReadOnlyAccess"
|
50
|
+
}
|
51
|
+
|
52
|
+
resource "aws_iam_role" "ecs_task_execution" {
|
53
|
+
name = local.ecs_task_execution_role_name
|
54
|
+
assume_role_policy = data.aws_iam_policy_document.ecs_task_execution_assume_role_policy.json
|
55
|
+
|
56
|
+
tags = {
|
57
|
+
Name = local.ecs_task_execution_role_name
|
58
|
+
}
|
59
|
+
}
|
60
|
+
|
61
|
+
data "aws_iam_policy_document" "ecs_task_execution_assume_role_policy" {
|
62
|
+
statement {
|
63
|
+
actions = ["sts:AssumeRole"]
|
64
|
+
|
65
|
+
principals {
|
66
|
+
type = "Service"
|
67
|
+
identifiers = ["ecs-tasks.amazonaws.com"]
|
68
|
+
}
|
69
|
+
}
|
70
|
+
}
|
71
|
+
|
72
|
+
# ECS task execution
|
73
|
+
resource "aws_iam_policy" "ecs_task_execution" {
|
74
|
+
name = local.ecs_task_execution_policy_name
|
75
|
+
policy = data.aws_iam_policy.ecs_task_execution.policy
|
76
|
+
}
|
77
|
+
|
78
|
+
resource "aws_iam_role_policy_attachment" "ecs_task_execution" {
|
79
|
+
role = aws_iam_role.ecs_task_execution.name
|
80
|
+
policy_arn = aws_iam_policy.ecs_task_execution.arn
|
81
|
+
}
|
82
|
+
|
83
|
+
data "aws_iam_policy" "ecs_task_execution" {
|
84
|
+
arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
|
85
|
+
}
|
86
|
+
|
87
|
+
# CloudWatch Events
|
88
|
+
resource "aws_iam_role" "cw_events" {
|
89
|
+
name = local.cw_events_role_name
|
90
|
+
assume_role_policy = data.aws_iam_policy_document.cw_events_assume_role_policy.json
|
91
|
+
}
|
92
|
+
|
93
|
+
data "aws_iam_policy_document" "cw_events_assume_role_policy" {
|
94
|
+
statement {
|
95
|
+
actions = ["sts:AssumeRole"]
|
96
|
+
|
97
|
+
principals {
|
98
|
+
type = "Service"
|
99
|
+
identifiers = ["events.amazonaws.com"]
|
100
|
+
}
|
101
|
+
}
|
102
|
+
}
|
103
|
+
|
104
|
+
resource "aws_iam_policy" "cw_events" {
|
105
|
+
name = local.cw_events_policy_name
|
106
|
+
policy = data.aws_iam_policy.cw_events.policy
|
107
|
+
}
|
108
|
+
|
109
|
+
resource "aws_iam_role_policy_attachment" "cw_events" {
|
110
|
+
role = aws_iam_role.cw_events.name
|
111
|
+
policy_arn = aws_iam_policy.cw_events.arn
|
112
|
+
}
|
113
|
+
|
114
|
+
data "aws_iam_policy" "cw_events" {
|
115
|
+
arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
|
116
|
+
}
|
117
|
+
|
118
|
+
locals {
|
119
|
+
bucket_write_policy_name = "${var.aws_recon_base_name}-bucket-write-policy"
|
120
|
+
ecs_task_execution_role_name = "${var.aws_recon_base_name}-ecs-task-execution-role"
|
121
|
+
ecs_task_execution_policy_name = "${var.aws_recon_base_name}-ecs-task-execution-policy"
|
122
|
+
cw_events_policy_name = "${var.aws_recon_base_name}-cw-events-policy"
|
123
|
+
cw_events_role_name = "${var.aws_recon_base_name}-cw-events-role"
|
124
|
+
aws_recon_task_role_name = "${var.aws_recon_base_name}-exec-role"
|
125
|
+
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
output "aws_recon_ecs_cluster" {
|
2
|
+
value = aws_ecs_cluster.aws_recon.name
|
3
|
+
}
|
4
|
+
|
5
|
+
output "aws_recon_ecs_scheduled_task" {
|
6
|
+
value = aws_cloudwatch_event_rule.default.name
|
7
|
+
}
|
8
|
+
|
9
|
+
output "aws_recon_s3_bucket" {
|
10
|
+
value = aws_s3_bucket.aws_recon.bucket
|
11
|
+
}
|
12
|
+
|
13
|
+
output "aws_recon_task_manual_run_command" {
|
14
|
+
value = "\nOne-off task run command:\n\naws ecs run-task --task-definition ${aws_ecs_task_definition.aws_recon_task.family} --cluster ${aws_ecs_cluster.aws_recon.name} --launch-type FARGATE --network-configuration \"awsvpcConfiguration={subnets=[${aws_subnet.subnet.id}],securityGroups=[${aws_security_group.sg.id}],assignPublicIp=ENABLED}\"\n"
|
15
|
+
}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
## Terraform Setup
|
2
|
+
|
3
|
+
This is an example module that can be used in its current form or modified for your specific environment. It builds the minimum components necessary to collect inventory on a schedule running AWS Recon as a Fargate scheduled task.
|
4
|
+
|
5
|
+
### Requirements
|
6
|
+
|
7
|
+
Before running this Terraform module, adjust your region accordingly in `main.tf`.
|
8
|
+
|
9
|
+
### What is created?
|
10
|
+
|
11
|
+
This Terraform example will deploy the following resources:
|
12
|
+
|
13
|
+
- an S3 bucket to store compressed JSON output files
|
14
|
+
- an IAM role for ECS task execution
|
15
|
+
- a Security Group for the ECS cluster/task
|
16
|
+
- a VPC and NGW for the ECS cluster/task
|
17
|
+
- an ECS/Fargate cluster
|
18
|
+
- an ECS task definition to run AWS Recon collection
|
19
|
+
- a CloudWatch event rule to trigger the ECS task
|
20
|
+
- a CloudTrail log group for ECS task logs
|
@@ -0,0 +1,20 @@
|
|
1
|
+
resource "aws_s3_bucket" "aws_recon" {
|
2
|
+
bucket = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}-${data.aws_iam_account_alias.current.id}"
|
3
|
+
acl = "private"
|
4
|
+
force_destroy = true
|
5
|
+
|
6
|
+
lifecycle_rule {
|
7
|
+
id = "expire-after-${var.retention_period}-days"
|
8
|
+
enabled = true
|
9
|
+
|
10
|
+
expiration {
|
11
|
+
days = var.retention_period
|
12
|
+
}
|
13
|
+
}
|
14
|
+
}
|
15
|
+
|
16
|
+
resource "random_id" "aws_recon" {
|
17
|
+
byte_length = 6
|
18
|
+
}
|
19
|
+
|
20
|
+
data "aws_iam_account_alias" "current" {}
|
@@ -0,0 +1,58 @@
|
|
1
|
+
variable "aws_recon_base_name" {
|
2
|
+
type = string
|
3
|
+
default = "aws-recon"
|
4
|
+
}
|
5
|
+
|
6
|
+
variable "aws_recon_container_name" {
|
7
|
+
type = string
|
8
|
+
default = "darkbitio/aws_recon"
|
9
|
+
}
|
10
|
+
|
11
|
+
variable "aws_recon_container_version" {
|
12
|
+
type = string
|
13
|
+
default = "latest"
|
14
|
+
}
|
15
|
+
|
16
|
+
variable "aws_regions" {
|
17
|
+
type = list(any)
|
18
|
+
default = [
|
19
|
+
"global",
|
20
|
+
# "af-south-1",
|
21
|
+
# "ap-east-1",
|
22
|
+
# "ap-northeast-1",
|
23
|
+
# "ap-northeast-2",
|
24
|
+
# "ap-northeast-3",
|
25
|
+
# "ap-south-1",
|
26
|
+
# "ap-southeast-1",
|
27
|
+
# "ap-southeast-2",
|
28
|
+
# "ca-central-1",
|
29
|
+
# "eu-central-1",
|
30
|
+
# "eu-north-1",
|
31
|
+
# "eu-south-1",
|
32
|
+
# "eu-west-1",
|
33
|
+
# "eu-west-2",
|
34
|
+
# "eu-west-3",
|
35
|
+
# "me-south-1",
|
36
|
+
# "sa-east-1",
|
37
|
+
"us-east-1",
|
38
|
+
"us-east-2",
|
39
|
+
"us-west-1",
|
40
|
+
"us-west-2",
|
41
|
+
]
|
42
|
+
}
|
43
|
+
|
44
|
+
# must be one of: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365
|
45
|
+
variable "retention_period" {
|
46
|
+
type = number
|
47
|
+
default = 30
|
48
|
+
}
|
49
|
+
|
50
|
+
variable "schedule_expression" {
|
51
|
+
type = string
|
52
|
+
default = "cron(4 * * * ? *)"
|
53
|
+
}
|
54
|
+
|
55
|
+
variable "base_subnet_cidr" {
|
56
|
+
type = string
|
57
|
+
default = "10.76.0.0/16"
|
58
|
+
}
|
@@ -0,0 +1,73 @@
|
|
1
|
+
|
2
|
+
# Create a VPC
|
3
|
+
resource "aws_vpc" "vpc" {
|
4
|
+
cidr_block = local.cidr_block
|
5
|
+
tags = {
|
6
|
+
Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
7
|
+
}
|
8
|
+
}
|
9
|
+
|
10
|
+
# Create subnet
|
11
|
+
resource "aws_subnet" "subnet" {
|
12
|
+
vpc_id = aws_vpc.vpc.id
|
13
|
+
cidr_block = local.subnet_cidr_block
|
14
|
+
availability_zone = data.aws_availability_zones.available.names[0]
|
15
|
+
|
16
|
+
tags = {
|
17
|
+
Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}-public"
|
18
|
+
}
|
19
|
+
}
|
20
|
+
|
21
|
+
resource "aws_security_group" "sg" {
|
22
|
+
name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
23
|
+
description = "Allow AWS Recon collection egress"
|
24
|
+
vpc_id = aws_vpc.vpc.id
|
25
|
+
|
26
|
+
egress {
|
27
|
+
from_port = 0
|
28
|
+
to_port = 0
|
29
|
+
protocol = "-1"
|
30
|
+
cidr_blocks = ["0.0.0.0/0"]
|
31
|
+
}
|
32
|
+
|
33
|
+
tags = {
|
34
|
+
Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
35
|
+
}
|
36
|
+
}
|
37
|
+
|
38
|
+
resource "aws_internet_gateway" "igw" {
|
39
|
+
vpc_id = aws_vpc.vpc.id
|
40
|
+
|
41
|
+
tags = {
|
42
|
+
Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
43
|
+
}
|
44
|
+
}
|
45
|
+
|
46
|
+
resource "aws_route_table" "rt" {
|
47
|
+
vpc_id = aws_vpc.vpc.id
|
48
|
+
|
49
|
+
route {
|
50
|
+
cidr_block = "0.0.0.0/0"
|
51
|
+
gateway_id = aws_internet_gateway.igw.id
|
52
|
+
}
|
53
|
+
|
54
|
+
tags = {
|
55
|
+
Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}"
|
56
|
+
}
|
57
|
+
}
|
58
|
+
|
59
|
+
resource "aws_route_table_association" "rt_association" {
|
60
|
+
subnet_id = aws_subnet.subnet.id
|
61
|
+
route_table_id = aws_route_table.rt.id
|
62
|
+
}
|
63
|
+
|
64
|
+
locals {
|
65
|
+
cidr_block = var.base_subnet_cidr
|
66
|
+
subnet_cidr_block = cidrsubnet(local.cidr_block, 8, 0)
|
67
|
+
}
|
68
|
+
|
69
|
+
data "aws_region" "current" {}
|
70
|
+
|
71
|
+
data "aws_availability_zones" "available" {
|
72
|
+
state = "available"
|
73
|
+
}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws_recon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Josh Larsen
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-04-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -244,6 +244,16 @@ files:
|
|
244
244
|
- lib/aws_recon/services.yaml
|
245
245
|
- lib/aws_recon/version.rb
|
246
246
|
- readme.md
|
247
|
+
- utils/cloudformation/aws-recon-cfn-template.yml
|
248
|
+
- utils/terraform/cloudwatch.tf
|
249
|
+
- utils/terraform/ecs.tf
|
250
|
+
- utils/terraform/iam.tf
|
251
|
+
- utils/terraform/main.tf
|
252
|
+
- utils/terraform/output.tf
|
253
|
+
- utils/terraform/readme.md
|
254
|
+
- utils/terraform/s3.tf
|
255
|
+
- utils/terraform/vars.tf
|
256
|
+
- utils/terraform/vpc.tf
|
247
257
|
homepage: https://github.com/darkbitio/aws-recon
|
248
258
|
licenses:
|
249
259
|
- MIT
|
@@ -266,5 +276,5 @@ requirements: []
|
|
266
276
|
rubygems_version: 3.0.8
|
267
277
|
signing_key:
|
268
278
|
specification_version: 4
|
269
|
-
summary: A multi-threaded AWS inventory collection
|
279
|
+
summary: A multi-threaded AWS security-focused inventory collection tool.
|
270
280
|
test_files: []
|